@method-ai-hq/sdk 0.0.0-stage → 0.14.2

package/dist/packages/sdk/src/method.js

@@ -0,0 +1,359 @@
+#!/usr/bin/env node
+import packageInfo from '../package.json' with { type: 'json' };
+import runtimeInfo from '@withmethod/runtime/package.json' with { type: 'json' };
+import { startRunWorker, waitForRun, cancelRun, workerAlive } from './run-worker.js';
+import { methodCache } from './prepare.js';
+import { restorePackage } from './method-files.js';
+import { publish } from './versions.js';
+import { runSaved } from './run-saved.js';
+import { bindInput, bindConnection } from './bindings.js';
+import { existsSync, readFileSync, realpathSync } from "node:fs";
+import { join, resolve } from "node:path";
+import { parseArgs } from "node:util";
+import { pathToFileURL } from "node:url";
+import { randomUUID } from "node:crypto";
+import { localAuthoring, authoringPath, readDocument, writeDocument } from "./authoring.js";
+import { MethodClient, DEFAULT_SERVER } from "./method-client.js";
+import { MethodSync } from "./method-sync.js";
+import { progressMain } from "./progress.js";
+import { parse, localMain } from "./local-cli.js";
+import { writePrivateJson } from "./files.js";
+import { loadWorkflow } from "../../workflow-language/src/validate.js";
+import { methodHelp, overviewHelp as help } from "./method-help.js";
+import { fixFor } from "@withmethod/runtime/runner.js";
+/** The next step for a failure before the first step. */
+export function setupFix(error) {
+    if (error.code === 'missing_secret') {
+        const names = (error.missing ?? []).join(' ');
+        return `No step ran. Run method secret find to see which nearby files hold ${names}, then method secret import FILE ${names}; or ask the user to enter each one with method secret set NAME (a private form in their browser). Never open or print a key file, and never ask for a value in chat.`;
+    }
+    return fixFor(error);
+}
+function safePath(path) {
+    return authoringPath(path);
+}
+async function dispatchRunWorker(args, flags, extra = []) {
+    if (process.env.METHOD_RUN_WORKER === '1' || (!/[/\\]method\.(?:js|ts)$/.test(process.argv[1] ?? '') && !flags.background))
+        return false;
+    if (flags.resume && !flags['run-dir'])
+        throw Error('Resume needs --run-dir.');
+    const directory = safePath(flags['run-dir'] ?? join(methodCache(), 'runs', randomUUID()));
+    const workerArgs = [...args.filter(arg => arg !== '--background'), ...extra];
+    if (!flags['run-dir'])
+        workerArgs.push('--run-dir', directory);
+    await startRunWorker(workerArgs, directory, flags.background);
+    return true;
+}
+export async function methodMain(args = process.argv.slice(2), clientFactory = server => new MethodClient(server)) {
+    const earlyHelp = methodHelp(args);
+    if (earlyHelp !== undefined) {
+        process.stdout.write(earlyHelp);
+        return;
+    }
+    if (['improve', 'proposals', 'apply'].includes(args[0] ?? ''))
+        return (await import('./proposals.js')).proposalCommand(args, clientFactory);
+    await (await import('./proposals.js')).autoApply(args, clientFactory);
+    if (args[0] === 'worker' || args[0] === 'keys')
+        return (await import('./worker.js')).productionCommand(args, clientFactory);
+    if (args[0] === '__run-data')
+        return (await import('./run-data.js')).runDataCommand(args.slice(1));
+    if (args[0] === 'connect' || args[0] === 'answer') {
+        const server = parseArgs({ args: args.slice(1), strict: false, options: { server: { type: 'string' } } }).values.server;
+        const client = clientFactory(typeof server === 'string' ? server : DEFAULT_SERVER);
+        const { connectCommand, answerCommand } = await import('./connect.js');
+        process.stdout.write(JSON.stringify(await (args[0] === 'connect' ? connectCommand : answerCommand)(args.slice(1), client), null, 2) + '\n');
+        return;
+    }
+    if (args[0] === 'browser' && args[1] === 'connect') {
+        const { configureBrowser } = await import('./browser.js');
+        const p = parseArgs({ args: args.slice(2), options: { name: { type: 'string' }, cdp: { type: 'string' } } });
+        process.stdout.write(JSON.stringify(configureBrowser(p.values.name ?? 'default', p.values.cdp)) + '\n');
+        return;
+    }
+    if (args[0] === '__worker') {
+        const file = args[1];
+        const directory = resolve(file, '..');
+        writePrivateJson(join(directory, 'worker.json'), { status: 'running', pid: process.pid });
+        try {
+            await methodMain(JSON.parse(readFileSync(file, 'utf8')).args, clientFactory);
+        }
+        catch (error) {
+            // A run that fails before its first step still prints one result that says why and what to do.
+            const failed = { status: error.code === 'needs_input' ? 'needs_input' : 'failed', run_dir: directory, code: error.code ?? 'setup_failed', error: error.message, ...(error.missing ? { missing: error.missing } : {}), fix: setupFix(error) };
+            if (!existsSync(join(directory, 'summary.json')))
+                writePrivateJson(join(directory, 'summary.json'), failed);
+            process.stdout.write(JSON.stringify(failed, null, 2) + '\n');
+            process.exitCode = error.code === 'needs_input' ? 2 : 1;
+        }
+        finally {
+            writePrivateJson(join(directory, 'worker.json'), { status: 'finished', pid: process.pid, exit_code: Number(process.exitCode ?? 0) });
+        }
+        return;
+    }
+    if (['wait', 'cancel', 'run-status'].includes(args[0] ?? '')) {
+        if (!args[1])
+            throw Error('Supply the local run directory.');
+        const directory = safePath(args[1]);
+        const result = args[0] === 'wait' ? await waitForRun(directory) : args[0] === 'cancel' ? cancelRun(directory) : { running: workerAlive(directory), ...(existsSync(join(directory, 'worker.json')) ? readDocument(join(directory, 'worker.json')) : {}) };
+        process.stdout.write(JSON.stringify(result) + '\n');
+        return;
+    }
+    if (args[0] === 'secret')
+        return (await import('./secrets.js')).secretCommand(args.slice(1));
+    if (args[0] === '__checks' && args[1])
+        return (await import('./method-issues.js')).checksWorker(args[1]);
+    if (args[0] === 'models' && !args.includes('--help'))
+        return (await import('./hosted-models.js')).modelsCommand(args.slice(1), clientFactory);
+    if (args.length === 1 && args[0] === "--version") {
+        process.stdout.write(`Method SDK ${packageInfo.version}; runtime ${runtimeInfo.version}; current format method/3.4\n`);
+        return;
+    }
+    if (["observe", "test", "case", "effect"].includes(args[0] ?? "") && !args.includes("--help")) {
+        const quality = await import("./quality.js");
+        if (args[0] === "observe")
+            return quality.observeCommand(args.slice(1));
+        if (args[0] === "test")
+            return quality.testCommand(args.slice(1));
+        if (args[0] === "case")
+            return quality.caseCommand(args.slice(1));
+        return quality.effectCommand(args.slice(1));
+    }
+    if (args[0] === 'run' && /^https?:/.test(args[1] ?? '')) {
+        const url = new URL(args[1]);
+        const match = url.pathname.match(/^\/methods\/([^/]+)$/);
+        if (match) {
+            if (url.username || url.password)
+                throw Error('Use a Method dashboard URL without credentials.');
+            args = ['run', decodeURIComponent(match[1]), ...args.slice(2), '--server', url.origin,
+                ...(url.searchParams.get('version') ? ['--version', url.searchParams.get('version')] : [])];
+        }
+    }
+    const localCommand = ["prompt", "inspect", "doctor"].includes(args[0] ?? "")
+        || ["run", "steps"].includes(args[0] ?? "") && /\.(method)$|^https?:\/\//i.test(args[1] ?? "");
+    if (localCommand) {
+        if (args[0] === 'run' && await dispatchRunWorker(args, parse(args).values))
+            return;
+        return localMain(args);
+    }
+    const requestedHelp = methodHelp(args);
+    if (requestedHelp !== undefined) {
+        process.stdout.write(requestedHelp);
+        return;
+    }
+    if (args[0] === "progress")
+        return progressMain(args.slice(1));
+    if (await localAuthoring(args))
+        return;
+    // Strip only Method flags. Pass run flags unchanged to the existing SDK parser.
+    const methodOptions = new Set([
+        "server",
+        "version",
+        "file",
+        "reason",
+        "method",
+        "connection",
+        "accept-failing-case",
+    ]);
+    const values = {}, rest = [];
+    for (let i = 0; i < args.length; i++) {
+        const match = args[i].match(/^--([^=]+)(?:=(.*))?$/);
+        if (match && methodOptions.has(match[1])) {
+            const value = match[2] ?? args[++i];
+            if (!value || value.startsWith("--"))
+                throw Error(`Supply --${match[1]}.`);
+            values[match[1]] = value;
+        }
+        else
+            rest.push(args[i]);
+    }
+    const [command, target, step] = rest;
+    if (!command || command === "--help" || command === "help") {
+        process.stdout.write(help);
+        return;
+    }
+    if (command === "sync") {
+        if (!target)
+            throw Error("Use method sync RUN_DIRECTORY.");
+        await MethodSync.retry(safePath(target), values.server ? new MethodClient(values.server) : undefined);
+        return;
+    }
+    const authorFile = command === "publish" || command === "explain" ? target : undefined;
+    if (command === "guard-keys")
+        return (await import("./guard-keys.js")).guardKeysCommand();
+    if (command === "config")
+        return (await import("./computer-settings.js")).configCommand(rest.slice(1));
+    if (command === "new-id") {
+        if (!target || rest.length !== 2)
+            throw Error("Use method new-id FILE.");
+        const { newMethodId, writeMethodId } = await import("./versions.js");
+        const id = newMethodId();
+        writeMethodId(safePath(target), id);
+        process.stdout.write(JSON.stringify({ file: safePath(target), method_id: id }, null, 2) + "\n");
+        return;
+    }
+    if (!["status", "login", "logout", "devices", "revoke", "list", "get", "steps", "step", "run", "runs", "logs", "publish", "explain", "bind", "state"].includes(command))
+        throw Error(help);
+    const client = clientFactory(values.server ?? DEFAULT_SERVER);
+    if (command === "login") {
+        await client.login();
+        return;
+    }
+    if (command === "logout") {
+        await client.logout();
+        process.stdout.write("Method is disconnected.\n");
+        return;
+    }
+    if (command === "status") {
+        let signedIn = false;
+        if (client.token()) {
+            try {
+                await client.request("/api/cli/me");
+                signedIn = true;
+            }
+            catch (error) {
+                if (!/^401:/.test(String(error.message)))
+                    throw error;
+            }
+        }
+        const update = await (await import("./update.js")).updateNotice();
+        process.stdout.write(JSON.stringify({ installed: true, server: client.server, signed_in: signedIn, ...(!signedIn ? { next: "Run method login to sign in with browser approval." } : {}), ...(update ? { update } : {}) }, null, 2) + "\n");
+        return;
+    }
+    // Without sign-in, explain prints one line and stops: script cards need a hosted model.
+    if (command === "explain") {
+        await (await import("./explain.js")).explainCommand(client, rest.slice(1));
+        return;
+    }
+    // The first real command starts browser approval if no credential is saved.
+    if (!client.token())
+        await client.login();
+    // Any signed-in command sends what is left in the outbox, in the background.
+    else
+        (await import("./outbox.js")).sendLeftovers(client);
+    if (command === 'run' && client.token()) {
+        try {
+            await client.request('/api/cli/me');
+        }
+        catch (error) {
+            if (String(error).includes('401:'))
+                await client.login();
+            else
+                throw error;
+        }
+    }
+    const print = (v) => process.stdout.write(JSON.stringify(v, null, 2) + "\n");
+    if (command === "devices") {
+        print(await client.request("/api/cli/devices"));
+        return;
+    }
+    if (command === "revoke") {
+        if (!target)
+            throw Error("Supply a device ID.");
+        print(await client.request(`/api/cli/devices/${encodeURIComponent(target)}/revoke`, "POST", {}));
+        return;
+    }
+    if (command === "list") {
+        print(await client.request("/api/methods"));
+        return;
+    }
+    if (command === "runs") {
+        print(await client.request(`/api/workspace/runs${values.method ? `?workflow_id=${encodeURIComponent(values.method)}` : ""}`));
+        return;
+    }
+    if (command === "logs") {
+        if (!target)
+            throw Error("Supply a run ID.");
+        print(await client.request(`/api/workspace/runs/${encodeURIComponent(target)}`));
+        return;
+    }
+    if (command === 'bind') {
+        if (target && step && values.connection) {
+            print(await bindConnection(client, target, step, values.connection, rest.includes('--upload')));
+            return;
+        }
+        if (!target || !step || !values.file)
+            throw Error('Use method bind ID NAME --file FOLDER [--upload].');
+        print(await bindInput(client, target, step, values.file, rest.includes('--upload')));
+        return;
+    }
+    if (command === 'state') {
+        if (!target)
+            throw Error('Supply a Method ID.');
+        const path = `/api/cli/methods/${encodeURIComponent(target)}/state`;
+        if (rest.includes('--enable')) {
+            if (!values.file)
+                throw Error('Supply --file with the initial state JSON.');
+            print(await client.request(path, 'POST', { action: 'enable', value: readDocument(values.file) }));
+        }
+        else if (rest.includes('--release')) {
+            const id = rest[rest.indexOf('--release') + 1];
+            if (!id)
+                throw Error('Supply the stopped run ID after --release. Inspect its actions first.');
+            print(await client.request(path, 'POST', { action: 'release', run_id: id }));
+        }
+        else
+            print(await client.request(path));
+        return;
+    }
+    if (command === "publish") {
+        if (!authorFile)
+            throw Error("Use method publish FILE [--reason TEXT] [--cloud|--workers] [--env NAME].");
+        print(await (await import("./cloud.js")).publishWithPlacement(client, rest, () => publish(client, safePath(authorFile), values.reason, (values["accept-failing-case"] ?? "").split(",").map(id => id.trim()).filter(Boolean))));
+        return;
+    }
+    if (!target)
+        throw Error(help);
+    const path = `/api/cli/methods/${encodeURIComponent(target)}`;
+    if (!["get", "steps", "step", "run"].includes(command))
+        throw Error(help);
+    const saved = await client.request(path +
+        (values.version ? `?version=${encodeURIComponent(values.version)}` : ""));
+    const workflow = command === "get" ? saved.workflow : loadWorkflow(saved.workflow);
+    if (command === "steps") {
+        print({ version_id: saved.version_id, steps: workflow.steps });
+        return;
+    }
+    if (command === "step") {
+        const found = step ? workflow.steps[step] : undefined;
+        if (!found)
+            throw Error("Step not found. Use method steps WORKFLOW_ID.");
+        print({ version_id: saved.version_id, step: found });
+        return;
+    }
+    if (command === "get") {
+        const parsed = parseArgs({
+            args: rest.slice(2),
+            options: { out: { type: "string" } },
+        });
+        if (parsed.values.out) {
+            const out = safePath(parsed.values.out);
+            if (existsSync(out))
+                throw Error("Choose a new output file; this file already exists.");
+            await restorePackage(saved, resolve(out, ".."), client);
+            // The id: line links the file to its Method; its content is the saved version.
+            const { methodContent, writeMethodId } = await import("./versions.js");
+            writeDocument(out, methodContent(workflow));
+            writeMethodId(out, saved.workflow_id);
+            (await import("./outbox.js")).recordCheckout(client.server, saved.workflow_id, saved.version_id, saved.version_number, methodContent(workflow));
+            print({
+                workflow_id: saved.workflow_id,
+                version_id: saved.version_id,
+                version_number: saved.version_number,
+                file: out,
+            });
+        }
+        else
+            print({ ...saved, workflow });
+        return;
+    }
+    const flags = parse(rest.slice(2)).values;
+    if (await dispatchRunWorker(args, flags, values.version ? [] : ['--version', saved.version_id]))
+        return;
+    await runSaved(saved, flags, client);
+}
+if (process.argv[1] &&
+    import.meta.url === pathToFileURL(realpathSync(process.argv[1])).href)
+    methodMain().catch((error) => {
+        process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`);
+        process.exitCode = 1;
+    });

package/dist/packages/sdk/src/model-checks.d.ts

@@ -0,0 +1,96 @@
+import type { ClassificationProvider } from '@withmethod/runtime';
+export type Level = 'error' | 'warning' | 'note';
+export type Issue = {
+    code: string;
+    level: Level;
+    step?: string;
+    field?: string;
+    file?: string;
+    line?: number;
+    message: string;
+    fix: string;
+    evidence?: {
+        probability?: number;
+        check?: string;
+        run_id?: string;
+    };
+    accepted?: string;
+};
+type Step = any;
+type Check = {
+    /** Change the version when the question or the inputs change; old cached answers then no longer apply. */
+    version: number;
+    level: Exclude<Level, 'error'>;
+    question: string;
+    /** Which part of the step the check reads; undefined when the check does not apply to the step. */
+    script?: (step: Step) => string | undefined;
+    applies: (step: Step) => boolean;
+    field: string;
+    message: (id: string) => string;
+    fix: string;
+};
+export declare const checks: Record<string, Check>;
+/**
+ * Checks shown to the user. A check moves here from shadow mode only with a results.json that shows its precision
+ * on the frozen held-out set meets its level (warning >= 0.90, note >= 0.75). Run npm run eval:checks first.
+ */
+export declare const shownChecks: string[];
+export declare const precisionBar: {
+    readonly warning: 0.9;
+    readonly note: 0.75;
+};
+/** An answer with a probability of yes at or above this is a finding. */
+export declare const findingThreshold = 0.5;
+/** The step as the check reads it: display text and accepts are left out, because they do not change what it does. */
+export declare function stepText(id: string, step: Step): string;
+export type Target = {
+    check: string;
+    version: number;
+    step: string;
+    inputs: Record<string, string>;
+    key: string;
+};
+/** The checks that apply to each step, with the exact inputs Jev receives. readText returns a file of the Method folder. */
+export declare function checkTargets(method: any, readText: (path: string) => string | undefined, steps?: Set<string>): Target[];
+export declare const checkKey: (check: string, version: number, inputs: Record<string, string>) => string;
+export type Result = {
+    check: string;
+    version: number;
+    step: string;
+    key: string;
+    answer: boolean;
+    probability: number;
+    model: string;
+    at: string;
+};
+export declare const checksDir: () => string;
+export declare function cached(key: string): Result | undefined;
+/**
+ * One yes/no question to Jev, as the two options yes and no (the form that every Method server accepts). The
+ * probability is that of yes.
+ */
+export declare const answerOptions: {
+    yes: string;
+    no: string;
+};
+export declare function askCheck(provider: ClassificationProvider, question: string, inputs: Record<string, string>, signal: AbortSignal): Promise<{
+    answer: boolean;
+    probability: number;
+    model: string;
+}>;
+export declare function evaluateTarget(provider: ClassificationProvider, target: Target, signal: AbortSignal): Promise<Result>;
+/**
+ * Run the targets in parallel. Cached answers come at once; new ones get waitMs in all. The others are returned as
+ * pending, so the caller can finish them in the background. A failed check is left out and is tried again next time.
+ */
+export declare function runChecks(provider: ClassificationProvider | undefined, targets: Target[], waitMs?: number): Promise<{
+    done: Result[];
+    pending: Target[];
+}>;
+/** Findings of shown checks become issues; shadow checks stay in the cache only. */
+export declare function modelIssues(results: Result[], shown?: string[]): Issue[];
+/** Cached shadow results for a set of keys, for audits. */
+export declare function shadowResults(keys?: Set<string>): Result[];
+/** The per-Method record of each step's text at the last validate, to show warnings only for changed steps. */
+export declare function changedSteps(file: string, method: any, readText: (path: string) => string | undefined): Set<string>;
+export {};

package/dist/packages/sdk/src/model-checks.js

@@ -0,0 +1,181 @@
+/**
+ * Model checks: yes/no questions to Jev through the Method account, for steps whose text changed. Each check starts
+ * in shadow mode: it is computed and recorded under ~/.cache/method/checks/, never shown. A check is shown only after
+ * its precision on the frozen labeled set (testkit/checks) meets its level: warning >= 0.90, note >= 0.75.
+ */
+import { createHash, randomUUID } from 'node:crypto';
+import { existsSync, mkdirSync, readFileSync, readdirSync, renameSync, writeFileSync } from 'node:fs';
+import { join } from 'node:path';
+import { serializeWorkflow } from '../../workflow-language/src/validate.js';
+import { methodCache } from './prepare.js';
+const promptKinds = ['call', 'agent'];
+export const checks = {
+    script_calls_model: {
+        version: 1, level: 'warning', field: 'do',
+        question: 'Does this script send text to a language model, an AI model, or an AI classifier (directly or through a helper library or company gateway), and use the model\'s answer? Fetching data, managing API keys, or counting records is not a model call.',
+        applies: step => step.do?.kind === 'run', script: step => step.do?.entrypoint,
+        message: id => `The script of ${id} appears to call a model.`,
+        fix: 'Move each model request into its own call, agent, or classify step with its prompt in the Method, and keep the fixed logic in the script.',
+    },
+    prompt_multiple_tasks: {
+        version: 1, level: 'note', field: 'do.prompt',
+        question: 'Does this step\'s prompt ask the model to do two or more separate tasks that could each be their own step with their own output, for example research and then write, or classify and then draft a reply? Several requirements for one output are one task.',
+        applies: step => promptKinds.includes(step.do?.kind),
+        message: id => `The prompt of ${id} appears to ask for more than one task.`,
+        fix: 'Give each task its own step, so that each prompt can be changed and compared alone.',
+    },
+    check_enforces_wording: {
+        version: 1, level: 'note', field: 'check',
+        question: 'Does this step\'s check fail an output because it does not use particular words, phrases, or a fixed wording, instead of checking facts, structure, or whether the output meets its goal? Requiring a field, a format such as a date or a link, or a value from the input is not enforcing wording.',
+        applies: step => !!step.check?.kind, script: step => step.check?.kind === 'run' ? step.check.entrypoint : undefined,
+        message: id => `The check of ${id} appears to require exact wording.`,
+        fix: 'Check what the output must contain or achieve, not the words it uses.',
+    },
+    classify_needs_math: {
+        version: 1, level: 'warning', field: 'do.question',
+        question: 'Does answering this classification step\'s question need arithmetic, counting, comparing numbers or dates, or another exact calculation that a script does reliably?',
+        applies: step => step.do?.kind === 'classify',
+        message: id => `The question of ${id} appears to need a calculation.`,
+        fix: 'Calculate the value in a run step, and classify only what needs judgment.',
+    },
+    classify_no_unclear: {
+        version: 1, level: 'note', field: 'do.options',
+        question: 'Can some inputs of this classification step fit none of its options, or be impossible to decide, while no option covers unclear, other, or none?',
+        applies: step => step.do?.kind === 'classify' && !!step.do.options,
+        message: id => `The options of ${id} have no choice for an unclear input.`,
+        fix: 'Add an option for unclear or other inputs, and decide what a later step does with it.',
+    },
+    prompt_no_missing_input: {
+        version: 1, level: 'warning', field: 'do.prompt',
+        question: 'Does this step\'s prompt refer to information, files, or data that are neither written in the prompt nor supplied by the step\'s in: values, so that the model must guess them? Tools, a browser, or a connection named in the step that can get the information count as supplied.',
+        applies: step => promptKinds.includes(step.do?.kind),
+        message: id => `The prompt of ${id} appears to need information that the step does not give it.`,
+        fix: 'Add the missing value to in: from an input or an earlier step, or write it in the prompt.',
+    },
+};
+/**
+ * Checks shown to the user. A check moves here from shadow mode only with a results.json that shows its precision
+ * on the frozen held-out set meets its level (warning >= 0.90, note >= 0.75). Run npm run eval:checks first.
+ */
+// Promoted by testkit/checks/results.json: held-out precision 1.00, recall 0.97, with real positives. The other checks stay in shadow mode until real positives clear their bar.
+export const shownChecks = ["script_calls_model"];
+export const precisionBar = { warning: 0.9, note: 0.75 };
+/** An answer with a probability of yes at or above this is a finding. */
+export const findingThreshold = 0.5;
+const limit = (text, max) => text.length > max ? `${text.slice(0, max)}\n[truncated]` : text;
+/** The step as the check reads it: display text and accepts are left out, because they do not change what it does. */
+export function stepText(id, step) {
+    const { reading: _reading, accept: _accept, ...rest } = step;
+    return limit(serializeWorkflow({ [id]: rest }), 16_000);
+}
+/** The checks that apply to each step, with the exact inputs Jev receives. readText returns a file of the Method folder. */
+export function checkTargets(method, readText, steps) {
+    const targets = [];
+    for (const [id, step] of Object.entries(method.steps ?? {})) {
+        if (steps && !steps.has(id))
+            continue;
+        for (const [name, check] of Object.entries(checks)) {
+            if (!check.applies(step))
+                continue;
+            const path = check.script?.(step);
+            const inputs = { step: stepText(id, step) };
+            if (path)
+                inputs.script = limit(readText(path) ?? '', 40_000);
+            targets.push({ check: name, version: check.version, step: id, inputs, key: checkKey(name, check.version, inputs) });
+        }
+    }
+    return targets;
+}
+export const checkKey = (check, version, inputs) => createHash('sha256').update(JSON.stringify([check, version, inputs.step, inputs.script ?? ''])).digest('hex');
+export const checksDir = () => join(methodCache(), 'checks');
+export function cached(key) {
+    const file = join(checksDir(), `${key}.json`);
+    try {
+        return existsSync(file) ? JSON.parse(readFileSync(file, 'utf8')) : undefined;
+    }
+    catch {
+        return undefined;
+    }
+}
+function store(result) {
+    mkdirSync(checksDir(), { recursive: true, mode: 0o700 });
+    const file = join(checksDir(), `${result.key}.json`), temporary = `${file}.${process.pid}.tmp`;
+    writeFileSync(temporary, JSON.stringify(result), { mode: 0o600 });
+    renameSync(temporary, file);
+}
+/**
+ * One yes/no question to Jev, as the two options yes and no (the form that every Method server accepts). The
+ * probability is that of yes.
+ */
+export const answerOptions = { yes: 'Yes: the question describes this step.', no: 'No: the question does not describe this step.' };
+export async function askCheck(provider, question, inputs, signal) {
+    const { model } = await provider.resolve(signal);
+    const answer = await provider.evaluate({ request_id: randomUUID(), model, question, options: answerOptions, inputs }, signal);
+    const probability = answer.probabilities.yes ?? 0;
+    return { answer: probability >= findingThreshold, probability, model };
+}
+export async function evaluateTarget(provider, target, signal) {
+    const answer = await askCheck(provider, checks[target.check].question, target.inputs, signal);
+    const result = { check: target.check, version: target.version, step: target.step, key: target.key, answer: answer.probability >= findingThreshold,
+        probability: answer.probability, model: answer.model, at: new Date().toISOString() };
+    store(result);
+    return result;
+}
+/**
+ * Run the targets in parallel. Cached answers come at once; new ones get waitMs in all. The others are returned as
+ * pending, so the caller can finish them in the background. A failed check is left out and is tried again next time.
+ */
+export async function runChecks(provider, targets, waitMs = 3000) {
+    const done = [], pending = [];
+    const open = targets.filter(target => { const hit = cached(target.key); if (hit)
+        done.push(hit); return !hit; });
+    if (!open.length)
+        return { done, pending };
+    if (!provider)
+        return { done, pending: open };
+    const controller = new AbortController();
+    const settled = new Set();
+    const work = open.map(target => evaluateTarget(provider, target, controller.signal)
+        .then(result => { done.push(result); }).catch(() => { }).finally(() => settled.add(target.key)));
+    let timer;
+    await Promise.race([Promise.all(work), new Promise(resolve => { timer = setTimeout(resolve, waitMs); })]);
+    clearTimeout(timer);
+    controller.abort();
+    pending.push(...open.filter(target => !settled.has(target.key)));
+    return { done, pending };
+}
+/** Findings of shown checks become issues; shadow checks stay in the cache only. */
+export function modelIssues(results, shown = shownChecks) {
+    return results.filter(result => result.answer && shown.includes(result.check)).map(result => {
+        const check = checks[result.check];
+        return { code: result.check, level: check.level, step: result.step, field: check.field, message: check.message(result.step), fix: check.fix,
+            evidence: { probability: result.probability, check: `${result.check}@${result.version}` } };
+    });
+}
+/** Cached shadow results for a set of keys, for audits. */
+export function shadowResults(keys) {
+    if (!existsSync(checksDir()))
+        return [];
+    return readdirSync(checksDir()).filter(name => /^[a-f0-9]{64}\.json$/.test(name) && (!keys || keys.has(name.slice(0, 64))))
+        .flatMap(name => { const hit = cached(name.slice(0, 64)); return hit ? [hit] : []; });
+}
+/** The per-Method record of each step's text at the last validate, to show warnings only for changed steps. */
+export function changedSteps(file, method, readText) {
+    const dir = join(checksDir(), 'validate'), record = join(dir, `${createHash('sha256').update(file).digest('hex').slice(0, 32)}.json`);
+    let before = {};
+    try {
+        before = existsSync(record) ? JSON.parse(readFileSync(record, 'utf8')).steps ?? {} : {};
+    }
+    catch { /* A new record. */ }
+    const now = {};
+    for (const [id, step] of Object.entries(method.steps ?? {})) {
+        const scripts = [step.do?.kind === 'run' ? step.do.entrypoint : undefined, step.check?.kind === 'run' ? step.check.entrypoint : undefined]
+            .filter(Boolean).map(path => readText(path) ?? '');
+        now[id] = createHash('sha256').update(JSON.stringify([stepText(id, step), ...scripts])).digest('hex');
+    }
+    mkdirSync(dir, { recursive: true, mode: 0o700 });
+    const temporary = `${record}.${process.pid}.tmp`;
+    writeFileSync(temporary, JSON.stringify({ file, steps: now }), { mode: 0o600 });
+    renameSync(temporary, record);
+    return new Set(Object.keys(now).filter(id => before[id] !== now[id]));
+}