@metrio-ai/cli 1.1.0 → 1.3.0

package/dist/commands/verify.js

@@ -0,0 +1,288 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.VERIFY_HELP = void 0;
+exports.parseVerifyArgs = parseVerifyArgs;
+exports.resolveTargets = resolveTargets;
+exports.runVerifyCommand = runVerifyCommand;
+/**
+ * `metrio verify` — runs cases.yaml against prompts and workflows and checks every reply.
+ * Execution uses the project API key, like `chat` and `run-batch`. Targets come from the
+ * workspace (.metrio/state.json) or from -p / -w with --cases. Every cases.yaml is
+ * validated before the first request.
+ */
+const fs = __importStar(require("fs"));
+const path = __importStar(require("path"));
+const client_1 = require("@metrio-ai/client");
+const errors_1 = require("../errors");
+const shared_1 = require("./shared");
+const sync_1 = require("./sync");
+const workspace_1 = require("../sync/workspace");
+const cases_1 = require("../verify/cases");
+const execute_1 = require("../verify/execute");
+const report_1 = require("../verify/report");
+exports.VERIFY_HELP = `metrio verify - run cases.yaml and check every reply
+
+Usage:
+  metrio verify [paths…] [options]               Workspace prompts/<name>/cases.yaml and
+                                                 workflows/<id>/cases.yaml
+  metrio verify -p <prompt-id> --cases <file>    One prompt, any cases file
+  metrio verify -w <workflow-id> --cases <file>  One workflow, any cases file
+
+In a workspace (see "metrio pull --help") each cases.yaml runs against the resource
+recorded in .metrio/state.json. Paths narrow the run: a resource directory, prompts/,
+workflows/ or the root.
+
+Options:
+  --version active|pushed|<n>   active (default): the live version; pushed: the version
+                                the last "metrio push" created; <n>: prompt version serial
+                                or workflow version
+                                (a number needs a single prompt or workflow path)
+  --concurrency <n>             Cases in flight (default 4)
+  --json                        Print results as JSON instead of the text report
+  --junit <file>                Also write a JUnit XML report
+  --fail-fast                   Start no new case after the first failure
+  --timezone <tz>               IANA timezone sent with every case
+  -e, --env <file>              Env file (default: .env in the current dir)
+  --api-key <key>               Project API key (default: METRIOAI_API_KEY); it must belong
+                                to the workspace's project
+  --base-url <url>              API base URL
+
+cases.yaml:
+  defaults:
+    channel: { User: { customer_id: "123" } }   # workflows only
+    variables: { region: TW }
+  cases:
+    - name: regular order
+      message: "我要訂 3 箱蘋果"          # or messages: [...] or messages_file: path
+      variables: { region: HK }           # merged key by key over defaults
+      expect:                             # optional; every listed assertion must hold
+        contains: ["蘋果"]                # reply text; also not_contains, regex, equals
+        json:                             # $.content $.structContent $.state $.tags, and
+          "$.state.order.qty": { equals: 3 }   # $.json (reply text parsed as JSON);
+          "$.tags": { contains: order }        # matchers: equals, exists, contains
+        path_includes: [classify, create_order]   # workflows: ordered node subsequence
+        terminal_node: create_order               # workflows
+        response_type: text                       # workflows: text | struct | none
+        max_latency_ms: 8000
+        max_tokens: 4000                          # input + output
+        max_llm_calls: 3                          # workflows
+
+Agent loop: pull → edit → push → verify --version pushed → push --activate
+
+Exit codes: 0 all passed, 1 a case failed / errored / was skipped, 2 invalid cases or options.
+`;
+function positive(raw, flag) {
+    const n = Number(raw);
+    if (!Number.isInteger(n) || n <= 0)
+        throw new errors_1.CliError(`${flag} must be a positive integer, got "${raw}"`);
+    return n;
+}
+function parseVerifyArgs(argv) {
+    const o = { paths: [], version: 'active', concurrency: 4, json: false, failFast: false };
+    for (let i = 0; i < argv.length; i += 1) {
+        const arg = argv[i];
+        const value = () => {
+            const v = argv[i + 1];
+            if (v === undefined)
+                throw new errors_1.CliError(`${arg} needs a value`);
+            i += 1;
+            return v;
+        };
+        switch (arg) {
+            case '--version': {
+                const v = value();
+                o.version = v === 'active' || v === 'pushed' ? v : positive(v, '--version');
+                break;
+            }
+            case '--concurrency':
+                o.concurrency = positive(value(), arg);
+                break;
+            case '--json':
+                o.json = true;
+                break;
+            case '--junit':
+                o.junit = value();
+                break;
+            case '--fail-fast':
+                o.failFast = true;
+                break;
+            case '-p':
+            case '--prompt-id':
+                o.promptId = positive(value(), arg);
+                break;
+            case '-w':
+            case '--workflow-id': {
+                const v = value().trim();
+                if (!v)
+                    throw new errors_1.CliError(`${arg} cannot be empty`);
+                o.workflowId = v;
+                break;
+            }
+            case '--cases':
+                o.cases = value();
+                break;
+            case '--timezone':
+                o.timezone = value();
+                break;
+            case '-e':
+            case '--env':
+                o.envFile = value();
+                break;
+            case '--api-key':
+                o.apiKey = value();
+                break;
+            case '--base-url':
+                o.baseUrl = value();
+                break;
+            default:
+                if (arg.startsWith('-'))
+                    throw new errors_1.CliError(`Unknown option ${arg}. Run "metrio verify --help".`);
+                o.paths.push(arg);
+        }
+    }
+    const direct = o.promptId !== undefined || o.workflowId !== undefined;
+    if (o.promptId !== undefined && o.workflowId !== undefined)
+        throw new errors_1.CliError('Use -p or -w, not both');
+    if (direct && !o.cases)
+        throw new errors_1.CliError('-p / -w need --cases <file>');
+    if (!direct && o.cases)
+        throw new errors_1.CliError('--cases needs -p <prompt-id> or -w <workflow-id>');
+    if (direct && o.paths.length)
+        throw new errors_1.CliError('Workspace paths cannot be combined with -p / -w');
+    if (direct && o.version === 'pushed') {
+        throw new errors_1.CliError('--version pushed reads .metrio/state.json; with -p / -w use active or a version number');
+    }
+    return o;
+}
+function directTarget(o) {
+    const kind = o.promptId !== undefined ? 'prompt' : 'workflow';
+    const file = o.cases;
+    let raw;
+    try {
+        if (!fs.statSync(file).isFile())
+            throw new Error('not a file');
+        raw = fs.readFileSync(file, 'utf8');
+    }
+    catch (err) {
+        throw new errors_1.CliError(`Cannot read cases file ${file}: ${err.message}`);
+    }
+    const ref = kind === 'prompt' ? o.promptId : o.workflowId;
+    return {
+        label: `${kind} ${ref}`,
+        kind,
+        ref,
+        version: typeof o.version === 'number' ? o.version : undefined,
+        cases: (0, cases_1.parseCasesYaml)(raw, { kind, file, baseDir: path.dirname(path.resolve(file)) }),
+    };
+}
+function resolveTargets(o) {
+    if (o.cases)
+        return [directTarget(o)];
+    const root = (0, workspace_1.findRoot)();
+    if (!root)
+        throw new errors_1.CliError('Not in a metrio workspace (no metrio.yaml). Run it inside one, or use -p / -w with --cases.');
+    const state = (0, workspace_1.readState)(root, (0, workspace_1.readProject)(root));
+    const listed = (0, workspace_1.listLocal)(root, ['prompt', 'workflow']);
+    // Filter by path first so an unrelated resource's cases.yaml is never read.
+    const inScope = (0, sync_1.filterEntries)(listed, root, o.paths);
+    const read = (rs) => rs.map((r) => ({ ...r, cases: (0, workspace_1.readResourceFile)(root, r.key, 'cases.yaml') }));
+    for (const p of o.paths) {
+        if (!read((0, sync_1.filterEntries)(listed, root, [p])).some((r) => r.cases !== undefined))
+            throw new errors_1.CliError(`${p}: no cases.yaml found`);
+    }
+    const selected = read(inScope).filter((r) => r.cases !== undefined);
+    if (!selected.length) {
+        throw new errors_1.CliError('No cases.yaml in this workspace. Add prompts/<name>/cases.yaml or workflows/<id>/cases.yaml.');
+    }
+    if (typeof o.version === 'number' && selected.length > 1) {
+        throw new errors_1.CliError(`--version <n> pins one resource; name a single prompt or workflow path (matched: ${selected.map((r) => r.key).join(', ')})`);
+    }
+    return selected.map((r) => {
+        const entry = state.resources[r.key];
+        if (!entry)
+            throw new errors_1.CliError(`${r.key} is not in .metrio/state.json yet; run "metrio push" first`);
+        const kind = r.kind;
+        let version;
+        if (typeof o.version === 'number')
+            version = o.version;
+        else if (o.version === 'pushed') {
+            version = kind === 'prompt' ? entry.pushed_serial : entry.pushed_workflow_version;
+            if (version === undefined) {
+                throw new errors_1.CliError(`${r.key}: no pushed version recorded in .metrio/state.json; run "metrio push"; a resource created by its first push (e.g. a new workflow) has none, so use --version active for it`);
+            }
+        }
+        const ref = kind === 'prompt' ? Number(entry.id) : entry.id;
+        if (kind === 'prompt' && (!Number.isInteger(ref) || ref <= 0)) {
+            throw new errors_1.CliError(`${r.key}: prompt id "${entry.id}" in .metrio/state.json is not a positive integer`);
+        }
+        return {
+            label: r.key,
+            kind,
+            ref,
+            version,
+            cases: (0, cases_1.parseCasesYaml)(r.cases, { kind, file: `${r.key}/cases.yaml`, baseDir: path.join(root, r.key), root }),
+        };
+    });
+}
+async function runVerifyCommand(argv) {
+    if (argv.includes('-h') || argv.includes('--help')) {
+        process.stdout.write(exports.VERIFY_HELP);
+        return;
+    }
+    const o = parseVerifyArgs(argv);
+    const targets = resolveTargets(o);
+    (0, shared_1.loadEnv)(o.envFile);
+    const client = new client_1.MetrioAI({ apiKey: (0, shared_1.resolveApiKey)(o.apiKey), baseUrl: (0, shared_1.resolveBaseUrl)(o.baseUrl) });
+    const results = await (0, execute_1.runAll)(client, targets, {
+        concurrency: o.concurrency,
+        failFast: o.failFast,
+        timezone: o.timezone,
+        onResult: (r, done, total) => process.stderr.write((0, sync_1.safe)(`[${done}/${total}] ${r.resource} › ${r.case} ${r.status}\n`)),
+    });
+    process.stdout.write(o.json ? (0, report_1.formatJson)(results) : (0, report_1.formatHuman)(results));
+    if (o.junit) {
+        try {
+            fs.writeFileSync(path.resolve(o.junit), (0, report_1.formatJunit)(results));
+        }
+        catch (err) {
+            process.stderr.write(`Error: cannot write --junit file ${o.junit} (${err.message})\n`);
+            process.exitCode = 1;
+        }
+    }
+    const s = (0, report_1.summarize)(results);
+    if (s.passed !== s.total)
+        process.exitCode = 1;
+}

package/dist/index.js

@@ -18,7 +18,9 @@ const chat_1 = require("./commands/chat");
 const prompt_1 = require("./commands/prompt");
 const workflow_1 = require("./commands/workflow");
 const tools_1 = require("./commands/tools");
+const verify_1 = require("./commands/verify");
 const run_1 = require("./manage/run");
+const sync_1 = require("./commands/sync");
 const registry_1 = require("./manage/registry");
 exports.HELP = `metrio - MetrioAI CLI
 
@@ -30,6 +32,7 @@ Run (project API key, see "metrio chat --help"):
   workflow run-batch    Run several workflow cases in one process
   prompt run-batch      Run several prompt cases in one process (metrio prompt --help)
   prompt create-version Create a prompt version from a file
+  verify [paths…]       Run cases.yaml and check replies ("metrio verify --help")
 
 Account (console OAuth):
   login                 Sign in to the Metrio console in your browser
@@ -40,6 +43,7 @@ Manage (console OAuth; run "metrio <group> --help"):
   project | prompt | version | skill | workflow | schedule | mcp | run | admin | model-tiers
                         e.g. metrio version create 12 -f system.md
                              metrio skill get 5 -o skill.md
+  pull | diff | push    Sync prompts, skills and workflows with local files ("metrio pull --help")
   tools                 List the console tools your login can call
   call <tool>           Call any console tool with JSON args (-f file | --args json)
 
@@ -81,6 +85,14 @@ async function main(argv) {
             return runMixedGroup('workflow', argv);
         case 'prompt':
             return runMixedGroup('prompt', argv);
+        case 'pull':
+            return (0, sync_1.runPullCommand)(argv.slice(1));
+        case 'diff':
+            return (0, sync_1.runDiffCommand)(argv.slice(1));
+        case 'push':
+            return (0, sync_1.runPushCommand)(argv.slice(1));
+        case 'verify':
+            return (0, verify_1.runVerifyCommand)(argv.slice(1));
         case 'login':
             return (0, auth_1.runLoginCommand)(argv.slice(1));
         case 'logout':

package/dist/manage/output.js

@@ -110,7 +110,12 @@ async function writeOutput(data, opts) {
         process.stdout.write(record.text.endsWith('\n') ? record.text : `${record.text}\n`);
         return;
     }
-    const rows = Array.isArray(data) ? data : Array.isArray(record.items) ? record.items : undefined;
+    const listKey = Array.isArray(record.items)
+        ? 'items'
+        : 'pagination' in record
+            ? Object.keys(record).find((k) => Array.isArray(record[k]))
+            : undefined;
+    const rows = Array.isArray(data) ? data : listKey ? record[listKey] : undefined;
     if (rows) {
         process.stdout.write(rows.length ? `${renderTable(rows)}\n` : '(none)\n');
         const p = asRecord(record.pagination);

package/dist/manage/run.js

@@ -7,6 +7,7 @@ const input_1 = require("./input");
 const output_1 = require("./output");
 const project_1 = require("./project");
 const registry_1 = require("./registry");
+const workspace_1 = require("../sync/workspace");
 const schema_1 = require("./schema");
 function parse(rest) {
     const parsed = { tokens: [], json: false, yes: false };
@@ -66,6 +67,27 @@ function isCommandPrefix(argv) {
     const words = argv.filter((a) => !a.startsWith('-'));
     return registry_1.COMMANDS.some((c) => words.length < c.path.length && words.every((w, i) => c.path[i] === w));
 }
+/**
+ * A prompt positional may name a tracked directory (prompts/<dir> or <dir>) instead of the
+ * numeric id — only for the workspace's own project, since ids are per project.
+ */
+function promptRef(raw, project) {
+    if (/^\d+$/.test(raw))
+        return raw;
+    const root = (0, workspace_1.findRoot)();
+    const dir = raw.replace(/\/+$/, '');
+    const key = dir.startsWith('prompts/') ? dir : `prompts/${dir}`;
+    // A broken workspace surfaces its own (informative) CliError.
+    const workspaceProject = root ? (0, workspace_1.readProject)(root) : undefined;
+    const entry = root ? (0, workspace_1.readState)(root, workspaceProject).resources[key] : undefined;
+    if (!entry || entry.kind !== 'prompt') {
+        throw new errors_1.CliError(`Unknown prompt "${raw}": use the numeric id, or a prompts/<dir> tracked in .metrio/state.json`);
+    }
+    if (project !== undefined && project !== workspaceProject) {
+        throw new errors_1.CliError(`"${raw}" is a directory of project ${workspaceProject}, but this command targets project ${String(project)}`);
+    }
+    return entry.id;
+}
 /** Runs one registry command; argv starts with the group, e.g. ['version', 'create', '12', '-f', 's.md']. */
 async function runManageCommand(argv, readStdin) {
     const match = (0, registry_1.findCommand)(argv);
@@ -107,7 +129,14 @@ async function runManageCommand(argv, readStdin) {
         put('project_slug', opts.project, 'project');
     for (const [k, v] of Object.entries(values))
         put(k, v, `--${(0, schema_1.kebab)(k)}`);
-    expected.forEach((arg, i) => put(arg.prop, (0, schema_1.coercePositional)(positionals[i], props[arg.prop], arg.label), arg.label));
+    // A prompt directory resolves against the final project_slug, so it waits for it.
+    let promptDir;
+    expected.forEach((arg, i) => {
+        if (arg.prop === 'prompt_external_id' && !/^\d+$/.test(positionals[i]))
+            promptDir = { prop: arg.prop, raw: positionals[i], label: arg.label };
+        else
+            put(arg.prop, (0, schema_1.coercePositional)(positionals[i], props[arg.prop], arg.label), arg.label);
+    });
     if ('project_slug' in props && !opts.project && args.project_slug === undefined) {
         try {
             const project = (0, project_1.resolveProject)(undefined);
@@ -119,6 +148,9 @@ async function runManageCommand(argv, readStdin) {
                 throw err;
         }
     }
+    if (promptDir) {
+        put(promptDir.prop, (0, schema_1.coercePositional)(promptRef(promptDir.raw, args.project_slug), props[promptDir.prop], promptDir.label), promptDir.label);
+    }
     if ((schema.required ?? []).includes('project_slug') && args.project_slug === undefined) {
         throw new errors_1.CliError('No project. Pass --project <slug>, set METRIO_PROJECT, or add "project: <slug>" to metrio.yaml.');
     }