@metrio-ai/cli 1.2.0 → 1.3.1

package/README.md

@@ -29,7 +29,7 @@ export METRIO_PROJECT=my-project  # or --project, or "project: my-project" in me
 
 metrio prompt list
 metrio version get 34 -o system.md          # the system prompt as a file
-metrio version create 12 -f system.md --model-name gpt-5
+metrio version create 12 -f system.md --model-provider openai --model-name gpt-5
 metrio version get 34 -o v.yaml && metrio version update 34 -f v.yaml   # round-trip
 metrio skill get 5 -o skill.md              # frontmatter (name, description, references) + body
 metrio skill update 5 -f skill.md
@@ -67,3 +67,30 @@ metrio version list pick-products     # prompt directories work wherever a promp
 - `pull` adopts untracked local files that are identical to the console. `pull` and `push` report per-resource failures, exit 1, and still save state for the resources that succeeded.
 - Renaming a prompt or skill directory is detected by content (status `renamed`); pull/push move the state entry. Workflow directories are named by workflow id and cannot be renamed locally; push refuses it.
 - Known limitation: renaming AND editing a prompt/skill directory in one go looks like a delete plus a new resource (push would create a second one; `--prune --yes` would delete the old one). Rename first, sync, then edit.
+
+## Verify prompts and workflows
+
+Put a `cases.yaml` next to a synced resource (`prompts/<name>/cases.yaml`, `workflows/<id>/cases.yaml`) and run it with the project API key (`METRIOAI_API_KEY` in `.env`, or `--api-key`; it must belong to the workspace's project):
+
+```yaml
+defaults:
+  variables: { region: TW }
+cases:
+  - name: regular order
+    message: "我要訂 3 箱蘋果"            # or messages: [...] or messages_file: path
+    expect:
+      contains: ["蘋果"]                  # not_contains, regex, equals read the reply text
+      json:
+        "$.state.order.qty": { equals: 3 }   # $.content $.structContent $.state $.tags $.json
+      path_includes: [classify, create_order]   # workflows only (also terminal_node,
+      max_latency_ms: 8000                      #   response_type, max_llm_calls)
+```
+
+```bash
+metrio push -m "new wording"          # creates a version that is not live yet
+metrio verify --version pushed        # run every cases.yaml against what you just pushed
+metrio push --activate                # make it live once the cases pass
+metrio verify -p 12 --cases cases.yaml --junit report.xml   # outside a workspace
+```
+
+Every `cases.yaml` is validated before the first request (a typo in an assertion is exit 2). A failed case prints each failing assertion with expected vs actual and exits 1. `--json` prints machine-readable results, `--junit <file>` writes JUnit XML, `--fail-fast` stops starting new cases after the first failure. Run `metrio verify --help` for the full format.

package/dist/commands/sync.js

@@ -38,6 +38,7 @@ exports.parseSyncArgs = parseSyncArgs;
 exports.openWorkspace = openWorkspace;
 exports.filterEntries = filterEntries;
 exports.loadEntries = loadEntries;
+exports.safe = safe;
 exports.runDiffCommand = runDiffCommand;
 exports.runPullCommand = runPullCommand;
 exports.runPushCommand = runPushCommand;

package/dist/commands/verify.js

@@ -0,0 +1,288 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.VERIFY_HELP = void 0;
+exports.parseVerifyArgs = parseVerifyArgs;
+exports.resolveTargets = resolveTargets;
+exports.runVerifyCommand = runVerifyCommand;
+/**
+ * `metrio verify` — runs cases.yaml against prompts and workflows and checks every reply.
+ * Execution uses the project API key, like `chat` and `run-batch`. Targets come from the
+ * workspace (.metrio/state.json) or from -p / -w with --cases. Every cases.yaml is
+ * validated before the first request.
+ */
+const fs = __importStar(require("fs"));
+const path = __importStar(require("path"));
+const client_1 = require("@metrio-ai/client");
+const errors_1 = require("../errors");
+const shared_1 = require("./shared");
+const sync_1 = require("./sync");
+const workspace_1 = require("../sync/workspace");
+const cases_1 = require("../verify/cases");
+const execute_1 = require("../verify/execute");
+const report_1 = require("../verify/report");
+exports.VERIFY_HELP = `metrio verify - run cases.yaml and check every reply
+
+Usage:
+  metrio verify [paths…] [options]               Workspace prompts/<name>/cases.yaml and
+                                                 workflows/<id>/cases.yaml
+  metrio verify -p <prompt-id> --cases <file>    One prompt, any cases file
+  metrio verify -w <workflow-id> --cases <file>  One workflow, any cases file
+
+In a workspace (see "metrio pull --help") each cases.yaml runs against the resource
+recorded in .metrio/state.json. Paths narrow the run: a resource directory, prompts/,
+workflows/ or the root.
+
+Options:
+  --version active|pushed|<n>   active (default): the live version; pushed: the version
+                                the last "metrio push" created; <n>: prompt version serial
+                                or workflow version
+                                (a number needs a single prompt or workflow path)
+  --concurrency <n>             Cases in flight (default 4)
+  --json                        Print results as JSON instead of the text report
+  --junit <file>                Also write a JUnit XML report
+  --fail-fast                   Start no new case after the first failure
+  --timezone <tz>               IANA timezone sent with every case
+  -e, --env <file>              Env file (default: .env in the current dir)
+  --api-key <key>               Project API key (default: METRIOAI_API_KEY); it must belong
+                                to the workspace's project
+  --base-url <url>              API base URL
+
+cases.yaml:
+  defaults:
+    channel: { User: { customer_id: "123" } }   # workflows only
+    variables: { region: TW }
+  cases:
+    - name: regular order
+      message: "我要訂 3 箱蘋果"          # or messages: [...] or messages_file: path
+      variables: { region: HK }           # merged key by key over defaults
+      expect:                             # optional; every listed assertion must hold
+        contains: ["蘋果"]                # reply text; also not_contains, regex, equals
+        json:                             # $.content $.structContent $.state $.tags, and
+          "$.state.order.qty": { equals: 3 }   # $.json (reply text parsed as JSON);
+          "$.tags": { contains: order }        # matchers: equals, exists, contains
+        path_includes: [classify, create_order]   # workflows: ordered node subsequence
+        terminal_node: create_order               # workflows
+        response_type: text                       # workflows: text | struct | none
+        max_latency_ms: 8000
+        max_tokens: 4000                          # input + output
+        max_llm_calls: 3                          # workflows
+
+Agent loop: pull → edit → push → verify --version pushed → push --activate
+
+Exit codes: 0 all passed, 1 a case failed / errored / was skipped, 2 invalid cases or options.
+`;
+function positive(raw, flag) {
+    const n = Number(raw);
+    if (!Number.isInteger(n) || n <= 0)
+        throw new errors_1.CliError(`${flag} must be a positive integer, got "${raw}"`);
+    return n;
+}
+function parseVerifyArgs(argv) {
+    const o = { paths: [], version: 'active', concurrency: 4, json: false, failFast: false };
+    for (let i = 0; i < argv.length; i += 1) {
+        const arg = argv[i];
+        const value = () => {
+            const v = argv[i + 1];
+            if (v === undefined)
+                throw new errors_1.CliError(`${arg} needs a value`);
+            i += 1;
+            return v;
+        };
+        switch (arg) {
+            case '--version': {
+                const v = value();
+                o.version = v === 'active' || v === 'pushed' ? v : positive(v, '--version');
+                break;
+            }
+            case '--concurrency':
+                o.concurrency = positive(value(), arg);
+                break;
+            case '--json':
+                o.json = true;
+                break;
+            case '--junit':
+                o.junit = value();
+                break;
+            case '--fail-fast':
+                o.failFast = true;
+                break;
+            case '-p':
+            case '--prompt-id':
+                o.promptId = positive(value(), arg);
+                break;
+            case '-w':
+            case '--workflow-id': {
+                const v = value().trim();
+                if (!v)
+                    throw new errors_1.CliError(`${arg} cannot be empty`);
+                o.workflowId = v;
+                break;
+            }
+            case '--cases':
+                o.cases = value();
+                break;
+            case '--timezone':
+                o.timezone = value();
+                break;
+            case '-e':
+            case '--env':
+                o.envFile = value();
+                break;
+            case '--api-key':
+                o.apiKey = value();
+                break;
+            case '--base-url':
+                o.baseUrl = value();
+                break;
+            default:
+                if (arg.startsWith('-'))
+                    throw new errors_1.CliError(`Unknown option ${arg}. Run "metrio verify --help".`);
+                o.paths.push(arg);
+        }
+    }
+    const direct = o.promptId !== undefined || o.workflowId !== undefined;
+    if (o.promptId !== undefined && o.workflowId !== undefined)
+        throw new errors_1.CliError('Use -p or -w, not both');
+    if (direct && !o.cases)
+        throw new errors_1.CliError('-p / -w need --cases <file>');
+    if (!direct && o.cases)
+        throw new errors_1.CliError('--cases needs -p <prompt-id> or -w <workflow-id>');
+    if (direct && o.paths.length)
+        throw new errors_1.CliError('Workspace paths cannot be combined with -p / -w');
+    if (direct && o.version === 'pushed') {
+        throw new errors_1.CliError('--version pushed reads .metrio/state.json; with -p / -w use active or a version number');
+    }
+    return o;
+}
+function directTarget(o) {
+    const kind = o.promptId !== undefined ? 'prompt' : 'workflow';
+    const file = o.cases;
+    let raw;
+    try {
+        if (!fs.statSync(file).isFile())
+            throw new Error('not a file');
+        raw = fs.readFileSync(file, 'utf8');
+    }
+    catch (err) {
+        throw new errors_1.CliError(`Cannot read cases file ${file}: ${err.message}`);
+    }
+    const ref = kind === 'prompt' ? o.promptId : o.workflowId;
+    return {
+        label: `${kind} ${ref}`,
+        kind,
+        ref,
+        version: typeof o.version === 'number' ? o.version : undefined,
+        cases: (0, cases_1.parseCasesYaml)(raw, { kind, file, baseDir: path.dirname(path.resolve(file)) }),
+    };
+}
+function resolveTargets(o) {
+    if (o.cases)
+        return [directTarget(o)];
+    const root = (0, workspace_1.findRoot)();
+    if (!root)
+        throw new errors_1.CliError('Not in a metrio workspace (no metrio.yaml). Run it inside one, or use -p / -w with --cases.');
+    const state = (0, workspace_1.readState)(root, (0, workspace_1.readProject)(root));
+    const listed = (0, workspace_1.listLocal)(root, ['prompt', 'workflow']);
+    // Filter by path first so an unrelated resource's cases.yaml is never read.
+    const inScope = (0, sync_1.filterEntries)(listed, root, o.paths);
+    const read = (rs) => rs.map((r) => ({ ...r, cases: (0, workspace_1.readResourceFile)(root, r.key, 'cases.yaml') }));
+    for (const p of o.paths) {
+        if (!read((0, sync_1.filterEntries)(listed, root, [p])).some((r) => r.cases !== undefined))
+            throw new errors_1.CliError(`${p}: no cases.yaml found`);
+    }
+    const selected = read(inScope).filter((r) => r.cases !== undefined);
+    if (!selected.length) {
+        throw new errors_1.CliError('No cases.yaml in this workspace. Add prompts/<name>/cases.yaml or workflows/<id>/cases.yaml.');
+    }
+    if (typeof o.version === 'number' && selected.length > 1) {
+        throw new errors_1.CliError(`--version <n> pins one resource; name a single prompt or workflow path (matched: ${selected.map((r) => r.key).join(', ')})`);
+    }
+    return selected.map((r) => {
+        const entry = state.resources[r.key];
+        if (!entry)
+            throw new errors_1.CliError(`${r.key} is not in .metrio/state.json yet; run "metrio push" first`);
+        const kind = r.kind;
+        let version;
+        if (typeof o.version === 'number')
+            version = o.version;
+        else if (o.version === 'pushed') {
+            version = kind === 'prompt' ? entry.pushed_serial : entry.pushed_workflow_version;
+            if (version === undefined) {
+                throw new errors_1.CliError(`${r.key}: no pushed version recorded in .metrio/state.json; run "metrio push"; a resource created by its first push (e.g. a new workflow) has none, so use --version active for it`);
+            }
+        }
+        const ref = kind === 'prompt' ? Number(entry.id) : entry.id;
+        if (kind === 'prompt' && (!Number.isInteger(ref) || ref <= 0)) {
+            throw new errors_1.CliError(`${r.key}: prompt id "${entry.id}" in .metrio/state.json is not a positive integer`);
+        }
+        return {
+            label: r.key,
+            kind,
+            ref,
+            version,
+            cases: (0, cases_1.parseCasesYaml)(r.cases, { kind, file: `${r.key}/cases.yaml`, baseDir: path.join(root, r.key), root }),
+        };
+    });
+}
+async function runVerifyCommand(argv) {
+    if (argv.includes('-h') || argv.includes('--help')) {
+        process.stdout.write(exports.VERIFY_HELP);
+        return;
+    }
+    const o = parseVerifyArgs(argv);
+    const targets = resolveTargets(o);
+    (0, shared_1.loadEnv)(o.envFile);
+    const client = new client_1.MetrioAI({ apiKey: (0, shared_1.resolveApiKey)(o.apiKey), baseUrl: (0, shared_1.resolveBaseUrl)(o.baseUrl) });
+    const results = await (0, execute_1.runAll)(client, targets, {
+        concurrency: o.concurrency,
+        failFast: o.failFast,
+        timezone: o.timezone,
+        onResult: (r, done, total) => process.stderr.write((0, sync_1.safe)(`[${done}/${total}] ${r.resource} › ${r.case} ${r.status}\n`)),
+    });
+    process.stdout.write(o.json ? (0, report_1.formatJson)(results) : (0, report_1.formatHuman)(results));
+    if (o.junit) {
+        try {
+            fs.writeFileSync(path.resolve(o.junit), (0, report_1.formatJunit)(results));
+        }
+        catch (err) {
+            process.stderr.write(`Error: cannot write --junit file ${o.junit} (${err.message})\n`);
+            process.exitCode = 1;
+        }
+    }
+    const s = (0, report_1.summarize)(results);
+    if (s.passed !== s.total)
+        process.exitCode = 1;
+}

package/dist/index.js

@@ -18,6 +18,7 @@ const chat_1 = require("./commands/chat");
 const prompt_1 = require("./commands/prompt");
 const workflow_1 = require("./commands/workflow");
 const tools_1 = require("./commands/tools");
+const verify_1 = require("./commands/verify");
 const run_1 = require("./manage/run");
 const sync_1 = require("./commands/sync");
 const registry_1 = require("./manage/registry");
@@ -31,6 +32,7 @@ Run (project API key, see "metrio chat --help"):
   workflow run-batch    Run several workflow cases in one process
   prompt run-batch      Run several prompt cases in one process (metrio prompt --help)
   prompt create-version Create a prompt version from a file
+  verify [paths…]       Run cases.yaml and check replies ("metrio verify --help")
 
 Account (console OAuth):
   login                 Sign in to the Metrio console in your browser
@@ -89,6 +91,8 @@ async function main(argv) {
             return (0, sync_1.runDiffCommand)(argv.slice(1));
         case 'push':
             return (0, sync_1.runPushCommand)(argv.slice(1));
+        case 'verify':
+            return (0, verify_1.runVerifyCommand)(argv.slice(1));
         case 'login':
             return (0, auth_1.runLoginCommand)(argv.slice(1));
         case 'logout':

package/dist/sync/workspace.js

@@ -43,6 +43,7 @@ exports.acquireLock = acquireLock;
 exports.hashFiles = hashFiles;
 exports.slugify = slugify;
 exports.readLocal = readLocal;
+exports.readResourceFile = readResourceFile;
 exports.writeLocal = writeLocal;
 exports.removeLocal = removeLocal;
 exports.listLocal = listLocal;
@@ -323,6 +324,16 @@ function readLocal(root, key, kind) {
     }
     return Object.keys(files).length ? files : undefined;
 }
+/** An extra file inside a resource directory (e.g. cases.yaml); undefined when absent. Refuses symlinks. */
+function readResourceFile(root, key, name) {
+    const file = path.join(resolveKey(root, key), name);
+    assertNotSymlink(file, `${key}/${name}`);
+    if (!fs.existsSync(file))
+        return undefined;
+    if (!fs.lstatSync(file).isFile())
+        throw new errors_1.CliError(`${key}/${name} is not a file`);
+    return fs.readFileSync(file, 'utf8');
+}
 function writeLocal(root, key, kind, files) {
     const targetPath = resolveKey(root, key);
     if (kind === 'skill') {

package/dist/verify/assert.js

@@ -0,0 +1,117 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.observePrompt = observePrompt;
+exports.observeWorkflow = observeWorkflow;
+exports.evaluate = evaluate;
+/**
+ * Turns a chat / workflow response into one Observation, and checks an Expect against it.
+ */
+const util_1 = require("util");
+const workflow_1 = require("../commands/workflow");
+const jsonpath_1 = require("./jsonpath");
+function parseJson(text) {
+    try {
+        return JSON.parse(text);
+    }
+    catch {
+        return undefined;
+    }
+}
+const isObject = (v) => typeof v === 'object' && v !== null && !Array.isArray(v);
+const pretty = (v) => JSON.stringify(v ?? null, null, 2);
+function observePrompt(res) {
+    const raw = res.response ?? '';
+    const parsed = parseJson(raw);
+    // A prompt using the final_response tool answers with that JSON as its response string.
+    const final = isObject(parsed) && typeof parsed.responseType === 'string' ? parsed : undefined;
+    const content = final ? (typeof final.content === 'string' ? final.content : '') : raw;
+    // undefined (not null) when absent, so `$.structContent… exists: false` holds.
+    const structContent = final?.structContent;
+    const json = final ? parseJson(content) : parsed;
+    return {
+        text: final?.responseType === 'struct' ? pretty(structContent) : content,
+        root: { content, structContent, state: {}, tags: [], json: json ?? null },
+        path: [],
+        terminalNodeId: null,
+        responseType: final ? final.responseType : null,
+        latencyMs: res.elapsedTime,
+        tokens: (res.inputTokens ?? 0) + (res.outputTokens ?? 0),
+        llmCalls: null,
+        ...(final?.status === 'failed' ? { failedReason: typeof final.reason === 'string' ? final.reason : '' } : {}),
+    };
+}
+function observeWorkflow(res) {
+    const shaped = (0, workflow_1.shapeWorkflowOutput)(res, 'summary');
+    const reply = res.reply ?? {};
+    const content = typeof reply.content === 'string' ? reply.content : '';
+    const structContent = reply.structContent;
+    const text = reply.responseType === 'struct' ? pretty(structContent) : reply.responseType === 'text' ? content : '';
+    return {
+        text,
+        root: {
+            content,
+            structContent,
+            state: shaped.state ?? {},
+            tags: shaped.tags ?? [],
+            json: parseJson(content) ?? null,
+        },
+        path: shaped.path,
+        terminalNodeId: shaped.terminalNodeId ?? null,
+        responseType: reply.responseType ?? null,
+        latencyMs: res.elapsedTime,
+        tokens: (res.inputTokens ?? 0) + (res.outputTokens ?? 0),
+        llmCalls: res.llmCalls ?? 0,
+    };
+}
+function containsValue(actual, wanted) {
+    if (Array.isArray(actual))
+        return actual.some((item) => (0, util_1.isDeepStrictEqual)(item, wanted));
+    if (typeof actual === 'string' && typeof wanted === 'string')
+        return actual.includes(wanted);
+    return false;
+}
+/** Ordered, not necessarily contiguous. */
+function isSubsequence(wanted, path) {
+    let i = 0;
+    for (const node of path)
+        if (node === wanted[i])
+            i += 1;
+    return i === wanted.length;
+}
+function evaluate(e, obs) {
+    const failures = [];
+    const fail = (assertion, expected, actual) => failures.push({ assertion, expected, actual });
+    for (const s of e.contains ?? [])
+        if (!obs.text.includes(s))
+            fail('contains', s, obs.text);
+    for (const s of e.not_contains ?? [])
+        if (obs.text.includes(s))
+            fail('not_contains', s, obs.text);
+    for (const r of e.regex ?? [])
+        if (!new RegExp(r, 'u').test(obs.text))
+            fail('regex', r, obs.text);
+    if (e.equals !== undefined && obs.text.trim() !== e.equals.trim())
+        fail('equals', e.equals, obs.text);
+    for (const check of e.json ?? []) {
+        const actual = (0, jsonpath_1.readPath)(obs.root, check.segments);
+        if (check.exists !== undefined && (actual !== undefined) !== check.exists)
+            fail(`json ${check.path} exists`, check.exists, actual);
+        if (check.equals && !(0, util_1.isDeepStrictEqual)(actual, check.equals.value))
+            fail(`json ${check.path} equals`, check.equals.value, actual);
+        if (check.contains && !containsValue(actual, check.contains.value))
+            fail(`json ${check.path} contains`, check.contains.value, actual);
+    }
+    if (e.path_includes && !isSubsequence(e.path_includes, obs.path))
+        fail('path_includes', e.path_includes, obs.path);
+    if (e.terminal_node !== undefined && obs.terminalNodeId !== e.terminal_node)
+        fail('terminal_node', e.terminal_node, obs.terminalNodeId);
+    if (e.response_type !== undefined && obs.responseType !== e.response_type)
+        fail('response_type', e.response_type, obs.responseType);
+    if (e.max_latency_ms !== undefined && obs.latencyMs > e.max_latency_ms)
+        fail('max_latency_ms', e.max_latency_ms, obs.latencyMs);
+    if (e.max_tokens !== undefined && obs.tokens > e.max_tokens)
+        fail('max_tokens', e.max_tokens, obs.tokens);
+    if (e.max_llm_calls !== undefined && (obs.llmCalls ?? 0) > e.max_llm_calls)
+        fail('max_llm_calls', e.max_llm_calls, obs.llmCalls);
+    return failures;
+}

package/dist/verify/cases.js

@@ -0,0 +1,302 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.parseCasesYaml = parseCasesYaml;
+/**
+ * cases.yaml for `metrio verify`. Everything is parsed and validated before any case runs,
+ * so a typo is exit 2 instead of a paid run that silently checks nothing.
+ */
+const fs = __importStar(require("fs"));
+const path = __importStar(require("path"));
+const yaml_1 = require("yaml");
+const errors_1 = require("../errors");
+const shared_1 = require("../commands/shared");
+const jsonpath_1 = require("./jsonpath");
+const WORKFLOW_ONLY = ['path_includes', 'terminal_node', 'response_type', 'max_llm_calls'];
+const EXPECT_KEYS = ['contains', 'not_contains', 'regex', 'equals', 'json', ...WORKFLOW_ONLY, 'max_latency_ms', 'max_tokens'];
+const MESSAGE_KEYS = ['role', 'content', 'text', 'file', 'mime', 'fileType'];
+const ROLES = ['system', 'user', 'assistant'];
+const FILE_TYPES = ['image', 'pdf', 'binary'];
+const CASE_KEYS = ['name', 'message', 'messages', 'messages_file', 'variables', 'channel', 'expect'];
+const JSON_ROOTS = ['content', 'structContent', 'state', 'tags', 'json'];
+const MATCHERS = ['equals', 'exists', 'contains'];
+const isObject = (v) => typeof v === 'object' && v !== null && !Array.isArray(v);
+function stringList(v, where) {
+    const list = typeof v === 'string' ? [v] : v;
+    if (!Array.isArray(list) || list.length === 0 || !list.every((s) => typeof s === 'string' && s !== '')) {
+        throw new Error(`${where} must be a non-empty string or a list of non-empty strings`);
+    }
+    return list;
+}
+function limit(v, where) {
+    if (typeof v !== 'number' || !Number.isFinite(v) || v < 0)
+        throw new Error(`${where} must be a number >= 0`);
+    return v;
+}
+function mapping(v, where) {
+    if (!isObject(v))
+        throw new Error(`${where} must be a mapping`);
+    return v;
+}
+/** A file a case reads, resolved against baseDir; in a workspace it must stay inside the root. */
+function localPath(p, opts, where) {
+    const full = path.resolve(opts.baseDir, p);
+    if (opts.root) {
+        let real;
+        try {
+            real = fs.realpathSync(full);
+        }
+        catch (err) {
+            throw new Error(`${where} "${p}": ${err.message}`);
+        }
+        const rel = path.relative(fs.realpathSync(opts.root), real);
+        if (rel === '..' || rel.startsWith(`..${path.sep}`) || path.isAbsolute(rel)) {
+            throw new Error(`${where} "${p}" is outside the metrio workspace`);
+        }
+    }
+    return full;
+}
+function parseJsonChecks(raw, where) {
+    if (!isObject(raw) || Object.keys(raw).length === 0)
+        throw new Error(`${where} must map JSON paths to matchers`);
+    return Object.entries(raw).map(([expr, m]) => {
+        const at = `${where}["${expr}"]`;
+        let segments;
+        try {
+            segments = (0, jsonpath_1.parsePath)(expr);
+        }
+        catch (err) {
+            throw new Error(`${at}: ${err.message}`);
+        }
+        if (typeof segments[0] !== 'string' || !JSON_ROOTS.includes(segments[0])) {
+            throw new Error(`${at}: a path starts at one of $.${JSON_ROOTS.join(', $.')}`);
+        }
+        if (!isObject(m) || Object.keys(m).length === 0)
+            throw new Error(`${at} must be a mapping of ${MATCHERS.join(' / ')}`);
+        const check = { path: expr, segments };
+        for (const [name, value] of Object.entries(m)) {
+            if (name === 'equals')
+                check.equals = { value };
+            else if (name === 'contains')
+                check.contains = { value };
+            else if (name === 'exists') {
+                if (typeof value !== 'boolean')
+                    throw new Error(`${at}.exists must be true or false`);
+                check.exists = value;
+            }
+            else
+                throw new Error(`${at}.${name}: unknown matcher (known: ${MATCHERS.join(', ')})`);
+        }
+        return check;
+    });
+}
+function parseExpect(raw, kind, where) {
+    if (raw === undefined || raw === null)
+        return {};
+    const e = {};
+    for (const [key, v] of Object.entries(mapping(raw, where))) {
+        const at = `${where}.${key}`;
+        if (!EXPECT_KEYS.includes(key))
+            throw new Error(`${at}: unknown assertion (known: ${EXPECT_KEYS.join(', ')})`);
+        if (kind === 'prompt' && WORKFLOW_ONLY.includes(key))
+            throw new Error(`${at}: only workflows support this assertion`);
+        switch (key) {
+            case 'contains':
+            case 'not_contains':
+            case 'path_includes':
+                e[key] = stringList(v, at);
+                break;
+            case 'regex':
+                e.regex = stringList(v, at).map((r) => {
+                    try {
+                        new RegExp(r, 'u');
+                    }
+                    catch (err) {
+                        throw new Error(`${at}: invalid regex ${JSON.stringify(r)} (${err.message})`);
+                    }
+                    return r;
+                });
+                break;
+            case 'equals':
+                if (typeof v !== 'string')
+                    throw new Error(`${at} must be a string`);
+                e.equals = v;
+                break;
+            case 'json':
+                e.json = parseJsonChecks(v, at);
+                break;
+            case 'terminal_node':
+                if (typeof v !== 'string' || v === '')
+                    throw new Error(`${at} must be a node id`);
+                e.terminal_node = v;
+                break;
+            case 'response_type':
+                if (v !== 'text' && v !== 'struct' && v !== 'none')
+                    throw new Error(`${at} must be text, struct or none`);
+                e.response_type = v;
+                break;
+            default:
+                e[key] = limit(v, at);
+        }
+    }
+    return e;
+}
+function parseVariables(v, kind, where) {
+    if (v === undefined)
+        return undefined;
+    const vars = mapping(v, where);
+    if (kind === 'prompt') {
+        for (const [name, value] of Object.entries(vars)) {
+            if (!['string', 'number', 'boolean'].includes(typeof value)) {
+                throw new Error(`${where}.${name} must be a string, number or boolean (prompt variables are text)`);
+            }
+        }
+    }
+    return vars;
+}
+function caseMessages(c, opts, where) {
+    const given = ['message', 'messages', 'messages_file'].filter((k) => c[k] !== undefined);
+    if (given.length !== 1)
+        throw new Error(`${where} needs exactly one of message, messages, messages_file`);
+    let inputs;
+    let baseDir = opts.baseDir;
+    if (c.message !== undefined) {
+        if (typeof c.message !== 'string' || c.message.trim() === '')
+            throw new Error(`${where}.message must be a non-empty string`);
+        inputs = [{ role: 'user', content: c.message }];
+    }
+    else if (c.messages !== undefined) {
+        if (!Array.isArray(c.messages) || c.messages.length === 0)
+            throw new Error(`${where}.messages must be a non-empty list`);
+        inputs = c.messages;
+    }
+    else {
+        if (typeof c.messages_file !== 'string' || c.messages_file === '')
+            throw new Error(`${where}.messages_file must be a path`);
+        const file = localPath(c.messages_file, opts, `${where}.messages_file`);
+        inputs = (0, shared_1.readMessagesFile)(file);
+        if (inputs.length === 0)
+            throw new Error(`${where}.messages_file has no messages`);
+        baseDir = path.dirname(file);
+    }
+    inputs.forEach((m, i) => {
+        if (!isObject(m))
+            throw new Error(`${where} message #${i} must be a mapping`);
+        for (const key of Object.keys(m)) {
+            if (!MESSAGE_KEYS.includes(key))
+                throw new Error(`${where} message #${i}: unknown key "${key}" (expected ${MESSAGE_KEYS.join(', ')})`);
+        }
+        const at = `${where} message #${i}`;
+        if (!ROLES.includes(m.role))
+            throw new Error(`${at}: invalid role ${JSON.stringify(m.role)} (expected ${ROLES.join(', ')})`);
+        for (const key of ['content', 'text']) {
+            if (m[key] !== undefined && typeof m[key] !== 'string')
+                throw new Error(`${at}: ${key} must be a string`);
+        }
+        if (m.file !== undefined && (typeof m.file !== 'string' || m.file === ''))
+            throw new Error(`${at}: file must be a path string`);
+        if (m.mime !== undefined && (typeof m.mime !== 'string' || m.mime === ''))
+            throw new Error(`${at}: mime must be a non-empty string`);
+        if (m.fileType !== undefined && !FILE_TYPES.includes(m.fileType)) {
+            throw new Error(`${at}: fileType must be one of ${FILE_TYPES.join(', ')}`);
+        }
+        if (typeof m.file === 'string') {
+            localPath(m.file, { ...opts, baseDir }, `${where} message #${i} file`);
+        }
+    });
+    return (0, shared_1.buildMessages)(inputs, baseDir);
+}
+function parseDoc(raw, opts) {
+    let doc;
+    try {
+        doc = (0, yaml_1.parse)(raw);
+    }
+    catch (err) {
+        throw new Error(`not valid YAML: ${err.message}`);
+    }
+    if (!isObject(doc))
+        throw new Error('must be a mapping with a "cases" list');
+    for (const key of Object.keys(doc)) {
+        if (key !== 'defaults' && key !== 'cases')
+            throw new Error(`unknown top-level key "${key}" (expected defaults, cases)`);
+    }
+    const defaults = mapping(doc.defaults ?? {}, 'defaults');
+    for (const key of Object.keys(defaults)) {
+        if (key !== 'variables' && key !== 'channel')
+            throw new Error(`defaults.${key}: unknown key (expected variables, channel)`);
+    }
+    if (opts.kind === 'prompt' && defaults.channel !== undefined)
+        throw new Error('defaults.channel: only workflows take a channel');
+    const defaultVars = parseVariables(defaults.variables, opts.kind, 'defaults.variables');
+    const defaultChannel = defaults.channel === undefined ? undefined : mapping(defaults.channel, 'defaults.channel');
+    if (!Array.isArray(doc.cases) || doc.cases.length === 0)
+        throw new Error('"cases" must be a non-empty list');
+    const names = new Set();
+    return doc.cases.map((c, i) => {
+        const where = `cases[${i}]`;
+        if (!isObject(c))
+            throw new Error(`${where} must be a mapping`);
+        for (const key of Object.keys(c)) {
+            if (!CASE_KEYS.includes(key))
+                throw new Error(`${where}.${key}: unknown key (expected ${CASE_KEYS.join(', ')})`);
+        }
+        if (typeof c.name !== 'string' || c.name.trim() === '')
+            throw new Error(`${where}.name must be a non-empty string`);
+        if (names.has(c.name))
+            throw new Error(`${where}.name "${c.name}" is used twice`);
+        names.add(c.name);
+        const at = `${where} (${c.name})`;
+        if (opts.kind === 'prompt' && c.channel !== undefined)
+            throw new Error(`${at}.channel: only workflows take a channel`);
+        const vars = parseVariables(c.variables, opts.kind, `${at}.variables`);
+        const channel = c.channel === undefined ? undefined : mapping(c.channel, `${at}.channel`);
+        return {
+            name: c.name,
+            messages: caseMessages(c, opts, at),
+            variables: vars || defaultVars ? { ...defaultVars, ...vars } : undefined,
+            channel: channel || defaultChannel ? { ...defaultChannel, ...channel } : undefined,
+            expect: parseExpect(c.expect, opts.kind, `${at}.expect`),
+        };
+    });
+}
+/** Parses and validates a cases.yaml; every problem is a CliError (exit 2) prefixed with opts.file. */
+function parseCasesYaml(raw, opts) {
+    try {
+        return parseDoc(raw, opts);
+    }
+    catch (err) {
+        throw new errors_1.CliError(`${opts.file}: ${err.message}`);
+    }
+}

package/dist/verify/execute.js

@@ -0,0 +1,109 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.runCase = runCase;
+exports.runAll = runAll;
+/**
+ * Runs verify cases with the project API key and checks each reply. A case never throws:
+ * an API failure becomes an `error` row so one bad case does not hide the others.
+ */
+const client_1 = require("@metrio-ai/client");
+const shared_1 = require("../commands/shared");
+const prompt_1 = require("../commands/prompt");
+const assert_1 = require("./assert");
+async function runCase(client, t, c, timezone) {
+    const base = { resource: t.label, case: c.name };
+    try {
+        let obs;
+        let version;
+        let taskId;
+        if (t.kind === 'prompt') {
+            const res = await client.chatCompletion({
+                promptId: t.ref,
+                messages: c.messages,
+                // A numeric tag is the prompt version serial.
+                tag: t.version !== undefined ? String(t.version) : undefined,
+                timezone,
+                variables: (0, prompt_1.mergeVariables)([], c.variables),
+            });
+            obs = (0, assert_1.observePrompt)(res);
+            version = t.version;
+            taskId = res.taskId;
+            if (obs.failedReason !== undefined) {
+                return {
+                    ...base,
+                    status: 'error',
+                    failures: [],
+                    error: { message: `prompt reply failed: ${obs.failedReason || 'no reason given'}` },
+                    latencyMs: obs.latencyMs,
+                    tokens: obs.tokens,
+                    ...(version !== undefined ? { version } : {}),
+                    ...(taskId ? { taskId } : {}),
+                };
+            }
+        }
+        else {
+            const res = await client.workflowCompletion({
+                workflowId: t.ref,
+                messages: c.messages,
+                version: t.version,
+                channel: c.channel,
+                variables: c.variables,
+                timezone,
+            });
+            version = res.version;
+            taskId = res.taskId;
+            // The backend answers 200 for a failed or skipped run; never let that pass.
+            const broken = res.skipped ? 'workflow skipped' : res.reply?.status === 'failed' ? 'workflow run failed' : undefined;
+            if (broken) {
+                return {
+                    ...base,
+                    status: 'error',
+                    failures: [],
+                    error: { message: `${broken}: ${res.reply?.reason || 'no reason given'}` },
+                    latencyMs: res.elapsedTime,
+                    tokens: (res.inputTokens ?? 0) + (res.outputTokens ?? 0),
+                    ...(version !== undefined ? { version } : {}),
+                    ...(taskId ? { taskId } : {}),
+                };
+            }
+            obs = (0, assert_1.observeWorkflow)(res);
+        }
+        const failures = (0, assert_1.evaluate)(c.expect, obs);
+        return {
+            ...base,
+            status: failures.length ? 'fail' : 'pass',
+            failures,
+            latencyMs: obs.latencyMs,
+            tokens: obs.tokens,
+            ...(obs.llmCalls !== null ? { llmCalls: obs.llmCalls } : {}),
+            ...(version !== undefined ? { version } : {}),
+            ...(taskId ? { taskId } : {}),
+        };
+    }
+    catch (err) {
+        return {
+            ...base,
+            status: 'error',
+            failures: [],
+            error: {
+                message: err instanceof Error ? err.message : String(err),
+                status: err instanceof client_1.MetrioApiError ? err.statusCode : undefined,
+            },
+        };
+    }
+}
+async function runAll(client, targets, opts) {
+    const jobs = targets.flatMap((t) => t.cases.map((c) => ({ t, c })));
+    let done = 0;
+    let stop = false;
+    return (0, shared_1.mapWithConcurrency)(jobs, opts.concurrency, async ({ t, c }) => {
+        const result = stop
+            ? { resource: t.label, case: c.name, status: 'skipped', failures: [] }
+            : await runCase(client, t, c, opts.timezone);
+        if (opts.failFast && (result.status === 'fail' || result.status === 'error'))
+            stop = true;
+        done += 1;
+        opts.onResult?.(result, done, jobs.length);
+        return result;
+    });
+}

package/dist/verify/jsonpath.js

@@ -0,0 +1,58 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.parsePath = parsePath;
+exports.readPath = readPath;
+function parsePath(expr) {
+    if (!expr.startsWith('$'))
+        throw new Error(`JSON path must start with "$": ${expr}`);
+    const segments = [];
+    let i = 1;
+    while (i < expr.length) {
+        if (expr[i] === '.') {
+            const name = /^[^.[\]]+/.exec(expr.slice(i + 1));
+            if (!name)
+                throw new Error(`Bad JSON path ${expr}: empty name at position ${i}`);
+            segments.push(name[0]);
+            i += 1 + name[0].length;
+        }
+        else if (expr[i] === '[') {
+            const rest = expr.slice(i);
+            const index = /^\[(\d+)\]/.exec(rest);
+            const quoted = /^\[(?:"((?:[^"\\]|\\.)*)"|'([^']*)')\]/.exec(rest);
+            if (index) {
+                segments.push(Number(index[1]));
+                i += index[0].length;
+            }
+            else if (quoted) {
+                segments.push(quoted[1] !== undefined ? JSON.parse(`"${quoted[1]}"`) : quoted[2]);
+                i += quoted[0].length;
+            }
+            else {
+                throw new Error(`Bad JSON path ${expr}: unsupported bracket at position ${i} (use [0] or ["key"])`);
+            }
+        }
+        else {
+            throw new Error(`Bad JSON path ${expr}: unexpected "${expr[i]}" at position ${i}`);
+        }
+    }
+    return segments;
+}
+/** The value at `segments`, or undefined when any step is missing (own properties only). */
+function readPath(root, segments) {
+    let current = root;
+    for (const segment of segments) {
+        if (typeof segment === 'number') {
+            if (!Array.isArray(current) || segment >= current.length)
+                return undefined;
+            current = current[segment];
+        }
+        else {
+            if (current === null || typeof current !== 'object' || Array.isArray(current))
+                return undefined;
+            if (!Object.prototype.hasOwnProperty.call(current, segment))
+                return undefined;
+            current = current[segment];
+        }
+    }
+    return current;
+}

package/dist/verify/report.js

@@ -0,0 +1,87 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.summarize = summarize;
+exports.formatHuman = formatHuman;
+exports.formatJson = formatJson;
+exports.formatJunit = formatJunit;
+/**
+ * verify output: a grouped text report (stdout default), JSON (--json) and JUnit XML (--junit).
+ * Reply text is model output: the text report escapes control characters, JUnit escapes XML.
+ */
+const sync_1 = require("../commands/sync");
+function summarize(results) {
+    const count = (s) => results.filter((r) => r.status === s).length;
+    return { total: results.length, passed: count('pass'), failed: count('fail'), errors: count('error'), skipped: count('skipped') };
+}
+const MAX = 300;
+function show(value) {
+    const s = value === undefined ? '(missing)' : JSON.stringify(value);
+    const chars = [...s];
+    return chars.length > MAX ? `${chars.slice(0, MAX).join('')}…` : s;
+}
+const describeFailure = (f) => `${f.assertion}: expected ${show(f.expected)}, got ${show(f.actual)}`;
+const LABEL = { pass: 'PASS ', fail: 'FAIL ', error: 'ERROR', skipped: 'SKIP ' };
+function formatHuman(results) {
+    const lines = [];
+    let current;
+    for (const r of results) {
+        if (r.resource !== current) {
+            current = r.resource;
+            lines.push(r.resource);
+        }
+        const stats = [
+            // Workflow runs report fractional milliseconds; the text report shows whole ones.
+            r.latencyMs !== undefined ? `${Math.round(r.latencyMs)}ms` : '',
+            r.tokens !== undefined ? `${r.tokens} tokens` : '',
+            r.version !== undefined ? `v${r.version}` : '',
+        ].filter(Boolean).join(', ');
+        lines.push(`  ${LABEL[r.status]} ${r.case}${stats ? ` (${stats})` : ''}`);
+        for (const f of r.failures)
+            lines.push(`      ${describeFailure(f)}`);
+        if (r.error)
+            lines.push(`      ${r.error.status ? `HTTP ${r.error.status}: ` : ''}${r.error.message}`);
+    }
+    const s = summarize(results);
+    lines.push(`${s.total} cases: ${s.passed} passed, ${s.failed} failed, ${s.errors} errors, ${s.skipped} skipped`);
+    return (0, sync_1.safe)(`${lines.join('\n')}\n`);
+}
+function formatJson(results) {
+    return `${JSON.stringify({ summary: summarize(results), results }, null, 2)}\n`;
+}
+const xml = (s) => s
+    .replace(/[\x00-\x08\x0b\x0c\x0e-\x1f\ufffe\uffff]|[\ud800-\udbff](?![\udc00-\udfff])|(?<![\ud800-\udbff])[\udc00-\udfff]/g, '')
+    .replace(/&/g, '&amp;')
+    .replace(/</g, '&lt;')
+    .replace(/>/g, '&gt;')
+    .replace(/"/g, '&quot;');
+function formatJunit(results) {
+    const counts = (s) => `tests="${s.total}" failures="${s.failed}" errors="${s.errors}" skipped="${s.skipped}"`;
+    const out = ['<?xml version="1.0" encoding="UTF-8"?>', `<testsuites name="metrio verify" ${counts(summarize(results))}>`];
+    for (const name of [...new Set(results.map((r) => r.resource))]) {
+        const rows = results.filter((r) => r.resource === name);
+        out.push(`  <testsuite name="${xml(name)}" ${counts(summarize(rows))}>`);
+        for (const r of rows) {
+            const time = r.latencyMs !== undefined ? ` time="${(r.latencyMs / 1000).toFixed(3)}"` : '';
+            const open = `    <testcase classname="${xml(name)}" name="${xml(r.case)}"${time}`;
+            if (r.status === 'pass') {
+                out.push(`${open}/>`);
+                continue;
+            }
+            out.push(`${open}>`);
+            if (r.status === 'fail') {
+                const body = r.failures.map(describeFailure).join('\n');
+                out.push(`      <failure message="${r.failures.length} assertion(s) failed">${xml(body)}</failure>`);
+            }
+            else if (r.status === 'error') {
+                out.push(`      <error message="${xml(r.error?.message ?? 'error')}"/>`);
+            }
+            else {
+                out.push('      <skipped/>');
+            }
+            out.push('    </testcase>');
+        }
+        out.push('  </testsuite>');
+    }
+    out.push('</testsuites>', '');
+    return out.join('\n');
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@metrio-ai/cli",
-  "version": "1.2.0",
+  "version": "1.3.1",
   "packageManager": "yarn@4.14.1",
   "description": "Metrio AI command-line interface: run prompts and workflows, manage Metrio resources",
   "bin": {