@metrio-ai/cli 1.1.0 → 1.3.0

package/dist/verify/cases.js

@@ -0,0 +1,302 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.parseCasesYaml = parseCasesYaml;
+/**
+ * cases.yaml for `metrio verify`. Everything is parsed and validated before any case runs,
+ * so a typo is exit 2 instead of a paid run that silently checks nothing.
+ */
+const fs = __importStar(require("fs"));
+const path = __importStar(require("path"));
+const yaml_1 = require("yaml");
+const errors_1 = require("../errors");
+const shared_1 = require("../commands/shared");
+const jsonpath_1 = require("./jsonpath");
+const WORKFLOW_ONLY = ['path_includes', 'terminal_node', 'response_type', 'max_llm_calls'];
+const EXPECT_KEYS = ['contains', 'not_contains', 'regex', 'equals', 'json', ...WORKFLOW_ONLY, 'max_latency_ms', 'max_tokens'];
+const MESSAGE_KEYS = ['role', 'content', 'text', 'file', 'mime', 'fileType'];
+const ROLES = ['system', 'user', 'assistant'];
+const FILE_TYPES = ['image', 'pdf', 'binary'];
+const CASE_KEYS = ['name', 'message', 'messages', 'messages_file', 'variables', 'channel', 'expect'];
+const JSON_ROOTS = ['content', 'structContent', 'state', 'tags', 'json'];
+const MATCHERS = ['equals', 'exists', 'contains'];
+const isObject = (v) => typeof v === 'object' && v !== null && !Array.isArray(v);
+function stringList(v, where) {
+    const list = typeof v === 'string' ? [v] : v;
+    if (!Array.isArray(list) || list.length === 0 || !list.every((s) => typeof s === 'string' && s !== '')) {
+        throw new Error(`${where} must be a non-empty string or a list of non-empty strings`);
+    }
+    return list;
+}
+function limit(v, where) {
+    if (typeof v !== 'number' || !Number.isFinite(v) || v < 0)
+        throw new Error(`${where} must be a number >= 0`);
+    return v;
+}
+function mapping(v, where) {
+    if (!isObject(v))
+        throw new Error(`${where} must be a mapping`);
+    return v;
+}
+/** A file a case reads, resolved against baseDir; in a workspace it must stay inside the root. */
+function localPath(p, opts, where) {
+    const full = path.resolve(opts.baseDir, p);
+    if (opts.root) {
+        let real;
+        try {
+            real = fs.realpathSync(full);
+        }
+        catch (err) {
+            throw new Error(`${where} "${p}": ${err.message}`);
+        }
+        const rel = path.relative(fs.realpathSync(opts.root), real);
+        if (rel === '..' || rel.startsWith(`..${path.sep}`) || path.isAbsolute(rel)) {
+            throw new Error(`${where} "${p}" is outside the metrio workspace`);
+        }
+    }
+    return full;
+}
+function parseJsonChecks(raw, where) {
+    if (!isObject(raw) || Object.keys(raw).length === 0)
+        throw new Error(`${where} must map JSON paths to matchers`);
+    return Object.entries(raw).map(([expr, m]) => {
+        const at = `${where}["${expr}"]`;
+        let segments;
+        try {
+            segments = (0, jsonpath_1.parsePath)(expr);
+        }
+        catch (err) {
+            throw new Error(`${at}: ${err.message}`);
+        }
+        if (typeof segments[0] !== 'string' || !JSON_ROOTS.includes(segments[0])) {
+            throw new Error(`${at}: a path starts at one of $.${JSON_ROOTS.join(', $.')}`);
+        }
+        if (!isObject(m) || Object.keys(m).length === 0)
+            throw new Error(`${at} must be a mapping of ${MATCHERS.join(' / ')}`);
+        const check = { path: expr, segments };
+        for (const [name, value] of Object.entries(m)) {
+            if (name === 'equals')
+                check.equals = { value };
+            else if (name === 'contains')
+                check.contains = { value };
+            else if (name === 'exists') {
+                if (typeof value !== 'boolean')
+                    throw new Error(`${at}.exists must be true or false`);
+                check.exists = value;
+            }
+            else
+                throw new Error(`${at}.${name}: unknown matcher (known: ${MATCHERS.join(', ')})`);
+        }
+        return check;
+    });
+}
+function parseExpect(raw, kind, where) {
+    if (raw === undefined || raw === null)
+        return {};
+    const e = {};
+    for (const [key, v] of Object.entries(mapping(raw, where))) {
+        const at = `${where}.${key}`;
+        if (!EXPECT_KEYS.includes(key))
+            throw new Error(`${at}: unknown assertion (known: ${EXPECT_KEYS.join(', ')})`);
+        if (kind === 'prompt' && WORKFLOW_ONLY.includes(key))
+            throw new Error(`${at}: only workflows support this assertion`);
+        switch (key) {
+            case 'contains':
+            case 'not_contains':
+            case 'path_includes':
+                e[key] = stringList(v, at);
+                break;
+            case 'regex':
+                e.regex = stringList(v, at).map((r) => {
+                    try {
+                        new RegExp(r, 'u');
+                    }
+                    catch (err) {
+                        throw new Error(`${at}: invalid regex ${JSON.stringify(r)} (${err.message})`);
+                    }
+                    return r;
+                });
+                break;
+            case 'equals':
+                if (typeof v !== 'string')
+                    throw new Error(`${at} must be a string`);
+                e.equals = v;
+                break;
+            case 'json':
+                e.json = parseJsonChecks(v, at);
+                break;
+            case 'terminal_node':
+                if (typeof v !== 'string' || v === '')
+                    throw new Error(`${at} must be a node id`);
+                e.terminal_node = v;
+                break;
+            case 'response_type':
+                if (v !== 'text' && v !== 'struct' && v !== 'none')
+                    throw new Error(`${at} must be text, struct or none`);
+                e.response_type = v;
+                break;
+            default:
+                e[key] = limit(v, at);
+        }
+    }
+    return e;
+}
+function parseVariables(v, kind, where) {
+    if (v === undefined)
+        return undefined;
+    const vars = mapping(v, where);
+    if (kind === 'prompt') {
+        for (const [name, value] of Object.entries(vars)) {
+            if (!['string', 'number', 'boolean'].includes(typeof value)) {
+                throw new Error(`${where}.${name} must be a string, number or boolean (prompt variables are text)`);
+            }
+        }
+    }
+    return vars;
+}
+function caseMessages(c, opts, where) {
+    const given = ['message', 'messages', 'messages_file'].filter((k) => c[k] !== undefined);
+    if (given.length !== 1)
+        throw new Error(`${where} needs exactly one of message, messages, messages_file`);
+    let inputs;
+    let baseDir = opts.baseDir;
+    if (c.message !== undefined) {
+        if (typeof c.message !== 'string' || c.message.trim() === '')
+            throw new Error(`${where}.message must be a non-empty string`);
+        inputs = [{ role: 'user', content: c.message }];
+    }
+    else if (c.messages !== undefined) {
+        if (!Array.isArray(c.messages) || c.messages.length === 0)
+            throw new Error(`${where}.messages must be a non-empty list`);
+        inputs = c.messages;
+    }
+    else {
+        if (typeof c.messages_file !== 'string' || c.messages_file === '')
+            throw new Error(`${where}.messages_file must be a path`);
+        const file = localPath(c.messages_file, opts, `${where}.messages_file`);
+        inputs = (0, shared_1.readMessagesFile)(file);
+        if (inputs.length === 0)
+            throw new Error(`${where}.messages_file has no messages`);
+        baseDir = path.dirname(file);
+    }
+    inputs.forEach((m, i) => {
+        if (!isObject(m))
+            throw new Error(`${where} message #${i} must be a mapping`);
+        for (const key of Object.keys(m)) {
+            if (!MESSAGE_KEYS.includes(key))
+                throw new Error(`${where} message #${i}: unknown key "${key}" (expected ${MESSAGE_KEYS.join(', ')})`);
+        }
+        const at = `${where} message #${i}`;
+        if (!ROLES.includes(m.role))
+            throw new Error(`${at}: invalid role ${JSON.stringify(m.role)} (expected ${ROLES.join(', ')})`);
+        for (const key of ['content', 'text']) {
+            if (m[key] !== undefined && typeof m[key] !== 'string')
+                throw new Error(`${at}: ${key} must be a string`);
+        }
+        if (m.file !== undefined && (typeof m.file !== 'string' || m.file === ''))
+            throw new Error(`${at}: file must be a path string`);
+        if (m.mime !== undefined && (typeof m.mime !== 'string' || m.mime === ''))
+            throw new Error(`${at}: mime must be a non-empty string`);
+        if (m.fileType !== undefined && !FILE_TYPES.includes(m.fileType)) {
+            throw new Error(`${at}: fileType must be one of ${FILE_TYPES.join(', ')}`);
+        }
+        if (typeof m.file === 'string') {
+            localPath(m.file, { ...opts, baseDir }, `${where} message #${i} file`);
+        }
+    });
+    return (0, shared_1.buildMessages)(inputs, baseDir);
+}
+function parseDoc(raw, opts) {
+    let doc;
+    try {
+        doc = (0, yaml_1.parse)(raw);
+    }
+    catch (err) {
+        throw new Error(`not valid YAML: ${err.message}`);
+    }
+    if (!isObject(doc))
+        throw new Error('must be a mapping with a "cases" list');
+    for (const key of Object.keys(doc)) {
+        if (key !== 'defaults' && key !== 'cases')
+            throw new Error(`unknown top-level key "${key}" (expected defaults, cases)`);
+    }
+    const defaults = mapping(doc.defaults ?? {}, 'defaults');
+    for (const key of Object.keys(defaults)) {
+        if (key !== 'variables' && key !== 'channel')
+            throw new Error(`defaults.${key}: unknown key (expected variables, channel)`);
+    }
+    if (opts.kind === 'prompt' && defaults.channel !== undefined)
+        throw new Error('defaults.channel: only workflows take a channel');
+    const defaultVars = parseVariables(defaults.variables, opts.kind, 'defaults.variables');
+    const defaultChannel = defaults.channel === undefined ? undefined : mapping(defaults.channel, 'defaults.channel');
+    if (!Array.isArray(doc.cases) || doc.cases.length === 0)
+        throw new Error('"cases" must be a non-empty list');
+    const names = new Set();
+    return doc.cases.map((c, i) => {
+        const where = `cases[${i}]`;
+        if (!isObject(c))
+            throw new Error(`${where} must be a mapping`);
+        for (const key of Object.keys(c)) {
+            if (!CASE_KEYS.includes(key))
+                throw new Error(`${where}.${key}: unknown key (expected ${CASE_KEYS.join(', ')})`);
+        }
+        if (typeof c.name !== 'string' || c.name.trim() === '')
+            throw new Error(`${where}.name must be a non-empty string`);
+        if (names.has(c.name))
+            throw new Error(`${where}.name "${c.name}" is used twice`);
+        names.add(c.name);
+        const at = `${where} (${c.name})`;
+        if (opts.kind === 'prompt' && c.channel !== undefined)
+            throw new Error(`${at}.channel: only workflows take a channel`);
+        const vars = parseVariables(c.variables, opts.kind, `${at}.variables`);
+        const channel = c.channel === undefined ? undefined : mapping(c.channel, `${at}.channel`);
+        return {
+            name: c.name,
+            messages: caseMessages(c, opts, at),
+            variables: vars || defaultVars ? { ...defaultVars, ...vars } : undefined,
+            channel: channel || defaultChannel ? { ...defaultChannel, ...channel } : undefined,
+            expect: parseExpect(c.expect, opts.kind, `${at}.expect`),
+        };
+    });
+}
+/** Parses and validates a cases.yaml; every problem is a CliError (exit 2) prefixed with opts.file. */
+function parseCasesYaml(raw, opts) {
+    try {
+        return parseDoc(raw, opts);
+    }
+    catch (err) {
+        throw new errors_1.CliError(`${opts.file}: ${err.message}`);
+    }
+}

package/dist/verify/execute.js

@@ -0,0 +1,109 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.runCase = runCase;
+exports.runAll = runAll;
+/**
+ * Runs verify cases with the project API key and checks each reply. A case never throws:
+ * an API failure becomes an `error` row so one bad case does not hide the others.
+ */
+const client_1 = require("@metrio-ai/client");
+const shared_1 = require("../commands/shared");
+const prompt_1 = require("../commands/prompt");
+const assert_1 = require("./assert");
+async function runCase(client, t, c, timezone) {
+    const base = { resource: t.label, case: c.name };
+    try {
+        let obs;
+        let version;
+        let taskId;
+        if (t.kind === 'prompt') {
+            const res = await client.chatCompletion({
+                promptId: t.ref,
+                messages: c.messages,
+                // A numeric tag is the prompt version serial.
+                tag: t.version !== undefined ? String(t.version) : undefined,
+                timezone,
+                variables: (0, prompt_1.mergeVariables)([], c.variables),
+            });
+            obs = (0, assert_1.observePrompt)(res);
+            version = t.version;
+            taskId = res.taskId;
+            if (obs.failedReason !== undefined) {
+                return {
+                    ...base,
+                    status: 'error',
+                    failures: [],
+                    error: { message: `prompt reply failed: ${obs.failedReason || 'no reason given'}` },
+                    latencyMs: obs.latencyMs,
+                    tokens: obs.tokens,
+                    ...(version !== undefined ? { version } : {}),
+                    ...(taskId ? { taskId } : {}),
+                };
+            }
+        }
+        else {
+            const res = await client.workflowCompletion({
+                workflowId: t.ref,
+                messages: c.messages,
+                version: t.version,
+                channel: c.channel,
+                variables: c.variables,
+                timezone,
+            });
+            version = res.version;
+            taskId = res.taskId;
+            // The backend answers 200 for a failed or skipped run; never let that pass.
+            const broken = res.skipped ? 'workflow skipped' : res.reply?.status === 'failed' ? 'workflow run failed' : undefined;
+            if (broken) {
+                return {
+                    ...base,
+                    status: 'error',
+                    failures: [],
+                    error: { message: `${broken}: ${res.reply?.reason || 'no reason given'}` },
+                    latencyMs: res.elapsedTime,
+                    tokens: (res.inputTokens ?? 0) + (res.outputTokens ?? 0),
+                    ...(version !== undefined ? { version } : {}),
+                    ...(taskId ? { taskId } : {}),
+                };
+            }
+            obs = (0, assert_1.observeWorkflow)(res);
+        }
+        const failures = (0, assert_1.evaluate)(c.expect, obs);
+        return {
+            ...base,
+            status: failures.length ? 'fail' : 'pass',
+            failures,
+            latencyMs: obs.latencyMs,
+            tokens: obs.tokens,
+            ...(obs.llmCalls !== null ? { llmCalls: obs.llmCalls } : {}),
+            ...(version !== undefined ? { version } : {}),
+            ...(taskId ? { taskId } : {}),
+        };
+    }
+    catch (err) {
+        return {
+            ...base,
+            status: 'error',
+            failures: [],
+            error: {
+                message: err instanceof Error ? err.message : String(err),
+                status: err instanceof client_1.MetrioApiError ? err.statusCode : undefined,
+            },
+        };
+    }
+}
+async function runAll(client, targets, opts) {
+    const jobs = targets.flatMap((t) => t.cases.map((c) => ({ t, c })));
+    let done = 0;
+    let stop = false;
+    return (0, shared_1.mapWithConcurrency)(jobs, opts.concurrency, async ({ t, c }) => {
+        const result = stop
+            ? { resource: t.label, case: c.name, status: 'skipped', failures: [] }
+            : await runCase(client, t, c, opts.timezone);
+        if (opts.failFast && (result.status === 'fail' || result.status === 'error'))
+            stop = true;
+        done += 1;
+        opts.onResult?.(result, done, jobs.length);
+        return result;
+    });
+}

package/dist/verify/jsonpath.js

@@ -0,0 +1,58 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.parsePath = parsePath;
+exports.readPath = readPath;
+function parsePath(expr) {
+    if (!expr.startsWith('$'))
+        throw new Error(`JSON path must start with "$": ${expr}`);
+    const segments = [];
+    let i = 1;
+    while (i < expr.length) {
+        if (expr[i] === '.') {
+            const name = /^[^.[\]]+/.exec(expr.slice(i + 1));
+            if (!name)
+                throw new Error(`Bad JSON path ${expr}: empty name at position ${i}`);
+            segments.push(name[0]);
+            i += 1 + name[0].length;
+        }
+        else if (expr[i] === '[') {
+            const rest = expr.slice(i);
+            const index = /^\[(\d+)\]/.exec(rest);
+            const quoted = /^\[(?:"((?:[^"\\]|\\.)*)"|'([^']*)')\]/.exec(rest);
+            if (index) {
+                segments.push(Number(index[1]));
+                i += index[0].length;
+            }
+            else if (quoted) {
+                segments.push(quoted[1] !== undefined ? JSON.parse(`"${quoted[1]}"`) : quoted[2]);
+                i += quoted[0].length;
+            }
+            else {
+                throw new Error(`Bad JSON path ${expr}: unsupported bracket at position ${i} (use [0] or ["key"])`);
+            }
+        }
+        else {
+            throw new Error(`Bad JSON path ${expr}: unexpected "${expr[i]}" at position ${i}`);
+        }
+    }
+    return segments;
+}
+/** The value at `segments`, or undefined when any step is missing (own properties only). */
+function readPath(root, segments) {
+    let current = root;
+    for (const segment of segments) {
+        if (typeof segment === 'number') {
+            if (!Array.isArray(current) || segment >= current.length)
+                return undefined;
+            current = current[segment];
+        }
+        else {
+            if (current === null || typeof current !== 'object' || Array.isArray(current))
+                return undefined;
+            if (!Object.prototype.hasOwnProperty.call(current, segment))
+                return undefined;
+            current = current[segment];
+        }
+    }
+    return current;
+}

package/dist/verify/report.js

@@ -0,0 +1,86 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.summarize = summarize;
+exports.formatHuman = formatHuman;
+exports.formatJson = formatJson;
+exports.formatJunit = formatJunit;
+/**
+ * verify output: a grouped text report (stdout default), JSON (--json) and JUnit XML (--junit).
+ * Reply text is model output: the text report escapes control characters, JUnit escapes XML.
+ */
+const sync_1 = require("../commands/sync");
+function summarize(results) {
+    const count = (s) => results.filter((r) => r.status === s).length;
+    return { total: results.length, passed: count('pass'), failed: count('fail'), errors: count('error'), skipped: count('skipped') };
+}
+const MAX = 300;
+function show(value) {
+    const s = value === undefined ? '(missing)' : JSON.stringify(value);
+    const chars = [...s];
+    return chars.length > MAX ? `${chars.slice(0, MAX).join('')}…` : s;
+}
+const describeFailure = (f) => `${f.assertion}: expected ${show(f.expected)}, got ${show(f.actual)}`;
+const LABEL = { pass: 'PASS ', fail: 'FAIL ', error: 'ERROR', skipped: 'SKIP ' };
+function formatHuman(results) {
+    const lines = [];
+    let current;
+    for (const r of results) {
+        if (r.resource !== current) {
+            current = r.resource;
+            lines.push(r.resource);
+        }
+        const stats = [
+            r.latencyMs !== undefined ? `${r.latencyMs}ms` : '',
+            r.tokens !== undefined ? `${r.tokens} tokens` : '',
+            r.version !== undefined ? `v${r.version}` : '',
+        ].filter(Boolean).join(', ');
+        lines.push(`  ${LABEL[r.status]} ${r.case}${stats ? ` (${stats})` : ''}`);
+        for (const f of r.failures)
+            lines.push(`      ${describeFailure(f)}`);
+        if (r.error)
+            lines.push(`      ${r.error.status ? `HTTP ${r.error.status}: ` : ''}${r.error.message}`);
+    }
+    const s = summarize(results);
+    lines.push(`${s.total} cases: ${s.passed} passed, ${s.failed} failed, ${s.errors} errors, ${s.skipped} skipped`);
+    return (0, sync_1.safe)(`${lines.join('\n')}\n`);
+}
+function formatJson(results) {
+    return `${JSON.stringify({ summary: summarize(results), results }, null, 2)}\n`;
+}
+const xml = (s) => s
+    .replace(/[\x00-\x08\x0b\x0c\x0e-\x1f\ufffe\uffff]|[\ud800-\udbff](?![\udc00-\udfff])|(?<![\ud800-\udbff])[\udc00-\udfff]/g, '')
+    .replace(/&/g, '&amp;')
+    .replace(/</g, '&lt;')
+    .replace(/>/g, '&gt;')
+    .replace(/"/g, '&quot;');
+function formatJunit(results) {
+    const counts = (s) => `tests="${s.total}" failures="${s.failed}" errors="${s.errors}" skipped="${s.skipped}"`;
+    const out = ['<?xml version="1.0" encoding="UTF-8"?>', `<testsuites name="metrio verify" ${counts(summarize(results))}>`];
+    for (const name of [...new Set(results.map((r) => r.resource))]) {
+        const rows = results.filter((r) => r.resource === name);
+        out.push(`  <testsuite name="${xml(name)}" ${counts(summarize(rows))}>`);
+        for (const r of rows) {
+            const time = r.latencyMs !== undefined ? ` time="${(r.latencyMs / 1000).toFixed(3)}"` : '';
+            const open = `    <testcase classname="${xml(name)}" name="${xml(r.case)}"${time}`;
+            if (r.status === 'pass') {
+                out.push(`${open}/>`);
+                continue;
+            }
+            out.push(`${open}>`);
+            if (r.status === 'fail') {
+                const body = r.failures.map(describeFailure).join('\n');
+                out.push(`      <failure message="${r.failures.length} assertion(s) failed">${xml(body)}</failure>`);
+            }
+            else if (r.status === 'error') {
+                out.push(`      <error message="${xml(r.error?.message ?? 'error')}"/>`);
+            }
+            else {
+                out.push('      <skipped/>');
+            }
+            out.push('    </testcase>');
+        }
+        out.push('  </testsuite>');
+    }
+    out.push('</testsuites>', '');
+    return out.join('\n');
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@metrio-ai/cli",
-  "version": "1.1.0",
+  "version": "1.3.0",
   "packageManager": "yarn@4.14.1",
   "description": "Metrio AI command-line interface: run prompts and workflows, manage Metrio resources",
   "bin": {
@@ -31,9 +31,11 @@
   "dependencies": {
     "@metrio-ai/client": "^1.5.2",
     "ajv": "^8.17.1",
+    "diff": "^5.2.0",
     "yaml": "^2.8.3"
   },
   "devDependencies": {
+    "@types/diff": "^5.2.3",
     "@types/jest": "^29.5.14",
     "@types/node": "^20.10.0",
     "jest": "^29.7.0",