@metrio-ai/cli 1.4.0 → 1.5.0

package/dist/sync/status.js

@@ -1,6 +1,8 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
+exports.renderRemoteAgents = renderRemoteAgents;
 exports.computeEntries = computeEntries;
+const render_1 = require("./render");
 const workspace_1 = require("./workspace");
 function proposedKey(r) {
     if (r.kind === 'workflow') {
@@ -8,12 +10,18 @@ function proposedKey(r) {
         return `${workspace_1.KIND_DIR.workflow}/${unsafe ? 'workflow-' + (0, workspace_1.slugify)(r.id, 'workflow') : r.id}`;
     }
     const slug = (0, workspace_1.slugify)(r.name, `${r.kind}-${r.id}`);
+    if (r.kind === 'agent')
+        return `${workspace_1.KIND_DIR.agent}/${slug}`;
     return r.kind === 'skill' ? `${workspace_1.KIND_DIR.skill}/${slug}.md` : `${workspace_1.KIND_DIR.prompt}/${slug}`;
 }
 function withSuffix(key, id) {
     return key.endsWith('.md') ? `${key.slice(0, -3)}-${id}.md` : `${key}-${id}`;
 }
-function computeEntries(root, state, remote, kinds) {
+/**
+ * The workspace key of each remote resource: its tracked key, else a proposed one (suffixed on
+ * collision). Agent files are then rendered with directory references to those keys.
+ */
+function assignKeys(state, remote, kinds) {
     const tracked = Object.entries(state.resources).filter(([, s]) => kinds.includes(s.kind));
     const remoteAt = new Map();
     const taken = new Set(tracked.map(([key]) => key));
@@ -29,6 +37,32 @@ function computeEntries(root, state, remote, kinds) {
         taken.add(key);
         remoteAt.set(key, r);
     }
+    // Agents reference prompts and agents by directory, so render them once every key is known.
+    const idToKey = (kind) => {
+        const map = new Map();
+        for (const [key, s] of Object.entries(state.resources))
+            if (s.kind === kind)
+                map.set(s.id, key);
+        for (const [key, r] of remoteAt)
+            if (r.kind === kind)
+                map.set(r.id, key);
+        return map;
+    };
+    if (remote.some((r) => r.kind === 'agent')) {
+        const refs = { prompts: idToKey('prompt'), agents: idToKey('agent') };
+        for (const r of remote)
+            if (r.kind === 'agent')
+                r.files = { 'agent.yaml': (0, render_1.renderAgent)(r.data, refs) };
+    }
+    return remoteAt;
+}
+/** Renders remote agent files with directory references exactly as computeEntries does for this state. */
+function renderRemoteAgents(state, remote, kinds) {
+    assignKeys(state, remote, kinds);
+}
+function computeEntries(root, state, remote, kinds) {
+    const tracked = Object.entries(state.resources).filter(([, s]) => kinds.includes(s.kind));
+    const remoteAt = assignKeys(state, remote, kinds);
     const locals = (0, workspace_1.listLocal)(root, kinds);
     const keys = new Set([...remoteAt.keys(), ...tracked.map(([k]) => k), ...locals.map((l) => l.key)]);
     const entries = [];

package/dist/sync/workspace.js

@@ -52,11 +52,12 @@ const fs = __importStar(require("fs"));
 const path = __importStar(require("path"));
 const yaml_1 = require("yaml");
 const errors_1 = require("../errors");
-exports.KIND_DIR = { prompt: 'prompts', skill: 'skills', workflow: 'workflows' };
+exports.KIND_DIR = { prompt: 'prompts', skill: 'skills', workflow: 'workflows', agent: 'agents' };
 exports.KIND_FILES = {
     prompt: ['prompt.yaml', 'system.md'],
     skill: ['skill.md'],
     workflow: ['workflow.yaml', 'definition.json'],
+    agent: ['agent.yaml'],
 };
 const CONFIG = 'metrio.yaml';
 /** Check a file path for symlinks; ENOENT is fine (file doesn't exist yet). */
@@ -264,7 +265,7 @@ function validateKey(key) {
     if (segments.length !== 2)
         throw new errors_1.CliError(`Unsafe resource path "${key}" in .metrio/state.json or console data`);
     // First segment must be a resource kind directory
-    if (!['prompts', 'skills', 'workflows'].includes(segments[0])) {
+    if (!Object.values(exports.KIND_DIR).includes(segments[0])) {
         throw new errors_1.CliError(`Unsafe resource path "${key}" in .metrio/state.json or console data`);
     }
     // Check each segment

package/dist/verify/agent-runs.js

@@ -0,0 +1,23 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.runAgentSync = runAgentSync;
+async function runAgentSync(o) {
+    const doFetch = o.fetchImpl ?? fetch;
+    const url = `${o.baseUrl.replace(/\/+$/, '')}/v1/agents/${o.agentId}/runs`;
+    const started = Date.now();
+    const res = await doFetch(url, {
+        method: 'POST',
+        headers: { 'Content-Type': 'application/json', metrioai_api_key: o.apiKey },
+        body: JSON.stringify({ messages: o.messages, ...(o.variables ? { variables: o.variables } : {}), ...(o.timezone ? { timezone: o.timezone } : {}), mode: 'sync' }),
+    });
+    const text = await res.text();
+    const latencyMs = Date.now() - started;
+    let body;
+    try {
+        body = JSON.parse(text);
+    }
+    catch {
+        body = { error: text.slice(0, 500) };
+    }
+    return { status: res.status, body, latencyMs };
+}

package/dist/verify/assert.js

@@ -2,6 +2,7 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.observePrompt = observePrompt;
 exports.observeWorkflow = observeWorkflow;
+exports.observeAgent = observeAgent;
 exports.evaluate = evaluate;
 /**
  * Turns a chat / workflow response into one Observation, and checks an Expect against it.
@@ -63,6 +64,22 @@ function observeWorkflow(res) {
         llmCalls: res.llmCalls ?? 0,
     };
 }
+/** Builds an Observation from a 2xx sync agent run body (AgentRunResponse). */
+function observeAgent(body, latencyMs) {
+    const text = typeof body?.output === 'string' ? body.output : '';
+    const usage = body?.usage ?? {};
+    return {
+        text,
+        root: { content: text, structContent: undefined, state: {}, tags: [], json: parseJson(text) ?? null },
+        path: [],
+        terminalNodeId: null,
+        responseType: null,
+        latencyMs,
+        tokens: (usage.input_tokens ?? 0) + (usage.output_tokens ?? 0),
+        llmCalls: null,
+        stopReason: typeof body?.stop_reason === 'string' ? body.stop_reason : null,
+    };
+}
 function containsValue(actual, wanted) {
     if (Array.isArray(actual))
         return actual.some((item) => (0, util_1.isDeepStrictEqual)(item, wanted));
@@ -107,6 +124,8 @@ function evaluate(e, obs) {
         fail('terminal_node', e.terminal_node, obs.terminalNodeId);
     if (e.response_type !== undefined && obs.responseType !== e.response_type)
         fail('response_type', e.response_type, obs.responseType);
+    if (e.stop_reason !== undefined && obs.stopReason !== e.stop_reason)
+        fail('stop_reason', e.stop_reason, obs.stopReason ?? null);
     if (e.max_latency_ms !== undefined && obs.latencyMs > e.max_latency_ms)
         fail('max_latency_ms', e.max_latency_ms, obs.latencyMs);
     if (e.max_tokens !== undefined && obs.tokens > e.max_tokens)

package/dist/verify/cases.js

@@ -45,7 +45,9 @@ const errors_1 = require("../errors");
 const shared_1 = require("../commands/shared");
 const jsonpath_1 = require("./jsonpath");
 const WORKFLOW_ONLY = ['path_includes', 'terminal_node', 'response_type', 'max_llm_calls'];
-const EXPECT_KEYS = ['contains', 'not_contains', 'regex', 'equals', 'json', ...WORKFLOW_ONLY, 'max_latency_ms', 'max_tokens'];
+const AGENT_ONLY = ['stop_reason'];
+const STOP_REASONS = ['final', 'terminal_tool', 'max_turns', 'interrupted', 'cancelled'];
+const EXPECT_KEYS = ['contains', 'not_contains', 'regex', 'equals', 'json', ...WORKFLOW_ONLY, ...AGENT_ONLY, 'max_latency_ms', 'max_tokens'];
 const MESSAGE_KEYS = ['role', 'content', 'text', 'file', 'mime', 'fileType'];
 const ROLES = ['system', 'user', 'assistant'];
 const FILE_TYPES = ['image', 'pdf', 'binary'];
@@ -130,8 +132,10 @@ function parseExpect(raw, kind, where) {
         const at = `${where}.${key}`;
         if (!EXPECT_KEYS.includes(key))
             throw new Error(`${at}: unknown assertion (known: ${EXPECT_KEYS.join(', ')})`);
-        if (kind === 'prompt' && WORKFLOW_ONLY.includes(key))
+        if (kind !== 'workflow' && WORKFLOW_ONLY.includes(key))
             throw new Error(`${at}: only workflows support this assertion`);
+        if (kind !== 'agent' && AGENT_ONLY.includes(key))
+            throw new Error(`${at}: only agents support this assertion`);
         switch (key) {
             case 'contains':
             case 'not_contains':
@@ -157,6 +161,11 @@ function parseExpect(raw, kind, where) {
             case 'json':
                 e.json = parseJsonChecks(v, at);
                 break;
+            case 'stop_reason':
+                if (typeof v !== 'string' || !STOP_REASONS.includes(v))
+                    throw new Error(`${at} must be one of ${STOP_REASONS.join(', ')}`);
+                e.stop_reason = v;
+                break;
             case 'terminal_node':
                 if (typeof v !== 'string' || v === '')
                     throw new Error(`${at} must be a node id`);
@@ -177,10 +186,10 @@ function parseVariables(v, kind, where) {
     if (v === undefined)
         return undefined;
     const vars = mapping(v, where);
-    if (kind === 'prompt') {
+    if (kind !== 'workflow') {
         for (const [name, value] of Object.entries(vars)) {
             if (!['string', 'number', 'boolean'].includes(typeof value)) {
-                throw new Error(`${where}.${name} must be a string, number or boolean (prompt variables are text)`);
+                throw new Error(`${where}.${name} must be a string, number or boolean (prompt and agent variables are text)`);
             }
         }
     }
@@ -257,7 +266,7 @@ function parseDoc(raw, opts) {
         if (key !== 'variables' && key !== 'channel')
             throw new Error(`defaults.${key}: unknown key (expected variables, channel)`);
     }
-    if (opts.kind === 'prompt' && defaults.channel !== undefined)
+    if (opts.kind !== 'workflow' && defaults.channel !== undefined)
         throw new Error('defaults.channel: only workflows take a channel');
     const defaultVars = parseVariables(defaults.variables, opts.kind, 'defaults.variables');
     const defaultChannel = defaults.channel === undefined ? undefined : mapping(defaults.channel, 'defaults.channel');
@@ -278,7 +287,7 @@ function parseDoc(raw, opts) {
             throw new Error(`${where}.name "${c.name}" is used twice`);
         names.add(c.name);
         const at = `${where} (${c.name})`;
-        if (opts.kind === 'prompt' && c.channel !== undefined)
+        if (opts.kind !== 'workflow' && c.channel !== undefined)
             throw new Error(`${at}.channel: only workflows take a channel`);
         const vars = parseVariables(c.variables, opts.kind, `${at}.variables`);
         const channel = c.channel === undefined ? undefined : mapping(c.channel, `${at}.channel`);

package/dist/verify/execute.js

@@ -9,10 +9,48 @@ exports.runAll = runAll;
 const client_1 = require("@metrio-ai/client");
 const shared_1 = require("../commands/shared");
 const prompt_1 = require("../commands/prompt");
+const agent_runs_1 = require("./agent-runs");
 const assert_1 = require("./assert");
-async function runCase(client, t, c, timezone) {
+const scrubKey = (s, key) => (key ? s.split(key).join('***') : s);
+async function runAgentCase(api, t, c, base, timezone) {
+    const res = await (0, agent_runs_1.runAgentSync)({
+        ...api,
+        agentId: t.ref,
+        messages: c.messages,
+        variables: (0, prompt_1.mergeVariables)([], c.variables),
+        timezone,
+    });
+    const taskId = res.body?.runId !== undefined ? String(res.body.runId) : undefined;
+    const ids = taskId ? { taskId } : {};
+    if (res.status < 200 || res.status >= 300) {
+        const detail = typeof res.body?.error === 'string' ? res.body.error : JSON.stringify(res.body?.error ?? '');
+        let extra = '';
+        if (res.status >= 400 && res.status < 500 && res.body?.details !== undefined) {
+            const d = typeof res.body.details === 'string' ? res.body.details : JSON.stringify(res.body.details);
+            extra = ` (${d.length > 200 ? `${d.slice(0, 200)}...` : d})`;
+        }
+        return { ...base, status: 'error', failures: [], error: { message: scrubKey(`http=${res.status} ${detail}${extra}`.trim(), api.apiKey), status: res.status }, latencyMs: res.latencyMs, ...ids };
+    }
+    if (typeof res.body?.status !== 'string') {
+        return { ...base, status: 'error', failures: [], error: { message: scrubKey(`unexpected response: http=${res.status} without a run status`, api.apiKey) }, latencyMs: res.latencyMs, ...ids };
+    }
+    if (res.body.status === 'failed') {
+        const err = res.body.error;
+        const detail = typeof err === 'string' ? err : (err?.message ?? 'no reason given');
+        return { ...base, status: 'error', failures: [], error: { message: scrubKey(`agent run failed: ${detail}`, api.apiKey) }, latencyMs: res.latencyMs, ...ids };
+    }
+    const obs = (0, assert_1.observeAgent)(res.body, res.latencyMs);
+    const failures = (0, assert_1.evaluate)(c.expect, obs);
+    return { ...base, status: failures.length ? 'fail' : 'pass', failures, latencyMs: obs.latencyMs, tokens: obs.tokens, ...ids };
+}
+async function runCase(client, t, c, timezone, agentApi) {
     const base = { resource: t.label, case: c.name };
     try {
+        if (t.kind === 'agent') {
+            if (!agentApi)
+                throw new Error('agent targets need agentApi (base URL and API key)');
+            return await runAgentCase(agentApi, t, c, base, timezone);
+        }
         let obs;
         let version;
         let taskId;
@@ -86,7 +124,7 @@ async function runCase(client, t, c, timezone) {
             status: 'error',
             failures: [],
             error: {
-                message: err instanceof Error ? err.message : String(err),
+                message: scrubKey(err instanceof Error ? err.message : String(err), agentApi?.apiKey ?? ''),
                 status: err instanceof client_1.MetrioApiError ? err.statusCode : undefined,
             },
         };
@@ -99,7 +137,7 @@ async function runAll(client, targets, opts) {
     return (0, shared_1.mapWithConcurrency)(jobs, opts.concurrency, async ({ t, c }) => {
         const result = stop
             ? { resource: t.label, case: c.name, status: 'skipped', failures: [] }
-            : await runCase(client, t, c, opts.timezone);
+            : await runCase(client, t, c, opts.timezone, opts.agentApi);
         if (opts.failFast && (result.status === 'fail' || result.status === 'error'))
             stop = true;
         done += 1;

package/package.json

@@ -1,7 +1,6 @@
 {
   "name": "@metrio-ai/cli",
-  "version": "1.4.0",
-  "packageManager": "yarn@4.14.1",
+  "version": "1.5.0",
   "description": "Metrio AI command-line interface: run prompts and workflows, manage Metrio resources",
   "bin": {
     "metrio": "dist/index.js"
@@ -13,7 +12,7 @@
     "build": "rm -rf dist && tsc -p tsconfig.build.json",
     "typecheck": "tsc -p tsconfig.json --noEmit",
     "test": "jest",
-    "prepublishOnly": "yarn build"
+    "prepublishOnly": "bun run build"
   },
   "engines": {
     "node": ">=18.0.0"