@metrio-ai/cli 1.0.0

package/dist/commands/prompt-create-version.js

@@ -0,0 +1,211 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.CREATE_VERSION_HELP = void 0;
+exports.parseCreateVersionArgs = parseCreateVersionArgs;
+exports.runCreateVersionCommand = runCreateVersionCommand;
+/**
+ * `metrio prompt create-version` — creates a new version of a prompt from a
+ * local file (prompt-as-code). The file's entire content becomes the system
+ * prompt. The created version does NOT go live unless --set-as-head is given;
+ * promotion is an explicit act so CI pushes are safe by default.
+ */
+const fs = __importStar(require("fs"));
+const path = __importStar(require("path"));
+const client_1 = require("@metrio-ai/client");
+const shared_1 = require("./shared");
+exports.CREATE_VERSION_HELP = `metrio prompt create-version - create a prompt version from a file
+
+Usage:
+  metrio prompt create-version -p <id> -f <file> [options]
+
+Required:
+  -p, --prompt-id <id>      Prompt ID (integer; or set METRIO_PROMPT_ID)
+  -f, --file <path>         Prompt file; the entire content becomes the system prompt
+
+Options:
+      --model-tier <tier>      Model tier (wins over provider/name)
+      --model-provider <p>     Model provider (pair with --model-name)
+      --model-name <m>         Model name (pair with --model-provider)
+      --set-as-head            Promote the new version to head (default: off)
+      --description <text>     Version description
+      --output-format <fmt>    Output format: "text" or "json"
+      --json                   Print the created version as JSON
+  -e, --env <file>          Env file to load (default: .env in the current dir)
+      --api-key <key>       API key (overrides env file / environment)
+      --base-url <url>      Base API URL (overrides env file / environment)
+  -h, --help                Show this help
+
+The new version is created with the project API key (METRIOAI_API_KEY) — the
+prompt must belong to that key's project. Without --set-as-head the version is
+created but not promoted, so pushing from CI never changes what runs.
+`;
+/** Parses argv for `prompt create-version`. Throws on malformed input. */
+function parseCreateVersionArgs(argv) {
+    const options = {
+        setAsHead: false,
+        json: false,
+        help: false,
+    };
+    const next = (i, flag) => {
+        const value = argv[i + 1];
+        if (value === undefined)
+            throw new Error(`Missing value for ${flag}`);
+        return value;
+    };
+    for (let i = 0; i < argv.length; i++) {
+        let arg = argv[i];
+        let inline;
+        // Accept both `--flag value` and `--flag=value`.
+        const eq = arg.indexOf('=');
+        if (arg.startsWith('--') && eq > 2) {
+            inline = arg.slice(eq + 1);
+            arg = arg.slice(0, eq);
+        }
+        const take = (flag) => (inline !== undefined ? inline : next(i++, flag));
+        const noValue = (flag) => {
+            if (inline !== undefined)
+                throw new Error(`${flag} does not take a value`);
+        };
+        switch (arg) {
+            case '-p':
+            case '--prompt-id': {
+                const raw = take(arg);
+                const id = Number(raw);
+                if (!Number.isInteger(id) || id <= 0) {
+                    throw new Error(`--prompt-id must be a positive integer, got "${raw}"`);
+                }
+                options.promptId = id;
+                break;
+            }
+            case '-f':
+            case '--file':
+                options.file = take(arg);
+                break;
+            case '--model-tier':
+                options.modelTier = take(arg);
+                break;
+            case '--model-provider':
+                options.modelProvider = take(arg);
+                break;
+            case '--model-name':
+                options.modelName = take(arg);
+                break;
+            case '--set-as-head':
+                noValue(arg);
+                options.setAsHead = true;
+                break;
+            case '--description':
+                options.description = take(arg);
+                break;
+            case '--output-format': {
+                const raw = take(arg);
+                if (raw !== 'text' && raw !== 'json') {
+                    throw new Error(`--output-format must be "text" or "json", got "${raw}"`);
+                }
+                options.outputFormat = raw;
+                break;
+            }
+            case '--json':
+                noValue(arg);
+                options.json = true;
+                break;
+            case '-e':
+            case '--env':
+                options.envFile = take(arg);
+                break;
+            case '--api-key':
+                options.apiKey = take(arg);
+                break;
+            case '--base-url':
+                options.baseUrl = take(arg);
+                break;
+            case '-h':
+            case '--help':
+                noValue(arg);
+                options.help = true;
+                break;
+            default:
+                throw new Error(`Unknown argument: ${arg}`);
+        }
+    }
+    return options;
+}
+/** The `prompt create-version` subcommand. */
+async function runCreateVersionCommand(argv) {
+    const options = parseCreateVersionArgs(argv);
+    if (options.help) {
+        process.stdout.write(exports.CREATE_VERSION_HELP);
+        return;
+    }
+    if (!options.file) {
+        throw new Error('Missing required option: --file');
+    }
+    let systemPrompt;
+    try {
+        systemPrompt = fs.readFileSync(path.resolve(options.file), 'utf8');
+    }
+    catch (err) {
+        throw new Error(`Cannot read prompt file: ${options.file} (${err.message})`);
+    }
+    if (systemPrompt.trim() === '') {
+        throw new Error(`Prompt file is empty: ${options.file}`);
+    }
+    (0, shared_1.loadEnv)(options.envFile);
+    const promptId = (0, shared_1.resolvePromptId)(options.promptId);
+    const client = new client_1.MetrioAI({
+        apiKey: (0, shared_1.resolveApiKey)(options.apiKey),
+        baseUrl: (0, shared_1.resolveBaseUrl)(options.baseUrl),
+    });
+    const res = await client.createPromptVersion({
+        promptId,
+        systemPrompt,
+        modelTier: options.modelTier,
+        modelProvider: options.modelProvider,
+        modelName: options.modelName,
+        // Omit when false so the request only carries explicit promotion.
+        setAsHead: options.setAsHead || undefined,
+        description: options.description,
+        outputFormat: options.outputFormat,
+    });
+    if (options.json) {
+        process.stdout.write(`${JSON.stringify(res, null, 2)}\n`);
+    }
+    else {
+        const v = res.version;
+        process.stdout.write(`Created version ${v.serial} (id ${v.id}) for prompt ${promptId}` +
+            `${options.setAsHead ? ', set as head' : ''}\n`);
+    }
+}

package/dist/commands/prompt.js

@@ -0,0 +1,284 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.parseCases = parseCases;
+exports.mergeVariables = mergeVariables;
+exports.runPromptCommand = runPromptCommand;
+/**
+ * `metrio prompt run-batch` — runs several prompt cases in one process.
+ *
+ * Each case is an independent chat completion; the win over a shell loop is
+ * paying process start-up and env loading once instead of per case. Output is
+ * JSONL in input order so results can be diffed across runs.
+ */
+const client_1 = require("@metrio-ai/client");
+const client_2 = require("@metrio-ai/client");
+const shared_1 = require("./shared");
+const prompt_create_version_1 = require("./prompt-create-version");
+const PROMPT_HELP = `metrio prompt - prompt subcommands
+
+Usage:
+  metrio prompt run-batch -p <id> --cases <file.jsonl> [options]
+  metrio prompt create-version -p <id> -f <file> [options]
+
+Run "metrio prompt create-version --help" for that command's options.
+The rest of this help covers run-batch:
+
+Required:
+  -p, --prompt-id <id>     Prompt ID to run (integer; or set METRIO_PROMPT_ID)
+      --cases <file>       JSONL cases file; "-" reads stdin
+
+Options:
+      --concurrency <n>    Cases to run at once (default: 4)
+  -t, --tag <tag>          Version tag applied to every case (a case may override)
+      --var <name=value>   Global template variable (repeatable; a case may override)
+      --request-tag <tag>  Request-level tag recorded on every run, e.g. "test"
+                            (repeatable; max 5; each 1-32 chars, lowercase
+                            alphanumeric start, may contain "-" and "_")
+      --timezone <tz>      IANA timezone, e.g. "Asia/Taipei"
+  -o, --out <file>         Write result JSONL here (default: stdout)
+  -e, --env <file>         Env file to load (default: .env in the current dir)
+      --api-key <key>      API key (overrides env file / environment)
+      --base-url <url>     Base API URL (overrides env file / environment)
+  -h, --help               Show this help
+
+Cases file format (one JSON object per line; blank and "//" lines are ignored):
+  {"id":"c1","messages":[{"role":"user","content":"…"}],"system":"…","variables":{"k":"v"},"tag":"v3"}
+
+"messages" uses the same format as the chat command's messages file, including
+"file" attachments — attachment paths resolve relative to the cases file (or the
+current directory when reading stdin). A case without an "id" is named by its
+line number, e.g. "case-3".
+
+Results are JSONL in input order, one line per case:
+  {"id":"c1","ok":true,"response":"…","inputTokens":120,"outputTokens":45,"elapsedTime":1240,"taskId":"…"}
+  {"id":"c2","ok":false,"error":{"message":"…","status":429}}
+
+Progress goes to stderr as each case finishes — "[2/3] c2 ok 1240ms" in
+completion order, then a "3 cases: 2 ok, 1 failed" tally — so stdout carries
+nothing but the result rows. Those rows are written once the whole batch has
+finished, not streamed per case, so watch stderr for live progress.
+
+A failing case does not stop the batch. The exit code is 1 if any case failed.
+`;
+/**
+ * Parses a JSONL cases file into prompt cases, validating the fields this
+ * command sends. Line numbering and the `case-<line>` fallback id come from
+ * the shared JSONL reader, so every batch command names cases the same way.
+ */
+function parseCases(raw) {
+    return (0, shared_1.parseJsonlObjects)(raw).map(({ line, value }) => {
+        const entry = value;
+        if (!Array.isArray(entry.messages) || entry.messages.length === 0) {
+            throw new Error(`Case on line ${line} must have a non-empty "messages" array`);
+        }
+        // A non-object `variables` (e.g. an array) would survive Object.entries in
+        // mergeVariables as bogus numeric-keyed variables and the case would run
+        // "ok" with unintended inputs — reject it here instead.
+        if (entry.variables !== undefined &&
+            (entry.variables === null ||
+                typeof entry.variables !== 'object' ||
+                Array.isArray(entry.variables))) {
+            throw new Error(`Case on line ${line}: "variables" must be a JSON object`);
+        }
+        return {
+            id: (0, shared_1.caseIdFor)(value, line),
+            messages: entry.messages,
+            system: entry.system,
+            variables: entry.variables,
+            tag: entry.tag,
+        };
+    });
+}
+/**
+ * Merges the global `--var` flags with a case's own variables. The case wins on
+ * conflict, so a batch can share defaults and still vary one field per case.
+ * Returns `undefined` when there is nothing to send.
+ */
+function mergeVariables(globals, caseVars) {
+    const merged = new Map();
+    for (const variable of globals)
+        merged.set(variable.name, variable.value);
+    if (caseVars) {
+        for (const [name, value] of Object.entries(caseVars))
+            merged.set(name, String(value));
+    }
+    if (merged.size === 0)
+        return undefined;
+    return [...merged].map(([name, value]) => ({ name, value }));
+}
+/** Parses argv for `prompt run-batch`. Throws on malformed input. */
+function parsePromptArgs(argv) {
+    const options = {
+        concurrency: 4,
+        variables: [],
+        tags: [],
+        help: false,
+    };
+    const next = (i, flag) => {
+        const value = argv[i + 1];
+        if (value === undefined)
+            throw new Error(`Missing value for ${flag}`);
+        return value;
+    };
+    for (let i = 0; i < argv.length; i++) {
+        let arg = argv[i];
+        let inline;
+        // Accept both `--flag value` and `--flag=value`.
+        const eq = arg.indexOf('=');
+        if (arg.startsWith('--') && eq > 2) {
+            inline = arg.slice(eq + 1);
+            arg = arg.slice(0, eq);
+        }
+        const take = (flag) => (inline !== undefined ? inline : next(i++, flag));
+        switch (arg) {
+            case '-p':
+            case '--prompt-id': {
+                const raw = take(arg);
+                const id = Number(raw);
+                if (!Number.isInteger(id) || id <= 0) {
+                    throw new Error(`--prompt-id must be a positive integer, got "${raw}"`);
+                }
+                options.promptId = id;
+                break;
+            }
+            case '--cases':
+                options.casesFile = take(arg);
+                break;
+            case '--concurrency': {
+                const raw = take(arg);
+                const n = Number(raw);
+                if (!Number.isInteger(n) || n <= 0) {
+                    throw new Error(`--concurrency must be a positive integer, got "${raw}"`);
+                }
+                options.concurrency = n;
+                break;
+            }
+            case '-t':
+            case '--tag':
+                options.tag = take(arg);
+                break;
+            case '--timezone':
+                options.timezone = take(arg);
+                break;
+            case '--var': {
+                const raw = take(arg);
+                const sep = raw.indexOf('=');
+                if (sep === -1) {
+                    throw new Error(`--var must be in the form name=value, got "${raw}"`);
+                }
+                const name = raw.slice(0, sep).trim();
+                if (!name)
+                    throw new Error(`--var name cannot be empty, got "${raw}"`);
+                options.variables.push({ name, value: raw.slice(sep + 1) });
+                break;
+            }
+            case '--request-tag': {
+                const tag = take(arg).trim();
+                if (!tag)
+                    throw new Error('--request-tag cannot be empty');
+                options.tags.push(tag);
+                break;
+            }
+            case '-o':
+            case '--out':
+                options.out = take(arg);
+                break;
+            case '-e':
+            case '--env':
+                options.envFile = take(arg);
+                break;
+            case '--api-key':
+                options.apiKey = take(arg);
+                break;
+            case '--base-url':
+                options.baseUrl = take(arg);
+                break;
+            case '-h':
+            case '--help':
+                if (inline !== undefined) {
+                    throw new Error('--help does not take a value');
+                }
+                options.help = true;
+                break;
+            default:
+                throw new Error(`Unknown argument: ${arg}`);
+        }
+    }
+    return options;
+}
+/** The `prompt run-batch` subcommand. */
+async function runPromptCommand(argv) {
+    const action = argv[0];
+    if (action === undefined || action === '-h' || action === '--help') {
+        process.stdout.write(PROMPT_HELP);
+        return;
+    }
+    if (action === 'create-version') {
+        await (0, prompt_create_version_1.runCreateVersionCommand)(argv.slice(1));
+        return;
+    }
+    if (action !== 'run-batch') {
+        throw new Error(`Unknown prompt command: "${action}". Expected "run-batch" or "create-version".`);
+    }
+    const options = parsePromptArgs(argv.slice(1));
+    if (options.help) {
+        process.stdout.write(PROMPT_HELP);
+        return;
+    }
+    if (!options.casesFile) {
+        throw new Error('Missing required option: --cases');
+    }
+    (0, shared_1.loadEnv)(options.envFile);
+    // Prompt ID may come from the flag or the env file; the flag wins.
+    const promptId = (0, shared_1.resolvePromptId)(options.promptId);
+    const { raw, baseDir } = await (0, shared_1.readCasesSource)(options.casesFile);
+    const cases = parseCases(raw);
+    const client = new client_1.MetrioAI({
+        apiKey: (0, shared_1.resolveApiKey)(options.apiKey),
+        baseUrl: (0, shared_1.resolveBaseUrl)(options.baseUrl),
+    });
+    let completed = 0;
+    const results = await (0, shared_1.mapWithConcurrency)(cases, options.concurrency, async (c) => {
+        const result = await runOneCase(client, promptId, c, baseDir, options);
+        completed++;
+        process.stderr.write(`[${completed}/${cases.length}] ${c.id} ${result.ok ? `ok ${result.elapsedTime}ms` : `failed: ${result.error?.message}`}\n`);
+        return result;
+    });
+    (0, shared_1.emitBatchResults)(results, options.out);
+}
+/** Runs a single case, converting any failure into a result row. */
+async function runOneCase(client, promptId, c, baseDir, options) {
+    try {
+        const messages = (0, shared_1.buildMessages)(c.messages, baseDir);
+        if (c.system !== undefined && c.system.trim() !== '') {
+            messages.unshift({ role: 'system', content: { type: 'text', text: c.system } });
+        }
+        const res = await client.chatCompletion({
+            promptId,
+            messages,
+            tag: c.tag ?? options.tag,
+            timezone: options.timezone,
+            variables: mergeVariables(options.variables, c.variables),
+            tags: options.tags.length ? options.tags : undefined,
+        });
+        return {
+            id: c.id,
+            ok: true,
+            response: res.response,
+            inputTokens: res.inputTokens,
+            outputTokens: res.outputTokens,
+            elapsedTime: res.elapsedTime,
+            taskId: res.taskId,
+        };
+    }
+    catch (err) {
+        return {
+            id: c.id,
+            ok: false,
+            error: {
+                message: err instanceof Error ? err.message : String(err),
+                status: err instanceof client_2.MetrioApiError ? err.statusCode : undefined,
+            },
+        };
+    }
+}