metergraph-cli 0.2.2 → 0.2.4

package/src/cli.js

@@ -1,3 +1,5 @@
+import { runPricing } from "./pricing.js";
+import { pricingText, pricingHtml } from "./pricing-output.js";
 import { READ_COMMANDS, parseArgs } from "./args.js";
 import { runLogin, runLogout } from "./auth-login.js";
 import { runDoctor } from "./doctor.js";
@@ -15,10 +17,23 @@ import {
   versionData,
 } from "./output.js";
 import { runRead } from "./read.js";
+import { runExperimentStart } from "./experiments.js";
+import { experimentsMessage, experimentsText } from "./experiments-output.js";
+import { toolsMessage, toolsText } from "./tools-output.js";
 import { readMessage, readText } from "./read-output.js";
 import { listSkillPack, runSkill, runSkillPack } from "./skill.js";
 import { runSetup } from "./setup.js";
 import { containsKnownCredential, preflightNonHostedSetup, prerequisiteGuide, resumeNonHostedRoute } from "./setup-deployment.js";
+import {
+  ASKABLE,
+  agentAccessOffer,
+  canAsk,
+  commandLine,
+  fillSetupArgv,
+  fillLoginArgv,
+  offerAgentAccess,
+  terminalPrompter,
+} from "./setup-interactive.js";
 import { VERSION } from "./constants.js";
 import { createStyle, fields, headline, paragraph, render, steps, toneFor } from "./style.js";
 import { runVerify } from "./verify.js";
@@ -26,12 +41,13 @@ import { verifyMessage, verifyText } from "./verify-output.js";
 
 // Runs one command and resolves with the exit code. With --json, stdout gets
 // exactly one JSON line and stderr stays empty. Without it, results go to
-// stdout and usage or internal errors go to stderr. Never reads stdin.
-export async function main(argv, { stdout, stderr }) {
+// stdout and usage or internal errors go to stderr. Only setup reads stdin,
+// and only to ask a person at a terminal (see setup-interactive.js).
+export async function main(argv, { stdout, stderr, stdin = null }) {
   const json = argv.includes("--json");
   let result;
   try {
-    result = await run(argv, { stdout, stderr });
+    result = await run(argv, { stdout, stderr, stdin });
   } catch {
     result = envelope({
       command: null,
@@ -45,9 +61,38 @@ export async function main(argv, { stdout, stderr }) {
   return result.exit_code;
 }
 
-async function run(argv, { stdout, stderr }) {
-  const parsed = parseArgs(argv);
+async function run(argv, { stdout, stderr, stdin }) {
+  let parsed = parseArgs(argv);
   const style = createStyle(stdout);
+  const say = (...lines) => stdout.write(lines.map((line) => `${line}\n`).join(""));
+  const asking = canAsk(argv, { stdin, stdout, stderr });
+
+  if (asking && parsed.command === "login") {
+    argv = fillLoginArgv(argv);
+    parsed = parseArgs(argv);
+  }
+
+  // A person at a terminal is asked for what setup is missing; the answers
+  // become flags and go through the same parsing as a script's.
+  if (parsed.command === "setup" && !parsed.ok && ASKABLE.has(parsed.code) && asking) {
+    const prompter = terminalPrompter(stdin, stdout);
+    let filled;
+    try {
+      filled = await fillSetupArgv(argv, { ask: prompter.ask, say });
+    } finally {
+      prompter.close();
+    }
+    if (filled === null) {
+      stderr.write("Setup was cancelled. Nothing was changed.\n");
+      return envelope({ command: "setup", outcome: "cancelled", reason: "questions_cancelled",
+        message: "Setup was cancelled. Nothing was changed." });
+    }
+    argv = filled;
+    parsed = parseArgs(argv);
+    // Only a command that parses is shown: a value the parser refuses may be
+    // a mistyped credential, and its error never echoes input.
+    if (parsed.ok) say("", `Running: ${commandLine(filled)}`, "");
+  }
 
   if (!parsed.ok) {
     const result = envelope({
@@ -57,13 +102,20 @@ async function run(argv, { stdout, stderr }) {
       message: parsed.message,
     });
     if (parsed.json) stdout.write(toJsonLine(result));
-    else stderr.write(`${createStyle(stderr).red("Error:")} ${parsed.message}\n`);
+    else {
+      const errorStyle = createStyle(stderr);
+      const [first, ...rest] = paragraph(errorStyle, `Error: ${parsed.message}`, "");
+      const help = parsed.command && parsed.command !== "help"
+        ? `metergraph ${parsed.command} --help` : "metergraph --help";
+      stderr.write(render([first.replace(/^Error:/, errorStyle.red("Error:")), ...rest,
+        "", ...steps(errorStyle, [["Help", help]])]));
+    }
     return result;
   }
 
   if (parsed.command === "help") {
-    const result = envelope({ command: "help", outcome: "ok", data: helpData(parsed.topic) });
-    stdout.write(parsed.json ? toJsonLine(result) : helpText(parsed.topic, style));
+    const result = envelope({ command: "help", outcome: "ok", data: helpData(parsed.topic, parsed.action) });
+    stdout.write(parsed.json ? toJsonLine(result) : helpText(parsed.topic, style, parsed.action, parsed.advanced));
     return result;
   }
 
@@ -73,6 +125,14 @@ async function run(argv, { stdout, stderr }) {
     return result;
   }
 
+  if (parsed.command === "pricing") {
+    const response = await runPricing(parsed);
+    const result = envelope({ command: `pricing ${parsed.action}`, ...response });
+    stdout.write(parsed.json ? toJsonLine(result) :
+      parsed.format === "html" && result.ok ? pricingHtml(result.data) : pricingText(result, parsed.format, style));
+    return result;
+  }
+
   if (parsed.command === "login" || parsed.command === "logout") {
     // Human progress goes to stderr, never with --json, so stdout carries
     // only the final result.
@@ -106,24 +166,36 @@ async function run(argv, { stdout, stderr }) {
       ? await runSetup({ ...options, expectedProfile: routed.profile ?? null, announce: pending.announce }, progress)
       : routed;
     if (pending.result) return pending.result;
-    const { outcome, reason, data } = response;
+    const { outcome, reason } = response;
+    // The customer-local bundle can give the agent more than Metadata
+    // without a sign in; say how.
+    const offer = outcome === "ok" && routed.profile === "local";
+    const data = offer ? { ...response.data, agent_access: agentAccessOffer(options.origin) } : response.data;
     let result = envelope({ command: "setup", outcome, reason, data,
       message: outcome === "ok" ? null : "Setup did not complete. Review the reason and retry safely." });
     if (containsKnownCredential(result, routed)) {
       result = envelope({ command: "setup", outcome: "unsupported", reason: "credential_echo" });
     }
-    const human = setupText(result, style);
+    const human = setupText(result, style, { asking });
     if (containsKnownCredential(human, routed)) {
       result = envelope({ command: "setup", outcome: "unsupported", reason: "credential_echo" });
       stdout.write(parsed.json ? toJsonLine(result) : "Setup stopped.\n");
-    } else {
-      stdout.write(parsed.json ? toJsonLine(result) : human);
+      return result;
+    }
+    stdout.write(parsed.json ? toJsonLine(result) : human);
+    if (offer && asking && result.ok) {
+      const prompter = terminalPrompter(stdin, stdout);
+      try {
+        await offerAgentAccess({ client: options.client, origin: options.origin, prompter, say });
+      } finally {
+        prompter.close();
+      }
     }
     return result;
   }
 
   if (READ_COMMANDS.includes(parsed.command)) {
-    const { outcome, reason, data } = await runRead(parsed);
+    const { outcome, reason, data, validateOutput } = await runRead(parsed);
     const message = readMessage(outcome, reason);
     const result = envelope({
       command: parsed.command,
@@ -132,7 +204,44 @@ async function run(argv, { stdout, stderr }) {
       data,
       message: outcome === "ok" ? null : message,
     });
-    stdout.write(parsed.json ? toJsonLine(result) : readText(result, message, style));
+    const text = parsed.json ? toJsonLine(result) : readText(result, message, style);
+    if (validateOutput && !validateOutput(text)) return refuseCredentialOutput(parsed, stdout, style, validateOutput);
+    stdout.write(text);
+    return result;
+  }
+
+  if (parsed.command === "experiments") {
+    // start is the one write: its own key session (experiments.js). The other
+    // actions are Metadata reads through runRead's shared session.
+    const { outcome, reason, data, validateOutput } = parsed.action === "start" ? await runExperimentStart(parsed) : await runRead(parsed);
+    const message = experimentsMessage(outcome, reason);
+    const result = envelope({
+      command: `experiments ${parsed.action}`,
+      outcome,
+      reason,
+      data,
+      message: outcome === "ok" ? null : message,
+    });
+    const text = parsed.json ? toJsonLine(result) : experimentsText(result, message, style, parsed.action);
+    if (validateOutput && !validateOutput(text)) return refuseCredentialOutput(parsed, stdout, style, validateOutput);
+    stdout.write(text);
+    return result;
+  }
+
+  if (parsed.command === "tools") {
+    // Any Agent Access tool, through runRead's verified session (tools.js).
+    const { outcome, reason, data, validateOutput } = await runRead(parsed);
+    const message = toolsMessage(outcome, reason);
+    const result = envelope({
+      command: `tools ${parsed.action}`,
+      outcome,
+      reason,
+      data,
+      message: outcome === "ok" ? null : message,
+    });
+    const text = parsed.json ? toJsonLine(result) : toolsText(result, message, style, parsed.action);
+    if (validateOutput && !validateOutput(text)) return refuseCredentialOutput(parsed, stdout, style, validateOutput);
+    stdout.write(text);
     return result;
   }
 
@@ -212,7 +321,7 @@ const SETUP_TITLES = {
   operator_handoff: "Setup is waiting on the operator",
 };
 
-function setupText(result, style) {
+function setupText(result, style, { asking = false } = {}) {
   const data = result.data;
   const status = data?.status ?? "not_ready";
   const title = SETUP_TITLES[status] ?? (result.ok ? `Setup: ${status}` : "Setup did not complete");
@@ -241,9 +350,23 @@ function setupText(result, style) {
   lines.push(...remoteLines(data, style));
   lines.push(...budgetLines(data, style));
   if (result.ok) lines.push(...steps(style, [["", "Instrument your application and verify an exact trace."]]));
+  // At a terminal the offer is a question asked next instead.
+  if (data?.agent_access && !asking) lines.push(...agentAccessLines(data.agent_access, style));
   return render(lines);
 }
 
+// How to give the agent more than Metadata on the customer-local bundle.
+function agentAccessLines(access, style) {
+  const offer = access.offer;
+  return ["", ...steps(style, [
+    ["", "Your coding agent has Metadata access. To let it write evaluations and start analyses, " +
+      "run this in the bundle directory. It shows what it grants and asks you to confirm, then writes " +
+      "the key to that file without printing it."],
+    ["In the bundle directory", style.bold(offer.command)],
+    ["Bundle without bin/agent-key", offer.keys_page_url],
+  ]), ""];
+}
+
 // What an operator handoff is waiting for, in fixed text: each unconfirmed
 // prerequisite with its guidance, and the next action.
 function handoffLines(data, style) {
@@ -302,3 +425,12 @@ function repositoryValue(repository) {
     default: return "not recorded (no owner/name git remote; pass --repository OWNER/NAME)";
   }
 }
+
+function refuseCredentialOutput(parsed, stdout, style, validateOutput) {
+  const reason = "credential_in_metadata_response";
+  const message = readMessage("verification_failed", reason);
+  const result = envelope({ command: parsed.command, outcome: "verification_failed", reason, message });
+  const text = parsed.json ? toJsonLine(result) : `${message}\n`;
+  if (validateOutput(text)) stdout.write(text);
+  return result;
+}

package/src/constants.js

@@ -64,9 +64,38 @@ export const LOGIN_MIN_TIMEOUT_MS = 1000;
 export const LOGIN_MAX_TIMEOUT_MS = 900000;
 export const AUTH_HTTP_TIMEOUT_MS = 15000;
 
-// The only scope this CLI requests. It never asks for agent:read (Debug) or
+// The scope sign in requests. It never asks for agent:read (Debug) or
 // agent:replay, and never falls back to them.
 export const METADATA_SCOPE = "agent:metadata";
+// The Debug role and the workflow permissions an owner adds to it, by the
+// names the Keys page uses. Writing and testing evaluations and starting
+// analyses need Debug with all three. Replay is never used by this CLI.
+export const DEBUG_SCOPE = "agent:read";
+export const REPLAY_SCOPE = "agent:replay";
+export const WORKFLOW_PERMISSIONS = Object.freeze([
+  Object.freeze({ field: "allow_evaluation_writes", scope: "agent:eval-write", label: "Allow evaluation writes" }),
+  Object.freeze({ field: "allow_evaluation_previews", scope: "agent:eval-preview", label: "Allow evaluation previews" }),
+  Object.freeze({ field: "allow_analysis_starts", scope: "agent:analysis-run", label: "Allow analysis starts" }),
+]);
+export const WORKFLOW_SCOPES = Object.freeze(WORKFLOW_PERMISSIONS.map(({ scope }) => scope).sort());
+// The permissions are offered where the deployment reports managed
+// evaluations; customer-owned AWS (byoc-core) and the open-source server do not.
+export const WORKFLOW_DEPLOYMENT_CAPABILITY = "evaluations";
+
+// The scope login asks for and a saved sign in records: Metadata, or Debug
+// followed by the chosen workflow permissions in WORKFLOW_SCOPES order.
+export function loginScope(role, permissions = []) {
+  if (role !== "debug") return METADATA_SCOPE;
+  return [DEBUG_SCOPE, ...WORKFLOW_SCOPES.filter((scope) => permissions.includes(scope))].join(" ");
+}
+
+// Whether a saved sign in's scope is one login asks for, in that exact form.
+export function isLoginScope(scope) {
+  if (scope === METADATA_SCOPE) return true;
+  if (typeof scope !== "string") return false;
+  const [first, ...rest] = scope.split(" ");
+  return first === DEBUG_SCOPE && scope === loginScope("debug", rest);
+}
 
 // The service's agent access contract version, as sent in schema_version by
 // GET /v1/agent/workspace and GET /v1/agent/capabilities. It is the
@@ -99,8 +128,15 @@ export const READ_PATHS = Object.freeze({
   routes: "/v1/agent/routes",
   traces: "/v1/agent/traces",
 });
+const PRICING_QUERY_KEYS = Object.freeze(["model", "provider", "channel", "region", "at", "limit",
+  "input_tokens", "output_tokens", "cache_read_tokens", "cache_write_tokens", "cache_write_5m_tokens", "cache_write_1h_tokens", "batch"]);
 export const READ_QUERY_KEYS = Object.freeze({
+  "/v1/pricing/lookup": PRICING_QUERY_KEYS,
+  "/v1/pricing/cost": PRICING_QUERY_KEYS,
+  "/v1/pricing/history": PRICING_QUERY_KEYS,
+  "/v1/pricing/market": PRICING_QUERY_KEYS,
   "/v1/agent/usage": Object.freeze(["days", "limit", "metadata"]),
+  "/v1/agent/experiments": Object.freeze(["workload_id", "limit"]),
   "/v1/agent/usage/aggregates": Object.freeze([
     "group_by", "days", "since", "until", "limit", "route", "provider", "model", "environment", "metadata",
   ]),
@@ -124,6 +160,31 @@ export const USAGE_MAX_FILTER_VALUES = 20;
 export const USAGE_SORTS = Object.freeze(["cost", "cost-per-call", "calls", "errors", "avg-latency", "p95-latency", "day"]);
 export const USAGE_COMPARISONS = Object.freeze(["previous", "previous-same-time"]);
 
+// Experiments: replay a workload's recent production calls on 1 to 3
+// treatments and compare them with production. The list, estimate and
+// summary are Metadata reads at these fixed paths (one experiment is
+// EXPERIMENT_PATHS.list plus "/" and its UUID). Starting one needs a
+// coding-agent key with Debug access and the EXPERIMENT_START_SCOPE grant.
+// The limits match the service's own.
+export const EXPERIMENT_PATHS = Object.freeze({
+  list: "/v1/agent/experiments",
+  estimate: "/v1/agent/experiments/estimate",
+  start: "/v1/agent/experiments",
+});
+export const EXPERIMENT_START_SCOPE = "agent:analysis-run";
+// The scopes a start key must carry: Debug and the start grant. It may also
+// carry the other workflow permissions; a key with
+// Replay, Metadata only or any other scope is refused.
+export const EXPERIMENT_START_KEY_SCOPES = Object.freeze([DEBUG_SCOPE, EXPERIMENT_START_SCOPE]);
+export const EXPERIMENT_LAST_N = Object.freeze({ min: 10, max: 40, default: 20 });
+export const EXPERIMENT_MAX_TREATMENTS = 3;
+export const EXPERIMENT_MAX_TRACKED_FIELDS = 10;
+export const EXPERIMENT_LIST_DEFAULT_LIMIT = 20;
+export const EXPERIMENT_LIST_MAX_LIMIT = 50;
+export const EXPERIMENT_TEXT_LIMITS = Object.freeze({ workload: 500, provider: 60, model: 200, label: 40, path: 200 });
+export const EXPERIMENT_MAX_TOKENS = 32000;
+export const EXPERIMENT_REASONING_EFFORTS = Object.freeze(["none", "minimal", "low", "medium", "high", "xhigh", "max"]);
+
 // Read commands share one total deadline for every request they make,
 // including a token refresh. It is never reset per request or page.
 export const READ_DEFAULT_TIMEOUT_MS = 15000;
@@ -171,6 +232,8 @@ export const EXIT_CODES = Object.freeze({
   rate_limited: 16,
   cancelled: 17,
   action_required: 18,
+  refused: 19,
+  unconfirmed: 20,
 });
 
 export const EXIT_CODE_MEANINGS = Object.freeze({
@@ -211,4 +274,10 @@ export const EXIT_CODE_MEANINGS = Object.freeze({
     "A person must act before this command can finish; data.next_action says what to do. " +
     "With --json --no-browser the command exits as soon as the approval URL exists, and a background " +
     "process waits for approval until --timeout-ms. After approval, run the same command again to continue.",
+  refused:
+    "The service refused the experiment for a stated reason, such as its budget, the daily cap, experiments " +
+    "already running or a request it cannot price; data.refusal says which. Nothing was started.",
+  unconfirmed:
+    "An experiment start reached the service but its result was not confirmed, so an experiment may have " +
+    "been recorded and may run. List the workload's experiments before starting again: each start creates one.",
 });