metergraph-cli 0.2.2 → 0.2.3

package/src/cli.js

@@ -1,3 +1,5 @@
+import { runPricing } from "./pricing.js";
+import { pricingText, pricingHtml } from "./pricing-output.js";
 import { READ_COMMANDS, parseArgs } from "./args.js";
 import { runLogin, runLogout } from "./auth-login.js";
 import { runDoctor } from "./doctor.js";
@@ -15,10 +17,21 @@ import {
   versionData,
 } from "./output.js";
 import { runRead } from "./read.js";
+import { runExperimentStart } from "./experiments.js";
+import { experimentsMessage, experimentsText } from "./experiments-output.js";
 import { readMessage, readText } from "./read-output.js";
 import { listSkillPack, runSkill, runSkillPack } from "./skill.js";
 import { runSetup } from "./setup.js";
 import { containsKnownCredential, preflightNonHostedSetup, prerequisiteGuide, resumeNonHostedRoute } from "./setup-deployment.js";
+import {
+  ASKABLE,
+  agentAccessOffer,
+  canAsk,
+  commandLine,
+  fillSetupArgv,
+  offerAgentAccess,
+  terminalPrompter,
+} from "./setup-interactive.js";
 import { VERSION } from "./constants.js";
 import { createStyle, fields, headline, paragraph, render, steps, toneFor } from "./style.js";
 import { runVerify } from "./verify.js";
@@ -26,12 +39,13 @@ import { verifyMessage, verifyText } from "./verify-output.js";
 
 // Runs one command and resolves with the exit code. With --json, stdout gets
 // exactly one JSON line and stderr stays empty. Without it, results go to
-// stdout and usage or internal errors go to stderr. Never reads stdin.
-export async function main(argv, { stdout, stderr }) {
+// stdout and usage or internal errors go to stderr. Only setup reads stdin,
+// and only to ask a person at a terminal (see setup-interactive.js).
+export async function main(argv, { stdout, stderr, stdin = null }) {
   const json = argv.includes("--json");
   let result;
   try {
-    result = await run(argv, { stdout, stderr });
+    result = await run(argv, { stdout, stderr, stdin });
   } catch {
     result = envelope({
       command: null,
@@ -45,9 +59,33 @@ export async function main(argv, { stdout, stderr }) {
   return result.exit_code;
 }
 
-async function run(argv, { stdout, stderr }) {
-  const parsed = parseArgs(argv);
+async function run(argv, { stdout, stderr, stdin }) {
+  let parsed = parseArgs(argv);
   const style = createStyle(stdout);
+  const say = (...lines) => stdout.write(lines.map((line) => `${line}\n`).join(""));
+  const asking = canAsk(argv, { stdin, stdout });
+
+  // A person at a terminal is asked for what setup is missing; the answers
+  // become flags and go through the same parsing as a script's.
+  if (parsed.command === "setup" && !parsed.ok && ASKABLE.has(parsed.code) && asking) {
+    const prompter = terminalPrompter(stdin, stdout);
+    let filled;
+    try {
+      filled = await fillSetupArgv(argv, { ask: prompter.ask, say });
+    } finally {
+      prompter.close();
+    }
+    if (filled === null) {
+      stderr.write("Setup was cancelled. Nothing was changed.\n");
+      return envelope({ command: "setup", outcome: "cancelled", reason: "questions_cancelled",
+        message: "Setup was cancelled. Nothing was changed." });
+    }
+    argv = filled;
+    parsed = parseArgs(argv);
+    // Only a command that parses is shown: a value the parser refuses may be
+    // a mistyped credential, and its error never echoes input.
+    if (parsed.ok) say("", `Running: ${commandLine(filled)}`, "");
+  }
 
   if (!parsed.ok) {
     const result = envelope({
@@ -73,6 +111,14 @@ async function run(argv, { stdout, stderr }) {
     return result;
   }
 
+  if (parsed.command === "pricing") {
+    const response = await runPricing(parsed);
+    const result = envelope({ command: `pricing ${parsed.action}`, ...response });
+    stdout.write(parsed.json ? toJsonLine(result) :
+      parsed.format === "html" && result.ok ? pricingHtml(result.data) : pricingText(result, parsed.format, style));
+    return result;
+  }
+
   if (parsed.command === "login" || parsed.command === "logout") {
     // Human progress goes to stderr, never with --json, so stdout carries
     // only the final result.
@@ -106,18 +152,30 @@ async function run(argv, { stdout, stderr }) {
       ? await runSetup({ ...options, expectedProfile: routed.profile ?? null, announce: pending.announce }, progress)
       : routed;
     if (pending.result) return pending.result;
-    const { outcome, reason, data } = response;
+    const { outcome, reason } = response;
+    // The customer-local bundle can give the agent more than Metadata
+    // without a sign in; say how.
+    const offer = outcome === "ok" && routed.profile === "local";
+    const data = offer ? { ...response.data, agent_access: agentAccessOffer(options.origin) } : response.data;
     let result = envelope({ command: "setup", outcome, reason, data,
       message: outcome === "ok" ? null : "Setup did not complete. Review the reason and retry safely." });
     if (containsKnownCredential(result, routed)) {
       result = envelope({ command: "setup", outcome: "unsupported", reason: "credential_echo" });
     }
-    const human = setupText(result, style);
+    const human = setupText(result, style, { asking });
     if (containsKnownCredential(human, routed)) {
       result = envelope({ command: "setup", outcome: "unsupported", reason: "credential_echo" });
       stdout.write(parsed.json ? toJsonLine(result) : "Setup stopped.\n");
-    } else {
-      stdout.write(parsed.json ? toJsonLine(result) : human);
+      return result;
+    }
+    stdout.write(parsed.json ? toJsonLine(result) : human);
+    if (offer && asking && result.ok) {
+      const prompter = terminalPrompter(stdin, stdout);
+      try {
+        await offerAgentAccess({ client: options.client, origin: options.origin, prompter, say });
+      } finally {
+        prompter.close();
+      }
     }
     return result;
   }
@@ -136,6 +194,22 @@ async function run(argv, { stdout, stderr }) {
     return result;
   }
 
+  if (parsed.command === "experiments") {
+    // start is the one write: its own key session (experiments.js). The other
+    // actions are Metadata reads through runRead's shared session.
+    const { outcome, reason, data } = parsed.action === "start" ? await runExperimentStart(parsed) : await runRead(parsed);
+    const message = experimentsMessage(outcome, reason);
+    const result = envelope({
+      command: `experiments ${parsed.action}`,
+      outcome,
+      reason,
+      data,
+      message: outcome === "ok" ? null : message,
+    });
+    stdout.write(parsed.json ? toJsonLine(result) : experimentsText(result, message, style, parsed.action));
+    return result;
+  }
+
   if (parsed.command === "verify") {
     const { outcome, reason, data } = await runVerify(parsed);
     const message = verifyMessage(outcome, reason);
@@ -212,7 +286,7 @@ const SETUP_TITLES = {
   operator_handoff: "Setup is waiting on the operator",
 };
 
-function setupText(result, style) {
+function setupText(result, style, { asking = false } = {}) {
   const data = result.data;
   const status = data?.status ?? "not_ready";
   const title = SETUP_TITLES[status] ?? (result.ok ? `Setup: ${status}` : "Setup did not complete");
@@ -241,9 +315,23 @@ function setupText(result, style) {
   lines.push(...remoteLines(data, style));
   lines.push(...budgetLines(data, style));
   if (result.ok) lines.push(...steps(style, [["", "Instrument your application and verify an exact trace."]]));
+  // At a terminal the offer is a question asked next instead.
+  if (data?.agent_access && !asking) lines.push(...agentAccessLines(data.agent_access, style));
   return render(lines);
 }
 
+// How to give the agent more than Metadata on the customer-local bundle.
+function agentAccessLines(access, style) {
+  const offer = access.offer;
+  return ["", ...steps(style, [
+    ["", "Your coding agent has Metadata access. To let it write evaluations and start analyses, " +
+      "run this in the bundle directory. It shows what it grants and asks you to confirm, then writes " +
+      "the key to that file without printing it."],
+    ["In the bundle directory", style.bold(offer.command)],
+    ["Bundle without bin/agent-key", offer.keys_page_url],
+  ]), ""];
+}
+
 // What an operator handoff is waiting for, in fixed text: each unconfirmed
 // prerequisite with its guidance, and the next action.
 function handoffLines(data, style) {

package/src/constants.js

@@ -99,8 +99,15 @@ export const READ_PATHS = Object.freeze({
   routes: "/v1/agent/routes",
   traces: "/v1/agent/traces",
 });
+const PRICING_QUERY_KEYS = Object.freeze(["model", "provider", "channel", "region", "at", "limit",
+  "input_tokens", "output_tokens", "cache_read_tokens", "cache_write_tokens", "cache_write_5m_tokens", "cache_write_1h_tokens", "batch"]);
 export const READ_QUERY_KEYS = Object.freeze({
+  "/v1/pricing/lookup": PRICING_QUERY_KEYS,
+  "/v1/pricing/cost": PRICING_QUERY_KEYS,
+  "/v1/pricing/history": PRICING_QUERY_KEYS,
+  "/v1/pricing/market": PRICING_QUERY_KEYS,
   "/v1/agent/usage": Object.freeze(["days", "limit", "metadata"]),
+  "/v1/agent/experiments": Object.freeze(["workload_id", "limit"]),
   "/v1/agent/usage/aggregates": Object.freeze([
     "group_by", "days", "since", "until", "limit", "route", "provider", "model", "environment", "metadata",
   ]),
@@ -124,6 +131,32 @@ export const USAGE_MAX_FILTER_VALUES = 20;
 export const USAGE_SORTS = Object.freeze(["cost", "cost-per-call", "calls", "errors", "avg-latency", "p95-latency", "day"]);
 export const USAGE_COMPARISONS = Object.freeze(["previous", "previous-same-time"]);
 
+// Experiments: replay a workload's recent production calls on 1 to 3
+// treatments and compare them with production. The list, estimate and
+// summary are Metadata reads at these fixed paths (one experiment is
+// EXPERIMENT_PATHS.list plus "/" and its UUID). Starting one is the only
+// write this CLI makes to a workspace: it needs a separate coding-agent key
+// with Debug access and the EXPERIMENT_START_SCOPE grant, used for that one
+// request and nothing else. The limits match the service's own.
+export const EXPERIMENT_PATHS = Object.freeze({
+  list: "/v1/agent/experiments",
+  estimate: "/v1/agent/experiments/estimate",
+  start: "/v1/agent/experiments",
+});
+export const EXPERIMENT_START_SCOPE = "agent:analysis-run";
+// Exactly the scopes a start key carries: Debug and the start grant. A key
+// with any other scope (Replay, other workflow grants, Metadata, ingest) is
+// refused, so the key in an agent's environment can do no more than this.
+export const EXPERIMENT_START_KEY_SCOPES = Object.freeze(["agent:read", EXPERIMENT_START_SCOPE]);
+export const EXPERIMENT_LAST_N = Object.freeze({ min: 10, max: 40, default: 20 });
+export const EXPERIMENT_MAX_TREATMENTS = 3;
+export const EXPERIMENT_MAX_TRACKED_FIELDS = 10;
+export const EXPERIMENT_LIST_DEFAULT_LIMIT = 20;
+export const EXPERIMENT_LIST_MAX_LIMIT = 50;
+export const EXPERIMENT_TEXT_LIMITS = Object.freeze({ workload: 500, provider: 60, model: 200, label: 40, path: 200 });
+export const EXPERIMENT_MAX_TOKENS = 32000;
+export const EXPERIMENT_REASONING_EFFORTS = Object.freeze(["none", "minimal", "low", "medium", "high", "xhigh", "max"]);
+
 // Read commands share one total deadline for every request they make,
 // including a token refresh. It is never reset per request or page.
 export const READ_DEFAULT_TIMEOUT_MS = 15000;
@@ -171,6 +204,8 @@ export const EXIT_CODES = Object.freeze({
   rate_limited: 16,
   cancelled: 17,
   action_required: 18,
+  refused: 19,
+  unconfirmed: 20,
 });
 
 export const EXIT_CODE_MEANINGS = Object.freeze({
@@ -211,4 +246,10 @@ export const EXIT_CODE_MEANINGS = Object.freeze({
     "A person must act before this command can finish; data.next_action says what to do. " +
     "With --json --no-browser the command exits as soon as the approval URL exists, and a background " +
     "process waits for approval until --timeout-ms. After approval, run the same command again to continue.",
+  refused:
+    "The service refused the experiment for a stated reason, such as its budget, the daily cap, experiments " +
+    "already running or a request it cannot price; data.refusal says which. Nothing was started.",
+  unconfirmed:
+    "An experiment start reached the service but its result was not confirmed, so an experiment may have " +
+    "been recorded and may run. List the workload's experiments before starting again: each start creates one.",
 });