metergraph-cli 0.2.3 → 0.2.4

package/src/cli.js

@@ -19,6 +19,7 @@ import {
 import { runRead } from "./read.js";
 import { runExperimentStart } from "./experiments.js";
 import { experimentsMessage, experimentsText } from "./experiments-output.js";
+import { toolsMessage, toolsText } from "./tools-output.js";
 import { readMessage, readText } from "./read-output.js";
 import { listSkillPack, runSkill, runSkillPack } from "./skill.js";
 import { runSetup } from "./setup.js";
@@ -29,6 +30,7 @@ import {
   canAsk,
   commandLine,
   fillSetupArgv,
+  fillLoginArgv,
   offerAgentAccess,
   terminalPrompter,
 } from "./setup-interactive.js";
@@ -63,7 +65,12 @@ async function run(argv, { stdout, stderr, stdin }) {
   let parsed = parseArgs(argv);
   const style = createStyle(stdout);
   const say = (...lines) => stdout.write(lines.map((line) => `${line}\n`).join(""));
-  const asking = canAsk(argv, { stdin, stdout });
+  const asking = canAsk(argv, { stdin, stdout, stderr });
+
+  if (asking && parsed.command === "login") {
+    argv = fillLoginArgv(argv);
+    parsed = parseArgs(argv);
+  }
 
   // A person at a terminal is asked for what setup is missing; the answers
   // become flags and go through the same parsing as a script's.
@@ -95,13 +102,20 @@ async function run(argv, { stdout, stderr, stdin }) {
       message: parsed.message,
     });
     if (parsed.json) stdout.write(toJsonLine(result));
-    else stderr.write(`${createStyle(stderr).red("Error:")} ${parsed.message}\n`);
+    else {
+      const errorStyle = createStyle(stderr);
+      const [first, ...rest] = paragraph(errorStyle, `Error: ${parsed.message}`, "");
+      const help = parsed.command && parsed.command !== "help"
+        ? `metergraph ${parsed.command} --help` : "metergraph --help";
+      stderr.write(render([first.replace(/^Error:/, errorStyle.red("Error:")), ...rest,
+        "", ...steps(errorStyle, [["Help", help]])]));
+    }
     return result;
   }
 
   if (parsed.command === "help") {
-    const result = envelope({ command: "help", outcome: "ok", data: helpData(parsed.topic) });
-    stdout.write(parsed.json ? toJsonLine(result) : helpText(parsed.topic, style));
+    const result = envelope({ command: "help", outcome: "ok", data: helpData(parsed.topic, parsed.action) });
+    stdout.write(parsed.json ? toJsonLine(result) : helpText(parsed.topic, style, parsed.action, parsed.advanced));
     return result;
   }
 
@@ -181,7 +195,7 @@ async function run(argv, { stdout, stderr, stdin }) {
   }
 
   if (READ_COMMANDS.includes(parsed.command)) {
-    const { outcome, reason, data } = await runRead(parsed);
+    const { outcome, reason, data, validateOutput } = await runRead(parsed);
     const message = readMessage(outcome, reason);
     const result = envelope({
       command: parsed.command,
@@ -190,14 +204,16 @@ async function run(argv, { stdout, stderr, stdin }) {
       data,
       message: outcome === "ok" ? null : message,
     });
-    stdout.write(parsed.json ? toJsonLine(result) : readText(result, message, style));
+    const text = parsed.json ? toJsonLine(result) : readText(result, message, style);
+    if (validateOutput && !validateOutput(text)) return refuseCredentialOutput(parsed, stdout, style, validateOutput);
+    stdout.write(text);
     return result;
   }
 
   if (parsed.command === "experiments") {
     // start is the one write: its own key session (experiments.js). The other
     // actions are Metadata reads through runRead's shared session.
-    const { outcome, reason, data } = parsed.action === "start" ? await runExperimentStart(parsed) : await runRead(parsed);
+    const { outcome, reason, data, validateOutput } = parsed.action === "start" ? await runExperimentStart(parsed) : await runRead(parsed);
     const message = experimentsMessage(outcome, reason);
     const result = envelope({
       command: `experiments ${parsed.action}`,
@@ -206,7 +222,26 @@ async function run(argv, { stdout, stderr, stdin }) {
       data,
       message: outcome === "ok" ? null : message,
     });
-    stdout.write(parsed.json ? toJsonLine(result) : experimentsText(result, message, style, parsed.action));
+    const text = parsed.json ? toJsonLine(result) : experimentsText(result, message, style, parsed.action);
+    if (validateOutput && !validateOutput(text)) return refuseCredentialOutput(parsed, stdout, style, validateOutput);
+    stdout.write(text);
+    return result;
+  }
+
+  if (parsed.command === "tools") {
+    // Any Agent Access tool, through runRead's verified session (tools.js).
+    const { outcome, reason, data, validateOutput } = await runRead(parsed);
+    const message = toolsMessage(outcome, reason);
+    const result = envelope({
+      command: `tools ${parsed.action}`,
+      outcome,
+      reason,
+      data,
+      message: outcome === "ok" ? null : message,
+    });
+    const text = parsed.json ? toJsonLine(result) : toolsText(result, message, style, parsed.action);
+    if (validateOutput && !validateOutput(text)) return refuseCredentialOutput(parsed, stdout, style, validateOutput);
+    stdout.write(text);
     return result;
   }
 
@@ -390,3 +425,12 @@ function repositoryValue(repository) {
     default: return "not recorded (no owner/name git remote; pass --repository OWNER/NAME)";
   }
 }
+
+function refuseCredentialOutput(parsed, stdout, style, validateOutput) {
+  const reason = "credential_in_metadata_response";
+  const message = readMessage("verification_failed", reason);
+  const result = envelope({ command: parsed.command, outcome: "verification_failed", reason, message });
+  const text = parsed.json ? toJsonLine(result) : `${message}\n`;
+  if (validateOutput(text)) stdout.write(text);
+  return result;
+}

package/src/constants.js

@@ -64,9 +64,38 @@ export const LOGIN_MIN_TIMEOUT_MS = 1000;
 export const LOGIN_MAX_TIMEOUT_MS = 900000;
 export const AUTH_HTTP_TIMEOUT_MS = 15000;
 
-// The only scope this CLI requests. It never asks for agent:read (Debug) or
+// The scope sign in requests. It never asks for agent:read (Debug) or
 // agent:replay, and never falls back to them.
 export const METADATA_SCOPE = "agent:metadata";
+// The Debug role and the workflow permissions an owner adds to it, by the
+// names the Keys page uses. Writing and testing evaluations and starting
+// analyses need Debug with all three. Replay is never used by this CLI.
+export const DEBUG_SCOPE = "agent:read";
+export const REPLAY_SCOPE = "agent:replay";
+export const WORKFLOW_PERMISSIONS = Object.freeze([
+  Object.freeze({ field: "allow_evaluation_writes", scope: "agent:eval-write", label: "Allow evaluation writes" }),
+  Object.freeze({ field: "allow_evaluation_previews", scope: "agent:eval-preview", label: "Allow evaluation previews" }),
+  Object.freeze({ field: "allow_analysis_starts", scope: "agent:analysis-run", label: "Allow analysis starts" }),
+]);
+export const WORKFLOW_SCOPES = Object.freeze(WORKFLOW_PERMISSIONS.map(({ scope }) => scope).sort());
+// The permissions are offered where the deployment reports managed
+// evaluations; customer-owned AWS (byoc-core) and the open-source server do not.
+export const WORKFLOW_DEPLOYMENT_CAPABILITY = "evaluations";
+
+// The scope login asks for and a saved sign in records: Metadata, or Debug
+// followed by the chosen workflow permissions in WORKFLOW_SCOPES order.
+export function loginScope(role, permissions = []) {
+  if (role !== "debug") return METADATA_SCOPE;
+  return [DEBUG_SCOPE, ...WORKFLOW_SCOPES.filter((scope) => permissions.includes(scope))].join(" ");
+}
+
+// Whether a saved sign in's scope is one login asks for, in that exact form.
+export function isLoginScope(scope) {
+  if (scope === METADATA_SCOPE) return true;
+  if (typeof scope !== "string") return false;
+  const [first, ...rest] = scope.split(" ");
+  return first === DEBUG_SCOPE && scope === loginScope("debug", rest);
+}
 
 // The service's agent access contract version, as sent in schema_version by
 // GET /v1/agent/workspace and GET /v1/agent/capabilities. It is the
@@ -134,20 +163,19 @@ export const USAGE_COMPARISONS = Object.freeze(["previous", "previous-same-time"
 // Experiments: replay a workload's recent production calls on 1 to 3
 // treatments and compare them with production. The list, estimate and
 // summary are Metadata reads at these fixed paths (one experiment is
-// EXPERIMENT_PATHS.list plus "/" and its UUID). Starting one is the only
-// write this CLI makes to a workspace: it needs a separate coding-agent key
-// with Debug access and the EXPERIMENT_START_SCOPE grant, used for that one
-// request and nothing else. The limits match the service's own.
+// EXPERIMENT_PATHS.list plus "/" and its UUID). Starting one needs a
+// coding-agent key with Debug access and the EXPERIMENT_START_SCOPE grant.
+// The limits match the service's own.
 export const EXPERIMENT_PATHS = Object.freeze({
   list: "/v1/agent/experiments",
   estimate: "/v1/agent/experiments/estimate",
   start: "/v1/agent/experiments",
 });
 export const EXPERIMENT_START_SCOPE = "agent:analysis-run";
-// Exactly the scopes a start key carries: Debug and the start grant. A key
-// with any other scope (Replay, other workflow grants, Metadata, ingest) is
-// refused, so the key in an agent's environment can do no more than this.
-export const EXPERIMENT_START_KEY_SCOPES = Object.freeze(["agent:read", EXPERIMENT_START_SCOPE]);
+// The scopes a start key must carry: Debug and the start grant. It may also
+// carry the other workflow permissions; a key with
+// Replay, Metadata only or any other scope is refused.
+export const EXPERIMENT_START_KEY_SCOPES = Object.freeze([DEBUG_SCOPE, EXPERIMENT_START_SCOPE]);
 export const EXPERIMENT_LAST_N = Object.freeze({ min: 10, max: 40, default: 20 });
 export const EXPERIMENT_MAX_TREATMENTS = 3;
 export const EXPERIMENT_MAX_TRACKED_FIELDS = 10;

package/src/experiments-output.js

@@ -1,6 +1,6 @@
 import { STATISTICS } from "./experiments-contract.js";
 import { readMessage } from "./read-output.js";
-import { PLAIN, fields, headline, paragraph, render, steps, toneFor } from "./style.js";
+import { PLAIN, fields, headline, paragraph, render, steps, toneFor, workspaceShown } from "./style.js";
 
 // Fixed text for experiments results. Nothing here contains a value from a
 // service response other than validated metadata; the service's own refusal
@@ -11,11 +11,12 @@ const MESSAGES = {
     "This deployment does not offer experiments to this credential, for example because the feature is off. " +
     "Nothing was read or started.",
   experiment_start_key_required:
-    "Starting an experiment needs a coding-agent key with Debug access and Allow analysis starts, in " +
-    "--agent-token-file or METERGRAPH_AGENT_TOKEN. A saved sign in is Metadata only. Nothing was sent.",
+    "Starting an experiment needs a coding-agent key with the Debug role and Allow analysis starts, in " +
+    "--agent-token-file or METERGRAPH_AGENT_TOKEN. A saved sign in is Metadata only. " +
+    "Nothing was sent.",
   experiment_start_not_granted:
-    "This key cannot start experiments: it needs exactly Debug access and the Allow analysis starts grant from a " +
-    "workspace owner, and nothing more. Nothing was started.",
+    "This key cannot start experiments: it needs Debug access and the Allow analysis starts grant from a " +
+    "workspace owner, and no Replay access. Nothing was started.",
   experiment_start_unavailable:
     "The service does not offer experiment starts to this key, for example because the feature is off or the " +
     "granting owner is no longer an owner. Nothing was started.",
@@ -165,7 +166,7 @@ export function experimentsText(result, message, style = PLAIN, action) {
       style.dim(action === "start" ? "Replays captured prompts within the server's budget" : "Metadata only")}`,
     "",
   ];
-  const rows = [["Origin", data.origin], ["Workspace", data.workspace?.id ?? null]];
+  const rows = [["Origin", data.origin], ["Workspace", workspaceShown(data.workspace)]];
   if (result.ok) {
     const [more, body] = resultLines(action, data.result);
     lines.push(...fields(style, [...rows, ...more]));

package/src/experiments.js

@@ -1,6 +1,6 @@
 import { agentKeySource, keySession, keyTarget, readAgentKey } from "./agent-key.js";
 import { readBinding, resolveProject } from "./auth-binding.js";
-import { Stop, openStore, readCredential, resolveConfigDir } from "./auth-store.js";
+import { Stop, openStore, readProjectCredential, resolveConfigDir } from "./auth-store.js";
 import { DEFAULT_ORIGIN, EXPERIMENT_PATHS, EXPERIMENT_START_SCOPE, METADATA_SCOPE, READ_MAX_BYTES } from "./constants.js";
 import { AGENT_URL_ENV, classifyEnvironment, handoffPages } from "./environment.js";
 import {
@@ -12,7 +12,8 @@ import {
 } from "./experiments-contract.js";
 import { parseJsonObject } from "./http.js";
 import { parseOrigin } from "./origin.js";
-import { holdsKnown } from "./read.js";
+import { workspaceLabel, workspacePinned } from "./read-contract.js";
+import { holdsKnown, outputHoldsKnown } from "./read.js";
 import { deadline, failureOf, send, trapSignals } from "./transport.js";
 
 // Experiments: replay a workload's recent production calls on 1 to 3
@@ -20,15 +21,15 @@ import { deadline, failureOf, send, trapSignals } from "./transport.js";
 //
 // list, estimate and get are Metadata reads. runRead (read.js) establishes
 // their session exactly as for every other read command, a saved grant or a
-// Metadata key, and calls experimentRead with it. They send only these
+// coding-agent key, and calls experimentRead with it. They send only these
 // requests and print only validated fields; no input, output or tracked
 // field value is ever requested.
 //
-// start is the only command that changes a workspace. It needs a separate
-// coding-agent key with Debug access and the agent:analysis-run grant, from
-// --agent-token-file or METERGRAPH_AGENT_TOKEN, verified for exactly that
-// (auth-oauth.js checkStartWorkspace and checkStartCapabilities) before it is
-// used for this one POST. A saved sign in is Metadata only and never starts
+// start changes a workspace. It needs a coding-agent key with Debug access
+// and the agent:analysis-run grant (Allow analysis starts), from
+// --agent-token-file or METERGRAPH_AGENT_TOKEN, verified before it is used
+// for this one POST (auth-oauth.js checkStartWorkspace and
+// checkStartCapabilities). A saved sign in is Metadata only and never starts
 // one. Each start creates a new experiment, so a start that succeeded is not
 // retried.
 
@@ -167,8 +168,8 @@ function pollOrOpen(experiment, { started = false } = {}) {
     return {
       kind: "poll_experiment",
       message: started
-        ? `The experiment is ${experiment.status}. Poll it in about ${POLL_AFTER_SECONDS} seconds with a Metadata key ` +
-          "or this project's sign in; the start key is not used for reads."
+        ? `The experiment is ${experiment.status}. Poll it in about ${POLL_AFTER_SECONDS} seconds with ` +
+          "this project's sign in or a coding-agent key."
         : `The experiment is ${experiment.status}. Run the same command again in about ${POLL_AFTER_SECONDS} seconds.`,
       command: `metergraph experiments get --id ${experiment.id} --json`,
       poll_after_seconds: POLL_AFTER_SECONDS,
@@ -217,6 +218,7 @@ export async function runExperimentStart(options) {
     origin: null,
     workspaceId: null,
     intendedWorkspace: null,
+    workspaceLabel: { slug: null, name: null },
     profile: null,
     authenticated: false,
     scopes: [],
@@ -234,9 +236,11 @@ export async function runExperimentStart(options) {
     const data = startData(ctx, notices, outcome, report);
     if (holdsKnown(data, known)) {
       return { outcome: "verification_failed", reason: "credential_in_metadata_response",
-        data: startData({ ...ctx, reason: "credential_in_metadata_response" }, new Set(), "verification_failed", null) };
+        data: startData({ ...ctx, reason: "credential_in_metadata_response", workspaceLabel: { slug: null, name: null } },
+          new Set(), "verification_failed", null),
+        validateOutput: (text) => !outputHoldsKnown(text, known) };
     }
-    return { outcome, reason, data };
+    return { outcome, reason, data, validateOutput: (text) => !outputHoldsKnown(text, known) };
   };
   const settle = (result) => {
     const interrupted =
@@ -257,7 +261,7 @@ export async function runExperimentStart(options) {
     if (binding !== null) {
       try {
         const store = openStore(resolveConfigDir(options.configDir), { create: false });
-        const record = store === null ? null : readCredential(store, binding.credential_slot);
+        const record = store === null ? null : readProjectCredential(store, binding.credential_slot, root);
         originConfirmed = record !== null && record.origin === binding.origin;
       } catch (error) {
         if (!(error instanceof Stop)) throw error;
@@ -279,6 +283,10 @@ export async function runExperimentStart(options) {
       return end(settled.outcome, reason);
     }
     ctx.profile = session.session.profile;
+    if (target.workspaceText !== null && !workspacePinned(session.documents.workspace?.workspace, target.workspaceText)) {
+      return end("verification_failed", "workspace_context_mismatch");
+    }
+    ctx.workspaceLabel = workspaceLabel(session.documents.workspace?.workspace);
     if (ctx.workspaceId === null) {
       ctx.workspaceId = session.session.workspaceId;
       notices.add("workspace_not_pinned");
@@ -335,7 +343,7 @@ function unconfirmed(response, failure) {
 function startData(ctx, notices, outcome, report) {
   return {
     origin: ctx.origin,
-    workspace: ctx.workspaceId === null ? null : { id: ctx.workspaceId },
+    workspace: ctx.workspaceId === null ? null : { id: ctx.workspaceId, ...ctx.workspaceLabel },
     deployment_profile: ctx.profile,
     environment: ctx.environment.kind,
     credential: "agent_key",
@@ -356,7 +364,7 @@ function startNextAction(outcome, ctx, report) {
       kind: "check_before_retry",
       message:
         "The start request reached the service, but its answer was not confirmed: an experiment may exist and " +
-        "may be running. List the newest experiments with a Metadata key or this project's sign in before " +
+        "may be running. List the newest experiments with this project's sign in or a coding-agent key before " +
         "starting again; each start creates a new experiment.",
       command: "metergraph experiments list --json",
     };
@@ -376,8 +384,9 @@ function startNextAction(outcome, ctx, report) {
       kind: "use_experiment_start_key",
       url: keys,
       message:
-        "Starting an experiment needs a coding-agent key with Debug access and Allow analysis starts, which only " +
-        "a workspace owner can grant on the Keys page. Pass it with --agent-token-file or METERGRAPH_AGENT_TOKEN; " +
+        "Starting an experiment needs a coding-agent key with the Debug role and Allow analysis starts, which " +
+        "only a workspace owner can grant on the Keys page (or, on the customer-local bundle, with " +
+        "bin/agent-key). Pass it with --agent-token-file or METERGRAPH_AGENT_TOKEN; " +
         "do not ask for it in chat. A start replays the sampled calls' captured prompts on each treatment's " +
         "provider within the server's budget. Or start the experiment in the dashboard.",
       required_scope: EXPERIMENT_START_SCOPE,