metergraph-cli 0.2.2 → 0.2.3

package/src/experiments.js

@@ -0,0 +1,397 @@
+import { agentKeySource, keySession, keyTarget, readAgentKey } from "./agent-key.js";
+import { readBinding, resolveProject } from "./auth-binding.js";
+import { Stop, openStore, readCredential, resolveConfigDir } from "./auth-store.js";
+import { DEFAULT_ORIGIN, EXPERIMENT_PATHS, EXPERIMENT_START_SCOPE, METADATA_SCOPE, READ_MAX_BYTES } from "./constants.js";
+import { AGENT_URL_ENV, classifyEnvironment, handoffPages } from "./environment.js";
+import {
+  experimentEstimate,
+  experimentRefusal,
+  experimentStarted,
+  experimentSummary,
+  experimentsList,
+} from "./experiments-contract.js";
+import { parseJsonObject } from "./http.js";
+import { parseOrigin } from "./origin.js";
+import { holdsKnown } from "./read.js";
+import { deadline, failureOf, send, trapSignals } from "./transport.js";
+
+// Experiments: replay a workload's recent production calls on 1 to 3
+// treatments and read the built-in statistics per arm against production.
+//
+// list, estimate and get are Metadata reads. runRead (read.js) establishes
+// their session exactly as for every other read command, a saved grant or a
+// Metadata key, and calls experimentRead with it. They send only these
+// requests and print only validated fields; no input, output or tracked
+// field value is ever requested.
+//
+// start is the only command that changes a workspace. It needs a separate
+// coding-agent key with Debug access and the agent:analysis-run grant, from
+// --agent-token-file or METERGRAPH_AGENT_TOKEN, verified for exactly that
+// (auth-oauth.js checkStartWorkspace and checkStartCapabilities) before it is
+// used for this one POST. A saved sign in is Metadata only and never starts
+// one. Each start creates a new experiment, so a start that succeeded is not
+// retried.
+
+const CAPABILITIES = Object.freeze({ list: "experiments", estimate: "experiment_estimate", get: "experiment" });
+const RETRY_AFTER = /^[0-9]{1,5}$/;
+// The service's refusal reasons with their own fixed message here; any other
+// is reported as experiment_refused with its code kept in data.refusal.
+export const REFUSAL_REASONS = Object.freeze([
+  "over_budget",
+  "no_replayable_calls",
+  "experiments_in_progress",
+  "daily_budget_exhausted",
+  "route_unpriced",
+  "invalid_reasoning_effort",
+  "invalid_settings",
+  "invalid_sample",
+  "invalid_treatment",
+  "invalid_treatments",
+  "invalid_tracked_fields",
+  "invalid_request",
+]);
+export const POLL_AFTER_SECONDS = 15;
+
+const isObject = (value) => value !== null && typeof value === "object" && !Array.isArray(value);
+
+// A Metadata capability this grant can read: listed, available and plainly
+// metadata only.
+function readable(entry) {
+  return isObject(entry) && entry.available === true && entry.content === false && entry.mutates === false &&
+    entry.external_calls === false && entry.privacy_class === "metadata" && entry.required_scope === METADATA_SCOPE;
+}
+
+function retryAfterOf(response) {
+  const header = response.headers["retry-after"];
+  const seconds = typeof header === "string" && RETRY_AFTER.test(header.trim()) ? Number(header.trim()) : null;
+  return seconds !== null && seconds <= 86400 ? seconds : null;
+}
+
+// One answer from an experiments endpoint as { ok: true, body } or a failure
+// with fixed tokens. A refusal keeps the service's code, reason and numbers.
+function answer(response, expected) {
+  if (response.kind === "response") {
+    const { status } = response;
+    if (status === 401) return { ok: false, outcome: "login_required", reason: "access_revoked" };
+    if (status === 403) return { ok: false, outcome: "permission_denied", reason: "forbidden" };
+    if (status === 429) return { ok: false, outcome: "rate_limited", reason: "rate_limited", retryAfter: retryAfterOf(response) };
+    if (status === 501) return { ok: false, outcome: "capability_unavailable", reason: "experiments_unavailable" };
+    if (status === 404 || status === 409 || status === 422) {
+      const refusal = experimentRefusal(parseJsonObject(response));
+      if (refusal === null) return { ok: false, outcome: "unsupported", reason: "request_rejected" };
+      const named = refusal.code === "not_found" ? "experiment_not_found" : refusal.reason;
+      const reason = named === "experiment_not_found" || REFUSAL_REASONS.includes(named) ? named : "experiment_refused";
+      return { ok: false, outcome: "refused", reason, refusal };
+    }
+  }
+  const failure = failureOf(response);
+  if (failure !== null) return { ok: false, ...failure };
+  if (response.status !== expected) return { ok: false, outcome: "unsupported", reason: "unexpected_status" };
+  const body = parseJsonObject(response);
+  if (body === null) return { ok: false, outcome: "verification_failed", reason: "experiment_response_invalid" };
+  return { ok: true, body };
+}
+
+// The list, estimate or summary read for runRead, with its verified session.
+export async function experimentRead(session, options, ctx, documents, signal, notices) {
+  if (!readable(documents.capabilities?.agent?.[CAPABILITIES[options.action]])) {
+    return { ok: false, outcome: "capability_unavailable", reason: "experiments_unavailable" };
+  }
+  const request = options.experiment;
+  const common = { signal, maxBytes: READ_MAX_BYTES, bearer: session.accessToken };
+  let response;
+  let validate;
+  if (options.action === "list") {
+    const query = new URLSearchParams();
+    if (request.workloadId !== null) query.set("workload_id", request.workloadId);
+    query.set("limit", String(request.limit));
+    response = await send(session.origin, EXPERIMENT_PATHS.list, { ...common, query });
+    validate = (body) => experimentsList(body, ctx, request, notices);
+  } else if (options.action === "estimate") {
+    response = await send(session.origin, EXPERIMENT_PATHS.estimate, { ...common, method: "POST", json: request.request });
+    validate = (body) => experimentEstimate(body, ctx, request.request, notices);
+  } else {
+    // The ID is a validated UUID, so the path is still a fixed shape.
+    response = await send(session.origin, `${EXPERIMENT_PATHS.list}/${request.id}`, common);
+    validate = (body) => experimentSummary(body, ctx, request, notices);
+  }
+  const read = answer(response, 200);
+  if (!read.ok) {
+    if (read.reason === "access_revoked" && ctx.credential === "agent_key") read.reason = "agent_key_rejected";
+    return read;
+  }
+  return validate(read.body);
+}
+
+// What an agent does next with an experiments result, or undefined for the
+// read commands' shared handling (keys, sign in, workspace, rate limits).
+export function experimentNextAction(outcome, ctx, options, report) {
+  if (options.command !== "experiments") return undefined;
+  const origin = ctx.origin;
+  if (outcome === "ok" && options.action === "estimate") {
+    return {
+      kind: report.blocking_reasons.length > 0 ? "change_experiment" : "start_experiment",
+      message: report.blocking_reasons.length > 0
+        ? "A start now would be refused: this estimate is above the experiment budget or the workspace's daily " +
+          "cap. Sample fewer calls, drop a treatment or lower max_tokens, then estimate again."
+        : 'Run the same command as "metergraph experiments start" to start it. A start needs a coding-agent key ' +
+          "with Debug access and Allow analysis starts, in --agent-token-file or METERGRAPH_AGENT_TOKEN; " +
+          "or start it in the dashboard.",
+      url: report.new_experiment_url,
+      start_granted: report.start_granted,
+    };
+  }
+  if (outcome === "ok" && options.action === "get") return pollOrOpen(report.experiment);
+  if (outcome === "refused" && ctx.reason === "experiment_not_found") {
+    return {
+      kind: "list_experiments",
+      message: "No experiment with this ID is visible to this credential's workspace. List them to find it.",
+      command: "metergraph experiments list --json",
+    };
+  }
+  if (outcome === "refused") return refusedAction(ctx.reason);
+  if (outcome === "capability_unavailable") {
+    return {
+      kind: "use_dashboard",
+      message:
+        "This deployment does not offer experiments to this credential: the feature may be off for it. " +
+        "Nothing was read or started.",
+      url: origin === null ? null : `${origin}/#experiments`,
+    };
+  }
+  return undefined;
+}
+
+function pollOrOpen(experiment, { started = false } = {}) {
+  if (!experiment.terminal) {
+    return {
+      kind: "poll_experiment",
+      message: started
+        ? `The experiment is ${experiment.status}. Poll it in about ${POLL_AFTER_SECONDS} seconds with a Metadata key ` +
+          "or this project's sign in; the start key is not used for reads."
+        : `The experiment is ${experiment.status}. Run the same command again in about ${POLL_AFTER_SECONDS} seconds.`,
+      command: `metergraph experiments get --id ${experiment.id} --json`,
+      poll_after_seconds: POLL_AFTER_SECONDS,
+    };
+  }
+  if (experiment.status === "failed") {
+    return {
+      kind: "open_experiment",
+      message:
+        "The experiment failed; its error_code says why. Statistics cover only the cases that finished. " +
+        "Inputs, outputs and tracked field values are in the dashboard.",
+      url: experiment.app_url,
+    };
+  }
+  return {
+    kind: "open_experiment",
+    message:
+      "The statistics show how each treatment differs from production on the same calls; they do not judge it. " +
+      "Inputs, outputs and tracked field values are in the dashboard.",
+    url: experiment.app_url,
+  };
+}
+
+function refusedAction(reason) {
+  if (reason === "experiments_in_progress" || reason === "daily_budget_exhausted") {
+    return {
+      kind: "retry_later",
+      message: reason === "experiments_in_progress"
+        ? "Two experiments are already running in this workspace. Start this one when one finishes."
+        : "This workspace has used its daily experiment budget. Try again later.",
+    };
+  }
+  return {
+    kind: "change_experiment",
+    message:
+      "Change the request and estimate it again: sample fewer calls, drop or change a treatment, lower max_tokens " +
+      "or pick a workload with captured calls. Nothing was started.",
+  };
+}
+
+// experiments start: one POST with a start key, then the experiment to poll.
+export async function runExperimentStart(options) {
+  const env = options.env ?? process.env;
+  const ctx = {
+    environment: options.environment ?? classifyEnvironment(env),
+    origin: null,
+    workspaceId: null,
+    intendedWorkspace: null,
+    profile: null,
+    authenticated: false,
+    scopes: [],
+    reason: null,
+    // Where the Keys page is before any origin is resolved.
+    keysOrigin: options.origin ?? (typeof env[AGENT_URL_ENV] === "string" ? parseOrigin(env[AGENT_URL_ENV]) : null) ??
+      DEFAULT_ORIGIN,
+  };
+  const notices = new Set();
+  const trap = trapSignals();
+  const limit = deadline(options.timeoutMs, trap.signal);
+  let known = [];
+  const end = (outcome, reason, report = null) => {
+    ctx.reason = reason;
+    const data = startData(ctx, notices, outcome, report);
+    if (holdsKnown(data, known)) {
+      return { outcome: "verification_failed", reason: "credential_in_metadata_response",
+        data: startData({ ...ctx, reason: "credential_in_metadata_response" }, new Set(), "verification_failed", null) };
+    }
+    return { outcome, reason, data };
+  };
+  const settle = (result) => {
+    const interrupted =
+      (result.outcome === "authorization_failed" && result.reason === "cancelled") ||
+      (result.outcome === "connection_failed" && result.reason === "timeout");
+    if (interrupted && limit.timedOut()) return { outcome: "connection_failed", reason: "timeout" };
+    if (interrupted && trap.signal.aborted) return { outcome: "cancelled", reason: "cancelled" };
+    return result;
+  };
+  try {
+    const root = resolveProject(options.project);
+    const binding = readBinding(root)?.binding ?? null;
+    const source = agentKeySource(options, env);
+    if (source === null) return end("permission_denied", "experiment_start_key_required");
+    // As for reads: a binding's non-hosted origin is confirmed by a saved
+    // sign in for it on this machine, or by --url or METERGRAPH_URL.
+    let originConfirmed = false;
+    if (binding !== null) {
+      try {
+        const store = openStore(resolveConfigDir(options.configDir), { create: false });
+        const record = store === null ? null : readCredential(store, binding.credential_slot);
+        originConfirmed = record !== null && record.origin === binding.origin;
+      } catch (error) {
+        if (!(error instanceof Stop)) throw error;
+      }
+    }
+    if (source.kind === "env") {
+      const shape = readAgentKey(source, env);
+      if (!shape.ok) return end(shape.outcome, shape.reason);
+    }
+    const target = keyTarget({ binding, options, env, originConfirmed });
+    if (!target.ok) return end(target.outcome, target.reason);
+    Object.assign(ctx, {
+      origin: target.origin, workspaceId: target.workspaceId, intendedWorkspace: target.workspaceId, profile: target.profile,
+    });
+    const session = await keySession({ target, source, env, signal: limit.signal, access: "experiment_start" });
+    if (!session.ok) {
+      const settled = settle(session);
+      const reason = settled.reason === "agent_key_insufficient_scope" ? "experiment_start_not_granted" : settled.reason;
+      return end(settled.outcome, reason);
+    }
+    ctx.profile = session.session.profile;
+    if (ctx.workspaceId === null) {
+      ctx.workspaceId = session.session.workspaceId;
+      notices.add("workspace_not_pinned");
+    }
+    known = session.knownCredentials;
+    ctx.authenticated = true;
+    ctx.scopes = session.session.scopes;
+    const request = options.experiment.request;
+    // The start has its own deadline: the service samples, prices and
+    // launches within the request, and an answer cut short is ambiguous.
+    const post = deadline(options.timeoutMs, trap.signal);
+    const response = await send(ctx.origin, EXPERIMENT_PATHS.start, {
+      method: "POST", json: request, signal: post.signal, maxBytes: READ_MAX_BYTES, bearer: session.session.accessToken,
+    });
+    const started = answer(response, 202);
+    if (!started.ok) {
+      if (unconfirmed(response, started)) return end("unconfirmed", "experiment_start_unconfirmed");
+      if (started.reason === "access_revoked") started.reason = "agent_key_rejected";
+      if (started.outcome === "permission_denied") started.reason = "experiment_start_not_granted";
+      if (started.reason === "timeout" && post.timedOut()) return end("connection_failed", "timeout");
+      if (trap.signal.aborted) return end("cancelled", "cancelled");
+      return end(started.outcome, started.reason, started.refusal ? { refusal: started.refusal } : null);
+    }
+    // Accepted: whatever its answer says from here, an experiment exists.
+    const value = experimentStarted(started.body, ctx, request, notices);
+    if (!value.ok) return end("unconfirmed", "experiment_start_unconfirmed");
+    if (value.value.experiment.status === "failed") return end("unhealthy", "experiment_launch_failed", value.value);
+    return end("ok", null, value.value);
+  } catch (error) {
+    if (error instanceof Stop) {
+      const settled = settle(error);
+      return end(settled.outcome, settled.reason);
+    }
+    throw error;
+  } finally {
+    trap.release();
+  }
+}
+
+// A start whose request was handed to the network and then failed, or that
+// the service answered with a server error, may have been recorded.
+function unconfirmed(response, failure) {
+  if (response.kind === "error") return response.sent === true;
+  if (response.status >= 200 && response.status < 300) return true;
+  if (failure.outcome === "unhealthy" && failure.reason === "server_error") return true;
+  if (response.status === 503) {
+    const body = parseJsonObject(response);
+    // A managed-workflow refusal is decided before anything is recorded.
+    return !(isObject(body?.error) && body.error.code === "managed_features_disabled");
+  }
+  return failure.outcome === "unsupported" && failure.reason === "response_too_large";
+}
+
+function startData(ctx, notices, outcome, report) {
+  return {
+    origin: ctx.origin,
+    workspace: ctx.workspaceId === null ? null : { id: ctx.workspaceId },
+    deployment_profile: ctx.profile,
+    environment: ctx.environment.kind,
+    credential: "agent_key",
+    authenticated: ctx.authenticated,
+    scopes: ctx.scopes,
+    result: outcome === "ok" || ctx.reason === "experiment_launch_failed" ? report : null,
+    refusal: outcome === "refused" ? report?.refusal ?? null : null,
+    notices: [...notices].sort(),
+    next_action: startNextAction(outcome, ctx, report),
+  };
+}
+
+function startNextAction(outcome, ctx, report) {
+  const keys = handoffPages(ctx.origin ?? ctx.keysOrigin).keys_url;
+  if (outcome === "ok") return pollOrOpen({ ...report.experiment, app_url: report.app_url }, { started: true });
+  if (outcome === "unconfirmed") {
+    return {
+      kind: "check_before_retry",
+      message:
+        "The start request reached the service, but its answer was not confirmed: an experiment may exist and " +
+        "may be running. List the newest experiments with a Metadata key or this project's sign in before " +
+        "starting again; each start creates a new experiment.",
+      command: "metergraph experiments list --json",
+    };
+  }
+  if (ctx.reason === "experiment_launch_failed") {
+    return {
+      kind: "start_again_later",
+      message:
+        "The service recorded the experiment but could not launch its runner, so it failed and nothing ran. " +
+        "Start it again later.",
+      url: report.app_url,
+    };
+  }
+  if (outcome === "refused") return refusedAction(ctx.reason);
+  if (["experiment_start_key_required", "experiment_start_not_granted", "experiment_start_unavailable"].includes(ctx.reason)) {
+    return {
+      kind: "use_experiment_start_key",
+      url: keys,
+      message:
+        "Starting an experiment needs a coding-agent key with Debug access and Allow analysis starts, which only " +
+        "a workspace owner can grant on the Keys page. Pass it with --agent-token-file or METERGRAPH_AGENT_TOKEN; " +
+        "do not ask for it in chat. A start replays the sampled calls' captured prompts on each treatment's " +
+        "provider within the server's budget. Or start the experiment in the dashboard.",
+      required_scope: EXPERIMENT_START_SCOPE,
+    };
+  }
+  if (ctx.reason === "agent_key_rejected") {
+    return { kind: "replace_agent_key", url: keys, message: "The service refused this key: it is unknown, expired or revoked." };
+  }
+  if (ctx.reason === "workspace_context_mismatch") {
+    return { kind: "use_workspace_key", url: keys, message: "This key belongs to a different workspace than this project or --workspace names." };
+  }
+  if (outcome === "rate_limited") return { kind: "retry_later", message: "Wait, then run the same command again." };
+  if (outcome === "capability_unavailable") {
+    return { kind: "use_dashboard", message: "This deployment does not offer experiments to this key. Nothing was started.", url: null };
+  }
+  return null;
+}

package/src/output.js

@@ -1,3 +1,4 @@
+import { PRICING_OPTIONS } from "./pricing-args.js";
 import {
   CONNECTION_GUIDE_URL,
   DEFAULT_ORIGIN,
@@ -232,6 +233,37 @@ const READ_BASE_OPTIONS = [
       "Without it, METERGRAPH_AGENT_TOKEN is used when this project has no saved sign in on this machine.",
   },
 ];
+const EXPERIMENTS_USAGE = [
+  "metergraph experiments list [--workload ID] [--limit N] [read options] [--json]",
+  "metergraph experiments estimate --workload ID --treatment SPEC [--treatment SPEC]... [--last-n N] " +
+    "[--sample-model MODEL] [--track PATH]... [read options] [--json]",
+  "metergraph experiments start (same as estimate) [--agent-token-file PATH] [--json]",
+  "metergraph experiments get --id UUID [read options] [--json]",
+];
+const EXPERIMENTS_OPTIONS = [
+  { name: "--workload", value: "ID", summary: "The workload (route) whose recent production calls are replayed. Required for estimate and start; a filter for list." },
+  {
+    name: "--treatment",
+    value: "SPEC",
+    summary:
+      "1 to 3 times. PROVIDER/MODEL, optionally followed by ,label=NAME ,temperature=0..2 ,max_tokens=1..32000 " +
+      "or ,reasoning_effort=none|minimal|low|medium|high|xhigh|max. Unset settings follow production.",
+  },
+  { name: "--last-n", value: "N", summary: "Replay the last N replayable calls, 10 to 40. Default 20." },
+  { name: "--sample-model", value: "MODEL", summary: "Only production calls made with this exact model, for a single-model control." },
+  { name: "--track", value: "PATH", summary: "Up to 10 times. A JSON path such as $.priority whose values the dashboard compares." },
+  { name: "--id", value: "UUID", summary: "The experiment to read (get)." },
+  { name: "--limit", value: "N", summary: "Experiments to list, 1 to 50. Default 20." },
+  ...READ_BASE_OPTIONS.filter((option) => option.name !== "--agent-token-file"),
+  {
+    name: "--agent-token-file",
+    value: "PATH",
+    summary:
+      "Private (0600) file holding a coding-agent key. For list, estimate and get: a key with Metadata access. " +
+      "For start: a key with Debug access and Allow analysis starts, used for that one request.",
+  },
+  JSON_OPTION,
+];
 const DAYS_OPTION = {
   name: "--days",
   value: "N",
@@ -424,6 +456,7 @@ export function helpData(topic) {
       "metergraph --help [--json]",
       "metergraph --version [--json]",
       "metergraph doctor [--url ORIGIN] [--timeout-ms N] [--json]",
+      "metergraph pricing models|lookup|calculate|cost|history|market [options] [--json]",
       ...SKILL_USAGE,
       ...SKILLS_USAGE,
       LOGIN_USAGE,
@@ -431,8 +464,12 @@ export function helpData(topic) {
       SETUP_USAGE,
       VERIFY_USAGE,
       ...READ_HELP.map((entry) => entry.usage),
+      ...EXPERIMENTS_USAGE,
     ],
     commands: [
+      { name: "pricing", summary: "Public model discovery, lookup and calculation. Current market board; historical charts in local reference mode.",
+        options: [...PRICING_OPTIONS.map(name => ({ name, value: "VALUE", summary: "See metergraph help pricing and docs/pricing.md." })),
+          { name: "--batch", value: null, summary: "Use batch pricing for cost." }, JSON_OPTION] },
       {
         name: "doctor",
         summary:
@@ -488,7 +525,7 @@ export function helpData(topic) {
         name: "setup",
         summary: "Guide browser sign in and workspace choice, approve an ingest-only key, and install the selected client skill. Does not verify application traffic.",
         options: [
-          { name: "--runtime", value: "RUNTIME", summary: "Required. local only; other runtimes get a handoff." },
+          { name: "--runtime", value: "RUNTIME", summary: "Required unless a person runs setup at a terminal (then local). local only; other runtimes get a handoff." },
           { name: "--url", value: "ORIGIN", summary: "Deployment origin. Defaults to an existing project binding, else the hosted origin." },
           { name: "--workspace", value: "UUID", summary: "Expected workspace; the browser must approve this exact workspace." },
           { name: "--project", value: "DIR", summary: "Project directory. Default: current directory." },
@@ -514,6 +551,14 @@ export function helpData(topic) {
         summary: "Poll Metadata for one exact trace in an explicit invocation window. Never sends application traffic.",
         options: VERIFY_OPTIONS,
       },
+      {
+        name: "experiments",
+        summary:
+          "Estimate, start, list and read experiments: a workload's recent production calls replayed on 1 to 3 " +
+          "treatments, with built-in statistics per arm next to production. list, estimate and get are Metadata " +
+          "reads; start needs a key with Debug access and Allow analysis starts and spends within the server's budget.",
+        options: EXPERIMENTS_OPTIONS,
+      },
       ...READ_HELP.map(({ name, summary, options, examples }) => ({
         name,
         summary,
@@ -535,6 +580,43 @@ export function helpData(topic) {
 
 export function helpText(topic, style = PLAIN) {
   const lines = [];
+  if (topic === "pricing") {
+    return `Usage: metergraph pricing models|lookup|calculate|cost|history|market [options] [--json]
+
+Public catalog prices. No account, project or token is required.
+models  [--model ID] [--provider ID | --channel ID] [--region ID] [--limit 1..100]
+lookup  --model ID [--provider ID | --channel ID] [--region ID] [--at YYYY-MM-DD]
+calculate (or cost) Same selectors; --input-tokens N --output-tokens N [--batch]
+        --cache-read-tokens N; --trace-format anthropic.messages|openai.responses|
+        openai.chat_completions|otel.gen_ai. Native format inferred for direct
+        Anthropic/OpenAI; otherwise otel.gen_ai. Only supplied counts are sent.
+        Anthropic also accepts --cache-write-tokens N or
+        --cache-write-5m-tokens N / --cache-write-1h-tokens N.
+market  --channel ID [--region ID] [--limit 1..100] (current rates, no history)
+history --model ID --channel ID [--region ID] [--format table|chart|html]
+        Requires --api reference --url LOOPBACK_ORIGIN; no public history API.
+
+All actions: --url ORIGIN --timeout-ms 100..30000 --json --api public|reference
+Default API: public; default origin: https://app.metergraph.dev.
+--at selects UTC midnight; without it public lookup/calculation use server time.
+Models and public market discovery are current-only. Filters on models run locally.
+History/market region defaults to global. Omitted input/output counts stay unknown;
+partial costs are known subtotals, not complete costs. OpenAI cache counts must
+not exceed total input. --json cannot be combined with --format.
+
+Examples:
+  metergraph pricing models --limit 10
+  metergraph pricing lookup --model gpt-5 --provider openai --region global --json
+  metergraph pricing calculate --model claude-sonnet-5 --provider anthropic --input-tokens 1000 --cache-read-tokens 800 --output-tokens 200
+  metergraph pricing market --channel openai-api --limit 5
+  metergraph pricing history --model openai/gpt-5.6-terra --channel openai-api --api reference --url http://127.0.0.1:8765 --format html > history.html
+
+Public API: GET /v1/pricing/models; POST /v1/pricing/lookup and /calculate.
+No credentials, traces, prompts or responses are loaded from project files.
+The CLI constructs metadata-only provider-format traces from the supplied counts.
+See docs/pricing.md for the supported contract, limitations and reference previews.
+`;
+  }
   if (topic === "doctor") {
     lines.push(
       "Usage: metergraph doctor [--url ORIGIN] [--timeout-ms N] [--json]",
@@ -560,6 +642,8 @@ export function helpText(topic, style = PLAIN) {
       "--skip-skill is explicit. Non-hosted setup requires an explicit origin, workspace and",
       "operator prerequisite attestation; OSS remains an operator handoff. No application",
       "traffic is claimed until an exact instrumented invocation is separately verified.",
+      "", "At a terminal without --json, setup asks for any of these it needs that are",
+      "missing, prints the full command, then runs it. Scripts and agents get no questions.",
       "", "Options:");
     lines.push(...optionLines(helpData(null).commands.find((entry) => entry.name === "setup").options, 25));
     lines.push("", ...AGENT_APPROVAL_HELP);
@@ -599,6 +683,30 @@ export function helpText(topic, style = PLAIN) {
       "Opening requires a server link that selects the verified workspace; older links get a manual handoff.",
       "", "Options:");
     lines.push(...optionLines(VERIFY_OPTIONS, 22));
+  } else if (topic === "experiments") {
+    lines.push(
+      "Usage:",
+      ...EXPERIMENTS_USAGE.map((usage) => `  ${usage}`),
+      "",
+      "An experiment replays a workload's last 10 to 40 replayable production calls on 1 to 3 treatments",
+      "(a provider, model and settings) and compares each with what production returned on the same calls:",
+      "cost per call, p50 latency and output tokens, output length, JSON parse rate, same top-level keys,",
+      "empty or refused and cut-off rates. The statistics show differences; they do not judge them.",
+      "",
+      "estimate prices it against the server's budget and daily cap and writes nothing. start takes the",
+      "same arguments, so it starts exactly what was priced; poll it with get until it is terminal.",
+      "Each start creates a new experiment, so a start that succeeded is never retried.",
+      "",
+      "list, estimate and get use this project's saved Metadata grant or a Metadata key, as the read",
+      "commands do, and never read inputs, outputs or tracked field values; those are in the dashboard.",
+      "start needs a coding-agent key with Debug access and Allow analysis starts, which only a workspace",
+      "owner can grant, in --agent-token-file or METERGRAPH_AGENT_TOKEN. The CLI checks the key has exactly",
+      "that before it sends the one start request, and never uses it for anything else. The service then",
+      "replays the sampled calls' captured prompts on each treatment's provider within its budget.",
+      "",
+      "Options:",
+    );
+    lines.push(...optionLines(EXPERIMENTS_OPTIONS, 22));
   } else if (READ_HELP.some((entry) => entry.name === topic)) {
     const entry = READ_HELP.find((candidate) => candidate.name === topic);
     lines.push(
@@ -699,6 +807,8 @@ export function helpText(topic, style = PLAIN) {
       "  metergraph usage [options]       Daily usage per route, Metadata only",
       "  metergraph routes [options]      Routes and evaluation contract versions",
       "  metergraph traces [options]      One page of trace metadata",
+      "  metergraph experiments list|estimate|start|get  Replay a workload on other models",
+      "  metergraph pricing <action>      Public lookup, cost, history and market board",
       "",
       'Run "metergraph help doctor" for doctor options.',
       'Run "metergraph help skill" for skill options.',
@@ -706,6 +816,7 @@ export function helpText(topic, style = PLAIN) {
       'Run "metergraph help login" or "metergraph help logout" for sign in options.',
       'Run "metergraph help verify" for exact-trace verification options.',
       'Run "metergraph help COMMAND" for status, context, capabilities, usage, routes or traces.',
+      'Run "metergraph help experiments" for experiment options.',
       "",
       ...AGENT_PATH_HELP,
     );