metergraph-cli 0.2.2 → 0.2.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +49 -5
- package/bin/metergraph.js +1 -1
- package/package.json +1 -1
- package/src/agent-key.js +6 -3
- package/src/args.js +218 -22
- package/src/auth-oauth.js +58 -5
- package/src/cli.js +98 -10
- package/src/constants.js +41 -0
- package/src/experiments-contract.js +462 -0
- package/src/experiments-output.js +189 -0
- package/src/experiments.js +397 -0
- package/src/output.js +112 -1
- package/src/pricing-args.js +163 -0
- package/src/pricing-output.js +206 -0
- package/src/pricing-reference.js +195 -0
- package/src/pricing.js +307 -0
- package/src/read-contract.js +23 -3
- package/src/read.js +15 -1
- package/src/setup-deployment.js +7 -0
- package/src/setup-interactive.js +341 -0
- package/src/transport.js +5 -3
package/README.md
CHANGED
|
@@ -53,7 +53,7 @@ npx --yes metergraph-cli@latest traces --limit 20 # one page of trace metadata
|
|
|
53
53
|
```
|
|
54
54
|
|
|
55
55
|
To install it globally instead, run `npm install -g metergraph-cli@latest`. The command is
|
|
56
|
-
`metergraph`. Pin an exact version, such as `metergraph-cli@0.2.
|
|
56
|
+
`metergraph`. Pin an exact version, such as `metergraph-cli@0.2.3`, when a
|
|
57
57
|
script must not change with the `latest` tag. Previews use the `next` tag.
|
|
58
58
|
|
|
59
59
|
## Requirements
|
|
@@ -109,7 +109,11 @@ The CLI is built to be run by Codex, Claude Code and Cursor as well as by people
|
|
|
109
109
|
`schema_version`, `command`, `ok`, `outcome`, `exit_code`, `data` and `error`.
|
|
110
110
|
Failures carry a fixed `error.reason` and, where useful, a `data.next_action` that
|
|
111
111
|
tells the agent what to do next.
|
|
112
|
-
- **No hidden prompts.**
|
|
112
|
+
- **No hidden prompts.** Only `setup` asks questions, and only a person at a terminal:
|
|
113
|
+
with stdin and stdout both terminals, no `--json` and no `CI`, it asks for what its
|
|
114
|
+
flags leave out and prints the full command before running it. Otherwise no command
|
|
115
|
+
reads stdin or asks a question, so an agent or script gets the same flag errors as
|
|
116
|
+
before. `--no-browser`
|
|
113
117
|
prints the sign in URL for a person to open instead of launching a browser. With
|
|
114
118
|
`--json`, a step that needs approval exits within seconds with 18
|
|
115
119
|
`action_required` and `data.next_action` set to
|
|
@@ -135,6 +139,8 @@ Suggested `AGENTS.md` or `CLAUDE.md` lines for a project that uses Metergraph:
|
|
|
135
139
|
|
|
136
140
|
| Command | What it does | Network |
|
|
137
141
|
| --- | --- | --- |
|
|
142
|
+
| `pricing models`, `pricing lookup`, `pricing calculate` (`cost` alias) | Discover models, look up rates and calculate costs through the public API without sign in | Yes |
|
|
143
|
+
| `pricing market`, `pricing history` | Current public market board; historical terminal/HTML charts require the local reference adapter ([guide](docs/pricing.md)) | Yes |
|
|
138
144
|
| `setup` | Sign in, choose a workspace, write the ingest key to `.env`, record the repository, install the skill | Yes |
|
|
139
145
|
| `verify` | Confirm that one exact trace, by trace ID or request ID, reached the workspace | Yes |
|
|
140
146
|
| `usage` | Daily calls, errors, cost, tokens and latency per route, or cost grouped by provider, model, route or UTC day with exact filters, rankings and a prior-period comparison; filtered by exact metadata or grouped by one metadata key | Yes |
|
|
@@ -143,6 +149,7 @@ Suggested `AGENTS.md` or `CLAUDE.md` lines for a project that uses Metergraph:
|
|
|
143
149
|
| `status` | Whether the project is signed in, and to which workspace | Yes |
|
|
144
150
|
| `context` | Workspace details and retention | Yes |
|
|
145
151
|
| `capabilities` | What the workspace makes available to this project | Yes |
|
|
152
|
+
| `experiments list`, `estimate`, `start`, `get` | Replay a workload's recent production calls on 1 to 3 other models or settings: price it, start it, poll it and read cost, latency and output statistics per treatment next to production | Yes |
|
|
146
153
|
| `login`, `logout` | Sign a project in or out without the rest of `setup` | Yes |
|
|
147
154
|
| `doctor` | Check that a service is reachable, healthy and supported, without credentials | Yes |
|
|
148
155
|
| `skill install`, `skill update` | Install or update the Metergraph skill for one agent | No |
|
|
@@ -152,10 +159,37 @@ Suggested `AGENTS.md` or `CLAUDE.md` lines for a project that uses Metergraph:
|
|
|
152
159
|
[CLI reference](https://github.com/metergraph/cli/blob/main/docs/reference.md) covers every option, what each command reads and
|
|
153
160
|
writes, and full JSON examples.
|
|
154
161
|
|
|
162
|
+
## Public pricing preview
|
|
163
|
+
|
|
164
|
+
```sh
|
|
165
|
+
metergraph pricing models --limit 10
|
|
166
|
+
metergraph pricing lookup --model gpt-5 --provider openai --region global --json
|
|
167
|
+
metergraph pricing calculate --model claude-sonnet-5 --provider anthropic \
|
|
168
|
+
--input-tokens 1000 --cache-read-tokens 800 --output-tokens 200
|
|
169
|
+
metergraph pricing market --channel openai-api --limit 5
|
|
170
|
+
```
|
|
171
|
+
|
|
172
|
+
These commands use the public Pricing API without an account or workspace.
|
|
173
|
+
`cost` is an alias for `calculate`. Inspect status and missing dimensions: an HTTP
|
|
174
|
+
success may still be partial or unpriced. The current market board uses discovery
|
|
175
|
+
plus bounded lookups; it does not invent historical prices or returns.
|
|
176
|
+
|
|
177
|
+
The stock-style history visualization is preserved in explicit local reference mode:
|
|
178
|
+
|
|
179
|
+
```sh
|
|
180
|
+
metergraph pricing history --model openai/gpt-5.6-terra --channel openai-api \
|
|
181
|
+
--api reference --url http://127.0.0.1:8765 --format html > history.html
|
|
182
|
+
```
|
|
183
|
+
|
|
184
|
+
See [the pricing guide](docs/pricing.md) for the API contract, input semantics and
|
|
185
|
+
local adapter. The public service does not document a history endpoint. This is
|
|
186
|
+
an exploratory preview; supplier pricing coverage and storage claims are not
|
|
187
|
+
independently verified by these command tests.
|
|
188
|
+
|
|
155
189
|
## Exit codes
|
|
156
190
|
|
|
157
191
|
Exit codes are stable. Changing one is a breaking change. Codes 10 to 18 are not in
|
|
158
|
-
`0.1.0
|
|
192
|
+
`0.1.0`, and codes 19 and 20 are not in `0.2.2` or earlier.
|
|
159
193
|
|
|
160
194
|
| Code | Outcome | Meaning |
|
|
161
195
|
| --- | --- | --- |
|
|
@@ -178,6 +212,8 @@ Exit codes are stable. Changing one is a breaking change. Codes 10 to 18 are not
|
|
|
178
212
|
| 16 | `rate_limited` | The service asked the CLI to slow down. Try again later. Only setup retries, after a short Retry-After, to keep its approval. |
|
|
179
213
|
| 17 | `cancelled` | A read command was interrupted before it finished. Read commands never change workspace configuration or telemetry. |
|
|
180
214
|
| 18 | `action_required` | A person must act before the command can finish; `data.next_action` says what to do. `login` and `setup` with `--json --no-browser` exit with this as soon as an approval URL exists (`next_action.kind: "open_url"`), while a background process waits for the approval until `--timeout-ms`. After approval, run the same command again to continue. |
|
|
215
|
+
| 19 | `refused` | The service refused an experiment for a stated reason, such as its budget, the daily cap, experiments already running or a model it cannot price; `data.refusal` says which. Nothing was started. |
|
|
216
|
+
| 20 | `unconfirmed` | An `experiments start` request reached the service but its result was not confirmed (a timeout, a dropped connection, a server error or an answer that does not match the contract), so an experiment may exist and may run. List experiments before starting again; each start creates one. |
|
|
181
217
|
|
|
182
218
|
## Security and privacy
|
|
183
219
|
|
|
@@ -185,7 +221,11 @@ Exit codes are stable. Changing one is a breaking change. Codes 10 to 18 are not
|
|
|
185
221
|
only to the Metergraph origin you point it at.
|
|
186
222
|
- **Least access.** Sign in grants only the Metadata scope, `agent:metadata`. The CLI
|
|
187
223
|
never requests Debug or Replay access, never falls back to broader access, and never
|
|
188
|
-
reads retained prompts, responses or other content. It calls no model provider.
|
|
224
|
+
reads retained prompts, responses or other content. It calls no model provider. The one
|
|
225
|
+
exception is `experiments start`: it uses a coding-agent key a workspace owner created
|
|
226
|
+
with exactly Debug access and Allow analysis starts, refuses a key with any other scope,
|
|
227
|
+
and sends it only with that one start request. The service then replays the sampled calls on the
|
|
228
|
+
chosen providers within its own budget. Every other command refuses such a key.
|
|
189
229
|
- **Credentials.** The sign in grant is stored per user in `~/.config/metergraph`
|
|
190
230
|
(`AppData\Roaming\Metergraph` on Windows), as owner-only files on Linux and macOS and
|
|
191
231
|
encrypted with DPAPI on Windows. The ingest key goes only into the project's private
|
|
@@ -194,7 +234,11 @@ Exit codes are stable. Changing one is a breaking change. Codes 10 to 18 are not
|
|
|
194
234
|
- **Files it writes in your project:** `.env` (from `setup`), the skill file, and
|
|
195
235
|
secret-free state under `.metergraph/` that is safe to commit. The skill installer
|
|
196
236
|
never replaces a file it did not write and has no force option.
|
|
197
|
-
- **
|
|
237
|
+
- **Public pricing.** Pricing reads send no credentials and return public catalog data.
|
|
238
|
+
`--format html` emits a standalone document; it does not launch a browser.
|
|
239
|
+
- **Network.** Workspace reads use `GET`; experiments estimate and start use `POST`.
|
|
240
|
+
Public pricing lookup/calculation use unauthenticated `POST` with metadata-only
|
|
241
|
+
inputs. Fixed paths, no redirects followed,
|
|
198
242
|
bounded response sizes and fixed timeouts. Mistyped arguments are never echoed, since
|
|
199
243
|
they may hold a secret. `--help`, `--version` and `skill` make no network requests.
|
|
200
244
|
|
package/bin/metergraph.js
CHANGED
|
@@ -7,5 +7,5 @@ import { main } from "../src/cli.js";
|
|
|
7
7
|
process.stdout.on("error", () => {});
|
|
8
8
|
|
|
9
9
|
const argv = process.argv.slice(2);
|
|
10
|
-
const io = { stdout: process.stdout, stderr: process.stderr };
|
|
10
|
+
const io = { stdout: process.stdout, stderr: process.stderr, stdin: process.stdin };
|
|
11
11
|
process.exitCode = handsOff(argv) ? await handOff(argv, io) : await main(argv, io);
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "metergraph-cli",
|
|
3
|
-
"version": "0.2.
|
|
3
|
+
"version": "0.2.3",
|
|
4
4
|
"description": "Metergraph command line tool with a read-only connection probe, a project skill installer, Metadata-only sign in and bounded workspace Metadata reads",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"bin": {
|
package/src/agent-key.js
CHANGED
|
@@ -84,7 +84,10 @@ export function keyTarget({ binding, options, env, originConfirmed = false }) {
|
|
|
84
84
|
// { ok: false, outcome, reason }
|
|
85
85
|
// verifiedProfile is the profile the caller already read from GET
|
|
86
86
|
// /v1/deployment under the same deadline, or null to read it here.
|
|
87
|
-
|
|
87
|
+
// access is "metadata" for the read commands, or "experiment_start" for
|
|
88
|
+
// experiments start, which needs Debug access and the start grant instead
|
|
89
|
+
// (verifyContext) and uses the key for that one request.
|
|
90
|
+
export async function keySession({ target, source, env, signal, verifiedProfile = null, access = "metadata" }) {
|
|
88
91
|
// A key already in the environment is checked for shape before any
|
|
89
92
|
// request; a key file is opened only once the origin has proven itself.
|
|
90
93
|
let key = source.kind === "env" ? readAgentKey(source, env) : null;
|
|
@@ -108,7 +111,7 @@ export async function keySession({ target, source, env, signal, verifiedProfile
|
|
|
108
111
|
const verified = await verifyContext(
|
|
109
112
|
target.origin,
|
|
110
113
|
token,
|
|
111
|
-
{ profile, workspaceId: target.workspaceId, anyWorkspace: target.workspaceId === null },
|
|
114
|
+
{ profile, workspaceId: target.workspaceId, anyWorkspace: target.workspaceId === null, access },
|
|
112
115
|
signal,
|
|
113
116
|
);
|
|
114
117
|
if (!verified.ok) {
|
|
@@ -125,7 +128,7 @@ export async function keySession({ target, source, env, signal, verifiedProfile
|
|
|
125
128
|
origin: target.origin,
|
|
126
129
|
workspaceId: verified.workspaceId,
|
|
127
130
|
profile,
|
|
128
|
-
scopes: [METADATA_SCOPE],
|
|
131
|
+
scopes: access === "metadata" ? [METADATA_SCOPE] : verified.scopes,
|
|
129
132
|
accessToken: token,
|
|
130
133
|
},
|
|
131
134
|
documents: verified.documents,
|
package/src/args.js
CHANGED
|
@@ -1,7 +1,16 @@
|
|
|
1
|
+
import { parsePricing, PRICING_ACTIONS, PRICING_OPTIONS } from "./pricing-args.js";
|
|
1
2
|
import { parseRepository } from "./repository-identity.js";
|
|
2
3
|
import {
|
|
3
4
|
DEFAULT_ORIGIN,
|
|
4
5
|
DEFAULT_TIMEOUT_MS,
|
|
6
|
+
EXPERIMENT_LAST_N,
|
|
7
|
+
EXPERIMENT_LIST_DEFAULT_LIMIT,
|
|
8
|
+
EXPERIMENT_LIST_MAX_LIMIT,
|
|
9
|
+
EXPERIMENT_MAX_TOKENS,
|
|
10
|
+
EXPERIMENT_MAX_TRACKED_FIELDS,
|
|
11
|
+
EXPERIMENT_MAX_TREATMENTS,
|
|
12
|
+
EXPERIMENT_REASONING_EFFORTS,
|
|
13
|
+
EXPERIMENT_TEXT_LIMITS,
|
|
5
14
|
HANDOFF_SKILL_CLIENTS,
|
|
6
15
|
HANDOFF_SKILL_RUNTIMES,
|
|
7
16
|
HANDOFF_LOGIN_RUNTIMES,
|
|
@@ -35,8 +44,9 @@ import { parseOrigin } from "./origin.js";
|
|
|
35
44
|
import { groupedRequest, hasUnsafeText, isCursor, isSafeFilter, readQuery } from "./read-contract.js";
|
|
36
45
|
|
|
37
46
|
export const READ_COMMANDS = Object.freeze(["status", "context", "capabilities", "usage", "routes", "traces"]);
|
|
38
|
-
const COMMANDS = new Set(["doctor", "help", "skill", "skills", "login", "logout", "setup", "verify", ...READ_COMMANDS]);
|
|
39
|
-
const HELP_TOPICS = new Set(["doctor", "skill", "skills", "login", "logout", "setup", "verify", ...READ_COMMANDS]);
|
|
47
|
+
const COMMANDS = new Set(["pricing", "doctor", "help", "skill", "skills", "login", "logout", "setup", "verify", "experiments", ...READ_COMMANDS]);
|
|
48
|
+
const HELP_TOPICS = new Set(["pricing", "doctor", "skill", "skills", "login", "logout", "setup", "verify", "experiments", ...READ_COMMANDS]);
|
|
49
|
+
export const EXPERIMENT_ACTIONS = Object.freeze(["list", "estimate", "start", "get"]);
|
|
40
50
|
const SKILL_ACTIONS = new Set(["install", "update"]);
|
|
41
51
|
const SKILLS_ACTIONS = new Set(["install", "update", "list"]);
|
|
42
52
|
const READ_BASE = ["--project", "--config-dir", "--timeout-ms"];
|
|
@@ -45,6 +55,7 @@ const READ_BASE = ["--project", "--config-dir", "--timeout-ms"];
|
|
|
45
55
|
// never an argument.
|
|
46
56
|
const READ_KEY = ["--url", "--workspace", "--agent-token-file"];
|
|
47
57
|
const OPTIONS = {
|
|
58
|
+
pricing: new Set(PRICING_OPTIONS),
|
|
48
59
|
doctor: new Set(["--url", "--timeout-ms"]),
|
|
49
60
|
skill: new Set(["--client", "--runtime", "--project"]),
|
|
50
61
|
skills: new Set(["--client", "--runtime", "--project"]),
|
|
@@ -63,6 +74,16 @@ const OPTIONS = {
|
|
|
63
74
|
...READ_BASE, ...READ_KEY, "--days", "--limit", "--route", "--status", "--cursor", "--metadata", "--metadata-json",
|
|
64
75
|
]),
|
|
65
76
|
verify: new Set([...READ_BASE, "--trace-id", "--request-id", "--since", "--until", "--source", "--days", "--poll-ms", "--max-attempts"]),
|
|
77
|
+
experiments: new Set([
|
|
78
|
+
...READ_BASE, ...READ_KEY, "--workload", "--limit", "--treatment", "--last-n", "--sample-model", "--track", "--id",
|
|
79
|
+
]),
|
|
80
|
+
};
|
|
81
|
+
// Which experiments options each action takes, beyond READ_BASE and READ_KEY.
|
|
82
|
+
const EXPERIMENT_OPTIONS = {
|
|
83
|
+
list: new Set(["--workload", "--limit"]),
|
|
84
|
+
estimate: new Set(["--workload", "--treatment", "--last-n", "--sample-model", "--track"]),
|
|
85
|
+
start: new Set(["--workload", "--treatment", "--last-n", "--sample-model", "--track"]),
|
|
86
|
+
get: new Set(["--id"]),
|
|
66
87
|
};
|
|
67
88
|
// Requests the read commands recognize and refuse, so a script gets an
|
|
68
89
|
// explicit unsupported result instead of an unknown argument or, worse, a
|
|
@@ -96,6 +117,7 @@ const REFUSED_MESSAGES = {
|
|
|
96
117
|
};
|
|
97
118
|
// Options that may be given more than once; their values are kept in order.
|
|
98
119
|
const REPEATABLE = new Set(["--metadata"]);
|
|
120
|
+
const EXPERIMENT_REPEATABLE = new Set(["--treatment", "--track"]);
|
|
99
121
|
// usage takes several values for each of its exact-match filters.
|
|
100
122
|
const USAGE_REPEATABLE = new Set(["--route", "--provider", "--model", "--environment"]);
|
|
101
123
|
// usage reads a time range and environments itself, through grouped usage.
|
|
@@ -106,6 +128,7 @@ const USAGE_GROUPED = [
|
|
|
106
128
|
];
|
|
107
129
|
// Options that take no value.
|
|
108
130
|
const FLAGS = {
|
|
131
|
+
pricing: new Set(["--batch"]),
|
|
109
132
|
login: new Set(["--signup", "--no-browser", "--reconnect"]),
|
|
110
133
|
setup: new Set(["--no-browser", "--repair", "--signup", "--reconnect", "--skip-skill", "--confirm-prerequisites", "--no-repository"]),
|
|
111
134
|
verify: new Set(["--open", "--no-browser"]),
|
|
@@ -147,7 +170,7 @@ export function parseArgs(argv) {
|
|
|
147
170
|
|
|
148
171
|
const fail = (code, message, outcome = "invalid_input") => ({
|
|
149
172
|
ok: false,
|
|
150
|
-
command: (command === "skill" || command === "skills") && action !== null
|
|
173
|
+
command: (command === "skill" || command === "skills" || command === "experiments") && action !== null
|
|
151
174
|
? `${command} ${action}`
|
|
152
175
|
: command ?? (version ? "version" : null),
|
|
153
176
|
json,
|
|
@@ -174,6 +197,10 @@ export function parseArgs(argv) {
|
|
|
174
197
|
topic = arg;
|
|
175
198
|
continue;
|
|
176
199
|
}
|
|
200
|
+
if (command === "pricing" && action === null && PRICING_ACTIONS.includes(arg)) {
|
|
201
|
+
action = arg;
|
|
202
|
+
continue;
|
|
203
|
+
}
|
|
177
204
|
if (command === "skill" && action === null && SKILL_ACTIONS.has(arg)) {
|
|
178
205
|
action = arg;
|
|
179
206
|
continue;
|
|
@@ -182,6 +209,10 @@ export function parseArgs(argv) {
|
|
|
182
209
|
action = arg;
|
|
183
210
|
continue;
|
|
184
211
|
}
|
|
212
|
+
if (command === "experiments" && action === null && EXPERIMENT_ACTIONS.includes(arg)) {
|
|
213
|
+
action = arg;
|
|
214
|
+
continue;
|
|
215
|
+
}
|
|
185
216
|
|
|
186
217
|
if (READ_COMMANDS.includes(command)) {
|
|
187
218
|
const equals = arg.indexOf("=");
|
|
@@ -212,7 +243,8 @@ export function parseArgs(argv) {
|
|
|
212
243
|
if (value === undefined) {
|
|
213
244
|
return fail("missing_value", `${name} requires a value.`);
|
|
214
245
|
}
|
|
215
|
-
if (REPEATABLE.has(name) || (command === "usage" && USAGE_REPEATABLE.has(name))
|
|
246
|
+
if (REPEATABLE.has(name) || (command === "usage" && USAGE_REPEATABLE.has(name)) ||
|
|
247
|
+
(command === "experiments" && EXPERIMENT_REPEATABLE.has(name))) {
|
|
216
248
|
values[name] = [...(values[name] ?? []), value];
|
|
217
249
|
continue;
|
|
218
250
|
}
|
|
@@ -242,6 +274,12 @@ export function parseArgs(argv) {
|
|
|
242
274
|
`Unknown skills subcommand at argument ${position}. Use install, update or list.`,
|
|
243
275
|
);
|
|
244
276
|
}
|
|
277
|
+
if (command === "experiments" && action === null && !arg.startsWith("-")) {
|
|
278
|
+
return fail(
|
|
279
|
+
"unknown_subcommand",
|
|
280
|
+
`Unknown experiments subcommand at argument ${position}. Use list, estimate, start or get.`,
|
|
281
|
+
);
|
|
282
|
+
}
|
|
245
283
|
return fail(
|
|
246
284
|
"unknown_argument",
|
|
247
285
|
`Unrecognized argument at position ${position}. Run "metergraph --help" for usage.`,
|
|
@@ -252,11 +290,13 @@ export function parseArgs(argv) {
|
|
|
252
290
|
if (command === null || command === "help") {
|
|
253
291
|
return { ok: true, command: "help", topic, json };
|
|
254
292
|
}
|
|
293
|
+
if (command === "pricing") return parsePricing(action, values, flags, json, fail);
|
|
255
294
|
if (command === "skill") return parseSkill(action, values, json, fail);
|
|
256
295
|
if (command === "skills") return parseSkills(action, values, json, fail);
|
|
257
296
|
if (command === "login") return parseLogin(values, flags, json, fail);
|
|
258
297
|
if (command === "setup") return parseSetup(values, flags, json, fail);
|
|
259
298
|
if (command === "verify") return parseVerify(values, flags, json, fail);
|
|
299
|
+
if (command === "experiments") return parseExperiments(action, values, json, fail);
|
|
260
300
|
if (READ_COMMANDS.includes(command)) return parseRead(command, values, flags, json, fail);
|
|
261
301
|
if (command === "logout") {
|
|
262
302
|
const paths = parsePaths(values, fail);
|
|
@@ -280,6 +320,177 @@ export function parseArgs(argv) {
|
|
|
280
320
|
return { ok: true, command: "doctor", origin, timeoutMs, json };
|
|
281
321
|
}
|
|
282
322
|
|
|
323
|
+
// --url, --workspace and --agent-token-file, shared by the read commands and
|
|
324
|
+
// experiments: { ok: true, origin, workspace, agentTokenFile } or a failure.
|
|
325
|
+
function parseKeyTarget(values, fail) {
|
|
326
|
+
let origin = null;
|
|
327
|
+
if (values["--url"] !== undefined) {
|
|
328
|
+
origin = parseOrigin(values["--url"]);
|
|
329
|
+
if (origin === null) return fail("invalid_url", INVALID_URL);
|
|
330
|
+
}
|
|
331
|
+
const rawWorkspace = values["--workspace"];
|
|
332
|
+
if (rawWorkspace !== undefined && !UUID.test(rawWorkspace)) {
|
|
333
|
+
return fail("invalid_workspace", "--workspace must be a workspace ID in UUID form.");
|
|
334
|
+
}
|
|
335
|
+
const agentTokenFile = values["--agent-token-file"] ?? null;
|
|
336
|
+
if (agentTokenFile !== null && (agentTokenFile === "" || agentTokenFile.includes("\0"))) {
|
|
337
|
+
return fail("invalid_agent_token_file", "--agent-token-file must name a private absolute file.");
|
|
338
|
+
}
|
|
339
|
+
return {
|
|
340
|
+
ok: true,
|
|
341
|
+
origin,
|
|
342
|
+
workspace: rawWorkspace === undefined ? null : rawWorkspace.toLowerCase(),
|
|
343
|
+
agentTokenFile,
|
|
344
|
+
};
|
|
345
|
+
}
|
|
346
|
+
|
|
347
|
+
// experiments list | estimate | start | get. estimate and start take the same
|
|
348
|
+
// request, so an agent starts exactly what it priced: the workload, 1 to 3
|
|
349
|
+
// --treatment PROVIDER/MODEL[,label=..][,temperature=..][,max_tokens=..]
|
|
350
|
+
// [,reasoning_effort=..], and optionally --last-n, --sample-model and up to ten
|
|
351
|
+
// --track JSON paths. The service validates every request in full; these
|
|
352
|
+
// checks only refuse what it would refuse before anything is sent.
|
|
353
|
+
function parseExperiments(action, values, json, fail) {
|
|
354
|
+
if (action === null) {
|
|
355
|
+
return fail(
|
|
356
|
+
"missing_subcommand",
|
|
357
|
+
'experiments requires a subcommand: list, estimate, start or get. Run "metergraph help experiments" for usage.',
|
|
358
|
+
);
|
|
359
|
+
}
|
|
360
|
+
for (const name of Object.keys(values)) {
|
|
361
|
+
if (!READ_BASE.includes(name) && !READ_KEY.includes(name) && !EXPERIMENT_OPTIONS[action].has(name)) {
|
|
362
|
+
return fail("unknown_argument", `experiments ${action} does not take ${name}. Run "metergraph help experiments" for usage.`);
|
|
363
|
+
}
|
|
364
|
+
}
|
|
365
|
+
const paths = parsePaths(values, fail);
|
|
366
|
+
if (!paths.ok) return paths;
|
|
367
|
+
const rawTimeout = values["--timeout-ms"];
|
|
368
|
+
const timeoutMs = rawTimeout === undefined
|
|
369
|
+
? READ_DEFAULT_TIMEOUT_MS
|
|
370
|
+
: parseTimeout(rawTimeout, READ_MIN_TIMEOUT_MS, READ_MAX_TIMEOUT_MS);
|
|
371
|
+
if (timeoutMs === null) {
|
|
372
|
+
return fail("invalid_timeout", `--timeout-ms must be a whole number from ${READ_MIN_TIMEOUT_MS} to ${READ_MAX_TIMEOUT_MS}.`);
|
|
373
|
+
}
|
|
374
|
+
const target = parseKeyTarget(values, fail);
|
|
375
|
+
if (!target.ok) return target;
|
|
376
|
+
delete target.ok;
|
|
377
|
+
const parsed = {
|
|
378
|
+
ok: true, command: "experiments", action, ...target, project: paths.project, configDir: paths.configDir,
|
|
379
|
+
timeoutMs, json, experiment: { id: null, workloadId: null, limit: null, request: null },
|
|
380
|
+
};
|
|
381
|
+
const workload = values["--workload"];
|
|
382
|
+
if (workload !== undefined && !experimentText(workload, EXPERIMENT_TEXT_LIMITS.workload)) {
|
|
383
|
+
return fail("invalid_workload", `--workload must be 1 to ${EXPERIMENT_TEXT_LIMITS.workload} characters without control characters.`);
|
|
384
|
+
}
|
|
385
|
+
if (action === "get") {
|
|
386
|
+
const id = values["--id"];
|
|
387
|
+
if (id === undefined || !UUID.test(id)) return fail("invalid_experiment_id", "--id must be an experiment ID in UUID form.");
|
|
388
|
+
parsed.experiment.id = id.toLowerCase();
|
|
389
|
+
return parsed;
|
|
390
|
+
}
|
|
391
|
+
if (action === "list") {
|
|
392
|
+
const limit = values["--limit"] === undefined
|
|
393
|
+
? EXPERIMENT_LIST_DEFAULT_LIMIT
|
|
394
|
+
: parseBounded(values["--limit"], 1, EXPERIMENT_LIST_MAX_LIMIT);
|
|
395
|
+
if (limit === null) return fail("invalid_limit", `--limit must be a whole number from 1 to ${EXPERIMENT_LIST_MAX_LIMIT}.`);
|
|
396
|
+
// The exact query list sends, so an over-long one fails here and not after sign in.
|
|
397
|
+
const query = new URLSearchParams({ ...(workload === undefined ? {} : { workload_id: workload }), limit: String(limit) });
|
|
398
|
+
if (query.toString().length > MAX_QUERY_LENGTH) {
|
|
399
|
+
return fail("query_too_long", "--workload is too long to send in one request once encoded.");
|
|
400
|
+
}
|
|
401
|
+
parsed.experiment.workloadId = workload ?? null;
|
|
402
|
+
parsed.experiment.limit = limit;
|
|
403
|
+
return parsed;
|
|
404
|
+
}
|
|
405
|
+
if (workload === undefined) return fail("missing_workload", "--workload is required: the workload (route) whose calls to replay.");
|
|
406
|
+
const specs = values["--treatment"] ?? [];
|
|
407
|
+
if (specs.length < 1 || specs.length > EXPERIMENT_MAX_TREATMENTS) {
|
|
408
|
+
return fail("invalid_treatments", `Give 1 to ${EXPERIMENT_MAX_TREATMENTS} --treatment values.`);
|
|
409
|
+
}
|
|
410
|
+
const treatments = [];
|
|
411
|
+
for (const spec of specs) {
|
|
412
|
+
const treatment = parseTreatment(spec);
|
|
413
|
+
if (treatment === null) {
|
|
414
|
+
return fail(
|
|
415
|
+
"invalid_treatment",
|
|
416
|
+
"--treatment must be PROVIDER/MODEL, optionally followed by ,label=NAME ,temperature=0..2 " +
|
|
417
|
+
`,max_tokens=1..${EXPERIMENT_MAX_TOKENS} or ,reasoning_effort=${EXPERIMENT_REASONING_EFFORTS.join("|")}.`,
|
|
418
|
+
);
|
|
419
|
+
}
|
|
420
|
+
treatments.push(treatment);
|
|
421
|
+
}
|
|
422
|
+
const labels = treatments.map((treatment) => treatment.label).filter((label) => label !== undefined);
|
|
423
|
+
if (new Set(labels.map((label) => label.toLowerCase())).size !== labels.length ||
|
|
424
|
+
labels.some((label) => label.toLowerCase() === "control")) {
|
|
425
|
+
return fail("invalid_treatments", 'Treatment labels must be unique and cannot be "control".');
|
|
426
|
+
}
|
|
427
|
+
const sample = { last_n: EXPERIMENT_LAST_N.default };
|
|
428
|
+
if (values["--last-n"] !== undefined) {
|
|
429
|
+
sample.last_n = parseBounded(values["--last-n"], EXPERIMENT_LAST_N.min, EXPERIMENT_LAST_N.max);
|
|
430
|
+
if (sample.last_n === null) {
|
|
431
|
+
return fail("invalid_last_n", `--last-n must be a whole number from ${EXPERIMENT_LAST_N.min} to ${EXPERIMENT_LAST_N.max}.`);
|
|
432
|
+
}
|
|
433
|
+
}
|
|
434
|
+
if (values["--sample-model"] !== undefined) {
|
|
435
|
+
if (!experimentText(values["--sample-model"], EXPERIMENT_TEXT_LIMITS.model)) {
|
|
436
|
+
return fail("invalid_sample_model", `--sample-model must be 1 to ${EXPERIMENT_TEXT_LIMITS.model} characters without control characters.`);
|
|
437
|
+
}
|
|
438
|
+
sample.model = values["--sample-model"];
|
|
439
|
+
}
|
|
440
|
+
const tracked = values["--track"] ?? [];
|
|
441
|
+
if (tracked.length > EXPERIMENT_MAX_TRACKED_FIELDS ||
|
|
442
|
+
tracked.some((field) => !experimentText(field, EXPERIMENT_TEXT_LIMITS.path) || !field.startsWith("$"))) {
|
|
443
|
+
return fail("invalid_tracked_fields", `Give at most ${EXPERIMENT_MAX_TRACKED_FIELDS} --track JSON paths, each starting with $.`);
|
|
444
|
+
}
|
|
445
|
+
parsed.experiment.request = { workload_id: workload, sample, treatments, tracked_fields: [...new Set(tracked)] };
|
|
446
|
+
return parsed;
|
|
447
|
+
}
|
|
448
|
+
|
|
449
|
+
function experimentText(value, max) {
|
|
450
|
+
return typeof value === "string" && value.trim() === value && value.length > 0 && value.length <= max && !hasUnsafeText(value);
|
|
451
|
+
}
|
|
452
|
+
|
|
453
|
+
const TREATMENT_KEYS = new Set(["label", "temperature", "max_tokens", "reasoning_effort"]);
|
|
454
|
+
|
|
455
|
+
// PROVIDER/MODEL[,key=value...] as a treatment, or null. The model may itself
|
|
456
|
+
// contain "/" (a gateway ID such as anthropic/claude-haiku-4.5).
|
|
457
|
+
function parseTreatment(spec) {
|
|
458
|
+
const [head, ...options] = spec.split(",");
|
|
459
|
+
const slash = head.indexOf("/");
|
|
460
|
+
if (slash <= 0) return null;
|
|
461
|
+
const provider = head.slice(0, slash);
|
|
462
|
+
const model = head.slice(slash + 1);
|
|
463
|
+
if (!experimentText(provider, EXPERIMENT_TEXT_LIMITS.provider) || !experimentText(model, EXPERIMENT_TEXT_LIMITS.model)) {
|
|
464
|
+
return null;
|
|
465
|
+
}
|
|
466
|
+
const treatment = { provider, model };
|
|
467
|
+
const settings = {};
|
|
468
|
+
for (const option of options) {
|
|
469
|
+
const equals = option.indexOf("=");
|
|
470
|
+
const key = option.slice(0, equals);
|
|
471
|
+
const value = option.slice(equals + 1);
|
|
472
|
+
if (equals <= 0 || !TREATMENT_KEYS.has(key) || Object.hasOwn(settings, key) || (key === "label" && treatment.label !== undefined)) {
|
|
473
|
+
return null;
|
|
474
|
+
}
|
|
475
|
+
if (key === "label") {
|
|
476
|
+
if (!experimentText(value, EXPERIMENT_TEXT_LIMITS.label)) return null;
|
|
477
|
+
treatment.label = value;
|
|
478
|
+
} else if (key === "temperature") {
|
|
479
|
+
if (!/^(?:[0-1](?:\.[0-9]{1,3})?|2(?:\.0{1,3})?)$/.test(value)) return null;
|
|
480
|
+
settings.temperature = Number(value);
|
|
481
|
+
} else if (key === "max_tokens") {
|
|
482
|
+
const tokens = /^[0-9]{1,5}$/.test(value) ? Number(value) : 0;
|
|
483
|
+
if (tokens < 1 || tokens > EXPERIMENT_MAX_TOKENS) return null;
|
|
484
|
+
settings.max_tokens = tokens;
|
|
485
|
+
} else {
|
|
486
|
+
if (!EXPERIMENT_REASONING_EFFORTS.includes(value)) return null;
|
|
487
|
+
settings.reasoning_effort = value;
|
|
488
|
+
}
|
|
489
|
+
}
|
|
490
|
+
if (Object.keys(settings).length > 0) treatment.settings = settings;
|
|
491
|
+
return treatment;
|
|
492
|
+
}
|
|
493
|
+
|
|
283
494
|
function parseVerify(values, flags, json, fail) {
|
|
284
495
|
const paths = parsePaths(values, fail);
|
|
285
496
|
if (!paths.ok) return paths;
|
|
@@ -515,24 +726,9 @@ function parseRead(command, values, flags, json, fail) {
|
|
|
515
726
|
}
|
|
516
727
|
// Where to read and with which key: the same for every read, grouped usage
|
|
517
728
|
// included.
|
|
518
|
-
|
|
519
|
-
if (
|
|
520
|
-
|
|
521
|
-
if (origin === null) return fail("invalid_url", INVALID_URL);
|
|
522
|
-
}
|
|
523
|
-
const rawWorkspace = values["--workspace"];
|
|
524
|
-
if (rawWorkspace !== undefined && !UUID.test(rawWorkspace)) {
|
|
525
|
-
return fail("invalid_workspace", "--workspace must be a workspace ID in UUID form.");
|
|
526
|
-
}
|
|
527
|
-
const agentTokenFile = values["--agent-token-file"] ?? null;
|
|
528
|
-
if (agentTokenFile !== null && (agentTokenFile === "" || agentTokenFile.includes("\0"))) {
|
|
529
|
-
return fail("invalid_agent_token_file", "--agent-token-file must name a private absolute file.");
|
|
530
|
-
}
|
|
531
|
-
const target = {
|
|
532
|
-
origin,
|
|
533
|
-
workspace: rawWorkspace === undefined ? null : rawWorkspace.toLowerCase(),
|
|
534
|
-
agentTokenFile,
|
|
535
|
-
};
|
|
729
|
+
const target = parseKeyTarget(values, fail);
|
|
730
|
+
if (!target.ok) return target;
|
|
731
|
+
delete target.ok;
|
|
536
732
|
if (command === "usage" && (USAGE_GROUPED.some((name) => values[name] !== undefined) || flags.has("--today"))) {
|
|
537
733
|
return parseGroupedUsage(values, flags, json, fail, { paths, timeoutMs, limit, target });
|
|
538
734
|
}
|
package/src/auth-oauth.js
CHANGED
|
@@ -1,6 +1,12 @@
|
|
|
1
1
|
import { createHash, randomBytes } from "node:crypto";
|
|
2
2
|
|
|
3
|
-
import {
|
|
3
|
+
import {
|
|
4
|
+
AGENT_CONTRACT_VERSION,
|
|
5
|
+
AUTH_PATHS,
|
|
6
|
+
EXPERIMENT_START_KEY_SCOPES,
|
|
7
|
+
EXPERIMENT_START_SCOPE,
|
|
8
|
+
METADATA_SCOPE,
|
|
9
|
+
} from "./constants.js";
|
|
4
10
|
import { parseJsonObject } from "./http.js";
|
|
5
11
|
import { failureOf, send } from "./transport.js";
|
|
6
12
|
|
|
@@ -396,14 +402,19 @@ function decodeClaims(token) {
|
|
|
396
402
|
// with nothing to pin it to passes anyWorkspace: true with workspaceId null,
|
|
397
403
|
// and then the workspace the service reports is checked for consistency and
|
|
398
404
|
// returned as the verified one.
|
|
399
|
-
|
|
405
|
+
// access is "metadata" (every read command and sign in) or
|
|
406
|
+
// "experiment_start" (experiments start only: a coding-agent key with Debug
|
|
407
|
+
// access and the start grant, checked for exactly that and used for nothing else).
|
|
408
|
+
export async function verifyContext(origin, accessToken, { profile, workspaceId, anyWorkspace = false, access = "metadata" }, signal) {
|
|
400
409
|
const workspace = await bearerJson(origin, AUTH_PATHS.workspace, accessToken, WORKSPACE_BYTES, signal, "workspace");
|
|
401
410
|
if (!workspace.ok) return workspace;
|
|
402
411
|
if (workspaceId === null && anyWorkspace === true) {
|
|
403
412
|
workspaceId = normalizeUuid(workspace.body.workspace?.id);
|
|
404
413
|
if (workspaceId === null) return stop("verification_failed", "workspace_response_invalid");
|
|
405
414
|
}
|
|
406
|
-
const checked =
|
|
415
|
+
const checked = access === "experiment_start"
|
|
416
|
+
? checkStartWorkspace(workspace.body, { profile, workspaceId })
|
|
417
|
+
: checkWorkspace(workspace.body, { profile, workspaceId });
|
|
407
418
|
if (!checked.ok) return checked;
|
|
408
419
|
|
|
409
420
|
const capabilities = await bearerJson(
|
|
@@ -415,11 +426,18 @@ export async function verifyContext(origin, accessToken, { profile, workspaceId,
|
|
|
415
426
|
"capabilities",
|
|
416
427
|
);
|
|
417
428
|
if (!capabilities.ok) return capabilities;
|
|
418
|
-
const allowed =
|
|
429
|
+
const allowed = access === "experiment_start"
|
|
430
|
+
? checkStartCapabilities(capabilities.body, { profile, workspaceId })
|
|
431
|
+
: checkCapabilities(capabilities.body, { profile, workspaceId });
|
|
419
432
|
if (!allowed.ok) return allowed;
|
|
420
433
|
// The verified documents stay in memory for read commands, which print
|
|
421
434
|
// only fields they validate themselves.
|
|
422
|
-
return {
|
|
435
|
+
return {
|
|
436
|
+
ok: true,
|
|
437
|
+
workspaceId,
|
|
438
|
+
scopes: checked.scopes ?? [METADATA_SCOPE],
|
|
439
|
+
documents: { workspace: workspace.body, capabilities: capabilities.body },
|
|
440
|
+
};
|
|
423
441
|
}
|
|
424
442
|
|
|
425
443
|
// Both documents carry the service's contract version string in
|
|
@@ -444,6 +462,41 @@ export function checkWorkspace(body, { profile, workspaceId }) {
|
|
|
444
462
|
return { ok: true };
|
|
445
463
|
}
|
|
446
464
|
|
|
465
|
+
// A start key's workspace document: the same identity checks as
|
|
466
|
+
// checkWorkspace, and exactly the scopes Debug and the start grant.
|
|
467
|
+
export function checkStartWorkspace(body, { profile, workspaceId }) {
|
|
468
|
+
const identity = checkWorkspace({ ...body, access: { scopes: [METADATA_SCOPE] } }, { profile, workspaceId });
|
|
469
|
+
if (!identity.ok) return identity;
|
|
470
|
+
const scopes = body.access?.scopes;
|
|
471
|
+
if (
|
|
472
|
+
!Array.isArray(scopes) ||
|
|
473
|
+
scopes.length !== EXPERIMENT_START_KEY_SCOPES.length ||
|
|
474
|
+
!EXPERIMENT_START_KEY_SCOPES.every((scope) => scopes.includes(scope))
|
|
475
|
+
) {
|
|
476
|
+
return stop("permission_denied", "experiment_start_not_granted");
|
|
477
|
+
}
|
|
478
|
+
return { ok: true, scopes: [...scopes].sort() };
|
|
479
|
+
}
|
|
480
|
+
|
|
481
|
+
// A start key's capabilities document: the same identity checks as
|
|
482
|
+
// checkCapabilities, and the start capability offered to this key.
|
|
483
|
+
export function checkStartCapabilities(body, { profile, workspaceId }) {
|
|
484
|
+
if (body.schema_version !== AGENT_CONTRACT_VERSION || !isObject(body.provenance) || !isObject(body.agent)) {
|
|
485
|
+
return stop("verification_failed", "capabilities_response_invalid");
|
|
486
|
+
}
|
|
487
|
+
if (normalizeUuid(body.provenance.workspace_id) !== workspaceId) {
|
|
488
|
+
return stop("verification_failed", "workspace_context_mismatch");
|
|
489
|
+
}
|
|
490
|
+
if (body.provenance.deployment_profile !== profile || body.deployment_profile !== profile) {
|
|
491
|
+
return stop("verification_failed", "profile_mismatch");
|
|
492
|
+
}
|
|
493
|
+
const start = body.agent.experiment_start;
|
|
494
|
+
if (!isObject(start) || start.available !== true || start.required_scope !== EXPERIMENT_START_SCOPE || start.mutates !== true) {
|
|
495
|
+
return stop("capability_unavailable", "experiment_start_unavailable");
|
|
496
|
+
}
|
|
497
|
+
return { ok: true };
|
|
498
|
+
}
|
|
499
|
+
|
|
447
500
|
export function checkCapabilities(body, { profile, workspaceId }) {
|
|
448
501
|
if (body.schema_version !== AGENT_CONTRACT_VERSION || !isObject(body.provenance) || !isObject(body.agent)) {
|
|
449
502
|
return stop("verification_failed", "capabilities_response_invalid");
|