@withgauge/cli 0.18.0 → 0.20.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +52 -58
- package/dist/index.js +199 -83
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -159,7 +159,7 @@ For automatic checks, connect the Gauge GitHub App with repository access and
|
|
|
159
159
|
Actions read permission. Set `org` and enable the repository and target branches
|
|
160
160
|
in **Settings → GitHub**. Waiting for inputs starts no sessions and spends no
|
|
161
161
|
session credits.
|
|
162
|
-
Fork pull requests are disabled by default.
|
|
162
|
+
Fork pull requests are disabled by default. Check **Allow fork pull requests** to let
|
|
163
163
|
workspace members authorize an evaluation from its GitHub check. Authorization
|
|
164
164
|
covers one PR revision and uses workspace credits; new revisions and explicit
|
|
165
165
|
reruns require fresh approval. Configured inputs determine when an authorized
|
|
@@ -391,75 +391,69 @@ funding by default, like Run Now in the web UI; `--bill-to-org` uses that org's
|
|
|
391
391
|
credits instead. Other users always use org credits and can launch only in orgs
|
|
392
392
|
they belong to.
|
|
393
393
|
|
|
394
|
-
## Optimizations
|
|
394
|
+
## Optimizations
|
|
395
395
|
|
|
396
|
-
|
|
397
|
-
against an eval or a preference prompt. Gauge measures a baseline, runs the
|
|
398
|
-
trials you author, judges them, and tells you which change won. It never writes
|
|
399
|
-
the change itself, and it never touches your repository. You edit on disk and
|
|
400
|
-
send the text; Gauge serves it to the sandboxed agent in place of the live page
|
|
401
|
-
or skill version.
|
|
402
|
-
|
|
403
|
-
A scripted or agent-driven loop:
|
|
396
|
+
Start by finding runs that encountered the surface you want to improve:
|
|
404
397
|
|
|
405
398
|
```sh
|
|
406
|
-
gauge
|
|
407
|
-
gauge optimizations
|
|
408
|
-
gauge
|
|
409
|
-
|
|
410
|
-
gauge optimizations
|
|
411
|
-
$EDITOR start.md
|
|
412
|
-
gauge optimizations trials add cl_1 \
|
|
413
|
-
--title "Lead with the env var" --page https://docs.acme.dev/start \
|
|
414
|
-
--body-file start.md --launch --idempotency-key start-env-var --yes
|
|
415
|
-
|
|
399
|
+
gauge instructions optimization
|
|
400
|
+
gauge optimizations baselines --eval ev_1 -o json
|
|
401
|
+
gauge runs insight run_1 -o json
|
|
402
|
+
gauge runs logs run_1 -o json
|
|
403
|
+
gauge optimizations start --eval ev_1 --from-run run_1 --yes
|
|
416
404
|
gauge optimizations watch cl_1
|
|
417
|
-
gauge optimizations trial cl_1 R1 # score, verdict, sessions, errors
|
|
418
|
-
gauge optimizations change cl_1 R1 --body # the winning text, to paste
|
|
419
|
-
gauge optimizations adopt cl_1 R1 --yes # mark the winner; baseline stays fixed
|
|
420
|
-
gauge optimizations handoff cl_1 # instructions to land the winner in your source
|
|
421
|
-
gauge optimizations complete cl_1 --yes
|
|
422
405
|
```
|
|
423
406
|
|
|
424
|
-
`
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
407
|
+
`baselines` returns observed owned surfaces grouped by run, source evidence,
|
|
408
|
+
judging status, eligibility (including the baseline's complete run set), and a
|
|
409
|
+
`nextCursor` for `--cursor`. It uses the UI baseline picker's compatibility rules
|
|
410
|
+
and recent 100-batch eligibility window. Missing evidence is unknown; the list
|
|
411
|
+
is an analysis summary, not an exhaustive inventory of every trace event.
|
|
412
|
+
`--from-run` inherits the eligible baseline's harness/model and reuses its runs.
|
|
413
|
+
A stale choice fails instead of silently buying a new baseline. `--target` is
|
|
414
|
+
an alternative when you need to establish a baseline; do not combine it with
|
|
415
|
+
`--from-run`. Use `--preference` instead of `--eval` for preference prompts.
|
|
431
416
|
|
|
432
417
|
```sh
|
|
433
|
-
gauge optimizations authoring cl_1 > context.json
|
|
434
|
-
gauge optimizations
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
# "lever": "CLI", "intervention": {"kind": "CLI_INTERVENTION", "after": <bundle.json>}}}]}
|
|
438
|
-
gauge optimizations round preview cl_1 --file round.json # prints previewDigest
|
|
418
|
+
gauge optimizations authoring cl_1 > context.json
|
|
419
|
+
gauge optimizations page cl_1 --url https://docs.acme.dev/auth -o json
|
|
420
|
+
# Author round.json using context.revision and the observed baseline evidence.
|
|
421
|
+
gauge optimizations round preview cl_1 --file round.json -o json
|
|
439
422
|
gauge optimizations round save cl_1 --file round.json --digest <previewDigest>
|
|
440
|
-
gauge optimizations trials remove cl_1 R1.2 --yes # drop one draft
|
|
441
423
|
gauge optimizations launch cl_1 --yes
|
|
424
|
+
gauge optimizations watch cl_1
|
|
425
|
+
gauge optimizations trial cl_1 R1.1 -o json
|
|
426
|
+
gauge optimizations adopt cl_1 R1.1 --yes
|
|
427
|
+
gauge optimizations handoff cl_1
|
|
428
|
+
gauge optimizations complete cl_1 --yes
|
|
442
429
|
```
|
|
443
430
|
|
|
444
|
-
A
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
the
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
`
|
|
462
|
-
|
|
431
|
+
A round is `{revision, trials: [{trial: {title, hypothesis, lever, intervention}}]}`.
|
|
432
|
+
Docs `REWRITE_DOCS` proposals supply `after.pages` with URL and exact `edits`
|
|
433
|
+
(`find`/`replace`), or a complete `body` and `rewriteReason`. Check the page's
|
|
434
|
+
`source`: `scrape` means current fetched content, not the captured response.
|
|
435
|
+
|
|
436
|
+
For CLI trials, inspect `cliIntervention.guidance` and `targets` in authoring
|
|
437
|
+
and use `runs logs` for the cited command evidence. `cli-change --file change.json`
|
|
438
|
+
accepts v3 `{schemaVersion:3, target, rules, shippingIntent}` and returns a sealed
|
|
439
|
+
bundle to put in `CLI_INTERVENTION.after`. It does not save or launch. The old
|
|
440
|
+
catalog/invocation-ID workflow is retired. Skills use `skill-files` and the
|
|
441
|
+
complete edited file bundle.
|
|
442
|
+
|
|
443
|
+
A checkpoint is a saved state of a baseline run. Resuming it avoids repeating
|
|
444
|
+
earlier work. Gauge automatically selects a compatible checkpoint before the
|
|
445
|
+
affected interaction for docs and CLI candidates, or starts fresh with a reason.
|
|
446
|
+
You do not select a checkpoint; omit checkpoint choices and fork flags. Skills
|
|
447
|
+
start fresh. Inspect
|
|
448
|
+
`runs[].candidateExecution` for actual starting mode and `candidateDelivery`
|
|
449
|
+
for exposure, observed artifacts, and evidence completeness. A score improvement
|
|
450
|
+
does not prove the change was exercised. Report sample counts and failed sessions.
|
|
451
|
+
|
|
452
|
+
Read `nextAction` on optimization detail; `watch` exits 4 when input is needed.
|
|
453
|
+
Round save replaces pending drafts but never launches. Use `--yes` for approved
|
|
454
|
+
spending and `trials add --idempotency-key` when retrying single-trial writes.
|
|
455
|
+
Adoption selects a winner without changing the baseline or deploying the change;
|
|
456
|
+
`handoff` supplies instructions to implement it in your source repository.
|
|
463
457
|
|
|
464
458
|
## Models, reports, and scripting
|
|
465
459
|
|
package/dist/index.js
CHANGED
|
@@ -577,6 +577,10 @@ var billingStatusSchema = z6.object({
|
|
|
577
577
|
holdsPerRun: workloadCreditStateSchema(z6.number()),
|
|
578
578
|
/** Can the org fund at least one more run of each workload right now. */
|
|
579
579
|
canRunByWorkload: workloadCreditStateSchema(z6.boolean()),
|
|
580
|
+
/** How the subscription is collected; null = no paid subscription. */
|
|
581
|
+
paymentType: z6.enum(["invoice", "card"]).nullable().optional(),
|
|
582
|
+
/** Invoiced subscriptions only: how far below zero the org may spend. */
|
|
583
|
+
creditLine: z6.object({ limitUsd: z6.number(), usedUsd: z6.number() }).nullable().optional(),
|
|
580
584
|
// --- Deprecated. Retained for v1 compatibility; removal needs an API-version
|
|
581
585
|
// decision. Overage is retired, so these are always inert.
|
|
582
586
|
/** @deprecated Use `currentSku.tierLabel`. */
|
|
@@ -597,7 +601,9 @@ var LEDGER_KINDS = [
|
|
|
597
601
|
"RUN_REFUND",
|
|
598
602
|
"OVERAGE_DEBIT",
|
|
599
603
|
"ADMIN_ADJUST",
|
|
600
|
-
"EXPIRY"
|
|
604
|
+
"EXPIRY",
|
|
605
|
+
"CREDIT_LINE_SETTLE",
|
|
606
|
+
"CREDIT_LINE_CARRY"
|
|
601
607
|
];
|
|
602
608
|
var ledgerKindSchema = z6.enum(LEDGER_KINDS);
|
|
603
609
|
var usageEntrySchema = z6.object({
|
|
@@ -3439,6 +3445,48 @@ var optimizationSummarySchema = z35.object({
|
|
|
3439
3445
|
var listOptimizationsResponseSchema = z35.object({
|
|
3440
3446
|
items: z35.array(optimizationSummarySchema)
|
|
3441
3447
|
});
|
|
3448
|
+
var baselineDiscoveryQuerySchema = z35.object({
|
|
3449
|
+
subjectKind: climbSubjectKindSchema,
|
|
3450
|
+
subjectId: z35.string().min(1),
|
|
3451
|
+
limit: z35.coerce.number().int().min(1).max(100).default(50),
|
|
3452
|
+
cursor: z35.string().min(1).optional()
|
|
3453
|
+
}).strict();
|
|
3454
|
+
var baselineRunSchema = z35.object({
|
|
3455
|
+
resolvedModel: z35.string().nullable(),
|
|
3456
|
+
runId: z35.string(),
|
|
3457
|
+
agent: z35.string(),
|
|
3458
|
+
model: z35.string(),
|
|
3459
|
+
status: z35.string(),
|
|
3460
|
+
createdAt: z35.string(),
|
|
3461
|
+
outcome: z35.object({
|
|
3462
|
+
passedCriteria: z35.number(),
|
|
3463
|
+
totalCriteria: z35.number(),
|
|
3464
|
+
judgingStatus: z35.string().nullable()
|
|
3465
|
+
}),
|
|
3466
|
+
surfaces: z35.array(
|
|
3467
|
+
z35.object({
|
|
3468
|
+
lever: z35.string(),
|
|
3469
|
+
label: z35.string(),
|
|
3470
|
+
targetRef: z35.unknown(),
|
|
3471
|
+
evidence: z35.unknown(),
|
|
3472
|
+
command: z35.unknown().optional()
|
|
3473
|
+
})
|
|
3474
|
+
),
|
|
3475
|
+
evidenceStatus: z35.object({
|
|
3476
|
+
surfaces: z35.string().nullable(),
|
|
3477
|
+
cli: z35.string().nullable()
|
|
3478
|
+
}),
|
|
3479
|
+
eligibility: z35.object({
|
|
3480
|
+
eligible: z35.boolean(),
|
|
3481
|
+
reason: z35.string().nullable(),
|
|
3482
|
+
baselineRunIds: z35.array(z35.string())
|
|
3483
|
+
})
|
|
3484
|
+
});
|
|
3485
|
+
var baselineDiscoveryResponseSchema = z35.object({
|
|
3486
|
+
items: z35.array(baselineRunSchema),
|
|
3487
|
+
nextCursor: z35.string().nullable(),
|
|
3488
|
+
note: z35.string()
|
|
3489
|
+
});
|
|
3442
3490
|
var targetSchema = z35.object({
|
|
3443
3491
|
agent: schedulableAgentSchema.optional(),
|
|
3444
3492
|
/** Empty or omitted means the agent's default model. */
|
|
@@ -3471,8 +3519,16 @@ var startOptimizationBodySchema = z35.object({
|
|
|
3471
3519
|
...planShape,
|
|
3472
3520
|
creditCap: z35.number().int().nonnegative().optional(),
|
|
3473
3521
|
targetScore: z35.number().min(0).max(1).optional(),
|
|
3474
|
-
target: targetSchema.extend({ agent: schedulableAgentSchema })
|
|
3475
|
-
|
|
3522
|
+
target: targetSchema.extend({ agent: schedulableAgentSchema }).optional(),
|
|
3523
|
+
fromRun: z35.string().min(1).optional()
|
|
3524
|
+
}).strict().superRefine(refinePlan).superRefine((value, ctx) => {
|
|
3525
|
+
if (!!value.target === !!value.fromRun)
|
|
3526
|
+
ctx.addIssue({
|
|
3527
|
+
code: "custom",
|
|
3528
|
+
message: "Pass target or fromRun, not both",
|
|
3529
|
+
path: ["target"]
|
|
3530
|
+
});
|
|
3531
|
+
});
|
|
3476
3532
|
var estimateOptimizationBodySchema = z35.object({
|
|
3477
3533
|
subjectKind: climbSubjectKindSchema,
|
|
3478
3534
|
subjectId: z35.string().min(1),
|
|
@@ -3671,6 +3727,7 @@ var skillFilesResponseSchema = z35.object({
|
|
|
3671
3727
|
files: overrideFilesSchema
|
|
3672
3728
|
});
|
|
3673
3729
|
var pageBodyResponseSchema = z35.object({
|
|
3730
|
+
source: z35.string().optional(),
|
|
3674
3731
|
url: z35.string(),
|
|
3675
3732
|
body: z35.string(),
|
|
3676
3733
|
contentType: z35.string().optional()
|
|
@@ -6008,15 +6065,23 @@ function register4(program2) {
|
|
|
6008
6065
|
// src/commands/billing.ts
|
|
6009
6066
|
var billing_exports2 = {};
|
|
6010
6067
|
__export(billing_exports2, {
|
|
6068
|
+
fmtOverage: () => fmtOverage,
|
|
6011
6069
|
register: () => register5
|
|
6012
6070
|
});
|
|
6013
6071
|
function fmtDate(iso) {
|
|
6014
6072
|
return iso ? iso.slice(0, 10) : "\u2014";
|
|
6015
6073
|
}
|
|
6074
|
+
function fmtOverage(s) {
|
|
6075
|
+
const line = s.creditLine;
|
|
6076
|
+
if (!line) return s.paymentType === "card" ? "n/a (card)" : "\u2014";
|
|
6077
|
+
if (line.limitUsd <= 0) return "off (credit line limit $0)";
|
|
6078
|
+
return `$${line.usedUsd.toFixed(2)} of $${line.limitUsd.toFixed(2)} credit line`;
|
|
6079
|
+
}
|
|
6016
6080
|
function statusLines(s) {
|
|
6017
6081
|
return [
|
|
6018
6082
|
["Plan", s.plan ?? "free"],
|
|
6019
6083
|
["Subscription", s.subscriptionStatus ?? "\u2014"],
|
|
6084
|
+
["Payment", s.paymentType ?? "\u2014"],
|
|
6020
6085
|
["Balance", `${s.balance} credits`],
|
|
6021
6086
|
[
|
|
6022
6087
|
"Allowance / cycle",
|
|
@@ -6028,10 +6093,7 @@ function statusLines(s) {
|
|
|
6028
6093
|
],
|
|
6029
6094
|
["Used this period", `${s.creditsUsedThisPeriod} credits`],
|
|
6030
6095
|
["Runs this period", String(s.runsThisPeriod)],
|
|
6031
|
-
[
|
|
6032
|
-
"Overage",
|
|
6033
|
-
s.overageEnabled ? `enabled (${s.overageThisPeriod} used${s.overageCapCredits === null ? ", uncapped" : `, cap ${s.overageCapCredits}`})` : "disabled"
|
|
6034
|
-
],
|
|
6096
|
+
["Overage", fmtOverage(s)],
|
|
6035
6097
|
["Can run eval", s.canRunByWorkload.eval ? "yes" : "no"],
|
|
6036
6098
|
["Can run visibility", s.canRunByWorkload.visibility ? "yes" : "no"],
|
|
6037
6099
|
["Can run generic", s.canRunByWorkload.generic ? "yes" : "no"],
|
|
@@ -6040,7 +6102,7 @@ function statusLines(s) {
|
|
|
6040
6102
|
}
|
|
6041
6103
|
function register5(program2) {
|
|
6042
6104
|
const group = program2.command("billing").description("Billing & credits");
|
|
6043
|
-
group.command("status").description("Show credit balance, plan, and overage").action(async (_opts, cmd) => {
|
|
6105
|
+
group.command("status").description("Show credit balance, plan, payment, and overage").action(async (_opts, cmd) => {
|
|
6044
6106
|
const org = resolveOrg(cmd);
|
|
6045
6107
|
const status = await createClient().get(
|
|
6046
6108
|
`/api/v1/orgs/${encodeURIComponent(org)}/billing`
|
|
@@ -8060,7 +8122,7 @@ description: Create and review Gauge eval prompts, model selections, and judging
|
|
|
8060
8122
|
- Native command inputs use \`type: "binary"\`, \`platform: "linux-x64"\`, selected \`paths\`, and a \`commands\` map from command names to explicit relative ELF executable files. CI must upload already unpacked files. Gauge restores executable permissions and exposes declared commands on PATH; compatible system libraries are required, nested archives and arbitrary installers are unsupported.
|
|
8061
8123
|
- Automatic GitHub PR checks support root \`gauge.json\` \`checks.paths\` as a suite gate and per-case frontmatter \`config.checks.paths\` as a narrower filter. Both accept positive repository-relative globs. Cases without filters run when the suite gate passes; changes to \`gauge.json\` select all cases and edits to a case select that case. Explicit CLI launches and GitHub reruns ignore both filters. Failed/unfinished previous checks, changed target branches, and unavailable diffs run conservatively. \`config.inputs\` selects inputs, not triggers.
|
|
8062
8124
|
- Pilot before scaling; confirm before spending credits. Creating definitions does not authorize launching runs. Report sample counts and distinguish incorrect behavior, missing evidence, environment failures, and budget exhaustion.
|
|
8063
|
-
- Gauge tests artifacts from the PR's successful build; preserve CI's existing checkout settings, including the default merge checkout. Fork PRs are disabled by default. In Gauge's GitHub settings,
|
|
8125
|
+
- Gauge tests artifacts from the PR's successful build; preserve CI's existing checkout settings, including the default merge checkout. Fork PRs are disabled by default. In Gauge's GitHub settings, check **Allow fork pull requests**; a workspace member then follows the GitHub check link to authorize that revision using workspace credits and connections. New revisions and explicit reruns need fresh approval. Inputs still determine readiness.
|
|
8064
8126
|
- For authenticated MCP access, connect the account in Gauge, complete OAuth in the browser when required, and put the versioned server reference in case \`config.mcpRefs\`. Gauge manages session credentials; keep tokens out of prompts and Git.
|
|
8065
8127
|
- To test local build output without the GitHub App, use \`gauge evals run --input name=path --yes -o json\`. Repeat \`--input\` for each input. This uses current evaluation files, including edits; other inputs keep their configured sources. Skills accept directories or archives, npm packages accept .tgz files, and file/binary inputs accept directories. Use \`gauge run-requests wait <id> -o json\` for the verdict, criterion results, and session links; failed checks exit nonzero while still returning JSON.
|
|
8066
8128
|
- Before committing setup changes, run \`gauge evals verify -o json\` for an offline working-tree checklist. Only attempted static checks are scored yes, no, or unknown; unknown means a specific declaration could not be resolved. Informational findings (such as zero cases) are separate. JSON includes a version, static status (passed/failed/incomplete), summary counts, global/per-input checks with stable IDs, facts, info, and excluded capabilities in notChecked. Exit 1 means a definite problem; exit 0 can still contain unknowns. Live delivery, account settings, and sandbox execution are outside this command's scope. It requires no login or PR and launches no sessions.
|
|
@@ -8168,7 +8230,7 @@ Gauge helps you understand and improve how coding agents discover and use your
|
|
|
8168
8230
|
product. It runs real agents in sandboxes under reproducible conditions, then
|
|
8169
8231
|
judges their work. Evals measure successful product use; Agent Preference
|
|
8170
8232
|
measures which tools agents choose and why. Optimizations test improvements to
|
|
8171
|
-
your docs and skills against those measurements.
|
|
8233
|
+
your docs, CLI, and skills against those measurements.
|
|
8172
8234
|
|
|
8173
8235
|
## Use Gauge and the CLI
|
|
8174
8236
|
|
|
@@ -8224,64 +8286,64 @@ Full guidance: \`gauge instructions preference\`.
|
|
|
8224
8286
|
|
|
8225
8287
|
### Optimization
|
|
8226
8288
|
|
|
8227
|
-
Gauge measures trials; you author
|
|
8228
|
-
|
|
8229
|
-
|
|
8230
|
-
|
|
8231
|
-
|
|
8232
|
-
|
|
8233
|
-
|
|
8234
|
-
|
|
8235
|
-
|
|
8289
|
+
Gauge measures trials; you author changes. First run \`gauge optimizations baselines\`
|
|
8290
|
+
for an eval or preference prompt to discover observed surfaces and eligible runs.
|
|
8291
|
+
Inspect source evidence, then start with \`--from-run\` to reuse the chosen baseline.
|
|
8292
|
+
Read authoring context, write a hypothesis, preview the complete change, save,
|
|
8293
|
+
and launch. Gauge resumes saved baseline states (checkpoints) when compatible, or starts fresh.
|
|
8294
|
+
Follow \`nextAction\`; watch exits 4 when input is needed. Verify actual change
|
|
8295
|
+
exposure, task scores, sample counts, and failed sessions separately. Keep the
|
|
8296
|
+
measurement fixed, confirm paid launches, and retry writes safely. Adopt the
|
|
8297
|
+
winner; use \`handoff\` for implementation in your source repository.
|
|
8236
8298
|
|
|
8237
8299
|
Full guidance: \`gauge instructions optimization\`.
|
|
8238
8300
|
`;
|
|
8239
8301
|
var OPTIMIZATIONS = `---
|
|
8240
8302
|
name: gauge-optimizations
|
|
8241
|
-
description:
|
|
8242
|
-
a Gauge eval or preference prompt.
|
|
8303
|
+
description: Discover observed surfaces, choose a baseline, and test docs, CLI,
|
|
8304
|
+
or skill changes against a Gauge eval or preference prompt.
|
|
8243
8305
|
---
|
|
8244
8306
|
|
|
8245
8307
|
# Optimizations
|
|
8246
8308
|
|
|
8247
|
-
Gauge measures
|
|
8248
|
-
against one change you wrote, judges the sessions, and reports which won.
|
|
8249
|
-
|
|
8250
|
-
Nothing touches your repository. A docs trial serves your text to the sandboxed
|
|
8251
|
-
agent in place of the live page. A skill trial swaps the version.
|
|
8309
|
+
Gauge measures; you author. Use -o json on reads. Discover before starting.
|
|
8252
8310
|
|
|
8253
8311
|
## The loop
|
|
8254
8312
|
|
|
8255
|
-
gauge optimizations
|
|
8256
|
-
gauge
|
|
8257
|
-
gauge
|
|
8258
|
-
|
|
8259
|
-
|
|
8260
|
-
|
|
8261
|
-
|
|
8262
|
-
|
|
8263
|
-
|
|
8264
|
-
|
|
8265
|
-
|
|
8266
|
-
|
|
8267
|
-
|
|
8268
|
-
|
|
8269
|
-
|
|
8270
|
-
|
|
8271
|
-
|
|
8272
|
-
|
|
8273
|
-
|
|
8274
|
-
-
|
|
8275
|
-
|
|
8276
|
-
|
|
8277
|
-
|
|
8278
|
-
|
|
8279
|
-
|
|
8280
|
-
|
|
8281
|
-
|
|
8282
|
-
|
|
8283
|
-
|
|
8284
|
-
|
|
8313
|
+
gauge optimizations baselines --eval <id> -o json
|
|
8314
|
+
gauge runs insight <run-id> -o json
|
|
8315
|
+
gauge runs logs <run-id> -o json
|
|
8316
|
+
gauge optimizations start --eval <id> --from-run <run-id> --yes
|
|
8317
|
+
|
|
8318
|
+
Discovery returns observed owned surfaces, evidence, judging, eligibility, and nextCursor.
|
|
8319
|
+
Choose a run that encountered the surface you want to improve. Missing evidence is unknown.
|
|
8320
|
+
--from-run reuses its eligible baseline (possibly several runs) and inherits its model.
|
|
8321
|
+
Use --preference for preference prompts, or --target <agent:model> to establish a baseline.
|
|
8322
|
+
A new baseline may spend credits. Only use --yes for approved spending.
|
|
8323
|
+
|
|
8324
|
+
gauge optimizations watch <id> # exit 4 = your turn, not failure
|
|
8325
|
+
gauge optimizations authoring <id> # targets, guidance, revision
|
|
8326
|
+
gauge optimizations page <id> --url <url> -o json
|
|
8327
|
+
|
|
8328
|
+
Follow nextAction: WAIT \u2192 watch; AUTHOR_TRIAL \u2192 author; LAUNCH_ROUND \u2192 launch.
|
|
8329
|
+
Check page.source: scrape is current content, not the captured baseline response.
|
|
8330
|
+
Write one hypothesis and the complete change in round.json, using the authoring revision.
|
|
8331
|
+
Docs use exact find/replace edits or body with rewriteReason; skills use the complete bundle.
|
|
8332
|
+
CLI: cli-change --file takes v3 {schemaVersion:3,target,rules,shippingIntent}; put its output in intervention.after.
|
|
8333
|
+
|
|
8334
|
+
gauge optimizations round preview <id> --file round.json -o json
|
|
8335
|
+
gauge optimizations round save <id> --file round.json --digest <previewDigest>
|
|
8336
|
+
gauge optimizations launch <id> --yes
|
|
8337
|
+
gauge optimizations watch <id>
|
|
8338
|
+
gauge optimizations trial <id> <trial> -o json
|
|
8339
|
+
|
|
8340
|
+
Checkpoints are saved states of baseline runs: resuming one avoids repeating earlier work.
|
|
8341
|
+
Gauge selects checkpoints automatically before the affected interaction, or starts fresh with a reason.
|
|
8342
|
+
Omit startingPoints and fork flags; you do not select a checkpoint. Check candidateExecution,
|
|
8343
|
+
candidateDelivery (exposure, observedArtifacts, evidenceComplete), scores, counts, and failed sessions.
|
|
8344
|
+
Better scores alone are not proof. Keep eval/prompt/criteria fixed; repeat if inconclusive.
|
|
8345
|
+
Adopt <id> <trial> selects the winner; handoff <id> gives source instructions; complete <id> finishes.
|
|
8346
|
+
Neither deploys. Single-trial retries: trials add --idempotency-key. Optional: gauge optimizations diagnose.
|
|
8285
8347
|
`;
|
|
8286
8348
|
var TOPICS = {
|
|
8287
8349
|
root: { name: "gauge", markdown: ROOT },
|
|
@@ -9256,6 +9318,46 @@ function register19(program2) {
|
|
|
9256
9318
|
else printItems(res.items.map(toRow3), "table");
|
|
9257
9319
|
}
|
|
9258
9320
|
);
|
|
9321
|
+
addSubjectOptions(
|
|
9322
|
+
group.command("baselines").description(
|
|
9323
|
+
"Discover which ordinary runs exposed which owned surfaces, and whether they can seed an optimization"
|
|
9324
|
+
)
|
|
9325
|
+
).option("--limit <n>", "page size (1\u2013100)", "50").option("--cursor <cursor>", "nextCursor from the previous page").action(
|
|
9326
|
+
async (opts, cmd) => {
|
|
9327
|
+
const response = await createClient().get(
|
|
9328
|
+
`${basePath(resolveOrg(cmd))}/baselines`,
|
|
9329
|
+
{
|
|
9330
|
+
...parseSubject(opts),
|
|
9331
|
+
limit: parseCount(opts.limit, "--limit"),
|
|
9332
|
+
cursor: opts.cursor
|
|
9333
|
+
}
|
|
9334
|
+
);
|
|
9335
|
+
if (resolveOutput(cmd) === "json") printJson(response);
|
|
9336
|
+
else {
|
|
9337
|
+
printItems(
|
|
9338
|
+
response.items.flatMap(
|
|
9339
|
+
(run) => (run.surfaces.length ? run.surfaces : [{ lever: "\u2014", label: "No recorded surface evidence" }]).map((surface) => ({
|
|
9340
|
+
run: run.runId,
|
|
9341
|
+
model: run.model || run.resolvedModel || run.agent,
|
|
9342
|
+
date: run.createdAt.slice(0, 10),
|
|
9343
|
+
criteria: run.outcome.totalCriteria ? `${run.outcome.passedCriteria}/${run.outcome.totalCriteria}` : "\u2014",
|
|
9344
|
+
status: run.status,
|
|
9345
|
+
surface: surface.lever,
|
|
9346
|
+
target: surface.label,
|
|
9347
|
+
eligible: run.eligibility.eligible ? "yes" : "no"
|
|
9348
|
+
}))
|
|
9349
|
+
),
|
|
9350
|
+
"table"
|
|
9351
|
+
);
|
|
9352
|
+
for (const run of response.items)
|
|
9353
|
+
if (run.eligibility.reason)
|
|
9354
|
+
console.log(`${run.runId}: ${run.eligibility.reason}`);
|
|
9355
|
+
console.log(response.note);
|
|
9356
|
+
if (response.nextCursor)
|
|
9357
|
+
console.log(`More: --cursor ${response.nextCursor}`);
|
|
9358
|
+
}
|
|
9359
|
+
}
|
|
9360
|
+
);
|
|
9259
9361
|
group.command("get <id>").description("One optimization: settings, score, and its trial ladder").action(async (id2, _opts, cmd) => {
|
|
9260
9362
|
const org = resolveOrg(cmd);
|
|
9261
9363
|
const detail = await createClient().get(itemPath(org, id2));
|
|
@@ -9278,25 +9380,29 @@ function register19(program2) {
|
|
|
9278
9380
|
addTargetOption(
|
|
9279
9381
|
addSubjectOptions(
|
|
9280
9382
|
group.command("start").description(
|
|
9281
|
-
"
|
|
9383
|
+
"Reuse a selected baseline with --from-run, or establish one with --target"
|
|
9282
9384
|
)
|
|
9283
9385
|
)
|
|
9386
|
+
).option(
|
|
9387
|
+
"--from-run <runId>",
|
|
9388
|
+
"reuse the eligible baseline containing this run; inherits its model"
|
|
9284
9389
|
).option("--name <name>", "optimization name (default: from the subject)").option("--yes", "skip the confirmation").action(
|
|
9285
9390
|
async (opts, cmd) => {
|
|
9286
9391
|
const org = resolveOrg(cmd);
|
|
9287
|
-
if (
|
|
9392
|
+
if (!!opts.target === !!opts.fromRun)
|
|
9288
9393
|
throw new CliError(
|
|
9289
9394
|
"usage",
|
|
9290
|
-
"--target <agent[:model]
|
|
9395
|
+
"pass --from-run <runId> or --target <agent[:model]>, not both; discover runs with `gauge optimizations baselines`"
|
|
9291
9396
|
);
|
|
9292
9397
|
const subject = parseSubject(opts);
|
|
9293
9398
|
await confirmOrAbort(
|
|
9294
|
-
`Start an optimization on ${subjectLabel(subject.subjectKind)} ${subject.subjectId}?
|
|
9399
|
+
opts.fromRun ? `Start an optimization from the baseline containing ${opts.fromRun}?` : `Start an optimization on ${subjectLabel(subject.subjectKind)} ${subject.subjectId}? A new baseline may spend credits.`,
|
|
9295
9400
|
opts.yes
|
|
9296
9401
|
);
|
|
9297
9402
|
const detail = await createClient().post(basePath(org), {
|
|
9298
9403
|
...subject,
|
|
9299
9404
|
...targetFromOpts(opts),
|
|
9405
|
+
...opts.fromRun ? { fromRun: opts.fromRun } : {},
|
|
9300
9406
|
name: opts.name
|
|
9301
9407
|
});
|
|
9302
9408
|
if (resolveOutput(cmd) === "json") printJson(detail);
|
|
@@ -9514,26 +9620,22 @@ function register19(program2) {
|
|
|
9514
9620
|
}
|
|
9515
9621
|
);
|
|
9516
9622
|
group.command("authoring <id>").description(
|
|
9517
|
-
"What the next round may change:
|
|
9623
|
+
"What the next round may change: exposed surfaces, CLI targets, baseline run evidence, revision"
|
|
9518
9624
|
).action(async (id2, _opts, cmd) => {
|
|
9519
9625
|
printJson(
|
|
9520
9626
|
await createClient().get(`${itemPath(resolveOrg(cmd), id2)}/authoring`)
|
|
9521
9627
|
);
|
|
9522
9628
|
});
|
|
9523
9629
|
group.command("invocation <id> <invocationId>").description(
|
|
9524
|
-
"
|
|
9525
|
-
).action(
|
|
9526
|
-
|
|
9527
|
-
|
|
9528
|
-
|
|
9529
|
-
|
|
9530
|
-
|
|
9531
|
-
)
|
|
9532
|
-
);
|
|
9533
|
-
}
|
|
9534
|
-
);
|
|
9630
|
+
"Retired: use authoring source run references and gauge runs logs"
|
|
9631
|
+
).action(() => {
|
|
9632
|
+
throw new CliError(
|
|
9633
|
+
"usage",
|
|
9634
|
+
"The invocation catalog is retired. Read `gauge optimizations authoring <id>` and `gauge runs logs <runId> --output json` for observed command evidence."
|
|
9635
|
+
);
|
|
9636
|
+
});
|
|
9535
9637
|
group.command("cli-change <id>").description(
|
|
9536
|
-
"
|
|
9638
|
+
"Validate and seal v3 {schemaVersion:3, target, rules, shippingIntent}; use the output as intervention.after in a round file. Saves nothing"
|
|
9537
9639
|
).requiredOption("--file <path>", "the change JSON; '-' for stdin").action(async (id2, opts, cmd) => {
|
|
9538
9640
|
printJson(
|
|
9539
9641
|
await createClient().post(
|
|
@@ -9724,7 +9826,15 @@ function register19(program2) {
|
|
|
9724
9826
|
{ url: opts.url }
|
|
9725
9827
|
);
|
|
9726
9828
|
if (resolveOutput(cmd) === "json") printJson(res);
|
|
9727
|
-
else
|
|
9829
|
+
else {
|
|
9830
|
+
if (res.source === "scrape")
|
|
9831
|
+
console.error(
|
|
9832
|
+
"Source: fresh scrape; this is not the captured baseline response."
|
|
9833
|
+
);
|
|
9834
|
+
else if (res.source === "none")
|
|
9835
|
+
console.error("No page body is available.");
|
|
9836
|
+
console.log(res.body);
|
|
9837
|
+
}
|
|
9728
9838
|
});
|
|
9729
9839
|
}
|
|
9730
9840
|
|
|
@@ -9733,16 +9843,14 @@ var org_exports = {};
|
|
|
9733
9843
|
__export(org_exports, {
|
|
9734
9844
|
register: () => register20
|
|
9735
9845
|
});
|
|
9736
|
-
function printSettings(s) {
|
|
9846
|
+
function printSettings(s, billingStatus) {
|
|
9737
9847
|
console.log(`slug: ${s.slug}`);
|
|
9738
9848
|
console.log(`name: ${s.name}`);
|
|
9739
9849
|
console.log(`website: ${s.website ?? "(none)"}`);
|
|
9740
9850
|
console.log(
|
|
9741
9851
|
`max spend/run: ${s.maxSpendUsd == null ? "(server default)" : `$${s.maxSpendUsd}`}`
|
|
9742
9852
|
);
|
|
9743
|
-
console.log(
|
|
9744
|
-
`overage: ${s.overageEnabled ? "on" : "off"}${s.overageEnabled ? ` (cap: ${s.overageCapCredits == null ? "uncapped" : `${s.overageCapCredits} credits`})` : ""}`
|
|
9745
|
-
);
|
|
9853
|
+
console.log(`overage: ${fmtOverage(billingStatus)}`);
|
|
9746
9854
|
console.log(`auto-join: ${s.autoJoinDomain ?? "off"}`);
|
|
9747
9855
|
}
|
|
9748
9856
|
function register20(program2) {
|
|
@@ -9751,11 +9859,19 @@ function register20(program2) {
|
|
|
9751
9859
|
);
|
|
9752
9860
|
group.command("get").description("Show the org's settings (edit them in the web app)").action(async (_opts, cmd) => {
|
|
9753
9861
|
const org = resolveOrg(cmd);
|
|
9754
|
-
const
|
|
9755
|
-
|
|
9756
|
-
|
|
9757
|
-
|
|
9758
|
-
|
|
9862
|
+
const client = createClient();
|
|
9863
|
+
const base = `/api/v1/orgs/${encodeURIComponent(org)}`;
|
|
9864
|
+
const [settings, billingStatus] = await Promise.all([
|
|
9865
|
+
client.get(base),
|
|
9866
|
+
client.get(`${base}/billing`)
|
|
9867
|
+
]);
|
|
9868
|
+
if (resolveOutput(cmd) === "json")
|
|
9869
|
+
printJson({
|
|
9870
|
+
...settings,
|
|
9871
|
+
paymentType: billingStatus.paymentType ?? null,
|
|
9872
|
+
creditLine: billingStatus.creditLine ?? null
|
|
9873
|
+
});
|
|
9874
|
+
else printSettings(settings, billingStatus);
|
|
9759
9875
|
});
|
|
9760
9876
|
}
|
|
9761
9877
|
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@withgauge/cli",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.20.0",
|
|
4
4
|
"description": "Gauge customer CLI — Agent Preference, evals, runs, and organization settings over the Gauge API",
|
|
5
5
|
"private": false,
|
|
6
6
|
"author": "Gauge <support@withgauge.com> (https://withgauge.com)",
|