@letterstory/cli 0.54.0 → 0.57.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +14 -0
- package/lib/cli.mjs +11 -3
- package/lib/commands/admin.mjs +366 -5
- package/lib/commands/evals.mjs +1 -1
- package/lib/commands/shredder.mjs +110 -1
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -359,6 +359,12 @@ letterstory admin tuning compare --title "Echo vs live" --candidates Hazel:21,As
|
|
|
359
359
|
letterstory admin tuning comparison <id> # per-measure verdicts (better / worse / within noise / refused)
|
|
360
360
|
letterstory admin tuning picks <id> # the blind queue; then: admin tuning pick <id> --cell <key> --pick left|right|tie
|
|
361
361
|
letterstory admin tuning reading-list <id> --n 10 # your next blind pairs as links (Text A / Text B), to read off the screen; pick in the console
|
|
362
|
+
letterstory admin tuning pair <id> --cell <key> # one pair named: live and candidate, both texts, the measures per arm (a key reads every pair)
|
|
363
|
+
letterstory admin tuning candidates --kernels all --patch '{"writer":"echo/echo"}' --note "Echo" # the same change as a candidate of each kernel
|
|
364
|
+
letterstory admin tuning session-new --title "Echo round 1" --kernel Hazel --input-file brief.md --kind outline [--version 22]
|
|
365
|
+
letterstory admin tuning propose <session> --text "Echo with rarity on; live with the shred after" --create # trials from words, over the dials
|
|
366
|
+
letterstory admin tuning trial-run <trial> --repeats 2 # runs advance on the server; poll `admin tuning session <id> --texts`
|
|
367
|
+
letterstory admin tuning trial-save <trial> --note "…" # a trial as a candidate version (never live); then `admin tuning compare`
|
|
362
368
|
letterstory admin tuning promote <experiment-id> --versions <service-kernel-id>:21 # from a concluded "ship" decision
|
|
363
369
|
letterstory admin experiments list --q colons # what we've tried
|
|
364
370
|
letterstory admin experiments new "We use too many colons" --as "claude-code"
|
|
@@ -573,6 +579,14 @@ letterstory shredder run <endpoint-id> --file post.md --format markdown
|
|
|
573
579
|
|
|
574
580
|
letterstory shredder run <endpoint-id> --file post.md --no-wait # just queue it
|
|
575
581
|
letterstory shredder job <job-id> # poll it yourself
|
|
582
|
+
|
|
583
|
+
# The shredder as ONE versioned file (like a kernel file; schema at /api/schema/shredder):
|
|
584
|
+
letterstory shredder file <endpoint-id> > shredder.json # the file the endpoint shreds with
|
|
585
|
+
letterstory shredder file <endpoint-id> --patch '{"rewriters":{"excludeWriterFamily":true}}'
|
|
586
|
+
letterstory shredder versions <endpoint-id> # your copy's versions and our releases
|
|
587
|
+
letterstory shredder pin <endpoint-id> <version-id|none> # pin one of our releases, or follow live
|
|
588
|
+
letterstory shredder offer <endpoint-id> [--accept] # our newer version, over your own changes
|
|
589
|
+
letterstory shredder families [--hours 6] # how each rewriter family is doing
|
|
576
590
|
```
|
|
577
591
|
|
|
578
592
|
Both follow loops poll every 15–20 s. `LETTERSTORY_POLL_INTERVAL_MS` overrides that — set
|
package/lib/cli.mjs
CHANGED
|
@@ -54,7 +54,7 @@ import {
|
|
|
54
54
|
} from "./commands.mjs";
|
|
55
55
|
|
|
56
56
|
// Keep in sync with cli/package.json.
|
|
57
|
-
export const VERSION = "0.
|
|
57
|
+
export const VERSION = "0.57.0";
|
|
58
58
|
|
|
59
59
|
// Flags that never take a value. Listing them explicitly means `deploy get --json <id>`
|
|
60
60
|
// can't accidentally swallow the id as --json's value.
|
|
@@ -718,7 +718,15 @@ Shredder (diversify content produced outside Letterstory; structure preserved):
|
|
|
718
718
|
job lands as error or degraded. Pace the polling with
|
|
719
719
|
LETTERSTORY_POLL_INTERVAL_MS.
|
|
720
720
|
shredder job <job-id> Poll a queued shred and print it when it lands
|
|
721
|
-
shredder runs [--endpoint <id>] [--limit <n>] Recent shred calls (telemetry), newest first
|
|
721
|
+
shredder runs [--endpoint <id>] [--limit <n>] Recent shred calls (telemetry), newest first:
|
|
722
|
+
the shredder version, level, seats and models each ran
|
|
723
|
+
shredder file <endpoint-id> Print the shredder file the endpoint shreds with
|
|
724
|
+
shredder file <endpoint-id> (--file f.json | --patch '<json>') [--note "…"]
|
|
725
|
+
Save a new version of the endpoint's own copy
|
|
726
|
+
(schema: GET /api/schema/shredder)
|
|
727
|
+
shredder versions <endpoint-id> Your copy's versions and our released ones
|
|
728
|
+
shredder pin <endpoint-id> <version-id|none> Pin one of our versions, or follow our live one
|
|
729
|
+
shredder offer <endpoint-id> [--accept] What our newer version would change; --accept takes it
|
|
722
730
|
|
|
723
731
|
Ledes (opening lines for cold email, chosen per prospect from a hand-approved library):
|
|
724
732
|
lede list Your lede endpoints + run counts
|
|
@@ -852,7 +860,7 @@ Feedback (tell us what's broken, missing, unclear, or wanted):
|
|
|
852
860
|
|
|
853
861
|
Evals (how the pipeline is doing on your posts, and the regression cases captured from them):
|
|
854
862
|
evals scoreboard [--collection <id>] [--days N] Every measure by stage: posts, failing, rate
|
|
855
|
-
evals cases [--status open|guarding|proposed|retired|rejected] [--stage research|brief|links|mentions|accuracy|pipeline]
|
|
863
|
+
evals cases [--status open|guarding|proposed|retired|rejected] [--stage research|brief|links|mentions|accuracy|pipeline|shred]
|
|
856
864
|
evals capture (--article <id> | --research-run <id> | --kernel-job <id>) --title "…"
|
|
857
865
|
(--description "…" | --body … | --file <path>) (--measure <key> | --assertion "…")
|
|
858
866
|
[--stage <s>] [--replay <stage>] [--auto-pen <id>] [--published-version N]
|
package/lib/commands/admin.mjs
CHANGED
|
@@ -424,12 +424,28 @@ async function cmdAdminTuningContext(ctx) {
|
|
|
424
424
|
* admin tuning kernel <service-kernel-id> [--version N|live] one version's file (JSON)
|
|
425
425
|
* admin tuning candidate <service-kernel-id> --file f.json [--note "…"] save a file as a candidate version
|
|
426
426
|
* admin tuning compare --title "…" --candidates Hazel:21,Aspen:20 [--experiment <id>] start a comparison
|
|
427
|
+
* (Hazel:23@22 reads v23 against the test version v22 instead of live; it cannot promote)
|
|
427
428
|
* admin tuning comparisons | comparison <id> list / read one (per-measure verdicts)
|
|
428
429
|
* admin tuning picks <comparison-id> the blind queue (next pairs)
|
|
429
430
|
* admin tuning pick <comparison-id> --cell <key> --pick left|right|tie record a blind pick
|
|
430
431
|
* admin tuning reading-list <comparison-id> [--n 10] your next blind pairs as links, to read off the screen
|
|
432
|
+
* admin tuning pair <comparison-id> --cell <key> one pair named: live and candidate, both texts, the measures per arm
|
|
433
|
+
* admin tuning candidates --kernels Hazel,Aspen|all --patch '{…}' [--base N] [--note "…"] the same changes as a candidate of each kernel
|
|
431
434
|
* admin tuning promote <experiment-id> --versions <service-kernel-id>:<version>,… promote from a "ship" decision
|
|
432
435
|
*
|
|
436
|
+
* Sessions: one input written many ways, side by side (the console's Write › Sessions). Runs advance
|
|
437
|
+
* on the server (the advance-tuning-runs cron), so a session finishes with no tab open; the blind
|
|
438
|
+
* read of the results is a person's, in the console.
|
|
439
|
+
* admin tuning base-kernels the kernels a session can run on (ids for --kernel)
|
|
440
|
+
* admin tuning base-versions <kernel-id> the versions a session on it can run against
|
|
441
|
+
* admin tuning sessions | session <id> [--texts] list / one session: trials, runs and their numbers (--texts adds each trial's latest text)
|
|
442
|
+
* admin tuning session-new --title "…" --kernel <id|name> (--input-file f.md | --input "…") [--kind article|outline|brief]
|
|
443
|
+
* [--version N] [--writers echo/echo,…] [--trials-from <session-id>]
|
|
444
|
+
* admin tuning propose <session-id> --text "Echo with rarity on; live with the shred after" [--create]
|
|
445
|
+
* admin tuning trial-add <session-id> [--name "…"] [--patch '{…}'] [--options '{…}'] [--from <trial-id>]
|
|
446
|
+
* admin tuning trial-run <trial-id> [--repeats 2] queue runs (guardrails apply)
|
|
447
|
+
* admin tuning trial-save <trial-id> [--note "…"] save a trial as a candidate version (never live)
|
|
448
|
+
*
|
|
433
449
|
* Research (Phase 3): our arena's versions as files, research comparisons, the deploy split.
|
|
434
450
|
* admin tuning arenas | arena <version-id> our arena's versions / one version's file and diff
|
|
435
451
|
* admin tuning arena-candidate --patch '{…}' | --file f.json [--of <version-id>] [--note "…"] save a candidate
|
|
@@ -437,7 +453,16 @@ async function cmdAdminTuningContext(ctx) {
|
|
|
437
453
|
* admin tuning research-inputs [--set v1] research's fixed inputs
|
|
438
454
|
* admin tuning research-split [--since-hours 72] [--at <iso>,…] what each deploy did to research in production
|
|
439
455
|
* admin tuning research-standing live's level on the fixed inputs, week by week
|
|
456
|
+
*
|
|
457
|
+
* Shred (Phase 4): the same, for shredder versions on shred's fixed inputs.
|
|
458
|
+
* admin tuning shred-compare --title "…" (--candidate <version-id> | --patch '{…}' | --null-check) [--keys …]
|
|
459
|
+
* admin tuning shred-inputs [--set v1] | shred-split [--since-hours 72] [--at <iso>,…] | shred-standing
|
|
440
460
|
* admin tuning promote-arena <experiment-id> --version <arena-version-id> promote our arena from a "ship" decision
|
|
461
|
+
*
|
|
462
|
+
* Shred (Phase 4): our shredder's versions as files.
|
|
463
|
+
* admin tuning shredders | shredder <version-id> our shredder's versions / one version's file and diff
|
|
464
|
+
* admin tuning shredder-candidate --patch '{…}' | --file f.json [--of <version-id>] [--note "…"] save a candidate
|
|
465
|
+
* admin tuning promote-shredder <experiment-id> --version <shredder-version-id> promote our shredder from a "ship" decision (blind picks required)
|
|
441
466
|
*/
|
|
442
467
|
/** `--flag '<json>'` → the parsed object, or undefined when the flag is absent. */
|
|
443
468
|
function jsonFlag(raw, name) {
|
|
@@ -452,6 +477,44 @@ function jsonFlag(raw, name) {
|
|
|
452
477
|
}
|
|
453
478
|
}
|
|
454
479
|
|
|
480
|
+
/**
|
|
481
|
+
* A session as an agent reads it: each trial's changes and runs with their headline numbers, and
|
|
482
|
+
* (with `texts`) the latest finished text per trial. The full session (traces, records) is `--json`.
|
|
483
|
+
*/
|
|
484
|
+
function sessionSummary(session, texts) {
|
|
485
|
+
const runsOf = (trialId) => session.runs.filter((r) => r.trialId === trialId);
|
|
486
|
+
return {
|
|
487
|
+
id: session.id,
|
|
488
|
+
title: session.title,
|
|
489
|
+
kernel: session.baseKernelName,
|
|
490
|
+
base_version: session.baseVersion ?? "live",
|
|
491
|
+
material_ready: session.material?.ready ?? true,
|
|
492
|
+
trials: session.trials.map((t) => {
|
|
493
|
+
const runs = runsOf(t.id);
|
|
494
|
+
const latest = [...runs].reverse().find((r) => r.status === "done" && r.outputText);
|
|
495
|
+
return compact({
|
|
496
|
+
id: t.id,
|
|
497
|
+
name: t.name,
|
|
498
|
+
file_patch: t.filePatch,
|
|
499
|
+
run_options: t.runOptions,
|
|
500
|
+
saved_as_version: t.savedAsVersion ?? undefined,
|
|
501
|
+
runs: runs.map((r) =>
|
|
502
|
+
compact({
|
|
503
|
+
id: r.id,
|
|
504
|
+
status: r.status,
|
|
505
|
+
error: r.error ?? undefined,
|
|
506
|
+
quality: r.metrics?.qualityTotal,
|
|
507
|
+
quality_scale: r.metrics?.qualityScale,
|
|
508
|
+
p_machine: r.metrics?.pMachine,
|
|
509
|
+
words: r.metrics?.words,
|
|
510
|
+
})
|
|
511
|
+
),
|
|
512
|
+
latest_text: texts && latest ? latest.outputText : undefined,
|
|
513
|
+
});
|
|
514
|
+
}),
|
|
515
|
+
};
|
|
516
|
+
}
|
|
517
|
+
|
|
455
518
|
async function cmdAdminTuning(ctx) {
|
|
456
519
|
const [sub, id] = ctx.positionals;
|
|
457
520
|
const { flags, io } = ctx;
|
|
@@ -508,12 +571,17 @@ async function cmdAdminTuning(ctx) {
|
|
|
508
571
|
throw new CliError(
|
|
509
572
|
'Usage: admin tuning compare --title "…" --candidates Hazel:21,Aspen:20 [--experiment <id>]'
|
|
510
573
|
);
|
|
574
|
+
// kernel:version, or kernel:version@control to read it against a test version instead of live.
|
|
511
575
|
const candidates = list.map((item) => {
|
|
512
576
|
const at = item.lastIndexOf(":");
|
|
513
|
-
const
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
577
|
+
const [v, c] = item.slice(at + 1).split("@");
|
|
578
|
+
const version = Number(v);
|
|
579
|
+
const control = c === undefined ? undefined : Number(c);
|
|
580
|
+
if (at < 1 || !Number.isInteger(version) || (control !== undefined && !Number.isInteger(control)))
|
|
581
|
+
throw new CliError(
|
|
582
|
+
`--candidates items are kernel:version or kernel:version@control, got "${item}"`
|
|
583
|
+
);
|
|
584
|
+
return compact({ kernel: item.slice(0, at), candidate_version: version, control_version: control });
|
|
517
585
|
});
|
|
518
586
|
const r = await staffRequest(
|
|
519
587
|
ctx,
|
|
@@ -556,6 +624,199 @@ async function cmdAdminTuning(ctx) {
|
|
|
556
624
|
);
|
|
557
625
|
return json ? printResult(io, flags, r) : io.log(r.text);
|
|
558
626
|
}
|
|
627
|
+
case "pair": {
|
|
628
|
+
const cell = flagStr(flags.cell);
|
|
629
|
+
if (!id || !cell) throw new CliError("Usage: admin tuning pair <comparison-id> --cell <key>");
|
|
630
|
+
return printResult(
|
|
631
|
+
io,
|
|
632
|
+
{ ...flags, json: true },
|
|
633
|
+
await staffRequest(ctx, "GET", `${T}/comparisons/${id}/pair?${new URLSearchParams({ cell })}`)
|
|
634
|
+
);
|
|
635
|
+
}
|
|
636
|
+
case "candidates": {
|
|
637
|
+
const patch = jsonFlag(flags.patch, "--patch");
|
|
638
|
+
const wanted = flagList(flags.kernels);
|
|
639
|
+
if (!patch || !wanted?.length)
|
|
640
|
+
throw new CliError(
|
|
641
|
+
'Usage: admin tuning candidates --kernels Hazel,Aspen|all --patch \'{"writer":"echo/echo"}\' [--base N] [--note "…"]'
|
|
642
|
+
);
|
|
643
|
+
const { kernels } = await staffRequest(ctx, "GET", `${T}/kernel-files`);
|
|
644
|
+
const all = wanted.length === 1 && wanted[0].toLowerCase() === "all";
|
|
645
|
+
const picked = all
|
|
646
|
+
? kernels
|
|
647
|
+
: wanted.map((name) => {
|
|
648
|
+
const k = kernels.find(
|
|
649
|
+
(x) => x.kernel.toLowerCase() === name.toLowerCase() || x.serviceKernelId === name
|
|
650
|
+
);
|
|
651
|
+
if (!k) throw new CliError(`No catalog kernel named ${name}.`);
|
|
652
|
+
return k;
|
|
653
|
+
});
|
|
654
|
+
const base = flagNum(flags.base);
|
|
655
|
+
const r = await staffRequest(
|
|
656
|
+
ctx,
|
|
657
|
+
"POST",
|
|
658
|
+
`${T}/kernel-files/candidates`,
|
|
659
|
+
compact({
|
|
660
|
+
targets: picked.map((k) => compact({ service_kernel_id: k.serviceKernelId, base })),
|
|
661
|
+
file_patch: patch,
|
|
662
|
+
note: flagStr(flags.note),
|
|
663
|
+
})
|
|
664
|
+
);
|
|
665
|
+
if (json) return printResult(io, flags, r);
|
|
666
|
+
return io.log(
|
|
667
|
+
r.outcomes
|
|
668
|
+
.map((o) =>
|
|
669
|
+
o.version
|
|
670
|
+
? `${(o.kernel ?? o.serviceKernelId).padEnd(12)} candidate v${o.version} (from v${o.base}): ${o.changes.join(", ") || "no change"}`
|
|
671
|
+
: `${(o.kernel ?? o.serviceKernelId).padEnd(12)} refused: ${o.error}`
|
|
672
|
+
)
|
|
673
|
+
.join("\n")
|
|
674
|
+
);
|
|
675
|
+
}
|
|
676
|
+
// --- Sessions: one input written many ways, side by side ---
|
|
677
|
+
case "base-kernels": {
|
|
678
|
+
const r = await staffRequest(ctx, "GET", `${T}/kernels`);
|
|
679
|
+
if (json) return printResult(io, flags, r);
|
|
680
|
+
return io.log(
|
|
681
|
+
r.kernels.map((k) => `${k.name.padEnd(28)} ${(k.orgName ?? "shared").padEnd(20)} ${k.id}`).join("\n")
|
|
682
|
+
);
|
|
683
|
+
}
|
|
684
|
+
case "base-versions": {
|
|
685
|
+
if (!id) throw new CliError("Usage: admin tuning base-versions <kernel-id>");
|
|
686
|
+
const r = await staffRequest(ctx, "GET", `${T}/kernels/${id}/versions`);
|
|
687
|
+
if (json) return printResult(io, flags, r);
|
|
688
|
+
return io.log(
|
|
689
|
+
r.versions
|
|
690
|
+
.map((v) => `v${v.version} ${v.status.padEnd(9)} ${v.writer}${v.note ? ` ${v.note}` : ""}`)
|
|
691
|
+
.join("\n") || "(no versions)"
|
|
692
|
+
);
|
|
693
|
+
}
|
|
694
|
+
case "sessions": {
|
|
695
|
+
const r = await staffRequest(ctx, "GET", `${T}?limit=${flagNum(flags.limit) ?? 30}`);
|
|
696
|
+
if (json) return printResult(io, flags, r);
|
|
697
|
+
return io.log(
|
|
698
|
+
r.sessions
|
|
699
|
+
.map(
|
|
700
|
+
(x) =>
|
|
701
|
+
`${x.id} ${x.title} · ${x.baseKernelName ?? "?"}${x.baseVersion ? ` v${x.baseVersion}` : ""} · ${x.trialCount} trials, ${x.runCount} runs`
|
|
702
|
+
)
|
|
703
|
+
.join("\n") || "(no sessions)"
|
|
704
|
+
);
|
|
705
|
+
}
|
|
706
|
+
case "session": {
|
|
707
|
+
if (!id) throw new CliError("Usage: admin tuning session <session-id> [--texts]");
|
|
708
|
+
const { session } = await staffRequest(ctx, "GET", `${T}/sessions/${id}`);
|
|
709
|
+
if (json) return printResult(io, flags, { session });
|
|
710
|
+
return printResult(io, { ...flags, json: true }, sessionSummary(session, flagBool(flags.texts)));
|
|
711
|
+
}
|
|
712
|
+
case "session-new": {
|
|
713
|
+
const title = flagStr(flags.title);
|
|
714
|
+
const kernel = flagStr(flags.kernel);
|
|
715
|
+
const path = flagStr(flags["input-file"]);
|
|
716
|
+
let input = flagStr(flags.input);
|
|
717
|
+
if (path) {
|
|
718
|
+
try {
|
|
719
|
+
input = readFileSync(path, "utf8");
|
|
720
|
+
} catch (err) {
|
|
721
|
+
throw new CliError(`--input-file ${path}: ${err.message}`);
|
|
722
|
+
}
|
|
723
|
+
}
|
|
724
|
+
if (!title || !kernel || !input)
|
|
725
|
+
throw new CliError(
|
|
726
|
+
'Usage: admin tuning session-new --title "…" --kernel <id|name> (--input-file f.md | --input "…") [--kind article|outline|brief] [--version N] [--writers a,b] [--trials-from <session-id>]'
|
|
727
|
+
);
|
|
728
|
+
let kernelId = kernel;
|
|
729
|
+
if (!/^[0-9a-f-]{36}$/i.test(kernel)) {
|
|
730
|
+
const { kernels } = await staffRequest(ctx, "GET", `${T}/kernels`);
|
|
731
|
+
const matches = kernels.filter((k) => k.name.toLowerCase() === kernel.toLowerCase());
|
|
732
|
+
const shared = matches.find((k) => !k.orgName) ?? matches[0];
|
|
733
|
+
if (!shared) throw new CliError(`No kernel named ${kernel} (admin tuning base-kernels lists them).`);
|
|
734
|
+
kernelId = shared.id;
|
|
735
|
+
}
|
|
736
|
+
const { session } = await staffRequest(
|
|
737
|
+
ctx,
|
|
738
|
+
"POST",
|
|
739
|
+
T,
|
|
740
|
+
compact({
|
|
741
|
+
title,
|
|
742
|
+
input_text: input,
|
|
743
|
+
input_kind: flagStr(flags.kind) ?? "unknown",
|
|
744
|
+
base_kernel_id: kernelId,
|
|
745
|
+
base_version: flagNum(flags.version),
|
|
746
|
+
trials_from_session_id: flagStr(flags["trials-from"]),
|
|
747
|
+
writers: (flagList(flags.writers) ?? []).map((model) => ({ model })),
|
|
748
|
+
})
|
|
749
|
+
);
|
|
750
|
+
if (json) return printResult(io, flags, { session });
|
|
751
|
+
return io.log(
|
|
752
|
+
`session ${session.id}: ${session.trials.length} trials on ${session.baseKernelName}${session.baseVersion ? ` v${session.baseVersion}` : " (live)"} → ${ctx.client.url}/superadmin/tuning/${session.id}`
|
|
753
|
+
);
|
|
754
|
+
}
|
|
755
|
+
case "propose": {
|
|
756
|
+
const text = flagStr(flags.text);
|
|
757
|
+
if (!id || !text)
|
|
758
|
+
throw new CliError(
|
|
759
|
+
'Usage: admin tuning propose <session-id> --text "Echo with rarity on; …" [--create]'
|
|
760
|
+
);
|
|
761
|
+
const r = await staffRequest(ctx, "POST", `${T}/sessions/${id}/propose`, { description: text });
|
|
762
|
+
if (flagBool(flags.create))
|
|
763
|
+
for (const t of r.trials)
|
|
764
|
+
await staffRequest(ctx, "POST", `${T}/sessions/${id}/trials`, {
|
|
765
|
+
name: t.name,
|
|
766
|
+
file_patch: t.file_patch,
|
|
767
|
+
run_options: t.run_options,
|
|
768
|
+
});
|
|
769
|
+
if (json) return printResult(io, flags, r);
|
|
770
|
+
return io.log(
|
|
771
|
+
[
|
|
772
|
+
...r.trials.map(
|
|
773
|
+
(t) => `${flagBool(flags.create) ? "created" : "proposed"} ${t.name}: ${t.changes.join(", ")}`
|
|
774
|
+
),
|
|
775
|
+
...r.unsupported.map((u) => `not expressible with the dials: ${u}`),
|
|
776
|
+
...r.rejected.map((x) => `dropped from ${x.trial}: ${x.dial} (${x.reason})`),
|
|
777
|
+
].join("\n") || "(nothing proposed)"
|
|
778
|
+
);
|
|
779
|
+
}
|
|
780
|
+
case "trial-add": {
|
|
781
|
+
if (!id)
|
|
782
|
+
throw new CliError(
|
|
783
|
+
"Usage: admin tuning trial-add <session-id> [--name] [--patch '{…}'] [--options '{…}'] [--from <trial-id>]"
|
|
784
|
+
);
|
|
785
|
+
const r = await staffRequest(
|
|
786
|
+
ctx,
|
|
787
|
+
"POST",
|
|
788
|
+
`${T}/sessions/${id}/trials`,
|
|
789
|
+
compact({
|
|
790
|
+
name: flagStr(flags.name),
|
|
791
|
+
file_patch: jsonFlag(flags.patch, "--patch"),
|
|
792
|
+
run_options: jsonFlag(flags.options, "--options"),
|
|
793
|
+
from_trial_id: flagStr(flags.from),
|
|
794
|
+
})
|
|
795
|
+
);
|
|
796
|
+
return json ? printResult(io, flags, r) : io.log(`trial ${r.trial.id} (${r.trial.name})`);
|
|
797
|
+
}
|
|
798
|
+
case "trial-run": {
|
|
799
|
+
if (!id) throw new CliError("Usage: admin tuning trial-run <trial-id> [--repeats N]");
|
|
800
|
+
const r = await staffRequest(ctx, "POST", `${T}/trials/${id}/run`, {
|
|
801
|
+
repeats: flagNum(flags.repeats) ?? 1,
|
|
802
|
+
});
|
|
803
|
+
return json
|
|
804
|
+
? printResult(io, flags, r)
|
|
805
|
+
: io.log(`queued ${r.runs.length} run(s): ${r.runs.map((x) => x.id).join(", ")}`);
|
|
806
|
+
}
|
|
807
|
+
case "trial-save": {
|
|
808
|
+
if (!id) throw new CliError('Usage: admin tuning trial-save <trial-id> [--note "…"]');
|
|
809
|
+
const r = await staffRequest(
|
|
810
|
+
ctx,
|
|
811
|
+
"POST",
|
|
812
|
+
`${T}/trials/${id}/candidate`,
|
|
813
|
+
compact({ note: flagStr(flags.note) })
|
|
814
|
+
);
|
|
815
|
+
if (json) return printResult(io, flags, r);
|
|
816
|
+
return io.log(
|
|
817
|
+
`${r.kernel}: saved as candidate v${r.version}${r.baseVersion ? ` (from v${r.baseVersion})` : ""}${r.leftBehind.length ? `; per-run, not saved: ${r.leftBehind.join(", ")}` : ""}`
|
|
818
|
+
);
|
|
819
|
+
}
|
|
559
820
|
// --- Research: our arena's versions, research comparisons, the deploy split ---
|
|
560
821
|
case "arenas": {
|
|
561
822
|
const r = await staffRequest(ctx, "GET", `${T}/arena-files`);
|
|
@@ -633,6 +894,56 @@ async function cmdAdminTuning(ctx) {
|
|
|
633
894
|
await staffRequest(ctx, "GET", `${T}/research/deploy-split${q.size ? `?${q}` : ""}`)
|
|
634
895
|
);
|
|
635
896
|
}
|
|
897
|
+
case "shred-compare": {
|
|
898
|
+
const title = flagStr(flags.title);
|
|
899
|
+
const versionId = flagStr(flags.candidate);
|
|
900
|
+
const patch = jsonFlag(flags.patch, "--patch");
|
|
901
|
+
const nullCheck = flagBool(flags["null-check"]);
|
|
902
|
+
const named = [versionId !== undefined, patch !== undefined, nullCheck === true].filter(Boolean).length;
|
|
903
|
+
if (!title || named !== 1)
|
|
904
|
+
throw new CliError(
|
|
905
|
+
"Usage: admin tuning shred-compare --title \"…\" (--candidate <version-id> | --patch '{…}' | --null-check) [--keys k1,k2] [--concurrency 6] [--experiment <id>]"
|
|
906
|
+
);
|
|
907
|
+
const candidate = versionId ? { version_id: versionId } : patch ? { patch } : { null_check: true };
|
|
908
|
+
const r = await staffRequest(
|
|
909
|
+
ctx,
|
|
910
|
+
"POST",
|
|
911
|
+
`${T}/comparisons`,
|
|
912
|
+
compact({
|
|
913
|
+
primitive: "shred",
|
|
914
|
+
title,
|
|
915
|
+
candidate,
|
|
916
|
+
keys: flagList(flags.keys),
|
|
917
|
+
concurrency: flagNum(flags.concurrency),
|
|
918
|
+
experiment_id: flagStr(flags.experiment),
|
|
919
|
+
})
|
|
920
|
+
);
|
|
921
|
+
return json ? printResult(io, flags, r) : io.log(`shred comparison ${r.comparison?.id ?? ""} started`);
|
|
922
|
+
}
|
|
923
|
+
case "shred-inputs": {
|
|
924
|
+
const set = flagStr(flags.set);
|
|
925
|
+
return printResult(
|
|
926
|
+
io,
|
|
927
|
+
{ ...flags, json: true },
|
|
928
|
+
await staffRequest(ctx, "GET", `${T}/shred/inputs${set ? `?${new URLSearchParams({ set })}` : ""}`)
|
|
929
|
+
);
|
|
930
|
+
}
|
|
931
|
+
case "shred-split": {
|
|
932
|
+
const q = new URLSearchParams(
|
|
933
|
+
compact({
|
|
934
|
+
since_hours: flagStr(flags["since-hours"]),
|
|
935
|
+
window_hours: flagStr(flags["window-hours"]),
|
|
936
|
+
at: flagStr(flags.at),
|
|
937
|
+
})
|
|
938
|
+
);
|
|
939
|
+
return printResult(
|
|
940
|
+
io,
|
|
941
|
+
{ ...flags, json: true },
|
|
942
|
+
await staffRequest(ctx, "GET", `${T}/shred/deploy-split${q.size ? `?${q}` : ""}`)
|
|
943
|
+
);
|
|
944
|
+
}
|
|
945
|
+
case "shred-standing":
|
|
946
|
+
return printResult(io, { ...flags, json: true }, await staffRequest(ctx, "GET", `${T}/shred/standing`));
|
|
636
947
|
case "research-standing":
|
|
637
948
|
return printResult(io, { ...flags, json: true }, await staffRequest(ctx, "GET", `${T}/research/standing`));
|
|
638
949
|
case "promote-arena": {
|
|
@@ -650,6 +961,56 @@ async function cmdAdminTuning(ctx) {
|
|
|
650
961
|
)
|
|
651
962
|
);
|
|
652
963
|
}
|
|
964
|
+
// --- Shred: our shredder's versions as files ---
|
|
965
|
+
case "shredders": {
|
|
966
|
+
const r = await staffRequest(ctx, "GET", `${T}/shredder-files`);
|
|
967
|
+
if (json) return printResult(io, flags, r);
|
|
968
|
+
return io.log(
|
|
969
|
+
r.versions
|
|
970
|
+
.map((v) => `${v.label.padEnd(18)} ${v.status.padEnd(9)} ${v.id}${v.note ? ` ${v.note}` : ""}`)
|
|
971
|
+
.join("\n") || "(no versions)"
|
|
972
|
+
);
|
|
973
|
+
}
|
|
974
|
+
case "shredder": {
|
|
975
|
+
if (!id) throw new CliError("Usage: admin tuning shredder <version-id>");
|
|
976
|
+
return printResult(
|
|
977
|
+
io,
|
|
978
|
+
{ ...flags, json: true },
|
|
979
|
+
await staffRequest(ctx, "GET", `${T}/shredder-files/${id}`)
|
|
980
|
+
);
|
|
981
|
+
}
|
|
982
|
+
case "shredder-candidate": {
|
|
983
|
+
const patch = jsonFlag(flags.patch, "--patch");
|
|
984
|
+
const file = jsonFile({ "shredder-file": flags.file }, "shredder");
|
|
985
|
+
if ((patch === undefined) === (file === undefined))
|
|
986
|
+
throw new CliError(
|
|
987
|
+
'Usage: admin tuning shredder-candidate --patch \'{"rewriters":{"excludeWriterFamily":true}}\' | --file shredder.json [--of <version-id>] [--note "…"]'
|
|
988
|
+
);
|
|
989
|
+
const r = await staffRequest(
|
|
990
|
+
ctx,
|
|
991
|
+
"POST",
|
|
992
|
+
`${T}/shredder-files`,
|
|
993
|
+
compact({ patch, file, patch_of: flagStr(flags.of), note: flagStr(flags.note) })
|
|
994
|
+
);
|
|
995
|
+
return json ? printResult(io, flags, r) : io.log(`saved candidate ${r.version.label} (${r.version.id})`);
|
|
996
|
+
}
|
|
997
|
+
case "promote-shredder": {
|
|
998
|
+
const version = flagStr(flags.version);
|
|
999
|
+
if (!id || !version)
|
|
1000
|
+
throw new CliError(
|
|
1001
|
+
"Usage: admin tuning promote-shredder <experiment-id> --version <shredder-version-id>"
|
|
1002
|
+
);
|
|
1003
|
+
return printResult(
|
|
1004
|
+
io,
|
|
1005
|
+
{ ...flags, json: true },
|
|
1006
|
+
await staffRequest(
|
|
1007
|
+
ctx,
|
|
1008
|
+
"POST",
|
|
1009
|
+
`${EXPERIMENTS}/${id}/promote`,
|
|
1010
|
+
compact({ shredder_version_id: version, actor: actorFrom(flags) })
|
|
1011
|
+
)
|
|
1012
|
+
);
|
|
1013
|
+
}
|
|
653
1014
|
case "promote": {
|
|
654
1015
|
const list = flagList(flags.versions);
|
|
655
1016
|
if (!id || !list?.length)
|
|
@@ -673,7 +1034,7 @@ async function cmdAdminTuning(ctx) {
|
|
|
673
1034
|
}
|
|
674
1035
|
default:
|
|
675
1036
|
throw new CliError(
|
|
676
|
-
"Usage: admin tuning kernels|kernel|candidate|compare|comparisons|comparison|picks|pick|reading-list|promote|arenas|arena|arena-candidate|research-compare|research-inputs|research-split|research-standing|promote-arena (see --help)"
|
|
1037
|
+
"Usage: admin tuning kernels|kernel|candidate|compare|comparisons|comparison|picks|pick|reading-list|pair|candidates|base-kernels|base-versions|sessions|session|session-new|propose|trial-add|trial-run|trial-save|promote|arenas|arena|arena-candidate|research-compare|research-inputs|research-split|research-standing|promote-arena|shred-compare|shred-inputs|shred-split|shred-standing|shredders|shredder|shredder-candidate|promote-shredder (see --help)"
|
|
677
1038
|
);
|
|
678
1039
|
}
|
|
679
1040
|
}
|
package/lib/commands/evals.mjs
CHANGED
|
@@ -16,7 +16,7 @@
|
|
|
16
16
|
import { CliError } from "../client.mjs";
|
|
17
17
|
import { flagStr, readBodyInput, printResult, ok, argsFromFlags } from "./shared.mjs";
|
|
18
18
|
|
|
19
|
-
const STAGES = ["research", "brief", "links", "mentions", "accuracy", "pipeline"];
|
|
19
|
+
const STAGES = ["research", "brief", "links", "mentions", "accuracy", "pipeline", "shred"];
|
|
20
20
|
const STATUSES = ["proposed", "open", "guarding", "retired", "rejected"];
|
|
21
21
|
|
|
22
22
|
/** `--keys a,b,c` (or repeated `--keys`) → ["a", "b", "c"]; undefined when unset. */
|
|
@@ -35,6 +35,23 @@ function readJsonFileFlag(flags, flag) {
|
|
|
35
35
|
}
|
|
36
36
|
}
|
|
37
37
|
|
|
38
|
+
/** `--<flag> '<json>'` inline, or `--<flag>-file <path>` → a JSON object, or undefined. */
|
|
39
|
+
function readJsonObjectFlag(flags, name) {
|
|
40
|
+
const inline = flagStr(flags[name]);
|
|
41
|
+
if (inline !== undefined) {
|
|
42
|
+
let parsed;
|
|
43
|
+
try {
|
|
44
|
+
parsed = JSON.parse(inline);
|
|
45
|
+
} catch (err) {
|
|
46
|
+
throw new CliError(`--${name} is not valid JSON: ${err.message}`);
|
|
47
|
+
}
|
|
48
|
+
if (parsed === null || typeof parsed !== "object" || Array.isArray(parsed))
|
|
49
|
+
throw new CliError(`--${name} must be a JSON object`);
|
|
50
|
+
return parsed;
|
|
51
|
+
}
|
|
52
|
+
return readJsonFileFlag(flags, `${name}-file`);
|
|
53
|
+
}
|
|
54
|
+
|
|
38
55
|
/**
|
|
39
56
|
* How every command in this file sources its tool's inputs — see `argsFromFlags` in
|
|
40
57
|
* shared.mjs and `scripts/parity-check.ts`, which diffs these keys against each tool's
|
|
@@ -68,6 +85,10 @@ export const ARGS = {
|
|
|
68
85
|
},
|
|
69
86
|
list_shredder_endpoints: {},
|
|
70
87
|
get_shred_balance: {},
|
|
88
|
+
// The rewriter families behind the shredder, as the platform's hourly watch reads them.
|
|
89
|
+
get_shred_family_health: {
|
|
90
|
+
hours: { flag: "hours", type: "num" },
|
|
91
|
+
},
|
|
71
92
|
create_shredder_endpoint: {
|
|
72
93
|
name: { flag: "name", required: true },
|
|
73
94
|
},
|
|
@@ -83,6 +104,33 @@ export const ARGS = {
|
|
|
83
104
|
endpoint_id: { positional: 1, label: "endpoint-id", required: true },
|
|
84
105
|
config: { value: (flags) => readJsonFileFlag(flags, "file") },
|
|
85
106
|
},
|
|
107
|
+
// The shredder as a versioned file (the arena file's twin): `shredder file <id>` prints it,
|
|
108
|
+
// `--file f.json` saves a whole file, `--patch '{…}'` only the fields to change.
|
|
109
|
+
get_shredder_file: {
|
|
110
|
+
endpoint_id: { positional: 0, label: "endpoint-id", required: true },
|
|
111
|
+
},
|
|
112
|
+
save_shredder_file: {
|
|
113
|
+
endpoint_id: { positional: 0, label: "endpoint-id", required: true },
|
|
114
|
+
file: { value: (flags) => readJsonFileFlag(flags, "file") },
|
|
115
|
+
patch: { value: (flags) => readJsonObjectFlag(flags, "patch") },
|
|
116
|
+
note: "note",
|
|
117
|
+
},
|
|
118
|
+
list_shredder_versions: {
|
|
119
|
+
endpoint_id: { positional: 0, label: "endpoint-id", required: true },
|
|
120
|
+
},
|
|
121
|
+
pin_shredder_version: {
|
|
122
|
+
endpoint_id: { positional: 0, label: "endpoint-id", required: true },
|
|
123
|
+
version_id: {
|
|
124
|
+
value: (_flags, positionals) => {
|
|
125
|
+
const v = positionals[1];
|
|
126
|
+
if (v === undefined) throw new CliError("Missing required <version-id|none>");
|
|
127
|
+
return v === "none" ? null : v;
|
|
128
|
+
},
|
|
129
|
+
},
|
|
130
|
+
},
|
|
131
|
+
accept_shredder_offer: {
|
|
132
|
+
endpoint_id: { positional: 0, label: "endpoint-id", required: true },
|
|
133
|
+
},
|
|
86
134
|
};
|
|
87
135
|
|
|
88
136
|
export async function cmdShredder(ctx) {
|
|
@@ -111,9 +159,19 @@ export async function cmdShredder(ctx) {
|
|
|
111
159
|
return shredderRuns(rest);
|
|
112
160
|
case "balance":
|
|
113
161
|
return shredderBalance(rest);
|
|
162
|
+
case "file":
|
|
163
|
+
return shredderFile(rest);
|
|
164
|
+
case "versions":
|
|
165
|
+
return shredderTool(rest, "list_shredder_versions");
|
|
166
|
+
case "pin":
|
|
167
|
+
return shredderTool(rest, "pin_shredder_version");
|
|
168
|
+
case "offer":
|
|
169
|
+
return rest.flags.accept === true ? shredderTool(rest, "accept_shredder_offer") : shredderOffer(rest);
|
|
170
|
+
case "families":
|
|
171
|
+
return shredderTool(rest, "get_shred_family_health");
|
|
114
172
|
default:
|
|
115
173
|
throw new CliError(
|
|
116
|
-
`Unknown shredder subcommand: ${sub ?? "(none)"}. Try: list, create, delete, endpoint, run, job, runs, balance`
|
|
174
|
+
`Unknown shredder subcommand: ${sub ?? "(none)"}. Try: list, create, delete, endpoint, run, job, runs, balance, file, versions, pin, offer, families`
|
|
117
175
|
);
|
|
118
176
|
}
|
|
119
177
|
}
|
|
@@ -284,3 +342,54 @@ async function shredderBalance(ctx) {
|
|
|
284
342
|
printResult(io, flags, result);
|
|
285
343
|
return 0;
|
|
286
344
|
}
|
|
345
|
+
|
|
346
|
+
/**
|
|
347
|
+
* `shredder file <endpoint-id>` prints the file the endpoint shreds with (the file itself, so
|
|
348
|
+
* `> f.json` round-trips); `shredder file <endpoint-id> --file f.json | --patch '{…}' [--note …]`
|
|
349
|
+
* saves a new live version of the endpoint's own copy.
|
|
350
|
+
*/
|
|
351
|
+
async function shredderFile(ctx) {
|
|
352
|
+
const { client, flags, io } = ctx;
|
|
353
|
+
const saving =
|
|
354
|
+
flagStr(flags.file) !== undefined ||
|
|
355
|
+
flagStr(flags.patch) !== undefined ||
|
|
356
|
+
flagStr(flags["patch-file"]) !== undefined;
|
|
357
|
+
if (!saving) {
|
|
358
|
+
const result = await client.callTool(
|
|
359
|
+
"get_shredder_file",
|
|
360
|
+
argsFromFlags(ARGS.get_shredder_file, flags, ctx.positionals)
|
|
361
|
+
);
|
|
362
|
+
printResult(io, flags, result, (r) => JSON.stringify(r?.file ?? r, null, 2));
|
|
363
|
+
return 0;
|
|
364
|
+
}
|
|
365
|
+
const result = await client.callTool(
|
|
366
|
+
"save_shredder_file",
|
|
367
|
+
argsFromFlags(ARGS.save_shredder_file, flags, ctx.positionals)
|
|
368
|
+
);
|
|
369
|
+
ok(
|
|
370
|
+
ctx,
|
|
371
|
+
`Saved ${toolField(result, "version")?.label ?? "a new version"}; the endpoint shreds with it from now on.`
|
|
372
|
+
);
|
|
373
|
+
printResult(io, flags, result);
|
|
374
|
+
return 0;
|
|
375
|
+
}
|
|
376
|
+
|
|
377
|
+
/** `shredder offer <endpoint-id>`: what our newer version would change; `--accept` takes it. */
|
|
378
|
+
async function shredderOffer(ctx) {
|
|
379
|
+
const { client, flags, io } = ctx;
|
|
380
|
+
const result = await client.callTool(
|
|
381
|
+
"get_shredder_file",
|
|
382
|
+
argsFromFlags(ARGS.get_shredder_file, flags, ctx.positionals)
|
|
383
|
+
);
|
|
384
|
+
const offer = toolField(result, "offer");
|
|
385
|
+
if (!offer) ok(ctx, "No offer: your endpoint shreds with our latest version, or a copy based on it.");
|
|
386
|
+
printResult(io, flags, result, (r) => JSON.stringify(r?.offer ?? null, null, 2));
|
|
387
|
+
return 0;
|
|
388
|
+
}
|
|
389
|
+
|
|
390
|
+
/** One shredder-file tool, its arguments from ARGS. */
|
|
391
|
+
async function shredderTool(ctx, tool) {
|
|
392
|
+
const result = await ctx.client.callTool(tool, argsFromFlags(ARGS[tool], ctx.flags, ctx.positionals));
|
|
393
|
+
printResult(ctx.io, ctx.flags, result);
|
|
394
|
+
return 0;
|
|
395
|
+
}
|