@letterstory/cli 0.55.0 → 0.57.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +13 -0
- package/lib/cli.mjs +11 -3
- package/lib/commands/admin.mjs +356 -5
- package/lib/commands/evals.mjs +1 -1
- package/lib/commands/shredder.mjs +110 -1
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -360,6 +360,11 @@ letterstory admin tuning comparison <id> # per-measure verdicts (b
|
|
|
360
360
|
letterstory admin tuning picks <id> # the blind queue; then: admin tuning pick <id> --cell <key> --pick left|right|tie
|
|
361
361
|
letterstory admin tuning reading-list <id> --n 10 # your next blind pairs as links (Text A / Text B), to read off the screen; pick in the console
|
|
362
362
|
letterstory admin tuning pair <id> --cell <key> # one pair named: live and candidate, both texts, the measures per arm (a key reads every pair)
|
|
363
|
+
letterstory admin tuning candidates --kernels all --patch '{"writer":"echo/echo"}' --note "Echo" # the same change as a candidate of each kernel
|
|
364
|
+
letterstory admin tuning session-new --title "Echo round 1" --kernel Hazel --input-file brief.md --kind outline [--version 22]
|
|
365
|
+
letterstory admin tuning propose <session> --text "Echo with rarity on; live with the shred after" --create # trials from words, over the dials
|
|
366
|
+
letterstory admin tuning trial-run <trial> --repeats 2 # runs advance on the server; poll `admin tuning session <id> --texts`
|
|
367
|
+
letterstory admin tuning trial-save <trial> --note "…" # a trial as a candidate version (never live); then `admin tuning compare`
|
|
363
368
|
letterstory admin tuning promote <experiment-id> --versions <service-kernel-id>:21 # from a concluded "ship" decision
|
|
364
369
|
letterstory admin experiments list --q colons # what we've tried
|
|
365
370
|
letterstory admin experiments new "We use too many colons" --as "claude-code"
|
|
@@ -574,6 +579,14 @@ letterstory shredder run <endpoint-id> --file post.md --format markdown
|
|
|
574
579
|
|
|
575
580
|
letterstory shredder run <endpoint-id> --file post.md --no-wait # just queue it
|
|
576
581
|
letterstory shredder job <job-id> # poll it yourself
|
|
582
|
+
|
|
583
|
+
# The shredder as ONE versioned file (like a kernel file; schema at /api/schema/shredder):
|
|
584
|
+
letterstory shredder file <endpoint-id> > shredder.json # the file the endpoint shreds with
|
|
585
|
+
letterstory shredder file <endpoint-id> --patch '{"rewriters":{"excludeWriterFamily":true}}'
|
|
586
|
+
letterstory shredder versions <endpoint-id> # your copy's versions and our releases
|
|
587
|
+
letterstory shredder pin <endpoint-id> <version-id|none> # pin one of our releases, or follow live
|
|
588
|
+
letterstory shredder offer <endpoint-id> [--accept] # our newer version, over your own changes
|
|
589
|
+
letterstory shredder families [--hours 6] # how each rewriter family is doing
|
|
577
590
|
```
|
|
578
591
|
|
|
579
592
|
Both follow loops poll every 15–20 s. `LETTERSTORY_POLL_INTERVAL_MS` overrides that — set
|
package/lib/cli.mjs
CHANGED
|
@@ -54,7 +54,7 @@ import {
|
|
|
54
54
|
} from "./commands.mjs";
|
|
55
55
|
|
|
56
56
|
// Keep in sync with cli/package.json.
|
|
57
|
-
export const VERSION = "0.
|
|
57
|
+
export const VERSION = "0.57.0";
|
|
58
58
|
|
|
59
59
|
// Flags that never take a value. Listing them explicitly means `deploy get --json <id>`
|
|
60
60
|
// can't accidentally swallow the id as --json's value.
|
|
@@ -718,7 +718,15 @@ Shredder (diversify content produced outside Letterstory; structure preserved):
|
|
|
718
718
|
job lands as error or degraded. Pace the polling with
|
|
719
719
|
LETTERSTORY_POLL_INTERVAL_MS.
|
|
720
720
|
shredder job <job-id> Poll a queued shred and print it when it lands
|
|
721
|
-
shredder runs [--endpoint <id>] [--limit <n>] Recent shred calls (telemetry), newest first
|
|
721
|
+
shredder runs [--endpoint <id>] [--limit <n>] Recent shred calls (telemetry), newest first:
|
|
722
|
+
the shredder version, level, seats and models each ran
|
|
723
|
+
shredder file <endpoint-id> Print the shredder file the endpoint shreds with
|
|
724
|
+
shredder file <endpoint-id> (--file f.json | --patch '<json>') [--note "…"]
|
|
725
|
+
Save a new version of the endpoint's own copy
|
|
726
|
+
(schema: GET /api/schema/shredder)
|
|
727
|
+
shredder versions <endpoint-id> Your copy's versions and our released ones
|
|
728
|
+
shredder pin <endpoint-id> <version-id|none> Pin one of our versions, or follow our live one
|
|
729
|
+
shredder offer <endpoint-id> [--accept] What our newer version would change; --accept takes it
|
|
722
730
|
|
|
723
731
|
Ledes (opening lines for cold email, chosen per prospect from a hand-approved library):
|
|
724
732
|
lede list Your lede endpoints + run counts
|
|
@@ -852,7 +860,7 @@ Feedback (tell us what's broken, missing, unclear, or wanted):
|
|
|
852
860
|
|
|
853
861
|
Evals (how the pipeline is doing on your posts, and the regression cases captured from them):
|
|
854
862
|
evals scoreboard [--collection <id>] [--days N] Every measure by stage: posts, failing, rate
|
|
855
|
-
evals cases [--status open|guarding|proposed|retired|rejected] [--stage research|brief|links|mentions|accuracy|pipeline]
|
|
863
|
+
evals cases [--status open|guarding|proposed|retired|rejected] [--stage research|brief|links|mentions|accuracy|pipeline|shred]
|
|
856
864
|
evals capture (--article <id> | --research-run <id> | --kernel-job <id>) --title "…"
|
|
857
865
|
(--description "…" | --body … | --file <path>) (--measure <key> | --assertion "…")
|
|
858
866
|
[--stage <s>] [--replay <stage>] [--auto-pen <id>] [--published-version N]
|
package/lib/commands/admin.mjs
CHANGED
|
@@ -424,13 +424,28 @@ async function cmdAdminTuningContext(ctx) {
|
|
|
424
424
|
* admin tuning kernel <service-kernel-id> [--version N|live] one version's file (JSON)
|
|
425
425
|
* admin tuning candidate <service-kernel-id> --file f.json [--note "…"] save a file as a candidate version
|
|
426
426
|
* admin tuning compare --title "…" --candidates Hazel:21,Aspen:20 [--experiment <id>] start a comparison
|
|
427
|
+
* (Hazel:23@22 reads v23 against the test version v22 instead of live; it cannot promote)
|
|
427
428
|
* admin tuning comparisons | comparison <id> list / read one (per-measure verdicts)
|
|
428
429
|
* admin tuning picks <comparison-id> the blind queue (next pairs)
|
|
429
430
|
* admin tuning pick <comparison-id> --cell <key> --pick left|right|tie record a blind pick
|
|
430
431
|
* admin tuning reading-list <comparison-id> [--n 10] your next blind pairs as links, to read off the screen
|
|
431
432
|
* admin tuning pair <comparison-id> --cell <key> one pair named: live and candidate, both texts, the measures per arm
|
|
433
|
+
* admin tuning candidates --kernels Hazel,Aspen|all --patch '{…}' [--base N] [--note "…"] the same changes as a candidate of each kernel
|
|
432
434
|
* admin tuning promote <experiment-id> --versions <service-kernel-id>:<version>,… promote from a "ship" decision
|
|
433
435
|
*
|
|
436
|
+
* Sessions: one input written many ways, side by side (the console's Write › Sessions). Runs advance
|
|
437
|
+
* on the server (the advance-tuning-runs cron), so a session finishes with no tab open; the blind
|
|
438
|
+
* read of the results is a person's, in the console.
|
|
439
|
+
* admin tuning base-kernels the kernels a session can run on (ids for --kernel)
|
|
440
|
+
* admin tuning base-versions <kernel-id> the versions a session on it can run against
|
|
441
|
+
* admin tuning sessions | session <id> [--texts] list / one session: trials, runs and their numbers (--texts adds each trial's latest text)
|
|
442
|
+
* admin tuning session-new --title "…" --kernel <id|name> (--input-file f.md | --input "…") [--kind article|outline|brief]
|
|
443
|
+
* [--version N] [--writers echo/echo,…] [--trials-from <session-id>]
|
|
444
|
+
* admin tuning propose <session-id> --text "Echo with rarity on; live with the shred after" [--create]
|
|
445
|
+
* admin tuning trial-add <session-id> [--name "…"] [--patch '{…}'] [--options '{…}'] [--from <trial-id>]
|
|
446
|
+
* admin tuning trial-run <trial-id> [--repeats 2] queue runs (guardrails apply)
|
|
447
|
+
* admin tuning trial-save <trial-id> [--note "…"] save a trial as a candidate version (never live)
|
|
448
|
+
*
|
|
434
449
|
* Research (Phase 3): our arena's versions as files, research comparisons, the deploy split.
|
|
435
450
|
* admin tuning arenas | arena <version-id> our arena's versions / one version's file and diff
|
|
436
451
|
* admin tuning arena-candidate --patch '{…}' | --file f.json [--of <version-id>] [--note "…"] save a candidate
|
|
@@ -438,7 +453,16 @@ async function cmdAdminTuningContext(ctx) {
|
|
|
438
453
|
* admin tuning research-inputs [--set v1] research's fixed inputs
|
|
439
454
|
* admin tuning research-split [--since-hours 72] [--at <iso>,…] what each deploy did to research in production
|
|
440
455
|
* admin tuning research-standing live's level on the fixed inputs, week by week
|
|
456
|
+
*
|
|
457
|
+
* Shred (Phase 4): the same, for shredder versions on shred's fixed inputs.
|
|
458
|
+
* admin tuning shred-compare --title "…" (--candidate <version-id> | --patch '{…}' | --null-check) [--keys …]
|
|
459
|
+
* admin tuning shred-inputs [--set v1] | shred-split [--since-hours 72] [--at <iso>,…] | shred-standing
|
|
441
460
|
* admin tuning promote-arena <experiment-id> --version <arena-version-id> promote our arena from a "ship" decision
|
|
461
|
+
*
|
|
462
|
+
* Shred (Phase 4): our shredder's versions as files.
|
|
463
|
+
* admin tuning shredders | shredder <version-id> our shredder's versions / one version's file and diff
|
|
464
|
+
* admin tuning shredder-candidate --patch '{…}' | --file f.json [--of <version-id>] [--note "…"] save a candidate
|
|
465
|
+
* admin tuning promote-shredder <experiment-id> --version <shredder-version-id> promote our shredder from a "ship" decision (blind picks required)
|
|
442
466
|
*/
|
|
443
467
|
/** `--flag '<json>'` → the parsed object, or undefined when the flag is absent. */
|
|
444
468
|
function jsonFlag(raw, name) {
|
|
@@ -453,6 +477,44 @@ function jsonFlag(raw, name) {
|
|
|
453
477
|
}
|
|
454
478
|
}
|
|
455
479
|
|
|
480
|
+
/**
|
|
481
|
+
* A session as an agent reads it: each trial's changes and runs with their headline numbers, and
|
|
482
|
+
* (with `texts`) the latest finished text per trial. The full session (traces, records) is `--json`.
|
|
483
|
+
*/
|
|
484
|
+
function sessionSummary(session, texts) {
|
|
485
|
+
const runsOf = (trialId) => session.runs.filter((r) => r.trialId === trialId);
|
|
486
|
+
return {
|
|
487
|
+
id: session.id,
|
|
488
|
+
title: session.title,
|
|
489
|
+
kernel: session.baseKernelName,
|
|
490
|
+
base_version: session.baseVersion ?? "live",
|
|
491
|
+
material_ready: session.material?.ready ?? true,
|
|
492
|
+
trials: session.trials.map((t) => {
|
|
493
|
+
const runs = runsOf(t.id);
|
|
494
|
+
const latest = [...runs].reverse().find((r) => r.status === "done" && r.outputText);
|
|
495
|
+
return compact({
|
|
496
|
+
id: t.id,
|
|
497
|
+
name: t.name,
|
|
498
|
+
file_patch: t.filePatch,
|
|
499
|
+
run_options: t.runOptions,
|
|
500
|
+
saved_as_version: t.savedAsVersion ?? undefined,
|
|
501
|
+
runs: runs.map((r) =>
|
|
502
|
+
compact({
|
|
503
|
+
id: r.id,
|
|
504
|
+
status: r.status,
|
|
505
|
+
error: r.error ?? undefined,
|
|
506
|
+
quality: r.metrics?.qualityTotal,
|
|
507
|
+
quality_scale: r.metrics?.qualityScale,
|
|
508
|
+
p_machine: r.metrics?.pMachine,
|
|
509
|
+
words: r.metrics?.words,
|
|
510
|
+
})
|
|
511
|
+
),
|
|
512
|
+
latest_text: texts && latest ? latest.outputText : undefined,
|
|
513
|
+
});
|
|
514
|
+
}),
|
|
515
|
+
};
|
|
516
|
+
}
|
|
517
|
+
|
|
456
518
|
async function cmdAdminTuning(ctx) {
|
|
457
519
|
const [sub, id] = ctx.positionals;
|
|
458
520
|
const { flags, io } = ctx;
|
|
@@ -509,12 +571,17 @@ async function cmdAdminTuning(ctx) {
|
|
|
509
571
|
throw new CliError(
|
|
510
572
|
'Usage: admin tuning compare --title "…" --candidates Hazel:21,Aspen:20 [--experiment <id>]'
|
|
511
573
|
);
|
|
574
|
+
// kernel:version, or kernel:version@control to read it against a test version instead of live.
|
|
512
575
|
const candidates = list.map((item) => {
|
|
513
576
|
const at = item.lastIndexOf(":");
|
|
514
|
-
const
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
577
|
+
const [v, c] = item.slice(at + 1).split("@");
|
|
578
|
+
const version = Number(v);
|
|
579
|
+
const control = c === undefined ? undefined : Number(c);
|
|
580
|
+
if (at < 1 || !Number.isInteger(version) || (control !== undefined && !Number.isInteger(control)))
|
|
581
|
+
throw new CliError(
|
|
582
|
+
`--candidates items are kernel:version or kernel:version@control, got "${item}"`
|
|
583
|
+
);
|
|
584
|
+
return compact({ kernel: item.slice(0, at), candidate_version: version, control_version: control });
|
|
518
585
|
});
|
|
519
586
|
const r = await staffRequest(
|
|
520
587
|
ctx,
|
|
@@ -566,6 +633,190 @@ async function cmdAdminTuning(ctx) {
|
|
|
566
633
|
await staffRequest(ctx, "GET", `${T}/comparisons/${id}/pair?${new URLSearchParams({ cell })}`)
|
|
567
634
|
);
|
|
568
635
|
}
|
|
636
|
+
case "candidates": {
|
|
637
|
+
const patch = jsonFlag(flags.patch, "--patch");
|
|
638
|
+
const wanted = flagList(flags.kernels);
|
|
639
|
+
if (!patch || !wanted?.length)
|
|
640
|
+
throw new CliError(
|
|
641
|
+
'Usage: admin tuning candidates --kernels Hazel,Aspen|all --patch \'{"writer":"echo/echo"}\' [--base N] [--note "…"]'
|
|
642
|
+
);
|
|
643
|
+
const { kernels } = await staffRequest(ctx, "GET", `${T}/kernel-files`);
|
|
644
|
+
const all = wanted.length === 1 && wanted[0].toLowerCase() === "all";
|
|
645
|
+
const picked = all
|
|
646
|
+
? kernels
|
|
647
|
+
: wanted.map((name) => {
|
|
648
|
+
const k = kernels.find(
|
|
649
|
+
(x) => x.kernel.toLowerCase() === name.toLowerCase() || x.serviceKernelId === name
|
|
650
|
+
);
|
|
651
|
+
if (!k) throw new CliError(`No catalog kernel named ${name}.`);
|
|
652
|
+
return k;
|
|
653
|
+
});
|
|
654
|
+
const base = flagNum(flags.base);
|
|
655
|
+
const r = await staffRequest(
|
|
656
|
+
ctx,
|
|
657
|
+
"POST",
|
|
658
|
+
`${T}/kernel-files/candidates`,
|
|
659
|
+
compact({
|
|
660
|
+
targets: picked.map((k) => compact({ service_kernel_id: k.serviceKernelId, base })),
|
|
661
|
+
file_patch: patch,
|
|
662
|
+
note: flagStr(flags.note),
|
|
663
|
+
})
|
|
664
|
+
);
|
|
665
|
+
if (json) return printResult(io, flags, r);
|
|
666
|
+
return io.log(
|
|
667
|
+
r.outcomes
|
|
668
|
+
.map((o) =>
|
|
669
|
+
o.version
|
|
670
|
+
? `${(o.kernel ?? o.serviceKernelId).padEnd(12)} candidate v${o.version} (from v${o.base}): ${o.changes.join(", ") || "no change"}`
|
|
671
|
+
: `${(o.kernel ?? o.serviceKernelId).padEnd(12)} refused: ${o.error}`
|
|
672
|
+
)
|
|
673
|
+
.join("\n")
|
|
674
|
+
);
|
|
675
|
+
}
|
|
676
|
+
// --- Sessions: one input written many ways, side by side ---
|
|
677
|
+
case "base-kernels": {
|
|
678
|
+
const r = await staffRequest(ctx, "GET", `${T}/kernels`);
|
|
679
|
+
if (json) return printResult(io, flags, r);
|
|
680
|
+
return io.log(
|
|
681
|
+
r.kernels.map((k) => `${k.name.padEnd(28)} ${(k.orgName ?? "shared").padEnd(20)} ${k.id}`).join("\n")
|
|
682
|
+
);
|
|
683
|
+
}
|
|
684
|
+
case "base-versions": {
|
|
685
|
+
if (!id) throw new CliError("Usage: admin tuning base-versions <kernel-id>");
|
|
686
|
+
const r = await staffRequest(ctx, "GET", `${T}/kernels/${id}/versions`);
|
|
687
|
+
if (json) return printResult(io, flags, r);
|
|
688
|
+
return io.log(
|
|
689
|
+
r.versions
|
|
690
|
+
.map((v) => `v${v.version} ${v.status.padEnd(9)} ${v.writer}${v.note ? ` ${v.note}` : ""}`)
|
|
691
|
+
.join("\n") || "(no versions)"
|
|
692
|
+
);
|
|
693
|
+
}
|
|
694
|
+
case "sessions": {
|
|
695
|
+
const r = await staffRequest(ctx, "GET", `${T}?limit=${flagNum(flags.limit) ?? 30}`);
|
|
696
|
+
if (json) return printResult(io, flags, r);
|
|
697
|
+
return io.log(
|
|
698
|
+
r.sessions
|
|
699
|
+
.map(
|
|
700
|
+
(x) =>
|
|
701
|
+
`${x.id} ${x.title} · ${x.baseKernelName ?? "?"}${x.baseVersion ? ` v${x.baseVersion}` : ""} · ${x.trialCount} trials, ${x.runCount} runs`
|
|
702
|
+
)
|
|
703
|
+
.join("\n") || "(no sessions)"
|
|
704
|
+
);
|
|
705
|
+
}
|
|
706
|
+
case "session": {
|
|
707
|
+
if (!id) throw new CliError("Usage: admin tuning session <session-id> [--texts]");
|
|
708
|
+
const { session } = await staffRequest(ctx, "GET", `${T}/sessions/${id}`);
|
|
709
|
+
if (json) return printResult(io, flags, { session });
|
|
710
|
+
return printResult(io, { ...flags, json: true }, sessionSummary(session, flagBool(flags.texts)));
|
|
711
|
+
}
|
|
712
|
+
case "session-new": {
|
|
713
|
+
const title = flagStr(flags.title);
|
|
714
|
+
const kernel = flagStr(flags.kernel);
|
|
715
|
+
const path = flagStr(flags["input-file"]);
|
|
716
|
+
let input = flagStr(flags.input);
|
|
717
|
+
if (path) {
|
|
718
|
+
try {
|
|
719
|
+
input = readFileSync(path, "utf8");
|
|
720
|
+
} catch (err) {
|
|
721
|
+
throw new CliError(`--input-file ${path}: ${err.message}`);
|
|
722
|
+
}
|
|
723
|
+
}
|
|
724
|
+
if (!title || !kernel || !input)
|
|
725
|
+
throw new CliError(
|
|
726
|
+
'Usage: admin tuning session-new --title "…" --kernel <id|name> (--input-file f.md | --input "…") [--kind article|outline|brief] [--version N] [--writers a,b] [--trials-from <session-id>]'
|
|
727
|
+
);
|
|
728
|
+
let kernelId = kernel;
|
|
729
|
+
if (!/^[0-9a-f-]{36}$/i.test(kernel)) {
|
|
730
|
+
const { kernels } = await staffRequest(ctx, "GET", `${T}/kernels`);
|
|
731
|
+
const matches = kernels.filter((k) => k.name.toLowerCase() === kernel.toLowerCase());
|
|
732
|
+
const shared = matches.find((k) => !k.orgName) ?? matches[0];
|
|
733
|
+
if (!shared) throw new CliError(`No kernel named ${kernel} (admin tuning base-kernels lists them).`);
|
|
734
|
+
kernelId = shared.id;
|
|
735
|
+
}
|
|
736
|
+
const { session } = await staffRequest(
|
|
737
|
+
ctx,
|
|
738
|
+
"POST",
|
|
739
|
+
T,
|
|
740
|
+
compact({
|
|
741
|
+
title,
|
|
742
|
+
input_text: input,
|
|
743
|
+
input_kind: flagStr(flags.kind) ?? "unknown",
|
|
744
|
+
base_kernel_id: kernelId,
|
|
745
|
+
base_version: flagNum(flags.version),
|
|
746
|
+
trials_from_session_id: flagStr(flags["trials-from"]),
|
|
747
|
+
writers: (flagList(flags.writers) ?? []).map((model) => ({ model })),
|
|
748
|
+
})
|
|
749
|
+
);
|
|
750
|
+
if (json) return printResult(io, flags, { session });
|
|
751
|
+
return io.log(
|
|
752
|
+
`session ${session.id}: ${session.trials.length} trials on ${session.baseKernelName}${session.baseVersion ? ` v${session.baseVersion}` : " (live)"} → ${ctx.client.url}/superadmin/tuning/${session.id}`
|
|
753
|
+
);
|
|
754
|
+
}
|
|
755
|
+
case "propose": {
|
|
756
|
+
const text = flagStr(flags.text);
|
|
757
|
+
if (!id || !text)
|
|
758
|
+
throw new CliError(
|
|
759
|
+
'Usage: admin tuning propose <session-id> --text "Echo with rarity on; …" [--create]'
|
|
760
|
+
);
|
|
761
|
+
const r = await staffRequest(ctx, "POST", `${T}/sessions/${id}/propose`, { description: text });
|
|
762
|
+
if (flagBool(flags.create))
|
|
763
|
+
for (const t of r.trials)
|
|
764
|
+
await staffRequest(ctx, "POST", `${T}/sessions/${id}/trials`, {
|
|
765
|
+
name: t.name,
|
|
766
|
+
file_patch: t.file_patch,
|
|
767
|
+
run_options: t.run_options,
|
|
768
|
+
});
|
|
769
|
+
if (json) return printResult(io, flags, r);
|
|
770
|
+
return io.log(
|
|
771
|
+
[
|
|
772
|
+
...r.trials.map(
|
|
773
|
+
(t) => `${flagBool(flags.create) ? "created" : "proposed"} ${t.name}: ${t.changes.join(", ")}`
|
|
774
|
+
),
|
|
775
|
+
...r.unsupported.map((u) => `not expressible with the dials: ${u}`),
|
|
776
|
+
...r.rejected.map((x) => `dropped from ${x.trial}: ${x.dial} (${x.reason})`),
|
|
777
|
+
].join("\n") || "(nothing proposed)"
|
|
778
|
+
);
|
|
779
|
+
}
|
|
780
|
+
case "trial-add": {
|
|
781
|
+
if (!id)
|
|
782
|
+
throw new CliError(
|
|
783
|
+
"Usage: admin tuning trial-add <session-id> [--name] [--patch '{…}'] [--options '{…}'] [--from <trial-id>]"
|
|
784
|
+
);
|
|
785
|
+
const r = await staffRequest(
|
|
786
|
+
ctx,
|
|
787
|
+
"POST",
|
|
788
|
+
`${T}/sessions/${id}/trials`,
|
|
789
|
+
compact({
|
|
790
|
+
name: flagStr(flags.name),
|
|
791
|
+
file_patch: jsonFlag(flags.patch, "--patch"),
|
|
792
|
+
run_options: jsonFlag(flags.options, "--options"),
|
|
793
|
+
from_trial_id: flagStr(flags.from),
|
|
794
|
+
})
|
|
795
|
+
);
|
|
796
|
+
return json ? printResult(io, flags, r) : io.log(`trial ${r.trial.id} (${r.trial.name})`);
|
|
797
|
+
}
|
|
798
|
+
case "trial-run": {
|
|
799
|
+
if (!id) throw new CliError("Usage: admin tuning trial-run <trial-id> [--repeats N]");
|
|
800
|
+
const r = await staffRequest(ctx, "POST", `${T}/trials/${id}/run`, {
|
|
801
|
+
repeats: flagNum(flags.repeats) ?? 1,
|
|
802
|
+
});
|
|
803
|
+
return json
|
|
804
|
+
? printResult(io, flags, r)
|
|
805
|
+
: io.log(`queued ${r.runs.length} run(s): ${r.runs.map((x) => x.id).join(", ")}`);
|
|
806
|
+
}
|
|
807
|
+
case "trial-save": {
|
|
808
|
+
if (!id) throw new CliError('Usage: admin tuning trial-save <trial-id> [--note "…"]');
|
|
809
|
+
const r = await staffRequest(
|
|
810
|
+
ctx,
|
|
811
|
+
"POST",
|
|
812
|
+
`${T}/trials/${id}/candidate`,
|
|
813
|
+
compact({ note: flagStr(flags.note) })
|
|
814
|
+
);
|
|
815
|
+
if (json) return printResult(io, flags, r);
|
|
816
|
+
return io.log(
|
|
817
|
+
`${r.kernel}: saved as candidate v${r.version}${r.baseVersion ? ` (from v${r.baseVersion})` : ""}${r.leftBehind.length ? `; per-run, not saved: ${r.leftBehind.join(", ")}` : ""}`
|
|
818
|
+
);
|
|
819
|
+
}
|
|
569
820
|
// --- Research: our arena's versions, research comparisons, the deploy split ---
|
|
570
821
|
case "arenas": {
|
|
571
822
|
const r = await staffRequest(ctx, "GET", `${T}/arena-files`);
|
|
@@ -643,6 +894,56 @@ async function cmdAdminTuning(ctx) {
|
|
|
643
894
|
await staffRequest(ctx, "GET", `${T}/research/deploy-split${q.size ? `?${q}` : ""}`)
|
|
644
895
|
);
|
|
645
896
|
}
|
|
897
|
+
case "shred-compare": {
|
|
898
|
+
const title = flagStr(flags.title);
|
|
899
|
+
const versionId = flagStr(flags.candidate);
|
|
900
|
+
const patch = jsonFlag(flags.patch, "--patch");
|
|
901
|
+
const nullCheck = flagBool(flags["null-check"]);
|
|
902
|
+
const named = [versionId !== undefined, patch !== undefined, nullCheck === true].filter(Boolean).length;
|
|
903
|
+
if (!title || named !== 1)
|
|
904
|
+
throw new CliError(
|
|
905
|
+
"Usage: admin tuning shred-compare --title \"…\" (--candidate <version-id> | --patch '{…}' | --null-check) [--keys k1,k2] [--concurrency 6] [--experiment <id>]"
|
|
906
|
+
);
|
|
907
|
+
const candidate = versionId ? { version_id: versionId } : patch ? { patch } : { null_check: true };
|
|
908
|
+
const r = await staffRequest(
|
|
909
|
+
ctx,
|
|
910
|
+
"POST",
|
|
911
|
+
`${T}/comparisons`,
|
|
912
|
+
compact({
|
|
913
|
+
primitive: "shred",
|
|
914
|
+
title,
|
|
915
|
+
candidate,
|
|
916
|
+
keys: flagList(flags.keys),
|
|
917
|
+
concurrency: flagNum(flags.concurrency),
|
|
918
|
+
experiment_id: flagStr(flags.experiment),
|
|
919
|
+
})
|
|
920
|
+
);
|
|
921
|
+
return json ? printResult(io, flags, r) : io.log(`shred comparison ${r.comparison?.id ?? ""} started`);
|
|
922
|
+
}
|
|
923
|
+
case "shred-inputs": {
|
|
924
|
+
const set = flagStr(flags.set);
|
|
925
|
+
return printResult(
|
|
926
|
+
io,
|
|
927
|
+
{ ...flags, json: true },
|
|
928
|
+
await staffRequest(ctx, "GET", `${T}/shred/inputs${set ? `?${new URLSearchParams({ set })}` : ""}`)
|
|
929
|
+
);
|
|
930
|
+
}
|
|
931
|
+
case "shred-split": {
|
|
932
|
+
const q = new URLSearchParams(
|
|
933
|
+
compact({
|
|
934
|
+
since_hours: flagStr(flags["since-hours"]),
|
|
935
|
+
window_hours: flagStr(flags["window-hours"]),
|
|
936
|
+
at: flagStr(flags.at),
|
|
937
|
+
})
|
|
938
|
+
);
|
|
939
|
+
return printResult(
|
|
940
|
+
io,
|
|
941
|
+
{ ...flags, json: true },
|
|
942
|
+
await staffRequest(ctx, "GET", `${T}/shred/deploy-split${q.size ? `?${q}` : ""}`)
|
|
943
|
+
);
|
|
944
|
+
}
|
|
945
|
+
case "shred-standing":
|
|
946
|
+
return printResult(io, { ...flags, json: true }, await staffRequest(ctx, "GET", `${T}/shred/standing`));
|
|
646
947
|
case "research-standing":
|
|
647
948
|
return printResult(io, { ...flags, json: true }, await staffRequest(ctx, "GET", `${T}/research/standing`));
|
|
648
949
|
case "promote-arena": {
|
|
@@ -660,6 +961,56 @@ async function cmdAdminTuning(ctx) {
|
|
|
660
961
|
)
|
|
661
962
|
);
|
|
662
963
|
}
|
|
964
|
+
// --- Shred: our shredder's versions as files ---
|
|
965
|
+
case "shredders": {
|
|
966
|
+
const r = await staffRequest(ctx, "GET", `${T}/shredder-files`);
|
|
967
|
+
if (json) return printResult(io, flags, r);
|
|
968
|
+
return io.log(
|
|
969
|
+
r.versions
|
|
970
|
+
.map((v) => `${v.label.padEnd(18)} ${v.status.padEnd(9)} ${v.id}${v.note ? ` ${v.note}` : ""}`)
|
|
971
|
+
.join("\n") || "(no versions)"
|
|
972
|
+
);
|
|
973
|
+
}
|
|
974
|
+
case "shredder": {
|
|
975
|
+
if (!id) throw new CliError("Usage: admin tuning shredder <version-id>");
|
|
976
|
+
return printResult(
|
|
977
|
+
io,
|
|
978
|
+
{ ...flags, json: true },
|
|
979
|
+
await staffRequest(ctx, "GET", `${T}/shredder-files/${id}`)
|
|
980
|
+
);
|
|
981
|
+
}
|
|
982
|
+
case "shredder-candidate": {
|
|
983
|
+
const patch = jsonFlag(flags.patch, "--patch");
|
|
984
|
+
const file = jsonFile({ "shredder-file": flags.file }, "shredder");
|
|
985
|
+
if ((patch === undefined) === (file === undefined))
|
|
986
|
+
throw new CliError(
|
|
987
|
+
'Usage: admin tuning shredder-candidate --patch \'{"rewriters":{"excludeWriterFamily":true}}\' | --file shredder.json [--of <version-id>] [--note "…"]'
|
|
988
|
+
);
|
|
989
|
+
const r = await staffRequest(
|
|
990
|
+
ctx,
|
|
991
|
+
"POST",
|
|
992
|
+
`${T}/shredder-files`,
|
|
993
|
+
compact({ patch, file, patch_of: flagStr(flags.of), note: flagStr(flags.note) })
|
|
994
|
+
);
|
|
995
|
+
return json ? printResult(io, flags, r) : io.log(`saved candidate ${r.version.label} (${r.version.id})`);
|
|
996
|
+
}
|
|
997
|
+
case "promote-shredder": {
|
|
998
|
+
const version = flagStr(flags.version);
|
|
999
|
+
if (!id || !version)
|
|
1000
|
+
throw new CliError(
|
|
1001
|
+
"Usage: admin tuning promote-shredder <experiment-id> --version <shredder-version-id>"
|
|
1002
|
+
);
|
|
1003
|
+
return printResult(
|
|
1004
|
+
io,
|
|
1005
|
+
{ ...flags, json: true },
|
|
1006
|
+
await staffRequest(
|
|
1007
|
+
ctx,
|
|
1008
|
+
"POST",
|
|
1009
|
+
`${EXPERIMENTS}/${id}/promote`,
|
|
1010
|
+
compact({ shredder_version_id: version, actor: actorFrom(flags) })
|
|
1011
|
+
)
|
|
1012
|
+
);
|
|
1013
|
+
}
|
|
663
1014
|
case "promote": {
|
|
664
1015
|
const list = flagList(flags.versions);
|
|
665
1016
|
if (!id || !list?.length)
|
|
@@ -683,7 +1034,7 @@ async function cmdAdminTuning(ctx) {
|
|
|
683
1034
|
}
|
|
684
1035
|
default:
|
|
685
1036
|
throw new CliError(
|
|
686
|
-
"Usage: admin tuning kernels|kernel|candidate|compare|comparisons|comparison|picks|pick|reading-list|pair|promote|arenas|arena|arena-candidate|research-compare|research-inputs|research-split|research-standing|promote-arena (see --help)"
|
|
1037
|
+
"Usage: admin tuning kernels|kernel|candidate|compare|comparisons|comparison|picks|pick|reading-list|pair|candidates|base-kernels|base-versions|sessions|session|session-new|propose|trial-add|trial-run|trial-save|promote|arenas|arena|arena-candidate|research-compare|research-inputs|research-split|research-standing|promote-arena|shred-compare|shred-inputs|shred-split|shred-standing|shredders|shredder|shredder-candidate|promote-shredder (see --help)"
|
|
687
1038
|
);
|
|
688
1039
|
}
|
|
689
1040
|
}
|
package/lib/commands/evals.mjs
CHANGED
|
@@ -16,7 +16,7 @@
|
|
|
16
16
|
import { CliError } from "../client.mjs";
|
|
17
17
|
import { flagStr, readBodyInput, printResult, ok, argsFromFlags } from "./shared.mjs";
|
|
18
18
|
|
|
19
|
-
const STAGES = ["research", "brief", "links", "mentions", "accuracy", "pipeline"];
|
|
19
|
+
const STAGES = ["research", "brief", "links", "mentions", "accuracy", "pipeline", "shred"];
|
|
20
20
|
const STATUSES = ["proposed", "open", "guarding", "retired", "rejected"];
|
|
21
21
|
|
|
22
22
|
/** `--keys a,b,c` (or repeated `--keys`) → ["a", "b", "c"]; undefined when unset. */
|
|
@@ -35,6 +35,23 @@ function readJsonFileFlag(flags, flag) {
|
|
|
35
35
|
}
|
|
36
36
|
}
|
|
37
37
|
|
|
38
|
+
/** `--<flag> '<json>'` inline, or `--<flag>-file <path>` → a JSON object, or undefined. */
|
|
39
|
+
function readJsonObjectFlag(flags, name) {
|
|
40
|
+
const inline = flagStr(flags[name]);
|
|
41
|
+
if (inline !== undefined) {
|
|
42
|
+
let parsed;
|
|
43
|
+
try {
|
|
44
|
+
parsed = JSON.parse(inline);
|
|
45
|
+
} catch (err) {
|
|
46
|
+
throw new CliError(`--${name} is not valid JSON: ${err.message}`);
|
|
47
|
+
}
|
|
48
|
+
if (parsed === null || typeof parsed !== "object" || Array.isArray(parsed))
|
|
49
|
+
throw new CliError(`--${name} must be a JSON object`);
|
|
50
|
+
return parsed;
|
|
51
|
+
}
|
|
52
|
+
return readJsonFileFlag(flags, `${name}-file`);
|
|
53
|
+
}
|
|
54
|
+
|
|
38
55
|
/**
|
|
39
56
|
* How every command in this file sources its tool's inputs — see `argsFromFlags` in
|
|
40
57
|
* shared.mjs and `scripts/parity-check.ts`, which diffs these keys against each tool's
|
|
@@ -68,6 +85,10 @@ export const ARGS = {
|
|
|
68
85
|
},
|
|
69
86
|
list_shredder_endpoints: {},
|
|
70
87
|
get_shred_balance: {},
|
|
88
|
+
// The rewriter families behind the shredder, as the platform's hourly watch reads them.
|
|
89
|
+
get_shred_family_health: {
|
|
90
|
+
hours: { flag: "hours", type: "num" },
|
|
91
|
+
},
|
|
71
92
|
create_shredder_endpoint: {
|
|
72
93
|
name: { flag: "name", required: true },
|
|
73
94
|
},
|
|
@@ -83,6 +104,33 @@ export const ARGS = {
|
|
|
83
104
|
endpoint_id: { positional: 1, label: "endpoint-id", required: true },
|
|
84
105
|
config: { value: (flags) => readJsonFileFlag(flags, "file") },
|
|
85
106
|
},
|
|
107
|
+
// The shredder as a versioned file (the arena file's twin): `shredder file <id>` prints it,
|
|
108
|
+
// `--file f.json` saves a whole file, `--patch '{…}'` only the fields to change.
|
|
109
|
+
get_shredder_file: {
|
|
110
|
+
endpoint_id: { positional: 0, label: "endpoint-id", required: true },
|
|
111
|
+
},
|
|
112
|
+
save_shredder_file: {
|
|
113
|
+
endpoint_id: { positional: 0, label: "endpoint-id", required: true },
|
|
114
|
+
file: { value: (flags) => readJsonFileFlag(flags, "file") },
|
|
115
|
+
patch: { value: (flags) => readJsonObjectFlag(flags, "patch") },
|
|
116
|
+
note: "note",
|
|
117
|
+
},
|
|
118
|
+
list_shredder_versions: {
|
|
119
|
+
endpoint_id: { positional: 0, label: "endpoint-id", required: true },
|
|
120
|
+
},
|
|
121
|
+
pin_shredder_version: {
|
|
122
|
+
endpoint_id: { positional: 0, label: "endpoint-id", required: true },
|
|
123
|
+
version_id: {
|
|
124
|
+
value: (_flags, positionals) => {
|
|
125
|
+
const v = positionals[1];
|
|
126
|
+
if (v === undefined) throw new CliError("Missing required <version-id|none>");
|
|
127
|
+
return v === "none" ? null : v;
|
|
128
|
+
},
|
|
129
|
+
},
|
|
130
|
+
},
|
|
131
|
+
accept_shredder_offer: {
|
|
132
|
+
endpoint_id: { positional: 0, label: "endpoint-id", required: true },
|
|
133
|
+
},
|
|
86
134
|
};
|
|
87
135
|
|
|
88
136
|
export async function cmdShredder(ctx) {
|
|
@@ -111,9 +159,19 @@ export async function cmdShredder(ctx) {
|
|
|
111
159
|
return shredderRuns(rest);
|
|
112
160
|
case "balance":
|
|
113
161
|
return shredderBalance(rest);
|
|
162
|
+
case "file":
|
|
163
|
+
return shredderFile(rest);
|
|
164
|
+
case "versions":
|
|
165
|
+
return shredderTool(rest, "list_shredder_versions");
|
|
166
|
+
case "pin":
|
|
167
|
+
return shredderTool(rest, "pin_shredder_version");
|
|
168
|
+
case "offer":
|
|
169
|
+
return rest.flags.accept === true ? shredderTool(rest, "accept_shredder_offer") : shredderOffer(rest);
|
|
170
|
+
case "families":
|
|
171
|
+
return shredderTool(rest, "get_shred_family_health");
|
|
114
172
|
default:
|
|
115
173
|
throw new CliError(
|
|
116
|
-
`Unknown shredder subcommand: ${sub ?? "(none)"}. Try: list, create, delete, endpoint, run, job, runs, balance`
|
|
174
|
+
`Unknown shredder subcommand: ${sub ?? "(none)"}. Try: list, create, delete, endpoint, run, job, runs, balance, file, versions, pin, offer, families`
|
|
117
175
|
);
|
|
118
176
|
}
|
|
119
177
|
}
|
|
@@ -284,3 +342,54 @@ async function shredderBalance(ctx) {
|
|
|
284
342
|
printResult(io, flags, result);
|
|
285
343
|
return 0;
|
|
286
344
|
}
|
|
345
|
+
|
|
346
|
+
/**
|
|
347
|
+
* `shredder file <endpoint-id>` prints the file the endpoint shreds with (the file itself, so
|
|
348
|
+
* `> f.json` round-trips); `shredder file <endpoint-id> --file f.json | --patch '{…}' [--note …]`
|
|
349
|
+
* saves a new live version of the endpoint's own copy.
|
|
350
|
+
*/
|
|
351
|
+
async function shredderFile(ctx) {
|
|
352
|
+
const { client, flags, io } = ctx;
|
|
353
|
+
const saving =
|
|
354
|
+
flagStr(flags.file) !== undefined ||
|
|
355
|
+
flagStr(flags.patch) !== undefined ||
|
|
356
|
+
flagStr(flags["patch-file"]) !== undefined;
|
|
357
|
+
if (!saving) {
|
|
358
|
+
const result = await client.callTool(
|
|
359
|
+
"get_shredder_file",
|
|
360
|
+
argsFromFlags(ARGS.get_shredder_file, flags, ctx.positionals)
|
|
361
|
+
);
|
|
362
|
+
printResult(io, flags, result, (r) => JSON.stringify(r?.file ?? r, null, 2));
|
|
363
|
+
return 0;
|
|
364
|
+
}
|
|
365
|
+
const result = await client.callTool(
|
|
366
|
+
"save_shredder_file",
|
|
367
|
+
argsFromFlags(ARGS.save_shredder_file, flags, ctx.positionals)
|
|
368
|
+
);
|
|
369
|
+
ok(
|
|
370
|
+
ctx,
|
|
371
|
+
`Saved ${toolField(result, "version")?.label ?? "a new version"}; the endpoint shreds with it from now on.`
|
|
372
|
+
);
|
|
373
|
+
printResult(io, flags, result);
|
|
374
|
+
return 0;
|
|
375
|
+
}
|
|
376
|
+
|
|
377
|
+
/** `shredder offer <endpoint-id>`: what our newer version would change; `--accept` takes it. */
|
|
378
|
+
async function shredderOffer(ctx) {
|
|
379
|
+
const { client, flags, io } = ctx;
|
|
380
|
+
const result = await client.callTool(
|
|
381
|
+
"get_shredder_file",
|
|
382
|
+
argsFromFlags(ARGS.get_shredder_file, flags, ctx.positionals)
|
|
383
|
+
);
|
|
384
|
+
const offer = toolField(result, "offer");
|
|
385
|
+
if (!offer) ok(ctx, "No offer: your endpoint shreds with our latest version, or a copy based on it.");
|
|
386
|
+
printResult(io, flags, result, (r) => JSON.stringify(r?.offer ?? null, null, 2));
|
|
387
|
+
return 0;
|
|
388
|
+
}
|
|
389
|
+
|
|
390
|
+
/** One shredder-file tool, its arguments from ARGS. */
|
|
391
|
+
async function shredderTool(ctx, tool) {
|
|
392
|
+
const result = await ctx.client.callTool(tool, argsFromFlags(ARGS[tool], ctx.flags, ctx.positionals));
|
|
393
|
+
printResult(ctx.io, ctx.flags, result);
|
|
394
|
+
return 0;
|
|
395
|
+
}
|