@basein/runner 0.3.27 → 0.3.28
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +13 -0
- package/dist/bin/bir-scenario.js +7 -5
- package/dist/bin/bir.js +36 -5
- package/dist/bin/scenario-edit.d.ts +102 -1
- package/dist/bin/scenario-edit.js +318 -23
- package/dist/control/server.d.ts +19 -0
- package/dist/control/server.js +80 -2
- package/dist/control/transcript.d.ts +9 -0
- package/dist/control/transcript.js +56 -9
- package/dist/record/recorder.d.ts +50 -0
- package/dist/record/remote-recorder.d.ts +9 -1
- package/dist/record/remote-recorder.js +38 -0
- package/dist/replay/controller.d.ts +41 -1
- package/dist/replay/controller.js +109 -4
- package/docs/calculatedReplayGuide.md +74 -1
- package/package.json +1 -1
|
@@ -8,8 +8,9 @@
|
|
|
8
8
|
* [--freeze | --unfreeze] [--note "why"] [--force --note "why"] [--revision <n>]
|
|
9
9
|
* bir scenario edit <runId|scnId> --param <name> --kind target|setting [--note "why"] [--revision <n>]
|
|
10
10
|
* bir scenario edits <runId|scnId>
|
|
11
|
-
* bir scenario undo <runId|scnId> [--edit <sedit_id|pedit_id> | --step <n> | --param <name>] [--force --note "why"]
|
|
11
|
+
* bir scenario undo <runId|scnId> [--edit <sedit_id|pedit_id|scor_id|simp_id> | --step <n> | --param <name>] [--force --note "why"]
|
|
12
12
|
* bir scenario calc <runId|scnId> [--force [--discard-edits]]
|
|
13
|
+
* bir scenario learning <runId|scnId> [--runs <n>] [--stop] [--again]
|
|
13
14
|
* bir scenario editing on|off|status
|
|
14
15
|
*
|
|
15
16
|
* WHY THIS IS A CLIENT AND NOTHING MORE. Whether a change may be saved is
|
|
@@ -47,6 +48,7 @@ export const SCENARIO_EDIT_SUBCOMMANDS = new Set([
|
|
|
47
48
|
"edits",
|
|
48
49
|
"undo",
|
|
49
50
|
"calc",
|
|
51
|
+
"learning",
|
|
50
52
|
"editing",
|
|
51
53
|
]);
|
|
52
54
|
/**
|
|
@@ -73,8 +75,9 @@ const USAGE = {
|
|
|
73
75
|
' [--freeze | --unfreeze] [--note "why"] [--force --note "why"] [--revision <n>] [--json]\n' +
|
|
74
76
|
' bir scenario edit <runId|scnId> --param <name> --kind target|setting [--note "why"] [--revision <n>] [--json]',
|
|
75
77
|
edits: "usage: bir scenario edits <runId|scnId> [--json]",
|
|
76
|
-
undo: 'usage: bir scenario undo <runId|scnId> [--edit <sedit_id|pedit_id|scor_id> | --step <n> | --param <name>] [--force --note "why"] [--json]',
|
|
78
|
+
undo: 'usage: bir scenario undo <runId|scnId> [--edit <sedit_id|pedit_id|scor_id|simp_id> | --step <n> | --param <name>] [--force --note "why"] [--json]',
|
|
77
79
|
calc: "usage: bir scenario calc <runId|scnId> [--force [--discard-edits]]",
|
|
80
|
+
learning: "usage: bir scenario learning <runId|scnId> [--runs <n>] [--stop] [--again] [--json]",
|
|
78
81
|
editing: "usage: bir scenario editing on|off|status",
|
|
79
82
|
};
|
|
80
83
|
export async function scenarioEditCommand(args, deps) {
|
|
@@ -91,10 +94,12 @@ export async function scenarioEditCommand(args, deps) {
|
|
|
91
94
|
return undo(args, deps);
|
|
92
95
|
case "calc":
|
|
93
96
|
return calc(args, deps);
|
|
97
|
+
case "learning":
|
|
98
|
+
return learning(args, deps);
|
|
94
99
|
case "editing":
|
|
95
100
|
return editing(args, deps);
|
|
96
101
|
default:
|
|
97
|
-
deps.out("usage: bir scenario <list|show|calc|check|edit|edits|undo|editing|replay> …");
|
|
102
|
+
deps.out("usage: bir scenario <list|show|calc|check|edit|edits|undo|editing|learning|replay> …");
|
|
98
103
|
return 2;
|
|
99
104
|
}
|
|
100
105
|
}
|
|
@@ -124,6 +129,13 @@ async function show(args, deps) {
|
|
|
124
129
|
}
|
|
125
130
|
if (args.step === undefined) {
|
|
126
131
|
out(jsonText(reply.body));
|
|
132
|
+
// Under the plan, the learning phase in one line (the service's
|
|
133
|
+
// learningPhase.md LP9) — for a person. Never with --json: scripts and
|
|
134
|
+
// the `bir` MCP server read this output as JSON, which carries the same
|
|
135
|
+
// facts as `learning`.
|
|
136
|
+
const line = args.json ? undefined : learningOfShown(reply.body);
|
|
137
|
+
if (line)
|
|
138
|
+
out(line);
|
|
127
139
|
return 0;
|
|
128
140
|
}
|
|
129
141
|
const served = reply.body;
|
|
@@ -618,17 +630,18 @@ function refusal(out, report, opening, hint) {
|
|
|
618
630
|
}
|
|
619
631
|
/**
|
|
620
632
|
* The service's history: step edits, and — from a service that has them —
|
|
621
|
-
* kind changes
|
|
622
|
-
*
|
|
633
|
+
* kind changes, corrections (autocorrect.md AC7) and the learning phase's
|
|
634
|
+
* changes (learningPhase.md LP9). An older service sends none of those lists,
|
|
635
|
+
* and there is nothing of them to print.
|
|
623
636
|
*/
|
|
624
637
|
function historyOf(body) {
|
|
625
638
|
const b = body;
|
|
639
|
+
const entries = (list) => Array.isArray(list) ? list.filter((c) => typeof c?.id === "string" && typeof c?.createdAt === "string") : [];
|
|
626
640
|
return {
|
|
627
641
|
edits: Array.isArray(b?.edits) ? b.edits : [],
|
|
628
642
|
paramEdits: Array.isArray(b?.paramEdits) ? b.paramEdits : [],
|
|
629
|
-
corrections:
|
|
630
|
-
|
|
631
|
-
: [],
|
|
643
|
+
corrections: entries(b?.corrections),
|
|
644
|
+
learning: entries(b?.learning),
|
|
632
645
|
};
|
|
633
646
|
}
|
|
634
647
|
async function fetchEdits(scenarioId, deps) {
|
|
@@ -656,16 +669,16 @@ async function listEdits(args, deps) {
|
|
|
656
669
|
out(jsonText(reply.body));
|
|
657
670
|
return 0;
|
|
658
671
|
}
|
|
659
|
-
const { edits, paramEdits, corrections } = historyOf(reply.body);
|
|
660
|
-
if (edits.length === 0 && paramEdits.length === 0 && corrections.length === 0) {
|
|
672
|
+
const { edits, paramEdits, corrections, learning } = historyOf(reply.body);
|
|
673
|
+
if (edits.length === 0 && paramEdits.length === 0 && corrections.length === 0 && learning.length === 0) {
|
|
661
674
|
out(`No hand edits of ${target}.`);
|
|
662
675
|
return 0;
|
|
663
676
|
}
|
|
664
677
|
// Ids are printed whole: they are what `undo --edit` takes. The detail line
|
|
665
678
|
// is indented rather than aligned under the date, which a 42-character id
|
|
666
679
|
// would push past most terminals' width. One list, newest first: a step's
|
|
667
|
-
// edits, a parameter's kind changes
|
|
668
|
-
//
|
|
680
|
+
// edits, a parameter's kind changes, the service's corrections of the plan
|
|
681
|
+
// and its learning phase's changes are walked back by the same `undo`.
|
|
669
682
|
const noteOf = (note) => (note ? ` "${note.replace(/\s+/g, " ")}"` : "");
|
|
670
683
|
const rows = [
|
|
671
684
|
...edits.map((e) => ({
|
|
@@ -683,6 +696,11 @@ async function listEdits(args, deps) {
|
|
|
683
696
|
line: `${c.id} ${when(c.createdAt)} correction ${String(c.state).padEnd(8)} revision ${c.chainRevisionBefore ?? "?"} → ${c.chainRevisionAfter ?? "—"}`,
|
|
684
697
|
detail: correctionSummary(c),
|
|
685
698
|
})),
|
|
699
|
+
...learning.map((c) => ({
|
|
700
|
+
at: c.createdAt,
|
|
701
|
+
line: `${c.id} ${when(c.createdAt)} learning ${String(c.state).padEnd(STATE_WIDTH)} revision ${c.revisionBefore ?? "?"} → ${c.revisionAfter ?? "—"}`,
|
|
702
|
+
detail: learningEditSummary(c),
|
|
703
|
+
})),
|
|
686
704
|
].sort((x, y) => (x.at < y.at ? 1 : x.at > y.at ? -1 : 0));
|
|
687
705
|
for (const r of rows) {
|
|
688
706
|
out(r.line);
|
|
@@ -828,13 +846,15 @@ export function newestUndoableCorrection(corrections) {
|
|
|
828
846
|
}
|
|
829
847
|
const isParamEditId = (id) => id.startsWith("pedit_");
|
|
830
848
|
const isCorrectionId = (id) => id.startsWith("scor_");
|
|
849
|
+
/** A change a learning phase made (the service's learningPhase.md LP9). */
|
|
850
|
+
const isLearningId = (id) => id.startsWith("simp_");
|
|
831
851
|
async function undo(args, deps) {
|
|
832
852
|
const { out } = deps;
|
|
833
853
|
const target = args.positionals[1];
|
|
834
854
|
if (!target)
|
|
835
855
|
return usageError(deps, "undo");
|
|
836
|
-
if (args.edit !== undefined && !/^(sedit|pedit|scor)_[\w-]+$/.test(args.edit)) {
|
|
837
|
-
return usageError(deps, "undo", `--edit takes a sedit_…, pedit_… or scor_… id, as \`bir scenario edits\` prints it; got ${args.edit}.`);
|
|
856
|
+
if (args.edit !== undefined && !/^(sedit|pedit|scor|simp)_[\w-]+$/.test(args.edit)) {
|
|
857
|
+
return usageError(deps, "undo", `--edit takes a sedit_…, pedit_… or scor_… id, or a learning change's simp_… id, as \`bir scenario edits\` prints it; got ${args.edit}.`);
|
|
838
858
|
}
|
|
839
859
|
const picks = [args.edit, args.step, args.param].filter((v) => v !== undefined).length;
|
|
840
860
|
if (picks > 1) {
|
|
@@ -872,11 +892,18 @@ async function undo(args, deps) {
|
|
|
872
892
|
];
|
|
873
893
|
const pick = candidates.sort((x, y) => (x.at < y.at ? 1 : x.at > y.at ? -1 : 0))[0];
|
|
874
894
|
if (!pick) {
|
|
895
|
+
// A learning phase's change is the service's, and is taken back by its
|
|
896
|
+
// id alone (learningPhase.md LP9): a plain undo leaves it be, and says
|
|
897
|
+
// how, when one is in use.
|
|
898
|
+
const learned = args.step === undefined && args.param === undefined ? newestUsedLearning(listed.learning) : undefined;
|
|
875
899
|
return localFailure(args, deps, "nothing_to_undo", args.step !== undefined
|
|
876
900
|
? `Nothing to undo: step ${args.step} of ${target} has no hand edit that can be undone.`
|
|
877
901
|
: args.param !== undefined
|
|
878
902
|
? `Nothing to undo: parameter ${args.param} of ${target} has no kind change that can be undone.`
|
|
879
|
-
: `Nothing to undo: ${target} has no hand edit that can be undone.`
|
|
903
|
+
: `Nothing to undo: ${target} has no hand edit that can be undone.` +
|
|
904
|
+
(learned
|
|
905
|
+
? ` Its learning phase's change ${learned.id} (${learned.state}) is taken back by its id: bir scenario undo ${target} --edit ${learned.id}`
|
|
906
|
+
: ""));
|
|
880
907
|
}
|
|
881
908
|
editId = pick.id;
|
|
882
909
|
}
|
|
@@ -907,6 +934,8 @@ async function undo(args, deps) {
|
|
|
907
934
|
return paramUndone(reply.body, editId, target, args, deps);
|
|
908
935
|
if (isCorrectionId(editId))
|
|
909
936
|
return correctionUndone(reply.body, editId, target, deps);
|
|
937
|
+
if (isLearningId(editId))
|
|
938
|
+
return learningUndone(reply.body, editId, target, deps);
|
|
910
939
|
if (answer?.report)
|
|
911
940
|
renderReport(out, answer.report, { brief: true, undo: true });
|
|
912
941
|
if (refused && answer?.report) {
|
|
@@ -964,6 +993,250 @@ function paramUndone(answer, editId, target, args, deps) {
|
|
|
964
993
|
deps.out(` One further back: bir scenario undo ${target}${further}`);
|
|
965
994
|
return 0;
|
|
966
995
|
}
|
|
996
|
+
/** A whole count the service sent, or null for anything else. */
|
|
997
|
+
const count = (value) => typeof value === "number" && Number.isFinite(value) && value >= 0 ? Math.round(value) : null;
|
|
998
|
+
const runsWord = (n) => `${n} run${n === 1 ? "" : "s"}`;
|
|
999
|
+
/** One line of the service's text: no line breaks, no runs of spaces. */
|
|
1000
|
+
const oneLine = (text) => (typeof text === "string" ? text.replace(/\s+/g, " ").trim() : "");
|
|
1001
|
+
/**
|
|
1002
|
+
* The learning phase in one line (the service's learningPhase.md LP1, LP9):
|
|
1003
|
+
* "Learning: 4 of 10 runs evaluated · change 2 on trial (1 of 2 runs)",
|
|
1004
|
+
* "Learning done after 10 runs", "Learning stopped after 4 runs", or "No
|
|
1005
|
+
* learning phase". `changes`, when known, give the change on trial its
|
|
1006
|
+
* number, oldest first; `waiting`, the changes that wait for their owner.
|
|
1007
|
+
*/
|
|
1008
|
+
export function learningLine(l, changes, waiting) {
|
|
1009
|
+
if (!l || (l.state !== "learning" && l.state !== "done" && l.state !== "stopped"))
|
|
1010
|
+
return "No learning phase";
|
|
1011
|
+
const evaluated = count(l.evaluated) ?? 0;
|
|
1012
|
+
if (l.state === "done")
|
|
1013
|
+
return `Learning done after ${runsWord(evaluated)}`;
|
|
1014
|
+
if (l.state === "stopped")
|
|
1015
|
+
return `Learning stopped after ${runsWord(evaluated)}`;
|
|
1016
|
+
const parts = [`Learning: ${evaluated} of ${count(l.runs) ?? "?"} runs evaluated`];
|
|
1017
|
+
const trial = l.trial && typeof l.trial === "object" ? l.trial : undefined;
|
|
1018
|
+
if (trial) {
|
|
1019
|
+
const at = changes ? changes.findIndex((c) => c?.id === trial.changeId) : -1;
|
|
1020
|
+
const needed = count(trial.needed);
|
|
1021
|
+
parts.push(`${at >= 0 ? `change ${at + 1}` : "a change"} on trial` +
|
|
1022
|
+
(needed !== null ? ` (${count(trial.done) ?? 0} of ${runsWord(needed)})` : ""));
|
|
1023
|
+
}
|
|
1024
|
+
const wait = count(waiting);
|
|
1025
|
+
if (wait)
|
|
1026
|
+
parts.push(`${wait} change${wait === 1 ? "" : "s"} waiting for you`);
|
|
1027
|
+
return parts.join(" · ");
|
|
1028
|
+
}
|
|
1029
|
+
/** "cost 40% under the original", from a ratio to the original; undefined for none. */
|
|
1030
|
+
function ratioWords(what, ratio) {
|
|
1031
|
+
if (typeof ratio !== "number" || !Number.isFinite(ratio) || ratio < 0)
|
|
1032
|
+
return undefined;
|
|
1033
|
+
const pct = Math.round(Math.abs(1 - ratio) * 100);
|
|
1034
|
+
if (pct === 0)
|
|
1035
|
+
return `${what} the same as the original`;
|
|
1036
|
+
return `${what} ${pct}% ${ratio < 1 ? "under" : "over"} the original`;
|
|
1037
|
+
}
|
|
1038
|
+
/**
|
|
1039
|
+
* A revision's three measures in one line, leaving out what is not known:
|
|
1040
|
+
* "Successful 4 of 6 · cost 40% under the original · time 35% under the
|
|
1041
|
+
* original". Undefined when none is.
|
|
1042
|
+
*/
|
|
1043
|
+
export function measuresLine(m) {
|
|
1044
|
+
if (!m || typeof m !== "object")
|
|
1045
|
+
return undefined;
|
|
1046
|
+
const parts = [];
|
|
1047
|
+
const evaluated = count(m.evaluated);
|
|
1048
|
+
const successful = count(m.successful);
|
|
1049
|
+
if (evaluated !== null && evaluated > 0 && successful !== null)
|
|
1050
|
+
parts.push(`Successful ${successful} of ${evaluated}`);
|
|
1051
|
+
const cost = ratioWords("cost", m.costRatio);
|
|
1052
|
+
if (cost)
|
|
1053
|
+
parts.push(cost);
|
|
1054
|
+
const time = ratioWords("time", m.speedRatio);
|
|
1055
|
+
if (time)
|
|
1056
|
+
parts.push(time);
|
|
1057
|
+
return parts.length > 0 ? parts.join(" · ") : undefined;
|
|
1058
|
+
}
|
|
1059
|
+
/** The width a learning change's state is padded to, beside its id, in `learning` (and `edits`). */
|
|
1060
|
+
const STATE_WIDTH = 12;
|
|
1061
|
+
/** The scenario's learning line for `show`: from `scenario.learning`, when the service sent one. */
|
|
1062
|
+
function learningOfShown(body) {
|
|
1063
|
+
const scenario = body?.scenario;
|
|
1064
|
+
const brief = scenario?.learning;
|
|
1065
|
+
if (!brief || typeof brief !== "object")
|
|
1066
|
+
return undefined;
|
|
1067
|
+
if (brief.state !== "learning" && brief.state !== "done" && brief.state !== "stopped")
|
|
1068
|
+
return undefined;
|
|
1069
|
+
const view = scenario?.learningView;
|
|
1070
|
+
return learningLine(brief, Array.isArray(view?.changes) ? view.changes : undefined, brief.waiting);
|
|
1071
|
+
}
|
|
1072
|
+
/**
|
|
1073
|
+
* The learning phase as a person reads it: its line, the three measures of
|
|
1074
|
+
* the revision the plan is at, the problems the last evaluated run showed,
|
|
1075
|
+
* and the changes — each with its id, which `bir scenario undo --edit` takes.
|
|
1076
|
+
*/
|
|
1077
|
+
function renderLearning(out, view, target) {
|
|
1078
|
+
const changes = Array.isArray(view.changes) ? view.changes.filter((c) => c && typeof c.id === "string") : [];
|
|
1079
|
+
const waiting = changes.filter((c) => c.state === "proposed").length;
|
|
1080
|
+
out(learningLine(view, changes, waiting));
|
|
1081
|
+
if (view.state !== "learning" && view.state !== "done" && view.state !== "stopped") {
|
|
1082
|
+
out(view.mode === "off"
|
|
1083
|
+
? " The service's learning phase is switched off (SCENARIO_LEARNING=off): no plan learns."
|
|
1084
|
+
: ` To start one: bir scenario learning ${target} --again [--runs <n>]`);
|
|
1085
|
+
return;
|
|
1086
|
+
}
|
|
1087
|
+
const revision = count(view.revision);
|
|
1088
|
+
const measures = measuresLine(view.measures?.current);
|
|
1089
|
+
out(` ${revision !== null ? `Revision ${revision}` : "Now"}: ${measures ?? "no run of it evaluated yet"}`);
|
|
1090
|
+
const problems = Array.isArray(view.problems) ? view.problems.filter((p) => p && typeof p.text === "string") : [];
|
|
1091
|
+
if (problems.length > 0) {
|
|
1092
|
+
out(" Found in the last run:");
|
|
1093
|
+
for (const p of problems) {
|
|
1094
|
+
const step = count(p.step);
|
|
1095
|
+
out(` ${p.code ?? "?"} ${oneLine(p.text)}${step !== null ? ` (step ${step})` : ""}`);
|
|
1096
|
+
}
|
|
1097
|
+
}
|
|
1098
|
+
if (changes.length > 0) {
|
|
1099
|
+
out(" Changes:");
|
|
1100
|
+
for (const c of changes) {
|
|
1101
|
+
const decision = oneLine(c.decision);
|
|
1102
|
+
out(` ${c.id} ${String(c.state ?? "?").padEnd(STATE_WIDTH)} ${String(c.aim ?? "?").padEnd(10)} ${oneLine(c.summary)}` +
|
|
1103
|
+
(decision ? ` — ${decision}` : ""));
|
|
1104
|
+
}
|
|
1105
|
+
if (waiting > 0)
|
|
1106
|
+
out(' A proposed change is tried when you press "Use" on the recording page.');
|
|
1107
|
+
}
|
|
1108
|
+
}
|
|
1109
|
+
/** The second line of a learning change in `edits`: what it did, what for, and what became of it. */
|
|
1110
|
+
export function learningEditSummary(c) {
|
|
1111
|
+
const parts = [];
|
|
1112
|
+
const summary = oneLine(c.summary);
|
|
1113
|
+
if (summary)
|
|
1114
|
+
parts.push(summary);
|
|
1115
|
+
if (c.aim)
|
|
1116
|
+
parts.push(`aims at ${c.aim === "successful" ? "successful runs" : c.aim}`);
|
|
1117
|
+
switch (c.state) {
|
|
1118
|
+
case "proposed":
|
|
1119
|
+
parts.push('shown, not used yet — "Use" on the recording page tries it');
|
|
1120
|
+
break;
|
|
1121
|
+
case "trial":
|
|
1122
|
+
parts.push("on trial: the next runs decide whether it is kept");
|
|
1123
|
+
break;
|
|
1124
|
+
case "kept":
|
|
1125
|
+
parts.push("kept");
|
|
1126
|
+
break;
|
|
1127
|
+
case "taken_back":
|
|
1128
|
+
parts.push("taken back");
|
|
1129
|
+
break;
|
|
1130
|
+
case "failed_check":
|
|
1131
|
+
parts.push("not used: its check failed");
|
|
1132
|
+
break;
|
|
1133
|
+
case "dismissed":
|
|
1134
|
+
parts.push("dismissed");
|
|
1135
|
+
break;
|
|
1136
|
+
case "replaced":
|
|
1137
|
+
parts.push("replaced by a recalculation");
|
|
1138
|
+
break;
|
|
1139
|
+
default:
|
|
1140
|
+
if (c.state)
|
|
1141
|
+
parts.push(String(c.state));
|
|
1142
|
+
}
|
|
1143
|
+
return parts.join("; ");
|
|
1144
|
+
}
|
|
1145
|
+
/** The learning change a plain `undo` would name, when nothing else can be undone: the newest in use. */
|
|
1146
|
+
function newestUsedLearning(learning) {
|
|
1147
|
+
return [...learning]
|
|
1148
|
+
.filter((c) => c.state === "trial" || c.state === "kept")
|
|
1149
|
+
.sort((x, y) => (x.createdAt < y.createdAt ? 1 : x.createdAt > y.createdAt ? -1 : 0))[0];
|
|
1150
|
+
}
|
|
1151
|
+
/** A learning change taken back: the plan as it was before it, and where the phase stands now. */
|
|
1152
|
+
function learningUndone(answer, editId, target, deps) {
|
|
1153
|
+
const change = answer?.change;
|
|
1154
|
+
const revision = answer?.chainRevision ?? change?.revisionAfter;
|
|
1155
|
+
deps.out(`The plan is back as it was before learning change ${change?.id ?? editId} (revision ${revision ?? "?"}).`);
|
|
1156
|
+
const summary = oneLine(change?.summary);
|
|
1157
|
+
if (summary)
|
|
1158
|
+
deps.out(` It was: ${summary}`);
|
|
1159
|
+
deps.out(` The learning phase now: bir scenario learning ${target}`);
|
|
1160
|
+
return 0;
|
|
1161
|
+
}
|
|
1162
|
+
/**
|
|
1163
|
+
* `bir scenario learning <id>` (the service's learningPhase.md LP9): the plan's
|
|
1164
|
+
* learning phase — from its calculation until N of its runs were evaluated,
|
|
1165
|
+
* in which the service improves it one change at a time. Without a flag it
|
|
1166
|
+
* only reads. `--runs <n>` sets N (1–50), `--stop` ends the phase now,
|
|
1167
|
+
* `--again` starts a new one (of `--runs` runs, when given). Each change is
|
|
1168
|
+
* the service's; this prints what it answers, in words, or unchanged with
|
|
1169
|
+
* `--json`.
|
|
1170
|
+
*/
|
|
1171
|
+
async function learning(args, deps) {
|
|
1172
|
+
const { out } = deps;
|
|
1173
|
+
const target = args.positionals[1];
|
|
1174
|
+
if (!target)
|
|
1175
|
+
return usageError(deps, "learning");
|
|
1176
|
+
if (args.stop && (args.runs !== undefined || args.again)) {
|
|
1177
|
+
return usageError(deps, "learning", "--stop ends the learning phase, and --runs and --again are for one that goes on; give one of them.");
|
|
1178
|
+
}
|
|
1179
|
+
if (args.runs !== undefined && (!Number.isInteger(args.runs) || args.runs < 1 || args.runs > 50)) {
|
|
1180
|
+
return usageError(deps, "learning", `--runs takes a number from 1 to 50: how many evaluated runs the learning phase lasts; got ${args.runs}.`);
|
|
1181
|
+
}
|
|
1182
|
+
const body = args.stop
|
|
1183
|
+
? { stop: true }
|
|
1184
|
+
: args.again
|
|
1185
|
+
? { again: true, ...(args.runs !== undefined ? { runs: args.runs } : {}) }
|
|
1186
|
+
: args.runs !== undefined
|
|
1187
|
+
? { runs: args.runs }
|
|
1188
|
+
: undefined;
|
|
1189
|
+
// A read is two requests for a `run_` id — which scenario, then its phase —
|
|
1190
|
+
// inside one bound, as `edits` reads; a change has no such bound.
|
|
1191
|
+
const clock = new ReadClock();
|
|
1192
|
+
const resolved = await resolveTarget(target, args, deps, body ? undefined : clock);
|
|
1193
|
+
if ("code" in resolved)
|
|
1194
|
+
return resolved.code;
|
|
1195
|
+
const path = `/scenarios/${seg(resolved.id)}/learning`;
|
|
1196
|
+
const reply = body ? await deps.service("POST", path, body) : await clock.within(deps.service("GET", path));
|
|
1197
|
+
if ("error" in reply || reply.status !== 200)
|
|
1198
|
+
return explainLearning(reply, args, deps, target, Boolean(body));
|
|
1199
|
+
if (args.json) {
|
|
1200
|
+
out(jsonText(reply.body));
|
|
1201
|
+
return 0;
|
|
1202
|
+
}
|
|
1203
|
+
const view = reply.body?.learning;
|
|
1204
|
+
if (!view || typeof view !== "object") {
|
|
1205
|
+
out("The service answered without the learning phase.");
|
|
1206
|
+
return 1;
|
|
1207
|
+
}
|
|
1208
|
+
renderLearning(out, view, target);
|
|
1209
|
+
return 0;
|
|
1210
|
+
}
|
|
1211
|
+
/**
|
|
1212
|
+
* The learning routes' refusals in words (contract: 400 `invalid_input`; 409
|
|
1213
|
+
* `not_ready`, `builtin`, `learning_off`); everything else as {@link explain}
|
|
1214
|
+
* says it. A 404 without the service's own code is a service older than the
|
|
1215
|
+
* learning phase.
|
|
1216
|
+
*/
|
|
1217
|
+
function explainLearning(reply, args, deps, target, change) {
|
|
1218
|
+
const verb = change ? "Not changed" : "Could not read the learning phase";
|
|
1219
|
+
if (!("error" in reply) && !args.json) {
|
|
1220
|
+
const code = reply.body?.error;
|
|
1221
|
+
if (reply.status === 404 && code !== "not_found") {
|
|
1222
|
+
deps.out(`${verb}: the service does not know the learning phase yet — it is older than this runner.`);
|
|
1223
|
+
return 1;
|
|
1224
|
+
}
|
|
1225
|
+
if (code === "builtin") {
|
|
1226
|
+
deps.out(`${verb} (builtin): a built-in plan (Update bir, Remove bir) has no learning phase.`);
|
|
1227
|
+
return 1;
|
|
1228
|
+
}
|
|
1229
|
+
if (code === "learning_off") {
|
|
1230
|
+
deps.out(`${verb} (learning_off): the service's learning phase is switched off (SCENARIO_LEARNING=off), so no plan learns.`);
|
|
1231
|
+
return 1;
|
|
1232
|
+
}
|
|
1233
|
+
}
|
|
1234
|
+
return explain(reply, args, deps, {
|
|
1235
|
+
verb,
|
|
1236
|
+
target,
|
|
1237
|
+
notFound: `No such scenario, or it is not yours: ${target}.`,
|
|
1238
|
+
});
|
|
1239
|
+
}
|
|
967
1240
|
// ── calc ──────────────────────────────────────────────────────────────────────
|
|
968
1241
|
/**
|
|
969
1242
|
* `calc` lives here because of one answer: a recalculation rebuilds every
|
|
@@ -1176,6 +1449,8 @@ function explain(reply, args, deps, e) {
|
|
|
1176
1449
|
const aboutParam = e.param !== undefined || (e.editId?.startsWith("pedit_") ?? false);
|
|
1177
1450
|
// A correction (autocorrect.md AC7) is about the whole plan.
|
|
1178
1451
|
const aboutCorrection = e.editId?.startsWith("scor_") ?? false;
|
|
1452
|
+
// So is a learning phase's change (learningPhase.md LP9).
|
|
1453
|
+
const aboutLearning = e.editId?.startsWith("simp_") ?? false;
|
|
1179
1454
|
if (reply.status === 404) {
|
|
1180
1455
|
out(e.notFound ??
|
|
1181
1456
|
`${e.verb} (not_found): no such scenario, step or edit — or it is not yours. Only a scenario's owner can see or change it.`);
|
|
@@ -1216,18 +1491,38 @@ function explain(reply, args, deps, e) {
|
|
|
1216
1491
|
return 1;
|
|
1217
1492
|
case "revision_changed": {
|
|
1218
1493
|
const now = typeof body.chainRevision === "number" ? ` — it is at revision ${body.chainRevision} now` : "";
|
|
1219
|
-
out(
|
|
1220
|
-
? `${e.verb} (revision_changed): the plan changed since it was read${now}.
|
|
1221
|
-
:
|
|
1494
|
+
out(aboutLearning
|
|
1495
|
+
? `${e.verb} (revision_changed): the plan changed since it was read${now}. \`bir scenario edits ${target}\` lists what changed it; then try again.`
|
|
1496
|
+
: aboutParam
|
|
1497
|
+
? `${e.verb} (revision_changed): the plan changed since it was read${now}. Show it again (\`bir scenario show ${target}\`), then try again.`
|
|
1498
|
+
: `${e.verb} (revision_changed): the plan changed since it was read${now}. Show it again (\`bir scenario show ${target}${stepFlag}\`), check again, then save again.`);
|
|
1222
1499
|
return 1;
|
|
1223
1500
|
}
|
|
1501
|
+
case "not_latest_change": {
|
|
1502
|
+
// A learning change is taken back exactly — the whole plan from before
|
|
1503
|
+
// it — so only while no later change stands on it (learningPhase.md LP6).
|
|
1504
|
+
const b = body;
|
|
1505
|
+
const latest = typeof b.latestChangeId === "string" ? b.latestChangeId : typeof b.latestEditId === "string" ? b.latestEditId : undefined;
|
|
1506
|
+
if (latest) {
|
|
1507
|
+
out(`${e.verb} (not_latest_change): the plan has changed since ${e.editId ?? "that change"} — ${latest} changed it after. Take back the newer one first, then this one:`);
|
|
1508
|
+
out(` bir scenario undo ${target} --edit ${latest}`);
|
|
1509
|
+
out(` bir scenario undo ${target} --edit ${e.editId ?? "<this change>"}`);
|
|
1510
|
+
}
|
|
1511
|
+
else {
|
|
1512
|
+
out(`${e.verb} (not_latest_change): the plan has changed since ${e.editId ?? "that change"}. Take back the newer change first — \`bir scenario edits ${target}\` lists them, newest first.`);
|
|
1513
|
+
}
|
|
1514
|
+
return 1;
|
|
1515
|
+
}
|
|
1516
|
+
case "not_used":
|
|
1517
|
+
out(`${e.verb} (not_used): ${e.editId ?? "that change"} was never used — it is proposed, dismissed or failed its check — so the plan never ran it, and there is nothing to take back.`);
|
|
1518
|
+
return 1;
|
|
1224
1519
|
case "not_latest_edit": {
|
|
1225
1520
|
// Undo walks back (D9), so this is a real way through now: undoing the
|
|
1226
1521
|
// newer entry puts the step back as this one left it, and then this one
|
|
1227
1522
|
// can be undone. It used to loop, when undo only toggled (C23).
|
|
1228
1523
|
const latest = typeof body.latestEditId === "string" ? body.latestEditId : "<the newer edit>";
|
|
1229
|
-
out(aboutCorrection
|
|
1230
|
-
? `${e.verb} (not_latest_edit): the plan has changed since that correction — ${latest} changed it after. Undo the newer one first, then this one:`
|
|
1524
|
+
out(aboutCorrection || aboutLearning
|
|
1525
|
+
? `${e.verb} (not_latest_edit): the plan has changed since that ${aboutLearning ? "change" : "correction"} — ${latest} changed it after. Undo the newer one first, then this one:`
|
|
1231
1526
|
: `${e.verb} (not_latest_edit): the ${aboutParam ? "parameter" : "step"} has changed since that edit — ${latest} changed it after. Undo the newer one first, then this one:`);
|
|
1232
1527
|
out(` bir scenario undo ${target} --edit ${latest}`);
|
|
1233
1528
|
out(` bir scenario undo ${target} --edit ${e.editId ?? "<this edit>"}`);
|
|
@@ -1235,8 +1530,8 @@ function explain(reply, args, deps, e) {
|
|
|
1235
1530
|
}
|
|
1236
1531
|
case "already_undone": {
|
|
1237
1532
|
const by = typeof body.revertedById === "string" ? body.revertedById : undefined;
|
|
1238
|
-
if (aboutCorrection) {
|
|
1239
|
-
out(`${e.verb} (already_undone): ${e.editId ?? "that correction"} was taken back already; \`bir scenario edits ${target}\` lists what the plan has now.`);
|
|
1533
|
+
if (aboutCorrection || aboutLearning) {
|
|
1534
|
+
out(`${e.verb} (already_undone): ${e.editId ?? (aboutLearning ? "that change" : "that correction")} was taken back already; \`bir scenario edits ${target}\` lists what the plan has now.`);
|
|
1240
1535
|
return 1;
|
|
1241
1536
|
}
|
|
1242
1537
|
out(`${e.verb} (already_undone): ${e.editId ?? "that edit"} was undone already${by ? `, by ${by}` : ""}. To redo it, undo that undo:`);
|
|
@@ -1246,7 +1541,7 @@ function explain(reply, args, deps, e) {
|
|
|
1246
1541
|
case "edit_replaced":
|
|
1247
1542
|
out(aboutParam
|
|
1248
1543
|
? `${e.verb} (edit_replaced): a recalculation dropped that parameter after the change. It stays in \`bir scenario edits\`, but there is nothing left to undo.`
|
|
1249
|
-
: `${e.verb} (edit_replaced): a recalculation replaced the plan after that ${aboutCorrection ? "correction" : "edit"}. It stays in \`bir scenario edits\`, but there is nothing left to undo.`);
|
|
1544
|
+
: `${e.verb} (edit_replaced): a recalculation replaced the plan after that ${aboutCorrection ? "correction" : aboutLearning ? "change" : "edit"}. It stays in \`bir scenario edits\`, but there is nothing left to undo.`);
|
|
1250
1545
|
return 1;
|
|
1251
1546
|
case "rate_limited": {
|
|
1252
1547
|
const ms = typeof body.retryAfterMs === "number" ? body.retryAfterMs : undefined;
|
package/dist/control/server.d.ts
CHANGED
|
@@ -995,6 +995,25 @@ export declare class ControlServer {
|
|
|
995
995
|
* to now otherwise.
|
|
996
996
|
*/
|
|
997
997
|
private costSince;
|
|
998
|
+
/**
|
|
999
|
+
* Where a turn's tokens end, for its reports' `turn` (the service's
|
|
1000
|
+
* learningPhase.md LP11): `until`, the next turn's mark, when that turn has
|
|
1001
|
+
* begun; a mark taken now otherwise, as {@link costSince} reads the
|
|
1002
|
+
* transcript now. Undefined for another host, which has no transcript, and
|
|
1003
|
+
* when the transcript cannot be read. Never throws.
|
|
1004
|
+
*/
|
|
1005
|
+
private tokensEnd;
|
|
1006
|
+
/**
|
|
1007
|
+
* The model calls and tokens of one report's window, for its `turn` (the
|
|
1008
|
+
* service's learningPhase.md LP11): the window its cost is read for, from
|
|
1009
|
+
* `from` to `end` — the plan's own retire, or the next plan's arm — and
|
|
1010
|
+
* otherwise to `to` ({@link tokensEnd}). Another host's run has the tokens
|
|
1011
|
+
* it handed over and no count of its calls, and only `whole`: for the
|
|
1012
|
+
* report that bills the run, when one report does. Undefined when nothing
|
|
1013
|
+
* can be said — no mark, no transcript — and then the report's `turn` says
|
|
1014
|
+
* null. Never throws.
|
|
1015
|
+
*/
|
|
1016
|
+
private turnUsage;
|
|
998
1017
|
private reportExecution;
|
|
999
1018
|
/**
|
|
1000
1019
|
* The plan's own calls of the turn, whole, as an answered turn's report
|
package/dist/control/server.js
CHANGED
|
@@ -31,7 +31,7 @@ import { BIR_CALL_ID, fingerprint, newCallId, parseQualifiedName, qualifyToolNam
|
|
|
31
31
|
import { StepIndexAllocator } from "./ordering.js";
|
|
32
32
|
import { pidAlive } from "./discovery.js";
|
|
33
33
|
import { pinCompatible } from "./pin-guard.js";
|
|
34
|
-
import { agentDayOf, answerBeforePrompt, contextForToolUse, intentForToolUse, markTranscriptUsage, promptCount, recentToolResults, sessionTranscriptPath, settledLastAssistantText, settledTextAfterToolUse, settledTurnEnd, thinkingForToolUse, usageBetween, usageSince, } from "./transcript.js";
|
|
34
|
+
import { agentDayOf, answerBeforePrompt, callsBetween, contextForToolUse, intentForToolUse, markTranscriptUsage, promptCount, recentToolResults, sessionTranscriptPath, settledLastAssistantText, settledTextAfterToolUse, settledTurnEnd, thinkingForToolUse, usageBetween, usageSince, } from "./transcript.js";
|
|
35
35
|
import { isHousekeepingCall } from "../record/housekeeping.js";
|
|
36
36
|
import { readsOnlyCall } from "../replay/reads-only.js";
|
|
37
37
|
import { asSent, userText } from "../record/prompt-text.js";
|
|
@@ -709,6 +709,22 @@ function priceUsage(usage) {
|
|
|
709
709
|
...(rateUnknown.length > 0 ? { rateUnknown } : {}),
|
|
710
710
|
};
|
|
711
711
|
}
|
|
712
|
+
/**
|
|
713
|
+
* The tokens of a usage list — the one its cost was priced from — under the
|
|
714
|
+
* names an execution report's `turn` gives them (the service's
|
|
715
|
+
* learningPhase.md LP11): every cache write, the 1-hour ones among them, in
|
|
716
|
+
* `cacheWriteTokens`.
|
|
717
|
+
*/
|
|
718
|
+
function turnTokens(usage) {
|
|
719
|
+
const sum = { inputTokens: 0, outputTokens: 0, cacheReadTokens: 0, cacheWriteTokens: 0 };
|
|
720
|
+
for (const u of usage) {
|
|
721
|
+
sum.inputTokens += u.inputTokens;
|
|
722
|
+
sum.outputTokens += u.outputTokens;
|
|
723
|
+
sum.cacheReadTokens += u.cacheReadTokens;
|
|
724
|
+
sum.cacheWriteTokens += u.cacheCreationTokens;
|
|
725
|
+
}
|
|
726
|
+
return sum;
|
|
727
|
+
}
|
|
712
728
|
/**
|
|
713
729
|
* What `/health` answers as `hostFields`: how much of another host's protocol
|
|
714
730
|
* this recorder understands. An adapter reads it to know which fields it may
|
|
@@ -3280,6 +3296,51 @@ export class ControlServer {
|
|
|
3280
3296
|
return { usd: 0, measured: false };
|
|
3281
3297
|
return priceUsage(until ? usageBetween(mark, until) : usageSince(session.transcriptPath, mark));
|
|
3282
3298
|
}
|
|
3299
|
+
/**
|
|
3300
|
+
* Where a turn's tokens end, for its reports' `turn` (the service's
|
|
3301
|
+
* learningPhase.md LP11): `until`, the next turn's mark, when that turn has
|
|
3302
|
+
* begun; a mark taken now otherwise, as {@link costSince} reads the
|
|
3303
|
+
* transcript now. Undefined for another host, which has no transcript, and
|
|
3304
|
+
* when the transcript cannot be read. Never throws.
|
|
3305
|
+
*/
|
|
3306
|
+
tokensEnd(session, until) {
|
|
3307
|
+
if (otherHost(session))
|
|
3308
|
+
return undefined;
|
|
3309
|
+
if (until)
|
|
3310
|
+
return until;
|
|
3311
|
+
try {
|
|
3312
|
+
return markTranscriptUsage(session.transcriptPath);
|
|
3313
|
+
}
|
|
3314
|
+
catch {
|
|
3315
|
+
return undefined;
|
|
3316
|
+
}
|
|
3317
|
+
}
|
|
3318
|
+
/**
|
|
3319
|
+
* The model calls and tokens of one report's window, for its `turn` (the
|
|
3320
|
+
* service's learningPhase.md LP11): the window its cost is read for, from
|
|
3321
|
+
* `from` to `end` — the plan's own retire, or the next plan's arm — and
|
|
3322
|
+
* otherwise to `to` ({@link tokensEnd}). Another host's run has the tokens
|
|
3323
|
+
* it handed over and no count of its calls, and only `whole`: for the
|
|
3324
|
+
* report that bills the run, when one report does. Undefined when nothing
|
|
3325
|
+
* can be said — no mark, no transcript — and then the report's `turn` says
|
|
3326
|
+
* null. Never throws.
|
|
3327
|
+
*/
|
|
3328
|
+
turnUsage(session, run, from, end, to, whole) {
|
|
3329
|
+
try {
|
|
3330
|
+
if (otherHost(session)) {
|
|
3331
|
+
if (!whole || !run.hostUsage?.length)
|
|
3332
|
+
return undefined;
|
|
3333
|
+
return { modelCalls: null, ...turnTokens(run.hostUsage) };
|
|
3334
|
+
}
|
|
3335
|
+
const last = end ?? to;
|
|
3336
|
+
if (from === undefined || last === undefined)
|
|
3337
|
+
return undefined;
|
|
3338
|
+
return { modelCalls: callsBetween(from, last) ?? null, ...turnTokens(usageBetween(from, last)) };
|
|
3339
|
+
}
|
|
3340
|
+
catch {
|
|
3341
|
+
return undefined;
|
|
3342
|
+
}
|
|
3343
|
+
}
|
|
3283
3344
|
reportExecution(session, run, durationMs, cost,
|
|
3284
3345
|
/** The run was closed before its task ended for the caller (another host only). */
|
|
3285
3346
|
cutShort = false,
|
|
@@ -3296,6 +3357,11 @@ export class ControlServer {
|
|
|
3296
3357
|
// Another host hands over one number for the whole run. It cannot be
|
|
3297
3358
|
// split between two plans, so with more than one neither is measured.
|
|
3298
3359
|
const unsplit = otherHost(session) && states.length > 1;
|
|
3360
|
+
// Where the turn's tokens end, read once for every report's `turn` (LP11):
|
|
3361
|
+
// the next turn's mark when that turn has begun, the transcript as it is
|
|
3362
|
+
// now otherwise — as the cost was read a moment ago. Only for a turn that
|
|
3363
|
+
// has a report to send it with.
|
|
3364
|
+
const tokensTo = states.length > 0 || run.lateMatch ? this.tokensEnd(session, until) : undefined;
|
|
3299
3365
|
// The plan ran to its end and the agent redid the task in a repeat the
|
|
3300
3366
|
// service took (newestRun.md N7): the prompt plan's report says so, with
|
|
3301
3367
|
// the repeat's id and the agent's own calls in it (autocorrect.md AC1).
|
|
@@ -3382,6 +3448,8 @@ export class ControlServer {
|
|
|
3382
3448
|
redoneBy: state === prompt && state.ranToEnd === true ? redoneBy : undefined,
|
|
3383
3449
|
chainDeriveUsd,
|
|
3384
3450
|
booksOwnSaving: booksOwn(state),
|
|
3451
|
+
// The model calls and tokens of the same window as the cost above.
|
|
3452
|
+
usage: this.turnUsage(session, run, state.armedBy === "intent" ? state.usageMark : run.usageMark, end, tokensTo, state.armedBy !== "intent" && !unsplit),
|
|
3385
3453
|
});
|
|
3386
3454
|
if (!report)
|
|
3387
3455
|
return;
|
|
@@ -3416,7 +3484,15 @@ export class ControlServer {
|
|
|
3416
3484
|
.catch(() => "")
|
|
3417
3485
|
.then((answer) => {
|
|
3418
3486
|
const calls = answer ? this.answerCallsOf(state.scenarioId, kept) : [];
|
|
3419
|
-
this.queue.push(() => recorder.reportExecution(answer
|
|
3487
|
+
this.queue.push(() => recorder.reportExecution(answer
|
|
3488
|
+
? {
|
|
3489
|
+
...report,
|
|
3490
|
+
answer,
|
|
3491
|
+
...(calls.length > 0 ? { answerCalls: calls } : {}),
|
|
3492
|
+
// How long the answer is, whole: the report carries 8 KiB of it (LP11).
|
|
3493
|
+
...(report.turn ? { turn: { ...report.turn, answerChars: answer.length } } : {}),
|
|
3494
|
+
}
|
|
3495
|
+
: report));
|
|
3420
3496
|
});
|
|
3421
3497
|
this.holdSeal(run, sent);
|
|
3422
3498
|
return;
|
|
@@ -3438,6 +3514,8 @@ export class ControlServer {
|
|
|
3438
3514
|
siblings: [...states, late],
|
|
3439
3515
|
runId: run.runId,
|
|
3440
3516
|
cutShort: cutShort || undefined,
|
|
3517
|
+
// The turn's own tokens only when its cost is this report's too.
|
|
3518
|
+
usage: alone ? this.turnUsage(session, run, run.usageMark, undefined, tokensTo, !unsplit) : undefined,
|
|
3441
3519
|
});
|
|
3442
3520
|
if (report)
|
|
3443
3521
|
this.queue.push(() => recorder.reportExecution(report));
|
|
@@ -217,6 +217,15 @@ export interface ModelUsage extends RateFlags {
|
|
|
217
217
|
* of pricing, under the model's name with its rate key after it.
|
|
218
218
|
*/
|
|
219
219
|
export type UsageMark = Map<string, ModelUsage>;
|
|
220
|
+
/**
|
|
221
|
+
* The model calls between two marks of one transcript, `from` the earlier:
|
|
222
|
+
* assistant messages, each counted once by its `message.id` as its tokens are
|
|
223
|
+
* ({@link totalUsage}), the sub-agents' files included — the calls behind the
|
|
224
|
+
* tokens {@link usageBetween} gives for the same two marks. No `mark` counts
|
|
225
|
+
* from the start, as there. Clamped at 0, as the tokens are, for a transcript
|
|
226
|
+
* rewritten between the two. Undefined when either mark was not made here.
|
|
227
|
+
*/
|
|
228
|
+
export declare function callsBetween(mark: UsageMark | undefined, now: UsageMark): number | undefined;
|
|
220
229
|
/**
|
|
221
230
|
* Snapshot cumulative usage at the moment a run starts.
|
|
222
231
|
*
|