@basein/runner 0.3.15 → 0.3.16
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +7 -0
- package/dist/bin/bir-scenario.js +8 -5
- package/dist/bin/bir.js +2 -1
- package/dist/bin/investigate.d.ts +71 -0
- package/dist/bin/investigate.js +145 -13
- package/dist/bin/scenario-edit.d.ts +45 -1
- package/dist/bin/scenario-edit.js +134 -14
- package/dist/control/server.d.ts +16 -0
- package/dist/control/server.js +119 -2
- package/dist/record/recorder.d.ts +39 -4
- package/dist/record/remote-recorder.d.ts +3 -4
- package/dist/record/remote-recorder.js +16 -0
- package/dist/record/switchable-recorder.d.ts +1 -5
- package/dist/replay/bundle.d.ts +10 -0
- package/dist/replay/bundle.js +23 -2
- package/dist/replay/controller.d.ts +133 -0
- package/dist/replay/controller.js +496 -59
- package/dist/replay/flatten.d.ts +39 -3
- package/dist/replay/flatten.js +42 -2
- package/dist/replay/handover.d.ts +39 -1
- package/dist/replay/handover.js +81 -2
- package/dist/replay/logic.d.ts +8 -0
- package/dist/replay/logic.js +19 -0
- package/dist/replay/plan.d.ts +165 -4
- package/dist/replay/plan.js +488 -25
- package/dist/replay/source-run.d.ts +14 -1
- package/dist/replay/source-run.js +25 -2
- package/dist/replay/targets.js +8 -1
- package/dist/replay/types.d.ts +52 -3
- package/dist/replay/types.js +14 -0
- package/dist/util/service-headers.d.ts +7 -1
- package/dist/util/service-headers.js +7 -1
- package/docs/BaseInstRunner.md +3 -1
- package/docs/calculatedReplay.md +117 -6
- package/docs/calculatedReplayGuide.md +41 -3
- package/package.json +1 -1
|
@@ -674,6 +674,10 @@ export class RemoteRecorder {
|
|
|
674
674
|
toolName: data.toolName,
|
|
675
675
|
toolOutput: out(toolOutput),
|
|
676
676
|
toolError: out(toolError),
|
|
677
|
+
// Omitted, not nulled, when absent: an ordinary answer is the body it
|
|
678
|
+
// has always been. Only a plan's own call recorded into a redo carries
|
|
679
|
+
// one (`byPlan`, the service's autocorrect.md AC2).
|
|
680
|
+
metadata: data.metadata,
|
|
677
681
|
}));
|
|
678
682
|
return id;
|
|
679
683
|
}
|
|
@@ -744,6 +748,12 @@ export class RemoteRecorder {
|
|
|
744
748
|
kind: "report",
|
|
745
749
|
bytes: sizeOf(prompt, error, ...(steps ?? []).flatMap((s) => [s.toolInput, s.toolOutput])) + 1024,
|
|
746
750
|
send: async () => {
|
|
751
|
+
// A redo whose run the service refused to create is no run of its:
|
|
752
|
+
// never named (the service's autocorrect.md AC1), even when the
|
|
753
|
+
// refusal came after the report was built.
|
|
754
|
+
const redoneBy = report.redoneBy && this.runs.get(report.redoneBy.runId)?.settled?.created.created === false
|
|
755
|
+
? undefined
|
|
756
|
+
: report.redoneBy;
|
|
747
757
|
const body = await this.request("POST", `/scenarios/${report.scenarioId}/executions`,
|
|
748
758
|
// Per-step verdicts and the headline failure (errorshandling.md) ride
|
|
749
759
|
// beside the costs. All optional on the service side, so an older
|
|
@@ -776,6 +786,9 @@ export class RemoteRecorder {
|
|
|
776
786
|
// Why nothing ran, when nothing ran. Additive: an older service drops
|
|
777
787
|
// the field and books the cost exactly as it always did.
|
|
778
788
|
declined: report.declined,
|
|
789
|
+
// The plan ran to its end and the agent redid the task (the
|
|
790
|
+
// service's autocorrect.md AC1). Additive, like `declined`.
|
|
791
|
+
redoneBy,
|
|
779
792
|
}, steps), { retryTransient: Boolean(report.ticket) });
|
|
780
793
|
const r = (body ?? {});
|
|
781
794
|
const failed = report.steps?.filter((s) => s.status === "failed").length ?? 0;
|
|
@@ -795,6 +808,9 @@ export class RemoteRecorder {
|
|
|
795
808
|
baselineEligible: report.baselineEligible === false ? false : undefined,
|
|
796
809
|
sharedWith: report.sharedWith?.length,
|
|
797
810
|
declined: report.declined,
|
|
811
|
+
// `bir investigate` prints it beside the outcome: "redone by run_…".
|
|
812
|
+
redoneBy: report.redoneBy?.runId,
|
|
813
|
+
ownCalls: report.redoneBy?.ownCalls,
|
|
798
814
|
});
|
|
799
815
|
},
|
|
800
816
|
});
|
|
@@ -90,11 +90,7 @@ export declare class SwitchableRecorder implements Recorder {
|
|
|
90
90
|
currentAccessToken(): string | undefined;
|
|
91
91
|
startRun(input: string, metadata?: Record<string, unknown>, options?: StartRunOptions): string;
|
|
92
92
|
recordToolSelected(runId: string, stepIndex: number, d: Parameters<Recorder["recordToolSelected"]>[2]): string;
|
|
93
|
-
recordToolResponse(runId: string, stepIndex: number, d:
|
|
94
|
-
toolName: string;
|
|
95
|
-
toolOutput?: string;
|
|
96
|
-
toolError?: string;
|
|
97
|
-
}): string;
|
|
93
|
+
recordToolResponse(runId: string, stepIndex: number, d: Parameters<Recorder["recordToolResponse"]>[2]): string;
|
|
98
94
|
recordFinalAnswer(runId: string, stepIndex: number, d: {
|
|
99
95
|
answer: string;
|
|
100
96
|
}): string;
|
package/dist/replay/bundle.d.ts
CHANGED
|
@@ -34,7 +34,17 @@ export interface BundleEntry {
|
|
|
34
34
|
response: string;
|
|
35
35
|
/** True when `response` is a recorded output, not a fresh run. */
|
|
36
36
|
recorded?: boolean;
|
|
37
|
+
/**
|
|
38
|
+
* Set on a step its test left out (the service's autocorrect.md AC5): no
|
|
39
|
+
* call was made, and the bundle says so in its place, so the model can tell
|
|
40
|
+
* a whole answer from a part. `text` is the test in words, when it has some.
|
|
41
|
+
*/
|
|
42
|
+
notNeeded?: {
|
|
43
|
+
text: string | null;
|
|
44
|
+
};
|
|
37
45
|
}
|
|
46
|
+
/** The test's words, one line and short, for a bundle or a note. */
|
|
47
|
+
export declare function testWords(text: string | null | undefined): string | null;
|
|
38
48
|
/** Cap a computed input for inclusion in a bundle. */
|
|
39
49
|
export declare function bundleInput(value: unknown): string;
|
|
40
50
|
export interface AssembleOptions {
|
package/dist/replay/bundle.js
CHANGED
|
@@ -30,6 +30,13 @@ export const BASH_CHANNEL_MAX = 25_000;
|
|
|
30
30
|
const MAX_INPUT_CHARS = 2_000;
|
|
31
31
|
/** Fixed structural overhead reserved per step (labels, separators). */
|
|
32
32
|
const PER_STEP_OVERHEAD = 80;
|
|
33
|
+
/** The test's words, one line and short, for a bundle or a note. */
|
|
34
|
+
export function testWords(text) {
|
|
35
|
+
const line = (text ?? "").trim().split(/\r?\n/, 1)[0]?.trim() ?? "";
|
|
36
|
+
if (!line)
|
|
37
|
+
return null;
|
|
38
|
+
return line.length > 200 ? `${line.slice(0, 200)}…` : line;
|
|
39
|
+
}
|
|
33
40
|
/** Cap a computed input for inclusion in a bundle. */
|
|
34
41
|
export function bundleInput(value) {
|
|
35
42
|
let text;
|
|
@@ -90,7 +97,9 @@ const SUBTASK_FOOTER = "\n\nThe calls above are done. Continue your task, callin
|
|
|
90
97
|
* Compose the bundle. `maxChars` is a hard ceiling on the returned string.
|
|
91
98
|
*/
|
|
92
99
|
export function assembleBundle(entries, maxChars, opts = {}) {
|
|
93
|
-
|
|
100
|
+
// A step left out has no response to share the budget with: only the steps
|
|
101
|
+
// that answered split it.
|
|
102
|
+
const n = entries.filter((e) => !e.notNeeded).length;
|
|
94
103
|
const anyRecorded = entries.some((e) => e.recorded);
|
|
95
104
|
// A hand-over outranks a sub-task: it says "continue" too, and it also has to
|
|
96
105
|
// say that the plan stopped short. A sub-task outranks the live and mixed
|
|
@@ -111,10 +120,22 @@ export function assembleBundle(entries, maxChars, opts = {}) {
|
|
|
111
120
|
: LIVE_FOOTER;
|
|
112
121
|
const fixedCost = header.length +
|
|
113
122
|
footer.length +
|
|
114
|
-
entries.reduce((sum, e) => sum +
|
|
123
|
+
entries.reduce((sum, e) => sum +
|
|
124
|
+
PER_STEP_OVERHEAD +
|
|
125
|
+
e.toolName.length +
|
|
126
|
+
e.input.length +
|
|
127
|
+
// A left-out step's line, which has no response to take from.
|
|
128
|
+
(e.notNeeded ? PER_STEP_OVERHEAD + (testWords(e.notNeeded.text)?.length ?? 0) : 0), 0);
|
|
115
129
|
const responseBudget = Math.max(0, maxChars - fixedCost);
|
|
116
130
|
const perResponse = n > 0 ? Math.floor(responseBudget / n) : responseBudget;
|
|
117
131
|
const sections = entries.map((e, i) => {
|
|
132
|
+
if (e.notNeeded) {
|
|
133
|
+
// Never "skipped" or "failed": the step was left out on purpose, because
|
|
134
|
+
// this request does not need it, and nothing is missing for its absence.
|
|
135
|
+
const why = testWords(e.notNeeded.text);
|
|
136
|
+
return (`\n\n--- Step ${i + 1}: ${e.toolName} (not needed) ---\n` +
|
|
137
|
+
`Not run: this request does not need it${why ? ` — it is only for when ${why}` : ""}.`);
|
|
138
|
+
}
|
|
118
139
|
const response = truncate(e.response, perResponse);
|
|
119
140
|
return (`\n\n--- Step ${i + 1}: ${e.toolName} ---\n` +
|
|
120
141
|
`Call: ${e.input}\n` +
|
|
@@ -514,6 +514,60 @@ export interface ReplayState {
|
|
|
514
514
|
* sample of what the task costs unaided.
|
|
515
515
|
*/
|
|
516
516
|
hostStop?: HostStop;
|
|
517
|
+
/**
|
|
518
|
+
* The served chain's revision, for the mark on the plan's own calls when
|
|
519
|
+
* they are recorded into the turn's newest run (autocorrect.md AC2).
|
|
520
|
+
*/
|
|
521
|
+
chainRevision?: number;
|
|
522
|
+
/**
|
|
523
|
+
* Keep every call this plan makes, whole, for the turn's newest run: set by
|
|
524
|
+
* the control server on a prompt plan whose match offered to record the
|
|
525
|
+
* turn as a repeat. A plan that runs to its end and is then redone by the
|
|
526
|
+
* agent has its calls recorded first into that repeat (AC2) — the report's
|
|
527
|
+
* copies are cut to 3,800 bytes, and the agent's work is built on them.
|
|
528
|
+
*/
|
|
529
|
+
keepCalls?: boolean;
|
|
530
|
+
/**
|
|
531
|
+
* Those calls, in the order the plan made them, keyed by frame and step —
|
|
532
|
+
* a sub-task called twice keeps both calls of each step.
|
|
533
|
+
*/
|
|
534
|
+
turnCalls?: Map<string, TurnCall>;
|
|
535
|
+
/** What they hold, in characters: bounded by {@link TURN_CALLS_MAX_CHARS}. */
|
|
536
|
+
turnCallChars?: number;
|
|
537
|
+
/**
|
|
538
|
+
* Steer mode: the steps a test just left out (AC5), and the note that tells
|
|
539
|
+
* the model so — riding on the `PostToolUse` that moved the cursor past
|
|
540
|
+
* them, or given in place of a call the model makes to one of them.
|
|
541
|
+
*/
|
|
542
|
+
skipNote?: {
|
|
543
|
+
text: string;
|
|
544
|
+
tools: Set<string>;
|
|
545
|
+
delivered: boolean;
|
|
546
|
+
refused?: number;
|
|
547
|
+
/**
|
|
548
|
+
* It says the sequence is complete — the steps left out were the plan's
|
|
549
|
+
* last. The only note that outlives the plan: every other one names a step
|
|
550
|
+
* to call next, which a retired plan no longer has (AC5 review).
|
|
551
|
+
*/
|
|
552
|
+
complete?: boolean;
|
|
553
|
+
};
|
|
554
|
+
}
|
|
555
|
+
/**
|
|
556
|
+
* One of a plan's own calls, kept whole for the turn's newest run (the
|
|
557
|
+
* service's autocorrect.md AC2): redacted and capped like a recorded call,
|
|
558
|
+
* never cut to what a report carries.
|
|
559
|
+
*/
|
|
560
|
+
export interface TurnCall {
|
|
561
|
+
scenarioId: string;
|
|
562
|
+
stepIndex: number;
|
|
563
|
+
chainRevision: number | null;
|
|
564
|
+
toolName: string;
|
|
565
|
+
/** Serialized and redacted, at most 64 KiB, as a recorded call's input is. */
|
|
566
|
+
input: string;
|
|
567
|
+
/** The tool's answer, when it gave one; on the same terms. */
|
|
568
|
+
output?: string;
|
|
569
|
+
/** What it reported instead, when it failed. */
|
|
570
|
+
error?: string;
|
|
517
571
|
}
|
|
518
572
|
/**
|
|
519
573
|
* Why a match did not arm. One code per gate of the ladder, so the caller can
|
|
@@ -522,10 +576,21 @@ export interface ReplayState {
|
|
|
522
576
|
export type DeclineCode = "replay_disabled" | "not_ready" | "flatten_failed" | "known_bad_first_step" | "nondeterministic_first_step" | "unusable_first_step" | "coverage" | "missing_target" | "no_derive_key"
|
|
523
577
|
/** The service handed over no plan: this runner is too old for it. */
|
|
524
578
|
| "runner_update_required"
|
|
579
|
+
/**
|
|
580
|
+
* The service handed over no plan: it holds it, because the agent did the
|
|
581
|
+
* task again after the plan's last run (the service's autocorrect.md AC4).
|
|
582
|
+
*/
|
|
583
|
+
| "held"
|
|
525
584
|
/** The session's host only records: its adapter asked for no plan with the prompt. */
|
|
526
585
|
| "host_records_only"
|
|
527
586
|
/** The plan names a tool the host's agent does not hold. */
|
|
528
587
|
| "host_tool_missing"
|
|
588
|
+
/**
|
|
589
|
+
* The plan has steps that run only when a test says so (the service's
|
|
590
|
+
* autocorrect.md AC5), and the session's host is another app, whose adapter
|
|
591
|
+
* steps through a plan call by call and knows no tests yet.
|
|
592
|
+
*/
|
|
593
|
+
| "host_tests_unsupported"
|
|
529
594
|
/**
|
|
530
595
|
* No gate said no: the plan was good, and late. Not ready when the prompt
|
|
531
596
|
* had to be answered (`arm_timeout`), or ready just after it had been
|
|
@@ -786,6 +851,50 @@ export declare class ReplayController {
|
|
|
786
851
|
* ({@link pinFailure}). `how` lets the caller say so outright.
|
|
787
852
|
*/
|
|
788
853
|
postTool(state: ReplayState, toolUseId: string, output: string, failed?: string, how?: PostToolOptions): boolean;
|
|
854
|
+
/**
|
|
855
|
+
* A steer plan whose cursor reached its end: in front of a parked step (or a
|
|
856
|
+
* call the service could not answer), which hands over; otherwise it ran to
|
|
857
|
+
* its end — and from here every call is the model's own.
|
|
858
|
+
*/
|
|
859
|
+
private finishSteer;
|
|
860
|
+
/**
|
|
861
|
+
* Move a plan's own cursor past the steps this request does not need, and
|
|
862
|
+
* act on what its tests decide (the service's autocorrect.md AC5). Every
|
|
863
|
+
* verdict is recorded as it is made. `stopped` means the plan handed over —
|
|
864
|
+
* the other half, a test that could not answer, or no step needed at all —
|
|
865
|
+
* and its note waits in {@link takeNote}. Otherwise the steps left out are
|
|
866
|
+
* named in a note for the model ({@link takeSkipNote}), and the caller goes
|
|
867
|
+
* on: a step is due, or the plan is done.
|
|
868
|
+
*
|
|
869
|
+
* Nothing at all for a plan without tests, which behaves exactly as before.
|
|
870
|
+
*/
|
|
871
|
+
private settleCursor;
|
|
872
|
+
/** What the model is told about steps a test left out (AC5). */
|
|
873
|
+
private skipNoteText;
|
|
874
|
+
/**
|
|
875
|
+
* The note naming the steps a test just left out, claimed for delivery on a
|
|
876
|
+
* `PostToolUse` (AC5). Returned once; undefined when there is none.
|
|
877
|
+
*/
|
|
878
|
+
takeSkipNote(state: ReplayState | undefined): string | undefined;
|
|
879
|
+
/**
|
|
880
|
+
* The same note, for a call the model is about to make to one of the steps
|
|
881
|
+
* it names — when the note has not reached the model yet. The control server
|
|
882
|
+
* gives it in place of that call, whether the plan is still steering or has
|
|
883
|
+
* just finished: a step left out must never run because the note was late.
|
|
884
|
+
*/
|
|
885
|
+
skipNoteFor(state: ReplayState | undefined, toolName: string): string | undefined;
|
|
886
|
+
/**
|
|
887
|
+
* The pin guard took a call to the next step as that step: a note still
|
|
888
|
+
* waiting to say that a step before it was left out has nothing left to
|
|
889
|
+
* stop, and goes — as it went at the call before the guard existed.
|
|
890
|
+
*/
|
|
891
|
+
pinTaken(state: ReplayState | undefined): void;
|
|
892
|
+
/**
|
|
893
|
+
* The plan's own calls of the turn, whole, in the order it made them — for
|
|
894
|
+
* the turn's newest run when the agent redoes the task (AC2). Empty unless
|
|
895
|
+
* the control server asked for them to be kept ({@link ReplayState.keepCalls}).
|
|
896
|
+
*/
|
|
897
|
+
turnCallsOf(state: ReplayState | undefined): TurnCall[];
|
|
789
898
|
/**
|
|
790
899
|
* The result of a call pinned under a plan that has since handed over
|
|
791
900
|
* because another call ran beside it (`concurrentCall`). Nothing threads —
|
|
@@ -854,6 +963,16 @@ export declare class ReplayController {
|
|
|
854
963
|
* usual cost.
|
|
855
964
|
*/
|
|
856
965
|
cutShort?: boolean;
|
|
966
|
+
/**
|
|
967
|
+
* The plan ran to its end and the agent redid the task in a repeat the
|
|
968
|
+
* service took (autocorrect.md AC1): that repeat's id and the agent's
|
|
969
|
+
* own calls in it. Only the control server knows, and only for the
|
|
970
|
+
* prompt's plan.
|
|
971
|
+
*/
|
|
972
|
+
redoneBy?: {
|
|
973
|
+
runId: string;
|
|
974
|
+
ownCalls: number;
|
|
975
|
+
};
|
|
857
976
|
}): ExecutionReport | undefined;
|
|
858
977
|
/**
|
|
859
978
|
* The step a failed replay is fairly blamed on: the first whose own logic
|
|
@@ -951,6 +1070,13 @@ export declare class ReplayController {
|
|
|
951
1070
|
* step.
|
|
952
1071
|
*/
|
|
953
1072
|
private stopKindOf;
|
|
1073
|
+
/**
|
|
1074
|
+
* The steps before flat position `bound` that ran, 1-based, and the ones
|
|
1075
|
+
* their tests left out (AC5) — the plan cursor's own, and a divergence
|
|
1076
|
+
* bundle's (`extra`). `left` is empty when no step was left out; `ran` is
|
|
1077
|
+
* then not filled, since every step before `bound` ran.
|
|
1078
|
+
*/
|
|
1079
|
+
private ranBefore;
|
|
954
1080
|
private noteFor;
|
|
955
1081
|
/**
|
|
956
1082
|
* The executor handed to a plan: dispatch a step to the proxy that owns its
|
|
@@ -1038,6 +1164,13 @@ export declare class ReplayController {
|
|
|
1038
1164
|
* the recording's input and output beside this run's verdict.
|
|
1039
1165
|
*/
|
|
1040
1166
|
private recordStep;
|
|
1167
|
+
/**
|
|
1168
|
+
* Keep one call of the plan whole (AC2): its arguments and its answer,
|
|
1169
|
+
* redacted and capped as a recorded call's are. Later news about the same
|
|
1170
|
+
* step replaces earlier, in place, so the order is the order the calls went
|
|
1171
|
+
* out. Past {@link TURN_CALLS_MAX_CHARS} a call keeps only a report's cut.
|
|
1172
|
+
*/
|
|
1173
|
+
private keepCall;
|
|
1041
1174
|
/** The step verdicts, in chain order, for the execution report. */
|
|
1042
1175
|
private stepResultsOf;
|
|
1043
1176
|
private logStep;
|