@basein/runner 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +201 -0
- package/README.md +276 -0
- package/dist/auth/client.d.ts +85 -0
- package/dist/auth/client.js +284 -0
- package/dist/bin/bir-hooks.d.ts +48 -0
- package/dist/bin/bir-hooks.js +201 -0
- package/dist/bin/bir-proxy.d.ts +45 -0
- package/dist/bin/bir-proxy.js +207 -0
- package/dist/bin/bir-scenario.d.ts +24 -0
- package/dist/bin/bir-scenario.js +177 -0
- package/dist/bin/bir.d.ts +21 -0
- package/dist/bin/bir.js +876 -0
- package/dist/config/adapters/claude-code.d.ts +76 -0
- package/dist/config/adapters/claude-code.js +181 -0
- package/dist/config/adapters/generic.d.ts +17 -0
- package/dist/config/adapters/generic.js +36 -0
- package/dist/config/generate.d.ts +127 -0
- package/dist/config/generate.js +114 -0
- package/dist/config/resolve.d.ts +68 -0
- package/dist/config/resolve.js +132 -0
- package/dist/control/client.d.ts +56 -0
- package/dist/control/client.js +86 -0
- package/dist/control/correlation.d.ts +86 -0
- package/dist/control/correlation.js +0 -0
- package/dist/control/discovery.d.ts +50 -0
- package/dist/control/discovery.js +123 -0
- package/dist/control/ordering.d.ts +38 -0
- package/dist/control/ordering.js +44 -0
- package/dist/control/paths.d.ts +32 -0
- package/dist/control/paths.js +56 -0
- package/dist/control/server.d.ts +272 -0
- package/dist/control/server.js +1131 -0
- package/dist/control/transcript.d.ts +75 -0
- package/dist/control/transcript.js +241 -0
- package/dist/index.d.ts +37 -0
- package/dist/index.js +32 -0
- package/dist/jsonrpc/framing.d.ts +49 -0
- package/dist/jsonrpc/framing.js +143 -0
- package/dist/jsonrpc/types.d.ts +52 -0
- package/dist/jsonrpc/types.js +46 -0
- package/dist/proxy/intercept.d.ts +55 -0
- package/dist/proxy/intercept.js +147 -0
- package/dist/proxy/relay.d.ts +97 -0
- package/dist/proxy/relay.js +166 -0
- package/dist/proxy/session.d.ts +116 -0
- package/dist/proxy/session.js +319 -0
- package/dist/record/housekeeping.d.ts +34 -0
- package/dist/record/housekeeping.js +39 -0
- package/dist/record/queue.d.ts +48 -0
- package/dist/record/queue.js +96 -0
- package/dist/record/recorder.d.ts +111 -0
- package/dist/record/recorder.js +39 -0
- package/dist/record/redact.d.ts +37 -0
- package/dist/record/redact.js +119 -0
- package/dist/record/remote-recorder.d.ts +110 -0
- package/dist/record/remote-recorder.js +301 -0
- package/dist/record/truncate.d.ts +36 -0
- package/dist/record/truncate.js +85 -0
- package/dist/replay/bundle.d.ts +36 -0
- package/dist/replay/bundle.js +89 -0
- package/dist/replay/controller.d.ts +300 -0
- package/dist/replay/controller.js +807 -0
- package/dist/replay/coverage.d.ts +41 -0
- package/dist/replay/coverage.js +56 -0
- package/dist/replay/derive.d.ts +58 -0
- package/dist/replay/derive.js +166 -0
- package/dist/replay/executor.d.ts +78 -0
- package/dist/replay/executor.js +233 -0
- package/dist/replay/logic.d.ts +31 -0
- package/dist/replay/logic.js +50 -0
- package/dist/replay/plan.d.ts +181 -0
- package/dist/replay/plan.js +397 -0
- package/dist/replay/pricing.d.ts +41 -0
- package/dist/replay/pricing.js +76 -0
- package/dist/replay/source-run.d.ts +50 -0
- package/dist/replay/source-run.js +98 -0
- package/dist/replay/tool-error.d.ts +22 -0
- package/dist/replay/tool-error.js +60 -0
- package/dist/replay/types.d.ts +116 -0
- package/dist/replay/types.js +35 -0
- package/dist/upstream/client.d.ts +78 -0
- package/dist/upstream/client.js +114 -0
- package/dist/upstream/http-client.d.ts +78 -0
- package/dist/upstream/http-client.js +261 -0
- package/dist/upstream/lazy-client.d.ts +31 -0
- package/dist/upstream/lazy-client.js +53 -0
- package/dist/upstream/stdio-client.d.ts +57 -0
- package/dist/upstream/stdio-client.js +203 -0
- package/dist/util/log.d.ts +27 -0
- package/dist/util/log.js +51 -0
- package/dist/util/version.d.ts +2 -0
- package/dist/util/version.js +40 -0
- package/docs/BaseInstRunner.md +621 -0
- package/docs/calculatedReplay.md +1185 -0
- package/docs/calculatedReplayGuide.md +448 -0
- package/docs/installRun.md +413 -0
- package/docs/mcpmark.md +752 -0
- package/docs/quickstart.md +201 -0
- package/docs/t-bench.md +394 -0
- package/package.json +56 -0
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* logic — the four stored-JS evaluators (docs/calculatedReplay.md §9).
|
|
3
|
+
*
|
|
4
|
+
* A calculated scenario carries JavaScript function *bodies*, one per hook
|
|
5
|
+
* point, each ending in `return { … }`. They are compiled with `new Function`
|
|
6
|
+
* and run here.
|
|
7
|
+
*
|
|
8
|
+
* PORTED FROM THE SERVICE, DELIBERATELY. BaseIn's `src/scenarios/logic.ts` is
|
|
9
|
+
* the copy the service itself evaluates in a dry replay, so it is the one that
|
|
10
|
+
* defines what a scenario *means*. Porting RRepeat's near-identical copy instead
|
|
11
|
+
* would let a scenario behave one way under `bir scenario replay --dry` and
|
|
12
|
+
* another way here — an untestable difference. Argument order and the
|
|
13
|
+
* plain-object guard both match the service exactly.
|
|
14
|
+
*
|
|
15
|
+
* THERE IS NO SANDBOX, and the doc says so plainly (§13.1). A `new Function`
|
|
16
|
+
* body gets its declared arguments and the global scope: `require`/`import` are
|
|
17
|
+
* unreachable under ESM, but `fetch`, `process` and `globalThis` are not. What
|
|
18
|
+
* makes this acceptable is provenance, not containment — the payload arrives
|
|
19
|
+
* only in an authenticated response from the configured `BIR_AUTH_URL`,
|
|
20
|
+
* describing the caller's own recorded runs — and replay is opt-in for exactly
|
|
21
|
+
* this reason.
|
|
22
|
+
*/
|
|
23
|
+
/** Every logic body must return a plain object; anything else is a bug upstream. */
|
|
24
|
+
function assertPlainObject(result, label) {
|
|
25
|
+
if (typeof result !== "object" || result === null || Array.isArray(result)) {
|
|
26
|
+
throw new Error(`${label} must return a plain object`);
|
|
27
|
+
}
|
|
28
|
+
return result;
|
|
29
|
+
}
|
|
30
|
+
/** Transform the derived parameters once, before any step runs. */
|
|
31
|
+
export function evalParamsLogic(code, parameters, intent) {
|
|
32
|
+
const fn = new Function("parameters", "intent", code);
|
|
33
|
+
return assertPlainObject(fn(parameters, intent), "paramsLogic");
|
|
34
|
+
}
|
|
35
|
+
/** Derive one step's tool input from params, intent and the accumulated respParams. */
|
|
36
|
+
export function evalToolInputLogic(code, parameters, intent, respParams = {}) {
|
|
37
|
+
const fn = new Function("parameters", "intent", "respParams", code);
|
|
38
|
+
return assertPlainObject(fn(parameters, intent, respParams), "toolInputLogic");
|
|
39
|
+
}
|
|
40
|
+
/** Extract keys from a step's real output, to merge into respParams for later steps. */
|
|
41
|
+
export function evalToolOutputLogic(code, toolOutput, parameters, intent, respParams = {}) {
|
|
42
|
+
const fn = new Function("toolOutput", "parameters", "intent", "respParams", code);
|
|
43
|
+
return assertPlainObject(fn(toolOutput, parameters, intent, respParams), "toolOutputLogic");
|
|
44
|
+
}
|
|
45
|
+
/** Build the final response model from everything the steps accumulated. */
|
|
46
|
+
export function evalResponseParamsLogic(code, respParams, parameters, intent) {
|
|
47
|
+
const fn = new Function("respParams", "parameters", "intent", code);
|
|
48
|
+
return assertPlainObject(fn(respParams, parameters, intent), "responseParamsLogic");
|
|
49
|
+
}
|
|
50
|
+
//# sourceMappingURL=logic.js.map
|
|
@@ -0,0 +1,181 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* ScenarioReplayPlan — the stateful driver for one matched turn
|
|
3
|
+
* (docs/calculatedReplay.md §9).
|
|
4
|
+
*
|
|
5
|
+
* In `direct` mode the whole plan runs inside one loop. In `steer` mode the same
|
|
6
|
+
* state is split across *separate HTTP requests* — `PreToolUse` computes step
|
|
7
|
+
* N's input, `PostToolUse` threads step N's output — which is why the plan is an
|
|
8
|
+
* object with a cursor rather than a function with a loop.
|
|
9
|
+
*
|
|
10
|
+
* THREE DIFFERENCES FROM RRepeat's `ScenarioReplayPlan`, each deliberate:
|
|
11
|
+
*
|
|
12
|
+
* 1. **Lifetime is a run, not a session.** The plan hangs off `RunState`. Here
|
|
13
|
+
* a run *is* a turn (`UserPromptSubmit → Stop`), which is exactly the plan's
|
|
14
|
+
* lifetime, so a second prompt cannot inherit a stale plan.
|
|
15
|
+
* 2. **`params` are a promise.** Arming must not wait on the derivation call —
|
|
16
|
+
* the prompt hook has a budget to keep (§10). `ready()` is what the first
|
|
17
|
+
* `PreToolUse` (or `/scenario/run`) awaits, and `paramsLogic` is applied
|
|
18
|
+
* once, there.
|
|
19
|
+
* 3. **`execute` takes a step, not a tool name.** The executor must see the
|
|
20
|
+
* step to know its reach, so it can choose between the proxy, a recorded
|
|
21
|
+
* output, and skipping.
|
|
22
|
+
*/
|
|
23
|
+
import type { ReplayMode } from "./coverage.js";
|
|
24
|
+
import type { ExecutionStage, SerializedScenario, SerializedScenarioStep } from "./types.js";
|
|
25
|
+
/**
|
|
26
|
+
* Runs one step's tool and returns its output as a string.
|
|
27
|
+
*
|
|
28
|
+
* Resolving with a failure message is a *result* — a tool that ran and failed.
|
|
29
|
+
* Rejecting means the step could not be **run here at all**, which is the only
|
|
30
|
+
* case a recorded output may stand in for.
|
|
31
|
+
*/
|
|
32
|
+
export type ExecuteStep = (step: SerializedScenarioStep, input: Record<string, unknown>) => Promise<string>;
|
|
33
|
+
/** Supplies a step's recorded output when its tool cannot run here (§8.1). */
|
|
34
|
+
export type RecordedOutputFor = (step: SerializedScenarioStep) => Promise<string | undefined>;
|
|
35
|
+
/**
|
|
36
|
+
* What one step did, told to the caller as it happens.
|
|
37
|
+
*
|
|
38
|
+
* `executed` / `recorded` / `skipped` describe where the *output* came from —
|
|
39
|
+
* the tool, the source run, or nowhere. `failed` is the different kind of news:
|
|
40
|
+
* the step's work did not happen. Either the scenario's **own logic** threw —
|
|
41
|
+
* emitted immediately before the throw propagates, the only moment anything
|
|
42
|
+
* knows which step it belonged to — or the tool ran and reported an error
|
|
43
|
+
* (stage `tool_call`), in which case the chain carries on, threading the error
|
|
44
|
+
* like any other output, but nothing may call the plan a success.
|
|
45
|
+
*/
|
|
46
|
+
export interface StepInfo {
|
|
47
|
+
step: SerializedScenarioStep;
|
|
48
|
+
input: Record<string, unknown>;
|
|
49
|
+
outcome: "executed" | "recorded" | "skipped" | "failed";
|
|
50
|
+
/** Which part of the step broke. Only set for `failed`. */
|
|
51
|
+
stage?: ExecutionStage;
|
|
52
|
+
error?: string;
|
|
53
|
+
derivedKeys?: string[];
|
|
54
|
+
ms: number;
|
|
55
|
+
}
|
|
56
|
+
export type StepObserver = (info: StepInfo) => void;
|
|
57
|
+
export interface ComposeResult {
|
|
58
|
+
text: string;
|
|
59
|
+
executed: number;
|
|
60
|
+
recorded: number;
|
|
61
|
+
skipped: number;
|
|
62
|
+
/**
|
|
63
|
+
* Of `executed`, how many tools ran and reported an error. Counted separately
|
|
64
|
+
* because the tool *did* run — the step is in the bundle, its output threaded —
|
|
65
|
+
* and yet the work did not happen, and an outcome must not say it did.
|
|
66
|
+
*/
|
|
67
|
+
errored: number;
|
|
68
|
+
}
|
|
69
|
+
export interface PlanOptions {
|
|
70
|
+
scenario: SerializedScenario;
|
|
71
|
+
/** Resolves with the derived parameters. Rejection degrades to `{}`. */
|
|
72
|
+
params: Promise<Record<string, unknown>>;
|
|
73
|
+
mode: ReplayMode;
|
|
74
|
+
respParamsInit?: Record<string, unknown>;
|
|
75
|
+
}
|
|
76
|
+
export declare class ScenarioReplayPlan {
|
|
77
|
+
readonly scenarioId: string;
|
|
78
|
+
readonly runId: string;
|
|
79
|
+
readonly mode: ReplayMode;
|
|
80
|
+
readonly intent: string;
|
|
81
|
+
private readonly scenario;
|
|
82
|
+
private readonly steps;
|
|
83
|
+
private readonly paramsPromise;
|
|
84
|
+
private params;
|
|
85
|
+
private respParams;
|
|
86
|
+
private stepIndex;
|
|
87
|
+
private readyPromise?;
|
|
88
|
+
constructor(opts: PlanOptions);
|
|
89
|
+
/**
|
|
90
|
+
* Await the derivation, then apply `paramsLogic` — once, however many callers
|
|
91
|
+
* race here. A rejected derivation resolves to `{}` rather than throwing: the
|
|
92
|
+
* caller decided long ago that a replay with fallback parameters beats no
|
|
93
|
+
* replay, and a throw here would abort a turn the user is waiting on.
|
|
94
|
+
*/
|
|
95
|
+
ready(): Promise<void>;
|
|
96
|
+
/** 0-based index of the step awaiting execution (== completed step count). */
|
|
97
|
+
get currentStepIndex(): number;
|
|
98
|
+
get stepCount(): number;
|
|
99
|
+
/** The parameters in force. Empty until {@link ready} resolves. */
|
|
100
|
+
get parameters(): Readonly<Record<string, unknown>>;
|
|
101
|
+
/** Everything the steps have accumulated so far. */
|
|
102
|
+
get accumulated(): Readonly<Record<string, unknown>>;
|
|
103
|
+
/** The step awaiting execution, or undefined when the plan is done. */
|
|
104
|
+
currentStep(): SerializedScenarioStep | undefined;
|
|
105
|
+
/** The tool the current step expects, or undefined when done. */
|
|
106
|
+
expectedTool(): string | undefined;
|
|
107
|
+
/** True once every step has been applied — the plan should be retired. */
|
|
108
|
+
isDone(): boolean;
|
|
109
|
+
/** Every step in order — for the directive, and for coverage reporting. */
|
|
110
|
+
allSteps(): readonly SerializedScenarioStep[];
|
|
111
|
+
/**
|
|
112
|
+
* True when `toolName` is one of this scenario's own tools.
|
|
113
|
+
*
|
|
114
|
+
* The question "is this call part of the plan?" is not the same as "is this
|
|
115
|
+
* the call I expected next", and conflating them is what made a plan die on
|
|
116
|
+
* `ToolSearch`. A host has housekeeping tools of its own — tool search, todo
|
|
117
|
+
* lists, file reads while it thinks — and none of them mean the model has
|
|
118
|
+
* abandoned the sequence.
|
|
119
|
+
*/
|
|
120
|
+
usesTool(toolName: string): boolean;
|
|
121
|
+
/**
|
|
122
|
+
* Compute the pinned input for the current step. Throws when there is no
|
|
123
|
+
* current step, or when the step's `toolInputLogic` fails — the caller treats
|
|
124
|
+
* a throw as divergence.
|
|
125
|
+
*/
|
|
126
|
+
toolInputForCurrentStep(): Record<string, unknown>;
|
|
127
|
+
/**
|
|
128
|
+
* Thread a real tool output into `respParams` and advance.
|
|
129
|
+
*
|
|
130
|
+
* `realOutput` must be serialized the way the step's output was **recorded** —
|
|
131
|
+
* for a wrapped MCP step that is the proxy's whole `CallToolResult`, not the
|
|
132
|
+
* host's rendering of it (§7.2). Returns the keys derived, for the audit line.
|
|
133
|
+
* Throws when `toolOutputLogic` fails; the caller aborts to a normal turn.
|
|
134
|
+
*/
|
|
135
|
+
applyOutput(realOutput: string): string[];
|
|
136
|
+
/** Build the final response model from everything the steps accumulated. */
|
|
137
|
+
responseModel(): Record<string, unknown>;
|
|
138
|
+
/**
|
|
139
|
+
* The directive injected via `UserPromptSubmit`'s `additionalContext`.
|
|
140
|
+
*
|
|
141
|
+
* In `direct` mode it asks for one call to the first-party tool, which runs the
|
|
142
|
+
* whole sequence server-side. In `steer` mode it names the sequence so the
|
|
143
|
+
* model initiates the expected calls — the arguments are supplied by the
|
|
144
|
+
* system, so the model need not compute them.
|
|
145
|
+
*/
|
|
146
|
+
steeringDirective(directToolName?: string): string;
|
|
147
|
+
/**
|
|
148
|
+
* Execute every **remaining** step and lay the results out as one bundle.
|
|
149
|
+
*
|
|
150
|
+
* Iteration starts at {@link currentStepIndex}, not at 0: in a steered turn the
|
|
151
|
+
* live session already ran — with side effects — every step before the one it
|
|
152
|
+
* diverged on, so re-running them would be wrong.
|
|
153
|
+
*
|
|
154
|
+
* There is no try/catch around the input or output logic: a throw there
|
|
155
|
+
* propagates out, and the caller abandons the whole bundle rather than
|
|
156
|
+
* shipping a half-computed one. The tool call itself is the exception — a
|
|
157
|
+
* rejection means the step could not be run *here*, and then a recorded output
|
|
158
|
+
* is not a shortcut past a real result, it is the only result that exists.
|
|
159
|
+
* Skipping instead would drop the step *and* stop threading `respParams`, so
|
|
160
|
+
* every later step reading from it fails too.
|
|
161
|
+
*/
|
|
162
|
+
composeBundle(maxChars: number, execute: ExecuteStep, recordedOutputFor?: RecordedOutputFor, onStep?: StepObserver): Promise<ComposeResult>;
|
|
163
|
+
/**
|
|
164
|
+
* Run the whole plan from the current cursor, advancing it as it goes.
|
|
165
|
+
*
|
|
166
|
+
* This is `direct` mode's driver, and unlike {@link composeBundle} it mutates
|
|
167
|
+
* the plan — the steps really did run, so the cursor and `respParams` must say
|
|
168
|
+
* so, which is what lets the outcome be `steered_full` and `stepsPinned` be
|
|
169
|
+
* truthful. A step that cannot run at all still advances the cursor: the plan
|
|
170
|
+
* is spent either way, and stalling it would leave a retired turn looking live.
|
|
171
|
+
*
|
|
172
|
+
* `maxChars` caps the bundle exactly as a divergence bundle is capped. The
|
|
173
|
+
* direct path once passed `Number.MAX_SAFE_INTEGER` here — fine at 10 KB, and
|
|
174
|
+
* an unbounded payload into the model's context for a scenario whose tools
|
|
175
|
+
* answer in megabytes (docs/mcpmark.md §12).
|
|
176
|
+
*/
|
|
177
|
+
runToCompletion(execute: ExecuteStep, recordedOutputFor?: RecordedOutputFor, onStep?: StepObserver, deadline?: number, maxChars?: number): Promise<ComposeResult & {
|
|
178
|
+
partial: boolean;
|
|
179
|
+
}>;
|
|
180
|
+
}
|
|
181
|
+
//# sourceMappingURL=plan.d.ts.map
|
|
@@ -0,0 +1,397 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* ScenarioReplayPlan — the stateful driver for one matched turn
|
|
3
|
+
* (docs/calculatedReplay.md §9).
|
|
4
|
+
*
|
|
5
|
+
* In `direct` mode the whole plan runs inside one loop. In `steer` mode the same
|
|
6
|
+
* state is split across *separate HTTP requests* — `PreToolUse` computes step
|
|
7
|
+
* N's input, `PostToolUse` threads step N's output — which is why the plan is an
|
|
8
|
+
* object with a cursor rather than a function with a loop.
|
|
9
|
+
*
|
|
10
|
+
* THREE DIFFERENCES FROM RRepeat's `ScenarioReplayPlan`, each deliberate:
|
|
11
|
+
*
|
|
12
|
+
* 1. **Lifetime is a run, not a session.** The plan hangs off `RunState`. Here
|
|
13
|
+
* a run *is* a turn (`UserPromptSubmit → Stop`), which is exactly the plan's
|
|
14
|
+
* lifetime, so a second prompt cannot inherit a stale plan.
|
|
15
|
+
* 2. **`params` are a promise.** Arming must not wait on the derivation call —
|
|
16
|
+
* the prompt hook has a budget to keep (§10). `ready()` is what the first
|
|
17
|
+
* `PreToolUse` (or `/scenario/run`) awaits, and `paramsLogic` is applied
|
|
18
|
+
* once, there.
|
|
19
|
+
* 3. **`execute` takes a step, not a tool name.** The executor must see the
|
|
20
|
+
* step to know its reach, so it can choose between the proxy, a recorded
|
|
21
|
+
* output, and skipping.
|
|
22
|
+
*/
|
|
23
|
+
import { evalParamsLogic, evalResponseParamsLogic, evalToolInputLogic, evalToolOutputLogic, } from "./logic.js";
|
|
24
|
+
import { assembleBundle, bundleInput, MAX_REPLAY_REASON } from "./bundle.js";
|
|
25
|
+
import { toolResultError } from "./tool-error.js";
|
|
26
|
+
/**
|
|
27
|
+
* Evaluate one of the scenario's logic bodies, naming the step and the stage to
|
|
28
|
+
* the observer if it throws — then rethrow, unchanged and uncaught.
|
|
29
|
+
*
|
|
30
|
+
* The rethrow is the point. The caller's contract is unaltered: a logic failure
|
|
31
|
+
* still abandons the whole bundle rather than shipping a half-computed one. All
|
|
32
|
+
* this adds is that the failure is now *attributable* — "step 2's tool input
|
|
33
|
+
* logic", rather than a stack trace the console cannot line up against a chain.
|
|
34
|
+
*/
|
|
35
|
+
function reportingStage(onStep, step, stage, input, startedAt, fn) {
|
|
36
|
+
try {
|
|
37
|
+
return fn();
|
|
38
|
+
}
|
|
39
|
+
catch (err) {
|
|
40
|
+
onStep?.({
|
|
41
|
+
step,
|
|
42
|
+
input,
|
|
43
|
+
outcome: "failed",
|
|
44
|
+
stage,
|
|
45
|
+
error: err instanceof Error ? err.message : String(err),
|
|
46
|
+
ms: Date.now() - startedAt,
|
|
47
|
+
});
|
|
48
|
+
throw err;
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
export class ScenarioReplayPlan {
|
|
52
|
+
scenarioId;
|
|
53
|
+
runId;
|
|
54
|
+
mode;
|
|
55
|
+
intent;
|
|
56
|
+
scenario;
|
|
57
|
+
steps;
|
|
58
|
+
paramsPromise;
|
|
59
|
+
params = {};
|
|
60
|
+
respParams;
|
|
61
|
+
stepIndex = 0;
|
|
62
|
+
readyPromise;
|
|
63
|
+
constructor(opts) {
|
|
64
|
+
this.scenario = opts.scenario;
|
|
65
|
+
this.scenarioId = opts.scenario.id;
|
|
66
|
+
this.runId = opts.scenario.runId;
|
|
67
|
+
this.intent = opts.scenario.intent ?? "";
|
|
68
|
+
this.steps = opts.scenario.steps ?? [];
|
|
69
|
+
this.mode = opts.mode;
|
|
70
|
+
this.paramsPromise = opts.params;
|
|
71
|
+
this.respParams = { ...(opts.respParamsInit ?? {}) };
|
|
72
|
+
}
|
|
73
|
+
/**
|
|
74
|
+
* Await the derivation, then apply `paramsLogic` — once, however many callers
|
|
75
|
+
* race here. A rejected derivation resolves to `{}` rather than throwing: the
|
|
76
|
+
* caller decided long ago that a replay with fallback parameters beats no
|
|
77
|
+
* replay, and a throw here would abort a turn the user is waiting on.
|
|
78
|
+
*/
|
|
79
|
+
ready() {
|
|
80
|
+
if (!this.readyPromise) {
|
|
81
|
+
this.readyPromise = this.paramsPromise
|
|
82
|
+
.catch(() => ({}))
|
|
83
|
+
.then((derived) => {
|
|
84
|
+
let params = { ...derived };
|
|
85
|
+
if (this.scenario.paramsLogic) {
|
|
86
|
+
// A throw here is real: the scenario's own logic is broken, and the
|
|
87
|
+
// caller must abort rather than run steps with unprocessed params.
|
|
88
|
+
params = evalParamsLogic(this.scenario.paramsLogic, params, this.intent);
|
|
89
|
+
}
|
|
90
|
+
this.params = params;
|
|
91
|
+
});
|
|
92
|
+
}
|
|
93
|
+
return this.readyPromise;
|
|
94
|
+
}
|
|
95
|
+
/** 0-based index of the step awaiting execution (== completed step count). */
|
|
96
|
+
get currentStepIndex() {
|
|
97
|
+
return this.stepIndex;
|
|
98
|
+
}
|
|
99
|
+
get stepCount() {
|
|
100
|
+
return this.steps.length;
|
|
101
|
+
}
|
|
102
|
+
/** The parameters in force. Empty until {@link ready} resolves. */
|
|
103
|
+
get parameters() {
|
|
104
|
+
return this.params;
|
|
105
|
+
}
|
|
106
|
+
/** Everything the steps have accumulated so far. */
|
|
107
|
+
get accumulated() {
|
|
108
|
+
return this.respParams;
|
|
109
|
+
}
|
|
110
|
+
/** The step awaiting execution, or undefined when the plan is done. */
|
|
111
|
+
currentStep() {
|
|
112
|
+
return this.steps[this.stepIndex];
|
|
113
|
+
}
|
|
114
|
+
/** The tool the current step expects, or undefined when done. */
|
|
115
|
+
expectedTool() {
|
|
116
|
+
return this.steps[this.stepIndex]?.toolName;
|
|
117
|
+
}
|
|
118
|
+
/** True once every step has been applied — the plan should be retired. */
|
|
119
|
+
isDone() {
|
|
120
|
+
return this.stepIndex >= this.steps.length;
|
|
121
|
+
}
|
|
122
|
+
/** Every step in order — for the directive, and for coverage reporting. */
|
|
123
|
+
allSteps() {
|
|
124
|
+
return this.steps;
|
|
125
|
+
}
|
|
126
|
+
/**
|
|
127
|
+
* True when `toolName` is one of this scenario's own tools.
|
|
128
|
+
*
|
|
129
|
+
* The question "is this call part of the plan?" is not the same as "is this
|
|
130
|
+
* the call I expected next", and conflating them is what made a plan die on
|
|
131
|
+
* `ToolSearch`. A host has housekeeping tools of its own — tool search, todo
|
|
132
|
+
* lists, file reads while it thinks — and none of them mean the model has
|
|
133
|
+
* abandoned the sequence.
|
|
134
|
+
*/
|
|
135
|
+
usesTool(toolName) {
|
|
136
|
+
return this.steps.some((s) => s.toolName === toolName);
|
|
137
|
+
}
|
|
138
|
+
/**
|
|
139
|
+
* Compute the pinned input for the current step. Throws when there is no
|
|
140
|
+
* current step, or when the step's `toolInputLogic` fails — the caller treats
|
|
141
|
+
* a throw as divergence.
|
|
142
|
+
*/
|
|
143
|
+
toolInputForCurrentStep() {
|
|
144
|
+
const step = this.steps[this.stepIndex];
|
|
145
|
+
if (!step)
|
|
146
|
+
throw new Error("no current step to compute input for");
|
|
147
|
+
return evalToolInputLogic(step.toolInputLogic, this.params, this.intent, this.respParams);
|
|
148
|
+
}
|
|
149
|
+
/**
|
|
150
|
+
* Thread a real tool output into `respParams` and advance.
|
|
151
|
+
*
|
|
152
|
+
* `realOutput` must be serialized the way the step's output was **recorded** —
|
|
153
|
+
* for a wrapped MCP step that is the proxy's whole `CallToolResult`, not the
|
|
154
|
+
* host's rendering of it (§7.2). Returns the keys derived, for the audit line.
|
|
155
|
+
* Throws when `toolOutputLogic` fails; the caller aborts to a normal turn.
|
|
156
|
+
*/
|
|
157
|
+
applyOutput(realOutput) {
|
|
158
|
+
const step = this.steps[this.stepIndex];
|
|
159
|
+
if (!step)
|
|
160
|
+
return [];
|
|
161
|
+
let derivedKeys = [];
|
|
162
|
+
if (step.toolOutputLogic) {
|
|
163
|
+
const derived = evalToolOutputLogic(step.toolOutputLogic, realOutput, this.params, this.intent, this.respParams);
|
|
164
|
+
derivedKeys = Object.keys(derived);
|
|
165
|
+
this.respParams = { ...this.respParams, ...derived };
|
|
166
|
+
}
|
|
167
|
+
this.stepIndex += 1;
|
|
168
|
+
return derivedKeys;
|
|
169
|
+
}
|
|
170
|
+
/** Build the final response model from everything the steps accumulated. */
|
|
171
|
+
responseModel() {
|
|
172
|
+
if (!this.scenario.responseParamsLogic)
|
|
173
|
+
return { ...this.respParams };
|
|
174
|
+
return evalResponseParamsLogic(this.scenario.responseParamsLogic, this.respParams, this.params, this.intent);
|
|
175
|
+
}
|
|
176
|
+
/**
|
|
177
|
+
* The directive injected via `UserPromptSubmit`'s `additionalContext`.
|
|
178
|
+
*
|
|
179
|
+
* In `direct` mode it asks for one call to the first-party tool, which runs the
|
|
180
|
+
* whole sequence server-side. In `steer` mode it names the sequence so the
|
|
181
|
+
* model initiates the expected calls — the arguments are supplied by the
|
|
182
|
+
* system, so the model need not compute them.
|
|
183
|
+
*/
|
|
184
|
+
steeringDirective(directToolName = "mcp__bir__run_scenario") {
|
|
185
|
+
// The intent is a paragraph the analyser wrote; it is context, not an
|
|
186
|
+
// instruction, so it is clipped rather than dumped whole into the prompt.
|
|
187
|
+
const intent = this.intent.length > 240 ? `${this.intent.slice(0, 240)}…` : this.intent;
|
|
188
|
+
const lines = this.steps.map((s, i) => {
|
|
189
|
+
const why = s.reasoning ? ` — ${s.reasoning}` : "";
|
|
190
|
+
return ` ${i + 1}. ${s.toolName}${why}`;
|
|
191
|
+
});
|
|
192
|
+
if (this.mode === "direct") {
|
|
193
|
+
// The sequence is listed even though the model does not call it itself:
|
|
194
|
+
// it is about to receive these results, and knowing what was run is what
|
|
195
|
+
// lets it tell a complete answer from a partial one. It is also the only
|
|
196
|
+
// place an operator watching the session can see what a replay is doing.
|
|
197
|
+
return [
|
|
198
|
+
"[BaseInstRunner calculated replay] A known-good tool sequence was recorded",
|
|
199
|
+
`for this request${intent ? ` (intent: ${intent})` : ""}.`,
|
|
200
|
+
"",
|
|
201
|
+
"It runs these tools, in order, and returns their results:",
|
|
202
|
+
"",
|
|
203
|
+
...lines,
|
|
204
|
+
"",
|
|
205
|
+
`Call ${directToolName} once, with no arguments, before any other tool.`,
|
|
206
|
+
"Then answer the user's request from the results it returns.",
|
|
207
|
+
].join("\n");
|
|
208
|
+
}
|
|
209
|
+
return [
|
|
210
|
+
"[BaseInstRunner calculated replay] A known-good tool sequence was previously",
|
|
211
|
+
"recorded for this request. Call exactly these tools, in order, one at a time:",
|
|
212
|
+
"",
|
|
213
|
+
...lines,
|
|
214
|
+
...(intent ? ["", `Context: ${intent}`] : []),
|
|
215
|
+
"",
|
|
216
|
+
"The system supplies the exact arguments for each call — you do not need to",
|
|
217
|
+
"compute them. Do not call any other tools until this sequence is complete,",
|
|
218
|
+
"then answer the user's request from the tool results.",
|
|
219
|
+
].join("\n");
|
|
220
|
+
}
|
|
221
|
+
/**
|
|
222
|
+
* Execute every **remaining** step and lay the results out as one bundle.
|
|
223
|
+
*
|
|
224
|
+
* Iteration starts at {@link currentStepIndex}, not at 0: in a steered turn the
|
|
225
|
+
* live session already ran — with side effects — every step before the one it
|
|
226
|
+
* diverged on, so re-running them would be wrong.
|
|
227
|
+
*
|
|
228
|
+
* There is no try/catch around the input or output logic: a throw there
|
|
229
|
+
* propagates out, and the caller abandons the whole bundle rather than
|
|
230
|
+
* shipping a half-computed one. The tool call itself is the exception — a
|
|
231
|
+
* rejection means the step could not be run *here*, and then a recorded output
|
|
232
|
+
* is not a shortcut past a real result, it is the only result that exists.
|
|
233
|
+
* Skipping instead would drop the step *and* stop threading `respParams`, so
|
|
234
|
+
* every later step reading from it fails too.
|
|
235
|
+
*/
|
|
236
|
+
async composeBundle(maxChars, execute, recordedOutputFor, onStep) {
|
|
237
|
+
// A local copy: composing must not corrupt the live plan's accumulation if
|
|
238
|
+
// the caller decides to keep steering afterwards.
|
|
239
|
+
const respParams = { ...this.respParams };
|
|
240
|
+
const entries = [];
|
|
241
|
+
let executed = 0;
|
|
242
|
+
let recordedCount = 0;
|
|
243
|
+
let skipped = 0;
|
|
244
|
+
let errored = 0;
|
|
245
|
+
for (const step of this.steps.slice(this.stepIndex)) {
|
|
246
|
+
const startedAt = Date.now();
|
|
247
|
+
const computed = reportingStage(onStep, step, "tool_input_logic", {}, startedAt, () => evalToolInputLogic(step.toolInputLogic, this.params, this.intent, respParams));
|
|
248
|
+
let response;
|
|
249
|
+
let recorded = false;
|
|
250
|
+
try {
|
|
251
|
+
response = await execute(step, computed);
|
|
252
|
+
executed += 1;
|
|
253
|
+
}
|
|
254
|
+
catch (err) {
|
|
255
|
+
const error = err instanceof Error ? err.message : String(err);
|
|
256
|
+
const fallback = (typeof step.recordedOutput === "string" && step.recordedOutput.length > 0
|
|
257
|
+
? step.recordedOutput
|
|
258
|
+
: undefined) ?? (await recordedOutputFor?.(step).catch(() => undefined));
|
|
259
|
+
if (!fallback) {
|
|
260
|
+
skipped += 1;
|
|
261
|
+
onStep?.({ step, input: computed, outcome: "skipped", error, ms: Date.now() - startedAt });
|
|
262
|
+
continue;
|
|
263
|
+
}
|
|
264
|
+
response = fallback;
|
|
265
|
+
recorded = true;
|
|
266
|
+
recordedCount += 1;
|
|
267
|
+
onStep?.({ step, input: computed, outcome: "recorded", error, ms: Date.now() - startedAt });
|
|
268
|
+
}
|
|
269
|
+
// Thread the output for later steps' inputs. A recorded output threads
|
|
270
|
+
// too: `toolOutputLogic` was authored against exactly this shape, and a
|
|
271
|
+
// stale value beats a missing one downstream.
|
|
272
|
+
let derivedKeys = [];
|
|
273
|
+
if (step.toolOutputLogic) {
|
|
274
|
+
const derived = reportingStage(onStep, step, "tool_output_logic", computed, startedAt, () => evalToolOutputLogic(step.toolOutputLogic, response, this.params, this.intent, respParams));
|
|
275
|
+
derivedKeys = Object.keys(derived);
|
|
276
|
+
Object.assign(respParams, derived);
|
|
277
|
+
}
|
|
278
|
+
if (!recorded) {
|
|
279
|
+
// A step's verdict is its output, not its resolution: a tool that ran
|
|
280
|
+
// and answered "Error: …" resolves like any other, and a chain of eight
|
|
281
|
+
// of them once logged ok=true eight times (docs/mcpmark.md §13).
|
|
282
|
+
const error = toolResultError(response);
|
|
283
|
+
if (error)
|
|
284
|
+
errored += 1;
|
|
285
|
+
onStep?.({
|
|
286
|
+
step,
|
|
287
|
+
input: computed,
|
|
288
|
+
outcome: error ? "failed" : "executed",
|
|
289
|
+
stage: error ? "tool_call" : undefined,
|
|
290
|
+
error,
|
|
291
|
+
derivedKeys,
|
|
292
|
+
ms: Date.now() - startedAt,
|
|
293
|
+
});
|
|
294
|
+
}
|
|
295
|
+
entries.push({
|
|
296
|
+
toolName: step.toolName,
|
|
297
|
+
input: bundleInput(computed),
|
|
298
|
+
response,
|
|
299
|
+
recorded,
|
|
300
|
+
});
|
|
301
|
+
}
|
|
302
|
+
return {
|
|
303
|
+
text: assembleBundle(entries, maxChars),
|
|
304
|
+
executed,
|
|
305
|
+
recorded: recordedCount,
|
|
306
|
+
skipped,
|
|
307
|
+
errored,
|
|
308
|
+
};
|
|
309
|
+
}
|
|
310
|
+
/**
|
|
311
|
+
* Run the whole plan from the current cursor, advancing it as it goes.
|
|
312
|
+
*
|
|
313
|
+
* This is `direct` mode's driver, and unlike {@link composeBundle} it mutates
|
|
314
|
+
* the plan — the steps really did run, so the cursor and `respParams` must say
|
|
315
|
+
* so, which is what lets the outcome be `steered_full` and `stepsPinned` be
|
|
316
|
+
* truthful. A step that cannot run at all still advances the cursor: the plan
|
|
317
|
+
* is spent either way, and stalling it would leave a retired turn looking live.
|
|
318
|
+
*
|
|
319
|
+
* `maxChars` caps the bundle exactly as a divergence bundle is capped. The
|
|
320
|
+
* direct path once passed `Number.MAX_SAFE_INTEGER` here — fine at 10 KB, and
|
|
321
|
+
* an unbounded payload into the model's context for a scenario whose tools
|
|
322
|
+
* answer in megabytes (docs/mcpmark.md §12).
|
|
323
|
+
*/
|
|
324
|
+
async runToCompletion(execute, recordedOutputFor, onStep, deadline, maxChars = MAX_REPLAY_REASON) {
|
|
325
|
+
const entries = [];
|
|
326
|
+
let executed = 0;
|
|
327
|
+
let recordedCount = 0;
|
|
328
|
+
let skipped = 0;
|
|
329
|
+
let errored = 0;
|
|
330
|
+
let partial = false;
|
|
331
|
+
while (!this.isDone()) {
|
|
332
|
+
if (deadline !== undefined && Date.now() >= deadline) {
|
|
333
|
+
partial = true;
|
|
334
|
+
break;
|
|
335
|
+
}
|
|
336
|
+
const step = this.steps[this.stepIndex];
|
|
337
|
+
const startedAt = Date.now();
|
|
338
|
+
const computed = reportingStage(onStep, step, "tool_input_logic", {}, startedAt, () => this.toolInputForCurrentStep());
|
|
339
|
+
let response;
|
|
340
|
+
let recorded = false;
|
|
341
|
+
try {
|
|
342
|
+
response = await execute(step, computed);
|
|
343
|
+
executed += 1;
|
|
344
|
+
}
|
|
345
|
+
catch (err) {
|
|
346
|
+
const error = err instanceof Error ? err.message : String(err);
|
|
347
|
+
const fallback = (typeof step.recordedOutput === "string" && step.recordedOutput.length > 0
|
|
348
|
+
? step.recordedOutput
|
|
349
|
+
: undefined) ?? (await recordedOutputFor?.(step).catch(() => undefined));
|
|
350
|
+
if (!fallback) {
|
|
351
|
+
skipped += 1;
|
|
352
|
+
onStep?.({ step, input: computed, outcome: "skipped", error, ms: Date.now() - startedAt });
|
|
353
|
+
// Advance past a step that cannot run, without threading anything.
|
|
354
|
+
this.stepIndex += 1;
|
|
355
|
+
continue;
|
|
356
|
+
}
|
|
357
|
+
response = fallback;
|
|
358
|
+
recorded = true;
|
|
359
|
+
recordedCount += 1;
|
|
360
|
+
onStep?.({ step, input: computed, outcome: "recorded", error, ms: Date.now() - startedAt });
|
|
361
|
+
}
|
|
362
|
+
const derivedKeys = reportingStage(onStep, step, "tool_output_logic", computed, startedAt, () => this.applyOutput(response));
|
|
363
|
+
if (!recorded) {
|
|
364
|
+
// A step's verdict is its output, not its resolution: a tool that ran
|
|
365
|
+
// and answered "Error: …" resolves like any other, and a chain of eight
|
|
366
|
+
// of them once logged ok=true eight times (docs/mcpmark.md §13).
|
|
367
|
+
const error = toolResultError(response);
|
|
368
|
+
if (error)
|
|
369
|
+
errored += 1;
|
|
370
|
+
onStep?.({
|
|
371
|
+
step,
|
|
372
|
+
input: computed,
|
|
373
|
+
outcome: error ? "failed" : "executed",
|
|
374
|
+
stage: error ? "tool_call" : undefined,
|
|
375
|
+
error,
|
|
376
|
+
derivedKeys,
|
|
377
|
+
ms: Date.now() - startedAt,
|
|
378
|
+
});
|
|
379
|
+
}
|
|
380
|
+
entries.push({
|
|
381
|
+
toolName: step.toolName,
|
|
382
|
+
input: bundleInput(computed),
|
|
383
|
+
response,
|
|
384
|
+
recorded,
|
|
385
|
+
});
|
|
386
|
+
}
|
|
387
|
+
return {
|
|
388
|
+
text: assembleBundle(entries, maxChars),
|
|
389
|
+
executed,
|
|
390
|
+
recorded: recordedCount,
|
|
391
|
+
skipped,
|
|
392
|
+
errored,
|
|
393
|
+
partial,
|
|
394
|
+
};
|
|
395
|
+
}
|
|
396
|
+
}
|
|
397
|
+
//# sourceMappingURL=plan.js.map
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* pricing — token cost, in $/million tokens (docs/calculatedReplay.md §11.3).
|
|
3
|
+
*
|
|
4
|
+
* COPIED FROM THE SERVICE, NOT FROM RRepeat. A saving is `baseline − actual`,
|
|
5
|
+
* where the baseline is priced by BaseIn's `src/scenarios/cost.ts` and the actual
|
|
6
|
+
* by this file. The two are maintained by hand in separate repositories, so any
|
|
7
|
+
* drift between them is fake money — and {@link PRICING_VERSION} is stamped on
|
|
8
|
+
* every execution report so the server can store a mismatched one *unmeasured*
|
|
9
|
+
* rather than silently differencing two different tables.
|
|
10
|
+
*
|
|
11
|
+
* The service owns the baseline, so the service's table is the one to match.
|
|
12
|
+
* That is not a stylistic preference: at the time of writing, RRepeat and BaseIn
|
|
13
|
+
* both declare version `2026-08-30` while pricing `claude-opus-4-6` at 3.0/15.0
|
|
14
|
+
* and 5.0/25.0 respectively. Equal versions are supposed to certify that two
|
|
15
|
+
* numbers are comparable. For that row, right now, they are not.
|
|
16
|
+
*
|
|
17
|
+
* Bump the version — on **every** side — whenever a row changes.
|
|
18
|
+
*/
|
|
19
|
+
/** Identifies {@link MODEL_PRICING}. Must equal the service's constant. */
|
|
20
|
+
export declare const PRICING_VERSION = "2026-08-30";
|
|
21
|
+
/**
|
|
22
|
+
* Tolerate provider prefixes (`anthropic/`, `us.anthropic.`), date snapshots and
|
|
23
|
+
* unseen version bumps by falling back to the longest matching family, so a new
|
|
24
|
+
* snapshot never silently prices at $0.
|
|
25
|
+
*/
|
|
26
|
+
export declare function resolveModelPricing(model: string): {
|
|
27
|
+
input: number;
|
|
28
|
+
output: number;
|
|
29
|
+
cacheRead: number;
|
|
30
|
+
cacheWrite: number;
|
|
31
|
+
} | undefined;
|
|
32
|
+
export declare function hasModelPricing(model: string): boolean;
|
|
33
|
+
export interface TokenUsage {
|
|
34
|
+
inputTokens: number;
|
|
35
|
+
outputTokens: number;
|
|
36
|
+
cacheReadTokens?: number;
|
|
37
|
+
cacheCreationTokens?: number;
|
|
38
|
+
}
|
|
39
|
+
/** Cost in USD. An unpriced model yields 0 — the caller reports `measured: false`. */
|
|
40
|
+
export declare function calculateCostUsd(model: string, usage: TokenUsage): number;
|
|
41
|
+
//# sourceMappingURL=pricing.d.ts.map
|