@basein/runner 0.2.0 → 0.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -2
- package/dist/auth/client.d.ts +12 -0
- package/dist/auth/client.js +41 -0
- package/dist/bin/bir-hooks.d.ts +18 -3
- package/dist/bin/bir-hooks.js +70 -4
- package/dist/bin/bir.js +56 -3
- package/dist/control/server.d.ts +84 -1
- package/dist/control/server.js +555 -51
- package/dist/control/transcript.d.ts +40 -0
- package/dist/control/transcript.js +105 -0
- package/dist/record/recorder.d.ts +178 -4
- package/dist/record/recorder.js +6 -0
- package/dist/record/remote-recorder.d.ts +20 -2
- package/dist/record/remote-recorder.js +66 -6
- package/dist/replay/bundle.d.ts +10 -1
- package/dist/replay/bundle.js +41 -3
- package/dist/replay/controller.d.ts +179 -6
- package/dist/replay/controller.js +584 -55
- package/dist/replay/coverage.js +2 -2
- package/dist/replay/derive.d.ts +54 -7
- package/dist/replay/derive.js +174 -17
- package/dist/replay/flatten.d.ts +125 -0
- package/dist/replay/flatten.js +182 -0
- package/dist/replay/handover.d.ts +60 -0
- package/dist/replay/handover.js +82 -0
- package/dist/replay/logic.d.ts +11 -0
- package/dist/replay/logic.js +17 -0
- package/dist/replay/plan.d.ts +105 -8
- package/dist/replay/plan.js +309 -47
- package/dist/replay/source-run.d.ts +24 -10
- package/dist/replay/source-run.js +65 -30
- package/dist/replay/types.d.ts +108 -5
- package/dist/replay/types.js +33 -3
- package/docs/calculatedReplay.md +16 -8
- package/docs/calculatedReplayGuide.md +4 -4
- package/package.json +1 -1
|
@@ -11,13 +11,14 @@
|
|
|
11
11
|
* degrades to "run the turn normally" or returns something the session can
|
|
12
12
|
* ignore. Nothing throws at a hook.
|
|
13
13
|
*/
|
|
14
|
-
import type { ExecutionReport } from "../record/recorder.js";
|
|
14
|
+
import type { ExecutionReport, MatchKey, MatchSegment, MatchVerdict } from "../record/recorder.js";
|
|
15
15
|
import { type ReplayMode, type StepReach } from "./coverage.js";
|
|
16
|
-
import { deriveParameters } from "./derive.js";
|
|
16
|
+
import { deriveParameters, type DeriveContext } from "./derive.js";
|
|
17
17
|
import { ProxyWorkQueue } from "./executor.js";
|
|
18
18
|
import { ScenarioReplayPlan } from "./plan.js";
|
|
19
19
|
import { SourceRunOutputs } from "./source-run.js";
|
|
20
|
-
import { type
|
|
20
|
+
import { type FlatStop } from "./flatten.js";
|
|
21
|
+
import { type ExecutionOutcome, type ExecutionStepResult, type FallbackKind, type SerializedScenario, type SerializedScenarioStep } from "./types.js";
|
|
21
22
|
/** The first-party tool a `direct` plan is delivered through (§6.3). */
|
|
22
23
|
export declare const DIRECT_TOOL_NAME = "mcp__bir__run_scenario";
|
|
23
24
|
/** How long a `/proxy/poll` is held open before it answers empty. */
|
|
@@ -38,10 +39,25 @@ export interface ReplayOptions {
|
|
|
38
39
|
enabled: boolean;
|
|
39
40
|
/** Minimum similarity to steer — deliberately above the service's detection threshold. */
|
|
40
41
|
minSimilarity: number;
|
|
42
|
+
/**
|
|
43
|
+
* Minimum similarity to steer a **segment** (`BIR_MIN_SEGMENT_STEER_SIMILARITY`,
|
|
44
|
+
* segmented.md R-OUT-11). The same number as the server's clear band, so a
|
|
45
|
+
* clear hand-out is always accepted and no score falls between "clear" and
|
|
46
|
+
* "accepted".
|
|
47
|
+
*/
|
|
48
|
+
minSegmentSimilarity?: number;
|
|
49
|
+
/**
|
|
50
|
+
* `BIR_SEGMENT_ARM`. Off means observe-only: the runner asks for no segment
|
|
51
|
+
* and arms none, and only logs what would have armed (R-OUT-10).
|
|
52
|
+
*/
|
|
53
|
+
segmentArm?: boolean;
|
|
41
54
|
/** `BIR_REPLAY_ALLOW_SERVERS`; undefined means every wrapped server. */
|
|
42
55
|
allowServers?: ReadonlySet<string>;
|
|
43
56
|
budgets?: Partial<ReplayBudgets>;
|
|
44
|
-
/**
|
|
57
|
+
/**
|
|
58
|
+
* The service: the lazy source-run fetch, and the parameter derivation this
|
|
59
|
+
* runner no longer needs a key of its own for (segmented.md R-PARAM-5).
|
|
60
|
+
*/
|
|
45
61
|
authUrl?: string;
|
|
46
62
|
authToken?: () => string;
|
|
47
63
|
/** Injected by tests. */
|
|
@@ -49,6 +65,46 @@ export interface ReplayOptions {
|
|
|
49
65
|
fetchImpl?: typeof fetch;
|
|
50
66
|
/** Anthropic key for derivation. Absent → recorded sample values, free. */
|
|
51
67
|
apiKey?: string;
|
|
68
|
+
/**
|
|
69
|
+
* Intent matching in the ReAct loop (fallbk.md §Runner 4): while the model is
|
|
70
|
+
* driving, each tool call's reasoning is matched against scenario intents.
|
|
71
|
+
* Only consulted when `enabled` is on as well. Omitted means on, with defaults.
|
|
72
|
+
*/
|
|
73
|
+
intentMatch?: Partial<IntentMatchOptions>;
|
|
74
|
+
}
|
|
75
|
+
export interface IntentMatchOptions {
|
|
76
|
+
/** `BIR_INTENT_MATCH`. */
|
|
77
|
+
enabled: boolean;
|
|
78
|
+
/**
|
|
79
|
+
* How long a `PreToolUse` waits for the service. A timeout is a miss.
|
|
80
|
+
*
|
|
81
|
+
* 4 000 ms, not the 1 500 of Part 1: a request whose best segment lands in
|
|
82
|
+
* the *not clear* band has one live question to ask under the server's own
|
|
83
|
+
* 2 500 ms attempt, and must still answer well inside the hook's 30 s
|
|
84
|
+
* (segmented.md R-HIT-11). A clear match or a miss answers as fast as before.
|
|
85
|
+
*/
|
|
86
|
+
budgetMs: number;
|
|
87
|
+
/** Plans intent matching may arm in one turn. */
|
|
88
|
+
maxPerTurn: number;
|
|
89
|
+
/**
|
|
90
|
+
* Probes one turn may *send*. Separate from `maxPerTurn`, and much larger:
|
|
91
|
+
* arming three plans is a claim on the turn, while counting hits is only
|
|
92
|
+
* bookkeeping, and it must not stop at the third arm (R-HIT-5).
|
|
93
|
+
*/
|
|
94
|
+
maxRequestsPerTurn: number;
|
|
95
|
+
}
|
|
96
|
+
export declare const DEFAULT_INTENT_MATCH: IntentMatchOptions;
|
|
97
|
+
/** The turn handed back to the model part-way (fallbk.md). */
|
|
98
|
+
export interface Handover {
|
|
99
|
+
kind: FallbackKind;
|
|
100
|
+
/** The scenario's own `stepIndex` of the step it stopped at. */
|
|
101
|
+
stepIndex: number;
|
|
102
|
+
toolName?: string;
|
|
103
|
+
error?: string;
|
|
104
|
+
/** What the model is told. Always built, even when delivery fails. */
|
|
105
|
+
note: string;
|
|
106
|
+
/** Set once the note reached the model, so it is never delivered twice. */
|
|
107
|
+
delivered: boolean;
|
|
52
108
|
}
|
|
53
109
|
/** One pinned call, remembered so its output is threaded from the right source. */
|
|
54
110
|
interface PinnedCall {
|
|
@@ -88,13 +144,76 @@ export interface ReplayState {
|
|
|
88
144
|
* `PreToolUse`, threaded in `PostToolUse`), and the second word on a step
|
|
89
145
|
* replaces the first rather than appending a second verdict for it.
|
|
90
146
|
*/
|
|
91
|
-
stepResults: Map<
|
|
147
|
+
stepResults: Map<string, ExecutionStepResult>;
|
|
92
148
|
armedAt: number;
|
|
93
149
|
reported: boolean;
|
|
94
150
|
sourceRun?: SourceRunOutputs;
|
|
151
|
+
/**
|
|
152
|
+
* Why the flattener stopped the plan, when it is what stopped it: a call this
|
|
153
|
+
* chain cannot run right now (segmented.md R-CALL-29). The note names the
|
|
154
|
+
* sub-task and the reason, and the report carries `unusable_call`.
|
|
155
|
+
*/
|
|
156
|
+
flatStop?: FlatStop;
|
|
95
157
|
/** Set once a plan has been retired, so nothing re-arms mid-turn. */
|
|
96
158
|
retired: boolean;
|
|
159
|
+
/** What armed it: the prompt, or a ReAct iteration's intent (fallbk.md). */
|
|
160
|
+
armedBy: "prompt" | "intent";
|
|
161
|
+
/**
|
|
162
|
+
* Which kind of row was handed out (segmented.md R-OUT-7). A `segment` is a
|
|
163
|
+
* named sub-task of a recording rather than a whole task, and it is judged
|
|
164
|
+
* against its own similarity gate. Absent from an older server, and then read
|
|
165
|
+
* as `scenario` (R-COMPAT-2).
|
|
166
|
+
*/
|
|
167
|
+
kind: "scenario" | "segment";
|
|
168
|
+
/** Where a handed-out segment sits in its recording (R-OUT-8). */
|
|
169
|
+
segment?: MatchSegment;
|
|
170
|
+
/** Which of the segment's two keys scored (R-OUT-3). For logs. */
|
|
171
|
+
key?: MatchKey;
|
|
172
|
+
/** The cheap model's answer, when the score sat in the not-clear band. */
|
|
173
|
+
verified?: MatchVerdict;
|
|
174
|
+
/**
|
|
175
|
+
* Segments of this plan's own recording that armed later in the same turn
|
|
176
|
+
* (R-MONEY-4). Their share comes off this plan's baseline, so one recording's
|
|
177
|
+
* steps are never counted in two baselines of one turn.
|
|
178
|
+
*/
|
|
179
|
+
sharedWith: string[];
|
|
180
|
+
/** Set when the plan handed the task to the model part-way (fallbk.md). */
|
|
181
|
+
handover?: Handover;
|
|
182
|
+
/**
|
|
183
|
+
* Which gate declined, as a fixed code (segmented.md R-HIT-14). The prose
|
|
184
|
+
* `why` that `decline()` logs stays free text; this is what code reads — the
|
|
185
|
+
* position rule (R-OUT-6) reports a position only for `known_bad_first_step`,
|
|
186
|
+
* and reading prose for that would break the first time a sentence was
|
|
187
|
+
* reworded.
|
|
188
|
+
*/
|
|
189
|
+
declined?: DeclineCode;
|
|
190
|
+
/**
|
|
191
|
+
* Opaque to the controller: the control server's transcript watermark taken
|
|
192
|
+
* when an intent match armed this, so each plan in a turn is billed for its
|
|
193
|
+
* own window and a turn's cost is never reported twice.
|
|
194
|
+
*/
|
|
195
|
+
usageMark?: unknown;
|
|
196
|
+
/**
|
|
197
|
+
* Takes another watermark. Set by the control server, which is the only side
|
|
198
|
+
* that holds the transcript path (segmented.md R-MONEY-3).
|
|
199
|
+
*/
|
|
200
|
+
markUsage?: () => unknown;
|
|
201
|
+
/**
|
|
202
|
+
* The watermark at a *completing* retire, and when it happened.
|
|
203
|
+
*
|
|
204
|
+
* A plan that finished its work stops costing at that moment: the agent's
|
|
205
|
+
* later work in the same turn is the agent's, not this plan's. A plan that
|
|
206
|
+
* handed over or diverged takes no mark, because the follow-on work is the
|
|
207
|
+
* direct consequence of its stopping (R-MONEY-3, R-FALL-5).
|
|
208
|
+
*/
|
|
209
|
+
retiredMark?: unknown;
|
|
210
|
+
retiredAt?: number;
|
|
97
211
|
}
|
|
212
|
+
/**
|
|
213
|
+
* Why a match did not arm. One code per gate of the ladder, so the caller can
|
|
214
|
+
* act on a decline without parsing the sentence that explains it.
|
|
215
|
+
*/
|
|
216
|
+
export type DeclineCode = "replay_disabled" | "not_ready" | "flatten_failed" | "similarity" | "known_bad_first_step" | "unusable_first_step" | "coverage" | "missing_target" | "no_derive_key";
|
|
98
217
|
/** What `PreToolUse` should do about this call. */
|
|
99
218
|
export type PreToolAction =
|
|
100
219
|
/** Pin the arguments and let the real tool run. */
|
|
@@ -125,8 +244,17 @@ export declare class ReplayController {
|
|
|
125
244
|
readonly enabled: boolean;
|
|
126
245
|
readonly work: ProxyWorkQueue;
|
|
127
246
|
readonly budgets: ReplayBudgets;
|
|
247
|
+
readonly intentMatch: IntentMatchOptions;
|
|
128
248
|
private readonly opts;
|
|
129
249
|
constructor(opts: ReplayOptions);
|
|
250
|
+
/**
|
|
251
|
+
* Whether a handed-out segment may actually arm (segmented.md R-OUT-10).
|
|
252
|
+
*
|
|
253
|
+
* Off by default: until an operator has read the observe-only logs and turned
|
|
254
|
+
* `BIR_SEGMENT_ARM=1` on, the runner asks for no segment and arms none. The
|
|
255
|
+
* server enforces the same thing from its side (R-OUT-9).
|
|
256
|
+
*/
|
|
257
|
+
get segmentArm(): boolean;
|
|
130
258
|
/** The plan's own delivery vehicle is never a scenario step. */
|
|
131
259
|
isDirectTool(toolName: string): boolean;
|
|
132
260
|
/**
|
|
@@ -144,9 +272,20 @@ export declare class ReplayController {
|
|
|
144
272
|
* baseline sample to contribute, and losing that is how a savings ledger ends
|
|
145
273
|
* up with a denominator nobody measured.
|
|
146
274
|
*/
|
|
147
|
-
arm(match: RunMatchLike, prompt: string, wrapped: ReadonlySet<string
|
|
275
|
+
arm(match: RunMatchLike, prompt: string, wrapped: ReadonlySet<string>, armedBy?: "prompt" | "intent", ctx?: DeriveContext): Promise<ReplayState>;
|
|
148
276
|
/** The directive to inject via `additionalContext`, or undefined when declined. */
|
|
149
277
|
directiveFor(state: ReplayState): string | undefined;
|
|
278
|
+
/**
|
|
279
|
+
* A hand-over note not yet delivered, claimed for delivery (fallbk.md D4).
|
|
280
|
+
* Returns it once and marks it delivered; undefined when there is none.
|
|
281
|
+
*/
|
|
282
|
+
takeNote(state: ReplayState | undefined): string | undefined;
|
|
283
|
+
/**
|
|
284
|
+
* Deliver text to the model in place of the tool call it was about to make:
|
|
285
|
+
* as genuine command output for `Bash`, as the denial reason otherwise — the
|
|
286
|
+
* two channels a divergence bundle already uses.
|
|
287
|
+
*/
|
|
288
|
+
deliverInstead(text: string, toolName: string): PreToolAction;
|
|
150
289
|
/**
|
|
151
290
|
* `PreToolUse`, while a plan is active.
|
|
152
291
|
*
|
|
@@ -195,6 +334,8 @@ export declare class ReplayController {
|
|
|
195
334
|
measured: boolean;
|
|
196
335
|
durationMs: number;
|
|
197
336
|
prompt?: string;
|
|
337
|
+
/** Every plan state of this turn, this one included (R-MONEY-5). */
|
|
338
|
+
siblings?: readonly ReplayState[];
|
|
198
339
|
}): ExecutionReport | undefined;
|
|
199
340
|
/**
|
|
200
341
|
* The step a failed replay is fairly blamed on: the first whose own logic
|
|
@@ -240,6 +381,17 @@ export declare class ReplayController {
|
|
|
240
381
|
* directive the moment the match lands; the first `PreToolUse` — or
|
|
241
382
|
* `/scenario/run`, which has no hook timeout at all — is where the wait lands.
|
|
242
383
|
*/
|
|
384
|
+
/**
|
|
385
|
+
* Where to ask the service to read this turn (segmented.md R-PARAM-5).
|
|
386
|
+
*
|
|
387
|
+
* Undefined when there is nobody to ask — an unauthenticated session, or one
|
|
388
|
+
* whose token has gone. `derive` then falls back to the recorded samples, and
|
|
389
|
+
* a scenario with a target declines, exactly as a keyless runner always did.
|
|
390
|
+
*
|
|
391
|
+
* The token is read here rather than captured, because the recorder refreshes
|
|
392
|
+
* it as a session outlives it.
|
|
393
|
+
*/
|
|
394
|
+
private deriveService;
|
|
243
395
|
private startDerivation;
|
|
244
396
|
/**
|
|
245
397
|
* Divergence (§8). Execute the remaining steps for real, then deliver.
|
|
@@ -248,6 +400,16 @@ export declare class ReplayController {
|
|
|
248
400
|
* nor re-injects again.
|
|
249
401
|
*/
|
|
250
402
|
private diverge;
|
|
403
|
+
/**
|
|
404
|
+
* Hand the rest of the task to the model (fallbk.md D4/D5).
|
|
405
|
+
*
|
|
406
|
+
* Retires the plan and builds the note; delivery is the caller's, because the
|
|
407
|
+
* channel depends on the hook that noticed. The outcome is `fell_back` when at
|
|
408
|
+
* least one step did its work under the plan, and `failed` when none did — a
|
|
409
|
+
* scenario that did nothing has nothing to book, and is a baseline sample.
|
|
410
|
+
*/
|
|
411
|
+
private handOver;
|
|
412
|
+
private noteFor;
|
|
251
413
|
/**
|
|
252
414
|
* The executor handed to a plan: dispatch a step to the proxy that owns its
|
|
253
415
|
* upstream.
|
|
@@ -295,6 +457,17 @@ export interface RunMatchLike {
|
|
|
295
457
|
similarity: number;
|
|
296
458
|
scenario: Record<string, unknown> | null;
|
|
297
459
|
executionTicket?: string;
|
|
460
|
+
fallback?: {
|
|
461
|
+
maxStepFailures: number;
|
|
462
|
+
};
|
|
463
|
+
/** All absent from an older server (segmented.md R-COMPAT-2). */
|
|
464
|
+
kind?: "scenario" | "segment";
|
|
465
|
+
segment?: MatchSegment;
|
|
466
|
+
key?: MatchKey;
|
|
467
|
+
verified?: MatchVerdict;
|
|
468
|
+
/** The frozen name, beside the runtime intent the plan runs on (R-INTENT-12). */
|
|
469
|
+
intentName?: string;
|
|
298
470
|
}
|
|
471
|
+
export declare function knownBadStepIndex(steps: readonly SerializedScenarioStep[], maxStepFailures: number | undefined): number | undefined;
|
|
299
472
|
export {};
|
|
300
473
|
//# sourceMappingURL=controller.d.ts.map
|