@basein/runner 0.2.0 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -0
- package/dist/auth/client.d.ts +12 -0
- package/dist/auth/client.js +41 -0
- package/dist/bin/bir-hooks.d.ts +13 -0
- package/dist/bin/bir-hooks.js +58 -0
- package/dist/bin/bir.js +42 -2
- package/dist/control/server.d.ts +84 -1
- package/dist/control/server.js +546 -51
- package/dist/control/transcript.d.ts +40 -0
- package/dist/control/transcript.js +105 -0
- package/dist/record/recorder.d.ts +178 -4
- package/dist/record/recorder.js +6 -0
- package/dist/record/remote-recorder.d.ts +20 -2
- package/dist/record/remote-recorder.js +66 -6
- package/dist/replay/bundle.d.ts +10 -1
- package/dist/replay/bundle.js +41 -3
- package/dist/replay/controller.d.ts +164 -5
- package/dist/replay/controller.js +556 -54
- package/dist/replay/coverage.js +2 -2
- package/dist/replay/derive.d.ts +20 -1
- package/dist/replay/derive.js +69 -12
- package/dist/replay/flatten.d.ts +125 -0
- package/dist/replay/flatten.js +182 -0
- package/dist/replay/handover.d.ts +60 -0
- package/dist/replay/handover.js +82 -0
- package/dist/replay/logic.d.ts +11 -0
- package/dist/replay/logic.js +17 -0
- package/dist/replay/plan.d.ts +105 -8
- package/dist/replay/plan.js +309 -47
- package/dist/replay/source-run.d.ts +24 -10
- package/dist/replay/source-run.js +65 -30
- package/dist/replay/types.d.ts +108 -5
- package/dist/replay/types.js +33 -3
- package/package.json +1 -1
|
@@ -11,13 +11,14 @@
|
|
|
11
11
|
* degrades to "run the turn normally" or returns something the session can
|
|
12
12
|
* ignore. Nothing throws at a hook.
|
|
13
13
|
*/
|
|
14
|
-
import type { ExecutionReport } from "../record/recorder.js";
|
|
14
|
+
import type { ExecutionReport, MatchKey, MatchSegment, MatchVerdict } from "../record/recorder.js";
|
|
15
15
|
import { type ReplayMode, type StepReach } from "./coverage.js";
|
|
16
|
-
import { deriveParameters } from "./derive.js";
|
|
16
|
+
import { deriveParameters, type DeriveContext } from "./derive.js";
|
|
17
17
|
import { ProxyWorkQueue } from "./executor.js";
|
|
18
18
|
import { ScenarioReplayPlan } from "./plan.js";
|
|
19
19
|
import { SourceRunOutputs } from "./source-run.js";
|
|
20
|
-
import { type
|
|
20
|
+
import { type FlatStop } from "./flatten.js";
|
|
21
|
+
import { type ExecutionOutcome, type ExecutionStepResult, type FallbackKind, type SerializedScenario, type SerializedScenarioStep } from "./types.js";
|
|
21
22
|
/** The first-party tool a `direct` plan is delivered through (§6.3). */
|
|
22
23
|
export declare const DIRECT_TOOL_NAME = "mcp__bir__run_scenario";
|
|
23
24
|
/** How long a `/proxy/poll` is held open before it answers empty. */
|
|
@@ -38,6 +39,18 @@ export interface ReplayOptions {
|
|
|
38
39
|
enabled: boolean;
|
|
39
40
|
/** Minimum similarity to steer — deliberately above the service's detection threshold. */
|
|
40
41
|
minSimilarity: number;
|
|
42
|
+
/**
|
|
43
|
+
* Minimum similarity to steer a **segment** (`BIR_MIN_SEGMENT_STEER_SIMILARITY`,
|
|
44
|
+
* segmented.md R-OUT-11). The same number as the server's clear band, so a
|
|
45
|
+
* clear hand-out is always accepted and no score falls between "clear" and
|
|
46
|
+
* "accepted".
|
|
47
|
+
*/
|
|
48
|
+
minSegmentSimilarity?: number;
|
|
49
|
+
/**
|
|
50
|
+
* `BIR_SEGMENT_ARM`. Off means observe-only: the runner asks for no segment
|
|
51
|
+
* and arms none, and only logs what would have armed (R-OUT-10).
|
|
52
|
+
*/
|
|
53
|
+
segmentArm?: boolean;
|
|
41
54
|
/** `BIR_REPLAY_ALLOW_SERVERS`; undefined means every wrapped server. */
|
|
42
55
|
allowServers?: ReadonlySet<string>;
|
|
43
56
|
budgets?: Partial<ReplayBudgets>;
|
|
@@ -49,6 +62,46 @@ export interface ReplayOptions {
|
|
|
49
62
|
fetchImpl?: typeof fetch;
|
|
50
63
|
/** Anthropic key for derivation. Absent → recorded sample values, free. */
|
|
51
64
|
apiKey?: string;
|
|
65
|
+
/**
|
|
66
|
+
* Intent matching in the ReAct loop (fallbk.md §Runner 4): while the model is
|
|
67
|
+
* driving, each tool call's reasoning is matched against scenario intents.
|
|
68
|
+
* Only consulted when `enabled` is on as well. Omitted means on, with defaults.
|
|
69
|
+
*/
|
|
70
|
+
intentMatch?: Partial<IntentMatchOptions>;
|
|
71
|
+
}
|
|
72
|
+
export interface IntentMatchOptions {
|
|
73
|
+
/** `BIR_INTENT_MATCH`. */
|
|
74
|
+
enabled: boolean;
|
|
75
|
+
/**
|
|
76
|
+
* How long a `PreToolUse` waits for the service. A timeout is a miss.
|
|
77
|
+
*
|
|
78
|
+
* 4 000 ms, not the 1 500 of Part 1: a request whose best segment lands in
|
|
79
|
+
* the *not clear* band has one live question to ask under the server's own
|
|
80
|
+
* 2 500 ms attempt, and must still answer well inside the hook's 30 s
|
|
81
|
+
* (segmented.md R-HIT-11). A clear match or a miss answers as fast as before.
|
|
82
|
+
*/
|
|
83
|
+
budgetMs: number;
|
|
84
|
+
/** Plans intent matching may arm in one turn. */
|
|
85
|
+
maxPerTurn: number;
|
|
86
|
+
/**
|
|
87
|
+
* Probes one turn may *send*. Separate from `maxPerTurn`, and much larger:
|
|
88
|
+
* arming three plans is a claim on the turn, while counting hits is only
|
|
89
|
+
* bookkeeping, and it must not stop at the third arm (R-HIT-5).
|
|
90
|
+
*/
|
|
91
|
+
maxRequestsPerTurn: number;
|
|
92
|
+
}
|
|
93
|
+
export declare const DEFAULT_INTENT_MATCH: IntentMatchOptions;
|
|
94
|
+
/** The turn handed back to the model part-way (fallbk.md). */
|
|
95
|
+
export interface Handover {
|
|
96
|
+
kind: FallbackKind;
|
|
97
|
+
/** The scenario's own `stepIndex` of the step it stopped at. */
|
|
98
|
+
stepIndex: number;
|
|
99
|
+
toolName?: string;
|
|
100
|
+
error?: string;
|
|
101
|
+
/** What the model is told. Always built, even when delivery fails. */
|
|
102
|
+
note: string;
|
|
103
|
+
/** Set once the note reached the model, so it is never delivered twice. */
|
|
104
|
+
delivered: boolean;
|
|
52
105
|
}
|
|
53
106
|
/** One pinned call, remembered so its output is threaded from the right source. */
|
|
54
107
|
interface PinnedCall {
|
|
@@ -88,13 +141,76 @@ export interface ReplayState {
|
|
|
88
141
|
* `PreToolUse`, threaded in `PostToolUse`), and the second word on a step
|
|
89
142
|
* replaces the first rather than appending a second verdict for it.
|
|
90
143
|
*/
|
|
91
|
-
stepResults: Map<
|
|
144
|
+
stepResults: Map<string, ExecutionStepResult>;
|
|
92
145
|
armedAt: number;
|
|
93
146
|
reported: boolean;
|
|
94
147
|
sourceRun?: SourceRunOutputs;
|
|
148
|
+
/**
|
|
149
|
+
* Why the flattener stopped the plan, when it is what stopped it: a call this
|
|
150
|
+
* chain cannot run right now (segmented.md R-CALL-29). The note names the
|
|
151
|
+
* sub-task and the reason, and the report carries `unusable_call`.
|
|
152
|
+
*/
|
|
153
|
+
flatStop?: FlatStop;
|
|
95
154
|
/** Set once a plan has been retired, so nothing re-arms mid-turn. */
|
|
96
155
|
retired: boolean;
|
|
156
|
+
/** What armed it: the prompt, or a ReAct iteration's intent (fallbk.md). */
|
|
157
|
+
armedBy: "prompt" | "intent";
|
|
158
|
+
/**
|
|
159
|
+
* Which kind of row was handed out (segmented.md R-OUT-7). A `segment` is a
|
|
160
|
+
* named sub-task of a recording rather than a whole task, and it is judged
|
|
161
|
+
* against its own similarity gate. Absent from an older server, and then read
|
|
162
|
+
* as `scenario` (R-COMPAT-2).
|
|
163
|
+
*/
|
|
164
|
+
kind: "scenario" | "segment";
|
|
165
|
+
/** Where a handed-out segment sits in its recording (R-OUT-8). */
|
|
166
|
+
segment?: MatchSegment;
|
|
167
|
+
/** Which of the segment's two keys scored (R-OUT-3). For logs. */
|
|
168
|
+
key?: MatchKey;
|
|
169
|
+
/** The cheap model's answer, when the score sat in the not-clear band. */
|
|
170
|
+
verified?: MatchVerdict;
|
|
171
|
+
/**
|
|
172
|
+
* Segments of this plan's own recording that armed later in the same turn
|
|
173
|
+
* (R-MONEY-4). Their share comes off this plan's baseline, so one recording's
|
|
174
|
+
* steps are never counted in two baselines of one turn.
|
|
175
|
+
*/
|
|
176
|
+
sharedWith: string[];
|
|
177
|
+
/** Set when the plan handed the task to the model part-way (fallbk.md). */
|
|
178
|
+
handover?: Handover;
|
|
179
|
+
/**
|
|
180
|
+
* Which gate declined, as a fixed code (segmented.md R-HIT-14). The prose
|
|
181
|
+
* `why` that `decline()` logs stays free text; this is what code reads — the
|
|
182
|
+
* position rule (R-OUT-6) reports a position only for `known_bad_first_step`,
|
|
183
|
+
* and reading prose for that would break the first time a sentence was
|
|
184
|
+
* reworded.
|
|
185
|
+
*/
|
|
186
|
+
declined?: DeclineCode;
|
|
187
|
+
/**
|
|
188
|
+
* Opaque to the controller: the control server's transcript watermark taken
|
|
189
|
+
* when an intent match armed this, so each plan in a turn is billed for its
|
|
190
|
+
* own window and a turn's cost is never reported twice.
|
|
191
|
+
*/
|
|
192
|
+
usageMark?: unknown;
|
|
193
|
+
/**
|
|
194
|
+
* Takes another watermark. Set by the control server, which is the only side
|
|
195
|
+
* that holds the transcript path (segmented.md R-MONEY-3).
|
|
196
|
+
*/
|
|
197
|
+
markUsage?: () => unknown;
|
|
198
|
+
/**
|
|
199
|
+
* The watermark at a *completing* retire, and when it happened.
|
|
200
|
+
*
|
|
201
|
+
* A plan that finished its work stops costing at that moment: the agent's
|
|
202
|
+
* later work in the same turn is the agent's, not this plan's. A plan that
|
|
203
|
+
* handed over or diverged takes no mark, because the follow-on work is the
|
|
204
|
+
* direct consequence of its stopping (R-MONEY-3, R-FALL-5).
|
|
205
|
+
*/
|
|
206
|
+
retiredMark?: unknown;
|
|
207
|
+
retiredAt?: number;
|
|
97
208
|
}
|
|
209
|
+
/**
|
|
210
|
+
* Why a match did not arm. One code per gate of the ladder, so the caller can
|
|
211
|
+
* act on a decline without parsing the sentence that explains it.
|
|
212
|
+
*/
|
|
213
|
+
export type DeclineCode = "replay_disabled" | "not_ready" | "flatten_failed" | "similarity" | "known_bad_first_step" | "unusable_first_step" | "coverage" | "missing_target" | "no_derive_key";
|
|
98
214
|
/** What `PreToolUse` should do about this call. */
|
|
99
215
|
export type PreToolAction =
|
|
100
216
|
/** Pin the arguments and let the real tool run. */
|
|
@@ -125,8 +241,17 @@ export declare class ReplayController {
|
|
|
125
241
|
readonly enabled: boolean;
|
|
126
242
|
readonly work: ProxyWorkQueue;
|
|
127
243
|
readonly budgets: ReplayBudgets;
|
|
244
|
+
readonly intentMatch: IntentMatchOptions;
|
|
128
245
|
private readonly opts;
|
|
129
246
|
constructor(opts: ReplayOptions);
|
|
247
|
+
/**
|
|
248
|
+
* Whether a handed-out segment may actually arm (segmented.md R-OUT-10).
|
|
249
|
+
*
|
|
250
|
+
* Off by default: until an operator has read the observe-only logs and turned
|
|
251
|
+
* `BIR_SEGMENT_ARM=1` on, the runner asks for no segment and arms none. The
|
|
252
|
+
* server enforces the same thing from its side (R-OUT-9).
|
|
253
|
+
*/
|
|
254
|
+
get segmentArm(): boolean;
|
|
130
255
|
/** The plan's own delivery vehicle is never a scenario step. */
|
|
131
256
|
isDirectTool(toolName: string): boolean;
|
|
132
257
|
/**
|
|
@@ -144,9 +269,20 @@ export declare class ReplayController {
|
|
|
144
269
|
* baseline sample to contribute, and losing that is how a savings ledger ends
|
|
145
270
|
* up with a denominator nobody measured.
|
|
146
271
|
*/
|
|
147
|
-
arm(match: RunMatchLike, prompt: string, wrapped: ReadonlySet<string
|
|
272
|
+
arm(match: RunMatchLike, prompt: string, wrapped: ReadonlySet<string>, armedBy?: "prompt" | "intent", ctx?: DeriveContext): Promise<ReplayState>;
|
|
148
273
|
/** The directive to inject via `additionalContext`, or undefined when declined. */
|
|
149
274
|
directiveFor(state: ReplayState): string | undefined;
|
|
275
|
+
/**
|
|
276
|
+
* A hand-over note not yet delivered, claimed for delivery (fallbk.md D4).
|
|
277
|
+
* Returns it once and marks it delivered; undefined when there is none.
|
|
278
|
+
*/
|
|
279
|
+
takeNote(state: ReplayState | undefined): string | undefined;
|
|
280
|
+
/**
|
|
281
|
+
* Deliver text to the model in place of the tool call it was about to make:
|
|
282
|
+
* as genuine command output for `Bash`, as the denial reason otherwise — the
|
|
283
|
+
* two channels a divergence bundle already uses.
|
|
284
|
+
*/
|
|
285
|
+
deliverInstead(text: string, toolName: string): PreToolAction;
|
|
150
286
|
/**
|
|
151
287
|
* `PreToolUse`, while a plan is active.
|
|
152
288
|
*
|
|
@@ -195,6 +331,8 @@ export declare class ReplayController {
|
|
|
195
331
|
measured: boolean;
|
|
196
332
|
durationMs: number;
|
|
197
333
|
prompt?: string;
|
|
334
|
+
/** Every plan state of this turn, this one included (R-MONEY-5). */
|
|
335
|
+
siblings?: readonly ReplayState[];
|
|
198
336
|
}): ExecutionReport | undefined;
|
|
199
337
|
/**
|
|
200
338
|
* The step a failed replay is fairly blamed on: the first whose own logic
|
|
@@ -248,6 +386,16 @@ export declare class ReplayController {
|
|
|
248
386
|
* nor re-injects again.
|
|
249
387
|
*/
|
|
250
388
|
private diverge;
|
|
389
|
+
/**
|
|
390
|
+
* Hand the rest of the task to the model (fallbk.md D4/D5).
|
|
391
|
+
*
|
|
392
|
+
* Retires the plan and builds the note; delivery is the caller's, because the
|
|
393
|
+
* channel depends on the hook that noticed. The outcome is `fell_back` when at
|
|
394
|
+
* least one step did its work under the plan, and `failed` when none did — a
|
|
395
|
+
* scenario that did nothing has nothing to book, and is a baseline sample.
|
|
396
|
+
*/
|
|
397
|
+
private handOver;
|
|
398
|
+
private noteFor;
|
|
251
399
|
/**
|
|
252
400
|
* The executor handed to a plan: dispatch a step to the proxy that owns its
|
|
253
401
|
* upstream.
|
|
@@ -295,6 +443,17 @@ export interface RunMatchLike {
|
|
|
295
443
|
similarity: number;
|
|
296
444
|
scenario: Record<string, unknown> | null;
|
|
297
445
|
executionTicket?: string;
|
|
446
|
+
fallback?: {
|
|
447
|
+
maxStepFailures: number;
|
|
448
|
+
};
|
|
449
|
+
/** All absent from an older server (segmented.md R-COMPAT-2). */
|
|
450
|
+
kind?: "scenario" | "segment";
|
|
451
|
+
segment?: MatchSegment;
|
|
452
|
+
key?: MatchKey;
|
|
453
|
+
verified?: MatchVerdict;
|
|
454
|
+
/** The frozen name, beside the runtime intent the plan runs on (R-INTENT-12). */
|
|
455
|
+
intentName?: string;
|
|
298
456
|
}
|
|
457
|
+
export declare function knownBadStepIndex(steps: readonly SerializedScenarioStep[], maxStepFailures: number | undefined): number | undefined;
|
|
299
458
|
export {};
|
|
300
459
|
//# sourceMappingURL=controller.d.ts.map
|