@basein/runner 0.2.0 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -0
- package/dist/auth/client.d.ts +12 -0
- package/dist/auth/client.js +41 -0
- package/dist/bin/bir-hooks.d.ts +13 -0
- package/dist/bin/bir-hooks.js +58 -0
- package/dist/bin/bir.js +42 -2
- package/dist/control/server.d.ts +84 -1
- package/dist/control/server.js +546 -51
- package/dist/control/transcript.d.ts +40 -0
- package/dist/control/transcript.js +105 -0
- package/dist/record/recorder.d.ts +178 -4
- package/dist/record/recorder.js +6 -0
- package/dist/record/remote-recorder.d.ts +20 -2
- package/dist/record/remote-recorder.js +66 -6
- package/dist/replay/bundle.d.ts +10 -1
- package/dist/replay/bundle.js +41 -3
- package/dist/replay/controller.d.ts +164 -5
- package/dist/replay/controller.js +556 -54
- package/dist/replay/coverage.js +2 -2
- package/dist/replay/derive.d.ts +20 -1
- package/dist/replay/derive.js +69 -12
- package/dist/replay/flatten.d.ts +125 -0
- package/dist/replay/flatten.js +182 -0
- package/dist/replay/handover.d.ts +60 -0
- package/dist/replay/handover.js +82 -0
- package/dist/replay/logic.d.ts +11 -0
- package/dist/replay/logic.js +17 -0
- package/dist/replay/plan.d.ts +105 -8
- package/dist/replay/plan.js +309 -47
- package/dist/replay/source-run.d.ts +24 -10
- package/dist/replay/source-run.js +65 -30
- package/dist/replay/types.d.ts +108 -5
- package/dist/replay/types.js +33 -3
- package/package.json +1 -1
|
@@ -21,73 +21,103 @@
|
|
|
21
21
|
*
|
|
22
22
|
* Fetched **lazily** — only when a step actually needs it — and cached for the
|
|
23
23
|
* turn, so a fully-executable replay never makes this call at all.
|
|
24
|
+
*
|
|
25
|
+
* A chain with calls draws on **several** recordings: a called segment's steps
|
|
26
|
+
* take their outputs from that segment's own recording, within its own range
|
|
27
|
+
* (segmented.md R-CALL-31). So the rows are cached per run id, and the cursors
|
|
28
|
+
* are kept per frame — a segment called twice in one chain replays from the
|
|
29
|
+
* start of its range both times, as two separate replays of it would.
|
|
24
30
|
*/
|
|
25
31
|
import { logDetail, logLine, errText } from "../util/log.js";
|
|
26
|
-
export class
|
|
32
|
+
export class SourceRunCache {
|
|
27
33
|
opts;
|
|
28
|
-
/**
|
|
29
|
-
|
|
34
|
+
/** run id → every recorded output of that run, in order. */
|
|
35
|
+
runs = new Map();
|
|
36
|
+
loading = new Map();
|
|
37
|
+
/** `frameId#toolName` → how many of that tool the frame has drawn. */
|
|
30
38
|
cursor = new Map();
|
|
31
|
-
loading;
|
|
32
39
|
constructor(opts) {
|
|
33
40
|
this.opts = opts;
|
|
34
41
|
}
|
|
35
42
|
/**
|
|
36
|
-
* The next recorded output for
|
|
37
|
-
*
|
|
38
|
-
*
|
|
43
|
+
* The next recorded output for this entry's tool, from this entry's own
|
|
44
|
+
* recording and range, or undefined when there is none left (or the run could
|
|
45
|
+
* not be fetched). Never throws — a missing recorded output means the step is
|
|
46
|
+
* skipped, which the caller already handles.
|
|
39
47
|
*/
|
|
40
|
-
async outputFor(
|
|
48
|
+
async outputFor(entry) {
|
|
49
|
+
const runId = entry.runId || this.opts.runId;
|
|
50
|
+
const toolName = entry.step.toolName;
|
|
51
|
+
if (!toolName)
|
|
52
|
+
return undefined;
|
|
41
53
|
try {
|
|
42
|
-
await this.load();
|
|
54
|
+
await this.load(runId);
|
|
43
55
|
}
|
|
44
56
|
catch (err) {
|
|
45
57
|
logLine("replay.source_run_failed", {
|
|
46
|
-
run:
|
|
58
|
+
run: runId,
|
|
47
59
|
why: "no recorded outputs available for steps that cannot run here",
|
|
48
60
|
error: errText(err),
|
|
49
61
|
});
|
|
50
62
|
return undefined;
|
|
51
63
|
}
|
|
52
|
-
const
|
|
53
|
-
|
|
64
|
+
const all = this.runs.get(runId) ?? [];
|
|
65
|
+
const inRange = all.filter((o) => o.toolName === toolName &&
|
|
66
|
+
(entry.stepFrom === null ||
|
|
67
|
+
entry.stepTo === null ||
|
|
68
|
+
(o.position >= entry.stepFrom && o.position <= entry.stepTo)));
|
|
69
|
+
if (inRange.length === 0)
|
|
54
70
|
return undefined;
|
|
55
|
-
const
|
|
56
|
-
this.cursor.
|
|
57
|
-
|
|
71
|
+
const key = `${entry.frame.id}#${toolName}`;
|
|
72
|
+
const at = this.cursor.get(key) ?? 0;
|
|
73
|
+
this.cursor.set(key, at + 1);
|
|
74
|
+
return inRange[at]?.output;
|
|
58
75
|
}
|
|
59
|
-
load() {
|
|
60
|
-
|
|
61
|
-
|
|
76
|
+
load(runId) {
|
|
77
|
+
let pending = this.loading.get(runId);
|
|
78
|
+
if (!pending) {
|
|
79
|
+
pending = this.fetchRun(runId);
|
|
80
|
+
this.loading.set(runId, pending);
|
|
62
81
|
}
|
|
63
|
-
return
|
|
82
|
+
return pending;
|
|
64
83
|
}
|
|
65
|
-
async fetchRun() {
|
|
84
|
+
async fetchRun(runId) {
|
|
66
85
|
const doFetch = this.opts.fetchImpl ?? fetch;
|
|
67
86
|
const controller = new AbortController();
|
|
68
87
|
const timer = setTimeout(() => controller.abort(), this.opts.timeoutMs ?? 10_000);
|
|
69
88
|
timer.unref?.();
|
|
70
89
|
try {
|
|
71
|
-
const res = await doFetch(`${this.opts.baseUrl.replace(/\/+$/, "")}/recordings/runs/${
|
|
90
|
+
const res = await doFetch(`${this.opts.baseUrl.replace(/\/+$/, "")}/recordings/runs/${runId}`, {
|
|
72
91
|
headers: { authorization: `Bearer ${this.opts.token()}` },
|
|
73
92
|
signal: controller.signal,
|
|
74
93
|
});
|
|
75
94
|
if (!res.ok)
|
|
76
95
|
throw new Error(`HTTP ${res.status}`);
|
|
77
96
|
const body = (await res.json());
|
|
78
|
-
const
|
|
97
|
+
const outputs = [];
|
|
98
|
+
// A `tool_response` belongs to the `tool_selected` before it, and it is
|
|
99
|
+
// *that* row's position the range is cut on: a response row may sit just
|
|
100
|
+
// outside a range whose call sits inside it.
|
|
101
|
+
const lastCall = new Map();
|
|
79
102
|
for (const row of body.steps ?? []) {
|
|
80
|
-
if (
|
|
103
|
+
if (!row.tool_name)
|
|
104
|
+
continue;
|
|
105
|
+
if (row.type === "tool_selected") {
|
|
106
|
+
lastCall.set(row.tool_name, row.step_index ?? -1);
|
|
81
107
|
continue;
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
108
|
+
}
|
|
109
|
+
if (row.type !== "tool_response")
|
|
110
|
+
continue;
|
|
111
|
+
outputs.push({
|
|
112
|
+
toolName: row.tool_name,
|
|
113
|
+
output: row.tool_output ?? "{}",
|
|
114
|
+
position: lastCall.get(row.tool_name) ?? row.step_index ?? -1,
|
|
115
|
+
});
|
|
85
116
|
}
|
|
86
|
-
this.
|
|
117
|
+
this.runs.set(runId, outputs);
|
|
87
118
|
logDetail("replay.source_run_loaded", {
|
|
88
|
-
run:
|
|
89
|
-
|
|
90
|
-
outputs: [...map.values()].reduce((n, l) => n + l.length, 0),
|
|
119
|
+
run: runId,
|
|
120
|
+
outputs: outputs.length,
|
|
91
121
|
});
|
|
92
122
|
}
|
|
93
123
|
finally {
|
|
@@ -95,4 +125,9 @@ export class SourceRunOutputs {
|
|
|
95
125
|
}
|
|
96
126
|
}
|
|
97
127
|
}
|
|
128
|
+
/**
|
|
129
|
+
* The name this class had before one chain could draw on more than one
|
|
130
|
+
* recording. Kept so nothing outside has to change its import.
|
|
131
|
+
*/
|
|
132
|
+
export { SourceRunCache as SourceRunOutputs };
|
|
98
133
|
//# sourceMappingURL=source-run.js.map
|
package/dist/replay/types.d.ts
CHANGED
|
@@ -11,16 +11,55 @@
|
|
|
11
11
|
* `"ready"` carries an **empty** `steps` array, by the service's own rule.
|
|
12
12
|
*/
|
|
13
13
|
/** One step: the tool, and the JS logic that recomputes its arguments. */
|
|
14
|
+
/**
|
|
15
|
+
* A segment inlined into the chain that calls it (segmented.md R-CALL-28).
|
|
16
|
+
*
|
|
17
|
+
* The caller's chain holds one *call* row where a stretch of steps used to be,
|
|
18
|
+
* and the service serves that segment's own chain inside it. Nothing is copied
|
|
19
|
+
* into the caller's scenario: this is one payload, built at serving time from
|
|
20
|
+
* the segment as it stands right now.
|
|
21
|
+
*/
|
|
22
|
+
export interface InlinedSegment {
|
|
23
|
+
id: string;
|
|
24
|
+
intent: string;
|
|
25
|
+
paramsObject: ParamsSchema | null;
|
|
26
|
+
runId: string;
|
|
27
|
+
stepFrom: number;
|
|
28
|
+
stepTo: number;
|
|
29
|
+
/**
|
|
30
|
+
* The segment's `chain_revision` as served. Sent back on every verdict about
|
|
31
|
+
* one of its steps, so a failure counted against a chain that has since been
|
|
32
|
+
* rebuilt is dropped (R-CALL-13, R-REUSE-5).
|
|
33
|
+
*/
|
|
34
|
+
chainRevision: number;
|
|
35
|
+
/** May itself contain call rows, to the depth limit. */
|
|
36
|
+
steps: SerializedScenarioStep[];
|
|
37
|
+
}
|
|
38
|
+
/** Why a call cannot run right now (segmented.md 9.9). */
|
|
39
|
+
export type CallUnusable = "switched_off" | "not_ready" | "missing" | "stale" | "unresolved" | "depth" | "unsupported";
|
|
14
40
|
export interface SerializedScenarioStep {
|
|
15
41
|
stepIndex: number;
|
|
16
|
-
/**
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
42
|
+
/**
|
|
43
|
+
* `"segment"` is a call: a sub-task this chain runs by calling a recorded
|
|
44
|
+
* scenario instead of carrying a copy of its steps (R-CALL-1). Absent means
|
|
45
|
+
* `"tool"`, which is what every older service sends.
|
|
46
|
+
*/
|
|
47
|
+
kind?: "tool" | "segment";
|
|
48
|
+
/** As the recorder wrote it — `mcp__<server>__<tool>` for an MCP call. Null on a call row. */
|
|
49
|
+
toolName: string | null;
|
|
50
|
+
/** JS body: `(parameters, intent, respParams) => object`. Null on a call row. */
|
|
51
|
+
toolInputLogic: string | null;
|
|
20
52
|
/** JS body: `(toolOutput, parameters, intent, respParams) => object`. */
|
|
21
53
|
toolOutputLogic: string | null;
|
|
22
54
|
/** The model's reasoning for this step, used in the steering directive. */
|
|
23
55
|
reasoning: string | null;
|
|
56
|
+
/**
|
|
57
|
+
* Execution reports that blamed this step with a `failed` verdict, counted by
|
|
58
|
+
* the service (fallbk.md D1). A step whose count is above the match's
|
|
59
|
+
* `fallback.maxStepFailures` is not attempted: the plan hands the turn to the
|
|
60
|
+
* model in front of it. Absent from an older service, which means never park.
|
|
61
|
+
*/
|
|
62
|
+
failureCount?: number;
|
|
24
63
|
/**
|
|
25
64
|
* The literal output recorded for this step.
|
|
26
65
|
*
|
|
@@ -30,12 +69,50 @@ export interface SerializedScenarioStep {
|
|
|
30
69
|
* {@link ../replay/source-run.ts} is what actually supplies recorded outputs.
|
|
31
70
|
*/
|
|
32
71
|
recordedOutput?: string | null;
|
|
72
|
+
/** Which segment answers this call right now, if one does. */
|
|
73
|
+
segmentId?: string | null;
|
|
74
|
+
/** What the sub-task is, in one line. The ask, frozen with the chain. */
|
|
75
|
+
callIntent?: string | null;
|
|
76
|
+
/** How many of the caller's recorded steps the call covers. */
|
|
77
|
+
callStepCount?: number | null;
|
|
78
|
+
/** JS body: `(parameters, intent, respParams) => <the segment's parameters>`. */
|
|
79
|
+
paramMapLogic?: string | null;
|
|
80
|
+
/** JS body: `(out, parameters, intent, respParams) => <keys the caller reads>`. */
|
|
81
|
+
resultMapLogic?: string | null;
|
|
82
|
+
/** The segment, inlined. Absent when the call is `unusable`. */
|
|
83
|
+
segment?: InlinedSegment | null;
|
|
84
|
+
/**
|
|
85
|
+
* Set when the call cannot run: the plan stops in front of it and the agent
|
|
86
|
+
* does that stretch itself (R-CALL-29, R-FALL-5).
|
|
87
|
+
*/
|
|
88
|
+
unusable?: CallUnusable;
|
|
89
|
+
/** How the answer was reached. Read only for the `plan.armed` line (R-CALL-32). */
|
|
90
|
+
resolution?: {
|
|
91
|
+
state: string;
|
|
92
|
+
score?: number;
|
|
93
|
+
verdict?: string;
|
|
94
|
+
resolvedAt?: string;
|
|
95
|
+
} | null;
|
|
33
96
|
}
|
|
97
|
+
/**
|
|
98
|
+
* What a parameter *is* (segmented.md R-PARAM-1). A `target` is what the task
|
|
99
|
+
* acts on and changes between requests; a `setting` is a value a user would
|
|
100
|
+
* never state. A setting that cannot be found takes its recorded sample; a
|
|
101
|
+
* target that cannot be found stops the plan, because a guessed target is how a
|
|
102
|
+
* replay does the right work on the wrong thing.
|
|
103
|
+
*
|
|
104
|
+
* Absent means target (R-PARAM-3): an older service, or a scenario the service
|
|
105
|
+
* has not classified yet, sends no kind, and the strict reading is the safe one.
|
|
106
|
+
*/
|
|
107
|
+
export type ParameterKind = "target" | "setting";
|
|
34
108
|
/** The `paramsObject` schema: one entry per parameter the scenario takes. */
|
|
35
109
|
export type ParamsSchema = Record<string, {
|
|
36
110
|
sampleValue: unknown;
|
|
37
111
|
description: string;
|
|
112
|
+
kind?: ParameterKind;
|
|
38
113
|
}>;
|
|
114
|
+
/** Whether this schema has anything the runner may not guess (R-PARAM-4). */
|
|
115
|
+
export declare function hasTargets(schema: ParamsSchema | null | undefined): boolean;
|
|
39
116
|
/** A `ready` scenario, as serialized by the BaseIn service on a match. */
|
|
40
117
|
export interface SerializedScenario {
|
|
41
118
|
id: string;
|
|
@@ -64,7 +141,25 @@ export interface SerializedScenario {
|
|
|
64
141
|
* How a matched run's execution is classified when it is reported back.
|
|
65
142
|
* Mirrors the service's zod enum on `POST /scenarios/:id/executions`.
|
|
66
143
|
*/
|
|
67
|
-
export type ExecutionOutcome = "steered_full" | "diverged" | "not_steered" | "failed"
|
|
144
|
+
export type ExecutionOutcome = "steered_full" | "diverged" | "not_steered" | "failed"
|
|
145
|
+
/**
|
|
146
|
+
* The plan ran some steps, then handed the rest of the task to the model — a
|
|
147
|
+
* step broke, or the next one is known to (fallbk.md D5). Books a saving
|
|
148
|
+
* against the whole turn's cost, which may be negative.
|
|
149
|
+
*/
|
|
150
|
+
| "fell_back";
|
|
151
|
+
/** Why a plan handed the turn to the model (fallbk.md). */
|
|
152
|
+
export type FallbackKind =
|
|
153
|
+
/** A step was attempted and broke; the model continues from it. */
|
|
154
|
+
"step_failed"
|
|
155
|
+
/** The plan stopped in front of a step that has failed too often, without trying it. */
|
|
156
|
+
| "known_bad_step"
|
|
157
|
+
/**
|
|
158
|
+
* The plan stopped in front of a *call* that cannot run — its segment is
|
|
159
|
+
* gone, switched off, out of date, or nested too deep (segmented.md
|
|
160
|
+
* R-CALL-29). Like a known-bad step it counts nothing against any step.
|
|
161
|
+
*/
|
|
162
|
+
| "unusable_call";
|
|
68
163
|
/**
|
|
69
164
|
* Upgrade order. `not_steered` is the floor — the control group, and the safe
|
|
70
165
|
* answer if a turn dies mid-flight — and each later decision point can only move
|
|
@@ -110,6 +205,14 @@ export interface ExecutionStepResult {
|
|
|
110
205
|
/** Message only — never a tool payload; this is rendered in a web page. */
|
|
111
206
|
error?: string;
|
|
112
207
|
durationMs?: number;
|
|
208
|
+
/**
|
|
209
|
+
* The scenario this step belongs to, when it ran inside a called segment: a
|
|
210
|
+
* failure is counted where a repair would happen (segmented.md R-CALL-13).
|
|
211
|
+
* Absent on a step of the reported scenario's own chain.
|
|
212
|
+
*/
|
|
213
|
+
scenarioId?: string;
|
|
214
|
+
/** That segment's `chain_revision` as served. Absent at depth 0. */
|
|
215
|
+
chainRevision?: number;
|
|
113
216
|
}
|
|
114
217
|
/** Narrowing guard for a payload that is worth arming a plan from. */
|
|
115
218
|
export declare function isReadyScenario(value: unknown): value is SerializedScenario;
|
package/dist/replay/types.js
CHANGED
|
@@ -10,6 +10,12 @@
|
|
|
10
10
|
* everywhere and nothing is assumed present. A scenario whose `state` is not
|
|
11
11
|
* `"ready"` carries an **empty** `steps` array, by the service's own rule.
|
|
12
12
|
*/
|
|
13
|
+
/** Whether this schema has anything the runner may not guess (R-PARAM-4). */
|
|
14
|
+
export function hasTargets(schema) {
|
|
15
|
+
if (!schema)
|
|
16
|
+
return false;
|
|
17
|
+
return Object.values(schema).some((p) => p?.kind !== "setting");
|
|
18
|
+
}
|
|
13
19
|
/**
|
|
14
20
|
* Upgrade order. `not_steered` is the floor — the control group, and the safe
|
|
15
21
|
* answer if a turn dies mid-flight — and each later decision point can only move
|
|
@@ -18,8 +24,9 @@
|
|
|
18
24
|
export const OUTCOME_RANK = {
|
|
19
25
|
not_steered: 0,
|
|
20
26
|
failed: 1,
|
|
21
|
-
|
|
22
|
-
|
|
27
|
+
fell_back: 2,
|
|
28
|
+
diverged: 3,
|
|
29
|
+
steered_full: 4,
|
|
23
30
|
};
|
|
24
31
|
/** Narrowing guard for a payload that is worth arming a plan from. */
|
|
25
32
|
export function isReadyScenario(value) {
|
|
@@ -30,6 +37,29 @@ export function isReadyScenario(value) {
|
|
|
30
37
|
s.state === "ready" &&
|
|
31
38
|
Array.isArray(s.steps) &&
|
|
32
39
|
s.steps.length > 0 &&
|
|
33
|
-
s.steps.every(
|
|
40
|
+
s.steps.every(readyStep));
|
|
41
|
+
}
|
|
42
|
+
/**
|
|
43
|
+
* One step of a ready chain (segmented.md 10.6 step 1).
|
|
44
|
+
*
|
|
45
|
+
* A call row carries no tool of its own, so the old test — a string `toolName`
|
|
46
|
+
* and a string `toolInputLogic` — would decline every chain that contains one.
|
|
47
|
+
* A call is ready when it says it cannot run (the plan stops in front of it and
|
|
48
|
+
* the agent takes over there) or when it carries a segment whose own steps pass
|
|
49
|
+
* this same test. A call row with neither is a payload we cannot run.
|
|
50
|
+
*/
|
|
51
|
+
function readyStep(step) {
|
|
52
|
+
if (!step)
|
|
53
|
+
return false;
|
|
54
|
+
if (step.kind === "segment") {
|
|
55
|
+
if (step.unusable)
|
|
56
|
+
return true;
|
|
57
|
+
const segment = step.segment;
|
|
58
|
+
return (!!segment &&
|
|
59
|
+
Array.isArray(segment.steps) &&
|
|
60
|
+
segment.steps.length > 0 &&
|
|
61
|
+
segment.steps.every(readyStep));
|
|
62
|
+
}
|
|
63
|
+
return typeof step.toolName === "string" && typeof step.toolInputLogic === "string";
|
|
34
64
|
}
|
|
35
65
|
//# sourceMappingURL=types.js.map
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@basein/runner",
|
|
3
|
-
"version": "0.2.
|
|
3
|
+
"version": "0.2.1",
|
|
4
4
|
"description": "A recording MCP proxy: sits between any MCP client and its MCP servers, executes each call on the client's behalf, and records the run as a reusable BaseIn scenario.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "MIT",
|