@basein/runner 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +201 -0
- package/README.md +276 -0
- package/dist/auth/client.d.ts +85 -0
- package/dist/auth/client.js +284 -0
- package/dist/bin/bir-hooks.d.ts +48 -0
- package/dist/bin/bir-hooks.js +201 -0
- package/dist/bin/bir-proxy.d.ts +45 -0
- package/dist/bin/bir-proxy.js +207 -0
- package/dist/bin/bir-scenario.d.ts +24 -0
- package/dist/bin/bir-scenario.js +177 -0
- package/dist/bin/bir.d.ts +21 -0
- package/dist/bin/bir.js +876 -0
- package/dist/config/adapters/claude-code.d.ts +76 -0
- package/dist/config/adapters/claude-code.js +181 -0
- package/dist/config/adapters/generic.d.ts +17 -0
- package/dist/config/adapters/generic.js +36 -0
- package/dist/config/generate.d.ts +127 -0
- package/dist/config/generate.js +114 -0
- package/dist/config/resolve.d.ts +68 -0
- package/dist/config/resolve.js +132 -0
- package/dist/control/client.d.ts +56 -0
- package/dist/control/client.js +86 -0
- package/dist/control/correlation.d.ts +86 -0
- package/dist/control/correlation.js +0 -0
- package/dist/control/discovery.d.ts +50 -0
- package/dist/control/discovery.js +123 -0
- package/dist/control/ordering.d.ts +38 -0
- package/dist/control/ordering.js +44 -0
- package/dist/control/paths.d.ts +32 -0
- package/dist/control/paths.js +56 -0
- package/dist/control/server.d.ts +272 -0
- package/dist/control/server.js +1131 -0
- package/dist/control/transcript.d.ts +75 -0
- package/dist/control/transcript.js +241 -0
- package/dist/index.d.ts +37 -0
- package/dist/index.js +32 -0
- package/dist/jsonrpc/framing.d.ts +49 -0
- package/dist/jsonrpc/framing.js +143 -0
- package/dist/jsonrpc/types.d.ts +52 -0
- package/dist/jsonrpc/types.js +46 -0
- package/dist/proxy/intercept.d.ts +55 -0
- package/dist/proxy/intercept.js +147 -0
- package/dist/proxy/relay.d.ts +97 -0
- package/dist/proxy/relay.js +166 -0
- package/dist/proxy/session.d.ts +116 -0
- package/dist/proxy/session.js +319 -0
- package/dist/record/housekeeping.d.ts +34 -0
- package/dist/record/housekeeping.js +39 -0
- package/dist/record/queue.d.ts +48 -0
- package/dist/record/queue.js +96 -0
- package/dist/record/recorder.d.ts +111 -0
- package/dist/record/recorder.js +39 -0
- package/dist/record/redact.d.ts +37 -0
- package/dist/record/redact.js +119 -0
- package/dist/record/remote-recorder.d.ts +110 -0
- package/dist/record/remote-recorder.js +301 -0
- package/dist/record/truncate.d.ts +36 -0
- package/dist/record/truncate.js +85 -0
- package/dist/replay/bundle.d.ts +36 -0
- package/dist/replay/bundle.js +89 -0
- package/dist/replay/controller.d.ts +300 -0
- package/dist/replay/controller.js +807 -0
- package/dist/replay/coverage.d.ts +41 -0
- package/dist/replay/coverage.js +56 -0
- package/dist/replay/derive.d.ts +58 -0
- package/dist/replay/derive.js +166 -0
- package/dist/replay/executor.d.ts +78 -0
- package/dist/replay/executor.js +233 -0
- package/dist/replay/logic.d.ts +31 -0
- package/dist/replay/logic.js +50 -0
- package/dist/replay/plan.d.ts +181 -0
- package/dist/replay/plan.js +397 -0
- package/dist/replay/pricing.d.ts +41 -0
- package/dist/replay/pricing.js +76 -0
- package/dist/replay/source-run.d.ts +50 -0
- package/dist/replay/source-run.js +98 -0
- package/dist/replay/tool-error.d.ts +22 -0
- package/dist/replay/tool-error.js +60 -0
- package/dist/replay/types.d.ts +116 -0
- package/dist/replay/types.js +35 -0
- package/dist/upstream/client.d.ts +78 -0
- package/dist/upstream/client.js +114 -0
- package/dist/upstream/http-client.d.ts +78 -0
- package/dist/upstream/http-client.js +261 -0
- package/dist/upstream/lazy-client.d.ts +31 -0
- package/dist/upstream/lazy-client.js +53 -0
- package/dist/upstream/stdio-client.d.ts +57 -0
- package/dist/upstream/stdio-client.js +203 -0
- package/dist/util/log.d.ts +27 -0
- package/dist/util/log.js +51 -0
- package/dist/util/version.d.ts +2 -0
- package/dist/util/version.js +40 -0
- package/docs/BaseInstRunner.md +621 -0
- package/docs/calculatedReplay.md +1185 -0
- package/docs/calculatedReplayGuide.md +448 -0
- package/docs/installRun.md +413 -0
- package/docs/mcpmark.md +752 -0
- package/docs/quickstart.md +201 -0
- package/docs/t-bench.md +394 -0
- package/package.json +56 -0
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* coverage — which rung of the ladder a scenario lands on (docs/calculatedReplay.md §5).
|
|
3
|
+
*
|
|
4
|
+
* The whole mode decision is a pure function of two things: the scenario's step
|
|
5
|
+
* list, and the set of server keys a `bir-proxy` has registered for this session.
|
|
6
|
+
* That is the §0 asymmetry made mechanical — a step whose tool we can reach on a
|
|
7
|
+
* connection we already hold costs nothing to run, and a step we cannot reach can
|
|
8
|
+
* only happen inside the live session.
|
|
9
|
+
*
|
|
10
|
+
* `live` covers three genuinely different cases and treats them identically,
|
|
11
|
+
* correctly: a built-in (`Bash`, `Read`, `Edit`, …), an MCP server the user chose
|
|
12
|
+
* not to wrap, and `claude-in-chrome` — which is `scope: "dynamic"`, appears in
|
|
13
|
+
* no config file, and therefore can never be wrapped at all.
|
|
14
|
+
*/
|
|
15
|
+
import type { SerializedScenarioStep } from "./types.js";
|
|
16
|
+
/** Where a step's tool can actually be executed. */
|
|
17
|
+
export type StepReach = "direct" | "live";
|
|
18
|
+
/** How the whole scenario will be run. */
|
|
19
|
+
export type ReplayMode = "direct" | "steer" | "none";
|
|
20
|
+
/**
|
|
21
|
+
* `direct` when the tool is `mcp__<server>__<tool>` for a server this control
|
|
22
|
+
* plane has a proxy for; `live` otherwise.
|
|
23
|
+
*
|
|
24
|
+
* `allow` is the `BIR_REPLAY_ALLOW_SERVERS` allowlist (§13.2, mitigation 2): a
|
|
25
|
+
* wrapped server outside it is deliberately demoted to `live`, so it can still be
|
|
26
|
+
* steered inside the session — where the user's own permission rules apply — but
|
|
27
|
+
* is never called unattended by us.
|
|
28
|
+
*/
|
|
29
|
+
export declare function reachOf(toolName: string, wrapped: ReadonlySet<string>, allow?: ReadonlySet<string>): StepReach;
|
|
30
|
+
/** The mode for a whole scenario: `direct` only when *every* step is reachable. */
|
|
31
|
+
export declare function modeFor(steps: readonly SerializedScenarioStep[], wrapped: ReadonlySet<string>, allow?: ReadonlySet<string>): ReplayMode;
|
|
32
|
+
/** Per-step reach, in step order — what `plan.armed` logs and `bir doctor` shows. */
|
|
33
|
+
export declare function coverageOf(steps: readonly SerializedScenarioStep[], wrapped: ReadonlySet<string>, allow?: ReadonlySet<string>): StepReach[];
|
|
34
|
+
/**
|
|
35
|
+
* Parse `BIR_REPLAY_ALLOW_SERVERS`. Unset (or empty) means *every wrapped
|
|
36
|
+
* server*, which is why the return is `undefined` rather than an empty set — an
|
|
37
|
+
* empty allowlist that silently allowed nothing would look like a broken replay,
|
|
38
|
+
* and one that silently allowed everything would defeat the flag.
|
|
39
|
+
*/
|
|
40
|
+
export declare function parseAllowList(value: string | undefined): ReadonlySet<string> | undefined;
|
|
41
|
+
//# sourceMappingURL=coverage.d.ts.map
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* coverage — which rung of the ladder a scenario lands on (docs/calculatedReplay.md §5).
|
|
3
|
+
*
|
|
4
|
+
* The whole mode decision is a pure function of two things: the scenario's step
|
|
5
|
+
* list, and the set of server keys a `bir-proxy` has registered for this session.
|
|
6
|
+
* That is the §0 asymmetry made mechanical — a step whose tool we can reach on a
|
|
7
|
+
* connection we already hold costs nothing to run, and a step we cannot reach can
|
|
8
|
+
* only happen inside the live session.
|
|
9
|
+
*
|
|
10
|
+
* `live` covers three genuinely different cases and treats them identically,
|
|
11
|
+
* correctly: a built-in (`Bash`, `Read`, `Edit`, …), an MCP server the user chose
|
|
12
|
+
* not to wrap, and `claude-in-chrome` — which is `scope: "dynamic"`, appears in
|
|
13
|
+
* no config file, and therefore can never be wrapped at all.
|
|
14
|
+
*/
|
|
15
|
+
import { parseQualifiedName } from "../control/correlation.js";
|
|
16
|
+
/**
|
|
17
|
+
* `direct` when the tool is `mcp__<server>__<tool>` for a server this control
|
|
18
|
+
* plane has a proxy for; `live` otherwise.
|
|
19
|
+
*
|
|
20
|
+
* `allow` is the `BIR_REPLAY_ALLOW_SERVERS` allowlist (§13.2, mitigation 2): a
|
|
21
|
+
* wrapped server outside it is deliberately demoted to `live`, so it can still be
|
|
22
|
+
* steered inside the session — where the user's own permission rules apply — but
|
|
23
|
+
* is never called unattended by us.
|
|
24
|
+
*/
|
|
25
|
+
export function reachOf(toolName, wrapped, allow) {
|
|
26
|
+
const mcp = parseQualifiedName(toolName);
|
|
27
|
+
if (!mcp || !wrapped.has(mcp.serverName))
|
|
28
|
+
return "live";
|
|
29
|
+
if (allow && !allow.has(mcp.serverName))
|
|
30
|
+
return "live";
|
|
31
|
+
return "direct";
|
|
32
|
+
}
|
|
33
|
+
/** The mode for a whole scenario: `direct` only when *every* step is reachable. */
|
|
34
|
+
export function modeFor(steps, wrapped, allow) {
|
|
35
|
+
if (steps.length === 0)
|
|
36
|
+
return "none";
|
|
37
|
+
return steps.every((s) => reachOf(s.toolName, wrapped, allow) === "direct") ? "direct" : "steer";
|
|
38
|
+
}
|
|
39
|
+
/** Per-step reach, in step order — what `plan.armed` logs and `bir doctor` shows. */
|
|
40
|
+
export function coverageOf(steps, wrapped, allow) {
|
|
41
|
+
return steps.map((s) => reachOf(s.toolName, wrapped, allow));
|
|
42
|
+
}
|
|
43
|
+
/**
|
|
44
|
+
* Parse `BIR_REPLAY_ALLOW_SERVERS`. Unset (or empty) means *every wrapped
|
|
45
|
+
* server*, which is why the return is `undefined` rather than an empty set — an
|
|
46
|
+
* empty allowlist that silently allowed nothing would look like a broken replay,
|
|
47
|
+
* and one that silently allowed everything would defeat the flag.
|
|
48
|
+
*/
|
|
49
|
+
export function parseAllowList(value) {
|
|
50
|
+
const names = (value ?? "")
|
|
51
|
+
.split(",")
|
|
52
|
+
.map((s) => s.trim())
|
|
53
|
+
.filter(Boolean);
|
|
54
|
+
return names.length > 0 ? new Set(names) : undefined;
|
|
55
|
+
}
|
|
56
|
+
//# sourceMappingURL=coverage.js.map
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* derive — fill a scenario's parameter schema from the live prompt
|
|
3
|
+
* (docs/calculatedReplay.md §16.3).
|
|
4
|
+
*
|
|
5
|
+
* The only tokens replay itself spends. One Haiku call, over raw `fetch`.
|
|
6
|
+
*
|
|
7
|
+
* NO SDK, ON PURPOSE. `bir-proxy` is spawned inside the host's process tree for
|
|
8
|
+
* *every* wrapped server; an npm install that drags a model SDK into that path is
|
|
9
|
+
* a startup-latency and supply-chain cost paid N times per session, for a feature
|
|
10
|
+
* most sessions never arm. This is ~40 lines of `fetch` and it keeps the package
|
|
11
|
+
* at zero runtime dependencies.
|
|
12
|
+
*
|
|
13
|
+
* NO KEY IS A SUPPORTED CONFIGURATION. Without `ANTHROPIC_API_KEY` the extraction
|
|
14
|
+
* is skipped entirely and every parameter takes the value it had in the recorded
|
|
15
|
+
* run — a plain replay of the recorded parameters, at zero cost. For a scenario
|
|
16
|
+
* whose parameters rarely change that is a complete, free replay; for one that
|
|
17
|
+
* keys off the prompt, it is why you want the key.
|
|
18
|
+
*
|
|
19
|
+
* The prompt and the parsing tolerances mirror RRepeat's
|
|
20
|
+
* `deriveParametersFromScenarioPayload` exactly, so the two produce the same
|
|
21
|
+
* parameters from the same prompt — otherwise a scenario would behave one way
|
|
22
|
+
* under RRepeat and another under BaseInstRunner.
|
|
23
|
+
*/
|
|
24
|
+
import type { ParamsSchema } from "./types.js";
|
|
25
|
+
export declare const DEFAULT_DERIVE_MODEL = "claude-haiku-4-5-20251001";
|
|
26
|
+
export interface DeriveOptions {
|
|
27
|
+
prompt: string;
|
|
28
|
+
intent: string;
|
|
29
|
+
paramsObject: ParamsSchema | null;
|
|
30
|
+
/** Absent → sample values, at zero cost. */
|
|
31
|
+
apiKey?: string;
|
|
32
|
+
model?: string;
|
|
33
|
+
signal?: AbortSignal;
|
|
34
|
+
/** Injected by tests. Defaults to global `fetch`. */
|
|
35
|
+
fetchImpl?: typeof fetch;
|
|
36
|
+
}
|
|
37
|
+
export interface DeriveResult {
|
|
38
|
+
params: Record<string, unknown>;
|
|
39
|
+
costUsd: number;
|
|
40
|
+
/** False when the values came from the recorded samples rather than the model. */
|
|
41
|
+
derived: boolean;
|
|
42
|
+
model?: string;
|
|
43
|
+
inputTokens?: number;
|
|
44
|
+
outputTokens?: number;
|
|
45
|
+
}
|
|
46
|
+
/**
|
|
47
|
+
* First balanced `{…}` in a text blob. Models wrap JSON in prose and fences more
|
|
48
|
+
* often than they emit it bare, and a failed parse here costs a whole replay.
|
|
49
|
+
*/
|
|
50
|
+
export declare function extractJsonObject(text: string): string | undefined;
|
|
51
|
+
/**
|
|
52
|
+
* Derive the scenario's parameters from `prompt`. Never throws: every failure
|
|
53
|
+
* path resolves with the recorded sample values, because a replay with stale
|
|
54
|
+
* parameters is worth more than no replay, and the caller's audit line is what
|
|
55
|
+
* records which happened.
|
|
56
|
+
*/
|
|
57
|
+
export declare function deriveParameters(opts: DeriveOptions): Promise<DeriveResult>;
|
|
58
|
+
//# sourceMappingURL=derive.d.ts.map
|
|
@@ -0,0 +1,166 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* derive — fill a scenario's parameter schema from the live prompt
|
|
3
|
+
* (docs/calculatedReplay.md §16.3).
|
|
4
|
+
*
|
|
5
|
+
* The only tokens replay itself spends. One Haiku call, over raw `fetch`.
|
|
6
|
+
*
|
|
7
|
+
* NO SDK, ON PURPOSE. `bir-proxy` is spawned inside the host's process tree for
|
|
8
|
+
* *every* wrapped server; an npm install that drags a model SDK into that path is
|
|
9
|
+
* a startup-latency and supply-chain cost paid N times per session, for a feature
|
|
10
|
+
* most sessions never arm. This is ~40 lines of `fetch` and it keeps the package
|
|
11
|
+
* at zero runtime dependencies.
|
|
12
|
+
*
|
|
13
|
+
* NO KEY IS A SUPPORTED CONFIGURATION. Without `ANTHROPIC_API_KEY` the extraction
|
|
14
|
+
* is skipped entirely and every parameter takes the value it had in the recorded
|
|
15
|
+
* run — a plain replay of the recorded parameters, at zero cost. For a scenario
|
|
16
|
+
* whose parameters rarely change that is a complete, free replay; for one that
|
|
17
|
+
* keys off the prompt, it is why you want the key.
|
|
18
|
+
*
|
|
19
|
+
* The prompt and the parsing tolerances mirror RRepeat's
|
|
20
|
+
* `deriveParametersFromScenarioPayload` exactly, so the two produce the same
|
|
21
|
+
* parameters from the same prompt — otherwise a scenario would behave one way
|
|
22
|
+
* under RRepeat and another under BaseInstRunner.
|
|
23
|
+
*/
|
|
24
|
+
import { calculateCostUsd } from "./pricing.js";
|
|
25
|
+
export const DEFAULT_DERIVE_MODEL = "claude-haiku-4-5-20251001";
|
|
26
|
+
const ANTHROPIC_URL = "https://api.anthropic.com/v1/messages";
|
|
27
|
+
const ANTHROPIC_VERSION = "2023-06-01";
|
|
28
|
+
/** Every parameter at its recorded sample value. The zero-cost fallback. */
|
|
29
|
+
function sampleValues(schema) {
|
|
30
|
+
const out = {};
|
|
31
|
+
for (const key of Object.keys(schema))
|
|
32
|
+
out[key] = schema[key].sampleValue;
|
|
33
|
+
return out;
|
|
34
|
+
}
|
|
35
|
+
/**
|
|
36
|
+
* First balanced `{…}` in a text blob. Models wrap JSON in prose and fences more
|
|
37
|
+
* often than they emit it bare, and a failed parse here costs a whole replay.
|
|
38
|
+
*/
|
|
39
|
+
export function extractJsonObject(text) {
|
|
40
|
+
const start = text.indexOf("{");
|
|
41
|
+
if (start === -1)
|
|
42
|
+
return undefined;
|
|
43
|
+
let depth = 0;
|
|
44
|
+
let inStr = false;
|
|
45
|
+
let esc = false;
|
|
46
|
+
for (let i = start; i < text.length; i += 1) {
|
|
47
|
+
const ch = text[i];
|
|
48
|
+
if (inStr) {
|
|
49
|
+
if (esc)
|
|
50
|
+
esc = false;
|
|
51
|
+
else if (ch === "\\")
|
|
52
|
+
esc = true;
|
|
53
|
+
else if (ch === '"')
|
|
54
|
+
inStr = false;
|
|
55
|
+
continue;
|
|
56
|
+
}
|
|
57
|
+
if (ch === '"')
|
|
58
|
+
inStr = true;
|
|
59
|
+
else if (ch === "{")
|
|
60
|
+
depth += 1;
|
|
61
|
+
else if (ch === "}") {
|
|
62
|
+
depth -= 1;
|
|
63
|
+
if (depth === 0)
|
|
64
|
+
return text.slice(start, i + 1);
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
return undefined;
|
|
68
|
+
}
|
|
69
|
+
function buildExtractionPrompt(prompt, intent, schema) {
|
|
70
|
+
const descriptions = Object.keys(schema)
|
|
71
|
+
.map((key) => {
|
|
72
|
+
const { description, sampleValue } = schema[key];
|
|
73
|
+
return `- ${key}: ${description} (sample: ${JSON.stringify(sampleValue)})`;
|
|
74
|
+
})
|
|
75
|
+
.join("\n");
|
|
76
|
+
return `You are extracting parameter values from a user prompt to execute a pre-defined scenario.
|
|
77
|
+
|
|
78
|
+
Scenario intent: ${intent}
|
|
79
|
+
|
|
80
|
+
Parameters needed (with sample values from past runs):
|
|
81
|
+
${descriptions}
|
|
82
|
+
|
|
83
|
+
User prompt:
|
|
84
|
+
${prompt}
|
|
85
|
+
|
|
86
|
+
Extract the parameter values from the user prompt. If a value is not explicitly mentioned, infer a reasonable default from the sample values above.
|
|
87
|
+
|
|
88
|
+
Respond ONLY with a valid JSON object mapping each parameter name to its value. No markdown fences, no prose.
|
|
89
|
+
Example: { "directory": "src/utils", "pattern": "*.ts" }`;
|
|
90
|
+
}
|
|
91
|
+
/**
|
|
92
|
+
* Derive the scenario's parameters from `prompt`. Never throws: every failure
|
|
93
|
+
* path resolves with the recorded sample values, because a replay with stale
|
|
94
|
+
* parameters is worth more than no replay, and the caller's audit line is what
|
|
95
|
+
* records which happened.
|
|
96
|
+
*/
|
|
97
|
+
export async function deriveParameters(opts) {
|
|
98
|
+
const schema = opts.paramsObject ?? {};
|
|
99
|
+
const keys = Object.keys(schema);
|
|
100
|
+
if (keys.length === 0)
|
|
101
|
+
return { params: {}, costUsd: 0, derived: false };
|
|
102
|
+
const apiKey = opts.apiKey ?? process.env.ANTHROPIC_API_KEY;
|
|
103
|
+
if (!apiKey)
|
|
104
|
+
return { params: sampleValues(schema), costUsd: 0, derived: false };
|
|
105
|
+
const model = opts.model ?? process.env.BIR_DERIVE_MODEL ?? DEFAULT_DERIVE_MODEL;
|
|
106
|
+
const doFetch = opts.fetchImpl ?? fetch;
|
|
107
|
+
const res = await doFetch(ANTHROPIC_URL, {
|
|
108
|
+
method: "POST",
|
|
109
|
+
headers: {
|
|
110
|
+
"content-type": "application/json",
|
|
111
|
+
"x-api-key": apiKey,
|
|
112
|
+
"anthropic-version": ANTHROPIC_VERSION,
|
|
113
|
+
},
|
|
114
|
+
body: JSON.stringify({
|
|
115
|
+
model,
|
|
116
|
+
max_tokens: 2024,
|
|
117
|
+
messages: [
|
|
118
|
+
{ role: "user", content: buildExtractionPrompt(opts.prompt, opts.intent, schema) },
|
|
119
|
+
],
|
|
120
|
+
}),
|
|
121
|
+
signal: opts.signal,
|
|
122
|
+
});
|
|
123
|
+
if (!res.ok) {
|
|
124
|
+
const detail = await res.text().catch(() => "");
|
|
125
|
+
throw new Error(`derive failed: HTTP ${res.status}${detail ? ` — ${detail.slice(0, 200)}` : ""}`);
|
|
126
|
+
}
|
|
127
|
+
const body = (await res.json());
|
|
128
|
+
const text = (body.content ?? [])
|
|
129
|
+
.filter((b) => b.type === "text")
|
|
130
|
+
.map((b) => b.text ?? "")
|
|
131
|
+
.join("");
|
|
132
|
+
const inputTokens = body.usage?.input_tokens ?? 0;
|
|
133
|
+
const outputTokens = body.usage?.output_tokens ?? 0;
|
|
134
|
+
const costUsd = calculateCostUsd(model, {
|
|
135
|
+
inputTokens,
|
|
136
|
+
outputTokens,
|
|
137
|
+
cacheReadTokens: body.usage?.cache_read_input_tokens ?? 0,
|
|
138
|
+
cacheCreationTokens: body.usage?.cache_creation_input_tokens ?? 0,
|
|
139
|
+
});
|
|
140
|
+
// `undefined` is not JSON, but models emit it; normalise before parsing so one
|
|
141
|
+
// stray token does not cost the whole extraction.
|
|
142
|
+
const normalized = text.replace(/:\s*undefined\b/g, ": null");
|
|
143
|
+
const cleaned = extractJsonObject(normalized) ??
|
|
144
|
+
normalized
|
|
145
|
+
.replace(/^```(?:json)?\s*/m, "")
|
|
146
|
+
.replace(/\s*```\s*$/m, "")
|
|
147
|
+
.trim();
|
|
148
|
+
let parsed;
|
|
149
|
+
try {
|
|
150
|
+
parsed = JSON.parse(cleaned);
|
|
151
|
+
}
|
|
152
|
+
catch {
|
|
153
|
+
throw new Error(`derive returned invalid JSON: ${text.slice(0, 200)}`);
|
|
154
|
+
}
|
|
155
|
+
if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) {
|
|
156
|
+
throw new Error("derive returned a non-object");
|
|
157
|
+
}
|
|
158
|
+
// A missing or null key falls back to its recorded sample, so a partial
|
|
159
|
+
// extraction still yields a runnable parameter set.
|
|
160
|
+
for (const key of keys) {
|
|
161
|
+
if (parsed[key] === null || parsed[key] === undefined)
|
|
162
|
+
parsed[key] = schema[key].sampleValue;
|
|
163
|
+
}
|
|
164
|
+
return { params: parsed, costUsd, derived: true, model, inputTokens, outputTokens };
|
|
165
|
+
}
|
|
166
|
+
//# sourceMappingURL=derive.js.map
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* executor — how the control server makes a proxy run a tool
|
|
3
|
+
* (docs/calculatedReplay.md §16.2).
|
|
4
|
+
*
|
|
5
|
+
* THE ASYMMETRY THIS EXISTS TO EXPLOIT. Every wrapped server is already reached
|
|
6
|
+
* through a `bir-proxy` holding an initialized, live `UpstreamClient`. Issuing a
|
|
7
|
+
* `tools/call` on that connection costs **zero tokens** — no second agent, no
|
|
8
|
+
* subprocess, no model in the loop. That is the whole reason this design has a
|
|
9
|
+
* free rung where RRepeat has an Agent SDK session.
|
|
10
|
+
*
|
|
11
|
+
* WHY LONG-POLLING, AND NOT A CALLBACK URL (D12). The control plane was
|
|
12
|
+
* one-directional: proxies dial out to report steps. Keeping it that way costs
|
|
13
|
+
* nothing here — the proxy holds one request open, the server answers it with
|
|
14
|
+
* work — and it avoids N new loopback listeners, N new tokens and N new things
|
|
15
|
+
* `bir doctor` must verify, for a latency saving of milliseconds on a path
|
|
16
|
+
* already dominated by the upstream's own round trip.
|
|
17
|
+
*
|
|
18
|
+
* A proxy that never polls (an old build, or one started without `BIR_REPLAY`)
|
|
19
|
+
* is not an error: `call` rejects with `no_proxy`, and the caller falls back to
|
|
20
|
+
* the step's recorded output.
|
|
21
|
+
*/
|
|
22
|
+
/** One dispatched tool call, as the proxy receives it. */
|
|
23
|
+
export interface Work {
|
|
24
|
+
workId: string;
|
|
25
|
+
/** Upstream-local name (`navigate_page`), not `mcp__server__navigate_page`. */
|
|
26
|
+
toolName: string;
|
|
27
|
+
arguments: unknown;
|
|
28
|
+
timeoutMs: number;
|
|
29
|
+
}
|
|
30
|
+
/** Rejection reason when no proxy is polling for a server. */
|
|
31
|
+
export declare const NO_PROXY = "no_proxy";
|
|
32
|
+
export declare class ProxyWorkQueue {
|
|
33
|
+
/** Work dispatched while its proxy was mid-round-trip, per server. */
|
|
34
|
+
private readonly pending;
|
|
35
|
+
/** Pollers currently parked, per server. */
|
|
36
|
+
private readonly waiters;
|
|
37
|
+
/** Work handed out and awaiting a result. */
|
|
38
|
+
private readonly inFlight;
|
|
39
|
+
/** serverName → when it last polled. A proxy between polls is still present. */
|
|
40
|
+
private readonly lastSeen;
|
|
41
|
+
private closed;
|
|
42
|
+
/** Servers a proxy is currently serving — what `bir doctor` reports. */
|
|
43
|
+
pollingServers(): string[];
|
|
44
|
+
/** True when a proxy for `serverName` is available to take work. */
|
|
45
|
+
hasPoller(serverName: string): boolean;
|
|
46
|
+
/** Parked now, or polled recently enough to be mid-round-trip. */
|
|
47
|
+
private isPresent;
|
|
48
|
+
/**
|
|
49
|
+
* `POST /proxy/poll`. Resolves with work, or with `undefined` at the poll
|
|
50
|
+
* deadline so the proxy re-polls. Work queued while this proxy was between
|
|
51
|
+
* polls is handed over immediately.
|
|
52
|
+
*/
|
|
53
|
+
waitForWork(serverName: string, pollDeadlineMs: number, signal?: AbortSignal): Promise<Work | undefined>;
|
|
54
|
+
/** `POST /proxy/result`. Unknown ids are ignored — a late result after a timeout. */
|
|
55
|
+
complete(workId: string, result?: unknown, error?: string): boolean;
|
|
56
|
+
/**
|
|
57
|
+
* Dispatch one call and await its result.
|
|
58
|
+
*
|
|
59
|
+
* Rejects with {@link NO_PROXY} when nothing is polling for `serverName`, and
|
|
60
|
+
* with `timeout` when the proxy takes the work but never answers. Both are
|
|
61
|
+
* "could not be run here", which is exactly the condition under which the
|
|
62
|
+
* caller may substitute a recorded output.
|
|
63
|
+
*/
|
|
64
|
+
call(serverName: string, toolName: string, args: unknown, timeoutMs: number): Promise<unknown>;
|
|
65
|
+
/** Remove queued work that timed out, so a later poll never gets stale work. */
|
|
66
|
+
private dropPending;
|
|
67
|
+
/** Fail everything in flight and release every poller. */
|
|
68
|
+
close(): void;
|
|
69
|
+
/**
|
|
70
|
+
* The oldest *live* poller. Waiters whose request has already gone are
|
|
71
|
+
* discarded rather than handed work they can never run.
|
|
72
|
+
*/
|
|
73
|
+
private takeWaiter;
|
|
74
|
+
private removeWaiter;
|
|
75
|
+
/** Drop a waiter's timer and abort listener. Idempotent. */
|
|
76
|
+
private detach;
|
|
77
|
+
}
|
|
78
|
+
//# sourceMappingURL=executor.d.ts.map
|
|
@@ -0,0 +1,233 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* executor — how the control server makes a proxy run a tool
|
|
3
|
+
* (docs/calculatedReplay.md §16.2).
|
|
4
|
+
*
|
|
5
|
+
* THE ASYMMETRY THIS EXISTS TO EXPLOIT. Every wrapped server is already reached
|
|
6
|
+
* through a `bir-proxy` holding an initialized, live `UpstreamClient`. Issuing a
|
|
7
|
+
* `tools/call` on that connection costs **zero tokens** — no second agent, no
|
|
8
|
+
* subprocess, no model in the loop. That is the whole reason this design has a
|
|
9
|
+
* free rung where RRepeat has an Agent SDK session.
|
|
10
|
+
*
|
|
11
|
+
* WHY LONG-POLLING, AND NOT A CALLBACK URL (D12). The control plane was
|
|
12
|
+
* one-directional: proxies dial out to report steps. Keeping it that way costs
|
|
13
|
+
* nothing here — the proxy holds one request open, the server answers it with
|
|
14
|
+
* work — and it avoids N new loopback listeners, N new tokens and N new things
|
|
15
|
+
* `bir doctor` must verify, for a latency saving of milliseconds on a path
|
|
16
|
+
* already dominated by the upstream's own round trip.
|
|
17
|
+
*
|
|
18
|
+
* A proxy that never polls (an old build, or one started without `BIR_REPLAY`)
|
|
19
|
+
* is not an error: `call` rejects with `no_proxy`, and the caller falls back to
|
|
20
|
+
* the step's recorded output.
|
|
21
|
+
*/
|
|
22
|
+
import { randomUUID } from "node:crypto";
|
|
23
|
+
import { logDetail } from "../util/log.js";
|
|
24
|
+
/** Rejection reason when no proxy is polling for a server. */
|
|
25
|
+
export const NO_PROXY = "no_proxy";
|
|
26
|
+
/**
|
|
27
|
+
* How long after its last poll a proxy is still considered present.
|
|
28
|
+
*
|
|
29
|
+
* A proxy is only parked *between* round trips: while it is POSTing one step's
|
|
30
|
+
* result and re-polling, nothing is waiting, and that window falls squarely
|
|
31
|
+
* between the steps of a multi-step plan. Treating it as absent would make every
|
|
32
|
+
* plan lose every step after the first.
|
|
33
|
+
*/
|
|
34
|
+
const PRESENT_MS = 60_000;
|
|
35
|
+
export class ProxyWorkQueue {
|
|
36
|
+
/** Work dispatched while its proxy was mid-round-trip, per server. */
|
|
37
|
+
pending = new Map();
|
|
38
|
+
/** Pollers currently parked, per server. */
|
|
39
|
+
waiters = new Map();
|
|
40
|
+
/** Work handed out and awaiting a result. */
|
|
41
|
+
inFlight = new Map();
|
|
42
|
+
/** serverName → when it last polled. A proxy between polls is still present. */
|
|
43
|
+
lastSeen = new Map();
|
|
44
|
+
closed = false;
|
|
45
|
+
/** Servers a proxy is currently serving — what `bir doctor` reports. */
|
|
46
|
+
pollingServers() {
|
|
47
|
+
const now = Date.now();
|
|
48
|
+
return [...this.lastSeen.entries()]
|
|
49
|
+
.filter(([, at]) => now - at <= PRESENT_MS)
|
|
50
|
+
.map(([name]) => name);
|
|
51
|
+
}
|
|
52
|
+
/** True when a proxy for `serverName` is available to take work. */
|
|
53
|
+
hasPoller(serverName) {
|
|
54
|
+
if ((this.waiters.get(serverName)?.length ?? 0) > 0)
|
|
55
|
+
return true;
|
|
56
|
+
return this.isPresent(serverName);
|
|
57
|
+
}
|
|
58
|
+
/** Parked now, or polled recently enough to be mid-round-trip. */
|
|
59
|
+
isPresent(serverName) {
|
|
60
|
+
const at = this.lastSeen.get(serverName);
|
|
61
|
+
return at !== undefined && Date.now() - at <= PRESENT_MS;
|
|
62
|
+
}
|
|
63
|
+
/**
|
|
64
|
+
* `POST /proxy/poll`. Resolves with work, or with `undefined` at the poll
|
|
65
|
+
* deadline so the proxy re-polls. Work queued while this proxy was between
|
|
66
|
+
* polls is handed over immediately.
|
|
67
|
+
*/
|
|
68
|
+
waitForWork(serverName, pollDeadlineMs, signal) {
|
|
69
|
+
if (this.closed)
|
|
70
|
+
return Promise.resolve(undefined);
|
|
71
|
+
this.lastSeen.set(serverName, Date.now());
|
|
72
|
+
const queued = this.pending.get(serverName);
|
|
73
|
+
if (queued && queued.length > 0) {
|
|
74
|
+
return Promise.resolve(queued.shift());
|
|
75
|
+
}
|
|
76
|
+
if (signal?.aborted)
|
|
77
|
+
return Promise.resolve(undefined);
|
|
78
|
+
return new Promise((resolve) => {
|
|
79
|
+
const list = this.waiters.get(serverName) ?? [];
|
|
80
|
+
const waiter = {
|
|
81
|
+
serverName,
|
|
82
|
+
resolve,
|
|
83
|
+
signal,
|
|
84
|
+
timer: setTimeout(() => {
|
|
85
|
+
this.removeWaiter(waiter);
|
|
86
|
+
resolve(undefined);
|
|
87
|
+
}, Math.max(0, pollDeadlineMs)),
|
|
88
|
+
};
|
|
89
|
+
if (signal) {
|
|
90
|
+
waiter.onAbort = () => {
|
|
91
|
+
logDetail("replay.poller_gone", { server: serverName, why: "its request closed" });
|
|
92
|
+
this.removeWaiter(waiter);
|
|
93
|
+
resolve(undefined);
|
|
94
|
+
};
|
|
95
|
+
signal.addEventListener("abort", waiter.onAbort, { once: true });
|
|
96
|
+
}
|
|
97
|
+
waiter.timer.unref?.();
|
|
98
|
+
list.push(waiter);
|
|
99
|
+
this.waiters.set(serverName, list);
|
|
100
|
+
});
|
|
101
|
+
}
|
|
102
|
+
/** `POST /proxy/result`. Unknown ids are ignored — a late result after a timeout. */
|
|
103
|
+
complete(workId, result, error) {
|
|
104
|
+
const entry = this.inFlight.get(workId);
|
|
105
|
+
if (!entry)
|
|
106
|
+
return false;
|
|
107
|
+
this.inFlight.delete(workId);
|
|
108
|
+
clearTimeout(entry.timer);
|
|
109
|
+
if (error)
|
|
110
|
+
entry.reject(new Error(error));
|
|
111
|
+
else
|
|
112
|
+
entry.resolve(result);
|
|
113
|
+
return true;
|
|
114
|
+
}
|
|
115
|
+
/**
|
|
116
|
+
* Dispatch one call and await its result.
|
|
117
|
+
*
|
|
118
|
+
* Rejects with {@link NO_PROXY} when nothing is polling for `serverName`, and
|
|
119
|
+
* with `timeout` when the proxy takes the work but never answers. Both are
|
|
120
|
+
* "could not be run here", which is exactly the condition under which the
|
|
121
|
+
* caller may substitute a recorded output.
|
|
122
|
+
*/
|
|
123
|
+
call(serverName, toolName, args, timeoutMs) {
|
|
124
|
+
if (this.closed)
|
|
125
|
+
return Promise.reject(new Error(NO_PROXY));
|
|
126
|
+
const work = { workId: "birwork_" + randomUUID(), toolName, arguments: args, timeoutMs };
|
|
127
|
+
const waiter = this.takeWaiter(serverName);
|
|
128
|
+
if (!waiter && !this.isPresent(serverName)) {
|
|
129
|
+
// No proxy has ever polled for this server, or one has been gone for a
|
|
130
|
+
// minute. Do NOT queue: the caller is inside a turn the user is waiting on,
|
|
131
|
+
// and work that sits until some proxy happens to appear would stall it past
|
|
132
|
+
// every budget. Fail fast, and let the recorded-output fallback decide.
|
|
133
|
+
return Promise.reject(new Error(NO_PROXY));
|
|
134
|
+
}
|
|
135
|
+
return new Promise((resolve, reject) => {
|
|
136
|
+
const entry = {
|
|
137
|
+
serverName,
|
|
138
|
+
resolve,
|
|
139
|
+
reject,
|
|
140
|
+
timer: setTimeout(() => {
|
|
141
|
+
this.inFlight.delete(work.workId);
|
|
142
|
+
this.dropPending(serverName, work.workId);
|
|
143
|
+
reject(new Error(`timeout after ${timeoutMs}ms`));
|
|
144
|
+
}, timeoutMs),
|
|
145
|
+
};
|
|
146
|
+
entry.timer.unref?.();
|
|
147
|
+
this.inFlight.set(work.workId, entry);
|
|
148
|
+
logDetail("replay.dispatch", {
|
|
149
|
+
server: serverName,
|
|
150
|
+
tool: toolName,
|
|
151
|
+
work: work.workId,
|
|
152
|
+
queued: waiter ? undefined : true,
|
|
153
|
+
});
|
|
154
|
+
if (waiter) {
|
|
155
|
+
waiter.resolve(work);
|
|
156
|
+
return;
|
|
157
|
+
}
|
|
158
|
+
// The proxy is mid-round-trip — POSTing the previous step's result, about
|
|
159
|
+
// to poll again. Hold the work for it; the per-call timeout above is what
|
|
160
|
+
// bounds the wait if it never comes back.
|
|
161
|
+
const list = this.pending.get(serverName) ?? [];
|
|
162
|
+
list.push(work);
|
|
163
|
+
this.pending.set(serverName, list);
|
|
164
|
+
});
|
|
165
|
+
}
|
|
166
|
+
/** Remove queued work that timed out, so a later poll never gets stale work. */
|
|
167
|
+
dropPending(serverName, workId) {
|
|
168
|
+
const list = this.pending.get(serverName);
|
|
169
|
+
if (!list)
|
|
170
|
+
return;
|
|
171
|
+
const i = list.findIndex((w) => w.workId === workId);
|
|
172
|
+
if (i >= 0)
|
|
173
|
+
list.splice(i, 1);
|
|
174
|
+
}
|
|
175
|
+
/** Fail everything in flight and release every poller. */
|
|
176
|
+
close() {
|
|
177
|
+
this.closed = true;
|
|
178
|
+
for (const [, list] of this.waiters) {
|
|
179
|
+
for (const waiter of list) {
|
|
180
|
+
this.detach(waiter);
|
|
181
|
+
waiter.resolve(undefined);
|
|
182
|
+
}
|
|
183
|
+
}
|
|
184
|
+
this.waiters.clear();
|
|
185
|
+
this.pending.clear();
|
|
186
|
+
// A closed queue has no proxies, whatever they were doing a moment ago.
|
|
187
|
+
this.lastSeen.clear();
|
|
188
|
+
for (const [, entry] of this.inFlight) {
|
|
189
|
+
clearTimeout(entry.timer);
|
|
190
|
+
entry.reject(new Error("control server closed"));
|
|
191
|
+
}
|
|
192
|
+
this.inFlight.clear();
|
|
193
|
+
}
|
|
194
|
+
/**
|
|
195
|
+
* The oldest *live* poller. Waiters whose request has already gone are
|
|
196
|
+
* discarded rather than handed work they can never run.
|
|
197
|
+
*/
|
|
198
|
+
takeWaiter(serverName) {
|
|
199
|
+
const list = this.waiters.get(serverName);
|
|
200
|
+
if (!list)
|
|
201
|
+
return undefined;
|
|
202
|
+
for (;;) {
|
|
203
|
+
const waiter = list.shift();
|
|
204
|
+
if (!waiter)
|
|
205
|
+
return undefined;
|
|
206
|
+
this.detach(waiter);
|
|
207
|
+
if (waiter.signal?.aborted) {
|
|
208
|
+
// Its proxy is gone. Release the promise and keep looking.
|
|
209
|
+
waiter.resolve(undefined);
|
|
210
|
+
continue;
|
|
211
|
+
}
|
|
212
|
+
return waiter;
|
|
213
|
+
}
|
|
214
|
+
}
|
|
215
|
+
removeWaiter(waiter) {
|
|
216
|
+
this.detach(waiter);
|
|
217
|
+
const list = this.waiters.get(waiter.serverName);
|
|
218
|
+
if (!list)
|
|
219
|
+
return;
|
|
220
|
+
const i = list.indexOf(waiter);
|
|
221
|
+
if (i >= 0)
|
|
222
|
+
list.splice(i, 1);
|
|
223
|
+
}
|
|
224
|
+
/** Drop a waiter's timer and abort listener. Idempotent. */
|
|
225
|
+
detach(waiter) {
|
|
226
|
+
clearTimeout(waiter.timer);
|
|
227
|
+
if (waiter.signal && waiter.onAbort) {
|
|
228
|
+
waiter.signal.removeEventListener("abort", waiter.onAbort);
|
|
229
|
+
waiter.onAbort = undefined;
|
|
230
|
+
}
|
|
231
|
+
}
|
|
232
|
+
}
|
|
233
|
+
//# sourceMappingURL=executor.js.map
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* logic — the four stored-JS evaluators (docs/calculatedReplay.md §9).
|
|
3
|
+
*
|
|
4
|
+
* A calculated scenario carries JavaScript function *bodies*, one per hook
|
|
5
|
+
* point, each ending in `return { … }`. They are compiled with `new Function`
|
|
6
|
+
* and run here.
|
|
7
|
+
*
|
|
8
|
+
* PORTED FROM THE SERVICE, DELIBERATELY. BaseIn's `src/scenarios/logic.ts` is
|
|
9
|
+
* the copy the service itself evaluates in a dry replay, so it is the one that
|
|
10
|
+
* defines what a scenario *means*. Porting RRepeat's near-identical copy instead
|
|
11
|
+
* would let a scenario behave one way under `bir scenario replay --dry` and
|
|
12
|
+
* another way here — an untestable difference. Argument order and the
|
|
13
|
+
* plain-object guard both match the service exactly.
|
|
14
|
+
*
|
|
15
|
+
* THERE IS NO SANDBOX, and the doc says so plainly (§13.1). A `new Function`
|
|
16
|
+
* body gets its declared arguments and the global scope: `require`/`import` are
|
|
17
|
+
* unreachable under ESM, but `fetch`, `process` and `globalThis` are not. What
|
|
18
|
+
* makes this acceptable is provenance, not containment — the payload arrives
|
|
19
|
+
* only in an authenticated response from the configured `BIR_AUTH_URL`,
|
|
20
|
+
* describing the caller's own recorded runs — and replay is opt-in for exactly
|
|
21
|
+
* this reason.
|
|
22
|
+
*/
|
|
23
|
+
/** Transform the derived parameters once, before any step runs. */
|
|
24
|
+
export declare function evalParamsLogic(code: string, parameters: Record<string, unknown>, intent: string): Record<string, unknown>;
|
|
25
|
+
/** Derive one step's tool input from params, intent and the accumulated respParams. */
|
|
26
|
+
export declare function evalToolInputLogic(code: string, parameters: Record<string, unknown>, intent: string, respParams?: Record<string, unknown>): Record<string, unknown>;
|
|
27
|
+
/** Extract keys from a step's real output, to merge into respParams for later steps. */
|
|
28
|
+
export declare function evalToolOutputLogic(code: string, toolOutput: string, parameters: Record<string, unknown>, intent: string, respParams?: Record<string, unknown>): Record<string, unknown>;
|
|
29
|
+
/** Build the final response model from everything the steps accumulated. */
|
|
30
|
+
export declare function evalResponseParamsLogic(code: string, respParams: Record<string, unknown>, parameters: Record<string, unknown>, intent: string): Record<string, unknown>;
|
|
31
|
+
//# sourceMappingURL=logic.d.ts.map
|