@basein/runner 0.2.1 → 0.2.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/dist/bin/bir-hooks.d.ts +5 -3
- package/dist/bin/bir-hooks.js +12 -4
- package/dist/bin/bir.js +33 -1
- package/dist/control/server.js +9 -0
- package/dist/record/recorder.d.ts +11 -0
- package/dist/record/remote-recorder.js +4 -0
- package/dist/replay/controller.d.ts +15 -1
- package/dist/replay/controller.js +88 -21
- package/dist/replay/derive.d.ts +34 -6
- package/dist/replay/derive.js +108 -8
- package/dist/replay/targets.d.ts +52 -0
- package/dist/replay/targets.js +133 -0
- package/docs/calculatedReplay.md +16 -8
- package/docs/calculatedReplayGuide.md +4 -4
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -201,8 +201,8 @@ what to do when they register, so there is one switch and no second copy of it:
|
|
|
201
201
|
| `BIR_REPLAY=1` | Enable replay. Nothing below matters until it is set. |
|
|
202
202
|
| `BIR_REPLAY_ALLOW_SERVERS` | Server keys eligible for **direct** execution. Unset means every wrapped server; setting it is the recommendation. |
|
|
203
203
|
| `BIR_MIN_STEER_SIMILARITY` | Minimum match similarity to replay (default `0.92`, above the service's own `0.9` detection threshold). |
|
|
204
|
-
| `ANTHROPIC_API_KEY` |
|
|
205
|
-
| `BIR_DERIVE_MODEL` | Derivation model (default `claude-haiku-4-5-20251001`). |
|
|
204
|
+
| `ANTHROPIC_API_KEY` | **Optional.** Working out what a new request is about — which fleet, which file, which date — is done for you by the service on its own key, as long as you are signed in. Set this only to keep that reading on your machine, on your key. Signed out *and* unset, a scenario whose values change between requests is declined rather than replayed on stale ones. |
|
|
205
|
+
| `BIR_DERIVE_MODEL` | Derivation model, when you set a key of your own (default `claude-haiku-4-5-20251001`). |
|
|
206
206
|
| `BIR_MATCH_BUDGET_MS` | How long the prompt hook waits for a match (default `2500`). |
|
|
207
207
|
| `BIR_DERIVE_BUDGET_MS` | How long the first `PreToolUse` waits for parameters (default `8000`). |
|
|
208
208
|
| `BIR_REPLAY_BUDGET_MS` / `BIR_STEP_TIMEOUT_MS` | Whole-plan and per-step ceilings for direct execution (default `120000` / `60000`). |
|
package/dist/bin/bir-hooks.d.ts
CHANGED
|
@@ -27,9 +27,11 @@
|
|
|
27
27
|
* BIR_REPLAY_ALLOW_SERVERS comma-separated server keys eligible for *direct*
|
|
28
28
|
* execution. Unset means every wrapped server
|
|
29
29
|
* BIR_MIN_STEER_SIMILARITY minimum match similarity to replay (default 0.92)
|
|
30
|
-
* ANTHROPIC_API_KEY
|
|
31
|
-
*
|
|
32
|
-
*
|
|
30
|
+
* ANTHROPIC_API_KEY OPTIONAL. Derivation — reading what this turn acts
|
|
31
|
+
* on — is done by the service for a signed-in
|
|
32
|
+
* runner. Set this only to keep that reading on this
|
|
33
|
+
* machine; signed out and unset, a scenario with a
|
|
34
|
+
* target does not run (segmented.md R-PARAM-5)
|
|
33
35
|
* BIR_DERIVE_MODEL derivation model (default claude-haiku-4-5-…)
|
|
34
36
|
* BIR_MATCH_BUDGET_MS prompt-hook match wait (default 2500)
|
|
35
37
|
* BIR_DERIVE_BUDGET_MS first PreToolUse derivation wait (default 8000)
|
package/dist/bin/bir-hooks.js
CHANGED
|
@@ -27,9 +27,11 @@
|
|
|
27
27
|
* BIR_REPLAY_ALLOW_SERVERS comma-separated server keys eligible for *direct*
|
|
28
28
|
* execution. Unset means every wrapped server
|
|
29
29
|
* BIR_MIN_STEER_SIMILARITY minimum match similarity to replay (default 0.92)
|
|
30
|
-
* ANTHROPIC_API_KEY
|
|
31
|
-
*
|
|
32
|
-
*
|
|
30
|
+
* ANTHROPIC_API_KEY OPTIONAL. Derivation — reading what this turn acts
|
|
31
|
+
* on — is done by the service for a signed-in
|
|
32
|
+
* runner. Set this only to keep that reading on this
|
|
33
|
+
* machine; signed out and unset, a scenario with a
|
|
34
|
+
* target does not run (segmented.md R-PARAM-5)
|
|
33
35
|
* BIR_DERIVE_MODEL derivation model (default claude-haiku-4-5-…)
|
|
34
36
|
* BIR_MATCH_BUDGET_MS prompt-hook match wait (default 2500)
|
|
35
37
|
* BIR_DERIVE_BUDGET_MS first PreToolUse derivation wait (default 8000)
|
|
@@ -182,7 +184,13 @@ function buildReplayOptions(auth) {
|
|
|
182
184
|
logLine("replay.enabled", {
|
|
183
185
|
minSimilarity: opts.minSimilarity,
|
|
184
186
|
allowServers: allowServers ? [...allowServers].join(",") : "(all wrapped)",
|
|
185
|
-
|
|
187
|
+
// Who reads what this turn acts on (segmented.md R-PARAM-5). Named at
|
|
188
|
+
// startup because the third state is a replay that silently never runs.
|
|
189
|
+
derive: process.env.ANTHROPIC_API_KEY
|
|
190
|
+
? "this machine (ANTHROPIC_API_KEY)"
|
|
191
|
+
: auth.session && auth.baseUrl
|
|
192
|
+
? "the service"
|
|
193
|
+
: "recorded sample values — sign in, or a scenario with a target will not run",
|
|
186
194
|
intentMatch: opts.intentMatch?.enabled ? "on" : "off",
|
|
187
195
|
why: "matched prompts will run their calculated scenario — steered steps are auto-approved",
|
|
188
196
|
});
|
package/dist/bin/bir.js
CHANGED
|
@@ -576,6 +576,17 @@ async function doctor(args) {
|
|
|
576
576
|
problems.push(`BIR_AUTH_URL in this shell (${shellAuthUrl}) is not a BaseIn service: ${shellAuthProblem}. ` +
|
|
577
577
|
`\`bir login\` and \`bir scenario\` here will fail — ${AUTH_URL_HINT}`);
|
|
578
578
|
}
|
|
579
|
+
// THE SCENARIO SERVER IS ON THE CRITICAL PATH OF THE FIRST TURN. A plan armed
|
|
580
|
+
// at `UserPromptSubmit` is delivered through `mcp__bir__run_scenario`, so a
|
|
581
|
+
// `bir` entry that starts with `npx` has to resolve and unpack the package
|
|
582
|
+
// before the tool exists at all — measured at ~13s, against a first tool call
|
|
583
|
+
// ~6s in. The turn does not fail; it just quietly runs the ordinary way, which
|
|
584
|
+
// is the failure this whole command exists to make visible.
|
|
585
|
+
const scenarioEntry = entries.find((e) => e.name === SCENARIO_SERVER_KEY);
|
|
586
|
+
if (scenarioEntry && !isRemote(scenarioEntry.config) && scenarioEntry.config.command === "npx") {
|
|
587
|
+
notes.push("the bir scenario server starts via npx, which can take longer than the first tool call " +
|
|
588
|
+
"of a turn — reinstall with `bir install --replay --global` to pin it to the installed copy");
|
|
589
|
+
}
|
|
579
590
|
const sidecar = readSidecar();
|
|
580
591
|
if (sidecar.controlPort && discovery && !discovery.url.endsWith(`:${sidecar.controlPort}`)) {
|
|
581
592
|
problems.push(`hooks were installed for port ${sidecar.controlPort} but the control server is on ${discovery.url}`);
|
|
@@ -600,7 +611,20 @@ async function doctor(args) {
|
|
|
600
611
|
if (replay?.enabled) {
|
|
601
612
|
out(`Replay : ON servers=${replay.allowServers?.join(",") || "(all wrapped)"} ` +
|
|
602
613
|
`minSimilarity=${replay.minSimilarity ?? "?"}`);
|
|
603
|
-
|
|
614
|
+
// An older control server sends only `deriveKey`; read it as the two states
|
|
615
|
+
// it could describe then.
|
|
616
|
+
const via = replay.deriveVia ?? (replay.deriveKey ? "key" : "samples");
|
|
617
|
+
const derive = via === "key"
|
|
618
|
+
? "this machine (ANTHROPIC_API_KEY)"
|
|
619
|
+
: via === "service"
|
|
620
|
+
? "the service (no key needed here)"
|
|
621
|
+
: "recorded sample values — scenarios with a target will NOT run";
|
|
622
|
+
out(` derive=${derive}`);
|
|
623
|
+
if (via === "samples") {
|
|
624
|
+
notes.push("nothing can read what this turn acts on: sign in with `bir login` so the " +
|
|
625
|
+
"service derives, or set ANTHROPIC_API_KEY to derive here. Until then a " +
|
|
626
|
+
"matched scenario with a target is declined and the agent does the task");
|
|
627
|
+
}
|
|
604
628
|
const idle = wantWrapped.filter((n) => !(replay.pollingProxies ?? []).includes(n));
|
|
605
629
|
if (idle.length > 0) {
|
|
606
630
|
notes.push(`these proxies are not polling for replay work: ${idle.join(", ")} — ` +
|
|
@@ -783,6 +807,14 @@ async function replayCommand(args) {
|
|
|
783
807
|
}
|
|
784
808
|
const r = body;
|
|
785
809
|
out(`params ${JSON.stringify(r.params ?? {})}`);
|
|
810
|
+
// The difference between "this scenario works" and "this scenario works on
|
|
811
|
+
// last week's values". A dry replay fills an unnamed target from the
|
|
812
|
+
// recording; a real turn never does, so without this line a green trace here
|
|
813
|
+
// is no evidence at all that a session would steer.
|
|
814
|
+
if (r.targetsFromSamples?.length) {
|
|
815
|
+
out(` ↑ not named by the prompt: ${r.targetsFromSamples.join(", ")}`);
|
|
816
|
+
out(" (a live turn finds these in an earlier step, or does not run)");
|
|
817
|
+
}
|
|
786
818
|
(r.steps ?? []).forEach((s, i) => {
|
|
787
819
|
out(`step ${i} ${s.toolName} ${JSON.stringify(s.input)}`);
|
|
788
820
|
const emitted = Object.keys(s.emitted ?? {});
|
package/dist/control/server.js
CHANGED
|
@@ -212,6 +212,15 @@ export class ControlServer {
|
|
|
212
212
|
minSimilarity: this.opts.replay?.minSimilarity ?? null,
|
|
213
213
|
allowServers: this.opts.replay?.allowServers ? [...this.opts.replay.allowServers] : null,
|
|
214
214
|
deriveKey: Boolean(this.opts.replay?.apiKey ?? process.env.ANTHROPIC_API_KEY),
|
|
215
|
+
// Who reads the turn for its parameters (segmented.md R-PARAM-5). The
|
|
216
|
+
// ordinary answer is `service`; a key here is an override; `samples`
|
|
217
|
+
// means no scenario with a target can run, which is the one state an
|
|
218
|
+
// operator must be able to see without reading a log.
|
|
219
|
+
deriveVia: (this.opts.replay?.apiKey ?? process.env.ANTHROPIC_API_KEY)
|
|
220
|
+
? "key"
|
|
221
|
+
: this.opts.replay?.authUrl && this.opts.replay?.authToken
|
|
222
|
+
? "service"
|
|
223
|
+
: "samples",
|
|
215
224
|
pollingProxies: this.replay.work.pollingServers(),
|
|
216
225
|
},
|
|
217
226
|
lossy: this.lossy,
|
|
@@ -263,6 +263,17 @@ export interface ExecutionReport {
|
|
|
263
263
|
* two baselines of one turn. The segments' own rows are untouched.
|
|
264
264
|
*/
|
|
265
265
|
sharedWith?: string[];
|
|
266
|
+
/**
|
|
267
|
+
* Which gate of the ladder declined, on a `not_steered` report.
|
|
268
|
+
*
|
|
269
|
+
* The one thing the recording page could never learn. "Replay on, scenario
|
|
270
|
+
* ready, prompt matched, nothing steered" has exactly one explanation and it
|
|
271
|
+
* used to live only in a stderr line on the machine that decided it — so the
|
|
272
|
+
* owner of a scenario that never runs had nothing to look at. It travels as
|
|
273
|
+
* the gate's own code rather than the sentence beside it: the prose is for a
|
|
274
|
+
* person reading the audit log, this is for the console.
|
|
275
|
+
*/
|
|
276
|
+
declined?: string;
|
|
266
277
|
}
|
|
267
278
|
/** Optional capability: reporting needs a service, and a NullRecorder has none. */
|
|
268
279
|
export interface ScenarioReporter {
|
|
@@ -212,6 +212,9 @@ export class RemoteRecorder {
|
|
|
212
212
|
fallbackKind: report.fallbackKind,
|
|
213
213
|
baselineEligible: report.baselineEligible,
|
|
214
214
|
sharedWith: report.sharedWith,
|
|
215
|
+
// Why nothing ran, when nothing ran. Additive: an older service drops
|
|
216
|
+
// the field and books the cost exactly as it always did.
|
|
217
|
+
declined: report.declined,
|
|
215
218
|
});
|
|
216
219
|
const r = (body ?? {});
|
|
217
220
|
const failed = report.steps?.filter((s) => s.status === "failed").length ?? 0;
|
|
@@ -228,6 +231,7 @@ export class RemoteRecorder {
|
|
|
228
231
|
stepsFailed: failed || undefined,
|
|
229
232
|
baselineEligible: report.baselineEligible === false ? false : undefined,
|
|
230
233
|
sharedWith: report.sharedWith?.length,
|
|
234
|
+
declined: report.declined,
|
|
231
235
|
});
|
|
232
236
|
});
|
|
233
237
|
}
|
|
@@ -54,7 +54,10 @@ export interface ReplayOptions {
|
|
|
54
54
|
/** `BIR_REPLAY_ALLOW_SERVERS`; undefined means every wrapped server. */
|
|
55
55
|
allowServers?: ReadonlySet<string>;
|
|
56
56
|
budgets?: Partial<ReplayBudgets>;
|
|
57
|
-
/**
|
|
57
|
+
/**
|
|
58
|
+
* The service: the lazy source-run fetch, and the parameter derivation this
|
|
59
|
+
* runner no longer needs a key of its own for (segmented.md R-PARAM-5).
|
|
60
|
+
*/
|
|
58
61
|
authUrl?: string;
|
|
59
62
|
authToken?: () => string;
|
|
60
63
|
/** Injected by tests. */
|
|
@@ -378,6 +381,17 @@ export declare class ReplayController {
|
|
|
378
381
|
* directive the moment the match lands; the first `PreToolUse` — or
|
|
379
382
|
* `/scenario/run`, which has no hook timeout at all — is where the wait lands.
|
|
380
383
|
*/
|
|
384
|
+
/**
|
|
385
|
+
* Where to ask the service to read this turn (segmented.md R-PARAM-5).
|
|
386
|
+
*
|
|
387
|
+
* Undefined when there is nobody to ask — an unauthenticated session, or one
|
|
388
|
+
* whose token has gone. `derive` then falls back to the recorded samples, and
|
|
389
|
+
* a scenario with a target declines, exactly as a keyless runner always did.
|
|
390
|
+
*
|
|
391
|
+
* The token is read here rather than captured, because the recorder refreshes
|
|
392
|
+
* it as a session outlives it.
|
|
393
|
+
*/
|
|
394
|
+
private deriveService;
|
|
381
395
|
private startDerivation;
|
|
382
396
|
/**
|
|
383
397
|
* Divergence (§8). Execute the remaining steps for real, then deliver.
|
|
@@ -25,6 +25,7 @@ import { ScenarioReplayPlan } from "./plan.js";
|
|
|
25
25
|
import { PRICING_VERSION } from "./pricing.js";
|
|
26
26
|
import { SourceRunOutputs } from "./source-run.js";
|
|
27
27
|
import { clampToFrameStart, flattenChain, } from "./flatten.js";
|
|
28
|
+
import { blockingTargets } from "./targets.js";
|
|
28
29
|
import { toolResultError } from "./tool-error.js";
|
|
29
30
|
import { OUTCOME_RANK, hasTargets, isReadyScenario, } from "./types.js";
|
|
30
31
|
/** The first-party tool a `direct` plan is delivered through (§6.3). */
|
|
@@ -233,26 +234,60 @@ export class ReplayController {
|
|
|
233
234
|
// because it is the only one that spends a model call and waits. A scenario
|
|
234
235
|
// whose parameters are all settings arms at once, as before: there is
|
|
235
236
|
// nothing the turn has to supply.
|
|
237
|
+
//
|
|
238
|
+
// A target the turn did not name stops the plan only when *this* plan could
|
|
239
|
+
// act on its recorded value; a target the chain discovers for itself does
|
|
240
|
+
// not ({@link ./targets.ts}). The unnamed value stays null either way, so
|
|
241
|
+
// nothing here ever runs on last week's.
|
|
236
242
|
if (hasTargets(scenario.paramsObject)) {
|
|
243
|
+
// Of the targets this answer left unnamed, the ones this plan could act
|
|
244
|
+
// on the recorded value of. Empty means the chain finds them for itself.
|
|
245
|
+
const blocking = (missing) => blockingTargets(missing, scenario.paramsObject, planned);
|
|
237
246
|
let result;
|
|
238
247
|
try {
|
|
239
248
|
result = await this.withBudget(derivation, this.budgets.deriveMs, "derivation");
|
|
240
249
|
}
|
|
241
250
|
catch (err) {
|
|
242
|
-
// Out of budget, or the call threw.
|
|
243
|
-
//
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
251
|
+
// Out of budget, or the call threw. The turn has not said what this task
|
|
252
|
+
// is to act on, so every target is unnamed (R-PARAM-3).
|
|
253
|
+
const blocked = blocking(targetKeys(scenario.paramsObject));
|
|
254
|
+
if (blocked.length > 0) {
|
|
255
|
+
return decline(`target ${blocked[0]} not found — ${errText(err)}`, "missing_target");
|
|
256
|
+
}
|
|
257
|
+
logLine("replay.targets_unread", {
|
|
258
|
+
scenario: scenario.id,
|
|
259
|
+
why: "the derivation did not answer, and this chain takes no target from the caller",
|
|
260
|
+
error: errText(err),
|
|
261
|
+
});
|
|
247
262
|
}
|
|
248
|
-
if (!result.derived) {
|
|
249
|
-
//
|
|
250
|
-
//
|
|
251
|
-
|
|
252
|
-
|
|
263
|
+
if (result && !result.derived) {
|
|
264
|
+
// Nobody read the turn — no key here and no service to ask, or the
|
|
265
|
+
// service declined. Settings can still take their recorded values, but a
|
|
266
|
+
// target is a guess nobody is allowed to make (R-PARAM-5). The reason
|
|
267
|
+
// travels into the line, because "it did not run" without one is how
|
|
268
|
+
// this stayed invisible before.
|
|
269
|
+
//
|
|
270
|
+
// Every target is judged, not `missing`: an answer that derived nothing
|
|
271
|
+
// carries the recorded sample for *all* of them, so a chain that reads
|
|
272
|
+
// one would run on last week's value rather than on nothing.
|
|
273
|
+
const blocked = blocking(targetKeys(scenario.paramsObject));
|
|
274
|
+
if (blocked.length > 0) {
|
|
275
|
+
return decline(`${result.reason ?? "no_derive_key"}: target ${blocked[0]}`, "no_derive_key");
|
|
276
|
+
}
|
|
253
277
|
}
|
|
254
|
-
if (result.missing.length > 0) {
|
|
255
|
-
|
|
278
|
+
else if (result && result.missing.length > 0) {
|
|
279
|
+
const blocked = blocking(result.missing);
|
|
280
|
+
if (blocked.length > 0) {
|
|
281
|
+
return decline(`target ${blocked[0]} not found`, "missing_target");
|
|
282
|
+
}
|
|
283
|
+
// Worth a line of its own: this is the difference between a scenario
|
|
284
|
+
// that never runs and one that does, and it is the first thing to look
|
|
285
|
+
// at when a replay acts on the wrong thing.
|
|
286
|
+
logLine("replay.targets_found_by_chain", {
|
|
287
|
+
scenario: scenario.id,
|
|
288
|
+
unnamed: result.missing.join(","),
|
|
289
|
+
why: "the chain computes these from its own steps; none is read from the caller",
|
|
290
|
+
});
|
|
256
291
|
}
|
|
257
292
|
}
|
|
258
293
|
// A plan armed by intent runs inside a task the agent is already doing, so
|
|
@@ -678,10 +713,14 @@ export class ReplayController {
|
|
|
678
713
|
const total = state.deriveCostUsd + d.sessionCostUsd + state.fallbackCostUsd;
|
|
679
714
|
const steps = this.stepResultsOf(state);
|
|
680
715
|
const isBaseline = state.outcome === "not_steered" || state.outcome === "failed";
|
|
681
|
-
// A costless decline is noise on both sides — *unless*
|
|
682
|
-
//
|
|
683
|
-
// service accepts a costless report that
|
|
684
|
-
|
|
716
|
+
// A costless decline is noise on both sides — *unless* it says something the
|
|
717
|
+
// recording page cannot learn any other way: a step that actually broke, or
|
|
718
|
+
// the gate that declined. The service accepts a costless report that carries
|
|
719
|
+
// either (its errorshandling.md).
|
|
720
|
+
if (isBaseline &&
|
|
721
|
+
total <= 0 &&
|
|
722
|
+
!state.declined &&
|
|
723
|
+
!steps?.some((s) => s.status === "failed")) {
|
|
685
724
|
return undefined;
|
|
686
725
|
}
|
|
687
726
|
const blame = this.blameStep(steps);
|
|
@@ -706,6 +745,10 @@ export class ReplayController {
|
|
|
706
745
|
errorStage: blame?.stage,
|
|
707
746
|
errorStepIndex: blame?.stepIndex,
|
|
708
747
|
errorToolName: blame?.toolName,
|
|
748
|
+
// Which gate said no. Only ever set on a decline, and the decline is
|
|
749
|
+
// always `not_steered`, so a steered report carries it byte-identically
|
|
750
|
+
// to before.
|
|
751
|
+
declined: state.declined,
|
|
709
752
|
// Where the plan handed the task to the model (fallbk.md). Only on
|
|
710
753
|
// `fell_back`: a hand-over before any step ran is `failed`, and a parked
|
|
711
754
|
// step found after a divergence is still a divergence.
|
|
@@ -886,6 +929,22 @@ export class ReplayController {
|
|
|
886
929
|
* directive the moment the match lands; the first `PreToolUse` — or
|
|
887
930
|
* `/scenario/run`, which has no hook timeout at all — is where the wait lands.
|
|
888
931
|
*/
|
|
932
|
+
/**
|
|
933
|
+
* Where to ask the service to read this turn (segmented.md R-PARAM-5).
|
|
934
|
+
*
|
|
935
|
+
* Undefined when there is nobody to ask — an unauthenticated session, or one
|
|
936
|
+
* whose token has gone. `derive` then falls back to the recorded samples, and
|
|
937
|
+
* a scenario with a target declines, exactly as a keyless runner always did.
|
|
938
|
+
*
|
|
939
|
+
* The token is read here rather than captured, because the recorder refreshes
|
|
940
|
+
* it as a session outlives it.
|
|
941
|
+
*/
|
|
942
|
+
deriveService(scenarioId) {
|
|
943
|
+
const token = this.opts.authToken?.();
|
|
944
|
+
if (!this.opts.authUrl || !token)
|
|
945
|
+
return undefined;
|
|
946
|
+
return { baseUrl: this.opts.authUrl, token, scenarioId };
|
|
947
|
+
}
|
|
889
948
|
startDerivation(scenario, prompt, state, ctx = {}) {
|
|
890
949
|
const derive = this.opts.deriveImpl ?? deriveParameters;
|
|
891
950
|
return derive({
|
|
@@ -893,6 +952,9 @@ export class ReplayController {
|
|
|
893
952
|
intent: scenario.intent ?? "",
|
|
894
953
|
paramsObject: scenario.paramsObject,
|
|
895
954
|
apiKey: this.opts.apiKey ?? process.env.ANTHROPIC_API_KEY,
|
|
955
|
+
// Where to ask when there is no key here, which is the ordinary case
|
|
956
|
+
// (segmented.md R-PARAM-5).
|
|
957
|
+
service: this.deriveService(scenario.id),
|
|
896
958
|
fetchImpl: this.opts.fetchImpl,
|
|
897
959
|
liveCall: ctx.liveCall,
|
|
898
960
|
recentResults: ctx.recentResults,
|
|
@@ -903,7 +965,12 @@ export class ReplayController {
|
|
|
903
965
|
scenario: scenario.id,
|
|
904
966
|
params: Object.keys(r.params).length,
|
|
905
967
|
costUsd: r.costUsd.toFixed(4),
|
|
906
|
-
source: r.derived
|
|
968
|
+
source: r.derived
|
|
969
|
+
? r.via === "service"
|
|
970
|
+
? "the service"
|
|
971
|
+
: "prompt"
|
|
972
|
+
: "recorded samples",
|
|
973
|
+
why: r.derived ? undefined : r.reason,
|
|
907
974
|
missing: r.missing.length > 0 ? r.missing.join(",") : undefined,
|
|
908
975
|
sampled: r.sampled.length > 0 ? r.sampled.join(",") : undefined,
|
|
909
976
|
});
|
|
@@ -1265,11 +1332,11 @@ export class ReplayController {
|
|
|
1265
1332
|
});
|
|
1266
1333
|
}
|
|
1267
1334
|
}
|
|
1268
|
-
/**
|
|
1269
|
-
function
|
|
1335
|
+
/** Every target of a schema. A parameter with no kind is one (R-PARAM-3). */
|
|
1336
|
+
function targetKeys(schema) {
|
|
1270
1337
|
if (!schema)
|
|
1271
|
-
return
|
|
1272
|
-
return Object.keys(schema).
|
|
1338
|
+
return [];
|
|
1339
|
+
return Object.keys(schema).filter((k) => schema[k]?.kind !== "setting");
|
|
1273
1340
|
}
|
|
1274
1341
|
/**
|
|
1275
1342
|
* Position of the first step whose failure count is above `maxStepFailures`
|
package/dist/replay/derive.d.ts
CHANGED
|
@@ -10,11 +10,20 @@
|
|
|
10
10
|
* most sessions never arm. This is ~40 lines of `fetch` and it keeps the package
|
|
11
11
|
* at zero runtime dependencies.
|
|
12
12
|
*
|
|
13
|
-
* NO KEY IS
|
|
14
|
-
*
|
|
15
|
-
*
|
|
16
|
-
*
|
|
17
|
-
*
|
|
13
|
+
* NO KEY IS THE ORDINARY CONFIGURATION (segmented.md R-PARAM-5). Without
|
|
14
|
+
* `ANTHROPIC_API_KEY` the reading is done by the **service**, over one authorized
|
|
15
|
+
* POST to `/scenarios/:id/derive`, under the key it already uses to calculate
|
|
16
|
+
* scenarios and to ask its live question. That is the path nearly every session
|
|
17
|
+
* takes: the people this runs for are inside Claude Code on a subscription and
|
|
18
|
+
* have no API key to give it.
|
|
19
|
+
*
|
|
20
|
+
* A key here is an override, not a requirement. It keeps the reading on this
|
|
21
|
+
* machine — the prompt never reaches the service — which is what an in-house
|
|
22
|
+
* deployment wants, and it is one round trip faster.
|
|
23
|
+
*
|
|
24
|
+
* NEITHER is still supported and still means the same thing it always did:
|
|
25
|
+
* settings take their recorded samples and a scenario with a target declines,
|
|
26
|
+
* because a target is never guessed.
|
|
18
27
|
*
|
|
19
28
|
* The prompt and the parsing tolerances mirror RRepeat's
|
|
20
29
|
* `deriveParametersFromScenarioPayload` exactly, so the two produce the same
|
|
@@ -34,12 +43,27 @@ export interface DeriveContext {
|
|
|
34
43
|
liveCall?: LiveCall;
|
|
35
44
|
recentResults?: string[];
|
|
36
45
|
}
|
|
46
|
+
/**
|
|
47
|
+
* The service, which derives when this runner has no key of its own
|
|
48
|
+
* (R-PARAM-5). All three fields or none: a session that is not signed in has
|
|
49
|
+
* nowhere to ask.
|
|
50
|
+
*/
|
|
51
|
+
export interface DeriveService {
|
|
52
|
+
/** `BIR_AUTH_URL`, the same base the recorder reports to. */
|
|
53
|
+
baseUrl: string;
|
|
54
|
+
/** A live access token. Read late — the recorder refreshes it mid-session. */
|
|
55
|
+
token: string;
|
|
56
|
+
/** Whose parameters to read. The service takes the schema from its own row. */
|
|
57
|
+
scenarioId: string;
|
|
58
|
+
}
|
|
37
59
|
export interface DeriveOptions extends DeriveContext {
|
|
38
60
|
prompt: string;
|
|
39
61
|
intent: string;
|
|
40
62
|
paramsObject: ParamsSchema | null;
|
|
41
|
-
/**
|
|
63
|
+
/** An override that keeps the reading on this machine. Absent → the service. */
|
|
42
64
|
apiKey?: string;
|
|
65
|
+
/** Used when there is no `apiKey`. Absent too → recorded samples. */
|
|
66
|
+
service?: DeriveService;
|
|
43
67
|
model?: string;
|
|
44
68
|
signal?: AbortSignal;
|
|
45
69
|
/** Injected by tests. Defaults to global `fetch`. */
|
|
@@ -50,6 +74,10 @@ export interface DeriveResult {
|
|
|
50
74
|
costUsd: number;
|
|
51
75
|
/** False when the values came from the recorded samples rather than the model. */
|
|
52
76
|
derived: boolean;
|
|
77
|
+
/** Who read the turn. For the audit line, and for nothing else. */
|
|
78
|
+
via?: "key" | "service" | "samples";
|
|
79
|
+
/** Why nothing was derived, when the service said. */
|
|
80
|
+
reason?: string;
|
|
53
81
|
model?: string;
|
|
54
82
|
inputTokens?: number;
|
|
55
83
|
outputTokens?: number;
|
package/dist/replay/derive.js
CHANGED
|
@@ -10,11 +10,20 @@
|
|
|
10
10
|
* most sessions never arm. This is ~40 lines of `fetch` and it keeps the package
|
|
11
11
|
* at zero runtime dependencies.
|
|
12
12
|
*
|
|
13
|
-
* NO KEY IS
|
|
14
|
-
*
|
|
15
|
-
*
|
|
16
|
-
*
|
|
17
|
-
*
|
|
13
|
+
* NO KEY IS THE ORDINARY CONFIGURATION (segmented.md R-PARAM-5). Without
|
|
14
|
+
* `ANTHROPIC_API_KEY` the reading is done by the **service**, over one authorized
|
|
15
|
+
* POST to `/scenarios/:id/derive`, under the key it already uses to calculate
|
|
16
|
+
* scenarios and to ask its live question. That is the path nearly every session
|
|
17
|
+
* takes: the people this runs for are inside Claude Code on a subscription and
|
|
18
|
+
* have no API key to give it.
|
|
19
|
+
*
|
|
20
|
+
* A key here is an override, not a requirement. It keeps the reading on this
|
|
21
|
+
* machine — the prompt never reaches the service — which is what an in-house
|
|
22
|
+
* deployment wants, and it is one round trip faster.
|
|
23
|
+
*
|
|
24
|
+
* NEITHER is still supported and still means the same thing it always did:
|
|
25
|
+
* settings take their recorded samples and a scenario with a target declines,
|
|
26
|
+
* because a target is never guessed.
|
|
18
27
|
*
|
|
19
28
|
* The prompt and the parsing tolerances mirror RRepeat's
|
|
20
29
|
* `deriveParametersFromScenarioPayload` exactly, so the two produce the same
|
|
@@ -120,14 +129,20 @@ export async function deriveParameters(opts) {
|
|
|
120
129
|
if (keys.length === 0)
|
|
121
130
|
return { params: {}, costUsd: 0, derived: false, missing: [], sampled: [] };
|
|
122
131
|
const apiKey = opts.apiKey ?? process.env.ANTHROPIC_API_KEY;
|
|
123
|
-
// No key
|
|
124
|
-
//
|
|
125
|
-
|
|
132
|
+
// No key of our own is the ordinary case: the service reads the turn instead,
|
|
133
|
+
// under the key it already spends on this account (R-PARAM-5).
|
|
134
|
+
if (!apiKey && opts.service)
|
|
135
|
+
return deriveViaService(opts, opts.service, schema, keys);
|
|
136
|
+
// No key and nowhere to ask — an unauthenticated session, or a deployment that
|
|
137
|
+
// turned service derivation off. `derived: false` is how the caller tells this
|
|
138
|
+
// apart from a real extraction: it declines any scenario that has a target and
|
|
139
|
+
// arms an all-settings one on its recorded values.
|
|
126
140
|
if (!apiKey)
|
|
127
141
|
return {
|
|
128
142
|
params: sampleValues(schema),
|
|
129
143
|
costUsd: 0,
|
|
130
144
|
derived: false,
|
|
145
|
+
via: "samples",
|
|
131
146
|
missing: [],
|
|
132
147
|
sampled: keys,
|
|
133
148
|
};
|
|
@@ -213,6 +228,7 @@ export async function deriveParameters(opts) {
|
|
|
213
228
|
params: parsed,
|
|
214
229
|
costUsd,
|
|
215
230
|
derived: true,
|
|
231
|
+
via: "key",
|
|
216
232
|
model,
|
|
217
233
|
inputTokens,
|
|
218
234
|
outputTokens,
|
|
@@ -220,4 +236,88 @@ export async function deriveParameters(opts) {
|
|
|
220
236
|
sampled,
|
|
221
237
|
};
|
|
222
238
|
}
|
|
239
|
+
/**
|
|
240
|
+
* Ask the service to read the turn (segmented.md R-PARAM-5).
|
|
241
|
+
*
|
|
242
|
+
* One authorized POST. The body carries the turn — the prompt, and whatever else
|
|
243
|
+
* this turn has said about its target — and never the schema: the service reads
|
|
244
|
+
* that from the scenario row it owns, so a stale copy here cannot change what is
|
|
245
|
+
* asked for.
|
|
246
|
+
*
|
|
247
|
+
* Never throws, and never worse than no key. A service that is off, out of
|
|
248
|
+
* budget, too slow or simply down answers `derived: false`, and so does every
|
|
249
|
+
* failure on this side, which is precisely the keyless behaviour this replaced:
|
|
250
|
+
* settings stand on their samples, and a target does not run.
|
|
251
|
+
*/
|
|
252
|
+
async function deriveViaService(opts, service, schema, keys) {
|
|
253
|
+
const samples = (reason) => ({
|
|
254
|
+
params: sampleValues(schema),
|
|
255
|
+
costUsd: 0,
|
|
256
|
+
derived: false,
|
|
257
|
+
via: "service",
|
|
258
|
+
reason,
|
|
259
|
+
missing: [],
|
|
260
|
+
sampled: keys,
|
|
261
|
+
});
|
|
262
|
+
const doFetch = opts.fetchImpl ?? fetch;
|
|
263
|
+
const url = `${service.baseUrl.replace(/\/+$/, "")}/scenarios/${service.scenarioId}/derive`;
|
|
264
|
+
let body;
|
|
265
|
+
try {
|
|
266
|
+
const res = await doFetch(url, {
|
|
267
|
+
method: "POST",
|
|
268
|
+
headers: {
|
|
269
|
+
"content-type": "application/json",
|
|
270
|
+
authorization: `Bearer ${service.token}`,
|
|
271
|
+
},
|
|
272
|
+
body: JSON.stringify({
|
|
273
|
+
prompt: opts.prompt,
|
|
274
|
+
liveCall: opts.liveCall,
|
|
275
|
+
recentResults: opts.recentResults,
|
|
276
|
+
}),
|
|
277
|
+
signal: opts.signal,
|
|
278
|
+
});
|
|
279
|
+
if (!res.ok)
|
|
280
|
+
return samples(`service HTTP ${res.status}`);
|
|
281
|
+
body = (await res.json());
|
|
282
|
+
}
|
|
283
|
+
catch (err) {
|
|
284
|
+
return samples(err instanceof Error ? err.message : String(err));
|
|
285
|
+
}
|
|
286
|
+
if (!body || typeof body !== "object")
|
|
287
|
+
return samples("service sent no answer");
|
|
288
|
+
if (body.derived !== true)
|
|
289
|
+
return samples(`service: ${body.reason ?? "declined"}`);
|
|
290
|
+
// Normalised again on this side, over *our* copy of the schema. The service
|
|
291
|
+
// answers in the same shape, but the plan runs on these values and the gate
|
|
292
|
+
// reads this `missing`: an older or newer service must not be able to leave a
|
|
293
|
+
// target unaccounted for.
|
|
294
|
+
const params = { ...(body.params ?? {}) };
|
|
295
|
+
const missing = [];
|
|
296
|
+
const sampled = [];
|
|
297
|
+
for (const key of keys) {
|
|
298
|
+
const value = params[key];
|
|
299
|
+
if (value !== null && value !== undefined)
|
|
300
|
+
continue;
|
|
301
|
+
if (isTarget(schema, key)) {
|
|
302
|
+
params[key] = null;
|
|
303
|
+
missing.push(key);
|
|
304
|
+
}
|
|
305
|
+
else {
|
|
306
|
+
params[key] = schema[key].sampleValue;
|
|
307
|
+
sampled.push(key);
|
|
308
|
+
}
|
|
309
|
+
}
|
|
310
|
+
return {
|
|
311
|
+
params,
|
|
312
|
+
// What the service spent on this turn, reported back on the execution so a
|
|
313
|
+
// replay's cost stays the whole truth about it (R-MONEY-2). The account is
|
|
314
|
+
// not billed for it; the service books it on its own ledger as well.
|
|
315
|
+
costUsd: typeof body.costUsd === "number" ? body.costUsd : 0,
|
|
316
|
+
derived: true,
|
|
317
|
+
via: "service",
|
|
318
|
+
model: body.model,
|
|
319
|
+
missing,
|
|
320
|
+
sampled,
|
|
321
|
+
};
|
|
322
|
+
}
|
|
223
323
|
//# sourceMappingURL=derive.js.map
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Which missing targets actually stop a plan (segmented.md R-PARAM-3).
|
|
3
|
+
*
|
|
4
|
+
* The rule a target exists for is "never do the right work on the wrong thing":
|
|
5
|
+
* a value the turn did not name must never be filled in from the recording. The
|
|
6
|
+
* gate that enforces it used to read `missing.length > 0` and decline, which is
|
|
7
|
+
* the same thing only when every target is something the chain *takes from the
|
|
8
|
+
* caller*.
|
|
9
|
+
*
|
|
10
|
+
* It often is not. An analysis routinely lists a value that a later step
|
|
11
|
+
* computes from an earlier step's output and keeps the caller's copy as a bare
|
|
12
|
+
* fallback:
|
|
13
|
+
*
|
|
14
|
+
* step 0 return { fleet: parameters.fleet };
|
|
15
|
+
* step 1 return { id: (respParams.rankedDeviceIds || [])[0] ?? parameters.id };
|
|
16
|
+
*
|
|
17
|
+
* `id` is classified a target — it is what the second call acts on and it does
|
|
18
|
+
* change between requests — but no prompt ever names it, because the chain
|
|
19
|
+
* discovers it. Declining on it meant a scenario that matched, was ready, was
|
|
20
|
+
* enabled and would have run perfectly never ran at all, on every turn, with the
|
|
21
|
+
* reason only in a stderr line.
|
|
22
|
+
*
|
|
23
|
+
* So the question is not "did the turn name every target" but the narrower one
|
|
24
|
+
* the rule actually cares about: **could this plan act on the recorded value of
|
|
25
|
+
* a target the turn did not name?** It could, in exactly two ways:
|
|
26
|
+
*
|
|
27
|
+
* 1. some step reads `parameters.<key>` outright, rather than as the fallback
|
|
28
|
+
* of a value computed at run time; or
|
|
29
|
+
* 2. some step's logic carries the recorded sample as a literal, so the value
|
|
30
|
+
* is baked into the chain whether or not anybody reads the parameter.
|
|
31
|
+
*
|
|
32
|
+
* Neither true means the recorded value cannot reach a tool. The missing target
|
|
33
|
+
* stays `null` all the way through — {@link ./derive.ts} sets it, `paramsLogic`
|
|
34
|
+
* copies it, the `??` passes over it — so a chain that can compute the value
|
|
35
|
+
* runs on this turn's value, and one that cannot calls a tool with `null` and
|
|
36
|
+
* falls back to the model. What never happens, either way, is last week's fleet.
|
|
37
|
+
*
|
|
38
|
+
* Only frame 0 is judged here. A called segment runs on the parameters its
|
|
39
|
+
* call's `paramMapLogic` builds, not on these, so that body is scanned as a
|
|
40
|
+
* consumer and the segment's own steps are not (R-CALL-29).
|
|
41
|
+
*/
|
|
42
|
+
import type { FlatEntry } from "./flatten.js";
|
|
43
|
+
import type { ParamsSchema } from "./types.js";
|
|
44
|
+
/**
|
|
45
|
+
* The subset of `missing` this plan must decline on.
|
|
46
|
+
*
|
|
47
|
+
* Empty means every target the turn left unnamed is one the chain finds for
|
|
48
|
+
* itself. `missing` is returned unchanged whenever the schema or the plan is
|
|
49
|
+
* missing, so a caller that knows less than this one is never made to guess.
|
|
50
|
+
*/
|
|
51
|
+
export declare function blockingTargets(missing: readonly string[], schema: ParamsSchema | null | undefined, planned: readonly FlatEntry[]): string[];
|
|
52
|
+
//# sourceMappingURL=targets.d.ts.map
|
|
@@ -0,0 +1,133 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Which missing targets actually stop a plan (segmented.md R-PARAM-3).
|
|
3
|
+
*
|
|
4
|
+
* The rule a target exists for is "never do the right work on the wrong thing":
|
|
5
|
+
* a value the turn did not name must never be filled in from the recording. The
|
|
6
|
+
* gate that enforces it used to read `missing.length > 0` and decline, which is
|
|
7
|
+
* the same thing only when every target is something the chain *takes from the
|
|
8
|
+
* caller*.
|
|
9
|
+
*
|
|
10
|
+
* It often is not. An analysis routinely lists a value that a later step
|
|
11
|
+
* computes from an earlier step's output and keeps the caller's copy as a bare
|
|
12
|
+
* fallback:
|
|
13
|
+
*
|
|
14
|
+
* step 0 return { fleet: parameters.fleet };
|
|
15
|
+
* step 1 return { id: (respParams.rankedDeviceIds || [])[0] ?? parameters.id };
|
|
16
|
+
*
|
|
17
|
+
* `id` is classified a target — it is what the second call acts on and it does
|
|
18
|
+
* change between requests — but no prompt ever names it, because the chain
|
|
19
|
+
* discovers it. Declining on it meant a scenario that matched, was ready, was
|
|
20
|
+
* enabled and would have run perfectly never ran at all, on every turn, with the
|
|
21
|
+
* reason only in a stderr line.
|
|
22
|
+
*
|
|
23
|
+
* So the question is not "did the turn name every target" but the narrower one
|
|
24
|
+
* the rule actually cares about: **could this plan act on the recorded value of
|
|
25
|
+
* a target the turn did not name?** It could, in exactly two ways:
|
|
26
|
+
*
|
|
27
|
+
* 1. some step reads `parameters.<key>` outright, rather than as the fallback
|
|
28
|
+
* of a value computed at run time; or
|
|
29
|
+
* 2. some step's logic carries the recorded sample as a literal, so the value
|
|
30
|
+
* is baked into the chain whether or not anybody reads the parameter.
|
|
31
|
+
*
|
|
32
|
+
* Neither true means the recorded value cannot reach a tool. The missing target
|
|
33
|
+
* stays `null` all the way through — {@link ./derive.ts} sets it, `paramsLogic`
|
|
34
|
+
* copies it, the `??` passes over it — so a chain that can compute the value
|
|
35
|
+
* runs on this turn's value, and one that cannot calls a tool with `null` and
|
|
36
|
+
* falls back to the model. What never happens, either way, is last week's fleet.
|
|
37
|
+
*
|
|
38
|
+
* Only frame 0 is judged here. A called segment runs on the parameters its
|
|
39
|
+
* call's `paramMapLogic` builds, not on these, so that body is scanned as a
|
|
40
|
+
* consumer and the segment's own steps are not (R-CALL-29).
|
|
41
|
+
*/
|
|
42
|
+
/**
|
|
43
|
+
* The subset of `missing` this plan must decline on.
|
|
44
|
+
*
|
|
45
|
+
* Empty means every target the turn left unnamed is one the chain finds for
|
|
46
|
+
* itself. `missing` is returned unchanged whenever the schema or the plan is
|
|
47
|
+
* missing, so a caller that knows less than this one is never made to guess.
|
|
48
|
+
*/
|
|
49
|
+
export function blockingTargets(missing, schema, planned) {
|
|
50
|
+
if (missing.length === 0)
|
|
51
|
+
return [];
|
|
52
|
+
if (!schema || planned.length === 0)
|
|
53
|
+
return [...missing];
|
|
54
|
+
const bodies = callerBodies(planned);
|
|
55
|
+
// Nothing to read the parameters: a plan with no logic at all cannot act on
|
|
56
|
+
// anything, but it is also not a shape worth reasoning about — keep it strict.
|
|
57
|
+
if (bodies.length === 0)
|
|
58
|
+
return [...missing];
|
|
59
|
+
return missing.filter((key) => bodies.some((body) => readsOutright(body, key) || carriesLiteral(body, schema[key]?.sampleValue)));
|
|
60
|
+
}
|
|
61
|
+
/**
|
|
62
|
+
* Every logic body that runs with the caller's own `parameters`.
|
|
63
|
+
*
|
|
64
|
+
* Both of a step's bodies, because `toolOutputLogic` is handed `parameters`
|
|
65
|
+
* too and a value baked in there travels just as far. Plus the `paramMapLogic`
|
|
66
|
+
* of each segment called directly from frame 0: it reads the caller's
|
|
67
|
+
* parameters to build the segment's.
|
|
68
|
+
*/
|
|
69
|
+
function callerBodies(planned) {
|
|
70
|
+
const bodies = [];
|
|
71
|
+
const seenFrames = new Set();
|
|
72
|
+
for (const entry of planned) {
|
|
73
|
+
if (entry.depth === 0) {
|
|
74
|
+
if (entry.step.toolInputLogic)
|
|
75
|
+
bodies.push(entry.step.toolInputLogic);
|
|
76
|
+
if (entry.step.toolOutputLogic)
|
|
77
|
+
bodies.push(entry.step.toolOutputLogic);
|
|
78
|
+
continue;
|
|
79
|
+
}
|
|
80
|
+
// The frame's own mapping, once, and only when its caller is frame 0.
|
|
81
|
+
const frame = entry.frame;
|
|
82
|
+
if (frame.depth !== 1 || seenFrames.has(frame.id))
|
|
83
|
+
continue;
|
|
84
|
+
seenFrames.add(frame.id);
|
|
85
|
+
if (frame.paramMapLogic)
|
|
86
|
+
bodies.push(frame.paramMapLogic);
|
|
87
|
+
}
|
|
88
|
+
return bodies;
|
|
89
|
+
}
|
|
90
|
+
/** `parameters.key`, `parameters["key"]`, `parameters['key']` — all three forms. */
|
|
91
|
+
function referencePattern(key) {
|
|
92
|
+
const k = escapeRegExp(key);
|
|
93
|
+
return new RegExp(`parameters\\s*(?:\\.\\s*${k}\\b|\\[\\s*["']${k}["']\\s*\\])`, "g");
|
|
94
|
+
}
|
|
95
|
+
/**
|
|
96
|
+
* Whether `body` reads `key` other than as a fallback.
|
|
97
|
+
*
|
|
98
|
+
* A reference is a fallback when what stands immediately before it is `??` or
|
|
99
|
+
* `||` — the shape an analysis writes when the chain computes the value and
|
|
100
|
+
* keeps the caller's as a last resort. Every other reference is a read: the
|
|
101
|
+
* step wants the caller's value and nothing else will do.
|
|
102
|
+
*/
|
|
103
|
+
function readsOutright(body, key) {
|
|
104
|
+
const pattern = referencePattern(key);
|
|
105
|
+
for (let m = pattern.exec(body); m; m = pattern.exec(body)) {
|
|
106
|
+
const before = body.slice(0, m.index).trimEnd();
|
|
107
|
+
if (!before.endsWith("??") && !before.endsWith("||"))
|
|
108
|
+
return true;
|
|
109
|
+
}
|
|
110
|
+
return false;
|
|
111
|
+
}
|
|
112
|
+
/**
|
|
113
|
+
* Whether `body` carries the recorded sample as a literal.
|
|
114
|
+
*
|
|
115
|
+
* This is the case the parameter reference cannot see: `return { id: 'dev_4411' }`
|
|
116
|
+
* acts on last week's device without mentioning `parameters` at all. Read
|
|
117
|
+
* generously — a sample too short to search for (`1`, `on`) counts as carried,
|
|
118
|
+
* because a false "this plan is safe" is the one answer this file must not give.
|
|
119
|
+
*/
|
|
120
|
+
function carriesLiteral(body, sample) {
|
|
121
|
+
if (sample === null || sample === undefined)
|
|
122
|
+
return false;
|
|
123
|
+
const text = typeof sample === "string" ? sample : JSON.stringify(sample);
|
|
124
|
+
if (typeof text !== "string" || text.length === 0)
|
|
125
|
+
return false;
|
|
126
|
+
if (text.length < 3)
|
|
127
|
+
return true;
|
|
128
|
+
return body.includes(text);
|
|
129
|
+
}
|
|
130
|
+
function escapeRegExp(text) {
|
|
131
|
+
return text.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
132
|
+
}
|
|
133
|
+
//# sourceMappingURL=targets.js.map
|
package/docs/calculatedReplay.md
CHANGED
|
@@ -853,7 +853,8 @@ fail because of BaseInstRunner.**
|
|
|
853
853
|
| Failure | Behaviour |
|
|
854
854
|
|---|---|
|
|
855
855
|
| `getMatch()` slower than the match budget | No plan. Ordinary turn. One `replay.decision` line |
|
|
856
|
-
| No `ANTHROPIC_API_KEY` |
|
|
856
|
+
| No `ANTHROPIC_API_KEY` | The service derives instead (segmented.md R-PARAM-5). Signed out as well: settings take their recorded `sampleValue`s, a scenario with a target declines `no_derive_key`, `deriveCostUsd: 0` |
|
|
857
|
+
| The service declines, is slow, or cannot be reached | The same, with its reason carried into the `replay.derived` and `replay.decision` lines. Never a throw |
|
|
857
858
|
| Derivation call fails or times out | Same as above, plus a `replay.derive_failed` line |
|
|
858
859
|
| `toolInputLogic` throws | Divergence (§8) — never a crash |
|
|
859
860
|
| `toolOutputLogic` throws | Retire the plan, abort to an ordinary turn, outcome `failed` |
|
|
@@ -982,7 +983,8 @@ export async function deriveParameters(opts: {
|
|
|
982
983
|
prompt: string;
|
|
983
984
|
intent: string;
|
|
984
985
|
paramsObject: Record<string, { sampleValue: unknown; description: string }> | null;
|
|
985
|
-
apiKey?: string; // ANTHROPIC_API_KEY
|
|
986
|
+
apiKey?: string; // ANTHROPIC_API_KEY — an override
|
|
987
|
+
service?: { baseUrl: string; token: string; scenarioId: string }; // the default path
|
|
986
988
|
model?: string; // default claude-haiku-4-5-20251001
|
|
987
989
|
signal?: AbortSignal;
|
|
988
990
|
}): Promise<{ params: Record<string, unknown>; costUsd: number; derived: boolean }>;
|
|
@@ -992,11 +994,16 @@ Behaviour, matching RRepeat's `deriveParametersFromScenarioPayload` exactly so t
|
|
|
992
994
|
same parameters from the same prompt:
|
|
993
995
|
|
|
994
996
|
- No `paramsObject` keys → `{}`, `costUsd: 0`.
|
|
995
|
-
- **No `apiKey`
|
|
996
|
-
|
|
997
|
-
|
|
998
|
-
|
|
999
|
-
from
|
|
997
|
+
- **No `apiKey` but a `service` → `POST {baseUrl}/scenarios/{id}/derive`** with the turn (prompt,
|
|
998
|
+
live call, recent results) and a bearer token, `derived` as the service answered. This is the
|
|
999
|
+
ordinary path: nearly every session runs inside Claude Code on a subscription and has no key of
|
|
1000
|
+
its own (segmented.md R-PARAM-5). The answer is re-normalised here against this runner's own copy
|
|
1001
|
+
of the schema, so the gate reads a `missing` computed from what will actually run.
|
|
1002
|
+
- **Neither → return every key's recorded `sampleValue`**, `costUsd: 0`, `derived: false`. Settings
|
|
1003
|
+
stand on their samples; a scenario with a target declines rather than run on a stale one.
|
|
1004
|
+
- With an `apiKey`, `POST https://api.anthropic.com/v1/messages` with the same extraction prompt,
|
|
1005
|
+
tolerant JSON extraction (first balanced `{…}`, `undefined` → `null`), and any `null`/missing
|
|
1006
|
+
**setting** filled from its `sampleValue` — a `null` target is named in `missing` instead.
|
|
1000
1007
|
|
|
1001
1008
|
Keeping this dependency-free is not purity. `bir-proxy` is spawned inside the host's process tree for
|
|
1002
1009
|
*every* wrapped server; an npm install that pulls a model SDK into that path is a startup-latency and
|
|
@@ -1041,7 +1048,8 @@ bir replay --scenario <scnId> --prompt "…" [--dry] alias; the Tier 2 / deb
|
|
|
1041
1048
|
source run's *recorded* outputs — no real tools, no side effects, one Haiku call. Without `--dry`,
|
|
1042
1049
|
`bir replay` runs the same plan through the same executor against the live proxies. `bir doctor`
|
|
1043
1050
|
gains a `replay` block: on/off, the effective `BIR_REPLAY_ALLOW_SERVERS` allowlist,
|
|
1044
|
-
`BIR_MIN_STEER_SIMILARITY`, and
|
|
1051
|
+
`BIR_MIN_STEER_SIMILARITY`, and who derives — this machine, the service, or nobody, which is the
|
|
1052
|
+
state in which a scenario with a target never runs (§13.2, mitigation 3).
|
|
1045
1053
|
|
|
1046
1054
|
---
|
|
1047
1055
|
|
|
@@ -34,7 +34,7 @@ end-to-end smoke test.
|
|
|
34
34
|
| 3 | `SIMILARITY_DETECTION_ENABLED=true` on the server (default) | otherwise no prompt ever matches |
|
|
35
35
|
| 4 | BaseInstRunnerMCP built and installed in your project | `bir status` |
|
|
36
36
|
| 5 | Tier 1 — the hooks wired and `bir-hooks` running | `bir doctor` |
|
|
37
|
-
| 6 | *(replay only)*
|
|
37
|
+
| 6 | *(replay only)* signed in, so the service can read what each request acts on | `bir doctor` — `derive=the service`; an `ANTHROPIC_API_KEY` here replaces it, see §5.3 |
|
|
38
38
|
|
|
39
39
|
Tier 1 is not optional for replay. A match is a match on **the prompt**, and a standalone proxy never
|
|
40
40
|
sees one (design §1.1). If `bir doctor` says `tier: standalone`, replay cannot arm, and that is the
|
|
@@ -229,7 +229,7 @@ bir doctor
|
|
|
229
229
|
|
|
230
230
|
```
|
|
231
231
|
replay ON servers=chrome-devtools,fleet-api minSimilarity=0.92
|
|
232
|
-
derive=
|
|
232
|
+
derive=the service (no key needed here)
|
|
233
233
|
control http://127.0.0.1:53411 sess=birsess_…
|
|
234
234
|
wrapped chrome-devtools ✓ proxy pid 41822 fleet-api ✓ proxy pid 41823
|
|
235
235
|
recording yes (https://your-basein-service)
|
|
@@ -246,8 +246,8 @@ there is exactly one switch and it cannot get out of step with itself.
|
|
|
246
246
|
| `BIR_REPLAY` | *(unset)* | `1` enables replay. Nothing below matters until it is set |
|
|
247
247
|
| `BIR_REPLAY_ALLOW_SERVERS` | *(all wrapped)* | Comma-separated server keys eligible for **direct** execution. **Set this** |
|
|
248
248
|
| `BIR_MIN_STEER_SIMILARITY` | `0.92` | Below this a match is detected but not replayed (§8) |
|
|
249
|
-
| `ANTHROPIC_API_KEY` | *(unset)* |
|
|
250
|
-
| `BIR_DERIVE_MODEL` | `claude-haiku-4-5-20251001` | The derivation model |
|
|
249
|
+
| `ANTHROPIC_API_KEY` | *(unset)* | **Not required.** Derivation — reading what this request acts on — is done by the service on its key for a signed-in runner. Set this to keep the reading on this machine instead: the prompt then never leaves it, and it is one round trip faster. Signed out *and* unset, only a scenario with nothing to work out replays |
|
|
250
|
+
| `BIR_DERIVE_MODEL` | `claude-haiku-4-5-20251001` | The derivation model, when this machine does the reading |
|
|
251
251
|
| `BIR_MATCH_BUDGET_MS` | `2500` | How long the prompt hook waits for a match before giving up |
|
|
252
252
|
| `BIR_DERIVE_BUDGET_MS` | `8000` | How long the first `PreToolUse` waits for parameters |
|
|
253
253
|
| `BIR_REPLAY_BUDGET_MS` | `120000` | Whole-plan ceiling for direct execution |
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@basein/runner",
|
|
3
|
-
"version": "0.2.
|
|
3
|
+
"version": "0.2.3",
|
|
4
4
|
"description": "A recording MCP proxy: sits between any MCP client and its MCP servers, executes each call on the client's behalf, and records the run as a reusable BaseIn scenario.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "MIT",
|