@basein/runner 0.3.8 → 0.3.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +46 -5
- package/dist/auth/client.d.ts +15 -0
- package/dist/auth/client.js +34 -12
- package/dist/bin/bir-hooks.d.ts +1 -1
- package/dist/bin/bir-hooks.js +1 -1
- package/dist/bin/bir.d.ts +2 -0
- package/dist/bin/bir.js +60 -6
- package/dist/bin/investigate.d.ts +5 -0
- package/dist/bin/investigate.js +117 -15
- package/dist/bin/setup.d.ts +12 -0
- package/dist/bin/setup.js +185 -56
- package/dist/bin/update.d.ts +81 -0
- package/dist/bin/update.js +194 -0
- package/dist/config/adapters/claude-code.d.ts +17 -0
- package/dist/config/adapters/claude-code.js +28 -1
- package/dist/config/generate.d.ts +14 -0
- package/dist/control/daemon.d.ts +26 -1
- package/dist/control/daemon.js +53 -3
- package/dist/control/pin-guard.d.ts +6 -1
- package/dist/control/pin-guard.js +15 -1
- package/dist/control/recorder-keeper.js +2 -1
- package/dist/control/server.d.ts +200 -2
- package/dist/control/server.js +741 -48
- package/dist/control/transcript.d.ts +4 -0
- package/dist/control/transcript.js +4 -0
- package/dist/proxy/session.js +1 -0
- package/dist/record/housekeeping.d.ts +7 -1
- package/dist/record/housekeeping.js +11 -3
- package/dist/record/recorder.d.ts +34 -0
- package/dist/record/remote-recorder.d.ts +14 -0
- package/dist/record/remote-recorder.js +107 -13
- package/dist/record/switchable-recorder.js +1 -0
- package/dist/record/truncate.d.ts +11 -0
- package/dist/record/truncate.js +11 -0
- package/dist/replay/controller.d.ts +146 -4
- package/dist/replay/controller.js +340 -31
- package/dist/replay/derive.js +2 -0
- package/dist/replay/handover.d.ts +13 -1
- package/dist/replay/handover.js +19 -0
- package/dist/replay/plan.d.ts +6 -0
- package/dist/replay/plan.js +13 -3
- package/dist/replay/safety.d.ts +15 -4
- package/dist/replay/safety.js +19 -6
- package/dist/replay/source-run.js +2 -1
- package/dist/replay/types.d.ts +16 -0
- package/dist/util/service-headers.d.ts +24 -0
- package/dist/util/service-headers.js +47 -0
- package/docs/BaseInstRunner.md +115 -1
- package/docs/calculatedReplay.md +239 -27
- package/docs/calculatedReplayGuide.md +134 -12
- package/docs/installRun.md +91 -10
- package/docs/quickstart.md +64 -9
- package/package.json +1 -1
- package/production.md +81 -16
- package/scripts/install.ps1 +25 -1
|
@@ -13,8 +13,8 @@
|
|
|
13
13
|
*/
|
|
14
14
|
import { parseQualifiedName } from "../control/correlation.js";
|
|
15
15
|
import { isHousekeepingCall } from "../record/housekeeping.js";
|
|
16
|
-
import { redact } from "../record/redact.js";
|
|
17
|
-
import { serializeCapped } from "../record/truncate.js";
|
|
16
|
+
import { REDACT_SCAN_MAX_CHARS, redact, redactJsonText, redactText } from "../record/redact.js";
|
|
17
|
+
import { capText, REPORT_PAYLOAD_MAX_BYTES, serializeCapped } from "../record/truncate.js";
|
|
18
18
|
import { logDetail, logLine, errText } from "../util/log.js";
|
|
19
19
|
import { serviceHealth, serviceTimeoutMs, timeoutSignal, } from "../util/service-health.js";
|
|
20
20
|
import { bashCommandFor } from "./bash-channel.js";
|
|
@@ -62,7 +62,7 @@ export const DEFAULT_BUDGETS = {
|
|
|
62
62
|
deriveMs: 8_000,
|
|
63
63
|
planMs: 100_000,
|
|
64
64
|
stepMs: 60_000,
|
|
65
|
-
hookDeriveMs:
|
|
65
|
+
hookDeriveMs: 8_000,
|
|
66
66
|
preToolMs: 9_000,
|
|
67
67
|
};
|
|
68
68
|
/**
|
|
@@ -311,15 +311,21 @@ export class ReplayController {
|
|
|
311
311
|
* Always returns a state — a declined match still has a ticket to redeem and a
|
|
312
312
|
* baseline sample to contribute, and losing that is how a savings ledger ends
|
|
313
313
|
* up with a denominator nobody measured.
|
|
314
|
+
*
|
|
315
|
+
* `host` is given for a session whose host is not Claude Code, and is kept
|
|
316
|
+
* on the state (see {@link HostContext}).
|
|
314
317
|
*/
|
|
315
|
-
async arm(match, prompt, wrapped, armedBy = "prompt", ctx = {}) {
|
|
318
|
+
async arm(match, prompt, wrapped, armedBy = "prompt", ctx = {}, host) {
|
|
316
319
|
const state = {
|
|
317
320
|
scenarioId: match.scenarioId,
|
|
318
321
|
ticket: match.executionTicket,
|
|
319
322
|
similarity: match.similarity,
|
|
320
323
|
matchedRunId: match.runId,
|
|
321
324
|
mode: "none",
|
|
322
|
-
|
|
325
|
+
// No proxy of this folder sees another host's calls, whatever a tool is
|
|
326
|
+
// named: every output is threaded from the host's own report.
|
|
327
|
+
wrapped: host ? new Set() : wrapped,
|
|
328
|
+
host,
|
|
323
329
|
pinned: new Map(),
|
|
324
330
|
outcome: "not_steered",
|
|
325
331
|
deriveCostUsd: 0,
|
|
@@ -364,6 +370,18 @@ export class ReplayController {
|
|
|
364
370
|
// Gate 2 — replay enabled.
|
|
365
371
|
if (!this.enabled)
|
|
366
372
|
return decline("BIR_REPLAY=0", "replay_disabled");
|
|
373
|
+
// And asked for. An adapter that only records never asks for a plan, and
|
|
374
|
+
// one started for it anyway would have nobody to step through it — while
|
|
375
|
+
// every call its model made itself would be read against the plan.
|
|
376
|
+
if (host?.recordsOnly) {
|
|
377
|
+
return decline(`the ${host.app} adapter of this run only records — it asked for no plan with its prompt`, "host_records_only");
|
|
378
|
+
}
|
|
379
|
+
// The service matched the prompt and handed over nothing, because this
|
|
380
|
+
// runner is older than its plans need (its RUNNER_MIN_CONTRACT). Its own
|
|
381
|
+
// code, ahead of "no scenario": the fix is an update, not a calculation.
|
|
382
|
+
if (match.withheld === "runner_update_required") {
|
|
383
|
+
return decline("the service hands this runner no plan until it is updated — run `bir update`", "runner_update_required");
|
|
384
|
+
}
|
|
367
385
|
// Gate 3 — a ready scenario with steps.
|
|
368
386
|
const scenario = match.scenario;
|
|
369
387
|
if (!match.scenarioId || !isReadyScenario(scenario)) {
|
|
@@ -428,7 +446,13 @@ export class ReplayController {
|
|
|
428
446
|
// built-in on `BIR_BUILTIN_TOOLS` counts as reachable: the runner
|
|
429
447
|
// executes it itself (replay/builtins.ts), so a chain that starts with a
|
|
430
448
|
// `Read` no longer forces steer mode.
|
|
431
|
-
|
|
449
|
+
//
|
|
450
|
+
// Another host's plan is always `steer`, every step `live`: its tools are
|
|
451
|
+
// in its own process, and a tool it happens to call `Read` or
|
|
452
|
+
// `mcp__fleet__list` is not the runner's to run or a proxy's to reach.
|
|
453
|
+
const allowServers = host ? undefined : this.opts.allowServers;
|
|
454
|
+
const builtinTools = host ? undefined : this.builtinTools;
|
|
455
|
+
const mode = modeFor(planned.map((e) => e.step), state.wrapped, allowServers, builtinTools);
|
|
432
456
|
if (mode === "none")
|
|
433
457
|
return decline("no step is executable", "coverage");
|
|
434
458
|
// Tool results feed derivation as context, and they are whatever the agent
|
|
@@ -441,11 +465,11 @@ export class ReplayController {
|
|
|
441
465
|
// target only a result names declines more often — one replay, never a
|
|
442
466
|
// command nobody asked for.
|
|
443
467
|
let deriveCtx = ctx;
|
|
444
|
-
if (ctx.recentResults?.length && sinkOfPlan(planned) === "command") {
|
|
468
|
+
if (ctx.recentResults?.length && sinkOfPlan(planned, host?.sinks) === "command") {
|
|
445
469
|
deriveCtx = { liveCall: ctx.liveCall };
|
|
446
470
|
logLine("replay.derive_context", {
|
|
447
471
|
scenario: scenario.id,
|
|
448
|
-
shell: hasShellStep(planned) || undefined,
|
|
472
|
+
shell: hasShellStep(planned, host?.sinks) || undefined,
|
|
449
473
|
why: "a step's input is run by a shell or a database (or its tool is named for one, such as *_query) — " +
|
|
450
474
|
"recent tool results are not read for its parameters, so a target only they name is not found",
|
|
451
475
|
});
|
|
@@ -527,7 +551,7 @@ export class ReplayController {
|
|
|
527
551
|
// same check still runs where the values settle later (`readyFor`).
|
|
528
552
|
// It travels as `missing_target`: the value read for a target could not
|
|
529
553
|
// be used, and no new code reaches a service that may not know it.
|
|
530
|
-
const sink = sinkOfPlan(planned);
|
|
554
|
+
const sink = sinkOfPlan(planned, host?.sinks);
|
|
531
555
|
if (result?.derived && sink) {
|
|
532
556
|
const bad = unsafeParameter(result.params, scenario.paramsObject, sink);
|
|
533
557
|
if (bad) {
|
|
@@ -590,6 +614,7 @@ export class ReplayController {
|
|
|
590
614
|
run: match.runId,
|
|
591
615
|
scenario: scenario.id,
|
|
592
616
|
mode,
|
|
617
|
+
host: host?.app,
|
|
593
618
|
steps: entries.length,
|
|
594
619
|
stopsAt: stopAt,
|
|
595
620
|
by: armedBy === "intent" ? "intent" : undefined,
|
|
@@ -602,11 +627,99 @@ export class ReplayController {
|
|
|
602
627
|
unusable: flat.stopReason?.reason,
|
|
603
628
|
// Which steps the service told this plan to stop in front of, and why.
|
|
604
629
|
stops: stopLines(entries),
|
|
605
|
-
coverage: coverageOf(entries.map((e) => e.step), wrapped,
|
|
630
|
+
coverage: coverageOf(entries.map((e) => e.step), state.wrapped, allowServers, builtinTools).join(","),
|
|
606
631
|
tools: entries.map((e) => e.step.toolName ?? "").join(","),
|
|
607
632
|
});
|
|
608
633
|
return state;
|
|
609
634
|
}
|
|
635
|
+
/**
|
|
636
|
+
* The armed plan as data: the steps it will run, in order, and where it
|
|
637
|
+
* stops when that is before its end. For a host that steps through the plan
|
|
638
|
+
* itself and must not have to pick the tool names out of a directive
|
|
639
|
+
* written for a model — a reworded sentence would break every adapter in
|
|
640
|
+
* the field. Built from the same entries the directive lists.
|
|
641
|
+
*/
|
|
642
|
+
planData(state) {
|
|
643
|
+
const plan = state.plan;
|
|
644
|
+
if (!plan || state.retired)
|
|
645
|
+
return undefined;
|
|
646
|
+
const steps = plan
|
|
647
|
+
.allEntries()
|
|
648
|
+
.slice(0, plan.stepCount)
|
|
649
|
+
.map((entry, stepIndex) => ({ stepIndex, toolName: entry.step.toolName ?? "" }));
|
|
650
|
+
return {
|
|
651
|
+
scenarioId: state.scenarioId,
|
|
652
|
+
mode: state.mode,
|
|
653
|
+
steps,
|
|
654
|
+
...(plan.stopsEarly() ? { stopAt: plan.stepCount } : {}),
|
|
655
|
+
};
|
|
656
|
+
}
|
|
657
|
+
/**
|
|
658
|
+
* Take back a plan that armed and has run nothing, as a decline.
|
|
659
|
+
*
|
|
660
|
+
* For what `arm` could not know: which tools the agent of another host
|
|
661
|
+
* holds. Such a host shows its tools only when its model is about to be
|
|
662
|
+
* called, with the first request for a step, and a plan that names a tool
|
|
663
|
+
* the agent lacks cannot be finished there. It ends before any step runs,
|
|
664
|
+
* booked like any gate's decline: `not_steered` under `code`, no step
|
|
665
|
+
* marked failed or missing — either would count against a plan that is
|
|
666
|
+
* not at fault. Returns false, changing nothing, once a step has run.
|
|
667
|
+
*/
|
|
668
|
+
declineArmed(state, code, why) {
|
|
669
|
+
if (!state.plan || state.retired)
|
|
670
|
+
return false;
|
|
671
|
+
if (state.stepsPinned > 0 || state.pinned.size > 0 || state.stepResults.size > 0)
|
|
672
|
+
return false;
|
|
673
|
+
state.declined = code;
|
|
674
|
+
logLine("replay.decision", {
|
|
675
|
+
verdict: "no-steer",
|
|
676
|
+
code,
|
|
677
|
+
run: state.matchedRunId,
|
|
678
|
+
scenario: state.scenarioId ?? undefined,
|
|
679
|
+
similarity: state.similarity.toFixed(3),
|
|
680
|
+
kind: state.kind,
|
|
681
|
+
why,
|
|
682
|
+
});
|
|
683
|
+
state.plan = undefined;
|
|
684
|
+
state.mode = "none";
|
|
685
|
+
state.stepsPlanned = 0;
|
|
686
|
+
this.retire(state, undefined);
|
|
687
|
+
return true;
|
|
688
|
+
}
|
|
689
|
+
/**
|
|
690
|
+
* Hand a live plan over for something its host did, not the plan: the
|
|
691
|
+
* model made a call of its own while the plan was live (`own_call`), the
|
|
692
|
+
* run was closed with the plan still live (`closed`), or the host was
|
|
693
|
+
* answered without a step's arguments because they were not ready in time
|
|
694
|
+
* (`late`).
|
|
695
|
+
*
|
|
696
|
+
* Only for a host that is not Claude Code, where a call the plan did not
|
|
697
|
+
* hand out is by definition the model's own — it is never pinned and never
|
|
698
|
+
* answered with a bundle. No step is marked failed, so nothing is counted
|
|
699
|
+
* against the plan; the outcome is `fell_back` when a step ran and `failed`
|
|
700
|
+
* when none did, as for every hand-over. The note is the caller's to
|
|
701
|
+
* deliver ({@link takeNote}). Returns false when there was no live plan.
|
|
702
|
+
*/
|
|
703
|
+
hostHandOver(state, why) {
|
|
704
|
+
const plan = state.plan;
|
|
705
|
+
if (!plan || state.retired)
|
|
706
|
+
return false;
|
|
707
|
+
const position = plan.currentStepIndex;
|
|
708
|
+
this.handOver(state, {
|
|
709
|
+
kind: "step_failed",
|
|
710
|
+
step: plan.currentStep(),
|
|
711
|
+
entry: plan.currentEntry(),
|
|
712
|
+
position,
|
|
713
|
+
error: why === "own_call"
|
|
714
|
+
? "the model made a call of its own, so the scenario stopped"
|
|
715
|
+
: why === "late"
|
|
716
|
+
? "its arguments were not ready in time"
|
|
717
|
+
: "the run ended before this step",
|
|
718
|
+
delivery: "above",
|
|
719
|
+
stopReason: "not_run",
|
|
720
|
+
});
|
|
721
|
+
return true;
|
|
722
|
+
}
|
|
610
723
|
/**
|
|
611
724
|
* The directive to inject via `additionalContext`, or undefined when declined.
|
|
612
725
|
*
|
|
@@ -650,6 +763,10 @@ export class ReplayController {
|
|
|
650
763
|
const plan = state.plan;
|
|
651
764
|
if (!plan || state.retired || state.mode !== "steer")
|
|
652
765
|
return false;
|
|
766
|
+
// Another host reads the plan as data and is handed each step's arguments
|
|
767
|
+
// when it asks for them: there is no preview to wait for.
|
|
768
|
+
if (state.host)
|
|
769
|
+
return false;
|
|
653
770
|
if (plan.parametersSettled)
|
|
654
771
|
return true;
|
|
655
772
|
const planned = plan.allEntries().slice(0, plan.stepCount);
|
|
@@ -695,9 +812,13 @@ export class ReplayController {
|
|
|
695
812
|
* Deliver text to the model in place of the tool call it was about to make:
|
|
696
813
|
* as genuine command output for `Bash`, as the denial reason otherwise — the
|
|
697
814
|
* two channels a divergence bundle already uses.
|
|
815
|
+
*
|
|
816
|
+
* `otherHost` is a session whose host is not Claude Code: a tool it calls
|
|
817
|
+
* `Bash` is its own, and its command is never rewritten. The text is always
|
|
818
|
+
* the refusal reason there.
|
|
698
819
|
*/
|
|
699
|
-
deliverInstead(text, toolName) {
|
|
700
|
-
if (toolName === "Bash") {
|
|
820
|
+
deliverInstead(text, toolName, otherHost = false) {
|
|
821
|
+
if (toolName === "Bash" && !otherHost) {
|
|
701
822
|
// A short command, whatever the text's size (bash-channel.ts): a long
|
|
702
823
|
// heredoc is cut by the shell, and a long output by the host.
|
|
703
824
|
const command = bashCommandFor(truncate(text, BASH_CHANNEL_MAX));
|
|
@@ -750,7 +871,8 @@ export class ReplayController {
|
|
|
750
871
|
// `bir scenario show` the model runs while a Bash step is expected must
|
|
751
872
|
// run as `bir`, never be swapped for the plan's command (D13). And before
|
|
752
873
|
// the wait for the parameters, which a `ToolSearch` has no use for.
|
|
753
|
-
|
|
874
|
+
const otherHost = state.host !== undefined;
|
|
875
|
+
if (isHousekeepingCall(toolName, toolInput, otherHost)) {
|
|
754
876
|
logDetail("replay.housekeeping", {
|
|
755
877
|
tool: toolName,
|
|
756
878
|
step: `${plan.currentStepIndex}/${plan.stepCount}`,
|
|
@@ -779,7 +901,13 @@ export class ReplayController {
|
|
|
779
901
|
// The model does the task itself — told so in place of this call.
|
|
780
902
|
const step = plan.currentStep();
|
|
781
903
|
if (step && ready.blame) {
|
|
782
|
-
this.recordStep(state, {
|
|
904
|
+
this.recordStep(state, {
|
|
905
|
+
step,
|
|
906
|
+
entry: plan.currentEntry(),
|
|
907
|
+
outcome: "failed",
|
|
908
|
+
stage: ready.stage,
|
|
909
|
+
error: ready.error,
|
|
910
|
+
});
|
|
783
911
|
}
|
|
784
912
|
this.handOver(state, {
|
|
785
913
|
kind: "step_failed",
|
|
@@ -791,7 +919,7 @@ export class ReplayController {
|
|
|
791
919
|
stopReason: "not_run",
|
|
792
920
|
});
|
|
793
921
|
const note = this.takeNote(state);
|
|
794
|
-
return note ? this.deliverInstead(note, toolName) : { kind: "abort" };
|
|
922
|
+
return note ? this.deliverInstead(note, toolName, otherHost) : { kind: "abort" };
|
|
795
923
|
}
|
|
796
924
|
// ONE CALL AT A TIME. A pin moves the cursor only when its output threads,
|
|
797
925
|
// which for a wrapped step is the proxy's report — landing either side of
|
|
@@ -829,6 +957,7 @@ export class ReplayController {
|
|
|
829
957
|
reach: this.reachFor(state, step),
|
|
830
958
|
toolName,
|
|
831
959
|
pinnedAt: Date.now(),
|
|
960
|
+
input,
|
|
832
961
|
});
|
|
833
962
|
logDetail("replay.pin", { n: plan.currentStepIndex, tool: toolName });
|
|
834
963
|
return { kind: "pin", input, stepIndex: plan.currentStepIndex };
|
|
@@ -850,6 +979,7 @@ export class ReplayController {
|
|
|
850
979
|
if (step) {
|
|
851
980
|
this.recordStep(state, {
|
|
852
981
|
step,
|
|
982
|
+
entry: plan.currentEntry(),
|
|
853
983
|
outcome: "failed",
|
|
854
984
|
stage: "tool_input_logic",
|
|
855
985
|
error: errText(err),
|
|
@@ -863,11 +993,29 @@ export class ReplayController {
|
|
|
863
993
|
});
|
|
864
994
|
const note = this.takeNote(state);
|
|
865
995
|
if (note)
|
|
866
|
-
return this.deliverInstead(note, toolName);
|
|
996
|
+
return this.deliverInstead(note, toolName, otherHost);
|
|
867
997
|
}
|
|
868
998
|
return { kind: "abort" };
|
|
869
999
|
}
|
|
870
1000
|
}
|
|
1001
|
+
if (state.host) {
|
|
1002
|
+
// Another host steps through the plan by asking for each step in turn,
|
|
1003
|
+
// so a request for anything but the next step is its adapter out of
|
|
1004
|
+
// step with the plan. Nothing can be recovered by running the rest
|
|
1005
|
+
// here — only the host can run its tools — so the plan stops, and the
|
|
1006
|
+
// model is told which steps ran. No step is marked failed.
|
|
1007
|
+
this.handOver(state, {
|
|
1008
|
+
kind: "step_failed",
|
|
1009
|
+
step: plan.currentStep(),
|
|
1010
|
+
entry: plan.currentEntry(),
|
|
1011
|
+
position: plan.currentStepIndex,
|
|
1012
|
+
error: `the scenario's next step is ${expected ?? "none"}, and the host asked for ${toolName}`,
|
|
1013
|
+
delivery: "above",
|
|
1014
|
+
stopReason: "not_run",
|
|
1015
|
+
});
|
|
1016
|
+
const note = this.takeNote(state);
|
|
1017
|
+
return note ? this.deliverInstead(note, toolName, true) : { kind: "abort" };
|
|
1018
|
+
}
|
|
871
1019
|
return await this.diverge(state, toolName, `expected ${expected ?? "no more tools"}, model called ${toolName}`, deadline, hook.signal);
|
|
872
1020
|
}
|
|
873
1021
|
/**
|
|
@@ -919,7 +1067,9 @@ export class ReplayController {
|
|
|
919
1067
|
if (inFlight.size > 0)
|
|
920
1068
|
state.lingering = inFlight;
|
|
921
1069
|
const note = this.takeNote(state);
|
|
922
|
-
return note
|
|
1070
|
+
return note
|
|
1071
|
+
? this.deliverInstead(note, toolName, state.host !== undefined)
|
|
1072
|
+
: { kind: "passthrough" };
|
|
923
1073
|
}
|
|
924
1074
|
/**
|
|
925
1075
|
* Wait until the plan's outstanding pins thread, or until waiting is no
|
|
@@ -1043,6 +1193,28 @@ export class ReplayController {
|
|
|
1043
1193
|
// reading it after the call would judge the *next* step, and reading it
|
|
1044
1194
|
// before assumes no other pin ever moved it.
|
|
1045
1195
|
const step = plan.allSteps()[pin.stepIndex];
|
|
1196
|
+
// The entry it ran as, so its verdict is kept under the scenario that owns
|
|
1197
|
+
// the step: a called segment numbers its steps from zero too, and without
|
|
1198
|
+
// this its step 0 replaced the caller's (segmented.md R-CALL-13).
|
|
1199
|
+
const entry = plan.allEntries()[pin.stepIndex];
|
|
1200
|
+
// THE HOST STOPPED THE STEP, and says so. Read before any failure: a
|
|
1201
|
+
// person's "no" arrives as an error result, and booked as one it would be
|
|
1202
|
+
// counted against a plan that did nothing wrong — after enough of them the
|
|
1203
|
+
// service stops serving the step. Nothing threads either way: after
|
|
1204
|
+
// `rejected` the tool never ran, and after `edited` its result answers a
|
|
1205
|
+
// call the plan did not plan, which its later steps were not computed for.
|
|
1206
|
+
if (how.hostStop && step) {
|
|
1207
|
+
this.hostStopped(state, pin, step, entry, output, how.hostStop);
|
|
1208
|
+
this.handOver(state, {
|
|
1209
|
+
kind: "step_failed",
|
|
1210
|
+
step,
|
|
1211
|
+
entry,
|
|
1212
|
+
position: pin.stepIndex,
|
|
1213
|
+
delivery: "above",
|
|
1214
|
+
stopReason: how.hostStop === "edited" ? "host_edited" : "host_rejected",
|
|
1215
|
+
});
|
|
1216
|
+
return false;
|
|
1217
|
+
}
|
|
1046
1218
|
// The output threads into the step at the cursor. One pin at a time
|
|
1047
1219
|
// (`preTool`) keeps that the pin's own step; if it ever is not, threading
|
|
1048
1220
|
// would feed one step's output to another's logic, so the plan stops.
|
|
@@ -1057,6 +1229,9 @@ export class ReplayController {
|
|
|
1057
1229
|
`the plan's cursor is at step ${plan.currentStepIndex}, not the pinned step ${pin.stepIndex}`;
|
|
1058
1230
|
const info = {
|
|
1059
1231
|
step,
|
|
1232
|
+
entry,
|
|
1233
|
+
input: pin.input,
|
|
1234
|
+
output: answerOf(output),
|
|
1060
1235
|
outcome: "failed",
|
|
1061
1236
|
stage: "tool_call",
|
|
1062
1237
|
error,
|
|
@@ -1068,7 +1243,7 @@ export class ReplayController {
|
|
|
1068
1243
|
this.handOver(state, {
|
|
1069
1244
|
kind: "step_failed",
|
|
1070
1245
|
step,
|
|
1071
|
-
entry
|
|
1246
|
+
entry,
|
|
1072
1247
|
position: pin.stepIndex,
|
|
1073
1248
|
error,
|
|
1074
1249
|
delivery: "above",
|
|
@@ -1090,6 +1265,10 @@ export class ReplayController {
|
|
|
1090
1265
|
if (step) {
|
|
1091
1266
|
this.recordStep(state, {
|
|
1092
1267
|
step,
|
|
1268
|
+
entry,
|
|
1269
|
+
input: pin.input,
|
|
1270
|
+
// What the logic threw on — the output is usually the explanation.
|
|
1271
|
+
output,
|
|
1093
1272
|
outcome: "failed",
|
|
1094
1273
|
stage: "tool_output_logic",
|
|
1095
1274
|
error: errText(err),
|
|
@@ -1125,6 +1304,9 @@ export class ReplayController {
|
|
|
1125
1304
|
if (step && toolError) {
|
|
1126
1305
|
const info = {
|
|
1127
1306
|
step,
|
|
1307
|
+
entry,
|
|
1308
|
+
input: pin.input,
|
|
1309
|
+
output,
|
|
1128
1310
|
outcome: "failed",
|
|
1129
1311
|
stage: "tool_call",
|
|
1130
1312
|
error: toolError,
|
|
@@ -1146,6 +1328,9 @@ export class ReplayController {
|
|
|
1146
1328
|
if (step) {
|
|
1147
1329
|
this.recordStep(state, {
|
|
1148
1330
|
step,
|
|
1331
|
+
entry,
|
|
1332
|
+
input: pin.input,
|
|
1333
|
+
output,
|
|
1149
1334
|
outcome: "executed",
|
|
1150
1335
|
ms: Date.now() - pin.pinnedAt,
|
|
1151
1336
|
});
|
|
@@ -1201,6 +1386,11 @@ export class ReplayController {
|
|
|
1201
1386
|
const step = plan.allSteps()[pin.stepIndex];
|
|
1202
1387
|
if (!step)
|
|
1203
1388
|
return;
|
|
1389
|
+
if (how.hostStop) {
|
|
1390
|
+
// The host stopped it: neither a failure nor work the plan did.
|
|
1391
|
+
this.hostStopped(state, pin, step, plan.allEntries()[pin.stepIndex], output, how.hostStop);
|
|
1392
|
+
return;
|
|
1393
|
+
}
|
|
1204
1394
|
const ms = Date.now() - pin.pinnedAt;
|
|
1205
1395
|
// Read the same way as a live pin's (`pinFailure`). A call that was sent
|
|
1206
1396
|
// and not seen through says so in its verdict: it may have done its work.
|
|
@@ -1210,13 +1400,23 @@ export class ReplayController {
|
|
|
1210
1400
|
? `${failure.error} — it was sent, and may or may not have completed`
|
|
1211
1401
|
: failure.error
|
|
1212
1402
|
: toolResultError(output);
|
|
1403
|
+
const entry = plan.allEntries()[pin.stepIndex];
|
|
1213
1404
|
if (error !== undefined) {
|
|
1214
|
-
const info = {
|
|
1405
|
+
const info = {
|
|
1406
|
+
step,
|
|
1407
|
+
entry,
|
|
1408
|
+
input: pin.input,
|
|
1409
|
+
output: answerOf(output),
|
|
1410
|
+
outcome: "failed",
|
|
1411
|
+
stage: "tool_call",
|
|
1412
|
+
error,
|
|
1413
|
+
ms,
|
|
1414
|
+
};
|
|
1215
1415
|
this.logStep(info);
|
|
1216
1416
|
this.recordStep(state, info);
|
|
1217
1417
|
return;
|
|
1218
1418
|
}
|
|
1219
|
-
this.recordStep(state, { step, outcome: "executed", ms });
|
|
1419
|
+
this.recordStep(state, { step, entry, input: pin.input, output, outcome: "executed", ms });
|
|
1220
1420
|
state.stepsPinned += 1;
|
|
1221
1421
|
this.upgrade(state, "fell_back");
|
|
1222
1422
|
logDetail("replay.thread_late", {
|
|
@@ -1225,6 +1425,39 @@ export class ReplayController {
|
|
|
1225
1425
|
why: "a step pinned before the hand-over finished — counted, not threaded",
|
|
1226
1426
|
});
|
|
1227
1427
|
}
|
|
1428
|
+
/**
|
|
1429
|
+
* Record a step its host stopped ({@link PostToolOptions.hostStop}) as
|
|
1430
|
+
* `skipped`, in plain words, and remember the stop for the report. Never
|
|
1431
|
+
* `failed`: the service counts failures against the plan. And not counted
|
|
1432
|
+
* in `stepsPinned`: the plan's own call did not run.
|
|
1433
|
+
*/
|
|
1434
|
+
hostStopped(state, pin, step, entry, output, stop) {
|
|
1435
|
+
const edited = stop === "edited";
|
|
1436
|
+
// After `rejected` nothing later can change what the report may claim.
|
|
1437
|
+
if (state.hostStop !== "rejected")
|
|
1438
|
+
state.hostStop = stop;
|
|
1439
|
+
logLine("replay.step_host_stopped", {
|
|
1440
|
+
scenario: state.scenarioId ?? undefined,
|
|
1441
|
+
n: step.stepIndex,
|
|
1442
|
+
tool: step.toolName ?? undefined,
|
|
1443
|
+
stop,
|
|
1444
|
+
why: edited
|
|
1445
|
+
? "a person changed this call before it ran — it ran with their arguments, and the plan stops there"
|
|
1446
|
+
: "the host did not run this step — a person said no or answered in its place, or a limit blocked it",
|
|
1447
|
+
});
|
|
1448
|
+
this.recordStep(state, {
|
|
1449
|
+
step,
|
|
1450
|
+
entry,
|
|
1451
|
+
// The plan's own arguments are what a refused call was asked with; a
|
|
1452
|
+
// changed call ran with other ones, which only the host knows.
|
|
1453
|
+
input: edited ? undefined : pin.input,
|
|
1454
|
+
output: answerOf(output),
|
|
1455
|
+
outcome: "skipped",
|
|
1456
|
+
stage: "tool_call",
|
|
1457
|
+
error: edited ? "a person changed this call" : "the host did not run this step",
|
|
1458
|
+
ms: Date.now() - pin.pinnedAt,
|
|
1459
|
+
});
|
|
1460
|
+
}
|
|
1228
1461
|
/** Whether a pinned call's output is threaded from the proxy's report (§7.2). */
|
|
1229
1462
|
threadsFromProxy(state, toolUseId) {
|
|
1230
1463
|
const pin = state.pinned.get(toolUseId) ?? state.lingering?.get(toolUseId);
|
|
@@ -1494,8 +1727,14 @@ export class ReplayController {
|
|
|
1494
1727
|
stepsPlanned: state.stepsPlanned,
|
|
1495
1728
|
stepsPinned: state.stepsPinned,
|
|
1496
1729
|
// A run its caller cancelled ended the turn early: its cost is no
|
|
1497
|
-
// measurement of the task, whatever the transcript says.
|
|
1498
|
-
|
|
1730
|
+
// measurement of the task, whatever the transcript says. So is one a
|
|
1731
|
+
// person said no in: the model is told not to run that step, the task
|
|
1732
|
+
// ends short of itself, and a verified saving booked against it would
|
|
1733
|
+
// be the cost of the work nobody did.
|
|
1734
|
+
measured: d.measured &&
|
|
1735
|
+
state.callerCancelled !== true &&
|
|
1736
|
+
state.hostStop !== "rejected" &&
|
|
1737
|
+
d.cutShort !== true,
|
|
1499
1738
|
pricingVersion: PRICING_VERSION,
|
|
1500
1739
|
prompt: d.prompt,
|
|
1501
1740
|
steps,
|
|
@@ -1524,7 +1763,14 @@ export class ReplayController {
|
|
|
1524
1763
|
// wrong before — a declined prompt scenario followed by an intent-armed
|
|
1525
1764
|
// plan, whose "baseline" was really the turn cut short at the second arm.
|
|
1526
1765
|
// A lone plan, armed or declined, sends nothing: the default is eligible.
|
|
1527
|
-
|
|
1766
|
+
//
|
|
1767
|
+
// Nor is a turn its host stopped a step of, or one that was cut short.
|
|
1768
|
+
// After a person's "no" the task ended early; after a changed call that
|
|
1769
|
+
// was the plan's first step, the turn made one model call fewer than
|
|
1770
|
+
// the task needs — the one the plan's call replaced.
|
|
1771
|
+
baselineEligible: d.siblings?.some((s) => s !== state && s.plan !== undefined) ||
|
|
1772
|
+
state.hostStop !== undefined ||
|
|
1773
|
+
d.cutShort === true
|
|
1528
1774
|
? false
|
|
1529
1775
|
: undefined,
|
|
1530
1776
|
};
|
|
@@ -1880,7 +2126,7 @@ export class ReplayController {
|
|
|
1880
2126
|
// done its work, and the model must know before it does the task itself.
|
|
1881
2127
|
if (stop?.reason === "uncertain" && stop.flatIndex !== undefined) {
|
|
1882
2128
|
const note = this.stopNote(state, plan, stop, "none");
|
|
1883
|
-
return this.deliverInstead(note, toolName);
|
|
2129
|
+
return this.deliverInstead(note, toolName, state.host !== undefined);
|
|
1884
2130
|
}
|
|
1885
2131
|
return { kind: "abort" };
|
|
1886
2132
|
}
|
|
@@ -1914,7 +2160,7 @@ export class ReplayController {
|
|
|
1914
2160
|
// Bash-clean delivery: the model reaches for `Bash` when it cannot call a
|
|
1915
2161
|
// scripted tool, and a genuine command output is trusted where a `deny`
|
|
1916
2162
|
// reason is read as adversarial interception.
|
|
1917
|
-
const action = this.deliverInstead(text, toolName);
|
|
2163
|
+
const action = this.deliverInstead(text, toolName, state.host !== undefined);
|
|
1918
2164
|
logDetail("replay.inject", { channel: action.kind, bytes: text.length });
|
|
1919
2165
|
return action;
|
|
1920
2166
|
}
|
|
@@ -1998,9 +2244,11 @@ export class ReplayController {
|
|
|
1998
2244
|
? "a second call arrived while the pinned step was still running — the agent continues from here"
|
|
1999
2245
|
: h.stopReason === "budget" || h.stopReason === "cancelled"
|
|
2000
2246
|
? "the run was cut short before this step — the agent continues from here"
|
|
2001
|
-
: h.stopReason === "
|
|
2002
|
-
? "
|
|
2003
|
-
:
|
|
2247
|
+
: h.stopReason === "host_rejected" || h.stopReason === "host_edited"
|
|
2248
|
+
? "the host stopped this step itself — no fault of the plan; the agent continues from here"
|
|
2249
|
+
: h.stopReason === "not_run" || h.stopReason === "uncertain"
|
|
2250
|
+
? "a step produced no result here — the agent continues from here"
|
|
2251
|
+
: "a step broke — the agent continues from here",
|
|
2004
2252
|
});
|
|
2005
2253
|
}
|
|
2006
2254
|
/**
|
|
@@ -2017,6 +2265,9 @@ export class ReplayController {
|
|
|
2017
2265
|
noteFor(state, h) {
|
|
2018
2266
|
const plan = state.plan;
|
|
2019
2267
|
const executed = h.ran ?? (h.ranThisStep ? h.position + 1 : h.position);
|
|
2268
|
+
// A step its host stopped was dealt with there — refused, answered, or
|
|
2269
|
+
// run with a person's arguments — so it is not among what is left to do.
|
|
2270
|
+
const handled = h.ranThisStep === true || h.stopReason === "host_rejected" || h.stopReason === "host_edited";
|
|
2020
2271
|
let params;
|
|
2021
2272
|
try {
|
|
2022
2273
|
params = plan?.parameters;
|
|
@@ -2047,7 +2298,7 @@ export class ReplayController {
|
|
|
2047
2298
|
inFlight: h.inFlight,
|
|
2048
2299
|
ranThisStep: h.ranThisStep,
|
|
2049
2300
|
totalSteps: plan?.totalSteps ?? state.stepsPlanned,
|
|
2050
|
-
remainingTools: plan ? plan.toolsFrom(
|
|
2301
|
+
remainingTools: plan ? plan.toolsFrom(handled ? h.position + 1 : h.position) : [],
|
|
2051
2302
|
params,
|
|
2052
2303
|
delivery: h.delivery,
|
|
2053
2304
|
});
|
|
@@ -2068,6 +2319,12 @@ export class ReplayController {
|
|
|
2068
2319
|
executeStep(state, opts = {}) {
|
|
2069
2320
|
return async (step, input) => {
|
|
2070
2321
|
const name = step.toolName ?? "";
|
|
2322
|
+
// Another host's tools are in its own process, whatever they are
|
|
2323
|
+
// named: one it calls `Bash` or `mcp__fleet__list` is not the runner's
|
|
2324
|
+
// built-in or a proxy's upstream. Nothing of such a plan runs here.
|
|
2325
|
+
if (state.host) {
|
|
2326
|
+
throw new StepRunError(`${name || "this step"} runs in the host's own process`, "not_run");
|
|
2327
|
+
}
|
|
2071
2328
|
const mcp = parseQualifiedName(name);
|
|
2072
2329
|
if (!mcp) {
|
|
2073
2330
|
// An allow-listed built-in runs *here*, against the session's own
|
|
@@ -2219,7 +2476,9 @@ export class ReplayController {
|
|
|
2219
2476
|
}
|
|
2220
2477
|
}
|
|
2221
2478
|
}
|
|
2222
|
-
const sink = opts.adHoc
|
|
2479
|
+
const sink = opts.adHoc
|
|
2480
|
+
? undefined
|
|
2481
|
+
: sinkOfPlan(plan.allEntries().slice(0, plan.stepCount), state.host?.sinks);
|
|
2223
2482
|
if (sink) {
|
|
2224
2483
|
const bad = unsafeParameter(plan.derivedValues, plan.paramsSchema, sink);
|
|
2225
2484
|
if (bad) {
|
|
@@ -2308,7 +2567,14 @@ export class ReplayController {
|
|
|
2308
2567
|
this.recordStep(state, info);
|
|
2309
2568
|
};
|
|
2310
2569
|
}
|
|
2311
|
-
/**
|
|
2570
|
+
/**
|
|
2571
|
+
* Upsert one step's verdict. Later news about a step replaces earlier news.
|
|
2572
|
+
*
|
|
2573
|
+
* The verdict keeps what the step sent and what came back, when the caller
|
|
2574
|
+
* has them. They are this run's, which the service can learn nowhere else:
|
|
2575
|
+
* a matched turn is not recorded, so without them the scenario's page shows
|
|
2576
|
+
* the recording's input and output beside this run's verdict.
|
|
2577
|
+
*/
|
|
2312
2578
|
recordStep(state, info) {
|
|
2313
2579
|
const status = info.outcome === "executed" ? "ok" : info.outcome;
|
|
2314
2580
|
// Keyed by the *flat* position, not by the step's own index: two scenarios
|
|
@@ -2325,6 +2591,16 @@ export class ReplayController {
|
|
|
2325
2591
|
? `${info.error.slice(0, MAX_STEP_ERROR_CHARS)}…`
|
|
2326
2592
|
: info.error
|
|
2327
2593
|
: undefined;
|
|
2594
|
+
// Payloads only for a step of the reported scenario's own chain. The
|
|
2595
|
+
// console still files an execution's verdicts by step number alone
|
|
2596
|
+
// (plan-services.md W1.9), and a called segment numbers its steps from
|
|
2597
|
+
// zero as well: a called step's call would be shown under the caller's
|
|
2598
|
+
// step of the same number — the very mix-up the payloads are there to end.
|
|
2599
|
+
const own = !info.entry || info.entry.depth === 0;
|
|
2600
|
+
// A step whose input logic threw never had an input: the `{}` the plan
|
|
2601
|
+
// reports there is a placeholder, not a call with no arguments.
|
|
2602
|
+
const sent = own && info.stage !== "tool_input_logic" ? info.input : undefined;
|
|
2603
|
+
const received = own ? info.output : undefined;
|
|
2328
2604
|
state.stepResults.set(key, {
|
|
2329
2605
|
stepIndex: info.step.stepIndex,
|
|
2330
2606
|
toolName: info.step.toolName ?? "",
|
|
@@ -2334,6 +2610,8 @@ export class ReplayController {
|
|
|
2334
2610
|
stage: info.stage ?? (status === "skipped" ? "tool_call" : undefined),
|
|
2335
2611
|
error,
|
|
2336
2612
|
durationMs: info.ms,
|
|
2613
|
+
toolInput: sent === undefined ? undefined : reportPayload(serializeCapped(redact(sent)), true),
|
|
2614
|
+
toolOutput: received === undefined ? undefined : reportPayload(received),
|
|
2337
2615
|
// The tool is gone from its server: the service repairs the plan without
|
|
2338
2616
|
// waiting out the grace (plan-services.md D8, kind 2).
|
|
2339
2617
|
toolMissing: info.toolMissing || undefined,
|
|
@@ -2507,6 +2785,37 @@ function pinFailure(pin, output, failed, how) {
|
|
|
2507
2785
|
: "the tool reported an error");
|
|
2508
2786
|
return { error, reason };
|
|
2509
2787
|
}
|
|
2788
|
+
/**
|
|
2789
|
+
* What a call that did not thread answered with, if it answered anything.
|
|
2790
|
+
* `null` is a proxy that lost the call (tool-error.ts, `isNoResult`), and `{}`
|
|
2791
|
+
* is how a hook that carried no `tool_response` reaches here: neither is an
|
|
2792
|
+
* output anybody received.
|
|
2793
|
+
*/
|
|
2794
|
+
function answerOf(output) {
|
|
2795
|
+
return isNoResult(output) || output.trim() === "{}" ? undefined : output;
|
|
2796
|
+
}
|
|
2797
|
+
/**
|
|
2798
|
+
* A step's input or output as its verdict carries it: redacted as a recorded
|
|
2799
|
+
* step's is, and cut — visibly — to what a report has room for.
|
|
2800
|
+
*
|
|
2801
|
+
* Cut as text rather than kept as JSON: these are read on the scenario's page,
|
|
2802
|
+
* never parsed, and the first kilobytes of the payload itself read better than
|
|
2803
|
+
* a preview of it inside a second layer of escapes.
|
|
2804
|
+
*
|
|
2805
|
+
* `clean` says it was redacted a moment ago — the input, serialized by the
|
|
2806
|
+
* caller. An output may not have been: the hook's view of a steered built-in
|
|
2807
|
+
* is threaded as the host gave it. So an output is redacted here, and as JSON,
|
|
2808
|
+
* where a secret is judged by its key as well as by its shape.
|
|
2809
|
+
*/
|
|
2810
|
+
function reportPayload(json, clean = false) {
|
|
2811
|
+
if (clean)
|
|
2812
|
+
return capText(json, REPORT_PAYLOAD_MAX_BYTES);
|
|
2813
|
+
// Larger than any producer here makes: only what a report keeps is scanned.
|
|
2814
|
+
if (json.length > REDACT_SCAN_MAX_CHARS) {
|
|
2815
|
+
return capText(json, REPORT_PAYLOAD_MAX_BYTES, redactText);
|
|
2816
|
+
}
|
|
2817
|
+
return capText(redactJsonText(json), REPORT_PAYLOAD_MAX_BYTES);
|
|
2818
|
+
}
|
|
2510
2819
|
/**
|
|
2511
2820
|
* The index a hand-over reports, in the *caller's* chain (R-CALL-32).
|
|
2512
2821
|
*
|