@basein/runner 0.3.25 → 0.3.27
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +27 -26
- package/dist/auth/client.d.ts +4 -1
- package/dist/auth/client.js +83 -14
- package/dist/bin/bir-hooks.js +4 -2
- package/dist/bin/bir.js +153 -37
- package/dist/bin/investigate.d.ts +13 -0
- package/dist/bin/investigate.js +72 -2
- package/dist/bin/scenario-edit.d.ts +5 -0
- package/dist/bin/scenario-edit.js +17 -1
- package/dist/config/adapters/claude-code.d.ts +51 -7
- package/dist/config/adapters/claude-code.js +105 -18
- package/dist/config/generate.d.ts +61 -13
- package/dist/config/generate.js +108 -20
- package/dist/config/resolve.d.ts +5 -0
- package/dist/config/resolve.js +7 -0
- package/dist/control/correlation.js +0 -0
- package/dist/control/daemon.d.ts +28 -0
- package/dist/control/daemon.js +62 -0
- package/dist/control/paths.d.ts +19 -1
- package/dist/control/paths.js +20 -2
- package/dist/control/pin-guard.js +8 -2
- package/dist/control/recorder-main.js +17 -5
- package/dist/control/server.d.ts +75 -11
- package/dist/control/server.js +440 -77
- package/dist/control/transcript.d.ts +64 -3
- package/dist/control/transcript.js +318 -37
- package/dist/jsonrpc/framing.d.ts +18 -3
- package/dist/jsonrpc/framing.js +57 -4
- package/dist/proxy/intercept.d.ts +4 -0
- package/dist/proxy/intercept.js +4 -0
- package/dist/proxy/relay.d.ts +13 -0
- package/dist/proxy/relay.js +12 -1
- package/dist/proxy/session.d.ts +8 -1
- package/dist/proxy/session.js +67 -10
- package/dist/record/recorder.d.ts +6 -1
- package/dist/record/redact.d.ts +18 -4
- package/dist/record/redact.js +538 -79
- package/dist/record/remote-recorder.d.ts +26 -5
- package/dist/record/remote-recorder.js +174 -18
- package/dist/record/truncate.js +3 -1
- package/dist/replay/bash-channel.d.ts +7 -1
- package/dist/replay/bash-channel.js +9 -3
- package/dist/replay/builtins.d.ts +14 -0
- package/dist/replay/builtins.js +460 -95
- package/dist/replay/bundle.d.ts +15 -11
- package/dist/replay/bundle.js +167 -33
- package/dist/replay/controller.d.ts +92 -1
- package/dist/replay/controller.js +246 -47
- package/dist/replay/derive.js +46 -4
- package/dist/replay/handover.d.ts +13 -0
- package/dist/replay/handover.js +18 -0
- package/dist/replay/plan.d.ts +41 -0
- package/dist/replay/plan.js +171 -3
- package/dist/replay/pricing.d.ts +61 -11
- package/dist/replay/pricing.js +135 -42
- package/dist/replay/reads-only.d.ts +48 -0
- package/dist/replay/reads-only.js +161 -0
- package/dist/replay/safety.js +43 -5
- package/dist/upstream/http-client.d.ts +6 -3
- package/dist/upstream/http-client.js +54 -13
- package/dist/upstream/stdio-client.d.ts +7 -0
- package/dist/upstream/stdio-client.js +1 -1
- package/dist/util/atomic-write.d.ts +9 -1
- package/dist/util/atomic-write.js +9 -6
- package/dist/util/service-health.d.ts +7 -0
- package/dist/util/service-health.js +10 -3
- package/package.json +1 -1
- package/production.md +2 -2
- package/scripts/install.ps1 +14 -2
|
@@ -22,7 +22,7 @@ import { BuiltinNotRunnable, BuiltinTimeout, FILE_BUILTINS, parseBashAllow, pars
|
|
|
22
22
|
import { BASH_CHANNEL_MAX, MAX_REPLAY_REASON, testWords, truncate } from "./bundle.js";
|
|
23
23
|
import { coverageOf, modeFor, reachOf } from "./coverage.js";
|
|
24
24
|
import { deriveParameters, localDay, } from "./derive.js";
|
|
25
|
-
import { buildHandoverNote, MAX_HANDOVER_NOTE, } from "./handover.js";
|
|
25
|
+
import { buildHandoverNote, buildUnseenResultNote, MAX_HANDOVER_NOTE, } from "./handover.js";
|
|
26
26
|
import { NO_PROXY, NoSessionProxyError, ProxyWorkQueue, QUEUE_CLOSED } from "./executor.js";
|
|
27
27
|
import { logicTimeoutMs, MAX_LOGIC_TIMEOUT_MS } from "./logic.js";
|
|
28
28
|
import { PREVIEW_BUDGET_MS, RecordedValueError, ScenarioReplayPlan, StepRunError, } from "./plan.js";
|
|
@@ -171,6 +171,56 @@ export const DEFAULT_INTENT_MATCH = {
|
|
|
171
171
|
maxPerTurn: 3,
|
|
172
172
|
maxRequestsPerTurn: 40,
|
|
173
173
|
};
|
|
174
|
+
/**
|
|
175
|
+
* Where the stopped step stood when a plan handed over, in the words the
|
|
176
|
+
* service reads (its recordings/repo.ts `handoverFamily`): did the step
|
|
177
|
+
* happen. Its tool ran and only its output could not be read: `ran`. It was
|
|
178
|
+
* sent and nothing came back: `uncertain`. Stopped in front of it — the plan's
|
|
179
|
+
* time ran out (`budget`), its caller left (`cancelled`), its test could not
|
|
180
|
+
* say whether this request needs it (`test_failed`) — it was never sent:
|
|
181
|
+
* `not_run`. Otherwise the stop's own reason (`not_run`, `failed`,
|
|
182
|
+
* `concurrent`, `host_rejected`, `host_edited`), and a plain failure, which
|
|
183
|
+
* carries none, reads `failed`.
|
|
184
|
+
*
|
|
185
|
+
* The service continues a hand-over only with work that was recorded where the
|
|
186
|
+
* step had not happened (`not_run`, `failed`): the agent's work there starts
|
|
187
|
+
* with that very step. Continued after a step that ran, or may have, the plan
|
|
188
|
+
* made of it ran the step a second time. The three stops in front of a step
|
|
189
|
+
* went out under their own names, which the service reads as "may have run":
|
|
190
|
+
* a plan stopped by its time budget was never continued.
|
|
191
|
+
*/
|
|
192
|
+
export function stopOf(h) {
|
|
193
|
+
if (h.ranThisStep)
|
|
194
|
+
return "ran";
|
|
195
|
+
if (h.inFlight)
|
|
196
|
+
return "uncertain";
|
|
197
|
+
if (h.stopReason === "budget" || h.stopReason === "cancelled" || h.stopReason === "test_failed")
|
|
198
|
+
return "not_run";
|
|
199
|
+
return h.stopReason ?? "failed";
|
|
200
|
+
}
|
|
201
|
+
/**
|
|
202
|
+
* Whether a run of the plan sent at least one of its calls: a step did its
|
|
203
|
+
* work, its tool answered with an error, its output could not be read after
|
|
204
|
+
* the tool ran, or it was sent and has not been seen through (a pin still out,
|
|
205
|
+
* a stop that may have happened). A step its test left out, one whose logic
|
|
206
|
+
* broke before its call, one that could not be sent and a recorded stand-in
|
|
207
|
+
* sent nothing — a plan that only did those has nothing a second run could
|
|
208
|
+
* repeat.
|
|
209
|
+
*/
|
|
210
|
+
export function sentACall(state) {
|
|
211
|
+
if (state.stepsPinned > 0 || state.pinned.size > 0 || (state.lingering?.size ?? 0) > 0)
|
|
212
|
+
return true;
|
|
213
|
+
const stop = state.handover?.stop;
|
|
214
|
+
if (stop === "ran" || stop === "uncertain" || stop === "host_edited")
|
|
215
|
+
return true;
|
|
216
|
+
for (const r of state.stepResults.values()) {
|
|
217
|
+
if (r.status === "ok")
|
|
218
|
+
return true;
|
|
219
|
+
if (r.status === "failed" && (r.stage === "tool_call" || r.stage === "tool_output_logic"))
|
|
220
|
+
return true;
|
|
221
|
+
}
|
|
222
|
+
return false;
|
|
223
|
+
}
|
|
174
224
|
/**
|
|
175
225
|
* How long a call waits for the previous pin to thread before it is judged to
|
|
176
226
|
* be running beside it: the control server's hook-view fallback
|
|
@@ -527,12 +577,23 @@ export class ReplayController {
|
|
|
527
577
|
// reading that did not answer carries every target at its recorded
|
|
528
578
|
// sample (derive.ts), and one of these must be the mark there too — never
|
|
529
579
|
// last week's task name under a step that would create the task.
|
|
530
|
-
const
|
|
531
|
-
|
|
580
|
+
const allTargets = hasTargets(scenario.paramsObject) ? targetKeys(scenario.paramsObject) : [];
|
|
581
|
+
const oneHalf = allTargets.length > 0
|
|
582
|
+
? testedOnlyTargets(allTargets, scenario.paramsObject, planned, scenario.paramsLogic)
|
|
532
583
|
: [];
|
|
533
|
-
|
|
584
|
+
// A reading that did not answer (no key, the service down, out of budget,
|
|
585
|
+
// too slow) names no target at all. Settings may stand on their recorded
|
|
586
|
+
// samples; a target never does, whichever way the plan reads it. Only the
|
|
587
|
+
// ones a step would act on outright make the gate below decline, and the
|
|
588
|
+
// rest are what targets.ts counts on: a chain that finds its own value
|
|
589
|
+
// (`(respParams.ranked || [])[0] ?? parameters.id`) keeps the caller's
|
|
590
|
+
// copy as a bare fallback, and that fallback is `null` there — as it is
|
|
591
|
+
// when the reading answered and found nothing. Left at its sample, the
|
|
592
|
+
// fallback was last week's device, and the step acted on it whenever the
|
|
593
|
+
// chain found nothing.
|
|
594
|
+
const params = derivation.then((r) => r.derived || allTargets.length === 0
|
|
534
595
|
? r.params
|
|
535
|
-
: { ...r.params, ...Object.fromEntries(
|
|
596
|
+
: { ...r.params, ...Object.fromEntries(allTargets.map((key) => [key, null])) });
|
|
536
597
|
// `params` is awaited by the plan, and by gate 7 below when there is a
|
|
537
598
|
// target. Without this, the losing race in gate 7 leaves `derivation`
|
|
538
599
|
// rejected and unhandled, which takes the process down on an unhandled
|
|
@@ -639,21 +700,24 @@ export class ReplayController {
|
|
|
639
700
|
stopAt,
|
|
640
701
|
subTask: isSubTask(state),
|
|
641
702
|
});
|
|
642
|
-
|
|
643
|
-
|
|
644
|
-
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
|
|
651
|
-
|
|
652
|
-
|
|
653
|
-
|
|
654
|
-
|
|
655
|
-
|
|
656
|
-
|
|
703
|
+
// A steer directive shows the arguments the steps will run with, so the
|
|
704
|
+
// call the model writes is one the pin guard accepts (plan.ts,
|
|
705
|
+
// `steeringDirective`). A direct one says which of this request's values
|
|
706
|
+
// the steps take, and puts this request's values where the recording's
|
|
707
|
+
// stand in its intent (B2 is11.md): "set Olga up" under a request for
|
|
708
|
+
// Carlos made a careful model check first, and that check retired the
|
|
709
|
+
// plan. Both need the parameters settled — which costs nothing when the
|
|
710
|
+
// derivation has already answered: gate 7 waited for it, or there was
|
|
711
|
+
// nothing to derive. One turn of the loop lets an answer that is already
|
|
712
|
+
// in hand through; one still on its way is not waited for here, and the
|
|
713
|
+
// directive leaves the values out (a direct one names each recorded value
|
|
714
|
+
// by its role instead). A `paramsLogic` that throws is not judged here:
|
|
715
|
+
// the first call that needs the parameters meets the same failure, as
|
|
716
|
+
// before.
|
|
717
|
+
if (!deriveAnswered)
|
|
718
|
+
await new Promise((resolve) => setImmediate(resolve));
|
|
719
|
+
if (deriveAnswered)
|
|
720
|
+
await state.plan.ready().catch(() => undefined);
|
|
657
721
|
state.mode = mode;
|
|
658
722
|
state.stepsPlanned = steps.length;
|
|
659
723
|
// The chain this plan was served at: what its own calls are marked with
|
|
@@ -665,7 +729,10 @@ export class ReplayController {
|
|
|
665
729
|
if (this.recordedFallback && this.opts.authUrl && this.opts.authToken) {
|
|
666
730
|
state.sourceRun = new SourceRunOutputs({
|
|
667
731
|
baseUrl: this.opts.authUrl,
|
|
668
|
-
|
|
732
|
+
// The run the current chain was calculated from: for a plan repaired
|
|
733
|
+
// from a fresh recording that is not the run it is stored under, and a
|
|
734
|
+
// stand-in taken from the old one is the output of another chain.
|
|
735
|
+
runId: scenario.sourceRunId ?? scenario.runId,
|
|
669
736
|
token: this.opts.authToken,
|
|
670
737
|
fetchImpl: this.opts.fetchImpl,
|
|
671
738
|
});
|
|
@@ -759,24 +826,63 @@ export class ReplayController {
|
|
|
759
826
|
* against the plan; the outcome is `fell_back` when a step ran and `failed`
|
|
760
827
|
* when none did, as for every hand-over. The note is the caller's to
|
|
761
828
|
* deliver ({@link takeNote}). Returns false when there was no live plan.
|
|
829
|
+
*
|
|
830
|
+
* A step the adapter was given and has not reported back on may be running:
|
|
831
|
+
* the plan stops after it, as for a call beside a pin ({@link concurrentCall}),
|
|
832
|
+
* and the hand-over says the step may have happened — never "did not run",
|
|
833
|
+
* which the service would continue with work that starts by running it
|
|
834
|
+
* again. Its result is still read when it lands.
|
|
762
835
|
*/
|
|
763
836
|
hostHandOver(state, why) {
|
|
764
837
|
const plan = state.plan;
|
|
765
838
|
if (!plan || state.retired)
|
|
766
839
|
return false;
|
|
767
|
-
const
|
|
768
|
-
|
|
769
|
-
|
|
770
|
-
|
|
771
|
-
|
|
772
|
-
|
|
773
|
-
|
|
774
|
-
|
|
775
|
-
|
|
776
|
-
|
|
777
|
-
|
|
778
|
-
|
|
779
|
-
|
|
840
|
+
const reason = why === "own_call"
|
|
841
|
+
? "the model made a call of its own, so the scenario stopped"
|
|
842
|
+
: why === "late"
|
|
843
|
+
? "its arguments were not ready in time"
|
|
844
|
+
: "the run ended before this step";
|
|
845
|
+
const outstanding = [...state.pinned.values()][0];
|
|
846
|
+
if (!outstanding) {
|
|
847
|
+
this.handOver(state, {
|
|
848
|
+
kind: "step_failed",
|
|
849
|
+
step: plan.currentStep(),
|
|
850
|
+
position: plan.currentStepIndex,
|
|
851
|
+
error: reason,
|
|
852
|
+
delivery: "above",
|
|
853
|
+
stopReason: "not_run",
|
|
854
|
+
});
|
|
855
|
+
return true;
|
|
856
|
+
}
|
|
857
|
+
const inFlight = new Map(state.pinned);
|
|
858
|
+
const next = outstanding.stepIndex + 1;
|
|
859
|
+
if (next < plan.stepCount) {
|
|
860
|
+
this.handOver(state, {
|
|
861
|
+
kind: "step_failed",
|
|
862
|
+
step: plan.allSteps()[next],
|
|
863
|
+
position: next,
|
|
864
|
+
error: reason,
|
|
865
|
+
delivery: "above",
|
|
866
|
+
stopReason: "not_run",
|
|
867
|
+
ran: outstanding.stepIndex,
|
|
868
|
+
inFlight: { stepIndex: outstanding.stepIndex, toolName: outstanding.toolName },
|
|
869
|
+
});
|
|
870
|
+
}
|
|
871
|
+
else {
|
|
872
|
+
// The step out is the plan's last: the stop is that step, sent and not answered.
|
|
873
|
+
this.handOver(state, {
|
|
874
|
+
kind: "step_failed",
|
|
875
|
+
step: plan.allSteps()[outstanding.stepIndex],
|
|
876
|
+
position: outstanding.stepIndex,
|
|
877
|
+
error: why === "own_call"
|
|
878
|
+
? "it had not reported back when the model made a call of its own"
|
|
879
|
+
: "it had not reported back when the run ended",
|
|
880
|
+
delivery: "above",
|
|
881
|
+
stopReason: "uncertain",
|
|
882
|
+
ran: outstanding.stepIndex,
|
|
883
|
+
});
|
|
884
|
+
}
|
|
885
|
+
state.lingering = inFlight;
|
|
780
886
|
return true;
|
|
781
887
|
}
|
|
782
888
|
/**
|
|
@@ -858,14 +964,27 @@ export class ReplayController {
|
|
|
858
964
|
*/
|
|
859
965
|
takeNote(state) {
|
|
860
966
|
const h = state?.handover;
|
|
861
|
-
if (
|
|
862
|
-
|
|
863
|
-
|
|
864
|
-
|
|
865
|
-
|
|
866
|
-
|
|
867
|
-
|
|
868
|
-
|
|
967
|
+
if (h && !h.delivered) {
|
|
968
|
+
h.delivered = true;
|
|
969
|
+
logDetail("replay.handover_delivered", {
|
|
970
|
+
scenario: state.scenarioId ?? undefined,
|
|
971
|
+
bytes: h.note.length,
|
|
972
|
+
});
|
|
973
|
+
return h.note;
|
|
974
|
+
}
|
|
975
|
+
// A plan that ran to its end with nobody there to read it has no
|
|
976
|
+
// hand-over, and this one note instead.
|
|
977
|
+
const u = state?.unseen;
|
|
978
|
+
if (u && !u.delivered) {
|
|
979
|
+
u.delivered = true;
|
|
980
|
+
logDetail("replay.unseen_delivered", { scenario: state.scenarioId ?? undefined, bytes: u.note.length });
|
|
981
|
+
return u.note;
|
|
982
|
+
}
|
|
983
|
+
return undefined;
|
|
984
|
+
}
|
|
985
|
+
/** Whether a note about this plan is still to be told to the model: a hand-over's, or the unseen results'. */
|
|
986
|
+
hasUndeliveredNote(state) {
|
|
987
|
+
return Boolean((state?.handover && !state.handover.delivered) || (state?.unseen && !state.unseen.delivered));
|
|
869
988
|
}
|
|
870
989
|
/**
|
|
871
990
|
* Deliver text to the model in place of the tool call it was about to make:
|
|
@@ -939,6 +1058,16 @@ export class ReplayController {
|
|
|
939
1058
|
});
|
|
940
1059
|
return { kind: "passthrough" };
|
|
941
1060
|
}
|
|
1061
|
+
// A CALL THAT ONLY LOOKS DOES NOT END A DIRECT PLAN (B2 is11.md). Asked
|
|
1062
|
+
// for `run_scenario` first, the model looked at something first instead —
|
|
1063
|
+
// in is11 its own notes, after the note had named another request's
|
|
1064
|
+
// person. Read as a divergence, this call would run every step here,
|
|
1065
|
+
// unattended; read as the model leaving the plan, it retired a plan that
|
|
1066
|
+
// would have done the task right. It is neither: it runs as written,
|
|
1067
|
+
// nothing runs in its place, and the plan waits for `run_scenario`. The
|
|
1068
|
+
// control server retires it at the first call that may change something.
|
|
1069
|
+
if (state.mode === "direct" && hook.readsOnly === true)
|
|
1070
|
+
return { kind: "passthrough" };
|
|
942
1071
|
// Everything below — the wait for the parameters, and a divergence that
|
|
943
1072
|
// executes the rest of the plan — happens inside this one hook, which the
|
|
944
1073
|
// host waits on for a bounded time. One deadline covers all of it — and
|
|
@@ -1709,6 +1838,12 @@ export class ReplayController {
|
|
|
1709
1838
|
const plan = state.plan;
|
|
1710
1839
|
if (!plan || state.retired)
|
|
1711
1840
|
return { ok: false, why: RUN_NO_PLAN };
|
|
1841
|
+
// Only a direct plan is run here, and only one with no step pinned for the
|
|
1842
|
+
// model's own call. A steer plan is stepped through by the model's calls,
|
|
1843
|
+
// and a step it pinned and that is still in flight would run a second time
|
|
1844
|
+
// under this run: the model told to call `run_scenario` by hand, beside it.
|
|
1845
|
+
if (state.mode !== "direct" || state.pinned.size > 0)
|
|
1846
|
+
return { ok: false, why: RUN_NO_PLAN };
|
|
1712
1847
|
if (state.running)
|
|
1713
1848
|
return state.running;
|
|
1714
1849
|
const running = this.runArmedOnce(state, plan, signal).catch((err) => {
|
|
@@ -1799,7 +1934,14 @@ export class ReplayController {
|
|
|
1799
1934
|
// out — is no saving at all: the turn it ended cost next to nothing and
|
|
1800
1935
|
// says nothing about what the task costs. `handOver` books `fell_back`
|
|
1801
1936
|
// (steps did run) or `failed`, and the report is not a measurement.
|
|
1802
|
-
|
|
1937
|
+
//
|
|
1938
|
+
// Whether the caller is gone is read here, after the run, and not only
|
|
1939
|
+
// from the stop's own reason: a caller who leaves during a step sees the
|
|
1940
|
+
// run go on to that step's end, and a step that then fails stops the run
|
|
1941
|
+
// for its own reason. The note about it went to a closed request, and the
|
|
1942
|
+
// model was never told which steps had run.
|
|
1943
|
+
const callerGone = stop.reason === "cancelled" || signal?.aborted === true;
|
|
1944
|
+
if (callerGone)
|
|
1803
1945
|
state.callerCancelled = true;
|
|
1804
1946
|
// Nothing ran *here*: there is no bundle, only the task — a recorded
|
|
1805
1947
|
// output is not a result that happened (plan.ts, DriveOptions).
|
|
@@ -1815,7 +1957,7 @@ export class ReplayController {
|
|
|
1815
1957
|
// A cancelled run's note is kept for the model's next call, alone —
|
|
1816
1958
|
// the bundle went to a caller who is gone — so it points at side
|
|
1817
1959
|
// effects in place, never at results "below" that nobody receives.
|
|
1818
|
-
delivery: live &&
|
|
1960
|
+
delivery: live && !callerGone ? "below" : "none",
|
|
1819
1961
|
stopReason: stop.reason,
|
|
1820
1962
|
ran: live ? undefined : 0,
|
|
1821
1963
|
// An output-logic failure stops *after* its step's tool ran: the step
|
|
@@ -1824,7 +1966,7 @@ export class ReplayController {
|
|
|
1824
1966
|
// The other half (AC5): what the steps from here are for.
|
|
1825
1967
|
whenText: stop.whenText,
|
|
1826
1968
|
});
|
|
1827
|
-
if (
|
|
1969
|
+
if (callerGone) {
|
|
1828
1970
|
// The caller is gone and reads nothing. The note stays undelivered: a
|
|
1829
1971
|
// client that timed out hands the task back to the model in this same
|
|
1830
1972
|
// turn, and the model's next call is where it learns which steps
|
|
@@ -1922,6 +2064,29 @@ export class ReplayController {
|
|
|
1922
2064
|
partial: result.partial || undefined,
|
|
1923
2065
|
ms: Date.now() - state.armedAt,
|
|
1924
2066
|
});
|
|
2067
|
+
// The caller left during the last step, which the run does not interrupt:
|
|
2068
|
+
// every step ran, and the answer goes to a closed request. Nothing stopped,
|
|
2069
|
+
// so there is no hand-over note, and the model was never told that the
|
|
2070
|
+
// steps ran — the next request of the same task ran them again, a second
|
|
2071
|
+
// row, a second mail. So the turn is no measurement of the task, and the
|
|
2072
|
+
// model is told once, at its next call or prompt (the server keeps the note
|
|
2073
|
+
// for it: `stashCancelledNote`).
|
|
2074
|
+
if (signal?.aborted) {
|
|
2075
|
+
state.callerCancelled = true;
|
|
2076
|
+
state.unseen = {
|
|
2077
|
+
note: buildUnseenResultNote({
|
|
2078
|
+
intent: plan.intent ?? "",
|
|
2079
|
+
executed: result.executed,
|
|
2080
|
+
totalSteps: plan.totalSteps ?? state.stepsPlanned,
|
|
2081
|
+
}),
|
|
2082
|
+
delivered: false,
|
|
2083
|
+
};
|
|
2084
|
+
logLine("replay.cancelled", {
|
|
2085
|
+
scenario: state.scenarioId ?? undefined,
|
|
2086
|
+
ran: `${result.executed}/${plan.stepCount}`,
|
|
2087
|
+
why: "the caller went away during the last step — every step ran and nobody read the results",
|
|
2088
|
+
});
|
|
2089
|
+
}
|
|
1925
2090
|
return {
|
|
1926
2091
|
ok: true,
|
|
1927
2092
|
text: result.text,
|
|
@@ -2025,8 +2190,15 @@ export class ReplayController {
|
|
|
2025
2190
|
// After a person's "no" the task ended early; after a changed call that
|
|
2026
2191
|
// was the plan's first step, the turn made one model call fewer than
|
|
2027
2192
|
// the task needs — the one the plan's call replaced.
|
|
2193
|
+
//
|
|
2194
|
+
// Nor one its caller cancelled, or one this plan was not armed for
|
|
2195
|
+
// because it had been cancelled a moment before: the steps it did
|
|
2196
|
+
// already ran in the turn before, so the agent's work is the rest of the
|
|
2197
|
+
// task, and its cost is no measure of the whole.
|
|
2028
2198
|
baselineEligible: d.siblings?.some((s) => s !== state && s.plan !== undefined) ||
|
|
2029
2199
|
state.hostStop !== undefined ||
|
|
2200
|
+
state.callerCancelled === true ||
|
|
2201
|
+
state.declined === "after_cancel" ||
|
|
2030
2202
|
d.cutShort === true
|
|
2031
2203
|
? false
|
|
2032
2204
|
: undefined,
|
|
@@ -2106,17 +2278,30 @@ export class ReplayController {
|
|
|
2106
2278
|
}
|
|
2107
2279
|
return out;
|
|
2108
2280
|
});
|
|
2281
|
+
// The plan stops in front of a step the service parked, or whose owner
|
|
2282
|
+
// froze it by hand (plan-services.md D8). An operator who asks for the run
|
|
2283
|
+
// by id turns the gates off, not that: a frozen step is the one that must
|
|
2284
|
+
// not run unattended — an install, a deploy — and the run ended up running
|
|
2285
|
+
// it, and every step after it. It stops there, and says so (`stopped`).
|
|
2286
|
+
const stopAt = stopIndex(scenario.steps, undefined);
|
|
2287
|
+
if (stopAt === 0) {
|
|
2288
|
+
return {
|
|
2289
|
+
ok: false,
|
|
2290
|
+
why: "the plan's first step is parked or frozen by hand, so it stops in front of it: nothing was run",
|
|
2291
|
+
};
|
|
2292
|
+
}
|
|
2109
2293
|
const plan = new ScenarioReplayPlan({
|
|
2110
2294
|
scenario,
|
|
2111
2295
|
params,
|
|
2112
2296
|
mode,
|
|
2113
2297
|
fallbackTargets: "samples",
|
|
2298
|
+
stopAt,
|
|
2114
2299
|
});
|
|
2115
2300
|
state.plan = plan;
|
|
2116
2301
|
if (this.opts.authUrl && this.opts.authToken) {
|
|
2117
2302
|
state.sourceRun = new SourceRunOutputs({
|
|
2118
2303
|
baseUrl: this.opts.authUrl,
|
|
2119
|
-
runId: scenario.runId,
|
|
2304
|
+
runId: scenario.sourceRunId ?? scenario.runId,
|
|
2120
2305
|
token: this.opts.authToken,
|
|
2121
2306
|
fetchImpl: this.opts.fetchImpl,
|
|
2122
2307
|
});
|
|
@@ -2366,7 +2551,10 @@ export class ReplayController {
|
|
|
2366
2551
|
// it as that half's evidence. Booked `diverged`, it never did.
|
|
2367
2552
|
const otherHalf = stop?.kind === "other_branch";
|
|
2368
2553
|
this.upgrade(state, broke || !didWork ? "failed" : otherHalf ? "fell_back" : "diverged");
|
|
2369
|
-
|
|
2554
|
+
// A step whose tool answered with an error did not do its work: it is not
|
|
2555
|
+
// among the steps the plan "ran", whose saving is booked and whose note
|
|
2556
|
+
// tells the model not to run them again (as `runArmedOnce` counts them).
|
|
2557
|
+
state.stepsPinned += Math.max(0, composed.executed - composed.errored);
|
|
2370
2558
|
// The headline number of this whole design: recovering from a divergence
|
|
2371
2559
|
// costs nothing, because the proxies were already connected.
|
|
2372
2560
|
logLine("replay.compose", {
|
|
@@ -2484,6 +2672,9 @@ export class ReplayController {
|
|
|
2484
2672
|
stepIndex: step?.stepIndex ?? position,
|
|
2485
2673
|
toolName: step?.toolName ?? undefined,
|
|
2486
2674
|
error: stop.error,
|
|
2675
|
+
...(stop.kind === "step_failed"
|
|
2676
|
+
? { stop: stopOf({ ranThisStep: stop.stage === "tool_output_logic", stopReason: stop.reason }) }
|
|
2677
|
+
: {}),
|
|
2487
2678
|
note,
|
|
2488
2679
|
delivered: true,
|
|
2489
2680
|
};
|
|
@@ -2507,6 +2698,7 @@ export class ReplayController {
|
|
|
2507
2698
|
stepIndex: h.step?.stepIndex ?? h.position,
|
|
2508
2699
|
toolName: h.step?.toolName ?? undefined,
|
|
2509
2700
|
error: h.error,
|
|
2701
|
+
...(h.kind === "step_failed" ? { stop: stopOf(h) } : {}),
|
|
2510
2702
|
note: this.noteFor(state, h),
|
|
2511
2703
|
delivered: false,
|
|
2512
2704
|
};
|
|
@@ -3152,6 +3344,13 @@ function classifyWorkError(err, serverName) {
|
|
|
3152
3344
|
if (message === QUEUE_CLOSED || /\bupstream is not available\b/i.test(message)) {
|
|
3153
3345
|
return new StepRunError(message, "not_run");
|
|
3154
3346
|
}
|
|
3347
|
+
// A timeout, an interrupt, a dropped connection: sent, and not seen through,
|
|
3348
|
+
// so its work may have happened — before the "MCP error" rule below, which
|
|
3349
|
+
// would call `MCP error -32001: Request timed out` a failure the tool
|
|
3350
|
+
// answered. The pinned path words it the same way ({@link pinFailure}). A
|
|
3351
|
+
// plan told "failed" there let the agent run the step again.
|
|
3352
|
+
if (UNCERTAIN_TEXT.test(message))
|
|
3353
|
+
return new StepRunError(message, "uncertain");
|
|
3155
3354
|
if (/^MCP error -?\d+/i.test(message))
|
|
3156
3355
|
return new StepRunError(message, "failed");
|
|
3157
3356
|
return new StepRunError(message, "uncertain");
|
package/dist/replay/derive.js
CHANGED
|
@@ -33,7 +33,7 @@
|
|
|
33
33
|
import { redactText } from "../record/redact.js";
|
|
34
34
|
import { isUnhealthyStatus, serviceHealth, timeoutSignal } from "../util/service-health.js";
|
|
35
35
|
import { serviceHeaders } from "../util/service-headers.js";
|
|
36
|
-
import { calculateCostUsd } from "./pricing.js";
|
|
36
|
+
import { calculateCostUsd, requestRates } from "./pricing.js";
|
|
37
37
|
export const DEFAULT_DERIVE_MODEL = "claude-haiku-4-5-20251001";
|
|
38
38
|
const ANTHROPIC_URL = "https://api.anthropic.com/v1/messages";
|
|
39
39
|
const ANTHROPIC_VERSION = "2023-06-01";
|
|
@@ -133,6 +133,44 @@ export function extractJsonObject(text) {
|
|
|
133
133
|
}
|
|
134
134
|
return undefined;
|
|
135
135
|
}
|
|
136
|
+
/**
|
|
137
|
+
* `text` with each bare `undefined` in its JSON made `null`, when that JSON
|
|
138
|
+
* does not parse as it came. `undefined` is not JSON, but models emit it
|
|
139
|
+
* (`{"a": undefined}`), and one stray token should not cost the extraction.
|
|
140
|
+
* A value that says it in words — `TypeError: undefined is not a function` —
|
|
141
|
+
* is a value, and was rewritten along with the rest: the plan was handed
|
|
142
|
+
* `TypeError: null is not a function`, silently. So the JSON is read as it
|
|
143
|
+
* came first, and only a token outside every string literal is changed: the
|
|
144
|
+
* walk takes each string from its opening quote to its closing one, so a
|
|
145
|
+
* quote inside a string never ends it early.
|
|
146
|
+
*/
|
|
147
|
+
function nullUndefined(text) {
|
|
148
|
+
const json = extractJsonObject(text);
|
|
149
|
+
if (json === undefined)
|
|
150
|
+
return text;
|
|
151
|
+
try {
|
|
152
|
+
JSON.parse(json);
|
|
153
|
+
return text;
|
|
154
|
+
}
|
|
155
|
+
catch {
|
|
156
|
+
// not JSON as it came: repaired below
|
|
157
|
+
}
|
|
158
|
+
let fixed = "";
|
|
159
|
+
let at = 0;
|
|
160
|
+
while (at < json.length) {
|
|
161
|
+
const open = json.indexOf('"', at);
|
|
162
|
+
fixed += json.slice(at, open < 0 ? json.length : open).replace(/\bundefined\b/g, "null");
|
|
163
|
+
if (open < 0)
|
|
164
|
+
break;
|
|
165
|
+
let end = open + 1;
|
|
166
|
+
while (end < json.length && json[end] !== '"')
|
|
167
|
+
end += json[end] === "\\" ? 2 : 1;
|
|
168
|
+
fixed += json.slice(open, end + 1);
|
|
169
|
+
at = end + 1;
|
|
170
|
+
}
|
|
171
|
+
const start = text.indexOf(json);
|
|
172
|
+
return text.slice(0, start) + fixed + text.slice(start + json.length);
|
|
173
|
+
}
|
|
136
174
|
export function buildExtractionPrompt(prompt, intent, schema, ctx = {}) {
|
|
137
175
|
// Each parameter wears its kind, because the last rule below treats the two
|
|
138
176
|
// differently and the model cannot tell them apart from the name alone
|
|
@@ -232,15 +270,19 @@ export async function deriveParameters(opts) {
|
|
|
232
270
|
.join("");
|
|
233
271
|
const inputTokens = body.usage?.input_tokens ?? 0;
|
|
234
272
|
const outputTokens = body.usage?.output_tokens ?? 0;
|
|
273
|
+
const cacheReadTokens = body.usage?.cache_read_input_tokens ?? 0;
|
|
274
|
+
const cacheCreationTokens = body.usage?.cache_creation_input_tokens ?? 0;
|
|
235
275
|
const costUsd = calculateCostUsd(model, {
|
|
236
276
|
inputTokens,
|
|
237
277
|
outputTokens,
|
|
238
|
-
cacheReadTokens
|
|
239
|
-
cacheCreationTokens
|
|
278
|
+
cacheReadTokens,
|
|
279
|
+
cacheCreationTokens,
|
|
280
|
+
// One request: priced by its own prompt length where the model is (pricing.ts).
|
|
281
|
+
...requestRates(model, { promptTokens: inputTokens + cacheReadTokens + cacheCreationTokens }),
|
|
240
282
|
});
|
|
241
283
|
// `undefined` is not JSON, but models emit it; normalise before parsing so one
|
|
242
284
|
// stray token does not cost the whole extraction.
|
|
243
|
-
const normalized = text
|
|
285
|
+
const normalized = nullUndefined(text);
|
|
244
286
|
const cleaned = extractJsonObject(normalized) ??
|
|
245
287
|
normalized
|
|
246
288
|
.replace(/^```(?:json)?\s*/m, "")
|
|
@@ -130,6 +130,19 @@ export interface HandoverNoteInput {
|
|
|
130
130
|
whenText?: string | null;
|
|
131
131
|
}
|
|
132
132
|
export declare function buildHandoverNote(n: HandoverNoteInput): string;
|
|
133
|
+
/**
|
|
134
|
+
* The note for a plan that ran every one of its steps while its caller was
|
|
135
|
+
* gone (Esc on `run_scenario`, or the client's timeout), so that the results
|
|
136
|
+
* reached nobody. Nothing stopped, so no hand-over note exists; without this
|
|
137
|
+
* one the model was never told the steps ran, and the next request of the same
|
|
138
|
+
* task ran them again — a row added twice, a mail sent twice.
|
|
139
|
+
*/
|
|
140
|
+
export declare function buildUnseenResultNote(n: {
|
|
141
|
+
intent: string;
|
|
142
|
+
/** How many steps ran, of how many the plan has. */
|
|
143
|
+
executed: number;
|
|
144
|
+
totalSteps: number;
|
|
145
|
+
}): string;
|
|
133
146
|
/** "Step 2", "Steps 1–3", "Steps 1, 3 and 5–6": 1-based numbers in order, runs joined. */
|
|
134
147
|
export declare function stepsPhrase(numbers: readonly number[]): string;
|
|
135
148
|
//# sourceMappingURL=handover.d.ts.map
|
package/dist/replay/handover.js
CHANGED
|
@@ -129,6 +129,24 @@ export function buildHandoverNote(n) {
|
|
|
129
129
|
}
|
|
130
130
|
return truncate(lines.join("\n"), MAX_HANDOVER_NOTE);
|
|
131
131
|
}
|
|
132
|
+
/**
|
|
133
|
+
* The note for a plan that ran every one of its steps while its caller was
|
|
134
|
+
* gone (Esc on `run_scenario`, or the client's timeout), so that the results
|
|
135
|
+
* reached nobody. Nothing stopped, so no hand-over note exists; without this
|
|
136
|
+
* one the model was never told the steps ran, and the next request of the same
|
|
137
|
+
* task ran them again — a row added twice, a mail sent twice.
|
|
138
|
+
*/
|
|
139
|
+
export function buildUnseenResultNote(n) {
|
|
140
|
+
const intent = n.intent.length > 240 ? `${n.intent.slice(0, 240)}…` : n.intent;
|
|
141
|
+
const lines = [
|
|
142
|
+
"[BaseInstRunner calculated replay] The calculated scenario for the earlier request ran while its result",
|
|
143
|
+
"was not being waited for, so nobody has seen what it did.",
|
|
144
|
+
];
|
|
145
|
+
if (intent)
|
|
146
|
+
lines.push(`Intent: ${intent}`);
|
|
147
|
+
lines.push(`${n.executed === n.totalSteps ? `All ${n.totalSteps}` : `${n.executed} of ${n.totalSteps}`} steps of the scenario ran, and their side effects are in place. Do not run them again.`, "If you need what they returned, look at the state they changed, or tell the user what ran and ask.");
|
|
148
|
+
return truncate(lines.join("\n"), MAX_HANDOVER_NOTE);
|
|
149
|
+
}
|
|
132
150
|
/** "Step 2", "Steps 1–3", "Steps 1, 3 and 5–6": 1-based numbers in order, runs joined. */
|
|
133
151
|
export function stepsPhrase(numbers) {
|
|
134
152
|
if (numbers.length === 0)
|
package/dist/replay/plan.d.ts
CHANGED
|
@@ -514,6 +514,47 @@ export declare class ScenarioReplayPlan {
|
|
|
514
514
|
* body gets at most `bodyMs` ({@link PREVIEW_LOGIC_MS}) of it.
|
|
515
515
|
*/
|
|
516
516
|
private previewArguments;
|
|
517
|
+
/**
|
|
518
|
+
* This request's value of `key`, as a directive may show it, or undefined
|
|
519
|
+
* (B2 is11.md): once the reading has settled, a plain value the reading
|
|
520
|
+
* took from the request — never a mark, and never a value a step works out
|
|
521
|
+
* (`from: "step"`): before the run nobody has that one, and the recorded one
|
|
522
|
+
* is another request's (issue 1212).
|
|
523
|
+
*/
|
|
524
|
+
private shownValueOf;
|
|
525
|
+
/**
|
|
526
|
+
* A text written from the recording — the intent, a step's reasoning — put
|
|
527
|
+
* right for this request, for a directive to show (B2 is11.md).
|
|
528
|
+
*
|
|
529
|
+
* Both name the recording's values: "set Olga up", "moves each lead to
|
|
530
|
+
* Olga". Shown beside a request for Carlos, the name tells the model the
|
|
531
|
+
* plan is for someone else. A careful model then checks before it runs
|
|
532
|
+
* anything, and a direct plan's first call that was not `run_scenario`
|
|
533
|
+
* retired it — so the replay was lost on exactly the turn replay is for: the
|
|
534
|
+
* same task with a new value. The steps never read the text: they take
|
|
535
|
+
* Carlos from the request.
|
|
536
|
+
*
|
|
537
|
+
* Each recorded value of a parameter found in the text, as a whole word, is
|
|
538
|
+
* replaced by this request's own value where the plan has one it may show
|
|
539
|
+
* ({@link shownValueOf}), and else by the parameter's role (`<assignee
|
|
540
|
+
* name>`): a target the request did not name, a value a step works out, or
|
|
541
|
+
* any value while the reading has not settled. A value the same as the
|
|
542
|
+
* recorded one stays as it is. One pass, longest value first, so nothing put
|
|
543
|
+
* in is replaced again.
|
|
544
|
+
*/
|
|
545
|
+
forThisRequest(text: string): string;
|
|
546
|
+
/** {@link forThisRequest} as one function, built once for a directive's many texts. */
|
|
547
|
+
private rewriter;
|
|
548
|
+
/**
|
|
549
|
+
* This request's own values, for the direct directive's line that says the
|
|
550
|
+
* steps take them (B2 is11.md): what the reading took from the request.
|
|
551
|
+
* The values the request names in words come first — a name, a title: what
|
|
552
|
+
* a reader holds the note against — those unlike the recording's first;
|
|
553
|
+
* then the values worked out from the day ("tomorrow" as a date). Targets
|
|
554
|
+
* only: never a setting, never a value a step works out, never a mark.
|
|
555
|
+
* Undefined until the reading has settled, or when it named none.
|
|
556
|
+
*/
|
|
557
|
+
requestValues(): string | undefined;
|
|
517
558
|
/**
|
|
518
559
|
* The directive injected via `UserPromptSubmit`'s `additionalContext`.
|
|
519
560
|
*
|