@basein/runner 0.3.25 → 0.3.26
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +27 -26
- package/dist/auth/client.d.ts +4 -1
- package/dist/auth/client.js +83 -14
- package/dist/bin/bir-hooks.js +4 -2
- package/dist/bin/bir.js +153 -37
- package/dist/bin/investigate.js +5 -0
- package/dist/bin/scenario-edit.d.ts +5 -0
- package/dist/bin/scenario-edit.js +17 -1
- package/dist/config/adapters/claude-code.d.ts +51 -7
- package/dist/config/adapters/claude-code.js +105 -18
- package/dist/config/generate.d.ts +61 -13
- package/dist/config/generate.js +108 -20
- package/dist/config/resolve.d.ts +5 -0
- package/dist/config/resolve.js +7 -0
- package/dist/control/correlation.js +0 -0
- package/dist/control/daemon.d.ts +28 -0
- package/dist/control/daemon.js +62 -0
- package/dist/control/paths.d.ts +19 -1
- package/dist/control/paths.js +20 -2
- package/dist/control/pin-guard.js +8 -2
- package/dist/control/recorder-main.js +17 -5
- package/dist/control/server.d.ts +63 -9
- package/dist/control/server.js +350 -62
- package/dist/control/transcript.d.ts +64 -3
- package/dist/control/transcript.js +318 -37
- package/dist/jsonrpc/framing.d.ts +18 -3
- package/dist/jsonrpc/framing.js +57 -4
- package/dist/proxy/intercept.d.ts +4 -0
- package/dist/proxy/intercept.js +4 -0
- package/dist/proxy/relay.d.ts +13 -0
- package/dist/proxy/relay.js +12 -1
- package/dist/proxy/session.d.ts +8 -1
- package/dist/proxy/session.js +67 -10
- package/dist/record/recorder.d.ts +6 -1
- package/dist/record/redact.d.ts +18 -4
- package/dist/record/redact.js +538 -79
- package/dist/record/remote-recorder.d.ts +26 -5
- package/dist/record/remote-recorder.js +174 -18
- package/dist/record/truncate.js +3 -1
- package/dist/replay/bash-channel.d.ts +7 -1
- package/dist/replay/bash-channel.js +9 -3
- package/dist/replay/builtins.d.ts +14 -0
- package/dist/replay/builtins.js +460 -95
- package/dist/replay/bundle.d.ts +15 -11
- package/dist/replay/bundle.js +167 -33
- package/dist/replay/controller.d.ts +62 -0
- package/dist/replay/controller.js +218 -32
- package/dist/replay/derive.js +46 -4
- package/dist/replay/handover.d.ts +13 -0
- package/dist/replay/handover.js +18 -0
- package/dist/replay/pricing.d.ts +61 -11
- package/dist/replay/pricing.js +135 -42
- package/dist/replay/safety.js +43 -5
- package/dist/upstream/http-client.d.ts +6 -3
- package/dist/upstream/http-client.js +54 -13
- package/dist/upstream/stdio-client.d.ts +7 -0
- package/dist/upstream/stdio-client.js +1 -1
- package/dist/util/atomic-write.d.ts +9 -1
- package/dist/util/atomic-write.js +9 -6
- package/dist/util/service-health.d.ts +7 -0
- package/dist/util/service-health.js +10 -3
- package/package.json +1 -1
- package/production.md +2 -2
- package/scripts/install.ps1 +14 -2
|
@@ -22,7 +22,7 @@ import { BuiltinNotRunnable, BuiltinTimeout, FILE_BUILTINS, parseBashAllow, pars
|
|
|
22
22
|
import { BASH_CHANNEL_MAX, MAX_REPLAY_REASON, testWords, truncate } from "./bundle.js";
|
|
23
23
|
import { coverageOf, modeFor, reachOf } from "./coverage.js";
|
|
24
24
|
import { deriveParameters, localDay, } from "./derive.js";
|
|
25
|
-
import { buildHandoverNote, MAX_HANDOVER_NOTE, } from "./handover.js";
|
|
25
|
+
import { buildHandoverNote, buildUnseenResultNote, MAX_HANDOVER_NOTE, } from "./handover.js";
|
|
26
26
|
import { NO_PROXY, NoSessionProxyError, ProxyWorkQueue, QUEUE_CLOSED } from "./executor.js";
|
|
27
27
|
import { logicTimeoutMs, MAX_LOGIC_TIMEOUT_MS } from "./logic.js";
|
|
28
28
|
import { PREVIEW_BUDGET_MS, RecordedValueError, ScenarioReplayPlan, StepRunError, } from "./plan.js";
|
|
@@ -171,6 +171,56 @@ export const DEFAULT_INTENT_MATCH = {
|
|
|
171
171
|
maxPerTurn: 3,
|
|
172
172
|
maxRequestsPerTurn: 40,
|
|
173
173
|
};
|
|
174
|
+
/**
|
|
175
|
+
* Where the stopped step stood when a plan handed over, in the words the
|
|
176
|
+
* service reads (its recordings/repo.ts `handoverFamily`): did the step
|
|
177
|
+
* happen. Its tool ran and only its output could not be read: `ran`. It was
|
|
178
|
+
* sent and nothing came back: `uncertain`. Stopped in front of it — the plan's
|
|
179
|
+
* time ran out (`budget`), its caller left (`cancelled`), its test could not
|
|
180
|
+
* say whether this request needs it (`test_failed`) — it was never sent:
|
|
181
|
+
* `not_run`. Otherwise the stop's own reason (`not_run`, `failed`,
|
|
182
|
+
* `concurrent`, `host_rejected`, `host_edited`), and a plain failure, which
|
|
183
|
+
* carries none, reads `failed`.
|
|
184
|
+
*
|
|
185
|
+
* The service continues a hand-over only with work that was recorded where the
|
|
186
|
+
* step had not happened (`not_run`, `failed`): the agent's work there starts
|
|
187
|
+
* with that very step. Continued after a step that ran, or may have, the plan
|
|
188
|
+
* made of it ran the step a second time. The three stops in front of a step
|
|
189
|
+
* went out under their own names, which the service reads as "may have run":
|
|
190
|
+
* a plan stopped by its time budget was never continued.
|
|
191
|
+
*/
|
|
192
|
+
export function stopOf(h) {
|
|
193
|
+
if (h.ranThisStep)
|
|
194
|
+
return "ran";
|
|
195
|
+
if (h.inFlight)
|
|
196
|
+
return "uncertain";
|
|
197
|
+
if (h.stopReason === "budget" || h.stopReason === "cancelled" || h.stopReason === "test_failed")
|
|
198
|
+
return "not_run";
|
|
199
|
+
return h.stopReason ?? "failed";
|
|
200
|
+
}
|
|
201
|
+
/**
|
|
202
|
+
* Whether a run of the plan sent at least one of its calls: a step did its
|
|
203
|
+
* work, its tool answered with an error, its output could not be read after
|
|
204
|
+
* the tool ran, or it was sent and has not been seen through (a pin still out,
|
|
205
|
+
* a stop that may have happened). A step its test left out, one whose logic
|
|
206
|
+
* broke before its call, one that could not be sent and a recorded stand-in
|
|
207
|
+
* sent nothing — a plan that only did those has nothing a second run could
|
|
208
|
+
* repeat.
|
|
209
|
+
*/
|
|
210
|
+
export function sentACall(state) {
|
|
211
|
+
if (state.stepsPinned > 0 || state.pinned.size > 0 || (state.lingering?.size ?? 0) > 0)
|
|
212
|
+
return true;
|
|
213
|
+
const stop = state.handover?.stop;
|
|
214
|
+
if (stop === "ran" || stop === "uncertain" || stop === "host_edited")
|
|
215
|
+
return true;
|
|
216
|
+
for (const r of state.stepResults.values()) {
|
|
217
|
+
if (r.status === "ok")
|
|
218
|
+
return true;
|
|
219
|
+
if (r.status === "failed" && (r.stage === "tool_call" || r.stage === "tool_output_logic"))
|
|
220
|
+
return true;
|
|
221
|
+
}
|
|
222
|
+
return false;
|
|
223
|
+
}
|
|
174
224
|
/**
|
|
175
225
|
* How long a call waits for the previous pin to thread before it is judged to
|
|
176
226
|
* be running beside it: the control server's hook-view fallback
|
|
@@ -527,12 +577,23 @@ export class ReplayController {
|
|
|
527
577
|
// reading that did not answer carries every target at its recorded
|
|
528
578
|
// sample (derive.ts), and one of these must be the mark there too — never
|
|
529
579
|
// last week's task name under a step that would create the task.
|
|
530
|
-
const
|
|
531
|
-
|
|
580
|
+
const allTargets = hasTargets(scenario.paramsObject) ? targetKeys(scenario.paramsObject) : [];
|
|
581
|
+
const oneHalf = allTargets.length > 0
|
|
582
|
+
? testedOnlyTargets(allTargets, scenario.paramsObject, planned, scenario.paramsLogic)
|
|
532
583
|
: [];
|
|
533
|
-
|
|
584
|
+
// A reading that did not answer (no key, the service down, out of budget,
|
|
585
|
+
// too slow) names no target at all. Settings may stand on their recorded
|
|
586
|
+
// samples; a target never does, whichever way the plan reads it. Only the
|
|
587
|
+
// ones a step would act on outright make the gate below decline, and the
|
|
588
|
+
// rest are what targets.ts counts on: a chain that finds its own value
|
|
589
|
+
// (`(respParams.ranked || [])[0] ?? parameters.id`) keeps the caller's
|
|
590
|
+
// copy as a bare fallback, and that fallback is `null` there — as it is
|
|
591
|
+
// when the reading answered and found nothing. Left at its sample, the
|
|
592
|
+
// fallback was last week's device, and the step acted on it whenever the
|
|
593
|
+
// chain found nothing.
|
|
594
|
+
const params = derivation.then((r) => r.derived || allTargets.length === 0
|
|
534
595
|
? r.params
|
|
535
|
-
: { ...r.params, ...Object.fromEntries(
|
|
596
|
+
: { ...r.params, ...Object.fromEntries(allTargets.map((key) => [key, null])) });
|
|
536
597
|
// `params` is awaited by the plan, and by gate 7 below when there is a
|
|
537
598
|
// target. Without this, the losing race in gate 7 leaves `derivation`
|
|
538
599
|
// rejected and unhandled, which takes the process down on an unhandled
|
|
@@ -665,7 +726,10 @@ export class ReplayController {
|
|
|
665
726
|
if (this.recordedFallback && this.opts.authUrl && this.opts.authToken) {
|
|
666
727
|
state.sourceRun = new SourceRunOutputs({
|
|
667
728
|
baseUrl: this.opts.authUrl,
|
|
668
|
-
|
|
729
|
+
// The run the current chain was calculated from: for a plan repaired
|
|
730
|
+
// from a fresh recording that is not the run it is stored under, and a
|
|
731
|
+
// stand-in taken from the old one is the output of another chain.
|
|
732
|
+
runId: scenario.sourceRunId ?? scenario.runId,
|
|
669
733
|
token: this.opts.authToken,
|
|
670
734
|
fetchImpl: this.opts.fetchImpl,
|
|
671
735
|
});
|
|
@@ -759,24 +823,63 @@ export class ReplayController {
|
|
|
759
823
|
* against the plan; the outcome is `fell_back` when a step ran and `failed`
|
|
760
824
|
* when none did, as for every hand-over. The note is the caller's to
|
|
761
825
|
* deliver ({@link takeNote}). Returns false when there was no live plan.
|
|
826
|
+
*
|
|
827
|
+
* A step the adapter was given and has not reported back on may be running:
|
|
828
|
+
* the plan stops after it, as for a call beside a pin ({@link concurrentCall}),
|
|
829
|
+
* and the hand-over says the step may have happened — never "did not run",
|
|
830
|
+
* which the service would continue with work that starts by running it
|
|
831
|
+
* again. Its result is still read when it lands.
|
|
762
832
|
*/
|
|
763
833
|
hostHandOver(state, why) {
|
|
764
834
|
const plan = state.plan;
|
|
765
835
|
if (!plan || state.retired)
|
|
766
836
|
return false;
|
|
767
|
-
const
|
|
768
|
-
|
|
769
|
-
|
|
770
|
-
|
|
771
|
-
|
|
772
|
-
|
|
773
|
-
|
|
774
|
-
|
|
775
|
-
|
|
776
|
-
|
|
777
|
-
|
|
778
|
-
|
|
779
|
-
|
|
837
|
+
const reason = why === "own_call"
|
|
838
|
+
? "the model made a call of its own, so the scenario stopped"
|
|
839
|
+
: why === "late"
|
|
840
|
+
? "its arguments were not ready in time"
|
|
841
|
+
: "the run ended before this step";
|
|
842
|
+
const outstanding = [...state.pinned.values()][0];
|
|
843
|
+
if (!outstanding) {
|
|
844
|
+
this.handOver(state, {
|
|
845
|
+
kind: "step_failed",
|
|
846
|
+
step: plan.currentStep(),
|
|
847
|
+
position: plan.currentStepIndex,
|
|
848
|
+
error: reason,
|
|
849
|
+
delivery: "above",
|
|
850
|
+
stopReason: "not_run",
|
|
851
|
+
});
|
|
852
|
+
return true;
|
|
853
|
+
}
|
|
854
|
+
const inFlight = new Map(state.pinned);
|
|
855
|
+
const next = outstanding.stepIndex + 1;
|
|
856
|
+
if (next < plan.stepCount) {
|
|
857
|
+
this.handOver(state, {
|
|
858
|
+
kind: "step_failed",
|
|
859
|
+
step: plan.allSteps()[next],
|
|
860
|
+
position: next,
|
|
861
|
+
error: reason,
|
|
862
|
+
delivery: "above",
|
|
863
|
+
stopReason: "not_run",
|
|
864
|
+
ran: outstanding.stepIndex,
|
|
865
|
+
inFlight: { stepIndex: outstanding.stepIndex, toolName: outstanding.toolName },
|
|
866
|
+
});
|
|
867
|
+
}
|
|
868
|
+
else {
|
|
869
|
+
// The step out is the plan's last: the stop is that step, sent and not answered.
|
|
870
|
+
this.handOver(state, {
|
|
871
|
+
kind: "step_failed",
|
|
872
|
+
step: plan.allSteps()[outstanding.stepIndex],
|
|
873
|
+
position: outstanding.stepIndex,
|
|
874
|
+
error: why === "own_call"
|
|
875
|
+
? "it had not reported back when the model made a call of its own"
|
|
876
|
+
: "it had not reported back when the run ended",
|
|
877
|
+
delivery: "above",
|
|
878
|
+
stopReason: "uncertain",
|
|
879
|
+
ran: outstanding.stepIndex,
|
|
880
|
+
});
|
|
881
|
+
}
|
|
882
|
+
state.lingering = inFlight;
|
|
780
883
|
return true;
|
|
781
884
|
}
|
|
782
885
|
/**
|
|
@@ -858,14 +961,27 @@ export class ReplayController {
|
|
|
858
961
|
*/
|
|
859
962
|
takeNote(state) {
|
|
860
963
|
const h = state?.handover;
|
|
861
|
-
if (
|
|
862
|
-
|
|
863
|
-
|
|
864
|
-
|
|
865
|
-
|
|
866
|
-
|
|
867
|
-
|
|
868
|
-
|
|
964
|
+
if (h && !h.delivered) {
|
|
965
|
+
h.delivered = true;
|
|
966
|
+
logDetail("replay.handover_delivered", {
|
|
967
|
+
scenario: state.scenarioId ?? undefined,
|
|
968
|
+
bytes: h.note.length,
|
|
969
|
+
});
|
|
970
|
+
return h.note;
|
|
971
|
+
}
|
|
972
|
+
// A plan that ran to its end with nobody there to read it has no
|
|
973
|
+
// hand-over, and this one note instead.
|
|
974
|
+
const u = state?.unseen;
|
|
975
|
+
if (u && !u.delivered) {
|
|
976
|
+
u.delivered = true;
|
|
977
|
+
logDetail("replay.unseen_delivered", { scenario: state.scenarioId ?? undefined, bytes: u.note.length });
|
|
978
|
+
return u.note;
|
|
979
|
+
}
|
|
980
|
+
return undefined;
|
|
981
|
+
}
|
|
982
|
+
/** Whether a note about this plan is still to be told to the model: a hand-over's, or the unseen results'. */
|
|
983
|
+
hasUndeliveredNote(state) {
|
|
984
|
+
return Boolean((state?.handover && !state.handover.delivered) || (state?.unseen && !state.unseen.delivered));
|
|
869
985
|
}
|
|
870
986
|
/**
|
|
871
987
|
* Deliver text to the model in place of the tool call it was about to make:
|
|
@@ -1709,6 +1825,12 @@ export class ReplayController {
|
|
|
1709
1825
|
const plan = state.plan;
|
|
1710
1826
|
if (!plan || state.retired)
|
|
1711
1827
|
return { ok: false, why: RUN_NO_PLAN };
|
|
1828
|
+
// Only a direct plan is run here, and only one with no step pinned for the
|
|
1829
|
+
// model's own call. A steer plan is stepped through by the model's calls,
|
|
1830
|
+
// and a step it pinned and that is still in flight would run a second time
|
|
1831
|
+
// under this run: the model told to call `run_scenario` by hand, beside it.
|
|
1832
|
+
if (state.mode !== "direct" || state.pinned.size > 0)
|
|
1833
|
+
return { ok: false, why: RUN_NO_PLAN };
|
|
1712
1834
|
if (state.running)
|
|
1713
1835
|
return state.running;
|
|
1714
1836
|
const running = this.runArmedOnce(state, plan, signal).catch((err) => {
|
|
@@ -1799,7 +1921,14 @@ export class ReplayController {
|
|
|
1799
1921
|
// out — is no saving at all: the turn it ended cost next to nothing and
|
|
1800
1922
|
// says nothing about what the task costs. `handOver` books `fell_back`
|
|
1801
1923
|
// (steps did run) or `failed`, and the report is not a measurement.
|
|
1802
|
-
|
|
1924
|
+
//
|
|
1925
|
+
// Whether the caller is gone is read here, after the run, and not only
|
|
1926
|
+
// from the stop's own reason: a caller who leaves during a step sees the
|
|
1927
|
+
// run go on to that step's end, and a step that then fails stops the run
|
|
1928
|
+
// for its own reason. The note about it went to a closed request, and the
|
|
1929
|
+
// model was never told which steps had run.
|
|
1930
|
+
const callerGone = stop.reason === "cancelled" || signal?.aborted === true;
|
|
1931
|
+
if (callerGone)
|
|
1803
1932
|
state.callerCancelled = true;
|
|
1804
1933
|
// Nothing ran *here*: there is no bundle, only the task — a recorded
|
|
1805
1934
|
// output is not a result that happened (plan.ts, DriveOptions).
|
|
@@ -1815,7 +1944,7 @@ export class ReplayController {
|
|
|
1815
1944
|
// A cancelled run's note is kept for the model's next call, alone —
|
|
1816
1945
|
// the bundle went to a caller who is gone — so it points at side
|
|
1817
1946
|
// effects in place, never at results "below" that nobody receives.
|
|
1818
|
-
delivery: live &&
|
|
1947
|
+
delivery: live && !callerGone ? "below" : "none",
|
|
1819
1948
|
stopReason: stop.reason,
|
|
1820
1949
|
ran: live ? undefined : 0,
|
|
1821
1950
|
// An output-logic failure stops *after* its step's tool ran: the step
|
|
@@ -1824,7 +1953,7 @@ export class ReplayController {
|
|
|
1824
1953
|
// The other half (AC5): what the steps from here are for.
|
|
1825
1954
|
whenText: stop.whenText,
|
|
1826
1955
|
});
|
|
1827
|
-
if (
|
|
1956
|
+
if (callerGone) {
|
|
1828
1957
|
// The caller is gone and reads nothing. The note stays undelivered: a
|
|
1829
1958
|
// client that timed out hands the task back to the model in this same
|
|
1830
1959
|
// turn, and the model's next call is where it learns which steps
|
|
@@ -1922,6 +2051,29 @@ export class ReplayController {
|
|
|
1922
2051
|
partial: result.partial || undefined,
|
|
1923
2052
|
ms: Date.now() - state.armedAt,
|
|
1924
2053
|
});
|
|
2054
|
+
// The caller left during the last step, which the run does not interrupt:
|
|
2055
|
+
// every step ran, and the answer goes to a closed request. Nothing stopped,
|
|
2056
|
+
// so there is no hand-over note, and the model was never told that the
|
|
2057
|
+
// steps ran — the next request of the same task ran them again, a second
|
|
2058
|
+
// row, a second mail. So the turn is no measurement of the task, and the
|
|
2059
|
+
// model is told once, at its next call or prompt (the server keeps the note
|
|
2060
|
+
// for it: `stashCancelledNote`).
|
|
2061
|
+
if (signal?.aborted) {
|
|
2062
|
+
state.callerCancelled = true;
|
|
2063
|
+
state.unseen = {
|
|
2064
|
+
note: buildUnseenResultNote({
|
|
2065
|
+
intent: plan.intent ?? "",
|
|
2066
|
+
executed: result.executed,
|
|
2067
|
+
totalSteps: plan.totalSteps ?? state.stepsPlanned,
|
|
2068
|
+
}),
|
|
2069
|
+
delivered: false,
|
|
2070
|
+
};
|
|
2071
|
+
logLine("replay.cancelled", {
|
|
2072
|
+
scenario: state.scenarioId ?? undefined,
|
|
2073
|
+
ran: `${result.executed}/${plan.stepCount}`,
|
|
2074
|
+
why: "the caller went away during the last step — every step ran and nobody read the results",
|
|
2075
|
+
});
|
|
2076
|
+
}
|
|
1925
2077
|
return {
|
|
1926
2078
|
ok: true,
|
|
1927
2079
|
text: result.text,
|
|
@@ -2025,8 +2177,15 @@ export class ReplayController {
|
|
|
2025
2177
|
// After a person's "no" the task ended early; after a changed call that
|
|
2026
2178
|
// was the plan's first step, the turn made one model call fewer than
|
|
2027
2179
|
// the task needs — the one the plan's call replaced.
|
|
2180
|
+
//
|
|
2181
|
+
// Nor one its caller cancelled, or one this plan was not armed for
|
|
2182
|
+
// because it had been cancelled a moment before: the steps it did
|
|
2183
|
+
// already ran in the turn before, so the agent's work is the rest of the
|
|
2184
|
+
// task, and its cost is no measure of the whole.
|
|
2028
2185
|
baselineEligible: d.siblings?.some((s) => s !== state && s.plan !== undefined) ||
|
|
2029
2186
|
state.hostStop !== undefined ||
|
|
2187
|
+
state.callerCancelled === true ||
|
|
2188
|
+
state.declined === "after_cancel" ||
|
|
2030
2189
|
d.cutShort === true
|
|
2031
2190
|
? false
|
|
2032
2191
|
: undefined,
|
|
@@ -2106,17 +2265,30 @@ export class ReplayController {
|
|
|
2106
2265
|
}
|
|
2107
2266
|
return out;
|
|
2108
2267
|
});
|
|
2268
|
+
// The plan stops in front of a step the service parked, or whose owner
|
|
2269
|
+
// froze it by hand (plan-services.md D8). An operator who asks for the run
|
|
2270
|
+
// by id turns the gates off, not that: a frozen step is the one that must
|
|
2271
|
+
// not run unattended — an install, a deploy — and the run ended up running
|
|
2272
|
+
// it, and every step after it. It stops there, and says so (`stopped`).
|
|
2273
|
+
const stopAt = stopIndex(scenario.steps, undefined);
|
|
2274
|
+
if (stopAt === 0) {
|
|
2275
|
+
return {
|
|
2276
|
+
ok: false,
|
|
2277
|
+
why: "the plan's first step is parked or frozen by hand, so it stops in front of it: nothing was run",
|
|
2278
|
+
};
|
|
2279
|
+
}
|
|
2109
2280
|
const plan = new ScenarioReplayPlan({
|
|
2110
2281
|
scenario,
|
|
2111
2282
|
params,
|
|
2112
2283
|
mode,
|
|
2113
2284
|
fallbackTargets: "samples",
|
|
2285
|
+
stopAt,
|
|
2114
2286
|
});
|
|
2115
2287
|
state.plan = plan;
|
|
2116
2288
|
if (this.opts.authUrl && this.opts.authToken) {
|
|
2117
2289
|
state.sourceRun = new SourceRunOutputs({
|
|
2118
2290
|
baseUrl: this.opts.authUrl,
|
|
2119
|
-
runId: scenario.runId,
|
|
2291
|
+
runId: scenario.sourceRunId ?? scenario.runId,
|
|
2120
2292
|
token: this.opts.authToken,
|
|
2121
2293
|
fetchImpl: this.opts.fetchImpl,
|
|
2122
2294
|
});
|
|
@@ -2366,7 +2538,10 @@ export class ReplayController {
|
|
|
2366
2538
|
// it as that half's evidence. Booked `diverged`, it never did.
|
|
2367
2539
|
const otherHalf = stop?.kind === "other_branch";
|
|
2368
2540
|
this.upgrade(state, broke || !didWork ? "failed" : otherHalf ? "fell_back" : "diverged");
|
|
2369
|
-
|
|
2541
|
+
// A step whose tool answered with an error did not do its work: it is not
|
|
2542
|
+
// among the steps the plan "ran", whose saving is booked and whose note
|
|
2543
|
+
// tells the model not to run them again (as `runArmedOnce` counts them).
|
|
2544
|
+
state.stepsPinned += Math.max(0, composed.executed - composed.errored);
|
|
2370
2545
|
// The headline number of this whole design: recovering from a divergence
|
|
2371
2546
|
// costs nothing, because the proxies were already connected.
|
|
2372
2547
|
logLine("replay.compose", {
|
|
@@ -2484,6 +2659,9 @@ export class ReplayController {
|
|
|
2484
2659
|
stepIndex: step?.stepIndex ?? position,
|
|
2485
2660
|
toolName: step?.toolName ?? undefined,
|
|
2486
2661
|
error: stop.error,
|
|
2662
|
+
...(stop.kind === "step_failed"
|
|
2663
|
+
? { stop: stopOf({ ranThisStep: stop.stage === "tool_output_logic", stopReason: stop.reason }) }
|
|
2664
|
+
: {}),
|
|
2487
2665
|
note,
|
|
2488
2666
|
delivered: true,
|
|
2489
2667
|
};
|
|
@@ -2507,6 +2685,7 @@ export class ReplayController {
|
|
|
2507
2685
|
stepIndex: h.step?.stepIndex ?? h.position,
|
|
2508
2686
|
toolName: h.step?.toolName ?? undefined,
|
|
2509
2687
|
error: h.error,
|
|
2688
|
+
...(h.kind === "step_failed" ? { stop: stopOf(h) } : {}),
|
|
2510
2689
|
note: this.noteFor(state, h),
|
|
2511
2690
|
delivered: false,
|
|
2512
2691
|
};
|
|
@@ -3152,6 +3331,13 @@ function classifyWorkError(err, serverName) {
|
|
|
3152
3331
|
if (message === QUEUE_CLOSED || /\bupstream is not available\b/i.test(message)) {
|
|
3153
3332
|
return new StepRunError(message, "not_run");
|
|
3154
3333
|
}
|
|
3334
|
+
// A timeout, an interrupt, a dropped connection: sent, and not seen through,
|
|
3335
|
+
// so its work may have happened — before the "MCP error" rule below, which
|
|
3336
|
+
// would call `MCP error -32001: Request timed out` a failure the tool
|
|
3337
|
+
// answered. The pinned path words it the same way ({@link pinFailure}). A
|
|
3338
|
+
// plan told "failed" there let the agent run the step again.
|
|
3339
|
+
if (UNCERTAIN_TEXT.test(message))
|
|
3340
|
+
return new StepRunError(message, "uncertain");
|
|
3155
3341
|
if (/^MCP error -?\d+/i.test(message))
|
|
3156
3342
|
return new StepRunError(message, "failed");
|
|
3157
3343
|
return new StepRunError(message, "uncertain");
|
package/dist/replay/derive.js
CHANGED
|
@@ -33,7 +33,7 @@
|
|
|
33
33
|
import { redactText } from "../record/redact.js";
|
|
34
34
|
import { isUnhealthyStatus, serviceHealth, timeoutSignal } from "../util/service-health.js";
|
|
35
35
|
import { serviceHeaders } from "../util/service-headers.js";
|
|
36
|
-
import { calculateCostUsd } from "./pricing.js";
|
|
36
|
+
import { calculateCostUsd, requestRates } from "./pricing.js";
|
|
37
37
|
export const DEFAULT_DERIVE_MODEL = "claude-haiku-4-5-20251001";
|
|
38
38
|
const ANTHROPIC_URL = "https://api.anthropic.com/v1/messages";
|
|
39
39
|
const ANTHROPIC_VERSION = "2023-06-01";
|
|
@@ -133,6 +133,44 @@ export function extractJsonObject(text) {
|
|
|
133
133
|
}
|
|
134
134
|
return undefined;
|
|
135
135
|
}
|
|
136
|
+
/**
|
|
137
|
+
* `text` with each bare `undefined` in its JSON made `null`, when that JSON
|
|
138
|
+
* does not parse as it came. `undefined` is not JSON, but models emit it
|
|
139
|
+
* (`{"a": undefined}`), and one stray token should not cost the extraction.
|
|
140
|
+
* A value that says it in words — `TypeError: undefined is not a function` —
|
|
141
|
+
* is a value, and was rewritten along with the rest: the plan was handed
|
|
142
|
+
* `TypeError: null is not a function`, silently. So the JSON is read as it
|
|
143
|
+
* came first, and only a token outside every string literal is changed: the
|
|
144
|
+
* walk takes each string from its opening quote to its closing one, so a
|
|
145
|
+
* quote inside a string never ends it early.
|
|
146
|
+
*/
|
|
147
|
+
function nullUndefined(text) {
|
|
148
|
+
const json = extractJsonObject(text);
|
|
149
|
+
if (json === undefined)
|
|
150
|
+
return text;
|
|
151
|
+
try {
|
|
152
|
+
JSON.parse(json);
|
|
153
|
+
return text;
|
|
154
|
+
}
|
|
155
|
+
catch {
|
|
156
|
+
// not JSON as it came: repaired below
|
|
157
|
+
}
|
|
158
|
+
let fixed = "";
|
|
159
|
+
let at = 0;
|
|
160
|
+
while (at < json.length) {
|
|
161
|
+
const open = json.indexOf('"', at);
|
|
162
|
+
fixed += json.slice(at, open < 0 ? json.length : open).replace(/\bundefined\b/g, "null");
|
|
163
|
+
if (open < 0)
|
|
164
|
+
break;
|
|
165
|
+
let end = open + 1;
|
|
166
|
+
while (end < json.length && json[end] !== '"')
|
|
167
|
+
end += json[end] === "\\" ? 2 : 1;
|
|
168
|
+
fixed += json.slice(open, end + 1);
|
|
169
|
+
at = end + 1;
|
|
170
|
+
}
|
|
171
|
+
const start = text.indexOf(json);
|
|
172
|
+
return text.slice(0, start) + fixed + text.slice(start + json.length);
|
|
173
|
+
}
|
|
136
174
|
export function buildExtractionPrompt(prompt, intent, schema, ctx = {}) {
|
|
137
175
|
// Each parameter wears its kind, because the last rule below treats the two
|
|
138
176
|
// differently and the model cannot tell them apart from the name alone
|
|
@@ -232,15 +270,19 @@ export async function deriveParameters(opts) {
|
|
|
232
270
|
.join("");
|
|
233
271
|
const inputTokens = body.usage?.input_tokens ?? 0;
|
|
234
272
|
const outputTokens = body.usage?.output_tokens ?? 0;
|
|
273
|
+
const cacheReadTokens = body.usage?.cache_read_input_tokens ?? 0;
|
|
274
|
+
const cacheCreationTokens = body.usage?.cache_creation_input_tokens ?? 0;
|
|
235
275
|
const costUsd = calculateCostUsd(model, {
|
|
236
276
|
inputTokens,
|
|
237
277
|
outputTokens,
|
|
238
|
-
cacheReadTokens
|
|
239
|
-
cacheCreationTokens
|
|
278
|
+
cacheReadTokens,
|
|
279
|
+
cacheCreationTokens,
|
|
280
|
+
// One request: priced by its own prompt length where the model is (pricing.ts).
|
|
281
|
+
...requestRates(model, { promptTokens: inputTokens + cacheReadTokens + cacheCreationTokens }),
|
|
240
282
|
});
|
|
241
283
|
// `undefined` is not JSON, but models emit it; normalise before parsing so one
|
|
242
284
|
// stray token does not cost the whole extraction.
|
|
243
|
-
const normalized = text
|
|
285
|
+
const normalized = nullUndefined(text);
|
|
244
286
|
const cleaned = extractJsonObject(normalized) ??
|
|
245
287
|
normalized
|
|
246
288
|
.replace(/^```(?:json)?\s*/m, "")
|
|
@@ -130,6 +130,19 @@ export interface HandoverNoteInput {
|
|
|
130
130
|
whenText?: string | null;
|
|
131
131
|
}
|
|
132
132
|
export declare function buildHandoverNote(n: HandoverNoteInput): string;
|
|
133
|
+
/**
|
|
134
|
+
* The note for a plan that ran every one of its steps while its caller was
|
|
135
|
+
* gone (Esc on `run_scenario`, or the client's timeout), so that the results
|
|
136
|
+
* reached nobody. Nothing stopped, so no hand-over note exists; without this
|
|
137
|
+
* one the model was never told the steps ran, and the next request of the same
|
|
138
|
+
* task ran them again — a row added twice, a mail sent twice.
|
|
139
|
+
*/
|
|
140
|
+
export declare function buildUnseenResultNote(n: {
|
|
141
|
+
intent: string;
|
|
142
|
+
/** How many steps ran, of how many the plan has. */
|
|
143
|
+
executed: number;
|
|
144
|
+
totalSteps: number;
|
|
145
|
+
}): string;
|
|
133
146
|
/** "Step 2", "Steps 1–3", "Steps 1, 3 and 5–6": 1-based numbers in order, runs joined. */
|
|
134
147
|
export declare function stepsPhrase(numbers: readonly number[]): string;
|
|
135
148
|
//# sourceMappingURL=handover.d.ts.map
|
package/dist/replay/handover.js
CHANGED
|
@@ -129,6 +129,24 @@ export function buildHandoverNote(n) {
|
|
|
129
129
|
}
|
|
130
130
|
return truncate(lines.join("\n"), MAX_HANDOVER_NOTE);
|
|
131
131
|
}
|
|
132
|
+
/**
|
|
133
|
+
* The note for a plan that ran every one of its steps while its caller was
|
|
134
|
+
* gone (Esc on `run_scenario`, or the client's timeout), so that the results
|
|
135
|
+
* reached nobody. Nothing stopped, so no hand-over note exists; without this
|
|
136
|
+
* one the model was never told the steps ran, and the next request of the same
|
|
137
|
+
* task ran them again — a row added twice, a mail sent twice.
|
|
138
|
+
*/
|
|
139
|
+
export function buildUnseenResultNote(n) {
|
|
140
|
+
const intent = n.intent.length > 240 ? `${n.intent.slice(0, 240)}…` : n.intent;
|
|
141
|
+
const lines = [
|
|
142
|
+
"[BaseInstRunner calculated replay] The calculated scenario for the earlier request ran while its result",
|
|
143
|
+
"was not being waited for, so nobody has seen what it did.",
|
|
144
|
+
];
|
|
145
|
+
if (intent)
|
|
146
|
+
lines.push(`Intent: ${intent}`);
|
|
147
|
+
lines.push(`${n.executed === n.totalSteps ? `All ${n.totalSteps}` : `${n.executed} of ${n.totalSteps}`} steps of the scenario ran, and their side effects are in place. Do not run them again.`, "If you need what they returned, look at the state they changed, or tell the user what ran and ask.");
|
|
148
|
+
return truncate(lines.join("\n"), MAX_HANDOVER_NOTE);
|
|
149
|
+
}
|
|
132
150
|
/** "Step 2", "Steps 1–3", "Steps 1, 3 and 5–6": 1-based numbers in order, runs joined. */
|
|
133
151
|
export function stepsPhrase(numbers) {
|
|
134
152
|
if (numbers.length === 0)
|
package/dist/replay/pricing.d.ts
CHANGED
|
@@ -15,27 +15,77 @@
|
|
|
15
15
|
* numbers are comparable. For that row, right now, they are not.
|
|
16
16
|
*
|
|
17
17
|
* Bump the version — on **every** side — whenever a row changes.
|
|
18
|
+
*
|
|
19
|
+
* WHAT A PRICE DEPENDS ON BESIDE THE MODEL (Anthropic's pricing page, read
|
|
20
|
+
* 2026-11-11). Until this version a turn was priced from four token counts per
|
|
21
|
+
* model, and three things the bill reads were not looked at:
|
|
22
|
+
*
|
|
23
|
+
* - WHICH CACHE WAS WRITTEN. A 5-minute cache write bills at 1.25x input, a
|
|
24
|
+
* 1-hour one at 2x. Claude Code writes the 1-hour cache: in the transcripts
|
|
25
|
+
* this was checked against, every cache write was a 1-hour one, and each was
|
|
26
|
+
* priced 37.5% too low.
|
|
27
|
+
* - THE REQUEST'S OWN PROMPT LENGTH, on Claude Haiku 5.5: a request whose prompt
|
|
28
|
+
* (input, cache reads and cache writes together) is over 100,000 tokens pays
|
|
29
|
+
* five times the price. Each request is priced on its own.
|
|
30
|
+
* - HOW THE REQUEST RAN: fast mode bills at twice the standard prices, and
|
|
31
|
+
* US-only inference at 1.1x.
|
|
32
|
+
*
|
|
33
|
+
* So a usage now says these of itself ({@link TokenUsage}), and whoever folds
|
|
34
|
+
* requests into totals keeps the requests that are priced differently apart
|
|
35
|
+
* ({@link requestRates}).
|
|
18
36
|
*/
|
|
19
37
|
/** Identifies {@link MODEL_PRICING}. Must equal the service's constant. */
|
|
20
|
-
export declare const PRICING_VERSION = "2026-
|
|
21
|
-
/**
|
|
22
|
-
|
|
23
|
-
* unseen version bumps by falling back to the longest matching family, so a new
|
|
24
|
-
* snapshot never silently prices at $0.
|
|
25
|
-
*/
|
|
26
|
-
export declare function resolveModelPricing(model: string): {
|
|
38
|
+
export declare const PRICING_VERSION = "2026-11-11";
|
|
39
|
+
/** One rate card, in $/million tokens. */
|
|
40
|
+
export interface ModelRates {
|
|
27
41
|
input: number;
|
|
28
42
|
output: number;
|
|
29
43
|
cacheRead: number;
|
|
44
|
+
/** A write to the 5-minute cache. */
|
|
30
45
|
cacheWrite: number;
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
46
|
+
/** A write to the 1-hour cache. */
|
|
47
|
+
cacheWrite1h: number;
|
|
48
|
+
}
|
|
49
|
+
/**
|
|
50
|
+
* A model's rate card, or undefined when the table has no row for it. With
|
|
51
|
+
* `longPrompt`, the card of a request over the model's prompt-length
|
|
52
|
+
* threshold, for a model that has one.
|
|
53
|
+
*/
|
|
54
|
+
export declare function resolveModelPricing(name: string, opts?: {
|
|
55
|
+
longPrompt?: boolean;
|
|
56
|
+
}): ModelRates | undefined;
|
|
57
|
+
export declare function hasModelPricing(name: string): boolean;
|
|
58
|
+
/** What of a request, beside its model and its token counts, sets its prices. */
|
|
59
|
+
export interface RateFlags {
|
|
60
|
+
/** Its prompt was over the model's long-prompt threshold (Haiku 5.5: 100,000 tokens). */
|
|
61
|
+
longPrompt?: boolean;
|
|
62
|
+
/** It ran in fast mode. */
|
|
63
|
+
fast?: boolean;
|
|
64
|
+
/** It ran on US-only inference. */
|
|
65
|
+
usOnly?: boolean;
|
|
66
|
+
}
|
|
67
|
+
export interface TokenUsage extends RateFlags {
|
|
34
68
|
inputTokens: number;
|
|
35
69
|
outputTokens: number;
|
|
36
70
|
cacheReadTokens?: number;
|
|
71
|
+
/** Every cache write, the 1-hour ones among them. */
|
|
37
72
|
cacheCreationTokens?: number;
|
|
73
|
+
/** Of {@link cacheCreationTokens}, the tokens written to the 1-hour cache. */
|
|
74
|
+
cacheCreation1hTokens?: number;
|
|
38
75
|
}
|
|
76
|
+
/**
|
|
77
|
+
* The flags of ONE request, from its own `usage` as the API reports it: its
|
|
78
|
+
* prompt length (input, cache reads and cache writes together), its `speed`
|
|
79
|
+
* and its `inference_geo`. Only what is so is set, so two requests priced
|
|
80
|
+
* alike give equal flags — and {@link rateKey} the same key.
|
|
81
|
+
*/
|
|
82
|
+
export declare function requestRates(name: string, request: {
|
|
83
|
+
promptTokens: number;
|
|
84
|
+
speed?: unknown;
|
|
85
|
+
inferenceGeo?: unknown;
|
|
86
|
+
}): RateFlags;
|
|
87
|
+
/** A short text that is equal for two sets of flags exactly when they price alike. "" for none. */
|
|
88
|
+
export declare function rateKey(flags: RateFlags): string;
|
|
39
89
|
/** Cost in USD. An unpriced model yields 0 — the caller reports `measured: false`. */
|
|
40
|
-
export declare function calculateCostUsd(
|
|
90
|
+
export declare function calculateCostUsd(name: string, usage: TokenUsage): number;
|
|
41
91
|
//# sourceMappingURL=pricing.d.ts.map
|