@basein/runner 0.3.17 → 0.3.19
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/bir-hooks.d.ts +11 -8
- package/dist/bin/bir-hooks.js +11 -8
- package/dist/bin/bir-scenario.d.ts +5 -1
- package/dist/bin/bir-scenario.js +155 -5
- package/dist/bin/bir.js +22 -9
- package/dist/bin/investigate.d.ts +9 -0
- package/dist/bin/investigate.js +26 -12
- package/dist/bin/scenario-edit.d.ts +7 -0
- package/dist/bin/scenario-edit.js +19 -2
- package/dist/bin/scenario-liveness.d.ts +12 -1
- package/dist/bin/scenario-liveness.js +8 -4
- package/dist/bin/setup.js +3 -3
- package/dist/control/client.d.ts +5 -0
- package/dist/control/client.js +7 -0
- package/dist/control/ensure-hook.d.ts +14 -3
- package/dist/control/ensure-hook.js +18 -5
- package/dist/control/recorder-main.d.ts +15 -0
- package/dist/control/recorder-main.js +33 -8
- package/dist/control/server.d.ts +192 -13
- package/dist/control/server.js +819 -41
- package/dist/control/transcript.d.ts +19 -0
- package/dist/control/transcript.js +57 -0
- package/dist/proxy/intercept.js +13 -0
- package/dist/proxy/session.d.ts +23 -0
- package/dist/proxy/session.js +71 -3
- package/dist/record/recorder.d.ts +156 -2
- package/dist/record/recorder.js +9 -0
- package/dist/record/remote-recorder.d.ts +59 -1
- package/dist/record/remote-recorder.js +297 -20
- package/dist/record/switchable-recorder.js +6 -0
- package/dist/replay/bundle.d.ts +8 -0
- package/dist/replay/bundle.js +20 -6
- package/dist/replay/condition-words.d.ts +28 -0
- package/dist/replay/condition-words.js +59 -0
- package/dist/replay/controller.d.ts +78 -7
- package/dist/replay/controller.js +318 -62
- package/dist/replay/derive.d.ts +15 -0
- package/dist/replay/derive.js +5 -0
- package/dist/replay/handover.d.ts +7 -0
- package/dist/replay/handover.js +89 -51
- package/dist/replay/plan.d.ts +101 -5
- package/dist/replay/plan.js +260 -18
- package/dist/replay/source-run.d.ts +2 -2
- package/dist/replay/source-run.js +3 -3
- package/dist/replay/targets.d.ts +40 -1
- package/dist/replay/targets.js +105 -3
- package/dist/replay/types.d.ts +25 -8
- package/docs/calculatedReplay.md +45 -5
- package/docs/calculatedReplayGuide.md +13 -4
- package/package.json +1 -1
- package/production.md +21 -7
|
@@ -21,20 +21,27 @@ import { bashCommandFor } from "./bash-channel.js";
|
|
|
21
21
|
import { BuiltinNotRunnable, BuiltinTimeout, parseBashAllow, parseBuiltinTools, runBuiltin, } from "./builtins.js";
|
|
22
22
|
import { BASH_CHANNEL_MAX, MAX_REPLAY_REASON, testWords, truncate } from "./bundle.js";
|
|
23
23
|
import { coverageOf, modeFor, reachOf } from "./coverage.js";
|
|
24
|
-
import { deriveParameters, } from "./derive.js";
|
|
24
|
+
import { deriveParameters, localDay, } from "./derive.js";
|
|
25
25
|
import { buildHandoverNote, MAX_HANDOVER_NOTE, } from "./handover.js";
|
|
26
26
|
import { NO_PROXY, NoSessionProxyError, ProxyWorkQueue, QUEUE_CLOSED } from "./executor.js";
|
|
27
27
|
import { logicTimeoutMs, MAX_LOGIC_TIMEOUT_MS } from "./logic.js";
|
|
28
|
-
import { PREVIEW_BUDGET_MS, ScenarioReplayPlan, StepRunError, } from "./plan.js";
|
|
28
|
+
import { PREVIEW_BUDGET_MS, RecordedValueError, ScenarioReplayPlan, StepRunError, } from "./plan.js";
|
|
29
29
|
import { PRICING_VERSION } from "./pricing.js";
|
|
30
30
|
import { hasShellStep, sinkOf, sinkOfPlan, unsafeParameter } from "./safety.js";
|
|
31
31
|
import { SourceRunOutputs } from "./source-run.js";
|
|
32
32
|
import { clampToFrameStart, flattenChain, frameEdges, } from "./flatten.js";
|
|
33
|
-
import { blockingTargets } from "./targets.js";
|
|
33
|
+
import { blockingTargets, testedOnlyTargets } from "./targets.js";
|
|
34
34
|
import { isNoResult, toolResultError } from "./tool-error.js";
|
|
35
35
|
import { OUTCOME_RANK, hasTargets, isReadyScenario, } from "./types.js";
|
|
36
36
|
/** The first-party tool a `direct` plan is delivered through (§6.3). */
|
|
37
37
|
export const DIRECT_TOOL_NAME = "mcp__bir__run_scenario";
|
|
38
|
+
/**
|
|
39
|
+
* `bir`'s own tool for saying a replay's results do not answer the request
|
|
40
|
+
* (bin/bir-scenario.ts; the service's review of calcscenarioautocorrect.md,
|
|
41
|
+
* item 19): the one call a finished bundle lets the agent make, where the
|
|
42
|
+
* session has it ({@link ReplayState.reportTool}).
|
|
43
|
+
*/
|
|
44
|
+
export const REPORT_TOOL_NAME = "mcp__bir__report_wrong_result";
|
|
38
45
|
/** How long a `/proxy/poll` is held open before it answers empty. */
|
|
39
46
|
export const POLL_HOLD_MS = 25_000;
|
|
40
47
|
/**
|
|
@@ -113,6 +120,19 @@ const DIRECT_RUN_MARGIN_MS = 5_000;
|
|
|
113
120
|
* Each is bounded by `logicTimeoutMs()` (logic.ts).
|
|
114
121
|
*/
|
|
115
122
|
const LOGIC_BODIES_OUTSIDE_PLAN = 6;
|
|
123
|
+
/**
|
|
124
|
+
* The agent's day as a system prompt said it, and only then (issue 1212): a
|
|
125
|
+
* line that says a day, written out or in words. Claude Code's own date
|
|
126
|
+
* reminder is this machine's day, which the runner knows better itself.
|
|
127
|
+
*/
|
|
128
|
+
function systemDay(day) {
|
|
129
|
+
if (!day || day.source !== "system")
|
|
130
|
+
return undefined;
|
|
131
|
+
if (day.day)
|
|
132
|
+
return day;
|
|
133
|
+
const words = /\b(jan(uary)?|feb(ruary)?|mar(ch)?|apr(il)?|may|june?|july?|aug(ust)?|sep(t(ember)?)?|oct(ober)?|nov(ember)?|dec(ember)?)\b/i;
|
|
134
|
+
return words.test(day.line) && /\b\d{4}\b/.test(day.line) ? day : undefined;
|
|
135
|
+
}
|
|
116
136
|
/** A derivation wait longer than this is not a budget a direct run can keep. */
|
|
117
137
|
const MAX_DIRECT_DERIVE_MS = 60_000;
|
|
118
138
|
/** However a direct run is squeezed, its plan keeps at least this. */
|
|
@@ -268,9 +288,11 @@ export class ReplayController {
|
|
|
268
288
|
/**
|
|
269
289
|
* Whether a handed-out segment may actually arm (segmented.md R-OUT-10).
|
|
270
290
|
*
|
|
271
|
-
*
|
|
272
|
-
* `BIR_SEGMENT_ARM=
|
|
273
|
-
*
|
|
291
|
+
* Only when the option says so. The recorder passes it on unless
|
|
292
|
+
* `BIR_SEGMENT_ARM=0` (from 0.3.19; the service's segmentsbreakdown.md
|
|
293
|
+
* SB-21); a controller built without it stays observe-only, asks for no
|
|
294
|
+
* segment and arms none. The server enforces the same thing from its side
|
|
295
|
+
* (R-OUT-9).
|
|
274
296
|
*/
|
|
275
297
|
get segmentArm() {
|
|
276
298
|
return this.opts.segmentArm === true;
|
|
@@ -369,6 +391,10 @@ export class ReplayController {
|
|
|
369
391
|
*/
|
|
370
392
|
async arm(match, prompt, wrapped, armedBy = "prompt", ctx = {}, host) {
|
|
371
393
|
const state = this.blankState(match, wrapped, armedBy, host);
|
|
394
|
+
// A decision note quotes the request (SB-18); a plan armed by intent was
|
|
395
|
+
// given the step's intent here, and its control server sets the request.
|
|
396
|
+
if (armedBy === "prompt")
|
|
397
|
+
state.request = prompt;
|
|
372
398
|
const isSegment = state.kind === "segment";
|
|
373
399
|
const decline = (why, code) => {
|
|
374
400
|
state.declined = code;
|
|
@@ -460,12 +486,18 @@ export class ReplayController {
|
|
|
460
486
|
const unusable = flat.stopAt === 0 && flat.stopReason ? flat.stopReason : undefined;
|
|
461
487
|
// The step that stops the plan, before the clamp: the reason is its own.
|
|
462
488
|
const stopping = entries[Math.min(...rawStop)]?.step ?? entries[0].step;
|
|
463
|
-
|
|
489
|
+
// The service never serves a decision at step 0 (segmentsbreakdown.md
|
|
490
|
+
// SB-5): should one come, the agent decides from the start, as it makes
|
|
491
|
+
// a judgement there. No new code reaches a service that may not know it.
|
|
492
|
+
const decisionFirst = stopping.stop?.kind === "decision";
|
|
493
|
+
const judgement = stopping.stop?.kind === "nondeterministic" || decisionFirst;
|
|
464
494
|
const why = unusable
|
|
465
495
|
? `its first step is a call that cannot run: ${unusable.reason}`
|
|
466
|
-
:
|
|
467
|
-
? "its first step
|
|
468
|
-
:
|
|
496
|
+
: decisionFirst
|
|
497
|
+
? "its first step is a decision the agent makes — the agent does this task"
|
|
498
|
+
: judgement
|
|
499
|
+
? "its first step needs a judgement the plan cannot compute — the agent does this task"
|
|
500
|
+
: `its first step has failed ${stopping.failureCount ?? 0} times — the agent does this task`;
|
|
469
501
|
return decline(why, unusable
|
|
470
502
|
? "unusable_first_step"
|
|
471
503
|
: judgement
|
|
@@ -506,7 +538,7 @@ export class ReplayController {
|
|
|
506
538
|
// command nobody asked for.
|
|
507
539
|
let deriveCtx = ctx;
|
|
508
540
|
if (ctx.recentResults?.length && sinkOfPlan(planned, host?.sinks) === "command") {
|
|
509
|
-
deriveCtx = { liveCall: ctx.liveCall };
|
|
541
|
+
deriveCtx = { liveCall: ctx.liveCall, agentDay: ctx.agentDay };
|
|
510
542
|
logLine("replay.derive_context", {
|
|
511
543
|
scenario: scenario.id,
|
|
512
544
|
shell: hasShellStep(planned, host?.sinks) || undefined,
|
|
@@ -515,11 +547,27 @@ export class ReplayController {
|
|
|
515
547
|
});
|
|
516
548
|
}
|
|
517
549
|
const derivation = this.startDerivation(scenario, prompt, state, deriveCtx);
|
|
550
|
+
// The agent's own day shows in the session only after the prompt hook
|
|
551
|
+
// (issue 1212): the plan's start looks again, and reads the values once
|
|
552
|
+
// more when the day is another one.
|
|
553
|
+
state.agentDayNow = deriveCtx.agentDay;
|
|
554
|
+
state.rederive = (day) => this.startDerivation(scenario, prompt, state, { ...deriveCtx, agentDay: () => day });
|
|
518
555
|
// Whether the derivation has answered yet — read, never waited on, when the
|
|
519
556
|
// directive is written (below).
|
|
520
557
|
let deriveAnswered = false;
|
|
521
558
|
derivation.then(() => (deriveAnswered = true), () => (deriveAnswered = true));
|
|
522
|
-
|
|
559
|
+
// The targets only one half of this plan reads (the service's review of
|
|
560
|
+
// calcscenarioautocorrect.md, item 11; targets.ts, `testedOnlyTargets`):
|
|
561
|
+
// the plan holds each as a mark when the request does not give it. A
|
|
562
|
+
// reading that did not answer carries every target at its recorded
|
|
563
|
+
// sample (derive.ts), and one of these must be the mark there too — never
|
|
564
|
+
// last week's task name under a step that would create the task.
|
|
565
|
+
const oneHalf = hasTargets(scenario.paramsObject)
|
|
566
|
+
? testedOnlyTargets(targetKeys(scenario.paramsObject), scenario.paramsObject, planned, scenario.paramsLogic)
|
|
567
|
+
: [];
|
|
568
|
+
const params = derivation.then((r) => r.derived || oneHalf.length === 0
|
|
569
|
+
? r.params
|
|
570
|
+
: { ...r.params, ...Object.fromEntries(oneHalf.map((key) => [key, null])) });
|
|
523
571
|
// `params` is awaited by the plan, and by gate 7 below when there is a
|
|
524
572
|
// target. Without this, the losing race in gate 7 leaves `derivation`
|
|
525
573
|
// rejected and unhandled, which takes the process down on an unhandled
|
|
@@ -578,11 +626,22 @@ export class ReplayController {
|
|
|
578
626
|
// Worth a line of its own: this is the difference between a scenario
|
|
579
627
|
// that never runs and one that does, and it is the first thing to look
|
|
580
628
|
// at when a replay acts on the wrong thing.
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
629
|
+
const forOneHalf = result.missing.filter((key) => oneHalf.includes(key));
|
|
630
|
+
const byChain = result.missing.filter((key) => !oneHalf.includes(key));
|
|
631
|
+
if (byChain.length > 0) {
|
|
632
|
+
logLine("replay.targets_found_by_chain", {
|
|
633
|
+
scenario: scenario.id,
|
|
634
|
+
unnamed: byChain.join(","),
|
|
635
|
+
why: "the chain computes these from its own steps; none is read from the caller",
|
|
636
|
+
});
|
|
637
|
+
}
|
|
638
|
+
if (forOneHalf.length > 0) {
|
|
639
|
+
logLine("replay.targets_for_one_half", {
|
|
640
|
+
scenario: scenario.id,
|
|
641
|
+
unnamed: forOneHalf.join(","),
|
|
642
|
+
why: "only steps that run under a test read these: the plan runs, and a step that needs one stops in front of its call for the agent",
|
|
643
|
+
});
|
|
644
|
+
}
|
|
586
645
|
}
|
|
587
646
|
// The values are in hand: one this plan must not template into its
|
|
588
647
|
// command (safety.ts) declines here, before any directive is injected,
|
|
@@ -700,6 +759,15 @@ export class ReplayController {
|
|
|
700
759
|
...(plan.stopsEarly() ? { stopAt: plan.stepCount } : {}),
|
|
701
760
|
};
|
|
702
761
|
}
|
|
762
|
+
/**
|
|
763
|
+
* Whether the plan's steps end in front of a decision (the service's
|
|
764
|
+
* segmentsbreakdown.md SB-6): it runs the steps before it, then hands the
|
|
765
|
+
* task back there. The control server gives such a plan a usage mark, so
|
|
766
|
+
* that it is billed only up to the stop (SB-26).
|
|
767
|
+
*/
|
|
768
|
+
stopsAtDecision(state) {
|
|
769
|
+
return state.plan?.stopEntry()?.step.stop?.kind === "decision";
|
|
770
|
+
}
|
|
703
771
|
/**
|
|
704
772
|
* Take back a plan that armed and has run nothing, as a decline.
|
|
705
773
|
*
|
|
@@ -1058,10 +1126,12 @@ export class ReplayController {
|
|
|
1058
1126
|
// a note, delivered in place of this call.
|
|
1059
1127
|
const step = plan.currentStep();
|
|
1060
1128
|
if (step) {
|
|
1129
|
+
// A value only the recording holds (issue 1212) is no fault of the step.
|
|
1130
|
+
const recorded = err instanceof RecordedValueError;
|
|
1061
1131
|
this.recordStep(state, {
|
|
1062
1132
|
step,
|
|
1063
1133
|
entry: plan.currentEntry(),
|
|
1064
|
-
outcome: "failed",
|
|
1134
|
+
outcome: recorded ? "skipped" : "failed",
|
|
1065
1135
|
stage: "tool_input_logic",
|
|
1066
1136
|
error: errText(err),
|
|
1067
1137
|
});
|
|
@@ -1071,6 +1141,7 @@ export class ReplayController {
|
|
|
1071
1141
|
position: plan.currentStepIndex,
|
|
1072
1142
|
error: errText(err),
|
|
1073
1143
|
delivery: "above",
|
|
1144
|
+
...(recorded ? { stopReason: "not_run" } : {}),
|
|
1074
1145
|
});
|
|
1075
1146
|
const note = this.takeNote(state);
|
|
1076
1147
|
if (note)
|
|
@@ -1772,7 +1843,7 @@ export class ReplayController {
|
|
|
1772
1843
|
const deadline = Date.now() + this.budgets.planMs;
|
|
1773
1844
|
let result;
|
|
1774
1845
|
try {
|
|
1775
|
-
result = await plan.runToCompletion(this.executeStep(state, { deadline }), this.recordedSource(state), this.observeStep(state), deadline, MAX_REPLAY_REASON, { stopOnFailure: true, recordedFallback: this.recordedFallback, signal });
|
|
1846
|
+
result = await plan.runToCompletion(this.executeStep(state, { deadline }), this.recordedSource(state), this.observeStep(state), deadline, MAX_REPLAY_REASON, { stopOnFailure: true, recordedFallback: this.recordedFallback, signal, reportTool: this.reportToolFor(state) });
|
|
1776
1847
|
}
|
|
1777
1848
|
catch (err) {
|
|
1778
1849
|
// The scenario's own logic failed. Retire and let the model do the work.
|
|
@@ -1952,16 +2023,20 @@ export class ReplayController {
|
|
|
1952
2023
|
const total = state.deriveCostUsd + d.sessionCostUsd + state.fallbackCostUsd;
|
|
1953
2024
|
const steps = this.stepResultsOf(state);
|
|
1954
2025
|
const isBaseline = state.outcome === "not_steered" || state.outcome === "failed";
|
|
2026
|
+
// The plan handed the task back by design, not by a fault: in front of the
|
|
2027
|
+
// other half of an "if … otherwise" (AC5), or at its decision (the
|
|
2028
|
+
// service's segmentsbreakdown.md SB-19). Such a stop is named whatever the
|
|
2029
|
+
// outcome, and the agent's work after it is the service's evidence.
|
|
2030
|
+
const handedBack = state.handover?.kind === "other_branch" || state.handover?.kind === "decision";
|
|
1955
2031
|
// A costless decline is noise on both sides — *unless* it says something the
|
|
1956
2032
|
// recording page cannot learn any other way: a step that actually broke, the
|
|
1957
|
-
// gate that declined, or a plan that
|
|
1958
|
-
//
|
|
1959
|
-
//
|
|
1960
|
-
// any of them (its errorshandling.md).
|
|
2033
|
+
// gate that declined, or a plan that handed the task back by design. The
|
|
2034
|
+
// service accepts a costless report that carries any of them (its
|
|
2035
|
+
// errorshandling.md).
|
|
1961
2036
|
if (isBaseline &&
|
|
1962
2037
|
total <= 0 &&
|
|
1963
2038
|
!state.declined &&
|
|
1964
|
-
|
|
2039
|
+
!handedBack &&
|
|
1965
2040
|
!steps?.some((s) => s.status === "failed")) {
|
|
1966
2041
|
return undefined;
|
|
1967
2042
|
}
|
|
@@ -2002,14 +2077,15 @@ export class ReplayController {
|
|
|
2002
2077
|
declined: state.declined,
|
|
2003
2078
|
// Where the plan handed the task to the model (fallbk.md). Only on
|
|
2004
2079
|
// `fell_back`: a hand-over before any step ran is `failed`, and a parked
|
|
2005
|
-
// step found after a divergence is still a divergence. Except
|
|
2006
|
-
//
|
|
2007
|
-
//
|
|
2080
|
+
// step found after a divergence is still a divergence. Except a plan
|
|
2081
|
+
// that handed the task back by design — the other half of an "if …
|
|
2082
|
+
// otherwise" (AC5), or its decision (SB-19) — named whatever the
|
|
2083
|
+
// outcome: the agent's work after it is the service's evidence, whether
|
|
2008
2084
|
// or not a step ran first.
|
|
2009
|
-
fallbackStepIndex: state.outcome === "fell_back" ||
|
|
2085
|
+
fallbackStepIndex: state.outcome === "fell_back" || handedBack
|
|
2010
2086
|
? state.handover?.stepIndex
|
|
2011
2087
|
: undefined,
|
|
2012
|
-
fallbackKind: state.outcome === "fell_back" ||
|
|
2088
|
+
fallbackKind: state.outcome === "fell_back" || handedBack
|
|
2013
2089
|
? state.handover?.kind
|
|
2014
2090
|
: undefined,
|
|
2015
2091
|
// The agent did not take the plan's result and redid the task (AC1).
|
|
@@ -2060,6 +2136,10 @@ export class ReplayController {
|
|
|
2060
2136
|
* there is nothing to be similar *to* and nothing to decline. It books no
|
|
2061
2137
|
* execution either — there is no ticket, and inventing a saving for a manual
|
|
2062
2138
|
* invocation is exactly the kind of number a ledger must never contain.
|
|
2139
|
+
*
|
|
2140
|
+
* One served stop is kept: a plan's decision (the service's
|
|
2141
|
+
* segmentsbreakdown.md SB-6). The run stops in front of it, and `stopped`
|
|
2142
|
+
* says so with the reason `decision`.
|
|
2063
2143
|
*/
|
|
2064
2144
|
async runAdHoc(scenario, prompt, wrapped) {
|
|
2065
2145
|
if (!isReadyScenario(scenario))
|
|
@@ -2124,16 +2204,25 @@ export class ReplayController {
|
|
|
2124
2204
|
}
|
|
2125
2205
|
return out;
|
|
2126
2206
|
});
|
|
2207
|
+
// The one served stop an operator's run keeps: a decision (the service's
|
|
2208
|
+
// segmentsbreakdown.md SB-6). The steps after it are one half of an "if …
|
|
2209
|
+
// otherwise", the half the recording took, and running them by id would
|
|
2210
|
+
// do that half's writes whatever the request. Every other served stop is
|
|
2211
|
+
// the plan's health, which an operator's run is there to try.
|
|
2212
|
+
const decisionAt = flat.entries.findIndex((e) => e.step.stop?.kind === "decision");
|
|
2213
|
+
const stopAt = decisionAt >= 0 && (flat.stopAt === undefined || decisionAt < flat.stopAt)
|
|
2214
|
+
? clampToFrameStart(flat.entries, decisionAt)
|
|
2215
|
+
: flat.stopAt;
|
|
2127
2216
|
const plan = new ScenarioReplayPlan({
|
|
2128
2217
|
scenario,
|
|
2129
2218
|
params,
|
|
2130
2219
|
mode,
|
|
2131
2220
|
entries: flat.entries,
|
|
2132
|
-
stopAt
|
|
2221
|
+
stopAt,
|
|
2133
2222
|
fallbackTargets: "samples",
|
|
2134
2223
|
});
|
|
2135
2224
|
state.plan = plan;
|
|
2136
|
-
state.flatStop = flat.stopReason;
|
|
2225
|
+
state.flatStop = stopAt === flat.stopAt ? flat.stopReason : undefined;
|
|
2137
2226
|
if (this.opts.authUrl && this.opts.authToken) {
|
|
2138
2227
|
state.sourceRun = new SourceRunOutputs({
|
|
2139
2228
|
baseUrl: this.opts.authUrl,
|
|
@@ -2180,6 +2269,15 @@ export class ReplayController {
|
|
|
2180
2269
|
}
|
|
2181
2270
|
const stop = result.stopped;
|
|
2182
2271
|
const stoppedStep = stop ? plan.allSteps()[stop.flatIndex ?? plan.stepCount] : undefined;
|
|
2272
|
+
// The plan's end in front of its decision: said so, by its own word.
|
|
2273
|
+
const atDecision = stop?.kind === "known_bad_step" && stoppedStep?.stop?.kind === "decision";
|
|
2274
|
+
if (atDecision) {
|
|
2275
|
+
logLine("replay.adhoc_decision", {
|
|
2276
|
+
scenario: scenario.id,
|
|
2277
|
+
step: stop.stepIndex,
|
|
2278
|
+
why: "bir replay stops in front of the plan's decision: the steps after it are the half the recording took",
|
|
2279
|
+
});
|
|
2280
|
+
}
|
|
2183
2281
|
return {
|
|
2184
2282
|
ok: true,
|
|
2185
2283
|
text: result.text,
|
|
@@ -2191,7 +2289,7 @@ export class ReplayController {
|
|
|
2191
2289
|
? {
|
|
2192
2290
|
step: stop.stepIndex,
|
|
2193
2291
|
tool: stoppedStep?.toolName ?? "",
|
|
2194
|
-
reason: stop.reason ?? stop.stage ?? stop.kind,
|
|
2292
|
+
reason: atDecision ? "decision" : (stop.reason ?? stop.stage ?? stop.kind),
|
|
2195
2293
|
error: stop.error,
|
|
2196
2294
|
}
|
|
2197
2295
|
: undefined,
|
|
@@ -2233,8 +2331,18 @@ export class ReplayController {
|
|
|
2233
2331
|
const service = this.deriveService(scenario.id);
|
|
2234
2332
|
const hasSchema = Object.keys(scenario.paramsObject ?? {}).length > 0;
|
|
2235
2333
|
state.deriveOnService = !apiKey && service !== undefined && hasSchema;
|
|
2334
|
+
// The day a system prompt told the agent it is — a benchmark's keeps its
|
|
2335
|
+
// agent on 2023-11-30 for ever — and else this machine's (issue 1212).
|
|
2336
|
+
// Never Claude Code's own date reminder: that is this machine's day, and
|
|
2337
|
+
// on a new day's first prompt still yesterday's.
|
|
2338
|
+
const agent = systemDay(ctx.agentDay?.());
|
|
2339
|
+
const today = agent?.day ?? localDay();
|
|
2340
|
+
state.deriveDay = today;
|
|
2341
|
+
state.deriveLine = agent?.line;
|
|
2236
2342
|
return derive({
|
|
2237
2343
|
prompt,
|
|
2344
|
+
today,
|
|
2345
|
+
...(agent?.line ? { todaySaid: agent.line } : {}),
|
|
2238
2346
|
intent: scenario.intent ?? "",
|
|
2239
2347
|
paramsObject: scenario.paramsObject,
|
|
2240
2348
|
apiKey,
|
|
@@ -2252,6 +2360,9 @@ export class ReplayController {
|
|
|
2252
2360
|
})
|
|
2253
2361
|
.then((r) => {
|
|
2254
2362
|
state.deriveCostUsd += r.costUsd;
|
|
2363
|
+
// The day the service read from: a plan on a fixed date reads its own.
|
|
2364
|
+
if (r.dayUsed)
|
|
2365
|
+
state.deriveDay = r.dayUsed;
|
|
2255
2366
|
logLine("replay.derived", {
|
|
2256
2367
|
scenario: scenario.id,
|
|
2257
2368
|
params: Object.keys(r.params).length,
|
|
@@ -2333,6 +2444,7 @@ export class ReplayController {
|
|
|
2333
2444
|
recordedFallback: this.recordedFallback,
|
|
2334
2445
|
deadline,
|
|
2335
2446
|
signal,
|
|
2447
|
+
reportTool: this.reportToolFor(state),
|
|
2336
2448
|
});
|
|
2337
2449
|
}
|
|
2338
2450
|
catch (err) {
|
|
@@ -2372,7 +2484,16 @@ export class ReplayController {
|
|
|
2372
2484
|
// when a step did its work — so the service takes the agent's work after
|
|
2373
2485
|
// it as that half's evidence. Booked `diverged`, it never did.
|
|
2374
2486
|
const otherHalf = stop?.kind === "other_branch";
|
|
2375
|
-
|
|
2487
|
+
// A bundle that ran up to the plan's decision and stopped there (the
|
|
2488
|
+
// service's segmentsbreakdown.md SB-17) is a hand-over too: the plan did
|
|
2489
|
+
// all it was for, and the agent decides. Not one of recorded outputs
|
|
2490
|
+
// only, which is no step done here — that stays a plain divergence.
|
|
2491
|
+
const atDecision = !stop &&
|
|
2492
|
+
plan.stopsEarly() &&
|
|
2493
|
+
composed.endedAt === plan.stepCount &&
|
|
2494
|
+
(composed.executed > 0 || composed.recorded === 0) &&
|
|
2495
|
+
this.stopKindOf(state, plan.stopEntry()) === "decision";
|
|
2496
|
+
this.upgrade(state, broke || !didWork ? "failed" : otherHalf || atDecision ? "fell_back" : "diverged");
|
|
2376
2497
|
state.stepsPinned += composed.executed;
|
|
2377
2498
|
// The headline number of this whole design: recovering from a divergence
|
|
2378
2499
|
// costs nothing, because the proxies were already connected.
|
|
@@ -2409,6 +2530,30 @@ export class ReplayController {
|
|
|
2409
2530
|
const note = this.stopNote(state, plan, stop, "none", composed.left);
|
|
2410
2531
|
return this.deliverInstead(note, toolName, state.host !== undefined);
|
|
2411
2532
|
}
|
|
2533
|
+
// Every step left before the decision was one this request does not
|
|
2534
|
+
// need: the plan has done its part, and the agent decides. Told in
|
|
2535
|
+
// place of its call, like the other half. The steps that ran are the
|
|
2536
|
+
// model's own calls above; the ones left out are named.
|
|
2537
|
+
if (atDecision) {
|
|
2538
|
+
const parked = plan.stopStep();
|
|
2539
|
+
state.handover = {
|
|
2540
|
+
kind: "decision",
|
|
2541
|
+
stepIndex: callerStepIndexOf(plan.stopEntry()) ?? parked.stepIndex,
|
|
2542
|
+
toolName: parked.toolName ?? undefined,
|
|
2543
|
+
note: this.noteFor(state, {
|
|
2544
|
+
kind: "decision",
|
|
2545
|
+
step: parked,
|
|
2546
|
+
entry: plan.stopEntry(),
|
|
2547
|
+
position: plan.stepCount,
|
|
2548
|
+
delivery: "above",
|
|
2549
|
+
executedAbove: plan.currentStepIndex,
|
|
2550
|
+
leftOut: composed.left,
|
|
2551
|
+
}),
|
|
2552
|
+
delivered: true,
|
|
2553
|
+
};
|
|
2554
|
+
this.decidedInDivergence(state, plan);
|
|
2555
|
+
return this.deliverInstead(state.handover.note, toolName, state.host !== undefined);
|
|
2556
|
+
}
|
|
2412
2557
|
// Every remaining step was left out by its test: the steps the session
|
|
2413
2558
|
// ran were the whole of the plan this request needs, and it ran to its
|
|
2414
2559
|
// end (newestRun.md N7 reads that).
|
|
@@ -2447,6 +2592,8 @@ export class ReplayController {
|
|
|
2447
2592
|
delivered: true,
|
|
2448
2593
|
};
|
|
2449
2594
|
text = `${state.handover.note}\n\n${composed.text}`;
|
|
2595
|
+
if (atDecision)
|
|
2596
|
+
this.decidedInDivergence(state, plan);
|
|
2450
2597
|
}
|
|
2451
2598
|
else if (!stop) {
|
|
2452
2599
|
// The bundle carries the rest of the chain, and its header tells the
|
|
@@ -2514,6 +2661,12 @@ export class ReplayController {
|
|
|
2514
2661
|
return;
|
|
2515
2662
|
const ran = state.stepsPinned;
|
|
2516
2663
|
this.retire(state, ran > 0 ? "fell_back" : "failed");
|
|
2664
|
+
// A plan stopped at its decision did all it was for: its bill ends here,
|
|
2665
|
+
// and the agent's decision and the half it takes are the agent's own
|
|
2666
|
+
// (the service's segmentsbreakdown.md SB-26). Only once a step ran: a
|
|
2667
|
+
// plan that did nothing is a baseline sample of the whole turn.
|
|
2668
|
+
if (h.kind === "decision" && ran > 0)
|
|
2669
|
+
this.markRetired(state);
|
|
2517
2670
|
state.handover = {
|
|
2518
2671
|
kind: h.kind,
|
|
2519
2672
|
// The *caller's* index: a fragment opens at the call row, not at a step
|
|
@@ -2540,21 +2693,23 @@ export class ReplayController {
|
|
|
2540
2693
|
? h.noneNeeded
|
|
2541
2694
|
? "no step of the plan was needed for this request — the agent does the task"
|
|
2542
2695
|
: `this request is the other half: the steps from here are only for when ${h.whenText ?? "their test says so"} — the agent continues`
|
|
2543
|
-
: h.kind === "
|
|
2544
|
-
? "
|
|
2545
|
-
: h.kind === "
|
|
2546
|
-
? "
|
|
2547
|
-
: h.kind === "
|
|
2548
|
-
? "the next step
|
|
2549
|
-
: h.
|
|
2550
|
-
? "
|
|
2551
|
-
: h.stopReason === "
|
|
2552
|
-
? "
|
|
2553
|
-
: h.stopReason === "
|
|
2554
|
-
? "the
|
|
2555
|
-
: h.stopReason === "
|
|
2556
|
-
? "
|
|
2557
|
-
:
|
|
2696
|
+
: h.kind === "decision"
|
|
2697
|
+
? "the plan ran the steps before its decision — the agent decides which way applies and continues"
|
|
2698
|
+
: h.kind === "unusable_call"
|
|
2699
|
+
? "a sub-task of this chain cannot run — the agent does that part itself"
|
|
2700
|
+
: h.kind === "known_bad_step"
|
|
2701
|
+
? "the next step has failed too often — the agent continues from here"
|
|
2702
|
+
: h.kind === "nondeterministic_step"
|
|
2703
|
+
? "the next step needs a judgement the plan cannot compute — the agent makes it"
|
|
2704
|
+
: h.stopReason === "concurrent"
|
|
2705
|
+
? "a second call arrived while the pinned step was still running — the agent continues from here"
|
|
2706
|
+
: h.stopReason === "budget" || h.stopReason === "cancelled"
|
|
2707
|
+
? "the run was cut short before this step — the agent continues from here"
|
|
2708
|
+
: h.stopReason === "host_rejected" || h.stopReason === "host_edited"
|
|
2709
|
+
? "the host stopped this step itself — no fault of the plan; the agent continues from here"
|
|
2710
|
+
: h.stopReason === "not_run" || h.stopReason === "uncertain"
|
|
2711
|
+
? "a step produced no result here — the agent continues from here"
|
|
2712
|
+
: "a step broke — the agent continues from here",
|
|
2558
2713
|
});
|
|
2559
2714
|
}
|
|
2560
2715
|
/**
|
|
@@ -2564,10 +2719,32 @@ export class ReplayController {
|
|
|
2564
2719
|
* step.
|
|
2565
2720
|
*/
|
|
2566
2721
|
stopKindOf(state, entry) {
|
|
2722
|
+
// The service serves a decision ahead of any other stop of its step
|
|
2723
|
+
// (the service's segmentsbreakdown.md SB-6), and so it is read first.
|
|
2724
|
+
if (entry?.step.stop?.kind === "decision")
|
|
2725
|
+
return "decision";
|
|
2567
2726
|
if (entry?.step.stop?.kind === "nondeterministic")
|
|
2568
2727
|
return "nondeterministic_step";
|
|
2569
2728
|
return state.flatStop ? "unusable_call" : "known_bad_step";
|
|
2570
2729
|
}
|
|
2730
|
+
/**
|
|
2731
|
+
* A divergence's bundle that ended at the plan's decision (SB-17): logged
|
|
2732
|
+
* as `handOver` logs every other hand-over, and its usage mark taken there
|
|
2733
|
+
* when a step of the plan ran (SB-26).
|
|
2734
|
+
*/
|
|
2735
|
+
decidedInDivergence(state, plan) {
|
|
2736
|
+
if (state.stepsPinned > 0)
|
|
2737
|
+
this.markRetired(state);
|
|
2738
|
+
logLine("replay.handover", {
|
|
2739
|
+
scenario: state.scenarioId ?? undefined,
|
|
2740
|
+
kind: "decision",
|
|
2741
|
+
step: `${plan.stepCount}/${plan.totalSteps}`,
|
|
2742
|
+
via: "divergence",
|
|
2743
|
+
ran: state.stepsPinned,
|
|
2744
|
+
outcome: state.outcome,
|
|
2745
|
+
why: "the plan ran the steps before its decision — the agent decides which way applies and continues",
|
|
2746
|
+
});
|
|
2747
|
+
}
|
|
2571
2748
|
/**
|
|
2572
2749
|
* The steps before flat position `bound` that ran, 1-based, and the ones
|
|
2573
2750
|
* their tests left out (AC5) — the plan cursor's own, and a divergence
|
|
@@ -2608,15 +2785,21 @@ export class ReplayController {
|
|
|
2608
2785
|
h.stopReason === "host_rejected" ||
|
|
2609
2786
|
h.stopReason === "host_edited" ||
|
|
2610
2787
|
h.stopReason === "result_map";
|
|
2788
|
+
// A chain that holds a decision (the service's segmentsbreakdown.md SB-18)
|
|
2789
|
+
// keeps the decision note's protections in every note, even when the plan
|
|
2790
|
+
// stops earlier for another reason: no intent, no parameters, and no step
|
|
2791
|
+
// from the decision on. Each points at the half the recording took, and an
|
|
2792
|
+
// agent that follows it there runs that half's segment: the plan decided.
|
|
2793
|
+
const decisionAt = plan ? plan.decisionIndex() : -1;
|
|
2611
2794
|
let params;
|
|
2612
2795
|
try {
|
|
2613
|
-
params = plan?.parameters;
|
|
2796
|
+
params = decisionAt >= 0 ? undefined : plan?.parameters;
|
|
2614
2797
|
}
|
|
2615
2798
|
catch {
|
|
2616
2799
|
params = undefined;
|
|
2617
2800
|
}
|
|
2618
2801
|
return buildHandoverNote({
|
|
2619
|
-
intent: plan?.intent ?? "",
|
|
2802
|
+
intent: decisionAt >= 0 ? "" : (plan?.intent ?? ""),
|
|
2620
2803
|
kind: h.kind,
|
|
2621
2804
|
stepIndex: h.position,
|
|
2622
2805
|
stepToolName: h.step?.toolName ?? undefined,
|
|
@@ -2651,13 +2834,17 @@ export class ReplayController {
|
|
|
2651
2834
|
totalSteps: plan?.totalSteps ?? state.stepsPlanned,
|
|
2652
2835
|
// A step that runs only under a test is marked so: the list must not
|
|
2653
2836
|
// read as steps this request needs (AC5).
|
|
2654
|
-
remainingTools: plan
|
|
2837
|
+
remainingTools: plan
|
|
2838
|
+
? plan.remainingFrom(handled ? h.position + 1 : h.position, decisionAt >= 0 ? decisionAt : undefined)
|
|
2839
|
+
: [],
|
|
2655
2840
|
params,
|
|
2656
2841
|
delivery: h.delivery,
|
|
2657
2842
|
whenText: h.whenText ?? null,
|
|
2658
2843
|
subTask: h.subTask ?? null,
|
|
2659
2844
|
// "None was needed" is the note's own wording for the end of the plan.
|
|
2660
2845
|
...(h.noneNeeded ? { stepIndex: plan?.totalSteps ?? state.stepsPlanned } : {}),
|
|
2846
|
+
// The request's own words for the choice (SB-18); never the service's.
|
|
2847
|
+
...(h.kind === "decision" ? { request: state.request ?? null } : {}),
|
|
2661
2848
|
});
|
|
2662
2849
|
}
|
|
2663
2850
|
/**
|
|
@@ -2768,6 +2955,21 @@ export class ReplayController {
|
|
|
2768
2955
|
}
|
|
2769
2956
|
return serializeCapped(redact(result));
|
|
2770
2957
|
}
|
|
2958
|
+
/**
|
|
2959
|
+
* The report tool a bundle may name, when this plan's session has it
|
|
2960
|
+
* ({@link ReplayState.reportTool}); undefined leaves the bundle's words as
|
|
2961
|
+
* they always were. Never for another host: its adapter reads no bundle.
|
|
2962
|
+
*/
|
|
2963
|
+
reportToolFor(state) {
|
|
2964
|
+
if (state.host || !state.reportTool)
|
|
2965
|
+
return undefined;
|
|
2966
|
+
try {
|
|
2967
|
+
return state.reportTool() ? REPORT_TOOL_NAME : undefined;
|
|
2968
|
+
}
|
|
2969
|
+
catch {
|
|
2970
|
+
return undefined;
|
|
2971
|
+
}
|
|
2972
|
+
}
|
|
2771
2973
|
/** Where recorded outputs come from, when they may stand in at all. */
|
|
2772
2974
|
recordedSource(state, force = false) {
|
|
2773
2975
|
if (!force && !this.recordedFallback)
|
|
@@ -2801,6 +3003,37 @@ export class ReplayController {
|
|
|
2801
3003
|
// carries the verdict, so the recording page says why it never runs.
|
|
2802
3004
|
blame: true,
|
|
2803
3005
|
});
|
|
3006
|
+
// The agent's own day may show only now (issue 1212): its host writes a
|
|
3007
|
+
// system prompt's "Today's date is …" just after the prompt hook. Values
|
|
3008
|
+
// read for another day are read again, before the first step runs.
|
|
3009
|
+
if (!opts.adHoc && state.rederive && state.agentDayNow && plan.currentStepIndex === 0) {
|
|
3010
|
+
const now = systemDay(state.agentDayNow());
|
|
3011
|
+
// A new line, unless it says the very day the values were read for. A
|
|
3012
|
+
// line with the day in words is read by the service.
|
|
3013
|
+
if (now && now.line !== state.deriveLine && (now.day === undefined || now.day !== state.deriveDay)) {
|
|
3014
|
+
logLine("replay.rederive", {
|
|
3015
|
+
scenario: state.scenarioId ?? undefined,
|
|
3016
|
+
from: state.deriveDay,
|
|
3017
|
+
to: now.day ?? "said in words",
|
|
3018
|
+
});
|
|
3019
|
+
state.deriveLine = now.line;
|
|
3020
|
+
if (now.day)
|
|
3021
|
+
state.deriveDay = now.day;
|
|
3022
|
+
const targets = targetKeys(plan.paramsSchema);
|
|
3023
|
+
// A reading that did not answer names no target, as at the prompt
|
|
3024
|
+
// (gate 7): never the recorded person or date in its place.
|
|
3025
|
+
const again = state.rederive(now).then((r) => {
|
|
3026
|
+
if (r.derived)
|
|
3027
|
+
return r;
|
|
3028
|
+
const params = { ...r.params };
|
|
3029
|
+
for (const key of targets)
|
|
3030
|
+
params[key] = null;
|
|
3031
|
+
return { ...r, params, missing: [...targets] };
|
|
3032
|
+
});
|
|
3033
|
+
again.then((r) => (state.rederivedMissing = r.missing), () => (state.rederivedMissing = [...targets]));
|
|
3034
|
+
plan.resetParams(again.then((r) => r.params));
|
|
3035
|
+
}
|
|
3036
|
+
}
|
|
2804
3037
|
if (!plan.parametersSettled) {
|
|
2805
3038
|
let failure;
|
|
2806
3039
|
const settled = plan.ready().then(() => true, (err) => {
|
|
@@ -2839,6 +3072,21 @@ export class ReplayController {
|
|
|
2839
3072
|
}
|
|
2840
3073
|
}
|
|
2841
3074
|
}
|
|
3075
|
+
// The second reading left a value unnamed that this plan would act on.
|
|
3076
|
+
const missingAgain = state.rederivedMissing;
|
|
3077
|
+
state.rederivedMissing = undefined;
|
|
3078
|
+
if (missingAgain && !opts.adHoc) {
|
|
3079
|
+
const planned = plan.allEntries().slice(0, plan.stepCount);
|
|
3080
|
+
const blocked = blockingTargets(missingAgain, plan.paramsSchema, planned, plan.paramsLogic);
|
|
3081
|
+
if (blocked.length > 0) {
|
|
3082
|
+
return {
|
|
3083
|
+
ok: false,
|
|
3084
|
+
error: `the value of ${blocked[0]} could not be read from this request for ${state.deriveDay}`,
|
|
3085
|
+
stage: "derive_params",
|
|
3086
|
+
blame: false,
|
|
3087
|
+
};
|
|
3088
|
+
}
|
|
3089
|
+
}
|
|
2842
3090
|
const sink = opts.adHoc
|
|
2843
3091
|
? undefined
|
|
2844
3092
|
: sinkOfPlan(plan.allEntries().slice(0, plan.stepCount), state.host?.sinks);
|
|
@@ -3103,17 +3351,10 @@ export class ReplayController {
|
|
|
3103
3351
|
this.upgrade(state, outcome);
|
|
3104
3352
|
// Only a plan that *finished* stops the clock (segmented.md R-MONEY-3). A
|
|
3105
3353
|
// hand-over or a divergence takes no mark: the agent's next tokens are the
|
|
3106
|
-
// consequence of this plan stopping, and belong to its bill.
|
|
3107
|
-
|
|
3108
|
-
|
|
3109
|
-
|
|
3110
|
-
state.retiredMark = state.markUsage?.();
|
|
3111
|
-
}
|
|
3112
|
-
catch {
|
|
3113
|
-
// An unreadable transcript leaves the mark undefined, and the window
|
|
3114
|
-
// falls back to the next arm or the end of the turn.
|
|
3115
|
-
}
|
|
3116
|
-
}
|
|
3354
|
+
// consequence of this plan stopping, and belong to its bill. (A decision
|
|
3355
|
+
// hand-over takes one of its own: `markRetired`.)
|
|
3356
|
+
if (completing)
|
|
3357
|
+
this.markRetired(state);
|
|
3117
3358
|
state.retired = true;
|
|
3118
3359
|
state.pinned.clear();
|
|
3119
3360
|
// A note naming a step to call next is void once the plan is: after a
|
|
@@ -3125,6 +3366,21 @@ export class ReplayController {
|
|
|
3125
3366
|
// A call waiting for a pin to thread has nothing left to wait for.
|
|
3126
3367
|
this.notifyPins(state);
|
|
3127
3368
|
}
|
|
3369
|
+
/**
|
|
3370
|
+
* Where the plan's work ended, for its bill: now, and the transcript's
|
|
3371
|
+
* watermark now. Taken at a completing retire, and at a decision hand-over
|
|
3372
|
+
* after a step ran (the service's segmentsbreakdown.md SB-26).
|
|
3373
|
+
*/
|
|
3374
|
+
markRetired(state) {
|
|
3375
|
+
state.retiredAt = Date.now();
|
|
3376
|
+
try {
|
|
3377
|
+
state.retiredMark = state.markUsage?.();
|
|
3378
|
+
}
|
|
3379
|
+
catch {
|
|
3380
|
+
// An unreadable transcript leaves the mark undefined, and the window
|
|
3381
|
+
// falls back to the next arm or the end of the turn.
|
|
3382
|
+
}
|
|
3383
|
+
}
|
|
3128
3384
|
}
|
|
3129
3385
|
/** Every target of a schema. A parameter with no kind is one (R-PARAM-3). */
|
|
3130
3386
|
function targetKeys(schema) {
|