@basein/runner 0.3.20 → 0.3.22
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +6 -5
- package/dist/bin/bir-hooks.d.ts +17 -19
- package/dist/bin/bir-hooks.js +17 -19
- package/dist/bin/bir-scenario.js +1 -1
- package/dist/bin/bir.js +1 -21
- package/dist/bin/investigate.d.ts +0 -9
- package/dist/bin/investigate.js +15 -43
- package/dist/bin/scenario-edit.js +13 -44
- package/dist/control/recorder-main.d.ts +0 -15
- package/dist/control/recorder-main.js +22 -42
- package/dist/control/server.d.ts +24 -137
- package/dist/control/server.js +107 -1001
- package/dist/record/recorder.d.ts +11 -142
- package/dist/record/recorder.js +0 -3
- package/dist/record/remote-recorder.d.ts +1 -9
- package/dist/record/remote-recorder.js +3 -49
- package/dist/record/switchable-recorder.js +0 -2
- package/dist/replay/builtins.d.ts +45 -9
- package/dist/replay/builtins.js +131 -59
- package/dist/replay/bundle.d.ts +2 -2
- package/dist/replay/bundle.js +3 -3
- package/dist/replay/controller.d.ts +34 -125
- package/dist/replay/controller.js +171 -528
- package/dist/replay/derive.d.ts +0 -10
- package/dist/replay/derive.js +0 -3
- package/dist/replay/handover.d.ts +2 -37
- package/dist/replay/handover.js +38 -106
- package/dist/replay/logic.d.ts +0 -11
- package/dist/replay/logic.js +0 -24
- package/dist/replay/plan.d.ts +43 -163
- package/dist/replay/plan.js +144 -556
- package/dist/replay/safety.d.ts +4 -5
- package/dist/replay/safety.js +7 -7
- package/dist/replay/source-run.d.ts +15 -24
- package/dist/replay/source-run.js +25 -58
- package/dist/replay/targets.d.ts +11 -21
- package/dist/replay/targets.js +30 -69
- package/dist/replay/types.d.ts +9 -96
- package/dist/replay/types.js +3 -16
- package/dist/util/service-headers.d.ts +1 -2
- package/dist/util/service-headers.js +1 -2
- package/docs/calculatedReplay.md +30 -81
- package/docs/calculatedReplayGuide.md +17 -34
- package/package.json +1 -1
- package/production.md +7 -36
- package/dist/replay/condition-words.d.ts +0 -28
- package/dist/replay/condition-words.js +0 -59
- package/dist/replay/flatten.d.ts +0 -161
- package/dist/replay/flatten.js +0 -222
- package/dist/replay/reads-only.d.ts +0 -29
- package/dist/replay/reads-only.js +0 -261
|
@@ -18,7 +18,7 @@ import { capText, DEFAULT_MAX_PAYLOAD_BYTES, REPORT_PAYLOAD_MAX_BYTES, serialize
|
|
|
18
18
|
import { logDetail, logLine, errText } from "../util/log.js";
|
|
19
19
|
import { serviceHealth, serviceTimeoutMs, timeoutSignal, } from "../util/service-health.js";
|
|
20
20
|
import { bashCommandFor } from "./bash-channel.js";
|
|
21
|
-
import { BuiltinNotRunnable, BuiltinTimeout, parseBashAllow, parseBuiltinTools, runBuiltin, } from "./builtins.js";
|
|
21
|
+
import { BuiltinNotRunnable, BuiltinTimeout, FILE_BUILTINS, parseBashAllow, parseBuiltinTools, parsePathAllow, runBuiltin, } from "./builtins.js";
|
|
22
22
|
import { BASH_CHANNEL_MAX, MAX_REPLAY_REASON, testWords, truncate } from "./bundle.js";
|
|
23
23
|
import { coverageOf, modeFor, reachOf } from "./coverage.js";
|
|
24
24
|
import { deriveParameters, localDay, } from "./derive.js";
|
|
@@ -29,7 +29,6 @@ import { PREVIEW_BUDGET_MS, RecordedValueError, ScenarioReplayPlan, StepRunError
|
|
|
29
29
|
import { PRICING_VERSION } from "./pricing.js";
|
|
30
30
|
import { hasShellStep, sinkOf, sinkOfPlan, unsafeParameter } from "./safety.js";
|
|
31
31
|
import { SourceRunOutputs } from "./source-run.js";
|
|
32
|
-
import { clampToFrameStart, flattenChain, frameEdges, } from "./flatten.js";
|
|
33
32
|
import { blockingTargets, testedOnlyTargets } from "./targets.js";
|
|
34
33
|
import { isNoResult, toolResultError } from "./tool-error.js";
|
|
35
34
|
import { OUTCOME_RANK, hasTargets, isReadyScenario, } from "./types.js";
|
|
@@ -168,7 +167,7 @@ function handedOver(note) {
|
|
|
168
167
|
}
|
|
169
168
|
export const DEFAULT_INTENT_MATCH = {
|
|
170
169
|
enabled: true,
|
|
171
|
-
budgetMs:
|
|
170
|
+
budgetMs: 1_500,
|
|
172
171
|
maxPerTurn: 3,
|
|
173
172
|
maxRequestsPerTurn: 40,
|
|
174
173
|
};
|
|
@@ -216,6 +215,8 @@ export class ReplayController {
|
|
|
216
215
|
builtinTools;
|
|
217
216
|
/** See {@link ReplayOptions.bashAllow}. */
|
|
218
217
|
bashAllow;
|
|
218
|
+
/** See {@link ReplayOptions.pathAllow}. */
|
|
219
|
+
pathAllow;
|
|
219
220
|
/** Calls waiting for a plan's pins to change (see `awaitPins`). */
|
|
220
221
|
pinWaiters = new WeakMap();
|
|
221
222
|
/**
|
|
@@ -277,25 +278,17 @@ export class ReplayController {
|
|
|
277
278
|
this.recordedFallback =
|
|
278
279
|
opts.recordedFallback ?? process.env.BIR_REPLAY_RECORDED_OUTPUTS === "1";
|
|
279
280
|
this.bashAllow = opts.bashAllow ?? parseBashAllow(process.env.BIR_BASH_ALLOW);
|
|
281
|
+
this.pathAllow = opts.pathAllow ?? parsePathAllow(process.env.BIR_PATH_ALLOW);
|
|
280
282
|
const builtins = opts.builtinTools ?? parseBuiltinTools(process.env.BIR_BUILTIN_TOOLS);
|
|
281
|
-
// A `Bash` that may run no command
|
|
282
|
-
//
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
*
|
|
291
|
-
* Only when the option says so. The recorder passes it on unless
|
|
292
|
-
* `BIR_SEGMENT_ARM=0` (from 0.3.19; the service's segmentsbreakdown.md
|
|
293
|
-
* SB-21); a controller built without it stays observe-only, asks for no
|
|
294
|
-
* segment and arms none. The server enforces the same thing from its side
|
|
295
|
-
* (R-OUT-9).
|
|
296
|
-
*/
|
|
297
|
-
get segmentArm() {
|
|
298
|
-
return this.opts.segmentArm === true;
|
|
283
|
+
// A `Bash` that may run no command, or a file tool that may touch no
|
|
284
|
+
// folder, is not reachable here. Left on the list, a chain of such steps
|
|
285
|
+
// would arm `direct` and stop at its first.
|
|
286
|
+
const reachable = (name) => name === "Bash"
|
|
287
|
+
? this.bashAllow.length > 0
|
|
288
|
+
: !FILE_BUILTINS.has(name) || this.pathAllow.length > 0;
|
|
289
|
+
this.builtinTools = [...builtins].every(reachable)
|
|
290
|
+
? builtins
|
|
291
|
+
: new Set([...builtins].filter(reachable));
|
|
299
292
|
}
|
|
300
293
|
/** The plan's own delivery vehicle is never a scenario step. */
|
|
301
294
|
isDirectTool(toolName) {
|
|
@@ -356,13 +349,6 @@ export class ReplayController {
|
|
|
356
349
|
reported: false,
|
|
357
350
|
retired: false,
|
|
358
351
|
armedBy,
|
|
359
|
-
// An older server sends no `kind`, and a whole-run scenario is the safe
|
|
360
|
-
// reading: it is what every hand-out was before segments existed.
|
|
361
|
-
kind: match.kind ?? "scenario",
|
|
362
|
-
segment: match.segment,
|
|
363
|
-
key: match.key,
|
|
364
|
-
verified: match.verified,
|
|
365
|
-
sharedWith: [],
|
|
366
352
|
};
|
|
367
353
|
}
|
|
368
354
|
/**
|
|
@@ -391,11 +377,6 @@ export class ReplayController {
|
|
|
391
377
|
*/
|
|
392
378
|
async arm(match, prompt, wrapped, armedBy = "prompt", ctx = {}, host) {
|
|
393
379
|
const state = this.blankState(match, wrapped, armedBy, host);
|
|
394
|
-
// A decision note quotes the request (SB-18); a plan armed by intent was
|
|
395
|
-
// given the step's intent here, and its control server sets the request.
|
|
396
|
-
if (armedBy === "prompt")
|
|
397
|
-
state.request = prompt;
|
|
398
|
-
const isSegment = state.kind === "segment";
|
|
399
380
|
const decline = (why, code) => {
|
|
400
381
|
state.declined = code;
|
|
401
382
|
logLine("replay.decision", {
|
|
@@ -404,19 +385,10 @@ export class ReplayController {
|
|
|
404
385
|
run: match.runId,
|
|
405
386
|
scenario: match.scenarioId ?? undefined,
|
|
406
387
|
similarity: match.similarity.toFixed(3),
|
|
407
|
-
kind: state.kind,
|
|
408
|
-
key: state.key,
|
|
409
|
-
verified: state.verified?.verdict,
|
|
410
388
|
why,
|
|
411
389
|
});
|
|
412
390
|
return state;
|
|
413
391
|
};
|
|
414
|
-
// The server never sends a segment to a runner that did not ask for one
|
|
415
|
-
// (R-OUT-9). One arriving anyway is declined here rather than run: an
|
|
416
|
-
// observe-only period that quietly armed something would be worthless.
|
|
417
|
-
if (isSegment && !this.segmentArm) {
|
|
418
|
-
return decline("segment hand-out while BIR_SEGMENT_ARM is off", "replay_disabled");
|
|
419
|
-
}
|
|
420
392
|
// Gate 2 — replay enabled.
|
|
421
393
|
if (!this.enabled)
|
|
422
394
|
return decline("BIR_REPLAY=0", "replay_disabled");
|
|
@@ -446,65 +418,24 @@ export class ReplayController {
|
|
|
446
418
|
}
|
|
447
419
|
// Gate 4 — similarity — is gone. A hand-out is the service's decision that
|
|
448
420
|
// this scenario does what the prompt asks: its match threshold, the
|
|
449
|
-
// same-request question under it, the intent pass
|
|
450
|
-
//
|
|
451
|
-
//
|
|
452
|
-
//
|
|
453
|
-
//
|
|
454
|
-
|
|
455
|
-
// Flatten the chain before the remaining gates, so every one of them judges
|
|
456
|
-
// the steps that will *really* run — the caller's own and, in order, those
|
|
457
|
-
// of every segment it calls (segmented.md R-CALL-29, R-CALL-30).
|
|
458
|
-
let flat;
|
|
459
|
-
try {
|
|
460
|
-
flat = flattenChain(scenario, {
|
|
461
|
-
range: match.segment
|
|
462
|
-
? { stepFrom: match.segment.stepFrom, stepTo: match.segment.stepTo }
|
|
463
|
-
: null,
|
|
464
|
-
});
|
|
465
|
-
}
|
|
466
|
-
catch (err) {
|
|
467
|
-
// A payload we cannot lay out is a payload we cannot run. It costs the
|
|
468
|
-
// turn nothing: the agent does the task (R-FALL-12).
|
|
469
|
-
logLine("replay.flatten_failed", { scenario: scenario.id, error: errText(err) });
|
|
470
|
-
return decline("the chain could not be flattened", "flatten_failed");
|
|
471
|
-
}
|
|
472
|
-
const entries = flat.entries;
|
|
473
|
-
if (entries.length === 0)
|
|
474
|
-
return decline("the chain has no runnable step", "not_ready");
|
|
421
|
+
// same-request question under it, the intent pass. Scoring it again here
|
|
422
|
+
// declined matches the service had made (a checked 0.85, a task line at
|
|
423
|
+
// 0.91), so every match now steers. What a plan cannot do — a step that
|
|
424
|
+
// keeps failing, a tool nobody wrapped, a target the prompt never named —
|
|
425
|
+
// the gates below still stop.
|
|
426
|
+
const steps = scenario.steps;
|
|
475
427
|
// Gate 5 — steps the plan must stop in front of (fallbk.md D3,
|
|
476
428
|
// plan-services.md D8): a step the service parked — failed more than its
|
|
477
|
-
// limit, retry not due — or one the calculation marked non-deterministic
|
|
478
|
-
|
|
479
|
-
// answer stops the plan the same way, and the earliest of them wins.
|
|
480
|
-
const badAt = stopIndex(entries.map((e) => e.step), match.fallback?.maxStepFailures);
|
|
481
|
-
const rawStop = [flat.stopAt, badAt].filter((n) => typeof n === "number");
|
|
482
|
-
// Clamped to the start of the outermost called frame it falls in: a plan
|
|
483
|
-
// never runs half a sub-task and leaves its result mapping unevaluated.
|
|
484
|
-
const stopAt = rawStop.length > 0 ? clampToFrameStart(entries, Math.min(...rawStop)) : undefined;
|
|
429
|
+
// limit, retry not due — or one the calculation marked non-deterministic.
|
|
430
|
+
const stopAt = stopIndex(steps, match.fallback?.maxStepFailures);
|
|
485
431
|
if (stopAt === 0) {
|
|
486
|
-
const
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
const judgement = stopping.stop?.kind === "nondeterministic" || decisionFirst;
|
|
494
|
-
const why = unusable
|
|
495
|
-
? `its first step is a call that cannot run: ${unusable.reason}`
|
|
496
|
-
: decisionFirst
|
|
497
|
-
? "its first step is a decision the agent makes — the agent does this task"
|
|
498
|
-
: judgement
|
|
499
|
-
? "its first step needs a judgement the plan cannot compute — the agent does this task"
|
|
500
|
-
: `its first step has failed ${stopping.failureCount ?? 0} times — the agent does this task`;
|
|
501
|
-
return decline(why, unusable
|
|
502
|
-
? "unusable_first_step"
|
|
503
|
-
: judgement
|
|
504
|
-
? "nondeterministic_first_step"
|
|
505
|
-
: "known_bad_first_step");
|
|
506
|
-
}
|
|
507
|
-
const planned = stopAt === undefined ? entries : entries.slice(0, stopAt);
|
|
432
|
+
const stopping = steps[0];
|
|
433
|
+
const judgement = stopping.stop?.kind === "nondeterministic";
|
|
434
|
+
return decline(judgement
|
|
435
|
+
? "its first step needs a judgement the plan cannot compute — the agent does this task"
|
|
436
|
+
: `its first step has failed ${stopping.failureCount ?? 0} times — the agent does this task`, judgement ? "nondeterministic_first_step" : "known_bad_first_step");
|
|
437
|
+
}
|
|
438
|
+
const planned = stopAt === undefined ? steps : steps.slice(0, stopAt);
|
|
508
439
|
// Another host's adapter steps through a plan call by call and knows no
|
|
509
440
|
// tests (AC5): asked for a step its test left out, it is refused, reads
|
|
510
441
|
// that as the plan's end, and the model's own calls after it are booked as
|
|
@@ -524,7 +455,7 @@ export class ReplayController {
|
|
|
524
455
|
// `mcp__fleet__list` is not the runner's to run or a proxy's to reach.
|
|
525
456
|
const allowServers = host ? undefined : this.opts.allowServers;
|
|
526
457
|
const builtinTools = host ? undefined : this.builtinTools;
|
|
527
|
-
const mode = modeFor(planned
|
|
458
|
+
const mode = modeFor(planned, state.wrapped, allowServers, builtinTools);
|
|
528
459
|
if (mode === "none")
|
|
529
460
|
return decline("no step is executable", "coverage");
|
|
530
461
|
// Tool results feed derivation as context, and they are whatever the agent
|
|
@@ -538,7 +469,7 @@ export class ReplayController {
|
|
|
538
469
|
// command nobody asked for.
|
|
539
470
|
let deriveCtx = ctx;
|
|
540
471
|
if (ctx.recentResults?.length && sinkOfPlan(planned, host?.sinks) === "command") {
|
|
541
|
-
deriveCtx = { liveCall: ctx.liveCall, agentDay: ctx.agentDay
|
|
472
|
+
deriveCtx = { liveCall: ctx.liveCall, agentDay: ctx.agentDay };
|
|
542
473
|
logLine("replay.derive_context", {
|
|
543
474
|
scenario: scenario.id,
|
|
544
475
|
shell: hasShellStep(planned, host?.sinks) || undefined,
|
|
@@ -665,14 +596,12 @@ export class ReplayController {
|
|
|
665
596
|
}
|
|
666
597
|
}
|
|
667
598
|
// A plan armed by intent runs inside a task the agent is already doing, so
|
|
668
|
-
// its directive and its bundles say "continue", never "answer the user"
|
|
669
|
-
// (segmented.md R-OUT-13).
|
|
599
|
+
// its directive and its bundles say "continue", never "answer the user".
|
|
670
600
|
state.plan = new ScenarioReplayPlan({
|
|
671
601
|
scenario,
|
|
672
602
|
params,
|
|
673
603
|
mode,
|
|
674
604
|
stopAt,
|
|
675
|
-
entries,
|
|
676
605
|
subTask: armedBy === "intent",
|
|
677
606
|
});
|
|
678
607
|
if (mode === "steer") {
|
|
@@ -691,17 +620,11 @@ export class ReplayController {
|
|
|
691
620
|
await state.plan.ready().catch(() => undefined);
|
|
692
621
|
}
|
|
693
622
|
state.mode = mode;
|
|
694
|
-
state.stepsPlanned =
|
|
623
|
+
state.stepsPlanned = steps.length;
|
|
695
624
|
// The chain this plan was served at: what its own calls are marked with
|
|
696
625
|
// when they are recorded into a redo of the turn (AC2).
|
|
697
626
|
if (typeof scenario.chainRevision === "number")
|
|
698
627
|
state.chainRevision = scenario.chainRevision;
|
|
699
|
-
// Why the plan stops, when a call is what stops it: the note names the
|
|
700
|
-
// sub-task rather than a tool (R-CALL-36).
|
|
701
|
-
state.flatStop =
|
|
702
|
-
typeof stopAt === "number" && flat.stopAt !== undefined && stopAt <= flat.stopAt
|
|
703
|
-
? flat.stopReason
|
|
704
|
-
: undefined;
|
|
705
628
|
// Recorded outputs are read only when they may stand in (opt-in), so a
|
|
706
629
|
// plan never fetches a run it will not show.
|
|
707
630
|
if (this.recordedFallback && this.opts.authUrl && this.opts.authToken) {
|
|
@@ -718,22 +641,16 @@ export class ReplayController {
|
|
|
718
641
|
scenario: scenario.id,
|
|
719
642
|
mode,
|
|
720
643
|
host: host?.app,
|
|
721
|
-
steps:
|
|
644
|
+
steps: steps.length,
|
|
722
645
|
stopsAt: stopAt,
|
|
723
646
|
by: armedBy === "intent" ? "intent" : undefined,
|
|
724
|
-
kind: state.kind,
|
|
725
|
-
key: state.key,
|
|
726
|
-
verified: state.verified?.verdict,
|
|
727
647
|
similarity: match.similarity.toFixed(3),
|
|
728
|
-
// What this chain calls, and what answers each call right now (R-CALL-32).
|
|
729
|
-
calls: callLines(scenario),
|
|
730
|
-
unusable: flat.stopReason?.reason,
|
|
731
648
|
// Which steps the service told this plan to stop in front of, and why.
|
|
732
|
-
stops: stopLines(
|
|
733
|
-
coverage: coverageOf(
|
|
734
|
-
tools:
|
|
735
|
-
// Which steps run only under a test, by
|
|
736
|
-
tests: testLines(
|
|
649
|
+
stops: stopLines(steps),
|
|
650
|
+
coverage: coverageOf(steps, state.wrapped, allowServers, builtinTools).join(","),
|
|
651
|
+
tools: steps.map((s) => s.toolName).join(","),
|
|
652
|
+
// Which steps run only under a test, by position (AC5).
|
|
653
|
+
tests: testLines(steps),
|
|
737
654
|
});
|
|
738
655
|
return state;
|
|
739
656
|
}
|
|
@@ -742,16 +659,16 @@ export class ReplayController {
|
|
|
742
659
|
* stops when that is before its end. For a host that steps through the plan
|
|
743
660
|
* itself and must not have to pick the tool names out of a directive
|
|
744
661
|
* written for a model — a reworded sentence would break every adapter in
|
|
745
|
-
* the field. Built from the same
|
|
662
|
+
* the field. Built from the same steps the directive lists.
|
|
746
663
|
*/
|
|
747
664
|
planData(state) {
|
|
748
665
|
const plan = state.plan;
|
|
749
666
|
if (!plan || state.retired)
|
|
750
667
|
return undefined;
|
|
751
668
|
const steps = plan
|
|
752
|
-
.
|
|
669
|
+
.allSteps()
|
|
753
670
|
.slice(0, plan.stepCount)
|
|
754
|
-
.map((
|
|
671
|
+
.map((step, stepIndex) => ({ stepIndex, toolName: step.toolName }));
|
|
755
672
|
return {
|
|
756
673
|
scenarioId: state.scenarioId,
|
|
757
674
|
mode: state.mode,
|
|
@@ -759,15 +676,6 @@ export class ReplayController {
|
|
|
759
676
|
...(plan.stopsEarly() ? { stopAt: plan.stepCount } : {}),
|
|
760
677
|
};
|
|
761
678
|
}
|
|
762
|
-
/**
|
|
763
|
-
* Whether the plan's steps end in front of a decision (the service's
|
|
764
|
-
* segmentsbreakdown.md SB-6): it runs the steps before it, then hands the
|
|
765
|
-
* task back there. The control server gives such a plan a usage mark, so
|
|
766
|
-
* that it is billed only up to the stop (SB-26).
|
|
767
|
-
*/
|
|
768
|
-
stopsAtDecision(state) {
|
|
769
|
-
return state.plan?.stopEntry()?.step.stop?.kind === "decision";
|
|
770
|
-
}
|
|
771
679
|
/**
|
|
772
680
|
* Take back a plan that armed and has run nothing, as a decline.
|
|
773
681
|
*
|
|
@@ -791,7 +699,6 @@ export class ReplayController {
|
|
|
791
699
|
run: state.matchedRunId,
|
|
792
700
|
scenario: state.scenarioId ?? undefined,
|
|
793
701
|
similarity: state.similarity.toFixed(3),
|
|
794
|
-
kind: state.kind,
|
|
795
702
|
why,
|
|
796
703
|
});
|
|
797
704
|
state.plan = undefined;
|
|
@@ -822,7 +729,6 @@ export class ReplayController {
|
|
|
822
729
|
this.handOver(state, {
|
|
823
730
|
kind: "step_failed",
|
|
824
731
|
step: plan.currentStep(),
|
|
825
|
-
entry: plan.currentEntry(),
|
|
826
732
|
position,
|
|
827
733
|
error: why === "own_call"
|
|
828
734
|
? "the model made a call of its own, so the scenario stopped"
|
|
@@ -883,8 +789,8 @@ export class ReplayController {
|
|
|
883
789
|
return false;
|
|
884
790
|
if (plan.parametersSettled)
|
|
885
791
|
return true;
|
|
886
|
-
const planned = plan.
|
|
887
|
-
if (!planned.some((
|
|
792
|
+
const planned = plan.allSteps().slice(0, plan.stepCount);
|
|
793
|
+
if (!planned.some((s) => pinnedToShape(s.toolName)))
|
|
888
794
|
return false;
|
|
889
795
|
if (state.deriveOnService && !serviceHealth.canWait())
|
|
890
796
|
return false;
|
|
@@ -1017,7 +923,6 @@ export class ReplayController {
|
|
|
1017
923
|
if (step && ready.blame) {
|
|
1018
924
|
this.recordStep(state, {
|
|
1019
925
|
step,
|
|
1020
|
-
entry: plan.currentEntry(),
|
|
1021
926
|
outcome: "failed",
|
|
1022
927
|
stage: ready.stage,
|
|
1023
928
|
error: ready.error,
|
|
@@ -1026,7 +931,6 @@ export class ReplayController {
|
|
|
1026
931
|
this.handOver(state, {
|
|
1027
932
|
kind: "step_failed",
|
|
1028
933
|
step,
|
|
1029
|
-
entry: plan.currentEntry(),
|
|
1030
934
|
position: plan.currentStepIndex,
|
|
1031
935
|
error: ready.error,
|
|
1032
936
|
delivery: "above",
|
|
@@ -1130,7 +1034,6 @@ export class ReplayController {
|
|
|
1130
1034
|
const recorded = err instanceof RecordedValueError;
|
|
1131
1035
|
this.recordStep(state, {
|
|
1132
1036
|
step,
|
|
1133
|
-
entry: plan.currentEntry(),
|
|
1134
1037
|
outcome: recorded ? "skipped" : "failed",
|
|
1135
1038
|
stage: "tool_input_logic",
|
|
1136
1039
|
error: errText(err),
|
|
@@ -1159,7 +1062,6 @@ export class ReplayController {
|
|
|
1159
1062
|
this.handOver(state, {
|
|
1160
1063
|
kind: "step_failed",
|
|
1161
1064
|
step: plan.currentStep(),
|
|
1162
|
-
entry: plan.currentEntry(),
|
|
1163
1065
|
position: plan.currentStepIndex,
|
|
1164
1066
|
error: `the scenario's next step is ${expected ?? "none"}, and the host asked for ${toolName}`,
|
|
1165
1067
|
delivery: "above",
|
|
@@ -1207,7 +1109,6 @@ export class ReplayController {
|
|
|
1207
1109
|
this.handOver(state, {
|
|
1208
1110
|
kind: "step_failed",
|
|
1209
1111
|
step: plan.allSteps()[next],
|
|
1210
|
-
entry: plan.allEntries()[next],
|
|
1211
1112
|
position: next,
|
|
1212
1113
|
delivery: "above",
|
|
1213
1114
|
stopReason: "concurrent",
|
|
@@ -1345,10 +1246,6 @@ export class ReplayController {
|
|
|
1345
1246
|
// reading it after the call would judge the *next* step, and reading it
|
|
1346
1247
|
// before assumes no other pin ever moved it.
|
|
1347
1248
|
const step = plan.allSteps()[pin.stepIndex];
|
|
1348
|
-
// The entry it ran as, so its verdict is kept under the scenario that owns
|
|
1349
|
-
// the step: a called segment numbers its steps from zero too, and without
|
|
1350
|
-
// this its step 0 replaced the caller's (segmented.md R-CALL-13).
|
|
1351
|
-
const entry = plan.allEntries()[pin.stepIndex];
|
|
1352
1249
|
// THE HOST STOPPED THE STEP, and says so. Read before any failure: a
|
|
1353
1250
|
// person's "no" arrives as an error result, and booked as one it would be
|
|
1354
1251
|
// counted against a plan that did nothing wrong — after enough of them the
|
|
@@ -1356,11 +1253,10 @@ export class ReplayController {
|
|
|
1356
1253
|
// `rejected` the tool never ran, and after `edited` its result answers a
|
|
1357
1254
|
// call the plan did not plan, which its later steps were not computed for.
|
|
1358
1255
|
if (how.hostStop && step) {
|
|
1359
|
-
this.hostStopped(state, pin, step,
|
|
1256
|
+
this.hostStopped(state, pin, step, output, how.hostStop);
|
|
1360
1257
|
this.handOver(state, {
|
|
1361
1258
|
kind: "step_failed",
|
|
1362
1259
|
step,
|
|
1363
|
-
entry,
|
|
1364
1260
|
position: pin.stepIndex,
|
|
1365
1261
|
delivery: "above",
|
|
1366
1262
|
stopReason: how.hostStop === "edited" ? "host_edited" : "host_rejected",
|
|
@@ -1381,7 +1277,6 @@ export class ReplayController {
|
|
|
1381
1277
|
`the plan's cursor is at step ${plan.currentStepIndex}, not the pinned step ${pin.stepIndex}`;
|
|
1382
1278
|
const info = {
|
|
1383
1279
|
step,
|
|
1384
|
-
entry,
|
|
1385
1280
|
input: pin.input,
|
|
1386
1281
|
output: answerOf(output),
|
|
1387
1282
|
outcome: "failed",
|
|
@@ -1395,7 +1290,6 @@ export class ReplayController {
|
|
|
1395
1290
|
this.handOver(state, {
|
|
1396
1291
|
kind: "step_failed",
|
|
1397
1292
|
step,
|
|
1398
|
-
entry,
|
|
1399
1293
|
position: pin.stepIndex,
|
|
1400
1294
|
error,
|
|
1401
1295
|
delivery: "above",
|
|
@@ -1417,7 +1311,6 @@ export class ReplayController {
|
|
|
1417
1311
|
if (step) {
|
|
1418
1312
|
this.recordStep(state, {
|
|
1419
1313
|
step,
|
|
1420
|
-
entry,
|
|
1421
1314
|
input: pin.input,
|
|
1422
1315
|
// What the logic threw on — the output is usually the explanation.
|
|
1423
1316
|
output,
|
|
@@ -1456,7 +1349,6 @@ export class ReplayController {
|
|
|
1456
1349
|
if (step && toolError) {
|
|
1457
1350
|
const info = {
|
|
1458
1351
|
step,
|
|
1459
|
-
entry,
|
|
1460
1352
|
input: pin.input,
|
|
1461
1353
|
output,
|
|
1462
1354
|
outcome: "failed",
|
|
@@ -1480,7 +1372,6 @@ export class ReplayController {
|
|
|
1480
1372
|
if (step) {
|
|
1481
1373
|
this.recordStep(state, {
|
|
1482
1374
|
step,
|
|
1483
|
-
entry,
|
|
1484
1375
|
input: pin.input,
|
|
1485
1376
|
output,
|
|
1486
1377
|
outcome: "executed",
|
|
@@ -1504,19 +1395,17 @@ export class ReplayController {
|
|
|
1504
1395
|
return done;
|
|
1505
1396
|
}
|
|
1506
1397
|
/**
|
|
1507
|
-
* A steer plan whose cursor reached its end: in front of a parked step
|
|
1508
|
-
*
|
|
1509
|
-
*
|
|
1398
|
+
* A steer plan whose cursor reached its end: in front of a parked step,
|
|
1399
|
+
* which hands over; otherwise it ran to its end — and from here every call
|
|
1400
|
+
* is the model's own.
|
|
1510
1401
|
*/
|
|
1511
1402
|
finishSteer(state, plan) {
|
|
1512
1403
|
if (plan.atStop()) {
|
|
1513
|
-
// Every planned step ran, and the next one is parked (fallbk.md D3)
|
|
1514
|
-
// is a call the service could not answer (segmented.md R-CALL-29).
|
|
1404
|
+
// Every planned step ran, and the next one is parked (fallbk.md D3).
|
|
1515
1405
|
const parked = plan.stopStep();
|
|
1516
1406
|
this.handOver(state, {
|
|
1517
|
-
kind:
|
|
1407
|
+
kind: stopKindOf(parked),
|
|
1518
1408
|
step: parked,
|
|
1519
|
-
entry: plan.stopEntry(),
|
|
1520
1409
|
position: plan.stepCount,
|
|
1521
1410
|
delivery: "above",
|
|
1522
1411
|
});
|
|
@@ -1553,34 +1442,28 @@ export class ReplayController {
|
|
|
1553
1442
|
settleCursor(state, plan, delivery) {
|
|
1554
1443
|
if (!plan.hasTests())
|
|
1555
1444
|
return "ready";
|
|
1556
|
-
// Never throws: a sub-task whose results cannot go back after its last
|
|
1557
|
-
// step was left out is a stop like any other (`result_map`).
|
|
1558
1445
|
const settled = plan.settleTests(this.observeStep(state));
|
|
1559
1446
|
// The recorded outputs of the steps left out are passed over, so a
|
|
1560
1447
|
// divergence's bundle later stands each step's own in (source-run.ts).
|
|
1561
1448
|
const source = state.sourceRun;
|
|
1562
1449
|
if (source) {
|
|
1563
1450
|
for (const l of settled.left) {
|
|
1564
|
-
|
|
1565
|
-
|
|
1566
|
-
|
|
1567
|
-
source.leftOut(entry);
|
|
1568
|
-
}
|
|
1451
|
+
const step = plan.allSteps()[l.position];
|
|
1452
|
+
if (step)
|
|
1453
|
+
source.leftOut(step);
|
|
1569
1454
|
}
|
|
1570
1455
|
}
|
|
1571
1456
|
if (settled.stop) {
|
|
1572
1457
|
const stop = settled.stop;
|
|
1573
|
-
const position = stop.
|
|
1458
|
+
const position = stop.position ?? plan.currentStepIndex;
|
|
1574
1459
|
this.handOver(state, {
|
|
1575
1460
|
kind: stop.kind,
|
|
1576
1461
|
step: plan.allSteps()[position],
|
|
1577
|
-
entry: plan.allEntries()[position],
|
|
1578
1462
|
position,
|
|
1579
1463
|
error: stop.error,
|
|
1580
1464
|
delivery,
|
|
1581
1465
|
stopReason: stop.reason,
|
|
1582
1466
|
whenText: stop.whenText,
|
|
1583
|
-
subTask: stop.subTask,
|
|
1584
1467
|
});
|
|
1585
1468
|
return "stopped";
|
|
1586
1469
|
}
|
|
@@ -1593,7 +1476,6 @@ export class ReplayController {
|
|
|
1593
1476
|
this.handOver(state, {
|
|
1594
1477
|
kind: "other_branch",
|
|
1595
1478
|
step: plan.allSteps()[0],
|
|
1596
|
-
entry: plan.allEntries()[0],
|
|
1597
1479
|
position: 0,
|
|
1598
1480
|
delivery,
|
|
1599
1481
|
noneNeeded: true,
|
|
@@ -1602,9 +1484,8 @@ export class ReplayController {
|
|
|
1602
1484
|
}
|
|
1603
1485
|
state.skipNote = {
|
|
1604
1486
|
text: this.skipNoteText(state, plan, settled.left),
|
|
1605
|
-
//
|
|
1606
|
-
|
|
1607
|
-
tools: new Set(settled.left.flatMap((l) => (l.tools.length > 0 ? l.tools : [l.toolName]))),
|
|
1487
|
+
// A call the model makes to any of these tools is one to a step left out.
|
|
1488
|
+
tools: new Set(settled.left.map((l) => l.toolName)),
|
|
1608
1489
|
delivered: false,
|
|
1609
1490
|
complete: plan.isDone() && !plan.atStop(),
|
|
1610
1491
|
};
|
|
@@ -1612,7 +1493,7 @@ export class ReplayController {
|
|
|
1612
1493
|
}
|
|
1613
1494
|
/** What the model is told about steps a test left out (AC5). */
|
|
1614
1495
|
skipNoteText(state, plan, left) {
|
|
1615
|
-
const named = left.map((l) => `step ${l.
|
|
1496
|
+
const named = left.map((l) => `step ${l.position + 1} (${l.toolName})`);
|
|
1616
1497
|
const texts = [...new Set(left.map((l) => l.text).filter((t) => !!t))];
|
|
1617
1498
|
const why = texts.length === 1 ? ` — only for when ${texts[0]}` : "";
|
|
1618
1499
|
const what = left.length === 1 ? `Not needed for this request: ${named[0]}${why}.` : `Not needed for this request: ${named.join(", ")}${why}.`;
|
|
@@ -1624,8 +1505,8 @@ export class ReplayController {
|
|
|
1624
1505
|
: "The sequence is complete: answer the user's request from the tool results.";
|
|
1625
1506
|
}
|
|
1626
1507
|
else {
|
|
1627
|
-
const
|
|
1628
|
-
next = `Leave ${left.length === 1 ? "it" : "them"} out, and call step ${plan.currentStepIndex + 1}${
|
|
1508
|
+
const step = plan.currentStep();
|
|
1509
|
+
next = `Leave ${left.length === 1 ? "it" : "them"} out, and call step ${plan.currentStepIndex + 1}${step?.toolName ? ` (${step.toolName})` : ""} next.`;
|
|
1629
1510
|
}
|
|
1630
1511
|
return `[BaseInstRunner calculated replay] ${what} ${next}`;
|
|
1631
1512
|
}
|
|
@@ -1687,7 +1568,7 @@ export class ReplayController {
|
|
|
1687
1568
|
return;
|
|
1688
1569
|
if (how.hostStop) {
|
|
1689
1570
|
// The host stopped it: neither a failure nor work the plan did.
|
|
1690
|
-
this.hostStopped(state, pin, step,
|
|
1571
|
+
this.hostStopped(state, pin, step, output, how.hostStop);
|
|
1691
1572
|
return;
|
|
1692
1573
|
}
|
|
1693
1574
|
const ms = Date.now() - pin.pinnedAt;
|
|
@@ -1699,11 +1580,9 @@ export class ReplayController {
|
|
|
1699
1580
|
? `${failure.error} — it was sent, and may or may not have completed`
|
|
1700
1581
|
: failure.error
|
|
1701
1582
|
: toolResultError(output);
|
|
1702
|
-
const entry = plan.allEntries()[pin.stepIndex];
|
|
1703
1583
|
if (error !== undefined) {
|
|
1704
1584
|
const info = {
|
|
1705
1585
|
step,
|
|
1706
|
-
entry,
|
|
1707
1586
|
input: pin.input,
|
|
1708
1587
|
output: answerOf(output),
|
|
1709
1588
|
outcome: "failed",
|
|
@@ -1715,7 +1594,7 @@ export class ReplayController {
|
|
|
1715
1594
|
this.recordStep(state, info);
|
|
1716
1595
|
return;
|
|
1717
1596
|
}
|
|
1718
|
-
this.recordStep(state, { step,
|
|
1597
|
+
this.recordStep(state, { step, input: pin.input, output, outcome: "executed", ms });
|
|
1719
1598
|
state.stepsPinned += 1;
|
|
1720
1599
|
this.upgrade(state, "fell_back");
|
|
1721
1600
|
logDetail("replay.thread_late", {
|
|
@@ -1730,7 +1609,7 @@ export class ReplayController {
|
|
|
1730
1609
|
* `failed`: the service counts failures against the plan. And not counted
|
|
1731
1610
|
* in `stepsPinned`: the plan's own call did not run.
|
|
1732
1611
|
*/
|
|
1733
|
-
hostStopped(state, pin, step,
|
|
1612
|
+
hostStopped(state, pin, step, output, stop) {
|
|
1734
1613
|
const edited = stop === "edited";
|
|
1735
1614
|
// After `rejected` nothing later can change what the report may claim.
|
|
1736
1615
|
if (state.hostStop !== "rejected")
|
|
@@ -1746,7 +1625,6 @@ export class ReplayController {
|
|
|
1746
1625
|
});
|
|
1747
1626
|
this.recordStep(state, {
|
|
1748
1627
|
step,
|
|
1749
|
-
entry,
|
|
1750
1628
|
// The plan's own arguments are what a refused call was asked with; a
|
|
1751
1629
|
// changed call ran with other ones, which only the host knows.
|
|
1752
1630
|
input: edited ? undefined : pin.input,
|
|
@@ -1823,7 +1701,6 @@ export class ReplayController {
|
|
|
1823
1701
|
if (step && ready.blame) {
|
|
1824
1702
|
this.recordStep(state, {
|
|
1825
1703
|
step,
|
|
1826
|
-
entry: plan.currentEntry(),
|
|
1827
1704
|
outcome: "failed",
|
|
1828
1705
|
stage: ready.stage,
|
|
1829
1706
|
error: ready.error,
|
|
@@ -1832,7 +1709,6 @@ export class ReplayController {
|
|
|
1832
1709
|
this.handOver(state, {
|
|
1833
1710
|
kind: "step_failed",
|
|
1834
1711
|
step,
|
|
1835
|
-
entry: plan.currentEntry(),
|
|
1836
1712
|
position: plan.currentStepIndex,
|
|
1837
1713
|
error: ready.error,
|
|
1838
1714
|
delivery: "none",
|
|
@@ -1866,7 +1742,7 @@ export class ReplayController {
|
|
|
1866
1742
|
// them alone is a baseline sample, not a hand-over "saving".
|
|
1867
1743
|
const ranOk = result.executed - result.errored;
|
|
1868
1744
|
state.stepsPinned += Math.max(0, ranOk);
|
|
1869
|
-
const position = stop.
|
|
1745
|
+
const position = stop.position ?? plan.stepCount;
|
|
1870
1746
|
const step = plan.allSteps()[position];
|
|
1871
1747
|
// Cut short by the clock, nothing failed: the steps that ran did their
|
|
1872
1748
|
// work and none is blamed. A divergence for the ledger — a partial
|
|
@@ -1884,11 +1760,10 @@ export class ReplayController {
|
|
|
1884
1760
|
const live = result.executed > 0;
|
|
1885
1761
|
this.handOver(state, {
|
|
1886
1762
|
// The plan only knows it stopped in front of its end; which kind of
|
|
1887
|
-
// stop that is — parked, a judgement
|
|
1888
|
-
//
|
|
1889
|
-
kind: stop.kind === "known_bad_step" ?
|
|
1763
|
+
// stop that is — parked, or a judgement — is decided here, where the
|
|
1764
|
+
// served reasons are.
|
|
1765
|
+
kind: stop.kind === "known_bad_step" ? stopKindOf(plan.stopStep()) : stop.kind,
|
|
1890
1766
|
step,
|
|
1891
|
-
entry: plan.allEntries()[position],
|
|
1892
1767
|
position,
|
|
1893
1768
|
error: stop.error,
|
|
1894
1769
|
// A cancelled run's note is kept for the model's next call, alone —
|
|
@@ -1899,12 +1774,9 @@ export class ReplayController {
|
|
|
1899
1774
|
ran: live ? undefined : 0,
|
|
1900
1775
|
// An output-logic failure stops *after* its step's tool ran: the step
|
|
1901
1776
|
// is done, its result is in the bundle, and it must not be run again.
|
|
1902
|
-
|
|
1903
|
-
// (`result_map`): that step did not run (AC5).
|
|
1904
|
-
ranThisStep: stop.stage === "tool_output_logic" && stop.reason !== "result_map",
|
|
1777
|
+
ranThisStep: stop.stage === "tool_output_logic",
|
|
1905
1778
|
// The other half (AC5): what the steps from here are for.
|
|
1906
1779
|
whenText: stop.whenText,
|
|
1907
|
-
subTask: stop.subTask,
|
|
1908
1780
|
});
|
|
1909
1781
|
if (stop.reason === "cancelled") {
|
|
1910
1782
|
// The caller is gone and reads nothing. The note stays undelivered: a
|
|
@@ -1941,7 +1813,6 @@ export class ReplayController {
|
|
|
1941
1813
|
this.handOver(state, {
|
|
1942
1814
|
kind: "other_branch",
|
|
1943
1815
|
step: plan.allSteps()[0],
|
|
1944
|
-
entry: plan.allEntries()[0],
|
|
1945
1816
|
position: 0,
|
|
1946
1817
|
delivery: "none",
|
|
1947
1818
|
ran: 0,
|
|
@@ -1956,7 +1827,6 @@ export class ReplayController {
|
|
|
1956
1827
|
this.handOver(state, {
|
|
1957
1828
|
kind: "step_failed",
|
|
1958
1829
|
step: plan.allSteps()[0],
|
|
1959
|
-
entry: plan.allEntries()[0],
|
|
1960
1830
|
position: 0,
|
|
1961
1831
|
error: "none of the scenario's steps could run here",
|
|
1962
1832
|
delivery: "none",
|
|
@@ -2023,20 +1893,16 @@ export class ReplayController {
|
|
|
2023
1893
|
const total = state.deriveCostUsd + d.sessionCostUsd + state.fallbackCostUsd;
|
|
2024
1894
|
const steps = this.stepResultsOf(state);
|
|
2025
1895
|
const isBaseline = state.outcome === "not_steered" || state.outcome === "failed";
|
|
2026
|
-
// The plan handed the task back by design, not by a fault: in front of the
|
|
2027
|
-
// other half of an "if … otherwise" (AC5), or at its decision (the
|
|
2028
|
-
// service's segmentsbreakdown.md SB-19). Such a stop is named whatever the
|
|
2029
|
-
// outcome, and the agent's work after it is the service's evidence.
|
|
2030
|
-
const handedBack = state.handover?.kind === "other_branch" || state.handover?.kind === "decision";
|
|
2031
1896
|
// A costless decline is noise on both sides — *unless* it says something the
|
|
2032
1897
|
// recording page cannot learn any other way: a step that actually broke, the
|
|
2033
|
-
// gate that declined, or a plan that
|
|
2034
|
-
//
|
|
2035
|
-
//
|
|
1898
|
+
// gate that declined, or a plan that stood in front of the other half of
|
|
1899
|
+
// its task (AC5), which the service takes the agent's work after as its
|
|
1900
|
+
// evidence of that half. The service accepts a costless report that carries
|
|
1901
|
+
// any of them (its errorshandling.md).
|
|
2036
1902
|
if (isBaseline &&
|
|
2037
1903
|
total <= 0 &&
|
|
2038
1904
|
!state.declined &&
|
|
2039
|
-
|
|
1905
|
+
state.handover?.kind !== "other_branch" &&
|
|
2040
1906
|
!steps?.some((s) => s.status === "failed")) {
|
|
2041
1907
|
return undefined;
|
|
2042
1908
|
}
|
|
@@ -2077,22 +1943,18 @@ export class ReplayController {
|
|
|
2077
1943
|
declined: state.declined,
|
|
2078
1944
|
// Where the plan handed the task to the model (fallbk.md). Only on
|
|
2079
1945
|
// `fell_back`: a hand-over before any step ran is `failed`, and a parked
|
|
2080
|
-
// step found after a divergence is still a divergence. Except
|
|
2081
|
-
//
|
|
2082
|
-
//
|
|
2083
|
-
// outcome: the agent's work after it is the service's evidence, whether
|
|
1946
|
+
// step found after a divergence is still a divergence. Except the other
|
|
1947
|
+
// half of an "if … otherwise" (AC5), named whatever the outcome: the
|
|
1948
|
+
// agent's work after it is the service's evidence of that half, whether
|
|
2084
1949
|
// or not a step ran first.
|
|
2085
|
-
fallbackStepIndex: state.outcome === "fell_back" ||
|
|
1950
|
+
fallbackStepIndex: state.outcome === "fell_back" || state.handover?.kind === "other_branch"
|
|
2086
1951
|
? state.handover?.stepIndex
|
|
2087
1952
|
: undefined,
|
|
2088
|
-
fallbackKind: state.outcome === "fell_back" ||
|
|
1953
|
+
fallbackKind: state.outcome === "fell_back" || state.handover?.kind === "other_branch"
|
|
2089
1954
|
? state.handover?.kind
|
|
2090
1955
|
: undefined,
|
|
2091
1956
|
// The agent did not take the plan's result and redid the task (AC1).
|
|
2092
1957
|
...(d.redoneBy ? { redoneBy: d.redoneBy } : {}),
|
|
2093
|
-
// Segments of this plan's recording that armed later in the turn
|
|
2094
|
-
// (segmented.md R-MONEY-4): their share comes off this plan's baseline.
|
|
2095
|
-
sharedWith: state.sharedWith.length > 0 ? [...state.sharedWith] : undefined,
|
|
2096
1958
|
// A turn in which another plan ran is not a measurement of what this task
|
|
2097
1959
|
// costs unaided, so it is no baseline sample (segmented.md R-MONEY-5).
|
|
2098
1960
|
// This covers two plans that both armed, and — the case that was silently
|
|
@@ -2136,26 +1998,11 @@ export class ReplayController {
|
|
|
2136
1998
|
* there is nothing to be similar *to* and nothing to decline. It books no
|
|
2137
1999
|
* execution either — there is no ticket, and inventing a saving for a manual
|
|
2138
2000
|
* invocation is exactly the kind of number a ledger must never contain.
|
|
2139
|
-
*
|
|
2140
|
-
* One served stop is kept: a plan's decision (the service's
|
|
2141
|
-
* segmentsbreakdown.md SB-6). The run stops in front of it, and `stopped`
|
|
2142
|
-
* says so with the reason `decision`.
|
|
2143
2001
|
*/
|
|
2144
2002
|
async runAdHoc(scenario, prompt, wrapped) {
|
|
2145
2003
|
if (!isReadyScenario(scenario))
|
|
2146
2004
|
return { ok: false, why: "scenario is not ready" };
|
|
2147
|
-
|
|
2148
|
-
// flat list, one frame per call (segmented.md R-CALL-37).
|
|
2149
|
-
let flat;
|
|
2150
|
-
try {
|
|
2151
|
-
flat = flattenChain(scenario);
|
|
2152
|
-
}
|
|
2153
|
-
catch (err) {
|
|
2154
|
-
return { ok: false, why: `the chain could not be flattened: ${errText(err)}` };
|
|
2155
|
-
}
|
|
2156
|
-
if (flat.entries.length === 0)
|
|
2157
|
-
return { ok: false, why: "the chain has no runnable step" };
|
|
2158
|
-
const mode = modeFor(flat.entries.map((e) => e.step), wrapped, this.opts.allowServers, this.builtinTools);
|
|
2005
|
+
const mode = modeFor(scenario.steps, wrapped, this.opts.allowServers, this.builtinTools);
|
|
2159
2006
|
if (mode === "none")
|
|
2160
2007
|
return { ok: false, why: "no step is executable" };
|
|
2161
2008
|
const state = {
|
|
@@ -2177,15 +2024,11 @@ export class ReplayController {
|
|
|
2177
2024
|
reported: true,
|
|
2178
2025
|
retired: false,
|
|
2179
2026
|
armedBy: "prompt",
|
|
2180
|
-
// `bir replay` runs a scenario an operator named by id: never a segment
|
|
2181
|
-
// hand-out, and nothing to share a baseline with.
|
|
2182
|
-
kind: "scenario",
|
|
2183
|
-
sharedWith: [],
|
|
2184
2027
|
};
|
|
2185
|
-
// `bir replay` keeps its own contract on a missing target
|
|
2186
|
-
//
|
|
2187
|
-
//
|
|
2188
|
-
//
|
|
2028
|
+
// `bir replay` keeps its own contract on a missing target: an operator
|
|
2029
|
+
// asked for this run by id, so a sample stands in and it never declines.
|
|
2030
|
+
// It says which, as the console's dry replay does (R-PARAM-7), so nobody
|
|
2031
|
+
// reads the output as proof the values were derived.
|
|
2189
2032
|
const schema = scenario.paramsObject ?? {};
|
|
2190
2033
|
const params = this.startDerivation(scenario, prompt, state).then((r) => {
|
|
2191
2034
|
const out = { ...r.params };
|
|
@@ -2204,25 +2047,13 @@ export class ReplayController {
|
|
|
2204
2047
|
}
|
|
2205
2048
|
return out;
|
|
2206
2049
|
});
|
|
2207
|
-
// The one served stop an operator's run keeps: a decision (the service's
|
|
2208
|
-
// segmentsbreakdown.md SB-6). The steps after it are one half of an "if …
|
|
2209
|
-
// otherwise", the half the recording took, and running them by id would
|
|
2210
|
-
// do that half's writes whatever the request. Every other served stop is
|
|
2211
|
-
// the plan's health, which an operator's run is there to try.
|
|
2212
|
-
const decisionAt = flat.entries.findIndex((e) => e.step.stop?.kind === "decision");
|
|
2213
|
-
const stopAt = decisionAt >= 0 && (flat.stopAt === undefined || decisionAt < flat.stopAt)
|
|
2214
|
-
? clampToFrameStart(flat.entries, decisionAt)
|
|
2215
|
-
: flat.stopAt;
|
|
2216
2050
|
const plan = new ScenarioReplayPlan({
|
|
2217
2051
|
scenario,
|
|
2218
2052
|
params,
|
|
2219
2053
|
mode,
|
|
2220
|
-
entries: flat.entries,
|
|
2221
|
-
stopAt,
|
|
2222
2054
|
fallbackTargets: "samples",
|
|
2223
2055
|
});
|
|
2224
2056
|
state.plan = plan;
|
|
2225
|
-
state.flatStop = stopAt === flat.stopAt ? flat.stopReason : undefined;
|
|
2226
2057
|
if (this.opts.authUrl && this.opts.authToken) {
|
|
2227
2058
|
state.sourceRun = new SourceRunOutputs({
|
|
2228
2059
|
baseUrl: this.opts.authUrl,
|
|
@@ -2238,8 +2069,8 @@ export class ReplayController {
|
|
|
2238
2069
|
coverage: coverageOf(scenario.steps, wrapped, this.opts.allowServers, this.builtinTools).join(","),
|
|
2239
2070
|
why: "bir replay — no match, an operator asked for it",
|
|
2240
2071
|
});
|
|
2241
|
-
// An operator's run keeps its own contract
|
|
2242
|
-
//
|
|
2072
|
+
// An operator's run keeps its own contract: out of time, the targets too
|
|
2073
|
+
// take their samples. What it never does is run on `{}`.
|
|
2243
2074
|
const ready = await this.readyFor(state, plan, this.adHocBudgets.deriveMs, { adHoc: true });
|
|
2244
2075
|
if (!ready.ok)
|
|
2245
2076
|
return { ok: false, why: ready.error };
|
|
@@ -2268,16 +2099,7 @@ export class ReplayController {
|
|
|
2268
2099
|
logDetail("replay.response_model_failed", { error: errText(err) });
|
|
2269
2100
|
}
|
|
2270
2101
|
const stop = result.stopped;
|
|
2271
|
-
const stoppedStep = stop ? plan.allSteps()[stop.
|
|
2272
|
-
// The plan's end in front of its decision: said so, by its own word.
|
|
2273
|
-
const atDecision = stop?.kind === "known_bad_step" && stoppedStep?.stop?.kind === "decision";
|
|
2274
|
-
if (atDecision) {
|
|
2275
|
-
logLine("replay.adhoc_decision", {
|
|
2276
|
-
scenario: scenario.id,
|
|
2277
|
-
step: stop.stepIndex,
|
|
2278
|
-
why: "bir replay stops in front of the plan's decision: the steps after it are the half the recording took",
|
|
2279
|
-
});
|
|
2280
|
-
}
|
|
2102
|
+
const stoppedStep = stop ? plan.allSteps()[stop.position ?? plan.stepCount] : undefined;
|
|
2281
2103
|
return {
|
|
2282
2104
|
ok: true,
|
|
2283
2105
|
text: result.text,
|
|
@@ -2289,7 +2111,7 @@ export class ReplayController {
|
|
|
2289
2111
|
? {
|
|
2290
2112
|
step: stop.stepIndex,
|
|
2291
2113
|
tool: stoppedStep?.toolName ?? "",
|
|
2292
|
-
reason:
|
|
2114
|
+
reason: stop.reason ?? stop.stage ?? stop.kind,
|
|
2293
2115
|
error: stop.error,
|
|
2294
2116
|
}
|
|
2295
2117
|
: undefined,
|
|
@@ -2352,8 +2174,6 @@ export class ReplayController {
|
|
|
2352
2174
|
fetchImpl: this.opts.fetchImpl,
|
|
2353
2175
|
liveCall: ctx.liveCall,
|
|
2354
2176
|
recentResults: ctx.recentResults,
|
|
2355
|
-
// A reading for a scenario that follows a plan's decision (offer1.md O4).
|
|
2356
|
-
...(ctx.atDecision ? { atDecision: ctx.atDecision } : {}),
|
|
2357
2177
|
// The request itself ends too. Without this a hung service or model held
|
|
2358
2178
|
// the socket for up to 300 s after every wait on it had given up. Long
|
|
2359
2179
|
// enough to cover both waits that can use its answer (the arm and the
|
|
@@ -2486,16 +2306,7 @@ export class ReplayController {
|
|
|
2486
2306
|
// when a step did its work — so the service takes the agent's work after
|
|
2487
2307
|
// it as that half's evidence. Booked `diverged`, it never did.
|
|
2488
2308
|
const otherHalf = stop?.kind === "other_branch";
|
|
2489
|
-
|
|
2490
|
-
// service's segmentsbreakdown.md SB-17) is a hand-over too: the plan did
|
|
2491
|
-
// all it was for, and the agent decides. Not one of recorded outputs
|
|
2492
|
-
// only, which is no step done here — that stays a plain divergence.
|
|
2493
|
-
const atDecision = !stop &&
|
|
2494
|
-
plan.stopsEarly() &&
|
|
2495
|
-
composed.endedAt === plan.stepCount &&
|
|
2496
|
-
(composed.executed > 0 || composed.recorded === 0) &&
|
|
2497
|
-
this.stopKindOf(state, plan.stopEntry()) === "decision";
|
|
2498
|
-
this.upgrade(state, broke || !didWork ? "failed" : otherHalf || atDecision ? "fell_back" : "diverged");
|
|
2309
|
+
this.upgrade(state, broke || !didWork ? "failed" : otherHalf ? "fell_back" : "diverged");
|
|
2499
2310
|
state.stepsPinned += composed.executed;
|
|
2500
2311
|
// The headline number of this whole design: recovering from a divergence
|
|
2501
2312
|
// costs nothing, because the proxies were already connected.
|
|
@@ -2506,7 +2317,7 @@ export class ReplayController {
|
|
|
2506
2317
|
recorded: composed.recorded,
|
|
2507
2318
|
skipped: composed.skipped,
|
|
2508
2319
|
errored: composed.errored || undefined,
|
|
2509
|
-
stopped: stop ? `${stop.
|
|
2320
|
+
stopped: stop ? `${stop.position ?? "?"}:${stop.reason ?? stop.stage ?? stop.kind}` : undefined,
|
|
2510
2321
|
bytes: composed.text.length,
|
|
2511
2322
|
costUsd: state.fallbackCostUsd.toFixed(2),
|
|
2512
2323
|
});
|
|
@@ -2522,40 +2333,16 @@ export class ReplayController {
|
|
|
2522
2333
|
// of recorded outputs only is last week's work. The model's own call goes
|
|
2523
2334
|
// ahead — unless a step was *sent* and never answered. That one may have
|
|
2524
2335
|
// done its work, and the model must know before it does the task itself.
|
|
2525
|
-
if (stop?.reason === "uncertain" && stop.
|
|
2336
|
+
if (stop?.reason === "uncertain" && stop.position !== undefined) {
|
|
2526
2337
|
const note = this.stopNote(state, plan, stop, "none", composed.left);
|
|
2527
2338
|
return this.deliverInstead(note, toolName, state.host !== undefined);
|
|
2528
2339
|
}
|
|
2529
2340
|
// The rest is the other half (AC5): the model is told so, in place of
|
|
2530
2341
|
// its call, before it does any of the plan's remaining steps.
|
|
2531
|
-
if (stop?.kind === "other_branch" && stop.
|
|
2342
|
+
if (stop?.kind === "other_branch" && stop.position !== undefined) {
|
|
2532
2343
|
const note = this.stopNote(state, plan, stop, "none", composed.left);
|
|
2533
2344
|
return this.deliverInstead(note, toolName, state.host !== undefined);
|
|
2534
2345
|
}
|
|
2535
|
-
// Every step left before the decision was one this request does not
|
|
2536
|
-
// need: the plan has done its part, and the agent decides. Told in
|
|
2537
|
-
// place of its call, like the other half. The steps that ran are the
|
|
2538
|
-
// model's own calls above; the ones left out are named.
|
|
2539
|
-
if (atDecision) {
|
|
2540
|
-
const parked = plan.stopStep();
|
|
2541
|
-
state.handover = {
|
|
2542
|
-
kind: "decision",
|
|
2543
|
-
stepIndex: callerStepIndexOf(plan.stopEntry()) ?? parked.stepIndex,
|
|
2544
|
-
toolName: parked.toolName ?? undefined,
|
|
2545
|
-
note: this.noteFor(state, {
|
|
2546
|
-
kind: "decision",
|
|
2547
|
-
step: parked,
|
|
2548
|
-
entry: plan.stopEntry(),
|
|
2549
|
-
position: plan.stepCount,
|
|
2550
|
-
delivery: "above",
|
|
2551
|
-
executedAbove: plan.currentStepIndex,
|
|
2552
|
-
leftOut: composed.left,
|
|
2553
|
-
}),
|
|
2554
|
-
delivered: true,
|
|
2555
|
-
};
|
|
2556
|
-
this.decidedInDivergence(state, plan);
|
|
2557
|
-
return this.deliverInstead(state.handover.note, toolName, state.host !== undefined);
|
|
2558
|
-
}
|
|
2559
2346
|
// Every remaining step was left out by its test: the steps the session
|
|
2560
2347
|
// ran were the whole of the plan this request needs, and it ran to its
|
|
2561
2348
|
// end (newestRun.md N7 reads that).
|
|
@@ -2570,13 +2357,13 @@ export class ReplayController {
|
|
|
2570
2357
|
// outcome stays as judged above, so the report carries no fallback fields
|
|
2571
2358
|
// (the other half of an "if … otherwise" excepted: see `buildReport`).
|
|
2572
2359
|
let text = composed.text;
|
|
2573
|
-
if (stop && (stop.kind === "step_failed" || stop.kind === "other_branch") && stop.
|
|
2360
|
+
if (stop && (stop.kind === "step_failed" || stop.kind === "other_branch") && stop.position !== undefined) {
|
|
2574
2361
|
const note = this.stopNote(state, plan, stop, "below", composed.left);
|
|
2575
2362
|
text = `${note}\n\n${composed.text}`;
|
|
2576
2363
|
}
|
|
2577
2364
|
else if (plan.stopsEarly() && composed.endedAt === plan.stepCount) {
|
|
2578
2365
|
const parked = plan.stopStep();
|
|
2579
|
-
const stopKind =
|
|
2366
|
+
const stopKind = stopKindOf(parked);
|
|
2580
2367
|
state.handover = {
|
|
2581
2368
|
kind: stopKind,
|
|
2582
2369
|
stepIndex: parked.stepIndex,
|
|
@@ -2584,7 +2371,6 @@ export class ReplayController {
|
|
|
2584
2371
|
note: this.noteFor(state, {
|
|
2585
2372
|
kind: stopKind,
|
|
2586
2373
|
step: parked,
|
|
2587
|
-
entry: plan.stopEntry(),
|
|
2588
2374
|
position: plan.stepCount,
|
|
2589
2375
|
delivery: "below",
|
|
2590
2376
|
executedAbove: plan.currentStepIndex,
|
|
@@ -2594,8 +2380,6 @@ export class ReplayController {
|
|
|
2594
2380
|
delivered: true,
|
|
2595
2381
|
};
|
|
2596
2382
|
text = `${state.handover.note}\n\n${composed.text}`;
|
|
2597
|
-
if (atDecision)
|
|
2598
|
-
this.decidedInDivergence(state, plan);
|
|
2599
2383
|
}
|
|
2600
2384
|
else if (!stop) {
|
|
2601
2385
|
// The bundle carries the rest of the chain, and its header tells the
|
|
@@ -2617,13 +2401,11 @@ export class ReplayController {
|
|
|
2617
2401
|
stopNote(state, plan, stop, delivery,
|
|
2618
2402
|
/** The steps the bundle left out (AC5): no step that ran. */
|
|
2619
2403
|
left = []) {
|
|
2620
|
-
const position = stop.
|
|
2404
|
+
const position = stop.position ?? plan.currentStepIndex;
|
|
2621
2405
|
const step = plan.allSteps()[position];
|
|
2622
|
-
const entry = plan.allEntries()[position];
|
|
2623
2406
|
const note = this.noteFor(state, {
|
|
2624
2407
|
kind: stop.kind,
|
|
2625
2408
|
step,
|
|
2626
|
-
entry,
|
|
2627
2409
|
position,
|
|
2628
2410
|
error: stop.error,
|
|
2629
2411
|
delivery,
|
|
@@ -2633,16 +2415,14 @@ export class ReplayController {
|
|
|
2633
2415
|
// steps that ran; "no step ran" would invite the model to run them again.
|
|
2634
2416
|
ran: delivery === "none" ? plan.currentStepIndex : undefined,
|
|
2635
2417
|
executedAbove: plan.currentStepIndex,
|
|
2636
|
-
// An output-logic failure stops after its step's tool ran (plan.ts)
|
|
2637
|
-
|
|
2638
|
-
ranThisStep: stop.stage === "tool_output_logic" && stop.reason !== "result_map",
|
|
2418
|
+
// An output-logic failure stops after its step's tool ran (plan.ts).
|
|
2419
|
+
ranThisStep: stop.stage === "tool_output_logic",
|
|
2639
2420
|
whenText: stop.whenText,
|
|
2640
|
-
subTask: stop.subTask,
|
|
2641
2421
|
leftOut: left,
|
|
2642
2422
|
});
|
|
2643
2423
|
state.handover ??= {
|
|
2644
2424
|
kind: stop.kind,
|
|
2645
|
-
stepIndex:
|
|
2425
|
+
stepIndex: step?.stepIndex ?? position,
|
|
2646
2426
|
toolName: step?.toolName ?? undefined,
|
|
2647
2427
|
error: stop.error,
|
|
2648
2428
|
note,
|
|
@@ -2663,17 +2443,9 @@ export class ReplayController {
|
|
|
2663
2443
|
return;
|
|
2664
2444
|
const ran = state.stepsPinned;
|
|
2665
2445
|
this.retire(state, ran > 0 ? "fell_back" : "failed");
|
|
2666
|
-
// A plan stopped at its decision did all it was for: its bill ends here,
|
|
2667
|
-
// and the agent's decision and the half it takes are the agent's own
|
|
2668
|
-
// (the service's segmentsbreakdown.md SB-26). Only once a step ran: a
|
|
2669
|
-
// plan that did nothing is a baseline sample of the whole turn.
|
|
2670
|
-
if (h.kind === "decision" && ran > 0)
|
|
2671
|
-
this.markRetired(state);
|
|
2672
2446
|
state.handover = {
|
|
2673
2447
|
kind: h.kind,
|
|
2674
|
-
|
|
2675
|
-
// number the caller's own chain does not have (R-CALL-32, R-FALL-7).
|
|
2676
|
-
stepIndex: callerStepIndexOf(h.entry) ?? h.step?.stepIndex ?? h.position,
|
|
2448
|
+
stepIndex: h.step?.stepIndex ?? h.position,
|
|
2677
2449
|
toolName: h.step?.toolName ?? undefined,
|
|
2678
2450
|
error: h.error,
|
|
2679
2451
|
note: this.noteFor(state, h),
|
|
@@ -2684,83 +2456,38 @@ export class ReplayController {
|
|
|
2684
2456
|
kind: h.kind,
|
|
2685
2457
|
step: `${h.position}/${state.plan?.totalSteps ?? state.stepsPlanned}`,
|
|
2686
2458
|
tool: h.step?.toolName ?? undefined,
|
|
2687
|
-
// The scenario that owns the stopped step, when it is not the matched one.
|
|
2688
|
-
owner: h.entry && h.entry.depth > 0 ? h.entry.scenarioId : undefined,
|
|
2689
|
-
unusable: state.flatStop?.reason,
|
|
2690
|
-
callIntent: state.flatStop?.callIntent ?? undefined,
|
|
2691
|
-
segment: state.flatStop?.segmentId ?? undefined,
|
|
2692
2459
|
ran,
|
|
2693
2460
|
outcome: state.outcome,
|
|
2694
2461
|
why: h.kind === "other_branch"
|
|
2695
2462
|
? h.noneNeeded
|
|
2696
2463
|
? "no step of the plan was needed for this request — the agent does the task"
|
|
2697
2464
|
: `this request is the other half: the steps from here are only for when ${h.whenText ?? "their test says so"} — the agent continues`
|
|
2698
|
-
: h.kind === "
|
|
2699
|
-
? "the
|
|
2700
|
-
: h.kind === "
|
|
2701
|
-
? "a
|
|
2702
|
-
: h.
|
|
2703
|
-
? "the
|
|
2704
|
-
: h.
|
|
2705
|
-
? "the
|
|
2706
|
-
: h.stopReason === "
|
|
2707
|
-
? "
|
|
2708
|
-
: h.stopReason === "
|
|
2709
|
-
? "
|
|
2710
|
-
:
|
|
2711
|
-
? "the host stopped this step itself — no fault of the plan; the agent continues from here"
|
|
2712
|
-
: h.stopReason === "not_run" || h.stopReason === "uncertain"
|
|
2713
|
-
? "a step produced no result here — the agent continues from here"
|
|
2714
|
-
: "a step broke — the agent continues from here",
|
|
2715
|
-
});
|
|
2716
|
-
}
|
|
2717
|
-
/**
|
|
2718
|
-
* Which kind of stop a plan's end is (plan-services.md D8). The stopping
|
|
2719
|
-
* step's own served reason wins — the calculation's verdict is more exact
|
|
2720
|
-
* than a counter — then a call the service could not answer, then a parked
|
|
2721
|
-
* step.
|
|
2722
|
-
*/
|
|
2723
|
-
stopKindOf(state, entry) {
|
|
2724
|
-
// The service serves a decision ahead of any other stop of its step
|
|
2725
|
-
// (the service's segmentsbreakdown.md SB-6), and so it is read first.
|
|
2726
|
-
if (entry?.step.stop?.kind === "decision")
|
|
2727
|
-
return "decision";
|
|
2728
|
-
if (entry?.step.stop?.kind === "nondeterministic")
|
|
2729
|
-
return "nondeterministic_step";
|
|
2730
|
-
return state.flatStop ? "unusable_call" : "known_bad_step";
|
|
2731
|
-
}
|
|
2732
|
-
/**
|
|
2733
|
-
* A divergence's bundle that ended at the plan's decision (SB-17): logged
|
|
2734
|
-
* as `handOver` logs every other hand-over, and its usage mark taken there
|
|
2735
|
-
* when a step of the plan ran (SB-26).
|
|
2736
|
-
*/
|
|
2737
|
-
decidedInDivergence(state, plan) {
|
|
2738
|
-
if (state.stepsPinned > 0)
|
|
2739
|
-
this.markRetired(state);
|
|
2740
|
-
logLine("replay.handover", {
|
|
2741
|
-
scenario: state.scenarioId ?? undefined,
|
|
2742
|
-
kind: "decision",
|
|
2743
|
-
step: `${plan.stepCount}/${plan.totalSteps}`,
|
|
2744
|
-
via: "divergence",
|
|
2745
|
-
ran: state.stepsPinned,
|
|
2746
|
-
outcome: state.outcome,
|
|
2747
|
-
why: "the plan ran the steps before its decision — the agent decides which way applies and continues",
|
|
2465
|
+
: h.kind === "known_bad_step"
|
|
2466
|
+
? "the next step has failed too often — the agent continues from here"
|
|
2467
|
+
: h.kind === "nondeterministic_step"
|
|
2468
|
+
? "the next step needs a judgement the plan cannot compute — the agent makes it"
|
|
2469
|
+
: h.stopReason === "concurrent"
|
|
2470
|
+
? "a second call arrived while the pinned step was still running — the agent continues from here"
|
|
2471
|
+
: h.stopReason === "budget" || h.stopReason === "cancelled"
|
|
2472
|
+
? "the run was cut short before this step — the agent continues from here"
|
|
2473
|
+
: h.stopReason === "host_rejected" || h.stopReason === "host_edited"
|
|
2474
|
+
? "the host stopped this step itself — no fault of the plan; the agent continues from here"
|
|
2475
|
+
: h.stopReason === "not_run" || h.stopReason === "uncertain"
|
|
2476
|
+
? "a step produced no result here — the agent continues from here"
|
|
2477
|
+
: "a step broke — the agent continues from here",
|
|
2748
2478
|
});
|
|
2749
2479
|
}
|
|
2750
2480
|
/**
|
|
2751
|
-
* The steps before
|
|
2752
|
-
*
|
|
2753
|
-
*
|
|
2754
|
-
*
|
|
2481
|
+
* The steps before position `bound` that ran, 1-based, and the ones their
|
|
2482
|
+
* tests left out (AC5) — the plan cursor's own, and a divergence bundle's
|
|
2483
|
+
* (`extra`). `left` is empty when no step was left out; `ran` is then not
|
|
2484
|
+
* filled, since every step before `bound` ran.
|
|
2755
2485
|
*/
|
|
2756
2486
|
ranBefore(plan, bound, extra = []) {
|
|
2757
|
-
const left = [...(plan?.leftOut() ?? []), ...extra].filter((l) => l.
|
|
2487
|
+
const left = [...(plan?.leftOut() ?? []), ...extra].filter((l) => l.position < bound);
|
|
2758
2488
|
if (left.length === 0)
|
|
2759
2489
|
return { ran: [], left };
|
|
2760
|
-
const out = new Set();
|
|
2761
|
-
for (const l of left)
|
|
2762
|
-
for (let i = l.flatIndex; i < Math.min(l.until, bound); i += 1)
|
|
2763
|
-
out.add(i);
|
|
2490
|
+
const out = new Set(left.map((l) => l.position));
|
|
2764
2491
|
const ran = [];
|
|
2765
2492
|
for (let i = 0; i < bound; i += 1)
|
|
2766
2493
|
if (!out.has(i))
|
|
@@ -2771,8 +2498,7 @@ export class ReplayController {
|
|
|
2771
2498
|
const plan = state.plan;
|
|
2772
2499
|
// The steps before the stop — or the bound a caller gives — less the ones
|
|
2773
2500
|
// their tests left out (AC5): a note must never say a step ran that did not.
|
|
2774
|
-
|
|
2775
|
-
const bound = h.ran ?? (h.ranThisStep || h.stopReason === "result_map" ? h.position + 1 : h.position);
|
|
2501
|
+
const bound = h.ran ?? (h.ranThisStep ? h.position + 1 : h.position);
|
|
2776
2502
|
const before = this.ranBefore(plan, bound, h.leftOut);
|
|
2777
2503
|
const skipped = before.left.length > 0;
|
|
2778
2504
|
const executed = skipped ? before.ran.length : bound;
|
|
@@ -2781,50 +2507,31 @@ export class ReplayController {
|
|
|
2781
2507
|
: before.ran.filter((n) => n <= h.executedAbove).length;
|
|
2782
2508
|
// A step its host stopped was dealt with there — refused, answered, or
|
|
2783
2509
|
// run with a person's arguments — so it is not among what is left to do.
|
|
2784
|
-
// Nor is the step a test left out, after which a sub-task's results could
|
|
2785
|
-
// not go back (`result_map`, AC5).
|
|
2786
2510
|
const handled = h.ranThisStep === true ||
|
|
2787
2511
|
h.stopReason === "host_rejected" ||
|
|
2788
|
-
h.stopReason === "host_edited"
|
|
2789
|
-
h.stopReason === "result_map";
|
|
2790
|
-
// A chain that holds a decision (the service's segmentsbreakdown.md SB-18)
|
|
2791
|
-
// keeps the decision note's protections in every note, even when the plan
|
|
2792
|
-
// stops earlier for another reason: no intent, no parameters, and no step
|
|
2793
|
-
// from the decision on. Each points at the half the recording took, and an
|
|
2794
|
-
// agent that follows it there runs that half's segment: the plan decided.
|
|
2795
|
-
const decisionAt = plan ? plan.decisionIndex() : -1;
|
|
2512
|
+
h.stopReason === "host_edited";
|
|
2796
2513
|
let params;
|
|
2797
2514
|
try {
|
|
2798
|
-
params =
|
|
2515
|
+
params = plan?.parameters;
|
|
2799
2516
|
}
|
|
2800
2517
|
catch {
|
|
2801
2518
|
params = undefined;
|
|
2802
2519
|
}
|
|
2803
2520
|
return buildHandoverNote({
|
|
2804
|
-
intent:
|
|
2521
|
+
intent: plan?.intent ?? "",
|
|
2805
2522
|
kind: h.kind,
|
|
2806
2523
|
stepIndex: h.position,
|
|
2807
2524
|
stepToolName: h.step?.toolName ?? undefined,
|
|
2808
2525
|
failureCount: h.step?.failureCount,
|
|
2809
2526
|
error: h.error,
|
|
2810
2527
|
stopReason: h.stopReason,
|
|
2811
|
-
unusable: state.flatStop
|
|
2812
|
-
? {
|
|
2813
|
-
reason: state.flatStop.reason,
|
|
2814
|
-
callIntent: state.flatStop.callIntent,
|
|
2815
|
-
segmentId: state.flatStop.segmentId,
|
|
2816
|
-
ownerScenarioId: state.flatStop.scenarioId,
|
|
2817
|
-
}
|
|
2818
|
-
: undefined,
|
|
2819
|
-
ownerScenarioId: h.entry && h.entry.depth > 0 ? h.entry.scenarioId : undefined,
|
|
2820
|
-
inSubTask: h.entry && h.entry.depth > 0 ? (h.entry.frame.callIntent ?? null) : null,
|
|
2821
2528
|
executed: Math.max(0, Math.min(executed, plan?.totalSteps ?? executed)),
|
|
2822
2529
|
...(skipped
|
|
2823
2530
|
? {
|
|
2824
2531
|
ranSteps: before.ran,
|
|
2825
2532
|
leftOut: before.left.map((l) => ({
|
|
2826
|
-
from: l.
|
|
2827
|
-
to:
|
|
2533
|
+
from: l.position + 1,
|
|
2534
|
+
to: l.position + 1,
|
|
2828
2535
|
toolName: l.toolName,
|
|
2829
2536
|
text: l.text,
|
|
2830
2537
|
})),
|
|
@@ -2836,17 +2543,12 @@ export class ReplayController {
|
|
|
2836
2543
|
totalSteps: plan?.totalSteps ?? state.stepsPlanned,
|
|
2837
2544
|
// A step that runs only under a test is marked so: the list must not
|
|
2838
2545
|
// read as steps this request needs (AC5).
|
|
2839
|
-
remainingTools: plan
|
|
2840
|
-
? plan.remainingFrom(handled ? h.position + 1 : h.position, decisionAt >= 0 ? decisionAt : undefined)
|
|
2841
|
-
: [],
|
|
2546
|
+
remainingTools: plan ? plan.remainingFrom(handled ? h.position + 1 : h.position) : [],
|
|
2842
2547
|
params,
|
|
2843
2548
|
delivery: h.delivery,
|
|
2844
2549
|
whenText: h.whenText ?? null,
|
|
2845
|
-
subTask: h.subTask ?? null,
|
|
2846
2550
|
// "None was needed" is the note's own wording for the end of the plan.
|
|
2847
2551
|
...(h.noneNeeded ? { stepIndex: plan?.totalSteps ?? state.stepsPlanned } : {}),
|
|
2848
|
-
// The request's own words for the choice (SB-18); never the service's.
|
|
2849
|
-
...(h.kind === "decision" ? { request: state.request ?? null } : {}),
|
|
2850
2552
|
});
|
|
2851
2553
|
}
|
|
2852
2554
|
/**
|
|
@@ -2873,11 +2575,12 @@ export class ReplayController {
|
|
|
2873
2575
|
}
|
|
2874
2576
|
const mcp = parseQualifiedName(name);
|
|
2875
2577
|
if (!mcp) {
|
|
2876
|
-
// An allow-listed built-in runs *here*,
|
|
2578
|
+
// An allow-listed built-in runs *here*, from the session's own
|
|
2877
2579
|
// project directory (replay/builtins.ts). Everything else about it —
|
|
2878
2580
|
// no session cwd (`bir replay`, an older server), off the list, no
|
|
2879
|
-
// runner-side implementation, a path
|
|
2880
|
-
// command `BIR_BASH_ALLOW` does not
|
|
2581
|
+
// runner-side implementation, a path outside the folders
|
|
2582
|
+
// `BIR_PATH_ALLOW` names, a `Bash` command `BIR_BASH_ALLOW` does not
|
|
2583
|
+
// allow — stays the old contract:
|
|
2881
2584
|
// `not_run`, so the session can do the step itself or its recorded
|
|
2882
2585
|
// output can stand in (`BIR_REPLAY_RECORDED_OUTPUTS=1`).
|
|
2883
2586
|
if (this.builtinTools.has(name) && state.sessionCwd) {
|
|
@@ -2939,6 +2642,9 @@ export class ReplayController {
|
|
|
2939
2642
|
cwd,
|
|
2940
2643
|
timeoutMs,
|
|
2941
2644
|
bashAllow: this.bashAllow,
|
|
2645
|
+
// Any folder unless `BIR_PATH_ALLOW` names some, as the agent's own
|
|
2646
|
+
// tools would reach it (up to 0.3.20, the project directory alone).
|
|
2647
|
+
pathAllow: this.pathAllow,
|
|
2942
2648
|
// The plan's Bash steps share a working directory, as the host's do.
|
|
2943
2649
|
bashCwd: state.bashCwd,
|
|
2944
2650
|
onBashCwd: (pwd) => {
|
|
@@ -2981,8 +2687,8 @@ export class ReplayController {
|
|
|
2981
2687
|
return undefined;
|
|
2982
2688
|
// A step a test left out passes over its recorded output, unless a
|
|
2983
2689
|
// correction added it (AC5, source-run.ts).
|
|
2984
|
-
return Object.assign((
|
|
2985
|
-
leftOut: (
|
|
2690
|
+
return Object.assign((step) => source.outputFor(step), {
|
|
2691
|
+
leftOut: (step) => source.leftOut(step),
|
|
2986
2692
|
});
|
|
2987
2693
|
}
|
|
2988
2694
|
/**
|
|
@@ -3061,7 +2767,7 @@ export class ReplayController {
|
|
|
3061
2767
|
}
|
|
3062
2768
|
if (!opts.adHoc) {
|
|
3063
2769
|
const schema = plan.paramsSchema;
|
|
3064
|
-
const planned = plan.
|
|
2770
|
+
const planned = plan.allSteps().slice(0, plan.stepCount);
|
|
3065
2771
|
const blocked = blockingTargets(targetKeys(schema), schema, planned, plan.paramsLogic);
|
|
3066
2772
|
if (blocked.length > 0) {
|
|
3067
2773
|
return {
|
|
@@ -3078,7 +2784,7 @@ export class ReplayController {
|
|
|
3078
2784
|
const missingAgain = state.rederivedMissing;
|
|
3079
2785
|
state.rederivedMissing = undefined;
|
|
3080
2786
|
if (missingAgain && !opts.adHoc) {
|
|
3081
|
-
const planned = plan.
|
|
2787
|
+
const planned = plan.allSteps().slice(0, plan.stepCount);
|
|
3082
2788
|
const blocked = blockingTargets(missingAgain, plan.paramsSchema, planned, plan.paramsLogic);
|
|
3083
2789
|
if (blocked.length > 0) {
|
|
3084
2790
|
return {
|
|
@@ -3091,7 +2797,7 @@ export class ReplayController {
|
|
|
3091
2797
|
}
|
|
3092
2798
|
const sink = opts.adHoc
|
|
3093
2799
|
? undefined
|
|
3094
|
-
: sinkOfPlan(plan.
|
|
2800
|
+
: sinkOfPlan(plan.allSteps().slice(0, plan.stepCount), state.host?.sinks);
|
|
3095
2801
|
if (sink) {
|
|
3096
2802
|
const bad = unsafeParameter(plan.derivedValues, plan.paramsSchema, sink);
|
|
3097
2803
|
if (bad) {
|
|
@@ -3190,12 +2896,7 @@ export class ReplayController {
|
|
|
3190
2896
|
*/
|
|
3191
2897
|
recordStep(state, info) {
|
|
3192
2898
|
const status = info.outcome === "executed" ? "ok" : info.outcome;
|
|
3193
|
-
|
|
3194
|
-
// in one plan both have a step 0, and one would otherwise overwrite the
|
|
3195
|
-
// other's verdict (segmented.md R-CALL-13).
|
|
3196
|
-
const key = info.entry
|
|
3197
|
-
? `${info.entry.scenarioId}#${info.entry.stepIndex}`
|
|
3198
|
-
: `#${info.step.stepIndex}`;
|
|
2899
|
+
const key = `#${info.step.stepIndex}`;
|
|
3199
2900
|
if (state.stepResults.size >= MAX_STEP_RESULTS && !state.stepResults.has(key)) {
|
|
3200
2901
|
return;
|
|
3201
2902
|
}
|
|
@@ -3204,29 +2905,19 @@ export class ReplayController {
|
|
|
3204
2905
|
? `${info.error.slice(0, MAX_STEP_ERROR_CHARS)}…`
|
|
3205
2906
|
: info.error
|
|
3206
2907
|
: undefined;
|
|
3207
|
-
// Payloads only for a step of the reported scenario's own chain. The
|
|
3208
|
-
// console still files an execution's verdicts by step number alone
|
|
3209
|
-
// (plan-services.md W1.9), and a called segment numbers its steps from
|
|
3210
|
-
// zero as well: a called step's call would be shown under the caller's
|
|
3211
|
-
// step of the same number — the very mix-up the payloads are there to end.
|
|
3212
|
-
const own = !info.entry || info.entry.depth === 0;
|
|
3213
2908
|
// A step whose input logic threw never had an input: the `{}` the plan
|
|
3214
2909
|
// reports there is a placeholder, not a call with no arguments. Nor had a
|
|
3215
2910
|
// step its test left out or stopped in front of (AC5).
|
|
3216
2911
|
const noCall = info.stage === "tool_input_logic" || info.stage === "when_logic" || status === "not_taken";
|
|
3217
|
-
const sent =
|
|
3218
|
-
const received =
|
|
2912
|
+
const sent = !noCall ? info.input : undefined;
|
|
2913
|
+
const received = !noCall ? info.output : undefined;
|
|
3219
2914
|
// The call, kept whole for the turn's newest run (AC2): only one that
|
|
3220
2915
|
// really went out — it answered, or it answered with an error.
|
|
3221
2916
|
if (state.keepCalls &&
|
|
3222
2917
|
!noCall &&
|
|
3223
2918
|
info.input !== undefined &&
|
|
3224
2919
|
(status === "ok" || (status === "failed" && info.stage === "tool_call"))) {
|
|
3225
|
-
|
|
3226
|
-
// makes two calls of each of its steps, and both are the turn's (AC5
|
|
3227
|
-
// review). A frame is one call of one sub-task.
|
|
3228
|
-
const callKey = info.entry ? `${info.entry.frame.id}#${info.entry.stepIndex}` : key;
|
|
3229
|
-
this.keepCall(state, callKey, info, status === "failed");
|
|
2920
|
+
this.keepCall(state, key, info, status === "failed");
|
|
3230
2921
|
}
|
|
3231
2922
|
state.stepResults.set(key, {
|
|
3232
2923
|
stepIndex: info.step.stepIndex,
|
|
@@ -3242,10 +2933,6 @@ export class ReplayController {
|
|
|
3242
2933
|
// The tool is gone from its server: the service repairs the plan without
|
|
3243
2934
|
// waiting out the grace (plan-services.md D8, kind 2).
|
|
3244
2935
|
toolMissing: info.toolMissing || undefined,
|
|
3245
|
-
// A verdict about a called segment's step is counted on that segment, at
|
|
3246
|
-
// the revision it was served (R-CALL-13).
|
|
3247
|
-
scenarioId: info.entry && info.entry.depth > 0 ? info.entry.scenarioId : undefined,
|
|
3248
|
-
chainRevision: info.entry && info.entry.depth > 0 ? info.entry.frame.chainRevision : undefined,
|
|
3249
2936
|
});
|
|
3250
2937
|
}
|
|
3251
2938
|
/**
|
|
@@ -3264,14 +2951,11 @@ export class ReplayController {
|
|
|
3264
2951
|
? serializeCapped(redact(info.input ?? {}))
|
|
3265
2952
|
: reportPayload(serializeCapped(redact(info.input ?? {})), true);
|
|
3266
2953
|
const answer = info.output === undefined ? undefined : room ? heldPayload(info.output) : reportPayload(info.output);
|
|
3267
|
-
const inSegment = info.entry && info.entry.depth > 0;
|
|
3268
2954
|
const call = {
|
|
3269
|
-
scenarioId:
|
|
2955
|
+
scenarioId: state.scenarioId ?? "",
|
|
3270
2956
|
stepIndex: info.step.stepIndex,
|
|
3271
|
-
chainRevision:
|
|
3272
|
-
|
|
3273
|
-
: (state.chainRevision ?? null),
|
|
3274
|
-
toolName: info.step.toolName ?? "",
|
|
2957
|
+
chainRevision: state.chainRevision ?? null,
|
|
2958
|
+
toolName: info.step.toolName,
|
|
3275
2959
|
input,
|
|
3276
2960
|
...(failed
|
|
3277
2961
|
? { error: info.error ? capText(redactText(info.error), DEFAULT_MAX_PAYLOAD_BYTES) : answer }
|
|
@@ -3295,8 +2979,6 @@ export class ReplayController {
|
|
|
3295
2979
|
if (info.outcome === "failed") {
|
|
3296
2980
|
logLine("replay.step_failed", {
|
|
3297
2981
|
n: info.step.stepIndex,
|
|
3298
|
-
scenario: info.entry && info.entry.depth > 0 ? info.entry.scenarioId : undefined,
|
|
3299
|
-
frame: info.entry?.frame.id,
|
|
3300
2982
|
tool: info.step.toolName ?? undefined,
|
|
3301
2983
|
stage: info.stage,
|
|
3302
2984
|
why: info.stage === "tool_call"
|
|
@@ -3328,8 +3010,7 @@ export class ReplayController {
|
|
|
3328
3010
|
const text = testWords(info.step.when?.text);
|
|
3329
3011
|
logLine("replay.step_not_taken", {
|
|
3330
3012
|
n: info.step.stepIndex,
|
|
3331
|
-
tool: info.step.toolName
|
|
3332
|
-
scenario: info.entry && info.entry.depth > 0 ? info.entry.scenarioId : undefined,
|
|
3013
|
+
tool: info.step.toolName,
|
|
3333
3014
|
why: `its test said this request does not need it${text ? ` — only for when ${text}` : ""}; left out, and the plan goes on`,
|
|
3334
3015
|
});
|
|
3335
3016
|
return;
|
|
@@ -3353,10 +3034,17 @@ export class ReplayController {
|
|
|
3353
3034
|
this.upgrade(state, outcome);
|
|
3354
3035
|
// Only a plan that *finished* stops the clock (segmented.md R-MONEY-3). A
|
|
3355
3036
|
// hand-over or a divergence takes no mark: the agent's next tokens are the
|
|
3356
|
-
// consequence of this plan stopping, and belong to its bill.
|
|
3357
|
-
|
|
3358
|
-
|
|
3359
|
-
|
|
3037
|
+
// consequence of this plan stopping, and belong to its bill.
|
|
3038
|
+
if (completing) {
|
|
3039
|
+
state.retiredAt = Date.now();
|
|
3040
|
+
try {
|
|
3041
|
+
state.retiredMark = state.markUsage?.();
|
|
3042
|
+
}
|
|
3043
|
+
catch {
|
|
3044
|
+
// An unreadable transcript leaves the mark undefined, and the window
|
|
3045
|
+
// falls back to the next arm or the end of the turn.
|
|
3046
|
+
}
|
|
3047
|
+
}
|
|
3360
3048
|
state.retired = true;
|
|
3361
3049
|
state.pinned.clear();
|
|
3362
3050
|
// A note naming a step to call next is void once the plan is: after a
|
|
@@ -3368,21 +3056,6 @@ export class ReplayController {
|
|
|
3368
3056
|
// A call waiting for a pin to thread has nothing left to wait for.
|
|
3369
3057
|
this.notifyPins(state);
|
|
3370
3058
|
}
|
|
3371
|
-
/**
|
|
3372
|
-
* Where the plan's work ended, for its bill: now, and the transcript's
|
|
3373
|
-
* watermark now. Taken at a completing retire, and at a decision hand-over
|
|
3374
|
-
* after a step ran (the service's segmentsbreakdown.md SB-26).
|
|
3375
|
-
*/
|
|
3376
|
-
markRetired(state) {
|
|
3377
|
-
state.retiredAt = Date.now();
|
|
3378
|
-
try {
|
|
3379
|
-
state.retiredMark = state.markUsage?.();
|
|
3380
|
-
}
|
|
3381
|
-
catch {
|
|
3382
|
-
// An unreadable transcript leaves the mark undefined, and the window
|
|
3383
|
-
// falls back to the next arm or the end of the turn.
|
|
3384
|
-
}
|
|
3385
|
-
}
|
|
3386
3059
|
}
|
|
3387
3060
|
/** Every target of a schema. A parameter with no kind is one (R-PARAM-3). */
|
|
3388
3061
|
function targetKeys(schema) {
|
|
@@ -3520,58 +3193,29 @@ function heldPayload(text) {
|
|
|
3520
3193
|
}
|
|
3521
3194
|
return capText(text, DEFAULT_MAX_PAYLOAD_BYTES, redactText);
|
|
3522
3195
|
}
|
|
3523
|
-
/** Whether
|
|
3524
|
-
function underTest(
|
|
3525
|
-
|
|
3526
|
-
return true;
|
|
3527
|
-
for (let f = entry.frame; f; f = f.parent)
|
|
3528
|
-
if (f.when)
|
|
3529
|
-
return true;
|
|
3530
|
-
return false;
|
|
3196
|
+
/** Whether a step runs only under a test (AC5). */
|
|
3197
|
+
function underTest(step) {
|
|
3198
|
+
return Boolean(step.when);
|
|
3531
3199
|
}
|
|
3532
|
-
/** The tests of a
|
|
3533
|
-
function testLines(
|
|
3534
|
-
|
|
3535
|
-
|
|
3536
|
-
const { startsAt } = frameEdges(entries);
|
|
3537
|
-
const lines = entries
|
|
3538
|
-
.map((e, i) => {
|
|
3539
|
-
const own = e.step.when;
|
|
3540
|
-
const call = (startsAt.get(i) ?? []).find((f) => f.when)?.when;
|
|
3541
|
-
const t = own ?? call;
|
|
3542
|
-
return t ? `${i}:${call && !own ? "call/" : ""}${t.otherwise}` : undefined;
|
|
3543
|
-
})
|
|
3200
|
+
/** The tests of a chain, by position, for the `plan.armed` line (AC5). */
|
|
3201
|
+
function testLines(steps) {
|
|
3202
|
+
const lines = steps
|
|
3203
|
+
.map((s, i) => (s.when ? `${i}:${s.when.otherwise}` : undefined))
|
|
3544
3204
|
.filter((s) => s !== undefined);
|
|
3545
3205
|
return lines.length > 0 ? lines.join(",") : undefined;
|
|
3546
3206
|
}
|
|
3547
3207
|
/**
|
|
3548
|
-
*
|
|
3549
|
-
*
|
|
3550
|
-
*
|
|
3551
|
-
* call row that opened its frame does. Reporting the segment's own index would
|
|
3552
|
-
* line the execution up against a step the caller's card does not have.
|
|
3208
|
+
* Which kind of stop a plan's end is (plan-services.md D8): the stopping
|
|
3209
|
+
* step's own served reason wins — the calculation's verdict is more exact
|
|
3210
|
+
* than a counter — then a parked step.
|
|
3553
3211
|
*/
|
|
3554
|
-
function
|
|
3555
|
-
|
|
3556
|
-
return entry?.stepIndex;
|
|
3557
|
-
let frame = entry.frame;
|
|
3558
|
-
while (frame.parent && frame.depth > 1)
|
|
3559
|
-
frame = frame.parent;
|
|
3560
|
-
return frame.callerStepIndex;
|
|
3561
|
-
}
|
|
3562
|
-
/** The calls in a served chain, for the `plan.armed` line (R-CALL-32). */
|
|
3563
|
-
function callLines(scenario) {
|
|
3564
|
-
const calls = (scenario.steps ?? []).filter((s) => s.kind === "segment");
|
|
3565
|
-
if (calls.length === 0)
|
|
3566
|
-
return undefined;
|
|
3567
|
-
return calls
|
|
3568
|
-
.map((s) => `${s.segmentId ?? "-"}:${s.unusable ?? s.resolution?.state ?? "resolved"}`)
|
|
3569
|
-
.join(",");
|
|
3212
|
+
function stopKindOf(step) {
|
|
3213
|
+
return step?.stop?.kind === "nondeterministic" ? "nondeterministic_step" : "known_bad_step";
|
|
3570
3214
|
}
|
|
3571
|
-
/** The served stops in a
|
|
3572
|
-
function stopLines(
|
|
3573
|
-
const lines =
|
|
3574
|
-
.map((
|
|
3215
|
+
/** The served stops in a chain, for the `plan.armed` line (plan-services.md D8). */
|
|
3216
|
+
function stopLines(steps) {
|
|
3217
|
+
const lines = steps
|
|
3218
|
+
.map((s, i) => (s.stop ? `${i}:${s.stop.kind}${s.stop.repairDue ? "/repair" : ""}` : undefined))
|
|
3575
3219
|
.filter((s) => s !== undefined);
|
|
3576
3220
|
return lines.length > 0 ? lines.join(",") : undefined;
|
|
3577
3221
|
}
|
|
@@ -3596,8 +3240,7 @@ function isKnownBad(step, maxStepFailures) {
|
|
|
3596
3240
|
*
|
|
3597
3241
|
* A step that carries `stop` was decided by the service — `null` means run it,
|
|
3598
3242
|
* even when its count is past the limit: its retry is due. A step without the
|
|
3599
|
-
* field came from an older service, and the old counter rule decides it.
|
|
3600
|
-
* can sit in one chain, since a called segment is served on its own terms.
|
|
3243
|
+
* field came from an older service, and the old counter rule decides it.
|
|
3601
3244
|
*/
|
|
3602
3245
|
export function stopIndex(steps, maxStepFailures) {
|
|
3603
3246
|
const i = steps.findIndex((s) => s.stop !== undefined ? s.stop !== null : isKnownBad(s, maxStepFailures));
|