@basein/runner 0.3.21 → 0.3.22
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/bir-hooks.d.ts +7 -16
- package/dist/bin/bir-hooks.js +7 -16
- package/dist/bin/bir-scenario.js +1 -1
- package/dist/bin/bir.js +1 -21
- package/dist/bin/investigate.d.ts +0 -9
- package/dist/bin/investigate.js +14 -42
- package/dist/bin/scenario-edit.js +13 -44
- package/dist/control/recorder-main.d.ts +0 -15
- package/dist/control/recorder-main.js +7 -39
- package/dist/control/server.d.ts +24 -137
- package/dist/control/server.js +101 -995
- package/dist/record/recorder.d.ts +11 -142
- package/dist/record/recorder.js +0 -3
- package/dist/record/remote-recorder.d.ts +1 -9
- package/dist/record/remote-recorder.js +3 -49
- package/dist/record/switchable-recorder.js +0 -2
- package/dist/replay/bundle.d.ts +2 -2
- package/dist/replay/bundle.js +3 -3
- package/dist/replay/controller.d.ts +19 -123
- package/dist/replay/controller.js +151 -518
- package/dist/replay/derive.d.ts +0 -10
- package/dist/replay/derive.js +0 -3
- package/dist/replay/handover.d.ts +2 -37
- package/dist/replay/handover.js +38 -106
- package/dist/replay/logic.d.ts +0 -11
- package/dist/replay/logic.js +0 -24
- package/dist/replay/plan.d.ts +43 -163
- package/dist/replay/plan.js +144 -556
- package/dist/replay/safety.d.ts +4 -5
- package/dist/replay/safety.js +7 -7
- package/dist/replay/source-run.d.ts +15 -24
- package/dist/replay/source-run.js +25 -58
- package/dist/replay/targets.d.ts +11 -21
- package/dist/replay/targets.js +30 -69
- package/dist/replay/types.d.ts +9 -96
- package/dist/replay/types.js +3 -16
- package/dist/util/service-headers.d.ts +1 -2
- package/dist/util/service-headers.js +1 -2
- package/docs/calculatedReplay.md +17 -74
- package/docs/calculatedReplayGuide.md +9 -30
- package/package.json +1 -1
- package/production.md +5 -35
- package/dist/replay/condition-words.d.ts +0 -28
- package/dist/replay/condition-words.js +0 -59
- package/dist/replay/flatten.d.ts +0 -161
- package/dist/replay/flatten.js +0 -222
- package/dist/replay/reads-only.d.ts +0 -29
- package/dist/replay/reads-only.js +0 -261
|
@@ -29,7 +29,6 @@ import { PREVIEW_BUDGET_MS, RecordedValueError, ScenarioReplayPlan, StepRunError
|
|
|
29
29
|
import { PRICING_VERSION } from "./pricing.js";
|
|
30
30
|
import { hasShellStep, sinkOf, sinkOfPlan, unsafeParameter } from "./safety.js";
|
|
31
31
|
import { SourceRunOutputs } from "./source-run.js";
|
|
32
|
-
import { clampToFrameStart, flattenChain, frameEdges, } from "./flatten.js";
|
|
33
32
|
import { blockingTargets, testedOnlyTargets } from "./targets.js";
|
|
34
33
|
import { isNoResult, toolResultError } from "./tool-error.js";
|
|
35
34
|
import { OUTCOME_RANK, hasTargets, isReadyScenario, } from "./types.js";
|
|
@@ -168,7 +167,7 @@ function handedOver(note) {
|
|
|
168
167
|
}
|
|
169
168
|
export const DEFAULT_INTENT_MATCH = {
|
|
170
169
|
enabled: true,
|
|
171
|
-
budgetMs:
|
|
170
|
+
budgetMs: 1_500,
|
|
172
171
|
maxPerTurn: 3,
|
|
173
172
|
maxRequestsPerTurn: 40,
|
|
174
173
|
};
|
|
@@ -291,18 +290,6 @@ export class ReplayController {
|
|
|
291
290
|
? builtins
|
|
292
291
|
: new Set([...builtins].filter(reachable));
|
|
293
292
|
}
|
|
294
|
-
/**
|
|
295
|
-
* Whether a handed-out segment may actually arm (segmented.md R-OUT-10).
|
|
296
|
-
*
|
|
297
|
-
* Only when the option says so. The recorder passes it on unless
|
|
298
|
-
* `BIR_SEGMENT_ARM=0` (from 0.3.19; the service's segmentsbreakdown.md
|
|
299
|
-
* SB-21); a controller built without it stays observe-only, asks for no
|
|
300
|
-
* segment and arms none. The server enforces the same thing from its side
|
|
301
|
-
* (R-OUT-9).
|
|
302
|
-
*/
|
|
303
|
-
get segmentArm() {
|
|
304
|
-
return this.opts.segmentArm === true;
|
|
305
|
-
}
|
|
306
293
|
/** The plan's own delivery vehicle is never a scenario step. */
|
|
307
294
|
isDirectTool(toolName) {
|
|
308
295
|
return toolName === DIRECT_TOOL_NAME;
|
|
@@ -362,13 +349,6 @@ export class ReplayController {
|
|
|
362
349
|
reported: false,
|
|
363
350
|
retired: false,
|
|
364
351
|
armedBy,
|
|
365
|
-
// An older server sends no `kind`, and a whole-run scenario is the safe
|
|
366
|
-
// reading: it is what every hand-out was before segments existed.
|
|
367
|
-
kind: match.kind ?? "scenario",
|
|
368
|
-
segment: match.segment,
|
|
369
|
-
key: match.key,
|
|
370
|
-
verified: match.verified,
|
|
371
|
-
sharedWith: [],
|
|
372
352
|
};
|
|
373
353
|
}
|
|
374
354
|
/**
|
|
@@ -397,11 +377,6 @@ export class ReplayController {
|
|
|
397
377
|
*/
|
|
398
378
|
async arm(match, prompt, wrapped, armedBy = "prompt", ctx = {}, host) {
|
|
399
379
|
const state = this.blankState(match, wrapped, armedBy, host);
|
|
400
|
-
// A decision note quotes the request (SB-18); a plan armed by intent was
|
|
401
|
-
// given the step's intent here, and its control server sets the request.
|
|
402
|
-
if (armedBy === "prompt")
|
|
403
|
-
state.request = prompt;
|
|
404
|
-
const isSegment = state.kind === "segment";
|
|
405
380
|
const decline = (why, code) => {
|
|
406
381
|
state.declined = code;
|
|
407
382
|
logLine("replay.decision", {
|
|
@@ -410,19 +385,10 @@ export class ReplayController {
|
|
|
410
385
|
run: match.runId,
|
|
411
386
|
scenario: match.scenarioId ?? undefined,
|
|
412
387
|
similarity: match.similarity.toFixed(3),
|
|
413
|
-
kind: state.kind,
|
|
414
|
-
key: state.key,
|
|
415
|
-
verified: state.verified?.verdict,
|
|
416
388
|
why,
|
|
417
389
|
});
|
|
418
390
|
return state;
|
|
419
391
|
};
|
|
420
|
-
// The server never sends a segment to a runner that did not ask for one
|
|
421
|
-
// (R-OUT-9). One arriving anyway is declined here rather than run: an
|
|
422
|
-
// observe-only period that quietly armed something would be worthless.
|
|
423
|
-
if (isSegment && !this.segmentArm) {
|
|
424
|
-
return decline("segment hand-out while BIR_SEGMENT_ARM is off", "replay_disabled");
|
|
425
|
-
}
|
|
426
392
|
// Gate 2 — replay enabled.
|
|
427
393
|
if (!this.enabled)
|
|
428
394
|
return decline("BIR_REPLAY=0", "replay_disabled");
|
|
@@ -452,65 +418,24 @@ export class ReplayController {
|
|
|
452
418
|
}
|
|
453
419
|
// Gate 4 — similarity — is gone. A hand-out is the service's decision that
|
|
454
420
|
// this scenario does what the prompt asks: its match threshold, the
|
|
455
|
-
// same-request question under it, the intent pass
|
|
456
|
-
//
|
|
457
|
-
//
|
|
458
|
-
//
|
|
459
|
-
//
|
|
460
|
-
|
|
461
|
-
// Flatten the chain before the remaining gates, so every one of them judges
|
|
462
|
-
// the steps that will *really* run — the caller's own and, in order, those
|
|
463
|
-
// of every segment it calls (segmented.md R-CALL-29, R-CALL-30).
|
|
464
|
-
let flat;
|
|
465
|
-
try {
|
|
466
|
-
flat = flattenChain(scenario, {
|
|
467
|
-
range: match.segment
|
|
468
|
-
? { stepFrom: match.segment.stepFrom, stepTo: match.segment.stepTo }
|
|
469
|
-
: null,
|
|
470
|
-
});
|
|
471
|
-
}
|
|
472
|
-
catch (err) {
|
|
473
|
-
// A payload we cannot lay out is a payload we cannot run. It costs the
|
|
474
|
-
// turn nothing: the agent does the task (R-FALL-12).
|
|
475
|
-
logLine("replay.flatten_failed", { scenario: scenario.id, error: errText(err) });
|
|
476
|
-
return decline("the chain could not be flattened", "flatten_failed");
|
|
477
|
-
}
|
|
478
|
-
const entries = flat.entries;
|
|
479
|
-
if (entries.length === 0)
|
|
480
|
-
return decline("the chain has no runnable step", "not_ready");
|
|
421
|
+
// same-request question under it, the intent pass. Scoring it again here
|
|
422
|
+
// declined matches the service had made (a checked 0.85, a task line at
|
|
423
|
+
// 0.91), so every match now steers. What a plan cannot do — a step that
|
|
424
|
+
// keeps failing, a tool nobody wrapped, a target the prompt never named —
|
|
425
|
+
// the gates below still stop.
|
|
426
|
+
const steps = scenario.steps;
|
|
481
427
|
// Gate 5 — steps the plan must stop in front of (fallbk.md D3,
|
|
482
428
|
// plan-services.md D8): a step the service parked — failed more than its
|
|
483
|
-
// limit, retry not due — or one the calculation marked non-deterministic
|
|
484
|
-
|
|
485
|
-
// answer stops the plan the same way, and the earliest of them wins.
|
|
486
|
-
const badAt = stopIndex(entries.map((e) => e.step), match.fallback?.maxStepFailures);
|
|
487
|
-
const rawStop = [flat.stopAt, badAt].filter((n) => typeof n === "number");
|
|
488
|
-
// Clamped to the start of the outermost called frame it falls in: a plan
|
|
489
|
-
// never runs half a sub-task and leaves its result mapping unevaluated.
|
|
490
|
-
const stopAt = rawStop.length > 0 ? clampToFrameStart(entries, Math.min(...rawStop)) : undefined;
|
|
429
|
+
// limit, retry not due — or one the calculation marked non-deterministic.
|
|
430
|
+
const stopAt = stopIndex(steps, match.fallback?.maxStepFailures);
|
|
491
431
|
if (stopAt === 0) {
|
|
492
|
-
const
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
const judgement = stopping.stop?.kind === "nondeterministic" || decisionFirst;
|
|
500
|
-
const why = unusable
|
|
501
|
-
? `its first step is a call that cannot run: ${unusable.reason}`
|
|
502
|
-
: decisionFirst
|
|
503
|
-
? "its first step is a decision the agent makes — the agent does this task"
|
|
504
|
-
: judgement
|
|
505
|
-
? "its first step needs a judgement the plan cannot compute — the agent does this task"
|
|
506
|
-
: `its first step has failed ${stopping.failureCount ?? 0} times — the agent does this task`;
|
|
507
|
-
return decline(why, unusable
|
|
508
|
-
? "unusable_first_step"
|
|
509
|
-
: judgement
|
|
510
|
-
? "nondeterministic_first_step"
|
|
511
|
-
: "known_bad_first_step");
|
|
512
|
-
}
|
|
513
|
-
const planned = stopAt === undefined ? entries : entries.slice(0, stopAt);
|
|
432
|
+
const stopping = steps[0];
|
|
433
|
+
const judgement = stopping.stop?.kind === "nondeterministic";
|
|
434
|
+
return decline(judgement
|
|
435
|
+
? "its first step needs a judgement the plan cannot compute — the agent does this task"
|
|
436
|
+
: `its first step has failed ${stopping.failureCount ?? 0} times — the agent does this task`, judgement ? "nondeterministic_first_step" : "known_bad_first_step");
|
|
437
|
+
}
|
|
438
|
+
const planned = stopAt === undefined ? steps : steps.slice(0, stopAt);
|
|
514
439
|
// Another host's adapter steps through a plan call by call and knows no
|
|
515
440
|
// tests (AC5): asked for a step its test left out, it is refused, reads
|
|
516
441
|
// that as the plan's end, and the model's own calls after it are booked as
|
|
@@ -530,7 +455,7 @@ export class ReplayController {
|
|
|
530
455
|
// `mcp__fleet__list` is not the runner's to run or a proxy's to reach.
|
|
531
456
|
const allowServers = host ? undefined : this.opts.allowServers;
|
|
532
457
|
const builtinTools = host ? undefined : this.builtinTools;
|
|
533
|
-
const mode = modeFor(planned
|
|
458
|
+
const mode = modeFor(planned, state.wrapped, allowServers, builtinTools);
|
|
534
459
|
if (mode === "none")
|
|
535
460
|
return decline("no step is executable", "coverage");
|
|
536
461
|
// Tool results feed derivation as context, and they are whatever the agent
|
|
@@ -544,7 +469,7 @@ export class ReplayController {
|
|
|
544
469
|
// command nobody asked for.
|
|
545
470
|
let deriveCtx = ctx;
|
|
546
471
|
if (ctx.recentResults?.length && sinkOfPlan(planned, host?.sinks) === "command") {
|
|
547
|
-
deriveCtx = { liveCall: ctx.liveCall, agentDay: ctx.agentDay
|
|
472
|
+
deriveCtx = { liveCall: ctx.liveCall, agentDay: ctx.agentDay };
|
|
548
473
|
logLine("replay.derive_context", {
|
|
549
474
|
scenario: scenario.id,
|
|
550
475
|
shell: hasShellStep(planned, host?.sinks) || undefined,
|
|
@@ -671,14 +596,12 @@ export class ReplayController {
|
|
|
671
596
|
}
|
|
672
597
|
}
|
|
673
598
|
// A plan armed by intent runs inside a task the agent is already doing, so
|
|
674
|
-
// its directive and its bundles say "continue", never "answer the user"
|
|
675
|
-
// (segmented.md R-OUT-13).
|
|
599
|
+
// its directive and its bundles say "continue", never "answer the user".
|
|
676
600
|
state.plan = new ScenarioReplayPlan({
|
|
677
601
|
scenario,
|
|
678
602
|
params,
|
|
679
603
|
mode,
|
|
680
604
|
stopAt,
|
|
681
|
-
entries,
|
|
682
605
|
subTask: armedBy === "intent",
|
|
683
606
|
});
|
|
684
607
|
if (mode === "steer") {
|
|
@@ -697,17 +620,11 @@ export class ReplayController {
|
|
|
697
620
|
await state.plan.ready().catch(() => undefined);
|
|
698
621
|
}
|
|
699
622
|
state.mode = mode;
|
|
700
|
-
state.stepsPlanned =
|
|
623
|
+
state.stepsPlanned = steps.length;
|
|
701
624
|
// The chain this plan was served at: what its own calls are marked with
|
|
702
625
|
// when they are recorded into a redo of the turn (AC2).
|
|
703
626
|
if (typeof scenario.chainRevision === "number")
|
|
704
627
|
state.chainRevision = scenario.chainRevision;
|
|
705
|
-
// Why the plan stops, when a call is what stops it: the note names the
|
|
706
|
-
// sub-task rather than a tool (R-CALL-36).
|
|
707
|
-
state.flatStop =
|
|
708
|
-
typeof stopAt === "number" && flat.stopAt !== undefined && stopAt <= flat.stopAt
|
|
709
|
-
? flat.stopReason
|
|
710
|
-
: undefined;
|
|
711
628
|
// Recorded outputs are read only when they may stand in (opt-in), so a
|
|
712
629
|
// plan never fetches a run it will not show.
|
|
713
630
|
if (this.recordedFallback && this.opts.authUrl && this.opts.authToken) {
|
|
@@ -724,22 +641,16 @@ export class ReplayController {
|
|
|
724
641
|
scenario: scenario.id,
|
|
725
642
|
mode,
|
|
726
643
|
host: host?.app,
|
|
727
|
-
steps:
|
|
644
|
+
steps: steps.length,
|
|
728
645
|
stopsAt: stopAt,
|
|
729
646
|
by: armedBy === "intent" ? "intent" : undefined,
|
|
730
|
-
kind: state.kind,
|
|
731
|
-
key: state.key,
|
|
732
|
-
verified: state.verified?.verdict,
|
|
733
647
|
similarity: match.similarity.toFixed(3),
|
|
734
|
-
// What this chain calls, and what answers each call right now (R-CALL-32).
|
|
735
|
-
calls: callLines(scenario),
|
|
736
|
-
unusable: flat.stopReason?.reason,
|
|
737
648
|
// Which steps the service told this plan to stop in front of, and why.
|
|
738
|
-
stops: stopLines(
|
|
739
|
-
coverage: coverageOf(
|
|
740
|
-
tools:
|
|
741
|
-
// Which steps run only under a test, by
|
|
742
|
-
tests: testLines(
|
|
649
|
+
stops: stopLines(steps),
|
|
650
|
+
coverage: coverageOf(steps, state.wrapped, allowServers, builtinTools).join(","),
|
|
651
|
+
tools: steps.map((s) => s.toolName).join(","),
|
|
652
|
+
// Which steps run only under a test, by position (AC5).
|
|
653
|
+
tests: testLines(steps),
|
|
743
654
|
});
|
|
744
655
|
return state;
|
|
745
656
|
}
|
|
@@ -748,16 +659,16 @@ export class ReplayController {
|
|
|
748
659
|
* stops when that is before its end. For a host that steps through the plan
|
|
749
660
|
* itself and must not have to pick the tool names out of a directive
|
|
750
661
|
* written for a model — a reworded sentence would break every adapter in
|
|
751
|
-
* the field. Built from the same
|
|
662
|
+
* the field. Built from the same steps the directive lists.
|
|
752
663
|
*/
|
|
753
664
|
planData(state) {
|
|
754
665
|
const plan = state.plan;
|
|
755
666
|
if (!plan || state.retired)
|
|
756
667
|
return undefined;
|
|
757
668
|
const steps = plan
|
|
758
|
-
.
|
|
669
|
+
.allSteps()
|
|
759
670
|
.slice(0, plan.stepCount)
|
|
760
|
-
.map((
|
|
671
|
+
.map((step, stepIndex) => ({ stepIndex, toolName: step.toolName }));
|
|
761
672
|
return {
|
|
762
673
|
scenarioId: state.scenarioId,
|
|
763
674
|
mode: state.mode,
|
|
@@ -765,15 +676,6 @@ export class ReplayController {
|
|
|
765
676
|
...(plan.stopsEarly() ? { stopAt: plan.stepCount } : {}),
|
|
766
677
|
};
|
|
767
678
|
}
|
|
768
|
-
/**
|
|
769
|
-
* Whether the plan's steps end in front of a decision (the service's
|
|
770
|
-
* segmentsbreakdown.md SB-6): it runs the steps before it, then hands the
|
|
771
|
-
* task back there. The control server gives such a plan a usage mark, so
|
|
772
|
-
* that it is billed only up to the stop (SB-26).
|
|
773
|
-
*/
|
|
774
|
-
stopsAtDecision(state) {
|
|
775
|
-
return state.plan?.stopEntry()?.step.stop?.kind === "decision";
|
|
776
|
-
}
|
|
777
679
|
/**
|
|
778
680
|
* Take back a plan that armed and has run nothing, as a decline.
|
|
779
681
|
*
|
|
@@ -797,7 +699,6 @@ export class ReplayController {
|
|
|
797
699
|
run: state.matchedRunId,
|
|
798
700
|
scenario: state.scenarioId ?? undefined,
|
|
799
701
|
similarity: state.similarity.toFixed(3),
|
|
800
|
-
kind: state.kind,
|
|
801
702
|
why,
|
|
802
703
|
});
|
|
803
704
|
state.plan = undefined;
|
|
@@ -828,7 +729,6 @@ export class ReplayController {
|
|
|
828
729
|
this.handOver(state, {
|
|
829
730
|
kind: "step_failed",
|
|
830
731
|
step: plan.currentStep(),
|
|
831
|
-
entry: plan.currentEntry(),
|
|
832
732
|
position,
|
|
833
733
|
error: why === "own_call"
|
|
834
734
|
? "the model made a call of its own, so the scenario stopped"
|
|
@@ -889,8 +789,8 @@ export class ReplayController {
|
|
|
889
789
|
return false;
|
|
890
790
|
if (plan.parametersSettled)
|
|
891
791
|
return true;
|
|
892
|
-
const planned = plan.
|
|
893
|
-
if (!planned.some((
|
|
792
|
+
const planned = plan.allSteps().slice(0, plan.stepCount);
|
|
793
|
+
if (!planned.some((s) => pinnedToShape(s.toolName)))
|
|
894
794
|
return false;
|
|
895
795
|
if (state.deriveOnService && !serviceHealth.canWait())
|
|
896
796
|
return false;
|
|
@@ -1023,7 +923,6 @@ export class ReplayController {
|
|
|
1023
923
|
if (step && ready.blame) {
|
|
1024
924
|
this.recordStep(state, {
|
|
1025
925
|
step,
|
|
1026
|
-
entry: plan.currentEntry(),
|
|
1027
926
|
outcome: "failed",
|
|
1028
927
|
stage: ready.stage,
|
|
1029
928
|
error: ready.error,
|
|
@@ -1032,7 +931,6 @@ export class ReplayController {
|
|
|
1032
931
|
this.handOver(state, {
|
|
1033
932
|
kind: "step_failed",
|
|
1034
933
|
step,
|
|
1035
|
-
entry: plan.currentEntry(),
|
|
1036
934
|
position: plan.currentStepIndex,
|
|
1037
935
|
error: ready.error,
|
|
1038
936
|
delivery: "above",
|
|
@@ -1136,7 +1034,6 @@ export class ReplayController {
|
|
|
1136
1034
|
const recorded = err instanceof RecordedValueError;
|
|
1137
1035
|
this.recordStep(state, {
|
|
1138
1036
|
step,
|
|
1139
|
-
entry: plan.currentEntry(),
|
|
1140
1037
|
outcome: recorded ? "skipped" : "failed",
|
|
1141
1038
|
stage: "tool_input_logic",
|
|
1142
1039
|
error: errText(err),
|
|
@@ -1165,7 +1062,6 @@ export class ReplayController {
|
|
|
1165
1062
|
this.handOver(state, {
|
|
1166
1063
|
kind: "step_failed",
|
|
1167
1064
|
step: plan.currentStep(),
|
|
1168
|
-
entry: plan.currentEntry(),
|
|
1169
1065
|
position: plan.currentStepIndex,
|
|
1170
1066
|
error: `the scenario's next step is ${expected ?? "none"}, and the host asked for ${toolName}`,
|
|
1171
1067
|
delivery: "above",
|
|
@@ -1213,7 +1109,6 @@ export class ReplayController {
|
|
|
1213
1109
|
this.handOver(state, {
|
|
1214
1110
|
kind: "step_failed",
|
|
1215
1111
|
step: plan.allSteps()[next],
|
|
1216
|
-
entry: plan.allEntries()[next],
|
|
1217
1112
|
position: next,
|
|
1218
1113
|
delivery: "above",
|
|
1219
1114
|
stopReason: "concurrent",
|
|
@@ -1351,10 +1246,6 @@ export class ReplayController {
|
|
|
1351
1246
|
// reading it after the call would judge the *next* step, and reading it
|
|
1352
1247
|
// before assumes no other pin ever moved it.
|
|
1353
1248
|
const step = plan.allSteps()[pin.stepIndex];
|
|
1354
|
-
// The entry it ran as, so its verdict is kept under the scenario that owns
|
|
1355
|
-
// the step: a called segment numbers its steps from zero too, and without
|
|
1356
|
-
// this its step 0 replaced the caller's (segmented.md R-CALL-13).
|
|
1357
|
-
const entry = plan.allEntries()[pin.stepIndex];
|
|
1358
1249
|
// THE HOST STOPPED THE STEP, and says so. Read before any failure: a
|
|
1359
1250
|
// person's "no" arrives as an error result, and booked as one it would be
|
|
1360
1251
|
// counted against a plan that did nothing wrong — after enough of them the
|
|
@@ -1362,11 +1253,10 @@ export class ReplayController {
|
|
|
1362
1253
|
// `rejected` the tool never ran, and after `edited` its result answers a
|
|
1363
1254
|
// call the plan did not plan, which its later steps were not computed for.
|
|
1364
1255
|
if (how.hostStop && step) {
|
|
1365
|
-
this.hostStopped(state, pin, step,
|
|
1256
|
+
this.hostStopped(state, pin, step, output, how.hostStop);
|
|
1366
1257
|
this.handOver(state, {
|
|
1367
1258
|
kind: "step_failed",
|
|
1368
1259
|
step,
|
|
1369
|
-
entry,
|
|
1370
1260
|
position: pin.stepIndex,
|
|
1371
1261
|
delivery: "above",
|
|
1372
1262
|
stopReason: how.hostStop === "edited" ? "host_edited" : "host_rejected",
|
|
@@ -1387,7 +1277,6 @@ export class ReplayController {
|
|
|
1387
1277
|
`the plan's cursor is at step ${plan.currentStepIndex}, not the pinned step ${pin.stepIndex}`;
|
|
1388
1278
|
const info = {
|
|
1389
1279
|
step,
|
|
1390
|
-
entry,
|
|
1391
1280
|
input: pin.input,
|
|
1392
1281
|
output: answerOf(output),
|
|
1393
1282
|
outcome: "failed",
|
|
@@ -1401,7 +1290,6 @@ export class ReplayController {
|
|
|
1401
1290
|
this.handOver(state, {
|
|
1402
1291
|
kind: "step_failed",
|
|
1403
1292
|
step,
|
|
1404
|
-
entry,
|
|
1405
1293
|
position: pin.stepIndex,
|
|
1406
1294
|
error,
|
|
1407
1295
|
delivery: "above",
|
|
@@ -1423,7 +1311,6 @@ export class ReplayController {
|
|
|
1423
1311
|
if (step) {
|
|
1424
1312
|
this.recordStep(state, {
|
|
1425
1313
|
step,
|
|
1426
|
-
entry,
|
|
1427
1314
|
input: pin.input,
|
|
1428
1315
|
// What the logic threw on — the output is usually the explanation.
|
|
1429
1316
|
output,
|
|
@@ -1462,7 +1349,6 @@ export class ReplayController {
|
|
|
1462
1349
|
if (step && toolError) {
|
|
1463
1350
|
const info = {
|
|
1464
1351
|
step,
|
|
1465
|
-
entry,
|
|
1466
1352
|
input: pin.input,
|
|
1467
1353
|
output,
|
|
1468
1354
|
outcome: "failed",
|
|
@@ -1486,7 +1372,6 @@ export class ReplayController {
|
|
|
1486
1372
|
if (step) {
|
|
1487
1373
|
this.recordStep(state, {
|
|
1488
1374
|
step,
|
|
1489
|
-
entry,
|
|
1490
1375
|
input: pin.input,
|
|
1491
1376
|
output,
|
|
1492
1377
|
outcome: "executed",
|
|
@@ -1510,19 +1395,17 @@ export class ReplayController {
|
|
|
1510
1395
|
return done;
|
|
1511
1396
|
}
|
|
1512
1397
|
/**
|
|
1513
|
-
* A steer plan whose cursor reached its end: in front of a parked step
|
|
1514
|
-
*
|
|
1515
|
-
*
|
|
1398
|
+
* A steer plan whose cursor reached its end: in front of a parked step,
|
|
1399
|
+
* which hands over; otherwise it ran to its end — and from here every call
|
|
1400
|
+
* is the model's own.
|
|
1516
1401
|
*/
|
|
1517
1402
|
finishSteer(state, plan) {
|
|
1518
1403
|
if (plan.atStop()) {
|
|
1519
|
-
// Every planned step ran, and the next one is parked (fallbk.md D3)
|
|
1520
|
-
// is a call the service could not answer (segmented.md R-CALL-29).
|
|
1404
|
+
// Every planned step ran, and the next one is parked (fallbk.md D3).
|
|
1521
1405
|
const parked = plan.stopStep();
|
|
1522
1406
|
this.handOver(state, {
|
|
1523
|
-
kind:
|
|
1407
|
+
kind: stopKindOf(parked),
|
|
1524
1408
|
step: parked,
|
|
1525
|
-
entry: plan.stopEntry(),
|
|
1526
1409
|
position: plan.stepCount,
|
|
1527
1410
|
delivery: "above",
|
|
1528
1411
|
});
|
|
@@ -1559,34 +1442,28 @@ export class ReplayController {
|
|
|
1559
1442
|
settleCursor(state, plan, delivery) {
|
|
1560
1443
|
if (!plan.hasTests())
|
|
1561
1444
|
return "ready";
|
|
1562
|
-
// Never throws: a sub-task whose results cannot go back after its last
|
|
1563
|
-
// step was left out is a stop like any other (`result_map`).
|
|
1564
1445
|
const settled = plan.settleTests(this.observeStep(state));
|
|
1565
1446
|
// The recorded outputs of the steps left out are passed over, so a
|
|
1566
1447
|
// divergence's bundle later stands each step's own in (source-run.ts).
|
|
1567
1448
|
const source = state.sourceRun;
|
|
1568
1449
|
if (source) {
|
|
1569
1450
|
for (const l of settled.left) {
|
|
1570
|
-
|
|
1571
|
-
|
|
1572
|
-
|
|
1573
|
-
source.leftOut(entry);
|
|
1574
|
-
}
|
|
1451
|
+
const step = plan.allSteps()[l.position];
|
|
1452
|
+
if (step)
|
|
1453
|
+
source.leftOut(step);
|
|
1575
1454
|
}
|
|
1576
1455
|
}
|
|
1577
1456
|
if (settled.stop) {
|
|
1578
1457
|
const stop = settled.stop;
|
|
1579
|
-
const position = stop.
|
|
1458
|
+
const position = stop.position ?? plan.currentStepIndex;
|
|
1580
1459
|
this.handOver(state, {
|
|
1581
1460
|
kind: stop.kind,
|
|
1582
1461
|
step: plan.allSteps()[position],
|
|
1583
|
-
entry: plan.allEntries()[position],
|
|
1584
1462
|
position,
|
|
1585
1463
|
error: stop.error,
|
|
1586
1464
|
delivery,
|
|
1587
1465
|
stopReason: stop.reason,
|
|
1588
1466
|
whenText: stop.whenText,
|
|
1589
|
-
subTask: stop.subTask,
|
|
1590
1467
|
});
|
|
1591
1468
|
return "stopped";
|
|
1592
1469
|
}
|
|
@@ -1599,7 +1476,6 @@ export class ReplayController {
|
|
|
1599
1476
|
this.handOver(state, {
|
|
1600
1477
|
kind: "other_branch",
|
|
1601
1478
|
step: plan.allSteps()[0],
|
|
1602
|
-
entry: plan.allEntries()[0],
|
|
1603
1479
|
position: 0,
|
|
1604
1480
|
delivery,
|
|
1605
1481
|
noneNeeded: true,
|
|
@@ -1608,9 +1484,8 @@ export class ReplayController {
|
|
|
1608
1484
|
}
|
|
1609
1485
|
state.skipNote = {
|
|
1610
1486
|
text: this.skipNoteText(state, plan, settled.left),
|
|
1611
|
-
//
|
|
1612
|
-
|
|
1613
|
-
tools: new Set(settled.left.flatMap((l) => (l.tools.length > 0 ? l.tools : [l.toolName]))),
|
|
1487
|
+
// A call the model makes to any of these tools is one to a step left out.
|
|
1488
|
+
tools: new Set(settled.left.map((l) => l.toolName)),
|
|
1614
1489
|
delivered: false,
|
|
1615
1490
|
complete: plan.isDone() && !plan.atStop(),
|
|
1616
1491
|
};
|
|
@@ -1618,7 +1493,7 @@ export class ReplayController {
|
|
|
1618
1493
|
}
|
|
1619
1494
|
/** What the model is told about steps a test left out (AC5). */
|
|
1620
1495
|
skipNoteText(state, plan, left) {
|
|
1621
|
-
const named = left.map((l) => `step ${l.
|
|
1496
|
+
const named = left.map((l) => `step ${l.position + 1} (${l.toolName})`);
|
|
1622
1497
|
const texts = [...new Set(left.map((l) => l.text).filter((t) => !!t))];
|
|
1623
1498
|
const why = texts.length === 1 ? ` — only for when ${texts[0]}` : "";
|
|
1624
1499
|
const what = left.length === 1 ? `Not needed for this request: ${named[0]}${why}.` : `Not needed for this request: ${named.join(", ")}${why}.`;
|
|
@@ -1630,8 +1505,8 @@ export class ReplayController {
|
|
|
1630
1505
|
: "The sequence is complete: answer the user's request from the tool results.";
|
|
1631
1506
|
}
|
|
1632
1507
|
else {
|
|
1633
|
-
const
|
|
1634
|
-
next = `Leave ${left.length === 1 ? "it" : "them"} out, and call step ${plan.currentStepIndex + 1}${
|
|
1508
|
+
const step = plan.currentStep();
|
|
1509
|
+
next = `Leave ${left.length === 1 ? "it" : "them"} out, and call step ${plan.currentStepIndex + 1}${step?.toolName ? ` (${step.toolName})` : ""} next.`;
|
|
1635
1510
|
}
|
|
1636
1511
|
return `[BaseInstRunner calculated replay] ${what} ${next}`;
|
|
1637
1512
|
}
|
|
@@ -1693,7 +1568,7 @@ export class ReplayController {
|
|
|
1693
1568
|
return;
|
|
1694
1569
|
if (how.hostStop) {
|
|
1695
1570
|
// The host stopped it: neither a failure nor work the plan did.
|
|
1696
|
-
this.hostStopped(state, pin, step,
|
|
1571
|
+
this.hostStopped(state, pin, step, output, how.hostStop);
|
|
1697
1572
|
return;
|
|
1698
1573
|
}
|
|
1699
1574
|
const ms = Date.now() - pin.pinnedAt;
|
|
@@ -1705,11 +1580,9 @@ export class ReplayController {
|
|
|
1705
1580
|
? `${failure.error} — it was sent, and may or may not have completed`
|
|
1706
1581
|
: failure.error
|
|
1707
1582
|
: toolResultError(output);
|
|
1708
|
-
const entry = plan.allEntries()[pin.stepIndex];
|
|
1709
1583
|
if (error !== undefined) {
|
|
1710
1584
|
const info = {
|
|
1711
1585
|
step,
|
|
1712
|
-
entry,
|
|
1713
1586
|
input: pin.input,
|
|
1714
1587
|
output: answerOf(output),
|
|
1715
1588
|
outcome: "failed",
|
|
@@ -1721,7 +1594,7 @@ export class ReplayController {
|
|
|
1721
1594
|
this.recordStep(state, info);
|
|
1722
1595
|
return;
|
|
1723
1596
|
}
|
|
1724
|
-
this.recordStep(state, { step,
|
|
1597
|
+
this.recordStep(state, { step, input: pin.input, output, outcome: "executed", ms });
|
|
1725
1598
|
state.stepsPinned += 1;
|
|
1726
1599
|
this.upgrade(state, "fell_back");
|
|
1727
1600
|
logDetail("replay.thread_late", {
|
|
@@ -1736,7 +1609,7 @@ export class ReplayController {
|
|
|
1736
1609
|
* `failed`: the service counts failures against the plan. And not counted
|
|
1737
1610
|
* in `stepsPinned`: the plan's own call did not run.
|
|
1738
1611
|
*/
|
|
1739
|
-
hostStopped(state, pin, step,
|
|
1612
|
+
hostStopped(state, pin, step, output, stop) {
|
|
1740
1613
|
const edited = stop === "edited";
|
|
1741
1614
|
// After `rejected` nothing later can change what the report may claim.
|
|
1742
1615
|
if (state.hostStop !== "rejected")
|
|
@@ -1752,7 +1625,6 @@ export class ReplayController {
|
|
|
1752
1625
|
});
|
|
1753
1626
|
this.recordStep(state, {
|
|
1754
1627
|
step,
|
|
1755
|
-
entry,
|
|
1756
1628
|
// The plan's own arguments are what a refused call was asked with; a
|
|
1757
1629
|
// changed call ran with other ones, which only the host knows.
|
|
1758
1630
|
input: edited ? undefined : pin.input,
|
|
@@ -1829,7 +1701,6 @@ export class ReplayController {
|
|
|
1829
1701
|
if (step && ready.blame) {
|
|
1830
1702
|
this.recordStep(state, {
|
|
1831
1703
|
step,
|
|
1832
|
-
entry: plan.currentEntry(),
|
|
1833
1704
|
outcome: "failed",
|
|
1834
1705
|
stage: ready.stage,
|
|
1835
1706
|
error: ready.error,
|
|
@@ -1838,7 +1709,6 @@ export class ReplayController {
|
|
|
1838
1709
|
this.handOver(state, {
|
|
1839
1710
|
kind: "step_failed",
|
|
1840
1711
|
step,
|
|
1841
|
-
entry: plan.currentEntry(),
|
|
1842
1712
|
position: plan.currentStepIndex,
|
|
1843
1713
|
error: ready.error,
|
|
1844
1714
|
delivery: "none",
|
|
@@ -1872,7 +1742,7 @@ export class ReplayController {
|
|
|
1872
1742
|
// them alone is a baseline sample, not a hand-over "saving".
|
|
1873
1743
|
const ranOk = result.executed - result.errored;
|
|
1874
1744
|
state.stepsPinned += Math.max(0, ranOk);
|
|
1875
|
-
const position = stop.
|
|
1745
|
+
const position = stop.position ?? plan.stepCount;
|
|
1876
1746
|
const step = plan.allSteps()[position];
|
|
1877
1747
|
// Cut short by the clock, nothing failed: the steps that ran did their
|
|
1878
1748
|
// work and none is blamed. A divergence for the ledger — a partial
|
|
@@ -1890,11 +1760,10 @@ export class ReplayController {
|
|
|
1890
1760
|
const live = result.executed > 0;
|
|
1891
1761
|
this.handOver(state, {
|
|
1892
1762
|
// The plan only knows it stopped in front of its end; which kind of
|
|
1893
|
-
// stop that is — parked, a judgement
|
|
1894
|
-
//
|
|
1895
|
-
kind: stop.kind === "known_bad_step" ?
|
|
1763
|
+
// stop that is — parked, or a judgement — is decided here, where the
|
|
1764
|
+
// served reasons are.
|
|
1765
|
+
kind: stop.kind === "known_bad_step" ? stopKindOf(plan.stopStep()) : stop.kind,
|
|
1896
1766
|
step,
|
|
1897
|
-
entry: plan.allEntries()[position],
|
|
1898
1767
|
position,
|
|
1899
1768
|
error: stop.error,
|
|
1900
1769
|
// A cancelled run's note is kept for the model's next call, alone —
|
|
@@ -1905,12 +1774,9 @@ export class ReplayController {
|
|
|
1905
1774
|
ran: live ? undefined : 0,
|
|
1906
1775
|
// An output-logic failure stops *after* its step's tool ran: the step
|
|
1907
1776
|
// is done, its result is in the bundle, and it must not be run again.
|
|
1908
|
-
|
|
1909
|
-
// (`result_map`): that step did not run (AC5).
|
|
1910
|
-
ranThisStep: stop.stage === "tool_output_logic" && stop.reason !== "result_map",
|
|
1777
|
+
ranThisStep: stop.stage === "tool_output_logic",
|
|
1911
1778
|
// The other half (AC5): what the steps from here are for.
|
|
1912
1779
|
whenText: stop.whenText,
|
|
1913
|
-
subTask: stop.subTask,
|
|
1914
1780
|
});
|
|
1915
1781
|
if (stop.reason === "cancelled") {
|
|
1916
1782
|
// The caller is gone and reads nothing. The note stays undelivered: a
|
|
@@ -1947,7 +1813,6 @@ export class ReplayController {
|
|
|
1947
1813
|
this.handOver(state, {
|
|
1948
1814
|
kind: "other_branch",
|
|
1949
1815
|
step: plan.allSteps()[0],
|
|
1950
|
-
entry: plan.allEntries()[0],
|
|
1951
1816
|
position: 0,
|
|
1952
1817
|
delivery: "none",
|
|
1953
1818
|
ran: 0,
|
|
@@ -1962,7 +1827,6 @@ export class ReplayController {
|
|
|
1962
1827
|
this.handOver(state, {
|
|
1963
1828
|
kind: "step_failed",
|
|
1964
1829
|
step: plan.allSteps()[0],
|
|
1965
|
-
entry: plan.allEntries()[0],
|
|
1966
1830
|
position: 0,
|
|
1967
1831
|
error: "none of the scenario's steps could run here",
|
|
1968
1832
|
delivery: "none",
|
|
@@ -2029,20 +1893,16 @@ export class ReplayController {
|
|
|
2029
1893
|
const total = state.deriveCostUsd + d.sessionCostUsd + state.fallbackCostUsd;
|
|
2030
1894
|
const steps = this.stepResultsOf(state);
|
|
2031
1895
|
const isBaseline = state.outcome === "not_steered" || state.outcome === "failed";
|
|
2032
|
-
// The plan handed the task back by design, not by a fault: in front of the
|
|
2033
|
-
// other half of an "if … otherwise" (AC5), or at its decision (the
|
|
2034
|
-
// service's segmentsbreakdown.md SB-19). Such a stop is named whatever the
|
|
2035
|
-
// outcome, and the agent's work after it is the service's evidence.
|
|
2036
|
-
const handedBack = state.handover?.kind === "other_branch" || state.handover?.kind === "decision";
|
|
2037
1896
|
// A costless decline is noise on both sides — *unless* it says something the
|
|
2038
1897
|
// recording page cannot learn any other way: a step that actually broke, the
|
|
2039
|
-
// gate that declined, or a plan that
|
|
2040
|
-
//
|
|
2041
|
-
//
|
|
1898
|
+
// gate that declined, or a plan that stood in front of the other half of
|
|
1899
|
+
// its task (AC5), which the service takes the agent's work after as its
|
|
1900
|
+
// evidence of that half. The service accepts a costless report that carries
|
|
1901
|
+
// any of them (its errorshandling.md).
|
|
2042
1902
|
if (isBaseline &&
|
|
2043
1903
|
total <= 0 &&
|
|
2044
1904
|
!state.declined &&
|
|
2045
|
-
|
|
1905
|
+
state.handover?.kind !== "other_branch" &&
|
|
2046
1906
|
!steps?.some((s) => s.status === "failed")) {
|
|
2047
1907
|
return undefined;
|
|
2048
1908
|
}
|
|
@@ -2083,22 +1943,18 @@ export class ReplayController {
|
|
|
2083
1943
|
declined: state.declined,
|
|
2084
1944
|
// Where the plan handed the task to the model (fallbk.md). Only on
|
|
2085
1945
|
// `fell_back`: a hand-over before any step ran is `failed`, and a parked
|
|
2086
|
-
// step found after a divergence is still a divergence. Except
|
|
2087
|
-
//
|
|
2088
|
-
//
|
|
2089
|
-
// outcome: the agent's work after it is the service's evidence, whether
|
|
1946
|
+
// step found after a divergence is still a divergence. Except the other
|
|
1947
|
+
// half of an "if … otherwise" (AC5), named whatever the outcome: the
|
|
1948
|
+
// agent's work after it is the service's evidence of that half, whether
|
|
2090
1949
|
// or not a step ran first.
|
|
2091
|
-
fallbackStepIndex: state.outcome === "fell_back" ||
|
|
1950
|
+
fallbackStepIndex: state.outcome === "fell_back" || state.handover?.kind === "other_branch"
|
|
2092
1951
|
? state.handover?.stepIndex
|
|
2093
1952
|
: undefined,
|
|
2094
|
-
fallbackKind: state.outcome === "fell_back" ||
|
|
1953
|
+
fallbackKind: state.outcome === "fell_back" || state.handover?.kind === "other_branch"
|
|
2095
1954
|
? state.handover?.kind
|
|
2096
1955
|
: undefined,
|
|
2097
1956
|
// The agent did not take the plan's result and redid the task (AC1).
|
|
2098
1957
|
...(d.redoneBy ? { redoneBy: d.redoneBy } : {}),
|
|
2099
|
-
// Segments of this plan's recording that armed later in the turn
|
|
2100
|
-
// (segmented.md R-MONEY-4): their share comes off this plan's baseline.
|
|
2101
|
-
sharedWith: state.sharedWith.length > 0 ? [...state.sharedWith] : undefined,
|
|
2102
1958
|
// A turn in which another plan ran is not a measurement of what this task
|
|
2103
1959
|
// costs unaided, so it is no baseline sample (segmented.md R-MONEY-5).
|
|
2104
1960
|
// This covers two plans that both armed, and — the case that was silently
|
|
@@ -2142,26 +1998,11 @@ export class ReplayController {
|
|
|
2142
1998
|
* there is nothing to be similar *to* and nothing to decline. It books no
|
|
2143
1999
|
* execution either — there is no ticket, and inventing a saving for a manual
|
|
2144
2000
|
* invocation is exactly the kind of number a ledger must never contain.
|
|
2145
|
-
*
|
|
2146
|
-
* One served stop is kept: a plan's decision (the service's
|
|
2147
|
-
* segmentsbreakdown.md SB-6). The run stops in front of it, and `stopped`
|
|
2148
|
-
* says so with the reason `decision`.
|
|
2149
2001
|
*/
|
|
2150
2002
|
async runAdHoc(scenario, prompt, wrapped) {
|
|
2151
2003
|
if (!isReadyScenario(scenario))
|
|
2152
2004
|
return { ok: false, why: "scenario is not ready" };
|
|
2153
|
-
|
|
2154
|
-
// flat list, one frame per call (segmented.md R-CALL-37).
|
|
2155
|
-
let flat;
|
|
2156
|
-
try {
|
|
2157
|
-
flat = flattenChain(scenario);
|
|
2158
|
-
}
|
|
2159
|
-
catch (err) {
|
|
2160
|
-
return { ok: false, why: `the chain could not be flattened: ${errText(err)}` };
|
|
2161
|
-
}
|
|
2162
|
-
if (flat.entries.length === 0)
|
|
2163
|
-
return { ok: false, why: "the chain has no runnable step" };
|
|
2164
|
-
const mode = modeFor(flat.entries.map((e) => e.step), wrapped, this.opts.allowServers, this.builtinTools);
|
|
2005
|
+
const mode = modeFor(scenario.steps, wrapped, this.opts.allowServers, this.builtinTools);
|
|
2165
2006
|
if (mode === "none")
|
|
2166
2007
|
return { ok: false, why: "no step is executable" };
|
|
2167
2008
|
const state = {
|
|
@@ -2183,15 +2024,11 @@ export class ReplayController {
|
|
|
2183
2024
|
reported: true,
|
|
2184
2025
|
retired: false,
|
|
2185
2026
|
armedBy: "prompt",
|
|
2186
|
-
// `bir replay` runs a scenario an operator named by id: never a segment
|
|
2187
|
-
// hand-out, and nothing to share a baseline with.
|
|
2188
|
-
kind: "scenario",
|
|
2189
|
-
sharedWith: [],
|
|
2190
2027
|
};
|
|
2191
|
-
// `bir replay` keeps its own contract on a missing target
|
|
2192
|
-
//
|
|
2193
|
-
//
|
|
2194
|
-
//
|
|
2028
|
+
// `bir replay` keeps its own contract on a missing target: an operator
|
|
2029
|
+
// asked for this run by id, so a sample stands in and it never declines.
|
|
2030
|
+
// It says which, as the console's dry replay does (R-PARAM-7), so nobody
|
|
2031
|
+
// reads the output as proof the values were derived.
|
|
2195
2032
|
const schema = scenario.paramsObject ?? {};
|
|
2196
2033
|
const params = this.startDerivation(scenario, prompt, state).then((r) => {
|
|
2197
2034
|
const out = { ...r.params };
|
|
@@ -2210,25 +2047,13 @@ export class ReplayController {
|
|
|
2210
2047
|
}
|
|
2211
2048
|
return out;
|
|
2212
2049
|
});
|
|
2213
|
-
// The one served stop an operator's run keeps: a decision (the service's
|
|
2214
|
-
// segmentsbreakdown.md SB-6). The steps after it are one half of an "if …
|
|
2215
|
-
// otherwise", the half the recording took, and running them by id would
|
|
2216
|
-
// do that half's writes whatever the request. Every other served stop is
|
|
2217
|
-
// the plan's health, which an operator's run is there to try.
|
|
2218
|
-
const decisionAt = flat.entries.findIndex((e) => e.step.stop?.kind === "decision");
|
|
2219
|
-
const stopAt = decisionAt >= 0 && (flat.stopAt === undefined || decisionAt < flat.stopAt)
|
|
2220
|
-
? clampToFrameStart(flat.entries, decisionAt)
|
|
2221
|
-
: flat.stopAt;
|
|
2222
2050
|
const plan = new ScenarioReplayPlan({
|
|
2223
2051
|
scenario,
|
|
2224
2052
|
params,
|
|
2225
2053
|
mode,
|
|
2226
|
-
entries: flat.entries,
|
|
2227
|
-
stopAt,
|
|
2228
2054
|
fallbackTargets: "samples",
|
|
2229
2055
|
});
|
|
2230
2056
|
state.plan = plan;
|
|
2231
|
-
state.flatStop = stopAt === flat.stopAt ? flat.stopReason : undefined;
|
|
2232
2057
|
if (this.opts.authUrl && this.opts.authToken) {
|
|
2233
2058
|
state.sourceRun = new SourceRunOutputs({
|
|
2234
2059
|
baseUrl: this.opts.authUrl,
|
|
@@ -2244,8 +2069,8 @@ export class ReplayController {
|
|
|
2244
2069
|
coverage: coverageOf(scenario.steps, wrapped, this.opts.allowServers, this.builtinTools).join(","),
|
|
2245
2070
|
why: "bir replay — no match, an operator asked for it",
|
|
2246
2071
|
});
|
|
2247
|
-
// An operator's run keeps its own contract
|
|
2248
|
-
//
|
|
2072
|
+
// An operator's run keeps its own contract: out of time, the targets too
|
|
2073
|
+
// take their samples. What it never does is run on `{}`.
|
|
2249
2074
|
const ready = await this.readyFor(state, plan, this.adHocBudgets.deriveMs, { adHoc: true });
|
|
2250
2075
|
if (!ready.ok)
|
|
2251
2076
|
return { ok: false, why: ready.error };
|
|
@@ -2274,16 +2099,7 @@ export class ReplayController {
|
|
|
2274
2099
|
logDetail("replay.response_model_failed", { error: errText(err) });
|
|
2275
2100
|
}
|
|
2276
2101
|
const stop = result.stopped;
|
|
2277
|
-
const stoppedStep = stop ? plan.allSteps()[stop.
|
|
2278
|
-
// The plan's end in front of its decision: said so, by its own word.
|
|
2279
|
-
const atDecision = stop?.kind === "known_bad_step" && stoppedStep?.stop?.kind === "decision";
|
|
2280
|
-
if (atDecision) {
|
|
2281
|
-
logLine("replay.adhoc_decision", {
|
|
2282
|
-
scenario: scenario.id,
|
|
2283
|
-
step: stop.stepIndex,
|
|
2284
|
-
why: "bir replay stops in front of the plan's decision: the steps after it are the half the recording took",
|
|
2285
|
-
});
|
|
2286
|
-
}
|
|
2102
|
+
const stoppedStep = stop ? plan.allSteps()[stop.position ?? plan.stepCount] : undefined;
|
|
2287
2103
|
return {
|
|
2288
2104
|
ok: true,
|
|
2289
2105
|
text: result.text,
|
|
@@ -2295,7 +2111,7 @@ export class ReplayController {
|
|
|
2295
2111
|
? {
|
|
2296
2112
|
step: stop.stepIndex,
|
|
2297
2113
|
tool: stoppedStep?.toolName ?? "",
|
|
2298
|
-
reason:
|
|
2114
|
+
reason: stop.reason ?? stop.stage ?? stop.kind,
|
|
2299
2115
|
error: stop.error,
|
|
2300
2116
|
}
|
|
2301
2117
|
: undefined,
|
|
@@ -2358,8 +2174,6 @@ export class ReplayController {
|
|
|
2358
2174
|
fetchImpl: this.opts.fetchImpl,
|
|
2359
2175
|
liveCall: ctx.liveCall,
|
|
2360
2176
|
recentResults: ctx.recentResults,
|
|
2361
|
-
// A reading for a scenario that follows a plan's decision (offer1.md O4).
|
|
2362
|
-
...(ctx.atDecision ? { atDecision: ctx.atDecision } : {}),
|
|
2363
2177
|
// The request itself ends too. Without this a hung service or model held
|
|
2364
2178
|
// the socket for up to 300 s after every wait on it had given up. Long
|
|
2365
2179
|
// enough to cover both waits that can use its answer (the arm and the
|
|
@@ -2492,16 +2306,7 @@ export class ReplayController {
|
|
|
2492
2306
|
// when a step did its work — so the service takes the agent's work after
|
|
2493
2307
|
// it as that half's evidence. Booked `diverged`, it never did.
|
|
2494
2308
|
const otherHalf = stop?.kind === "other_branch";
|
|
2495
|
-
|
|
2496
|
-
// service's segmentsbreakdown.md SB-17) is a hand-over too: the plan did
|
|
2497
|
-
// all it was for, and the agent decides. Not one of recorded outputs
|
|
2498
|
-
// only, which is no step done here — that stays a plain divergence.
|
|
2499
|
-
const atDecision = !stop &&
|
|
2500
|
-
plan.stopsEarly() &&
|
|
2501
|
-
composed.endedAt === plan.stepCount &&
|
|
2502
|
-
(composed.executed > 0 || composed.recorded === 0) &&
|
|
2503
|
-
this.stopKindOf(state, plan.stopEntry()) === "decision";
|
|
2504
|
-
this.upgrade(state, broke || !didWork ? "failed" : otherHalf || atDecision ? "fell_back" : "diverged");
|
|
2309
|
+
this.upgrade(state, broke || !didWork ? "failed" : otherHalf ? "fell_back" : "diverged");
|
|
2505
2310
|
state.stepsPinned += composed.executed;
|
|
2506
2311
|
// The headline number of this whole design: recovering from a divergence
|
|
2507
2312
|
// costs nothing, because the proxies were already connected.
|
|
@@ -2512,7 +2317,7 @@ export class ReplayController {
|
|
|
2512
2317
|
recorded: composed.recorded,
|
|
2513
2318
|
skipped: composed.skipped,
|
|
2514
2319
|
errored: composed.errored || undefined,
|
|
2515
|
-
stopped: stop ? `${stop.
|
|
2320
|
+
stopped: stop ? `${stop.position ?? "?"}:${stop.reason ?? stop.stage ?? stop.kind}` : undefined,
|
|
2516
2321
|
bytes: composed.text.length,
|
|
2517
2322
|
costUsd: state.fallbackCostUsd.toFixed(2),
|
|
2518
2323
|
});
|
|
@@ -2528,40 +2333,16 @@ export class ReplayController {
|
|
|
2528
2333
|
// of recorded outputs only is last week's work. The model's own call goes
|
|
2529
2334
|
// ahead — unless a step was *sent* and never answered. That one may have
|
|
2530
2335
|
// done its work, and the model must know before it does the task itself.
|
|
2531
|
-
if (stop?.reason === "uncertain" && stop.
|
|
2336
|
+
if (stop?.reason === "uncertain" && stop.position !== undefined) {
|
|
2532
2337
|
const note = this.stopNote(state, plan, stop, "none", composed.left);
|
|
2533
2338
|
return this.deliverInstead(note, toolName, state.host !== undefined);
|
|
2534
2339
|
}
|
|
2535
2340
|
// The rest is the other half (AC5): the model is told so, in place of
|
|
2536
2341
|
// its call, before it does any of the plan's remaining steps.
|
|
2537
|
-
if (stop?.kind === "other_branch" && stop.
|
|
2342
|
+
if (stop?.kind === "other_branch" && stop.position !== undefined) {
|
|
2538
2343
|
const note = this.stopNote(state, plan, stop, "none", composed.left);
|
|
2539
2344
|
return this.deliverInstead(note, toolName, state.host !== undefined);
|
|
2540
2345
|
}
|
|
2541
|
-
// Every step left before the decision was one this request does not
|
|
2542
|
-
// need: the plan has done its part, and the agent decides. Told in
|
|
2543
|
-
// place of its call, like the other half. The steps that ran are the
|
|
2544
|
-
// model's own calls above; the ones left out are named.
|
|
2545
|
-
if (atDecision) {
|
|
2546
|
-
const parked = plan.stopStep();
|
|
2547
|
-
state.handover = {
|
|
2548
|
-
kind: "decision",
|
|
2549
|
-
stepIndex: callerStepIndexOf(plan.stopEntry()) ?? parked.stepIndex,
|
|
2550
|
-
toolName: parked.toolName ?? undefined,
|
|
2551
|
-
note: this.noteFor(state, {
|
|
2552
|
-
kind: "decision",
|
|
2553
|
-
step: parked,
|
|
2554
|
-
entry: plan.stopEntry(),
|
|
2555
|
-
position: plan.stepCount,
|
|
2556
|
-
delivery: "above",
|
|
2557
|
-
executedAbove: plan.currentStepIndex,
|
|
2558
|
-
leftOut: composed.left,
|
|
2559
|
-
}),
|
|
2560
|
-
delivered: true,
|
|
2561
|
-
};
|
|
2562
|
-
this.decidedInDivergence(state, plan);
|
|
2563
|
-
return this.deliverInstead(state.handover.note, toolName, state.host !== undefined);
|
|
2564
|
-
}
|
|
2565
2346
|
// Every remaining step was left out by its test: the steps the session
|
|
2566
2347
|
// ran were the whole of the plan this request needs, and it ran to its
|
|
2567
2348
|
// end (newestRun.md N7 reads that).
|
|
@@ -2576,13 +2357,13 @@ export class ReplayController {
|
|
|
2576
2357
|
// outcome stays as judged above, so the report carries no fallback fields
|
|
2577
2358
|
// (the other half of an "if … otherwise" excepted: see `buildReport`).
|
|
2578
2359
|
let text = composed.text;
|
|
2579
|
-
if (stop && (stop.kind === "step_failed" || stop.kind === "other_branch") && stop.
|
|
2360
|
+
if (stop && (stop.kind === "step_failed" || stop.kind === "other_branch") && stop.position !== undefined) {
|
|
2580
2361
|
const note = this.stopNote(state, plan, stop, "below", composed.left);
|
|
2581
2362
|
text = `${note}\n\n${composed.text}`;
|
|
2582
2363
|
}
|
|
2583
2364
|
else if (plan.stopsEarly() && composed.endedAt === plan.stepCount) {
|
|
2584
2365
|
const parked = plan.stopStep();
|
|
2585
|
-
const stopKind =
|
|
2366
|
+
const stopKind = stopKindOf(parked);
|
|
2586
2367
|
state.handover = {
|
|
2587
2368
|
kind: stopKind,
|
|
2588
2369
|
stepIndex: parked.stepIndex,
|
|
@@ -2590,7 +2371,6 @@ export class ReplayController {
|
|
|
2590
2371
|
note: this.noteFor(state, {
|
|
2591
2372
|
kind: stopKind,
|
|
2592
2373
|
step: parked,
|
|
2593
|
-
entry: plan.stopEntry(),
|
|
2594
2374
|
position: plan.stepCount,
|
|
2595
2375
|
delivery: "below",
|
|
2596
2376
|
executedAbove: plan.currentStepIndex,
|
|
@@ -2600,8 +2380,6 @@ export class ReplayController {
|
|
|
2600
2380
|
delivered: true,
|
|
2601
2381
|
};
|
|
2602
2382
|
text = `${state.handover.note}\n\n${composed.text}`;
|
|
2603
|
-
if (atDecision)
|
|
2604
|
-
this.decidedInDivergence(state, plan);
|
|
2605
2383
|
}
|
|
2606
2384
|
else if (!stop) {
|
|
2607
2385
|
// The bundle carries the rest of the chain, and its header tells the
|
|
@@ -2623,13 +2401,11 @@ export class ReplayController {
|
|
|
2623
2401
|
stopNote(state, plan, stop, delivery,
|
|
2624
2402
|
/** The steps the bundle left out (AC5): no step that ran. */
|
|
2625
2403
|
left = []) {
|
|
2626
|
-
const position = stop.
|
|
2404
|
+
const position = stop.position ?? plan.currentStepIndex;
|
|
2627
2405
|
const step = plan.allSteps()[position];
|
|
2628
|
-
const entry = plan.allEntries()[position];
|
|
2629
2406
|
const note = this.noteFor(state, {
|
|
2630
2407
|
kind: stop.kind,
|
|
2631
2408
|
step,
|
|
2632
|
-
entry,
|
|
2633
2409
|
position,
|
|
2634
2410
|
error: stop.error,
|
|
2635
2411
|
delivery,
|
|
@@ -2639,16 +2415,14 @@ export class ReplayController {
|
|
|
2639
2415
|
// steps that ran; "no step ran" would invite the model to run them again.
|
|
2640
2416
|
ran: delivery === "none" ? plan.currentStepIndex : undefined,
|
|
2641
2417
|
executedAbove: plan.currentStepIndex,
|
|
2642
|
-
// An output-logic failure stops after its step's tool ran (plan.ts)
|
|
2643
|
-
|
|
2644
|
-
ranThisStep: stop.stage === "tool_output_logic" && stop.reason !== "result_map",
|
|
2418
|
+
// An output-logic failure stops after its step's tool ran (plan.ts).
|
|
2419
|
+
ranThisStep: stop.stage === "tool_output_logic",
|
|
2645
2420
|
whenText: stop.whenText,
|
|
2646
|
-
subTask: stop.subTask,
|
|
2647
2421
|
leftOut: left,
|
|
2648
2422
|
});
|
|
2649
2423
|
state.handover ??= {
|
|
2650
2424
|
kind: stop.kind,
|
|
2651
|
-
stepIndex:
|
|
2425
|
+
stepIndex: step?.stepIndex ?? position,
|
|
2652
2426
|
toolName: step?.toolName ?? undefined,
|
|
2653
2427
|
error: stop.error,
|
|
2654
2428
|
note,
|
|
@@ -2669,17 +2443,9 @@ export class ReplayController {
|
|
|
2669
2443
|
return;
|
|
2670
2444
|
const ran = state.stepsPinned;
|
|
2671
2445
|
this.retire(state, ran > 0 ? "fell_back" : "failed");
|
|
2672
|
-
// A plan stopped at its decision did all it was for: its bill ends here,
|
|
2673
|
-
// and the agent's decision and the half it takes are the agent's own
|
|
2674
|
-
// (the service's segmentsbreakdown.md SB-26). Only once a step ran: a
|
|
2675
|
-
// plan that did nothing is a baseline sample of the whole turn.
|
|
2676
|
-
if (h.kind === "decision" && ran > 0)
|
|
2677
|
-
this.markRetired(state);
|
|
2678
2446
|
state.handover = {
|
|
2679
2447
|
kind: h.kind,
|
|
2680
|
-
|
|
2681
|
-
// number the caller's own chain does not have (R-CALL-32, R-FALL-7).
|
|
2682
|
-
stepIndex: callerStepIndexOf(h.entry) ?? h.step?.stepIndex ?? h.position,
|
|
2448
|
+
stepIndex: h.step?.stepIndex ?? h.position,
|
|
2683
2449
|
toolName: h.step?.toolName ?? undefined,
|
|
2684
2450
|
error: h.error,
|
|
2685
2451
|
note: this.noteFor(state, h),
|
|
@@ -2690,83 +2456,38 @@ export class ReplayController {
|
|
|
2690
2456
|
kind: h.kind,
|
|
2691
2457
|
step: `${h.position}/${state.plan?.totalSteps ?? state.stepsPlanned}`,
|
|
2692
2458
|
tool: h.step?.toolName ?? undefined,
|
|
2693
|
-
// The scenario that owns the stopped step, when it is not the matched one.
|
|
2694
|
-
owner: h.entry && h.entry.depth > 0 ? h.entry.scenarioId : undefined,
|
|
2695
|
-
unusable: state.flatStop?.reason,
|
|
2696
|
-
callIntent: state.flatStop?.callIntent ?? undefined,
|
|
2697
|
-
segment: state.flatStop?.segmentId ?? undefined,
|
|
2698
2459
|
ran,
|
|
2699
2460
|
outcome: state.outcome,
|
|
2700
2461
|
why: h.kind === "other_branch"
|
|
2701
2462
|
? h.noneNeeded
|
|
2702
2463
|
? "no step of the plan was needed for this request — the agent does the task"
|
|
2703
2464
|
: `this request is the other half: the steps from here are only for when ${h.whenText ?? "their test says so"} — the agent continues`
|
|
2704
|
-
: h.kind === "
|
|
2705
|
-
? "the
|
|
2706
|
-
: h.kind === "
|
|
2707
|
-
? "a
|
|
2708
|
-
: h.
|
|
2709
|
-
? "the
|
|
2710
|
-
: h.
|
|
2711
|
-
? "the
|
|
2712
|
-
: h.stopReason === "
|
|
2713
|
-
? "
|
|
2714
|
-
: h.stopReason === "
|
|
2715
|
-
? "
|
|
2716
|
-
:
|
|
2717
|
-
? "the host stopped this step itself — no fault of the plan; the agent continues from here"
|
|
2718
|
-
: h.stopReason === "not_run" || h.stopReason === "uncertain"
|
|
2719
|
-
? "a step produced no result here — the agent continues from here"
|
|
2720
|
-
: "a step broke — the agent continues from here",
|
|
2721
|
-
});
|
|
2722
|
-
}
|
|
2723
|
-
/**
|
|
2724
|
-
* Which kind of stop a plan's end is (plan-services.md D8). The stopping
|
|
2725
|
-
* step's own served reason wins — the calculation's verdict is more exact
|
|
2726
|
-
* than a counter — then a call the service could not answer, then a parked
|
|
2727
|
-
* step.
|
|
2728
|
-
*/
|
|
2729
|
-
stopKindOf(state, entry) {
|
|
2730
|
-
// The service serves a decision ahead of any other stop of its step
|
|
2731
|
-
// (the service's segmentsbreakdown.md SB-6), and so it is read first.
|
|
2732
|
-
if (entry?.step.stop?.kind === "decision")
|
|
2733
|
-
return "decision";
|
|
2734
|
-
if (entry?.step.stop?.kind === "nondeterministic")
|
|
2735
|
-
return "nondeterministic_step";
|
|
2736
|
-
return state.flatStop ? "unusable_call" : "known_bad_step";
|
|
2737
|
-
}
|
|
2738
|
-
/**
|
|
2739
|
-
* A divergence's bundle that ended at the plan's decision (SB-17): logged
|
|
2740
|
-
* as `handOver` logs every other hand-over, and its usage mark taken there
|
|
2741
|
-
* when a step of the plan ran (SB-26).
|
|
2742
|
-
*/
|
|
2743
|
-
decidedInDivergence(state, plan) {
|
|
2744
|
-
if (state.stepsPinned > 0)
|
|
2745
|
-
this.markRetired(state);
|
|
2746
|
-
logLine("replay.handover", {
|
|
2747
|
-
scenario: state.scenarioId ?? undefined,
|
|
2748
|
-
kind: "decision",
|
|
2749
|
-
step: `${plan.stepCount}/${plan.totalSteps}`,
|
|
2750
|
-
via: "divergence",
|
|
2751
|
-
ran: state.stepsPinned,
|
|
2752
|
-
outcome: state.outcome,
|
|
2753
|
-
why: "the plan ran the steps before its decision — the agent decides which way applies and continues",
|
|
2465
|
+
: h.kind === "known_bad_step"
|
|
2466
|
+
? "the next step has failed too often — the agent continues from here"
|
|
2467
|
+
: h.kind === "nondeterministic_step"
|
|
2468
|
+
? "the next step needs a judgement the plan cannot compute — the agent makes it"
|
|
2469
|
+
: h.stopReason === "concurrent"
|
|
2470
|
+
? "a second call arrived while the pinned step was still running — the agent continues from here"
|
|
2471
|
+
: h.stopReason === "budget" || h.stopReason === "cancelled"
|
|
2472
|
+
? "the run was cut short before this step — the agent continues from here"
|
|
2473
|
+
: h.stopReason === "host_rejected" || h.stopReason === "host_edited"
|
|
2474
|
+
? "the host stopped this step itself — no fault of the plan; the agent continues from here"
|
|
2475
|
+
: h.stopReason === "not_run" || h.stopReason === "uncertain"
|
|
2476
|
+
? "a step produced no result here — the agent continues from here"
|
|
2477
|
+
: "a step broke — the agent continues from here",
|
|
2754
2478
|
});
|
|
2755
2479
|
}
|
|
2756
2480
|
/**
|
|
2757
|
-
* The steps before
|
|
2758
|
-
*
|
|
2759
|
-
*
|
|
2760
|
-
*
|
|
2481
|
+
* The steps before position `bound` that ran, 1-based, and the ones their
|
|
2482
|
+
* tests left out (AC5) — the plan cursor's own, and a divergence bundle's
|
|
2483
|
+
* (`extra`). `left` is empty when no step was left out; `ran` is then not
|
|
2484
|
+
* filled, since every step before `bound` ran.
|
|
2761
2485
|
*/
|
|
2762
2486
|
ranBefore(plan, bound, extra = []) {
|
|
2763
|
-
const left = [...(plan?.leftOut() ?? []), ...extra].filter((l) => l.
|
|
2487
|
+
const left = [...(plan?.leftOut() ?? []), ...extra].filter((l) => l.position < bound);
|
|
2764
2488
|
if (left.length === 0)
|
|
2765
2489
|
return { ran: [], left };
|
|
2766
|
-
const out = new Set();
|
|
2767
|
-
for (const l of left)
|
|
2768
|
-
for (let i = l.flatIndex; i < Math.min(l.until, bound); i += 1)
|
|
2769
|
-
out.add(i);
|
|
2490
|
+
const out = new Set(left.map((l) => l.position));
|
|
2770
2491
|
const ran = [];
|
|
2771
2492
|
for (let i = 0; i < bound; i += 1)
|
|
2772
2493
|
if (!out.has(i))
|
|
@@ -2777,8 +2498,7 @@ export class ReplayController {
|
|
|
2777
2498
|
const plan = state.plan;
|
|
2778
2499
|
// The steps before the stop — or the bound a caller gives — less the ones
|
|
2779
2500
|
// their tests left out (AC5): a note must never say a step ran that did not.
|
|
2780
|
-
|
|
2781
|
-
const bound = h.ran ?? (h.ranThisStep || h.stopReason === "result_map" ? h.position + 1 : h.position);
|
|
2501
|
+
const bound = h.ran ?? (h.ranThisStep ? h.position + 1 : h.position);
|
|
2782
2502
|
const before = this.ranBefore(plan, bound, h.leftOut);
|
|
2783
2503
|
const skipped = before.left.length > 0;
|
|
2784
2504
|
const executed = skipped ? before.ran.length : bound;
|
|
@@ -2787,50 +2507,31 @@ export class ReplayController {
|
|
|
2787
2507
|
: before.ran.filter((n) => n <= h.executedAbove).length;
|
|
2788
2508
|
// A step its host stopped was dealt with there — refused, answered, or
|
|
2789
2509
|
// run with a person's arguments — so it is not among what is left to do.
|
|
2790
|
-
// Nor is the step a test left out, after which a sub-task's results could
|
|
2791
|
-
// not go back (`result_map`, AC5).
|
|
2792
2510
|
const handled = h.ranThisStep === true ||
|
|
2793
2511
|
h.stopReason === "host_rejected" ||
|
|
2794
|
-
h.stopReason === "host_edited"
|
|
2795
|
-
h.stopReason === "result_map";
|
|
2796
|
-
// A chain that holds a decision (the service's segmentsbreakdown.md SB-18)
|
|
2797
|
-
// keeps the decision note's protections in every note, even when the plan
|
|
2798
|
-
// stops earlier for another reason: no intent, no parameters, and no step
|
|
2799
|
-
// from the decision on. Each points at the half the recording took, and an
|
|
2800
|
-
// agent that follows it there runs that half's segment: the plan decided.
|
|
2801
|
-
const decisionAt = plan ? plan.decisionIndex() : -1;
|
|
2512
|
+
h.stopReason === "host_edited";
|
|
2802
2513
|
let params;
|
|
2803
2514
|
try {
|
|
2804
|
-
params =
|
|
2515
|
+
params = plan?.parameters;
|
|
2805
2516
|
}
|
|
2806
2517
|
catch {
|
|
2807
2518
|
params = undefined;
|
|
2808
2519
|
}
|
|
2809
2520
|
return buildHandoverNote({
|
|
2810
|
-
intent:
|
|
2521
|
+
intent: plan?.intent ?? "",
|
|
2811
2522
|
kind: h.kind,
|
|
2812
2523
|
stepIndex: h.position,
|
|
2813
2524
|
stepToolName: h.step?.toolName ?? undefined,
|
|
2814
2525
|
failureCount: h.step?.failureCount,
|
|
2815
2526
|
error: h.error,
|
|
2816
2527
|
stopReason: h.stopReason,
|
|
2817
|
-
unusable: state.flatStop
|
|
2818
|
-
? {
|
|
2819
|
-
reason: state.flatStop.reason,
|
|
2820
|
-
callIntent: state.flatStop.callIntent,
|
|
2821
|
-
segmentId: state.flatStop.segmentId,
|
|
2822
|
-
ownerScenarioId: state.flatStop.scenarioId,
|
|
2823
|
-
}
|
|
2824
|
-
: undefined,
|
|
2825
|
-
ownerScenarioId: h.entry && h.entry.depth > 0 ? h.entry.scenarioId : undefined,
|
|
2826
|
-
inSubTask: h.entry && h.entry.depth > 0 ? (h.entry.frame.callIntent ?? null) : null,
|
|
2827
2528
|
executed: Math.max(0, Math.min(executed, plan?.totalSteps ?? executed)),
|
|
2828
2529
|
...(skipped
|
|
2829
2530
|
? {
|
|
2830
2531
|
ranSteps: before.ran,
|
|
2831
2532
|
leftOut: before.left.map((l) => ({
|
|
2832
|
-
from: l.
|
|
2833
|
-
to:
|
|
2533
|
+
from: l.position + 1,
|
|
2534
|
+
to: l.position + 1,
|
|
2834
2535
|
toolName: l.toolName,
|
|
2835
2536
|
text: l.text,
|
|
2836
2537
|
})),
|
|
@@ -2842,17 +2543,12 @@ export class ReplayController {
|
|
|
2842
2543
|
totalSteps: plan?.totalSteps ?? state.stepsPlanned,
|
|
2843
2544
|
// A step that runs only under a test is marked so: the list must not
|
|
2844
2545
|
// read as steps this request needs (AC5).
|
|
2845
|
-
remainingTools: plan
|
|
2846
|
-
? plan.remainingFrom(handled ? h.position + 1 : h.position, decisionAt >= 0 ? decisionAt : undefined)
|
|
2847
|
-
: [],
|
|
2546
|
+
remainingTools: plan ? plan.remainingFrom(handled ? h.position + 1 : h.position) : [],
|
|
2848
2547
|
params,
|
|
2849
2548
|
delivery: h.delivery,
|
|
2850
2549
|
whenText: h.whenText ?? null,
|
|
2851
|
-
subTask: h.subTask ?? null,
|
|
2852
2550
|
// "None was needed" is the note's own wording for the end of the plan.
|
|
2853
2551
|
...(h.noneNeeded ? { stepIndex: plan?.totalSteps ?? state.stepsPlanned } : {}),
|
|
2854
|
-
// The request's own words for the choice (SB-18); never the service's.
|
|
2855
|
-
...(h.kind === "decision" ? { request: state.request ?? null } : {}),
|
|
2856
2552
|
});
|
|
2857
2553
|
}
|
|
2858
2554
|
/**
|
|
@@ -2991,8 +2687,8 @@ export class ReplayController {
|
|
|
2991
2687
|
return undefined;
|
|
2992
2688
|
// A step a test left out passes over its recorded output, unless a
|
|
2993
2689
|
// correction added it (AC5, source-run.ts).
|
|
2994
|
-
return Object.assign((
|
|
2995
|
-
leftOut: (
|
|
2690
|
+
return Object.assign((step) => source.outputFor(step), {
|
|
2691
|
+
leftOut: (step) => source.leftOut(step),
|
|
2996
2692
|
});
|
|
2997
2693
|
}
|
|
2998
2694
|
/**
|
|
@@ -3071,7 +2767,7 @@ export class ReplayController {
|
|
|
3071
2767
|
}
|
|
3072
2768
|
if (!opts.adHoc) {
|
|
3073
2769
|
const schema = plan.paramsSchema;
|
|
3074
|
-
const planned = plan.
|
|
2770
|
+
const planned = plan.allSteps().slice(0, plan.stepCount);
|
|
3075
2771
|
const blocked = blockingTargets(targetKeys(schema), schema, planned, plan.paramsLogic);
|
|
3076
2772
|
if (blocked.length > 0) {
|
|
3077
2773
|
return {
|
|
@@ -3088,7 +2784,7 @@ export class ReplayController {
|
|
|
3088
2784
|
const missingAgain = state.rederivedMissing;
|
|
3089
2785
|
state.rederivedMissing = undefined;
|
|
3090
2786
|
if (missingAgain && !opts.adHoc) {
|
|
3091
|
-
const planned = plan.
|
|
2787
|
+
const planned = plan.allSteps().slice(0, plan.stepCount);
|
|
3092
2788
|
const blocked = blockingTargets(missingAgain, plan.paramsSchema, planned, plan.paramsLogic);
|
|
3093
2789
|
if (blocked.length > 0) {
|
|
3094
2790
|
return {
|
|
@@ -3101,7 +2797,7 @@ export class ReplayController {
|
|
|
3101
2797
|
}
|
|
3102
2798
|
const sink = opts.adHoc
|
|
3103
2799
|
? undefined
|
|
3104
|
-
: sinkOfPlan(plan.
|
|
2800
|
+
: sinkOfPlan(plan.allSteps().slice(0, plan.stepCount), state.host?.sinks);
|
|
3105
2801
|
if (sink) {
|
|
3106
2802
|
const bad = unsafeParameter(plan.derivedValues, plan.paramsSchema, sink);
|
|
3107
2803
|
if (bad) {
|
|
@@ -3200,12 +2896,7 @@ export class ReplayController {
|
|
|
3200
2896
|
*/
|
|
3201
2897
|
recordStep(state, info) {
|
|
3202
2898
|
const status = info.outcome === "executed" ? "ok" : info.outcome;
|
|
3203
|
-
|
|
3204
|
-
// in one plan both have a step 0, and one would otherwise overwrite the
|
|
3205
|
-
// other's verdict (segmented.md R-CALL-13).
|
|
3206
|
-
const key = info.entry
|
|
3207
|
-
? `${info.entry.scenarioId}#${info.entry.stepIndex}`
|
|
3208
|
-
: `#${info.step.stepIndex}`;
|
|
2899
|
+
const key = `#${info.step.stepIndex}`;
|
|
3209
2900
|
if (state.stepResults.size >= MAX_STEP_RESULTS && !state.stepResults.has(key)) {
|
|
3210
2901
|
return;
|
|
3211
2902
|
}
|
|
@@ -3214,29 +2905,19 @@ export class ReplayController {
|
|
|
3214
2905
|
? `${info.error.slice(0, MAX_STEP_ERROR_CHARS)}…`
|
|
3215
2906
|
: info.error
|
|
3216
2907
|
: undefined;
|
|
3217
|
-
// Payloads only for a step of the reported scenario's own chain. The
|
|
3218
|
-
// console still files an execution's verdicts by step number alone
|
|
3219
|
-
// (plan-services.md W1.9), and a called segment numbers its steps from
|
|
3220
|
-
// zero as well: a called step's call would be shown under the caller's
|
|
3221
|
-
// step of the same number — the very mix-up the payloads are there to end.
|
|
3222
|
-
const own = !info.entry || info.entry.depth === 0;
|
|
3223
2908
|
// A step whose input logic threw never had an input: the `{}` the plan
|
|
3224
2909
|
// reports there is a placeholder, not a call with no arguments. Nor had a
|
|
3225
2910
|
// step its test left out or stopped in front of (AC5).
|
|
3226
2911
|
const noCall = info.stage === "tool_input_logic" || info.stage === "when_logic" || status === "not_taken";
|
|
3227
|
-
const sent =
|
|
3228
|
-
const received =
|
|
2912
|
+
const sent = !noCall ? info.input : undefined;
|
|
2913
|
+
const received = !noCall ? info.output : undefined;
|
|
3229
2914
|
// The call, kept whole for the turn's newest run (AC2): only one that
|
|
3230
2915
|
// really went out — it answered, or it answered with an error.
|
|
3231
2916
|
if (state.keepCalls &&
|
|
3232
2917
|
!noCall &&
|
|
3233
2918
|
info.input !== undefined &&
|
|
3234
2919
|
(status === "ok" || (status === "failed" && info.stage === "tool_call"))) {
|
|
3235
|
-
|
|
3236
|
-
// makes two calls of each of its steps, and both are the turn's (AC5
|
|
3237
|
-
// review). A frame is one call of one sub-task.
|
|
3238
|
-
const callKey = info.entry ? `${info.entry.frame.id}#${info.entry.stepIndex}` : key;
|
|
3239
|
-
this.keepCall(state, callKey, info, status === "failed");
|
|
2920
|
+
this.keepCall(state, key, info, status === "failed");
|
|
3240
2921
|
}
|
|
3241
2922
|
state.stepResults.set(key, {
|
|
3242
2923
|
stepIndex: info.step.stepIndex,
|
|
@@ -3252,10 +2933,6 @@ export class ReplayController {
|
|
|
3252
2933
|
// The tool is gone from its server: the service repairs the plan without
|
|
3253
2934
|
// waiting out the grace (plan-services.md D8, kind 2).
|
|
3254
2935
|
toolMissing: info.toolMissing || undefined,
|
|
3255
|
-
// A verdict about a called segment's step is counted on that segment, at
|
|
3256
|
-
// the revision it was served (R-CALL-13).
|
|
3257
|
-
scenarioId: info.entry && info.entry.depth > 0 ? info.entry.scenarioId : undefined,
|
|
3258
|
-
chainRevision: info.entry && info.entry.depth > 0 ? info.entry.frame.chainRevision : undefined,
|
|
3259
2936
|
});
|
|
3260
2937
|
}
|
|
3261
2938
|
/**
|
|
@@ -3274,14 +2951,11 @@ export class ReplayController {
|
|
|
3274
2951
|
? serializeCapped(redact(info.input ?? {}))
|
|
3275
2952
|
: reportPayload(serializeCapped(redact(info.input ?? {})), true);
|
|
3276
2953
|
const answer = info.output === undefined ? undefined : room ? heldPayload(info.output) : reportPayload(info.output);
|
|
3277
|
-
const inSegment = info.entry && info.entry.depth > 0;
|
|
3278
2954
|
const call = {
|
|
3279
|
-
scenarioId:
|
|
2955
|
+
scenarioId: state.scenarioId ?? "",
|
|
3280
2956
|
stepIndex: info.step.stepIndex,
|
|
3281
|
-
chainRevision:
|
|
3282
|
-
|
|
3283
|
-
: (state.chainRevision ?? null),
|
|
3284
|
-
toolName: info.step.toolName ?? "",
|
|
2957
|
+
chainRevision: state.chainRevision ?? null,
|
|
2958
|
+
toolName: info.step.toolName,
|
|
3285
2959
|
input,
|
|
3286
2960
|
...(failed
|
|
3287
2961
|
? { error: info.error ? capText(redactText(info.error), DEFAULT_MAX_PAYLOAD_BYTES) : answer }
|
|
@@ -3305,8 +2979,6 @@ export class ReplayController {
|
|
|
3305
2979
|
if (info.outcome === "failed") {
|
|
3306
2980
|
logLine("replay.step_failed", {
|
|
3307
2981
|
n: info.step.stepIndex,
|
|
3308
|
-
scenario: info.entry && info.entry.depth > 0 ? info.entry.scenarioId : undefined,
|
|
3309
|
-
frame: info.entry?.frame.id,
|
|
3310
2982
|
tool: info.step.toolName ?? undefined,
|
|
3311
2983
|
stage: info.stage,
|
|
3312
2984
|
why: info.stage === "tool_call"
|
|
@@ -3338,8 +3010,7 @@ export class ReplayController {
|
|
|
3338
3010
|
const text = testWords(info.step.when?.text);
|
|
3339
3011
|
logLine("replay.step_not_taken", {
|
|
3340
3012
|
n: info.step.stepIndex,
|
|
3341
|
-
tool: info.step.toolName
|
|
3342
|
-
scenario: info.entry && info.entry.depth > 0 ? info.entry.scenarioId : undefined,
|
|
3013
|
+
tool: info.step.toolName,
|
|
3343
3014
|
why: `its test said this request does not need it${text ? ` — only for when ${text}` : ""}; left out, and the plan goes on`,
|
|
3344
3015
|
});
|
|
3345
3016
|
return;
|
|
@@ -3363,10 +3034,17 @@ export class ReplayController {
|
|
|
3363
3034
|
this.upgrade(state, outcome);
|
|
3364
3035
|
// Only a plan that *finished* stops the clock (segmented.md R-MONEY-3). A
|
|
3365
3036
|
// hand-over or a divergence takes no mark: the agent's next tokens are the
|
|
3366
|
-
// consequence of this plan stopping, and belong to its bill.
|
|
3367
|
-
|
|
3368
|
-
|
|
3369
|
-
|
|
3037
|
+
// consequence of this plan stopping, and belong to its bill.
|
|
3038
|
+
if (completing) {
|
|
3039
|
+
state.retiredAt = Date.now();
|
|
3040
|
+
try {
|
|
3041
|
+
state.retiredMark = state.markUsage?.();
|
|
3042
|
+
}
|
|
3043
|
+
catch {
|
|
3044
|
+
// An unreadable transcript leaves the mark undefined, and the window
|
|
3045
|
+
// falls back to the next arm or the end of the turn.
|
|
3046
|
+
}
|
|
3047
|
+
}
|
|
3370
3048
|
state.retired = true;
|
|
3371
3049
|
state.pinned.clear();
|
|
3372
3050
|
// A note naming a step to call next is void once the plan is: after a
|
|
@@ -3378,21 +3056,6 @@ export class ReplayController {
|
|
|
3378
3056
|
// A call waiting for a pin to thread has nothing left to wait for.
|
|
3379
3057
|
this.notifyPins(state);
|
|
3380
3058
|
}
|
|
3381
|
-
/**
|
|
3382
|
-
* Where the plan's work ended, for its bill: now, and the transcript's
|
|
3383
|
-
* watermark now. Taken at a completing retire, and at a decision hand-over
|
|
3384
|
-
* after a step ran (the service's segmentsbreakdown.md SB-26).
|
|
3385
|
-
*/
|
|
3386
|
-
markRetired(state) {
|
|
3387
|
-
state.retiredAt = Date.now();
|
|
3388
|
-
try {
|
|
3389
|
-
state.retiredMark = state.markUsage?.();
|
|
3390
|
-
}
|
|
3391
|
-
catch {
|
|
3392
|
-
// An unreadable transcript leaves the mark undefined, and the window
|
|
3393
|
-
// falls back to the next arm or the end of the turn.
|
|
3394
|
-
}
|
|
3395
|
-
}
|
|
3396
3059
|
}
|
|
3397
3060
|
/** Every target of a schema. A parameter with no kind is one (R-PARAM-3). */
|
|
3398
3061
|
function targetKeys(schema) {
|
|
@@ -3530,58 +3193,29 @@ function heldPayload(text) {
|
|
|
3530
3193
|
}
|
|
3531
3194
|
return capText(text, DEFAULT_MAX_PAYLOAD_BYTES, redactText);
|
|
3532
3195
|
}
|
|
3533
|
-
/** Whether
|
|
3534
|
-
function underTest(
|
|
3535
|
-
|
|
3536
|
-
return true;
|
|
3537
|
-
for (let f = entry.frame; f; f = f.parent)
|
|
3538
|
-
if (f.when)
|
|
3539
|
-
return true;
|
|
3540
|
-
return false;
|
|
3196
|
+
/** Whether a step runs only under a test (AC5). */
|
|
3197
|
+
function underTest(step) {
|
|
3198
|
+
return Boolean(step.when);
|
|
3541
3199
|
}
|
|
3542
|
-
/** The tests of a
|
|
3543
|
-
function testLines(
|
|
3544
|
-
|
|
3545
|
-
|
|
3546
|
-
const { startsAt } = frameEdges(entries);
|
|
3547
|
-
const lines = entries
|
|
3548
|
-
.map((e, i) => {
|
|
3549
|
-
const own = e.step.when;
|
|
3550
|
-
const call = (startsAt.get(i) ?? []).find((f) => f.when)?.when;
|
|
3551
|
-
const t = own ?? call;
|
|
3552
|
-
return t ? `${i}:${call && !own ? "call/" : ""}${t.otherwise}` : undefined;
|
|
3553
|
-
})
|
|
3200
|
+
/** The tests of a chain, by position, for the `plan.armed` line (AC5). */
|
|
3201
|
+
function testLines(steps) {
|
|
3202
|
+
const lines = steps
|
|
3203
|
+
.map((s, i) => (s.when ? `${i}:${s.when.otherwise}` : undefined))
|
|
3554
3204
|
.filter((s) => s !== undefined);
|
|
3555
3205
|
return lines.length > 0 ? lines.join(",") : undefined;
|
|
3556
3206
|
}
|
|
3557
3207
|
/**
|
|
3558
|
-
*
|
|
3559
|
-
*
|
|
3560
|
-
*
|
|
3561
|
-
* call row that opened its frame does. Reporting the segment's own index would
|
|
3562
|
-
* line the execution up against a step the caller's card does not have.
|
|
3208
|
+
* Which kind of stop a plan's end is (plan-services.md D8): the stopping
|
|
3209
|
+
* step's own served reason wins — the calculation's verdict is more exact
|
|
3210
|
+
* than a counter — then a parked step.
|
|
3563
3211
|
*/
|
|
3564
|
-
function
|
|
3565
|
-
|
|
3566
|
-
return entry?.stepIndex;
|
|
3567
|
-
let frame = entry.frame;
|
|
3568
|
-
while (frame.parent && frame.depth > 1)
|
|
3569
|
-
frame = frame.parent;
|
|
3570
|
-
return frame.callerStepIndex;
|
|
3571
|
-
}
|
|
3572
|
-
/** The calls in a served chain, for the `plan.armed` line (R-CALL-32). */
|
|
3573
|
-
function callLines(scenario) {
|
|
3574
|
-
const calls = (scenario.steps ?? []).filter((s) => s.kind === "segment");
|
|
3575
|
-
if (calls.length === 0)
|
|
3576
|
-
return undefined;
|
|
3577
|
-
return calls
|
|
3578
|
-
.map((s) => `${s.segmentId ?? "-"}:${s.unusable ?? s.resolution?.state ?? "resolved"}`)
|
|
3579
|
-
.join(",");
|
|
3212
|
+
function stopKindOf(step) {
|
|
3213
|
+
return step?.stop?.kind === "nondeterministic" ? "nondeterministic_step" : "known_bad_step";
|
|
3580
3214
|
}
|
|
3581
|
-
/** The served stops in a
|
|
3582
|
-
function stopLines(
|
|
3583
|
-
const lines =
|
|
3584
|
-
.map((
|
|
3215
|
+
/** The served stops in a chain, for the `plan.armed` line (plan-services.md D8). */
|
|
3216
|
+
function stopLines(steps) {
|
|
3217
|
+
const lines = steps
|
|
3218
|
+
.map((s, i) => (s.stop ? `${i}:${s.stop.kind}${s.stop.repairDue ? "/repair" : ""}` : undefined))
|
|
3585
3219
|
.filter((s) => s !== undefined);
|
|
3586
3220
|
return lines.length > 0 ? lines.join(",") : undefined;
|
|
3587
3221
|
}
|
|
@@ -3606,8 +3240,7 @@ function isKnownBad(step, maxStepFailures) {
|
|
|
3606
3240
|
*
|
|
3607
3241
|
* A step that carries `stop` was decided by the service — `null` means run it,
|
|
3608
3242
|
* even when its count is past the limit: its retry is due. A step without the
|
|
3609
|
-
* field came from an older service, and the old counter rule decides it.
|
|
3610
|
-
* can sit in one chain, since a called segment is served on its own terms.
|
|
3243
|
+
* field came from an older service, and the old counter rule decides it.
|
|
3611
3244
|
*/
|
|
3612
3245
|
export function stopIndex(steps, maxStepFailures) {
|
|
3613
3246
|
const i = steps.findIndex((s) => s.stop !== undefined ? s.stop !== null : isKnownBad(s, maxStepFailures));
|