@basein/runner 0.3.18 → 0.3.20
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/bir-hooks.d.ts +11 -8
- package/dist/bin/bir-hooks.js +11 -8
- package/dist/bin/bir.js +20 -7
- package/dist/bin/investigate.d.ts +9 -0
- package/dist/bin/investigate.js +31 -12
- package/dist/bin/scenario-edit.js +10 -0
- package/dist/control/recorder-main.d.ts +15 -0
- package/dist/control/recorder-main.js +33 -8
- package/dist/control/server.d.ts +91 -11
- package/dist/control/server.js +736 -49
- package/dist/record/recorder.d.ts +47 -4
- package/dist/record/recorder.js +3 -0
- package/dist/record/remote-recorder.d.ts +9 -1
- package/dist/record/remote-recorder.js +30 -0
- package/dist/record/switchable-recorder.js +2 -0
- package/dist/replay/condition-words.d.ts +28 -0
- package/dist/replay/condition-words.js +59 -0
- package/dist/replay/controller.d.ts +45 -6
- package/dist/replay/controller.js +184 -51
- package/dist/replay/derive.d.ts +10 -0
- package/dist/replay/derive.js +3 -0
- package/dist/replay/handover.d.ts +7 -0
- package/dist/replay/handover.js +89 -51
- package/dist/replay/plan.d.ts +11 -4
- package/dist/replay/plan.js +28 -8
- package/dist/replay/reads-only.d.ts +29 -0
- package/dist/replay/reads-only.js +261 -0
- package/dist/replay/types.d.ts +19 -3
- package/docs/calculatedReplay.md +45 -5
- package/docs/calculatedReplayGuide.md +16 -4
- package/package.json +1 -1
- package/production.md +34 -7
|
@@ -288,9 +288,11 @@ export class ReplayController {
|
|
|
288
288
|
/**
|
|
289
289
|
* Whether a handed-out segment may actually arm (segmented.md R-OUT-10).
|
|
290
290
|
*
|
|
291
|
-
*
|
|
292
|
-
* `BIR_SEGMENT_ARM=
|
|
293
|
-
*
|
|
291
|
+
* Only when the option says so. The recorder passes it on unless
|
|
292
|
+
* `BIR_SEGMENT_ARM=0` (from 0.3.19; the service's segmentsbreakdown.md
|
|
293
|
+
* SB-21); a controller built without it stays observe-only, asks for no
|
|
294
|
+
* segment and arms none. The server enforces the same thing from its side
|
|
295
|
+
* (R-OUT-9).
|
|
294
296
|
*/
|
|
295
297
|
get segmentArm() {
|
|
296
298
|
return this.opts.segmentArm === true;
|
|
@@ -389,6 +391,10 @@ export class ReplayController {
|
|
|
389
391
|
*/
|
|
390
392
|
async arm(match, prompt, wrapped, armedBy = "prompt", ctx = {}, host) {
|
|
391
393
|
const state = this.blankState(match, wrapped, armedBy, host);
|
|
394
|
+
// A decision note quotes the request (SB-18); a plan armed by intent was
|
|
395
|
+
// given the step's intent here, and its control server sets the request.
|
|
396
|
+
if (armedBy === "prompt")
|
|
397
|
+
state.request = prompt;
|
|
392
398
|
const isSegment = state.kind === "segment";
|
|
393
399
|
const decline = (why, code) => {
|
|
394
400
|
state.declined = code;
|
|
@@ -480,12 +486,18 @@ export class ReplayController {
|
|
|
480
486
|
const unusable = flat.stopAt === 0 && flat.stopReason ? flat.stopReason : undefined;
|
|
481
487
|
// The step that stops the plan, before the clamp: the reason is its own.
|
|
482
488
|
const stopping = entries[Math.min(...rawStop)]?.step ?? entries[0].step;
|
|
483
|
-
|
|
489
|
+
// The service never serves a decision at step 0 (segmentsbreakdown.md
|
|
490
|
+
// SB-5): should one come, the agent decides from the start, as it makes
|
|
491
|
+
// a judgement there. No new code reaches a service that may not know it.
|
|
492
|
+
const decisionFirst = stopping.stop?.kind === "decision";
|
|
493
|
+
const judgement = stopping.stop?.kind === "nondeterministic" || decisionFirst;
|
|
484
494
|
const why = unusable
|
|
485
495
|
? `its first step is a call that cannot run: ${unusable.reason}`
|
|
486
|
-
:
|
|
487
|
-
? "its first step
|
|
488
|
-
:
|
|
496
|
+
: decisionFirst
|
|
497
|
+
? "its first step is a decision the agent makes — the agent does this task"
|
|
498
|
+
: judgement
|
|
499
|
+
? "its first step needs a judgement the plan cannot compute — the agent does this task"
|
|
500
|
+
: `its first step has failed ${stopping.failureCount ?? 0} times — the agent does this task`;
|
|
489
501
|
return decline(why, unusable
|
|
490
502
|
? "unusable_first_step"
|
|
491
503
|
: judgement
|
|
@@ -526,7 +538,7 @@ export class ReplayController {
|
|
|
526
538
|
// command nobody asked for.
|
|
527
539
|
let deriveCtx = ctx;
|
|
528
540
|
if (ctx.recentResults?.length && sinkOfPlan(planned, host?.sinks) === "command") {
|
|
529
|
-
deriveCtx = { liveCall: ctx.liveCall, agentDay: ctx.agentDay };
|
|
541
|
+
deriveCtx = { liveCall: ctx.liveCall, agentDay: ctx.agentDay, atDecision: ctx.atDecision };
|
|
530
542
|
logLine("replay.derive_context", {
|
|
531
543
|
scenario: scenario.id,
|
|
532
544
|
shell: hasShellStep(planned, host?.sinks) || undefined,
|
|
@@ -747,6 +759,15 @@ export class ReplayController {
|
|
|
747
759
|
...(plan.stopsEarly() ? { stopAt: plan.stepCount } : {}),
|
|
748
760
|
};
|
|
749
761
|
}
|
|
762
|
+
/**
|
|
763
|
+
* Whether the plan's steps end in front of a decision (the service's
|
|
764
|
+
* segmentsbreakdown.md SB-6): it runs the steps before it, then hands the
|
|
765
|
+
* task back there. The control server gives such a plan a usage mark, so
|
|
766
|
+
* that it is billed only up to the stop (SB-26).
|
|
767
|
+
*/
|
|
768
|
+
stopsAtDecision(state) {
|
|
769
|
+
return state.plan?.stopEntry()?.step.stop?.kind === "decision";
|
|
770
|
+
}
|
|
750
771
|
/**
|
|
751
772
|
* Take back a plan that armed and has run nothing, as a decline.
|
|
752
773
|
*
|
|
@@ -2002,16 +2023,20 @@ export class ReplayController {
|
|
|
2002
2023
|
const total = state.deriveCostUsd + d.sessionCostUsd + state.fallbackCostUsd;
|
|
2003
2024
|
const steps = this.stepResultsOf(state);
|
|
2004
2025
|
const isBaseline = state.outcome === "not_steered" || state.outcome === "failed";
|
|
2026
|
+
// The plan handed the task back by design, not by a fault: in front of the
|
|
2027
|
+
// other half of an "if … otherwise" (AC5), or at its decision (the
|
|
2028
|
+
// service's segmentsbreakdown.md SB-19). Such a stop is named whatever the
|
|
2029
|
+
// outcome, and the agent's work after it is the service's evidence.
|
|
2030
|
+
const handedBack = state.handover?.kind === "other_branch" || state.handover?.kind === "decision";
|
|
2005
2031
|
// A costless decline is noise on both sides — *unless* it says something the
|
|
2006
2032
|
// recording page cannot learn any other way: a step that actually broke, the
|
|
2007
|
-
// gate that declined, or a plan that
|
|
2008
|
-
//
|
|
2009
|
-
//
|
|
2010
|
-
// any of them (its errorshandling.md).
|
|
2033
|
+
// gate that declined, or a plan that handed the task back by design. The
|
|
2034
|
+
// service accepts a costless report that carries any of them (its
|
|
2035
|
+
// errorshandling.md).
|
|
2011
2036
|
if (isBaseline &&
|
|
2012
2037
|
total <= 0 &&
|
|
2013
2038
|
!state.declined &&
|
|
2014
|
-
|
|
2039
|
+
!handedBack &&
|
|
2015
2040
|
!steps?.some((s) => s.status === "failed")) {
|
|
2016
2041
|
return undefined;
|
|
2017
2042
|
}
|
|
@@ -2052,14 +2077,15 @@ export class ReplayController {
|
|
|
2052
2077
|
declined: state.declined,
|
|
2053
2078
|
// Where the plan handed the task to the model (fallbk.md). Only on
|
|
2054
2079
|
// `fell_back`: a hand-over before any step ran is `failed`, and a parked
|
|
2055
|
-
// step found after a divergence is still a divergence. Except
|
|
2056
|
-
//
|
|
2057
|
-
//
|
|
2080
|
+
// step found after a divergence is still a divergence. Except a plan
|
|
2081
|
+
// that handed the task back by design — the other half of an "if …
|
|
2082
|
+
// otherwise" (AC5), or its decision (SB-19) — named whatever the
|
|
2083
|
+
// outcome: the agent's work after it is the service's evidence, whether
|
|
2058
2084
|
// or not a step ran first.
|
|
2059
|
-
fallbackStepIndex: state.outcome === "fell_back" ||
|
|
2085
|
+
fallbackStepIndex: state.outcome === "fell_back" || handedBack
|
|
2060
2086
|
? state.handover?.stepIndex
|
|
2061
2087
|
: undefined,
|
|
2062
|
-
fallbackKind: state.outcome === "fell_back" ||
|
|
2088
|
+
fallbackKind: state.outcome === "fell_back" || handedBack
|
|
2063
2089
|
? state.handover?.kind
|
|
2064
2090
|
: undefined,
|
|
2065
2091
|
// The agent did not take the plan's result and redid the task (AC1).
|
|
@@ -2110,6 +2136,10 @@ export class ReplayController {
|
|
|
2110
2136
|
* there is nothing to be similar *to* and nothing to decline. It books no
|
|
2111
2137
|
* execution either — there is no ticket, and inventing a saving for a manual
|
|
2112
2138
|
* invocation is exactly the kind of number a ledger must never contain.
|
|
2139
|
+
*
|
|
2140
|
+
* One served stop is kept: a plan's decision (the service's
|
|
2141
|
+
* segmentsbreakdown.md SB-6). The run stops in front of it, and `stopped`
|
|
2142
|
+
* says so with the reason `decision`.
|
|
2113
2143
|
*/
|
|
2114
2144
|
async runAdHoc(scenario, prompt, wrapped) {
|
|
2115
2145
|
if (!isReadyScenario(scenario))
|
|
@@ -2174,16 +2204,25 @@ export class ReplayController {
|
|
|
2174
2204
|
}
|
|
2175
2205
|
return out;
|
|
2176
2206
|
});
|
|
2207
|
+
// The one served stop an operator's run keeps: a decision (the service's
|
|
2208
|
+
// segmentsbreakdown.md SB-6). The steps after it are one half of an "if …
|
|
2209
|
+
// otherwise", the half the recording took, and running them by id would
|
|
2210
|
+
// do that half's writes whatever the request. Every other served stop is
|
|
2211
|
+
// the plan's health, which an operator's run is there to try.
|
|
2212
|
+
const decisionAt = flat.entries.findIndex((e) => e.step.stop?.kind === "decision");
|
|
2213
|
+
const stopAt = decisionAt >= 0 && (flat.stopAt === undefined || decisionAt < flat.stopAt)
|
|
2214
|
+
? clampToFrameStart(flat.entries, decisionAt)
|
|
2215
|
+
: flat.stopAt;
|
|
2177
2216
|
const plan = new ScenarioReplayPlan({
|
|
2178
2217
|
scenario,
|
|
2179
2218
|
params,
|
|
2180
2219
|
mode,
|
|
2181
2220
|
entries: flat.entries,
|
|
2182
|
-
stopAt
|
|
2221
|
+
stopAt,
|
|
2183
2222
|
fallbackTargets: "samples",
|
|
2184
2223
|
});
|
|
2185
2224
|
state.plan = plan;
|
|
2186
|
-
state.flatStop = flat.stopReason;
|
|
2225
|
+
state.flatStop = stopAt === flat.stopAt ? flat.stopReason : undefined;
|
|
2187
2226
|
if (this.opts.authUrl && this.opts.authToken) {
|
|
2188
2227
|
state.sourceRun = new SourceRunOutputs({
|
|
2189
2228
|
baseUrl: this.opts.authUrl,
|
|
@@ -2230,6 +2269,15 @@ export class ReplayController {
|
|
|
2230
2269
|
}
|
|
2231
2270
|
const stop = result.stopped;
|
|
2232
2271
|
const stoppedStep = stop ? plan.allSteps()[stop.flatIndex ?? plan.stepCount] : undefined;
|
|
2272
|
+
// The plan's end in front of its decision: said so, by its own word.
|
|
2273
|
+
const atDecision = stop?.kind === "known_bad_step" && stoppedStep?.stop?.kind === "decision";
|
|
2274
|
+
if (atDecision) {
|
|
2275
|
+
logLine("replay.adhoc_decision", {
|
|
2276
|
+
scenario: scenario.id,
|
|
2277
|
+
step: stop.stepIndex,
|
|
2278
|
+
why: "bir replay stops in front of the plan's decision: the steps after it are the half the recording took",
|
|
2279
|
+
});
|
|
2280
|
+
}
|
|
2233
2281
|
return {
|
|
2234
2282
|
ok: true,
|
|
2235
2283
|
text: result.text,
|
|
@@ -2241,7 +2289,7 @@ export class ReplayController {
|
|
|
2241
2289
|
? {
|
|
2242
2290
|
step: stop.stepIndex,
|
|
2243
2291
|
tool: stoppedStep?.toolName ?? "",
|
|
2244
|
-
reason: stop.reason ?? stop.stage ?? stop.kind,
|
|
2292
|
+
reason: atDecision ? "decision" : (stop.reason ?? stop.stage ?? stop.kind),
|
|
2245
2293
|
error: stop.error,
|
|
2246
2294
|
}
|
|
2247
2295
|
: undefined,
|
|
@@ -2304,6 +2352,8 @@ export class ReplayController {
|
|
|
2304
2352
|
fetchImpl: this.opts.fetchImpl,
|
|
2305
2353
|
liveCall: ctx.liveCall,
|
|
2306
2354
|
recentResults: ctx.recentResults,
|
|
2355
|
+
// A reading for a scenario that follows a plan's decision (offer1.md O4).
|
|
2356
|
+
...(ctx.atDecision ? { atDecision: ctx.atDecision } : {}),
|
|
2307
2357
|
// The request itself ends too. Without this a hung service or model held
|
|
2308
2358
|
// the socket for up to 300 s after every wait on it had given up. Long
|
|
2309
2359
|
// enough to cover both waits that can use its answer (the arm and the
|
|
@@ -2436,7 +2486,16 @@ export class ReplayController {
|
|
|
2436
2486
|
// when a step did its work — so the service takes the agent's work after
|
|
2437
2487
|
// it as that half's evidence. Booked `diverged`, it never did.
|
|
2438
2488
|
const otherHalf = stop?.kind === "other_branch";
|
|
2439
|
-
|
|
2489
|
+
// A bundle that ran up to the plan's decision and stopped there (the
|
|
2490
|
+
// service's segmentsbreakdown.md SB-17) is a hand-over too: the plan did
|
|
2491
|
+
// all it was for, and the agent decides. Not one of recorded outputs
|
|
2492
|
+
// only, which is no step done here — that stays a plain divergence.
|
|
2493
|
+
const atDecision = !stop &&
|
|
2494
|
+
plan.stopsEarly() &&
|
|
2495
|
+
composed.endedAt === plan.stepCount &&
|
|
2496
|
+
(composed.executed > 0 || composed.recorded === 0) &&
|
|
2497
|
+
this.stopKindOf(state, plan.stopEntry()) === "decision";
|
|
2498
|
+
this.upgrade(state, broke || !didWork ? "failed" : otherHalf || atDecision ? "fell_back" : "diverged");
|
|
2440
2499
|
state.stepsPinned += composed.executed;
|
|
2441
2500
|
// The headline number of this whole design: recovering from a divergence
|
|
2442
2501
|
// costs nothing, because the proxies were already connected.
|
|
@@ -2473,6 +2532,30 @@ export class ReplayController {
|
|
|
2473
2532
|
const note = this.stopNote(state, plan, stop, "none", composed.left);
|
|
2474
2533
|
return this.deliverInstead(note, toolName, state.host !== undefined);
|
|
2475
2534
|
}
|
|
2535
|
+
// Every step left before the decision was one this request does not
|
|
2536
|
+
// need: the plan has done its part, and the agent decides. Told in
|
|
2537
|
+
// place of its call, like the other half. The steps that ran are the
|
|
2538
|
+
// model's own calls above; the ones left out are named.
|
|
2539
|
+
if (atDecision) {
|
|
2540
|
+
const parked = plan.stopStep();
|
|
2541
|
+
state.handover = {
|
|
2542
|
+
kind: "decision",
|
|
2543
|
+
stepIndex: callerStepIndexOf(plan.stopEntry()) ?? parked.stepIndex,
|
|
2544
|
+
toolName: parked.toolName ?? undefined,
|
|
2545
|
+
note: this.noteFor(state, {
|
|
2546
|
+
kind: "decision",
|
|
2547
|
+
step: parked,
|
|
2548
|
+
entry: plan.stopEntry(),
|
|
2549
|
+
position: plan.stepCount,
|
|
2550
|
+
delivery: "above",
|
|
2551
|
+
executedAbove: plan.currentStepIndex,
|
|
2552
|
+
leftOut: composed.left,
|
|
2553
|
+
}),
|
|
2554
|
+
delivered: true,
|
|
2555
|
+
};
|
|
2556
|
+
this.decidedInDivergence(state, plan);
|
|
2557
|
+
return this.deliverInstead(state.handover.note, toolName, state.host !== undefined);
|
|
2558
|
+
}
|
|
2476
2559
|
// Every remaining step was left out by its test: the steps the session
|
|
2477
2560
|
// ran were the whole of the plan this request needs, and it ran to its
|
|
2478
2561
|
// end (newestRun.md N7 reads that).
|
|
@@ -2511,6 +2594,8 @@ export class ReplayController {
|
|
|
2511
2594
|
delivered: true,
|
|
2512
2595
|
};
|
|
2513
2596
|
text = `${state.handover.note}\n\n${composed.text}`;
|
|
2597
|
+
if (atDecision)
|
|
2598
|
+
this.decidedInDivergence(state, plan);
|
|
2514
2599
|
}
|
|
2515
2600
|
else if (!stop) {
|
|
2516
2601
|
// The bundle carries the rest of the chain, and its header tells the
|
|
@@ -2578,6 +2663,12 @@ export class ReplayController {
|
|
|
2578
2663
|
return;
|
|
2579
2664
|
const ran = state.stepsPinned;
|
|
2580
2665
|
this.retire(state, ran > 0 ? "fell_back" : "failed");
|
|
2666
|
+
// A plan stopped at its decision did all it was for: its bill ends here,
|
|
2667
|
+
// and the agent's decision and the half it takes are the agent's own
|
|
2668
|
+
// (the service's segmentsbreakdown.md SB-26). Only once a step ran: a
|
|
2669
|
+
// plan that did nothing is a baseline sample of the whole turn.
|
|
2670
|
+
if (h.kind === "decision" && ran > 0)
|
|
2671
|
+
this.markRetired(state);
|
|
2581
2672
|
state.handover = {
|
|
2582
2673
|
kind: h.kind,
|
|
2583
2674
|
// The *caller's* index: a fragment opens at the call row, not at a step
|
|
@@ -2604,21 +2695,23 @@ export class ReplayController {
|
|
|
2604
2695
|
? h.noneNeeded
|
|
2605
2696
|
? "no step of the plan was needed for this request — the agent does the task"
|
|
2606
2697
|
: `this request is the other half: the steps from here are only for when ${h.whenText ?? "their test says so"} — the agent continues`
|
|
2607
|
-
: h.kind === "
|
|
2608
|
-
? "
|
|
2609
|
-
: h.kind === "
|
|
2610
|
-
? "
|
|
2611
|
-
: h.kind === "
|
|
2612
|
-
? "the next step
|
|
2613
|
-
: h.
|
|
2614
|
-
? "
|
|
2615
|
-
: h.stopReason === "
|
|
2616
|
-
? "
|
|
2617
|
-
: h.stopReason === "
|
|
2618
|
-
? "the
|
|
2619
|
-
: h.stopReason === "
|
|
2620
|
-
? "
|
|
2621
|
-
:
|
|
2698
|
+
: h.kind === "decision"
|
|
2699
|
+
? "the plan ran the steps before its decision — the agent decides which way applies and continues"
|
|
2700
|
+
: h.kind === "unusable_call"
|
|
2701
|
+
? "a sub-task of this chain cannot run — the agent does that part itself"
|
|
2702
|
+
: h.kind === "known_bad_step"
|
|
2703
|
+
? "the next step has failed too often — the agent continues from here"
|
|
2704
|
+
: h.kind === "nondeterministic_step"
|
|
2705
|
+
? "the next step needs a judgement the plan cannot compute — the agent makes it"
|
|
2706
|
+
: h.stopReason === "concurrent"
|
|
2707
|
+
? "a second call arrived while the pinned step was still running — the agent continues from here"
|
|
2708
|
+
: h.stopReason === "budget" || h.stopReason === "cancelled"
|
|
2709
|
+
? "the run was cut short before this step — the agent continues from here"
|
|
2710
|
+
: h.stopReason === "host_rejected" || h.stopReason === "host_edited"
|
|
2711
|
+
? "the host stopped this step itself — no fault of the plan; the agent continues from here"
|
|
2712
|
+
: h.stopReason === "not_run" || h.stopReason === "uncertain"
|
|
2713
|
+
? "a step produced no result here — the agent continues from here"
|
|
2714
|
+
: "a step broke — the agent continues from here",
|
|
2622
2715
|
});
|
|
2623
2716
|
}
|
|
2624
2717
|
/**
|
|
@@ -2628,10 +2721,32 @@ export class ReplayController {
|
|
|
2628
2721
|
* step.
|
|
2629
2722
|
*/
|
|
2630
2723
|
stopKindOf(state, entry) {
|
|
2724
|
+
// The service serves a decision ahead of any other stop of its step
|
|
2725
|
+
// (the service's segmentsbreakdown.md SB-6), and so it is read first.
|
|
2726
|
+
if (entry?.step.stop?.kind === "decision")
|
|
2727
|
+
return "decision";
|
|
2631
2728
|
if (entry?.step.stop?.kind === "nondeterministic")
|
|
2632
2729
|
return "nondeterministic_step";
|
|
2633
2730
|
return state.flatStop ? "unusable_call" : "known_bad_step";
|
|
2634
2731
|
}
|
|
2732
|
+
/**
|
|
2733
|
+
* A divergence's bundle that ended at the plan's decision (SB-17): logged
|
|
2734
|
+
* as `handOver` logs every other hand-over, and its usage mark taken there
|
|
2735
|
+
* when a step of the plan ran (SB-26).
|
|
2736
|
+
*/
|
|
2737
|
+
decidedInDivergence(state, plan) {
|
|
2738
|
+
if (state.stepsPinned > 0)
|
|
2739
|
+
this.markRetired(state);
|
|
2740
|
+
logLine("replay.handover", {
|
|
2741
|
+
scenario: state.scenarioId ?? undefined,
|
|
2742
|
+
kind: "decision",
|
|
2743
|
+
step: `${plan.stepCount}/${plan.totalSteps}`,
|
|
2744
|
+
via: "divergence",
|
|
2745
|
+
ran: state.stepsPinned,
|
|
2746
|
+
outcome: state.outcome,
|
|
2747
|
+
why: "the plan ran the steps before its decision — the agent decides which way applies and continues",
|
|
2748
|
+
});
|
|
2749
|
+
}
|
|
2635
2750
|
/**
|
|
2636
2751
|
* The steps before flat position `bound` that ran, 1-based, and the ones
|
|
2637
2752
|
* their tests left out (AC5) — the plan cursor's own, and a divergence
|
|
@@ -2672,15 +2787,21 @@ export class ReplayController {
|
|
|
2672
2787
|
h.stopReason === "host_rejected" ||
|
|
2673
2788
|
h.stopReason === "host_edited" ||
|
|
2674
2789
|
h.stopReason === "result_map";
|
|
2790
|
+
// A chain that holds a decision (the service's segmentsbreakdown.md SB-18)
|
|
2791
|
+
// keeps the decision note's protections in every note, even when the plan
|
|
2792
|
+
// stops earlier for another reason: no intent, no parameters, and no step
|
|
2793
|
+
// from the decision on. Each points at the half the recording took, and an
|
|
2794
|
+
// agent that follows it there runs that half's segment: the plan decided.
|
|
2795
|
+
const decisionAt = plan ? plan.decisionIndex() : -1;
|
|
2675
2796
|
let params;
|
|
2676
2797
|
try {
|
|
2677
|
-
params = plan?.parameters;
|
|
2798
|
+
params = decisionAt >= 0 ? undefined : plan?.parameters;
|
|
2678
2799
|
}
|
|
2679
2800
|
catch {
|
|
2680
2801
|
params = undefined;
|
|
2681
2802
|
}
|
|
2682
2803
|
return buildHandoverNote({
|
|
2683
|
-
intent: plan?.intent ?? "",
|
|
2804
|
+
intent: decisionAt >= 0 ? "" : (plan?.intent ?? ""),
|
|
2684
2805
|
kind: h.kind,
|
|
2685
2806
|
stepIndex: h.position,
|
|
2686
2807
|
stepToolName: h.step?.toolName ?? undefined,
|
|
@@ -2715,13 +2836,17 @@ export class ReplayController {
|
|
|
2715
2836
|
totalSteps: plan?.totalSteps ?? state.stepsPlanned,
|
|
2716
2837
|
// A step that runs only under a test is marked so: the list must not
|
|
2717
2838
|
// read as steps this request needs (AC5).
|
|
2718
|
-
remainingTools: plan
|
|
2839
|
+
remainingTools: plan
|
|
2840
|
+
? plan.remainingFrom(handled ? h.position + 1 : h.position, decisionAt >= 0 ? decisionAt : undefined)
|
|
2841
|
+
: [],
|
|
2719
2842
|
params,
|
|
2720
2843
|
delivery: h.delivery,
|
|
2721
2844
|
whenText: h.whenText ?? null,
|
|
2722
2845
|
subTask: h.subTask ?? null,
|
|
2723
2846
|
// "None was needed" is the note's own wording for the end of the plan.
|
|
2724
2847
|
...(h.noneNeeded ? { stepIndex: plan?.totalSteps ?? state.stepsPlanned } : {}),
|
|
2848
|
+
// The request's own words for the choice (SB-18); never the service's.
|
|
2849
|
+
...(h.kind === "decision" ? { request: state.request ?? null } : {}),
|
|
2725
2850
|
});
|
|
2726
2851
|
}
|
|
2727
2852
|
/**
|
|
@@ -3228,17 +3353,10 @@ export class ReplayController {
|
|
|
3228
3353
|
this.upgrade(state, outcome);
|
|
3229
3354
|
// Only a plan that *finished* stops the clock (segmented.md R-MONEY-3). A
|
|
3230
3355
|
// hand-over or a divergence takes no mark: the agent's next tokens are the
|
|
3231
|
-
// consequence of this plan stopping, and belong to its bill.
|
|
3232
|
-
|
|
3233
|
-
|
|
3234
|
-
|
|
3235
|
-
state.retiredMark = state.markUsage?.();
|
|
3236
|
-
}
|
|
3237
|
-
catch {
|
|
3238
|
-
// An unreadable transcript leaves the mark undefined, and the window
|
|
3239
|
-
// falls back to the next arm or the end of the turn.
|
|
3240
|
-
}
|
|
3241
|
-
}
|
|
3356
|
+
// consequence of this plan stopping, and belong to its bill. (A decision
|
|
3357
|
+
// hand-over takes one of its own: `markRetired`.)
|
|
3358
|
+
if (completing)
|
|
3359
|
+
this.markRetired(state);
|
|
3242
3360
|
state.retired = true;
|
|
3243
3361
|
state.pinned.clear();
|
|
3244
3362
|
// A note naming a step to call next is void once the plan is: after a
|
|
@@ -3250,6 +3368,21 @@ export class ReplayController {
|
|
|
3250
3368
|
// A call waiting for a pin to thread has nothing left to wait for.
|
|
3251
3369
|
this.notifyPins(state);
|
|
3252
3370
|
}
|
|
3371
|
+
/**
|
|
3372
|
+
* Where the plan's work ended, for its bill: now, and the transcript's
|
|
3373
|
+
* watermark now. Taken at a completing retire, and at a decision hand-over
|
|
3374
|
+
* after a step ran (the service's segmentsbreakdown.md SB-26).
|
|
3375
|
+
*/
|
|
3376
|
+
markRetired(state) {
|
|
3377
|
+
state.retiredAt = Date.now();
|
|
3378
|
+
try {
|
|
3379
|
+
state.retiredMark = state.markUsage?.();
|
|
3380
|
+
}
|
|
3381
|
+
catch {
|
|
3382
|
+
// An unreadable transcript leaves the mark undefined, and the window
|
|
3383
|
+
// falls back to the next arm or the end of the turn.
|
|
3384
|
+
}
|
|
3385
|
+
}
|
|
3253
3386
|
}
|
|
3254
3387
|
/** Every target of a schema. A parameter with no kind is one (R-PARAM-3). */
|
|
3255
3388
|
function targetKeys(schema) {
|
package/dist/replay/derive.d.ts
CHANGED
|
@@ -53,6 +53,16 @@ export interface DeriveContext {
|
|
|
53
53
|
* into the session only after the prompt hook (issue 1212).
|
|
54
54
|
*/
|
|
55
55
|
agentDay?: () => AgentDay | undefined;
|
|
56
|
+
/**
|
|
57
|
+
* A reading for a scenario that can follow a plan's decision, made while the
|
|
58
|
+
* agent decides (the service's offer1.md O4): the plan, and its decision
|
|
59
|
+
* step. With it the service reads this turn's results without a live call,
|
|
60
|
+
* and the prompt as the user's request. Only the service reads it.
|
|
61
|
+
*/
|
|
62
|
+
atDecision?: {
|
|
63
|
+
scenarioId: string;
|
|
64
|
+
stepIndex: number;
|
|
65
|
+
};
|
|
56
66
|
}
|
|
57
67
|
/**
|
|
58
68
|
* The service, which derives when this runner has no key of its own
|
package/dist/replay/derive.js
CHANGED
|
@@ -347,6 +347,9 @@ async function deriveViaService(opts, service, schema, keys) {
|
|
|
347
347
|
// 2023-11-30 …"), which the service turns into the day it reads from
|
|
348
348
|
// (issue 1212). Older services ignore it.
|
|
349
349
|
...(opts.todaySaid ? { todaySaid: opts.todaySaid } : {}),
|
|
350
|
+
// A reading made at a plan's decision (offer1.md O4): the service then
|
|
351
|
+
// takes this turn's results with no live call. Older services ignore it.
|
|
352
|
+
...(opts.atDecision ? { atDecision: opts.atDecision } : {}),
|
|
350
353
|
}),
|
|
351
354
|
signal: opts.signal ?? timeoutSignal(),
|
|
352
355
|
});
|
|
@@ -156,6 +156,13 @@ export interface HandoverNoteInput {
|
|
|
156
156
|
* from there on are for, which this request is not (autocorrect.md AC5).
|
|
157
157
|
*/
|
|
158
158
|
whenText?: string | null;
|
|
159
|
+
/**
|
|
160
|
+
* For `decision`: the live request the plan serves, the user's own words.
|
|
161
|
+
* Its sentences with a condition word are quoted, so the agent sees the
|
|
162
|
+
* choice that is its to make (the service's segmentsbreakdown.md SB-18).
|
|
163
|
+
* Read for no other kind.
|
|
164
|
+
*/
|
|
165
|
+
request?: string | null;
|
|
159
166
|
}
|
|
160
167
|
export declare function buildHandoverNote(n: HandoverNoteInput): string;
|
|
161
168
|
/** "Step 2", "Steps 1–3", "Steps 1, 3 and 5–6": 1-based numbers in order, runs joined. */
|
package/dist/replay/handover.js
CHANGED
|
@@ -11,6 +11,7 @@
|
|
|
11
11
|
* and a tool's own payload is already in front of it where it ran.
|
|
12
12
|
*/
|
|
13
13
|
import { truncate } from "./bundle.js";
|
|
14
|
+
import { conditionSentences } from "./condition-words.js";
|
|
14
15
|
/** Hard ceiling on a note. Long enough to carry derived params, short enough to read. */
|
|
15
16
|
export const MAX_HANDOVER_NOTE = 4_000;
|
|
16
17
|
const MAX_ERROR_CHARS = 500;
|
|
@@ -26,6 +27,8 @@ const UNUSABLE_LINE = {
|
|
|
26
27
|
unsupported: "this runner cannot run",
|
|
27
28
|
};
|
|
28
29
|
export function buildHandoverNote(n) {
|
|
30
|
+
if (n.kind === "decision")
|
|
31
|
+
return decisionNote(n);
|
|
29
32
|
const intent = n.intent.length > 240 ? `${n.intent.slice(0, 240)}…` : n.intent;
|
|
30
33
|
const human = n.stepIndex + 1;
|
|
31
34
|
const tool = n.stepToolName ? ` (${n.stepToolName})` : "";
|
|
@@ -35,6 +38,91 @@ export function buildHandoverNote(n) {
|
|
|
35
38
|
];
|
|
36
39
|
if (intent)
|
|
37
40
|
lines.push(`Intent: ${intent}`);
|
|
41
|
+
lines.push(...ranLines(n));
|
|
42
|
+
if (n.kind === "unusable_call") {
|
|
43
|
+
// A call is a stretch of work, not a tool: the agent is told what the
|
|
44
|
+
// sub-task was, not which tool did not run.
|
|
45
|
+
const what = n.unusable?.callIntent ? ` ("${truncate(n.unusable.callIntent, 200)}")` : "";
|
|
46
|
+
const why = UNUSABLE_LINE[n.unusable?.reason ?? ""] ?? "that cannot run here";
|
|
47
|
+
lines.push(`Step ${human} of ${n.totalSteps} is a sub-task${what} ${why}.`);
|
|
48
|
+
}
|
|
49
|
+
else if (n.kind === "known_bad_step") {
|
|
50
|
+
const times = n.failureCount ? ` ${n.failureCount} times` : "";
|
|
51
|
+
// A parked step inside a called segment names the scenario that owns it:
|
|
52
|
+
// that is where the repair happens, and the agent's own step numbering has
|
|
53
|
+
// no row for it (R-CALL-30).
|
|
54
|
+
const owner = n.inSubTask
|
|
55
|
+
? `, in sub-task "${truncate(n.inSubTask, 120)}"${n.ownerScenarioId ? `, scenario ${n.ownerScenarioId}` : ""}`
|
|
56
|
+
: "";
|
|
57
|
+
lines.push(`Step ${human} of ${n.totalSteps}${tool.replace(/\)$/, `${owner})`)} did not run: it has failed${times} before, so the scenario stops in front of it.`);
|
|
58
|
+
}
|
|
59
|
+
else if (n.kind === "nondeterministic_step") {
|
|
60
|
+
// The calculation could not write code for this step's input: it is a
|
|
61
|
+
// judgement, and the agent is the one to make it (plan-services.md D8).
|
|
62
|
+
lines.push(`Step ${human} of ${n.totalSteps}${tool} did not run: its input needs a judgement the scenario cannot compute. Make that choice yourself and carry on.`);
|
|
63
|
+
}
|
|
64
|
+
else if (n.kind === "other_branch") {
|
|
65
|
+
// The plan holds one half of an "if … otherwise" and this request is the
|
|
66
|
+
// other (autocorrect.md AC5). The steps from here are not wrong, they are
|
|
67
|
+
// not this request's: the model is told what they are for, never to run
|
|
68
|
+
// them, and the list of their tools is left out below for that reason.
|
|
69
|
+
const why = n.whenText ? `they are only for when ${n.whenText}, and that is not so for this request` : "this request is the other case of the task, which they are not for";
|
|
70
|
+
lines.push(n.stepIndex >= n.totalSteps
|
|
71
|
+
? "None of the scenario's steps was needed for this request. Do the task as this request needs it."
|
|
72
|
+
: `Step ${human} of ${n.totalSteps}${tool} and the steps after it did not run: ${why}. Do the rest of the task yourself, as this request needs it.`);
|
|
73
|
+
}
|
|
74
|
+
else {
|
|
75
|
+
lines.push(stepFailedLine(n, human, tool));
|
|
76
|
+
}
|
|
77
|
+
// Not for the other half: those tools are the ones this request must not
|
|
78
|
+
// run, and naming them invites exactly that.
|
|
79
|
+
if (n.remainingTools.length > 0 && n.kind !== "other_branch") {
|
|
80
|
+
lines.push(`The remaining planned steps were: ${n.remainingTools.join(", ")}.`);
|
|
81
|
+
}
|
|
82
|
+
if (n.params && Object.keys(n.params).length > 0) {
|
|
83
|
+
let json;
|
|
84
|
+
try {
|
|
85
|
+
json = JSON.stringify(n.params);
|
|
86
|
+
}
|
|
87
|
+
catch {
|
|
88
|
+
json = "[unserializable]";
|
|
89
|
+
}
|
|
90
|
+
lines.push(`Parameters derived for this request: ${truncate(json, MAX_PARAMS_CHARS)}`);
|
|
91
|
+
}
|
|
92
|
+
return truncate(lines.join("\n"), MAX_HANDOVER_NOTE);
|
|
93
|
+
}
|
|
94
|
+
/**
|
|
95
|
+
* The note of a plan stopped at its decision (the service's
|
|
96
|
+
* segmentsbreakdown.md SB-18): what ran, and the request's own words for the
|
|
97
|
+
* choice the agent now makes. Nothing that names the recorded half: no
|
|
98
|
+
* intent, no remaining steps, no parameters, and never the condition the
|
|
99
|
+
* service keeps (`stop.why`). Each would hint which way the recording went.
|
|
100
|
+
*/
|
|
101
|
+
function decisionNote(n) {
|
|
102
|
+
// Where the results of the steps that ran are, as the ran-steps line says.
|
|
103
|
+
const above = Math.max(0, Math.min(n.executedAbove ?? 0, n.executed));
|
|
104
|
+
const delivery = above >= n.executed ? "above" : n.delivery;
|
|
105
|
+
const from = n.executed === 0
|
|
106
|
+
? ""
|
|
107
|
+
: delivery === "above"
|
|
108
|
+
? " from the results above"
|
|
109
|
+
: delivery === "below"
|
|
110
|
+
? above > 0
|
|
111
|
+
? " from the results above and below"
|
|
112
|
+
: " from the results below"
|
|
113
|
+
: "";
|
|
114
|
+
const quote = conditionSentences(n.request);
|
|
115
|
+
const ask = quote ? `This request asks for a decision: “${quote}”.` : "This task has a decision to make here.";
|
|
116
|
+
const lines = [
|
|
117
|
+
"[BaseInstRunner calculated replay] The calculated scenario for this request ran the steps before a decision, and stopped there.",
|
|
118
|
+
...ranLines(n),
|
|
119
|
+
`${ask} Decide${from} which way applies, then continue the task, calling tools as needed.`,
|
|
120
|
+
];
|
|
121
|
+
return truncate(lines.join("\n"), MAX_HANDOVER_NOTE);
|
|
122
|
+
}
|
|
123
|
+
/** What ran before the stop, and what of it the model may assume: the lines every note carries. */
|
|
124
|
+
function ranLines(n) {
|
|
125
|
+
const lines = [];
|
|
38
126
|
// Steps left out by their tests sit among those before the stop (AC5): the
|
|
39
127
|
// ones that ran are then named, never "steps 1–N", which would say a step
|
|
40
128
|
// ran that did not.
|
|
@@ -100,57 +188,7 @@ export function buildHandoverNote(n) {
|
|
|
100
188
|
lines.push(`Step ${n.inFlight.stepIndex + 1} of ${n.totalSteps}${inTool} was started and has not reported back: ` +
|
|
101
189
|
"it may not have run, or may still complete. Check whether its work happened before you repeat it.");
|
|
102
190
|
}
|
|
103
|
-
|
|
104
|
-
// A call is a stretch of work, not a tool: the agent is told what the
|
|
105
|
-
// sub-task was, not which tool did not run.
|
|
106
|
-
const what = n.unusable?.callIntent ? ` ("${truncate(n.unusable.callIntent, 200)}")` : "";
|
|
107
|
-
const why = UNUSABLE_LINE[n.unusable?.reason ?? ""] ?? "that cannot run here";
|
|
108
|
-
lines.push(`Step ${human} of ${n.totalSteps} is a sub-task${what} ${why}.`);
|
|
109
|
-
}
|
|
110
|
-
else if (n.kind === "known_bad_step") {
|
|
111
|
-
const times = n.failureCount ? ` ${n.failureCount} times` : "";
|
|
112
|
-
// A parked step inside a called segment names the scenario that owns it:
|
|
113
|
-
// that is where the repair happens, and the agent's own step numbering has
|
|
114
|
-
// no row for it (R-CALL-30).
|
|
115
|
-
const owner = n.inSubTask
|
|
116
|
-
? `, in sub-task "${truncate(n.inSubTask, 120)}"${n.ownerScenarioId ? `, scenario ${n.ownerScenarioId}` : ""}`
|
|
117
|
-
: "";
|
|
118
|
-
lines.push(`Step ${human} of ${n.totalSteps}${tool.replace(/\)$/, `${owner})`)} did not run: it has failed${times} before, so the scenario stops in front of it.`);
|
|
119
|
-
}
|
|
120
|
-
else if (n.kind === "nondeterministic_step") {
|
|
121
|
-
// The calculation could not write code for this step's input: it is a
|
|
122
|
-
// judgement, and the agent is the one to make it (plan-services.md D8).
|
|
123
|
-
lines.push(`Step ${human} of ${n.totalSteps}${tool} did not run: its input needs a judgement the scenario cannot compute. Make that choice yourself and carry on.`);
|
|
124
|
-
}
|
|
125
|
-
else if (n.kind === "other_branch") {
|
|
126
|
-
// The plan holds one half of an "if … otherwise" and this request is the
|
|
127
|
-
// other (autocorrect.md AC5). The steps from here are not wrong, they are
|
|
128
|
-
// not this request's: the model is told what they are for, never to run
|
|
129
|
-
// them, and the list of their tools is left out below for that reason.
|
|
130
|
-
const why = n.whenText ? `they are only for when ${n.whenText}, and that is not so for this request` : "this request is the other case of the task, which they are not for";
|
|
131
|
-
lines.push(n.stepIndex >= n.totalSteps
|
|
132
|
-
? "None of the scenario's steps was needed for this request. Do the task as this request needs it."
|
|
133
|
-
: `Step ${human} of ${n.totalSteps}${tool} and the steps after it did not run: ${why}. Do the rest of the task yourself, as this request needs it.`);
|
|
134
|
-
}
|
|
135
|
-
else {
|
|
136
|
-
lines.push(stepFailedLine(n, human, tool));
|
|
137
|
-
}
|
|
138
|
-
// Not for the other half: those tools are the ones this request must not
|
|
139
|
-
// run, and naming them invites exactly that.
|
|
140
|
-
if (n.remainingTools.length > 0 && n.kind !== "other_branch") {
|
|
141
|
-
lines.push(`The remaining planned steps were: ${n.remainingTools.join(", ")}.`);
|
|
142
|
-
}
|
|
143
|
-
if (n.params && Object.keys(n.params).length > 0) {
|
|
144
|
-
let json;
|
|
145
|
-
try {
|
|
146
|
-
json = JSON.stringify(n.params);
|
|
147
|
-
}
|
|
148
|
-
catch {
|
|
149
|
-
json = "[unserializable]";
|
|
150
|
-
}
|
|
151
|
-
lines.push(`Parameters derived for this request: ${truncate(json, MAX_PARAMS_CHARS)}`);
|
|
152
|
-
}
|
|
153
|
-
return truncate(lines.join("\n"), MAX_HANDOVER_NOTE);
|
|
191
|
+
return lines;
|
|
154
192
|
}
|
|
155
193
|
/** "Step 2", "Steps 1–3", "Steps 1, 3 and 5–6": 1-based numbers in order, runs joined. */
|
|
156
194
|
export function stepsPhrase(numbers) {
|