@basein/runner 0.3.18 → 0.3.19
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/bir-hooks.d.ts +11 -8
- package/dist/bin/bir-hooks.js +11 -8
- package/dist/bin/bir.js +20 -7
- package/dist/bin/investigate.d.ts +9 -0
- package/dist/bin/investigate.js +26 -12
- package/dist/bin/scenario-edit.js +10 -0
- package/dist/control/recorder-main.d.ts +15 -0
- package/dist/control/recorder-main.js +33 -8
- package/dist/control/server.d.ts +27 -11
- package/dist/control/server.js +100 -20
- package/dist/record/recorder.d.ts +10 -2
- package/dist/replay/condition-words.d.ts +28 -0
- package/dist/replay/condition-words.js +59 -0
- package/dist/replay/controller.d.ts +45 -6
- package/dist/replay/controller.js +181 -50
- package/dist/replay/handover.d.ts +7 -0
- package/dist/replay/handover.js +89 -51
- package/dist/replay/plan.d.ts +11 -4
- package/dist/replay/plan.js +28 -8
- package/dist/replay/types.d.ts +19 -3
- package/docs/calculatedReplay.md +45 -5
- package/docs/calculatedReplayGuide.md +13 -4
- package/package.json +1 -1
- package/production.md +21 -7
|
@@ -335,13 +335,21 @@ export interface IntentMatchRequest {
|
|
|
335
335
|
stepPosition?: number;
|
|
336
336
|
/** Recordings this turn must not count hits against (R-HIT-7). */
|
|
337
337
|
excludeRuns?: string[];
|
|
338
|
-
/**
|
|
338
|
+
/**
|
|
339
|
+
* Per plan of this turn: its recording, and how far it got in its own chain
|
|
340
|
+
* — the step it handed over at (a decision included), or its last step when
|
|
341
|
+
* it ran to its end (R-OUT-6). One entry per plan; the service maps each onto
|
|
342
|
+
* the recording and keeps the largest (its segmentsbreakdown.md D18).
|
|
343
|
+
*/
|
|
339
344
|
ranThrough?: Array<{
|
|
340
345
|
runId: string;
|
|
341
346
|
position: number;
|
|
342
347
|
scenarioId?: string;
|
|
343
348
|
}>;
|
|
344
|
-
/**
|
|
349
|
+
/**
|
|
350
|
+
* Present only while segment arming is on (R-OUT-9, R-OUT-10): the
|
|
351
|
+
* recorder's default from 0.3.19, absent with `BIR_SEGMENT_ARM=0`.
|
|
352
|
+
*/
|
|
345
353
|
acceptSegments?: boolean;
|
|
346
354
|
/** Always true from this runner (R-CALL-28). */
|
|
347
355
|
supportsCalls?: boolean;
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* condition-words — the request's own words for its decision (the service's
|
|
3
|
+
* segmentsbreakdown.md SB-18).
|
|
4
|
+
*
|
|
5
|
+
* When a plan stops at its decision, the hand-over note quotes the sentences
|
|
6
|
+
* of the live request that hold a word for a condition ("if … otherwise"), so
|
|
7
|
+
* the agent sees which choice is its own to make. Only the request's words:
|
|
8
|
+
* never the condition the service keeps, which is written from the recorded
|
|
9
|
+
* run and would hint at the half the recording took.
|
|
10
|
+
*
|
|
11
|
+
* The word lists are the service's (its src/scenarios/condition.ts), copied
|
|
12
|
+
* exactly. Keep the two the same: the service decides which requests may hold
|
|
13
|
+
* a condition with them, and this quotes the sentences they find.
|
|
14
|
+
*/
|
|
15
|
+
/** The longest quote a decision note carries, in characters. */
|
|
16
|
+
export declare const MAX_DECISION_QUOTE = 600;
|
|
17
|
+
/** Whether a text has a word for a condition in it: the service's `mayHoldCondition`. */
|
|
18
|
+
export declare function mayHoldCondition(text: string): boolean;
|
|
19
|
+
/**
|
|
20
|
+
* The sentences of `request` that hold a word for a condition, in order,
|
|
21
|
+
* each on one line of single spaces, joined by a space. Cut to at most `max`
|
|
22
|
+
* characters, ending in "…" when cut. Null when no sentence holds one.
|
|
23
|
+
*
|
|
24
|
+
* Lines are sentences of their own: the request is split at line breaks
|
|
25
|
+
* first, then at {@link SENTENCE_BREAK}.
|
|
26
|
+
*/
|
|
27
|
+
export declare function conditionSentences(request: string | null | undefined, max?: number): string | null;
|
|
28
|
+
//# sourceMappingURL=condition-words.d.ts.map
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* condition-words — the request's own words for its decision (the service's
|
|
3
|
+
* segmentsbreakdown.md SB-18).
|
|
4
|
+
*
|
|
5
|
+
* When a plan stops at its decision, the hand-over note quotes the sentences
|
|
6
|
+
* of the live request that hold a word for a condition ("if … otherwise"), so
|
|
7
|
+
* the agent sees which choice is its own to make. Only the request's words:
|
|
8
|
+
* never the condition the service keeps, which is written from the recorded
|
|
9
|
+
* run and would hint at the half the recording took.
|
|
10
|
+
*
|
|
11
|
+
* The word lists are the service's (its src/scenarios/condition.ts), copied
|
|
12
|
+
* exactly. Keep the two the same: the service decides which requests may hold
|
|
13
|
+
* a condition with them, and this quotes the sentences they find.
|
|
14
|
+
*/
|
|
15
|
+
/** Words that say a part of a task depends on something: English, and the Hebrew the service reads. */
|
|
16
|
+
const CONDITION_WORDS = /\b(if|otherwise|else|unless|only when|only if|in case|whether)\b/i;
|
|
17
|
+
const HEBREW_CONDITION = /(^|[\s,.;:!?"'()\-])(אם|אחרת|אלא אם|במקרה ש|רק אם|רק כאשר)($|[\s,.;:!?"'()\-])/;
|
|
18
|
+
/** The longest quote a decision note carries, in characters. */
|
|
19
|
+
export const MAX_DECISION_QUOTE = 600;
|
|
20
|
+
/** Whether a text has a word for a condition in it: the service's `mayHoldCondition`. */
|
|
21
|
+
export function mayHoldCondition(text) {
|
|
22
|
+
return CONDITION_WORDS.test(text) || HEBREW_CONDITION.test(text);
|
|
23
|
+
}
|
|
24
|
+
/**
|
|
25
|
+
* Where one sentence ends and the next begins: after `.`, `!`, `?` or `…` and
|
|
26
|
+
* white space, in front of a capital letter, a letter of a script with no
|
|
27
|
+
* capitals (Hebrew), a quote mark or a bracket. Never in front of a digit or a
|
|
28
|
+
* small letter, so "Nov. 20" and "e.g. the" stay in one sentence.
|
|
29
|
+
*/
|
|
30
|
+
const SENTENCE_BREAK = /(?<=[.!?…])\s+(?=[\p{Lu}\p{Lo}"'“‘(\[])/u;
|
|
31
|
+
/**
|
|
32
|
+
* The sentences of `request` that hold a word for a condition, in order,
|
|
33
|
+
* each on one line of single spaces, joined by a space. Cut to at most `max`
|
|
34
|
+
* characters, ending in "…" when cut. Null when no sentence holds one.
|
|
35
|
+
*
|
|
36
|
+
* Lines are sentences of their own: the request is split at line breaks
|
|
37
|
+
* first, then at {@link SENTENCE_BREAK}.
|
|
38
|
+
*/
|
|
39
|
+
export function conditionSentences(request, max = MAX_DECISION_QUOTE) {
|
|
40
|
+
if (typeof request !== "string" || request.trim() === "")
|
|
41
|
+
return null;
|
|
42
|
+
const sentences = request
|
|
43
|
+
.replace(/\r\n?/g, "\n")
|
|
44
|
+
.split("\n")
|
|
45
|
+
.flatMap((line) => line.split(SENTENCE_BREAK))
|
|
46
|
+
.map((s) => s.replace(/\s+/g, " ").trim())
|
|
47
|
+
.filter((s) => s !== "" && mayHoldCondition(s));
|
|
48
|
+
if (sentences.length === 0)
|
|
49
|
+
return null;
|
|
50
|
+
const quote = sentences.join(" ");
|
|
51
|
+
if (quote.length <= max)
|
|
52
|
+
return quote;
|
|
53
|
+
let cut = quote.slice(0, Math.max(0, max - 1));
|
|
54
|
+
// Never half of a character written as two code units (an emoji).
|
|
55
|
+
if (/[\uD800-\uDBFF]$/.test(cut))
|
|
56
|
+
cut = cut.slice(0, -1);
|
|
57
|
+
return `${cut.trimEnd()}…`;
|
|
58
|
+
}
|
|
59
|
+
//# sourceMappingURL=condition-words.js.map
|
|
@@ -240,8 +240,11 @@ export interface ReplayOptions {
|
|
|
240
240
|
/** On unless `BIR_REPLAY=0`. Everything here is inert when false. */
|
|
241
241
|
enabled: boolean;
|
|
242
242
|
/**
|
|
243
|
-
*
|
|
244
|
-
*
|
|
243
|
+
* Whether a handed-out segment may run mid-task. The recorder sets it from
|
|
244
|
+
* `BIR_SEGMENT_ARM`: on unless that is `0`, from runner 0.3.19 (the
|
|
245
|
+
* service's segmentsbreakdown.md SB-21). Omitted, as by a test or an
|
|
246
|
+
* embedder, it is off: observe-only, where the runner asks for no segment,
|
|
247
|
+
* arms none, and only logs what would have armed (R-OUT-10).
|
|
245
248
|
*/
|
|
246
249
|
segmentArm?: boolean;
|
|
247
250
|
/** `BIR_REPLAY_ALLOW_SERVERS`; undefined means every wrapped server. */
|
|
@@ -398,6 +401,14 @@ export interface ReplayState {
|
|
|
398
401
|
retired: boolean;
|
|
399
402
|
/** What armed it: the prompt, or a ReAct iteration's intent (fallbk.md). */
|
|
400
403
|
armedBy: "prompt" | "intent";
|
|
404
|
+
/**
|
|
405
|
+
* The user's request this plan serves, in their own words: the prompt for a
|
|
406
|
+
* plan the prompt armed (set in `arm`), the turn's prompt for one an intent
|
|
407
|
+
* match armed (set by the control server). Read only for the note of a plan
|
|
408
|
+
* stopped at its decision, which quotes the request's condition sentences
|
|
409
|
+
* (the service's segmentsbreakdown.md SB-18). Never sent anywhere.
|
|
410
|
+
*/
|
|
411
|
+
request?: string;
|
|
401
412
|
/**
|
|
402
413
|
* The Claude Code session this plan serves, set by the control server. Its
|
|
403
414
|
* steps run on that session's own proxies (executor.ts). Absent for `bir
|
|
@@ -485,7 +496,10 @@ export interface ReplayState {
|
|
|
485
496
|
* A plan that finished its work stops costing at that moment: the agent's
|
|
486
497
|
* later work in the same turn is the agent's, not this plan's. A plan that
|
|
487
498
|
* handed over or diverged takes no mark, because the follow-on work is the
|
|
488
|
-
* direct consequence of its stopping (R-MONEY-3, R-FALL-5)
|
|
499
|
+
* direct consequence of its stopping (R-MONEY-3, R-FALL-5) — except a plan
|
|
500
|
+
* stopped at its decision after a step ran: it did all it was for, and the
|
|
501
|
+
* agent's decision and the half it chose are the agent's own (the service's
|
|
502
|
+
* segmentsbreakdown.md SB-26).
|
|
489
503
|
*/
|
|
490
504
|
retiredMark?: unknown;
|
|
491
505
|
retiredAt?: number;
|
|
@@ -687,9 +701,11 @@ export declare class ReplayController {
|
|
|
687
701
|
/**
|
|
688
702
|
* Whether a handed-out segment may actually arm (segmented.md R-OUT-10).
|
|
689
703
|
*
|
|
690
|
-
*
|
|
691
|
-
* `BIR_SEGMENT_ARM=
|
|
692
|
-
*
|
|
704
|
+
* Only when the option says so. The recorder passes it on unless
|
|
705
|
+
* `BIR_SEGMENT_ARM=0` (from 0.3.19; the service's segmentsbreakdown.md
|
|
706
|
+
* SB-21); a controller built without it stays observe-only, asks for no
|
|
707
|
+
* segment and arms none. The server enforces the same thing from its side
|
|
708
|
+
* (R-OUT-9).
|
|
693
709
|
*/
|
|
694
710
|
get segmentArm(): boolean;
|
|
695
711
|
/** The plan's own delivery vehicle is never a scenario step. */
|
|
@@ -732,6 +748,13 @@ export declare class ReplayController {
|
|
|
732
748
|
* the field. Built from the same entries the directive lists.
|
|
733
749
|
*/
|
|
734
750
|
planData(state: ReplayState): PlanData | undefined;
|
|
751
|
+
/**
|
|
752
|
+
* Whether the plan's steps end in front of a decision (the service's
|
|
753
|
+
* segmentsbreakdown.md SB-6): it runs the steps before it, then hands the
|
|
754
|
+
* task back there. The control server gives such a plan a usage mark, so
|
|
755
|
+
* that it is billed only up to the stop (SB-26).
|
|
756
|
+
*/
|
|
757
|
+
stopsAtDecision(state: ReplayState): boolean;
|
|
735
758
|
/**
|
|
736
759
|
* Take back a plan that armed and has run nothing, as a decline.
|
|
737
760
|
*
|
|
@@ -1021,6 +1044,10 @@ export declare class ReplayController {
|
|
|
1021
1044
|
* there is nothing to be similar *to* and nothing to decline. It books no
|
|
1022
1045
|
* execution either — there is no ticket, and inventing a saving for a manual
|
|
1023
1046
|
* invocation is exactly the kind of number a ledger must never contain.
|
|
1047
|
+
*
|
|
1048
|
+
* One served stop is kept: a plan's decision (the service's
|
|
1049
|
+
* segmentsbreakdown.md SB-6). The run stops in front of it, and `stopped`
|
|
1050
|
+
* says so with the reason `decision`.
|
|
1024
1051
|
*/
|
|
1025
1052
|
runAdHoc(scenario: SerializedScenario, prompt: string, wrapped: ReadonlySet<string>): Promise<{
|
|
1026
1053
|
ok: boolean;
|
|
@@ -1096,6 +1123,12 @@ export declare class ReplayController {
|
|
|
1096
1123
|
* step.
|
|
1097
1124
|
*/
|
|
1098
1125
|
private stopKindOf;
|
|
1126
|
+
/**
|
|
1127
|
+
* A divergence's bundle that ended at the plan's decision (SB-17): logged
|
|
1128
|
+
* as `handOver` logs every other hand-over, and its usage mark taken there
|
|
1129
|
+
* when a step of the plan ran (SB-26).
|
|
1130
|
+
*/
|
|
1131
|
+
private decidedInDivergence;
|
|
1099
1132
|
/**
|
|
1100
1133
|
* The steps before flat position `bound` that ran, 1-based, and the ones
|
|
1101
1134
|
* their tests left out (AC5) — the plan cursor's own, and a divergence
|
|
@@ -1209,6 +1242,12 @@ export declare class ReplayController {
|
|
|
1209
1242
|
/** Outcomes only move forward; `not_steered` is the floor (§11.1). */
|
|
1210
1243
|
private upgrade;
|
|
1211
1244
|
private retire;
|
|
1245
|
+
/**
|
|
1246
|
+
* Where the plan's work ended, for its bill: now, and the transcript's
|
|
1247
|
+
* watermark now. Taken at a completing retire, and at a decision hand-over
|
|
1248
|
+
* after a step ran (the service's segmentsbreakdown.md SB-26).
|
|
1249
|
+
*/
|
|
1250
|
+
private markRetired;
|
|
1212
1251
|
}
|
|
1213
1252
|
/** The subset of `RunMatch` replay consumes. Structural, so tests need no recorder. */
|
|
1214
1253
|
export interface RunMatchLike {
|
|
@@ -288,9 +288,11 @@ export class ReplayController {
|
|
|
288
288
|
/**
|
|
289
289
|
* Whether a handed-out segment may actually arm (segmented.md R-OUT-10).
|
|
290
290
|
*
|
|
291
|
-
*
|
|
292
|
-
* `BIR_SEGMENT_ARM=
|
|
293
|
-
*
|
|
291
|
+
* Only when the option says so. The recorder passes it on unless
|
|
292
|
+
* `BIR_SEGMENT_ARM=0` (from 0.3.19; the service's segmentsbreakdown.md
|
|
293
|
+
* SB-21); a controller built without it stays observe-only, asks for no
|
|
294
|
+
* segment and arms none. The server enforces the same thing from its side
|
|
295
|
+
* (R-OUT-9).
|
|
294
296
|
*/
|
|
295
297
|
get segmentArm() {
|
|
296
298
|
return this.opts.segmentArm === true;
|
|
@@ -389,6 +391,10 @@ export class ReplayController {
|
|
|
389
391
|
*/
|
|
390
392
|
async arm(match, prompt, wrapped, armedBy = "prompt", ctx = {}, host) {
|
|
391
393
|
const state = this.blankState(match, wrapped, armedBy, host);
|
|
394
|
+
// A decision note quotes the request (SB-18); a plan armed by intent was
|
|
395
|
+
// given the step's intent here, and its control server sets the request.
|
|
396
|
+
if (armedBy === "prompt")
|
|
397
|
+
state.request = prompt;
|
|
392
398
|
const isSegment = state.kind === "segment";
|
|
393
399
|
const decline = (why, code) => {
|
|
394
400
|
state.declined = code;
|
|
@@ -480,12 +486,18 @@ export class ReplayController {
|
|
|
480
486
|
const unusable = flat.stopAt === 0 && flat.stopReason ? flat.stopReason : undefined;
|
|
481
487
|
// The step that stops the plan, before the clamp: the reason is its own.
|
|
482
488
|
const stopping = entries[Math.min(...rawStop)]?.step ?? entries[0].step;
|
|
483
|
-
|
|
489
|
+
// The service never serves a decision at step 0 (segmentsbreakdown.md
|
|
490
|
+
// SB-5): should one come, the agent decides from the start, as it makes
|
|
491
|
+
// a judgement there. No new code reaches a service that may not know it.
|
|
492
|
+
const decisionFirst = stopping.stop?.kind === "decision";
|
|
493
|
+
const judgement = stopping.stop?.kind === "nondeterministic" || decisionFirst;
|
|
484
494
|
const why = unusable
|
|
485
495
|
? `its first step is a call that cannot run: ${unusable.reason}`
|
|
486
|
-
:
|
|
487
|
-
? "its first step
|
|
488
|
-
:
|
|
496
|
+
: decisionFirst
|
|
497
|
+
? "its first step is a decision the agent makes — the agent does this task"
|
|
498
|
+
: judgement
|
|
499
|
+
? "its first step needs a judgement the plan cannot compute — the agent does this task"
|
|
500
|
+
: `its first step has failed ${stopping.failureCount ?? 0} times — the agent does this task`;
|
|
489
501
|
return decline(why, unusable
|
|
490
502
|
? "unusable_first_step"
|
|
491
503
|
: judgement
|
|
@@ -747,6 +759,15 @@ export class ReplayController {
|
|
|
747
759
|
...(plan.stopsEarly() ? { stopAt: plan.stepCount } : {}),
|
|
748
760
|
};
|
|
749
761
|
}
|
|
762
|
+
/**
|
|
763
|
+
* Whether the plan's steps end in front of a decision (the service's
|
|
764
|
+
* segmentsbreakdown.md SB-6): it runs the steps before it, then hands the
|
|
765
|
+
* task back there. The control server gives such a plan a usage mark, so
|
|
766
|
+
* that it is billed only up to the stop (SB-26).
|
|
767
|
+
*/
|
|
768
|
+
stopsAtDecision(state) {
|
|
769
|
+
return state.plan?.stopEntry()?.step.stop?.kind === "decision";
|
|
770
|
+
}
|
|
750
771
|
/**
|
|
751
772
|
* Take back a plan that armed and has run nothing, as a decline.
|
|
752
773
|
*
|
|
@@ -2002,16 +2023,20 @@ export class ReplayController {
|
|
|
2002
2023
|
const total = state.deriveCostUsd + d.sessionCostUsd + state.fallbackCostUsd;
|
|
2003
2024
|
const steps = this.stepResultsOf(state);
|
|
2004
2025
|
const isBaseline = state.outcome === "not_steered" || state.outcome === "failed";
|
|
2026
|
+
// The plan handed the task back by design, not by a fault: in front of the
|
|
2027
|
+
// other half of an "if … otherwise" (AC5), or at its decision (the
|
|
2028
|
+
// service's segmentsbreakdown.md SB-19). Such a stop is named whatever the
|
|
2029
|
+
// outcome, and the agent's work after it is the service's evidence.
|
|
2030
|
+
const handedBack = state.handover?.kind === "other_branch" || state.handover?.kind === "decision";
|
|
2005
2031
|
// A costless decline is noise on both sides — *unless* it says something the
|
|
2006
2032
|
// recording page cannot learn any other way: a step that actually broke, the
|
|
2007
|
-
// gate that declined, or a plan that
|
|
2008
|
-
//
|
|
2009
|
-
//
|
|
2010
|
-
// any of them (its errorshandling.md).
|
|
2033
|
+
// gate that declined, or a plan that handed the task back by design. The
|
|
2034
|
+
// service accepts a costless report that carries any of them (its
|
|
2035
|
+
// errorshandling.md).
|
|
2011
2036
|
if (isBaseline &&
|
|
2012
2037
|
total <= 0 &&
|
|
2013
2038
|
!state.declined &&
|
|
2014
|
-
|
|
2039
|
+
!handedBack &&
|
|
2015
2040
|
!steps?.some((s) => s.status === "failed")) {
|
|
2016
2041
|
return undefined;
|
|
2017
2042
|
}
|
|
@@ -2052,14 +2077,15 @@ export class ReplayController {
|
|
|
2052
2077
|
declined: state.declined,
|
|
2053
2078
|
// Where the plan handed the task to the model (fallbk.md). Only on
|
|
2054
2079
|
// `fell_back`: a hand-over before any step ran is `failed`, and a parked
|
|
2055
|
-
// step found after a divergence is still a divergence. Except
|
|
2056
|
-
//
|
|
2057
|
-
//
|
|
2080
|
+
// step found after a divergence is still a divergence. Except a plan
|
|
2081
|
+
// that handed the task back by design — the other half of an "if …
|
|
2082
|
+
// otherwise" (AC5), or its decision (SB-19) — named whatever the
|
|
2083
|
+
// outcome: the agent's work after it is the service's evidence, whether
|
|
2058
2084
|
// or not a step ran first.
|
|
2059
|
-
fallbackStepIndex: state.outcome === "fell_back" ||
|
|
2085
|
+
fallbackStepIndex: state.outcome === "fell_back" || handedBack
|
|
2060
2086
|
? state.handover?.stepIndex
|
|
2061
2087
|
: undefined,
|
|
2062
|
-
fallbackKind: state.outcome === "fell_back" ||
|
|
2088
|
+
fallbackKind: state.outcome === "fell_back" || handedBack
|
|
2063
2089
|
? state.handover?.kind
|
|
2064
2090
|
: undefined,
|
|
2065
2091
|
// The agent did not take the plan's result and redid the task (AC1).
|
|
@@ -2110,6 +2136,10 @@ export class ReplayController {
|
|
|
2110
2136
|
* there is nothing to be similar *to* and nothing to decline. It books no
|
|
2111
2137
|
* execution either — there is no ticket, and inventing a saving for a manual
|
|
2112
2138
|
* invocation is exactly the kind of number a ledger must never contain.
|
|
2139
|
+
*
|
|
2140
|
+
* One served stop is kept: a plan's decision (the service's
|
|
2141
|
+
* segmentsbreakdown.md SB-6). The run stops in front of it, and `stopped`
|
|
2142
|
+
* says so with the reason `decision`.
|
|
2113
2143
|
*/
|
|
2114
2144
|
async runAdHoc(scenario, prompt, wrapped) {
|
|
2115
2145
|
if (!isReadyScenario(scenario))
|
|
@@ -2174,16 +2204,25 @@ export class ReplayController {
|
|
|
2174
2204
|
}
|
|
2175
2205
|
return out;
|
|
2176
2206
|
});
|
|
2207
|
+
// The one served stop an operator's run keeps: a decision (the service's
|
|
2208
|
+
// segmentsbreakdown.md SB-6). The steps after it are one half of an "if …
|
|
2209
|
+
// otherwise", the half the recording took, and running them by id would
|
|
2210
|
+
// do that half's writes whatever the request. Every other served stop is
|
|
2211
|
+
// the plan's health, which an operator's run is there to try.
|
|
2212
|
+
const decisionAt = flat.entries.findIndex((e) => e.step.stop?.kind === "decision");
|
|
2213
|
+
const stopAt = decisionAt >= 0 && (flat.stopAt === undefined || decisionAt < flat.stopAt)
|
|
2214
|
+
? clampToFrameStart(flat.entries, decisionAt)
|
|
2215
|
+
: flat.stopAt;
|
|
2177
2216
|
const plan = new ScenarioReplayPlan({
|
|
2178
2217
|
scenario,
|
|
2179
2218
|
params,
|
|
2180
2219
|
mode,
|
|
2181
2220
|
entries: flat.entries,
|
|
2182
|
-
stopAt
|
|
2221
|
+
stopAt,
|
|
2183
2222
|
fallbackTargets: "samples",
|
|
2184
2223
|
});
|
|
2185
2224
|
state.plan = plan;
|
|
2186
|
-
state.flatStop = flat.stopReason;
|
|
2225
|
+
state.flatStop = stopAt === flat.stopAt ? flat.stopReason : undefined;
|
|
2187
2226
|
if (this.opts.authUrl && this.opts.authToken) {
|
|
2188
2227
|
state.sourceRun = new SourceRunOutputs({
|
|
2189
2228
|
baseUrl: this.opts.authUrl,
|
|
@@ -2230,6 +2269,15 @@ export class ReplayController {
|
|
|
2230
2269
|
}
|
|
2231
2270
|
const stop = result.stopped;
|
|
2232
2271
|
const stoppedStep = stop ? plan.allSteps()[stop.flatIndex ?? plan.stepCount] : undefined;
|
|
2272
|
+
// The plan's end in front of its decision: said so, by its own word.
|
|
2273
|
+
const atDecision = stop?.kind === "known_bad_step" && stoppedStep?.stop?.kind === "decision";
|
|
2274
|
+
if (atDecision) {
|
|
2275
|
+
logLine("replay.adhoc_decision", {
|
|
2276
|
+
scenario: scenario.id,
|
|
2277
|
+
step: stop.stepIndex,
|
|
2278
|
+
why: "bir replay stops in front of the plan's decision: the steps after it are the half the recording took",
|
|
2279
|
+
});
|
|
2280
|
+
}
|
|
2233
2281
|
return {
|
|
2234
2282
|
ok: true,
|
|
2235
2283
|
text: result.text,
|
|
@@ -2241,7 +2289,7 @@ export class ReplayController {
|
|
|
2241
2289
|
? {
|
|
2242
2290
|
step: stop.stepIndex,
|
|
2243
2291
|
tool: stoppedStep?.toolName ?? "",
|
|
2244
|
-
reason: stop.reason ?? stop.stage ?? stop.kind,
|
|
2292
|
+
reason: atDecision ? "decision" : (stop.reason ?? stop.stage ?? stop.kind),
|
|
2245
2293
|
error: stop.error,
|
|
2246
2294
|
}
|
|
2247
2295
|
: undefined,
|
|
@@ -2436,7 +2484,16 @@ export class ReplayController {
|
|
|
2436
2484
|
// when a step did its work — so the service takes the agent's work after
|
|
2437
2485
|
// it as that half's evidence. Booked `diverged`, it never did.
|
|
2438
2486
|
const otherHalf = stop?.kind === "other_branch";
|
|
2439
|
-
|
|
2487
|
+
// A bundle that ran up to the plan's decision and stopped there (the
|
|
2488
|
+
// service's segmentsbreakdown.md SB-17) is a hand-over too: the plan did
|
|
2489
|
+
// all it was for, and the agent decides. Not one of recorded outputs
|
|
2490
|
+
// only, which is no step done here — that stays a plain divergence.
|
|
2491
|
+
const atDecision = !stop &&
|
|
2492
|
+
plan.stopsEarly() &&
|
|
2493
|
+
composed.endedAt === plan.stepCount &&
|
|
2494
|
+
(composed.executed > 0 || composed.recorded === 0) &&
|
|
2495
|
+
this.stopKindOf(state, plan.stopEntry()) === "decision";
|
|
2496
|
+
this.upgrade(state, broke || !didWork ? "failed" : otherHalf || atDecision ? "fell_back" : "diverged");
|
|
2440
2497
|
state.stepsPinned += composed.executed;
|
|
2441
2498
|
// The headline number of this whole design: recovering from a divergence
|
|
2442
2499
|
// costs nothing, because the proxies were already connected.
|
|
@@ -2473,6 +2530,30 @@ export class ReplayController {
|
|
|
2473
2530
|
const note = this.stopNote(state, plan, stop, "none", composed.left);
|
|
2474
2531
|
return this.deliverInstead(note, toolName, state.host !== undefined);
|
|
2475
2532
|
}
|
|
2533
|
+
// Every step left before the decision was one this request does not
|
|
2534
|
+
// need: the plan has done its part, and the agent decides. Told in
|
|
2535
|
+
// place of its call, like the other half. The steps that ran are the
|
|
2536
|
+
// model's own calls above; the ones left out are named.
|
|
2537
|
+
if (atDecision) {
|
|
2538
|
+
const parked = plan.stopStep();
|
|
2539
|
+
state.handover = {
|
|
2540
|
+
kind: "decision",
|
|
2541
|
+
stepIndex: callerStepIndexOf(plan.stopEntry()) ?? parked.stepIndex,
|
|
2542
|
+
toolName: parked.toolName ?? undefined,
|
|
2543
|
+
note: this.noteFor(state, {
|
|
2544
|
+
kind: "decision",
|
|
2545
|
+
step: parked,
|
|
2546
|
+
entry: plan.stopEntry(),
|
|
2547
|
+
position: plan.stepCount,
|
|
2548
|
+
delivery: "above",
|
|
2549
|
+
executedAbove: plan.currentStepIndex,
|
|
2550
|
+
leftOut: composed.left,
|
|
2551
|
+
}),
|
|
2552
|
+
delivered: true,
|
|
2553
|
+
};
|
|
2554
|
+
this.decidedInDivergence(state, plan);
|
|
2555
|
+
return this.deliverInstead(state.handover.note, toolName, state.host !== undefined);
|
|
2556
|
+
}
|
|
2476
2557
|
// Every remaining step was left out by its test: the steps the session
|
|
2477
2558
|
// ran were the whole of the plan this request needs, and it ran to its
|
|
2478
2559
|
// end (newestRun.md N7 reads that).
|
|
@@ -2511,6 +2592,8 @@ export class ReplayController {
|
|
|
2511
2592
|
delivered: true,
|
|
2512
2593
|
};
|
|
2513
2594
|
text = `${state.handover.note}\n\n${composed.text}`;
|
|
2595
|
+
if (atDecision)
|
|
2596
|
+
this.decidedInDivergence(state, plan);
|
|
2514
2597
|
}
|
|
2515
2598
|
else if (!stop) {
|
|
2516
2599
|
// The bundle carries the rest of the chain, and its header tells the
|
|
@@ -2578,6 +2661,12 @@ export class ReplayController {
|
|
|
2578
2661
|
return;
|
|
2579
2662
|
const ran = state.stepsPinned;
|
|
2580
2663
|
this.retire(state, ran > 0 ? "fell_back" : "failed");
|
|
2664
|
+
// A plan stopped at its decision did all it was for: its bill ends here,
|
|
2665
|
+
// and the agent's decision and the half it takes are the agent's own
|
|
2666
|
+
// (the service's segmentsbreakdown.md SB-26). Only once a step ran: a
|
|
2667
|
+
// plan that did nothing is a baseline sample of the whole turn.
|
|
2668
|
+
if (h.kind === "decision" && ran > 0)
|
|
2669
|
+
this.markRetired(state);
|
|
2581
2670
|
state.handover = {
|
|
2582
2671
|
kind: h.kind,
|
|
2583
2672
|
// The *caller's* index: a fragment opens at the call row, not at a step
|
|
@@ -2604,21 +2693,23 @@ export class ReplayController {
|
|
|
2604
2693
|
? h.noneNeeded
|
|
2605
2694
|
? "no step of the plan was needed for this request — the agent does the task"
|
|
2606
2695
|
: `this request is the other half: the steps from here are only for when ${h.whenText ?? "their test says so"} — the agent continues`
|
|
2607
|
-
: h.kind === "
|
|
2608
|
-
? "
|
|
2609
|
-
: h.kind === "
|
|
2610
|
-
? "
|
|
2611
|
-
: h.kind === "
|
|
2612
|
-
? "the next step
|
|
2613
|
-
: h.
|
|
2614
|
-
? "
|
|
2615
|
-
: h.stopReason === "
|
|
2616
|
-
? "
|
|
2617
|
-
: h.stopReason === "
|
|
2618
|
-
? "the
|
|
2619
|
-
: h.stopReason === "
|
|
2620
|
-
? "
|
|
2621
|
-
:
|
|
2696
|
+
: h.kind === "decision"
|
|
2697
|
+
? "the plan ran the steps before its decision — the agent decides which way applies and continues"
|
|
2698
|
+
: h.kind === "unusable_call"
|
|
2699
|
+
? "a sub-task of this chain cannot run — the agent does that part itself"
|
|
2700
|
+
: h.kind === "known_bad_step"
|
|
2701
|
+
? "the next step has failed too often — the agent continues from here"
|
|
2702
|
+
: h.kind === "nondeterministic_step"
|
|
2703
|
+
? "the next step needs a judgement the plan cannot compute — the agent makes it"
|
|
2704
|
+
: h.stopReason === "concurrent"
|
|
2705
|
+
? "a second call arrived while the pinned step was still running — the agent continues from here"
|
|
2706
|
+
: h.stopReason === "budget" || h.stopReason === "cancelled"
|
|
2707
|
+
? "the run was cut short before this step — the agent continues from here"
|
|
2708
|
+
: h.stopReason === "host_rejected" || h.stopReason === "host_edited"
|
|
2709
|
+
? "the host stopped this step itself — no fault of the plan; the agent continues from here"
|
|
2710
|
+
: h.stopReason === "not_run" || h.stopReason === "uncertain"
|
|
2711
|
+
? "a step produced no result here — the agent continues from here"
|
|
2712
|
+
: "a step broke — the agent continues from here",
|
|
2622
2713
|
});
|
|
2623
2714
|
}
|
|
2624
2715
|
/**
|
|
@@ -2628,10 +2719,32 @@ export class ReplayController {
|
|
|
2628
2719
|
* step.
|
|
2629
2720
|
*/
|
|
2630
2721
|
stopKindOf(state, entry) {
|
|
2722
|
+
// The service serves a decision ahead of any other stop of its step
|
|
2723
|
+
// (the service's segmentsbreakdown.md SB-6), and so it is read first.
|
|
2724
|
+
if (entry?.step.stop?.kind === "decision")
|
|
2725
|
+
return "decision";
|
|
2631
2726
|
if (entry?.step.stop?.kind === "nondeterministic")
|
|
2632
2727
|
return "nondeterministic_step";
|
|
2633
2728
|
return state.flatStop ? "unusable_call" : "known_bad_step";
|
|
2634
2729
|
}
|
|
2730
|
+
/**
|
|
2731
|
+
* A divergence's bundle that ended at the plan's decision (SB-17): logged
|
|
2732
|
+
* as `handOver` logs every other hand-over, and its usage mark taken there
|
|
2733
|
+
* when a step of the plan ran (SB-26).
|
|
2734
|
+
*/
|
|
2735
|
+
decidedInDivergence(state, plan) {
|
|
2736
|
+
if (state.stepsPinned > 0)
|
|
2737
|
+
this.markRetired(state);
|
|
2738
|
+
logLine("replay.handover", {
|
|
2739
|
+
scenario: state.scenarioId ?? undefined,
|
|
2740
|
+
kind: "decision",
|
|
2741
|
+
step: `${plan.stepCount}/${plan.totalSteps}`,
|
|
2742
|
+
via: "divergence",
|
|
2743
|
+
ran: state.stepsPinned,
|
|
2744
|
+
outcome: state.outcome,
|
|
2745
|
+
why: "the plan ran the steps before its decision — the agent decides which way applies and continues",
|
|
2746
|
+
});
|
|
2747
|
+
}
|
|
2635
2748
|
/**
|
|
2636
2749
|
* The steps before flat position `bound` that ran, 1-based, and the ones
|
|
2637
2750
|
* their tests left out (AC5) — the plan cursor's own, and a divergence
|
|
@@ -2672,15 +2785,21 @@ export class ReplayController {
|
|
|
2672
2785
|
h.stopReason === "host_rejected" ||
|
|
2673
2786
|
h.stopReason === "host_edited" ||
|
|
2674
2787
|
h.stopReason === "result_map";
|
|
2788
|
+
// A chain that holds a decision (the service's segmentsbreakdown.md SB-18)
|
|
2789
|
+
// keeps the decision note's protections in every note, even when the plan
|
|
2790
|
+
// stops earlier for another reason: no intent, no parameters, and no step
|
|
2791
|
+
// from the decision on. Each points at the half the recording took, and an
|
|
2792
|
+
// agent that follows it there runs that half's segment: the plan decided.
|
|
2793
|
+
const decisionAt = plan ? plan.decisionIndex() : -1;
|
|
2675
2794
|
let params;
|
|
2676
2795
|
try {
|
|
2677
|
-
params = plan?.parameters;
|
|
2796
|
+
params = decisionAt >= 0 ? undefined : plan?.parameters;
|
|
2678
2797
|
}
|
|
2679
2798
|
catch {
|
|
2680
2799
|
params = undefined;
|
|
2681
2800
|
}
|
|
2682
2801
|
return buildHandoverNote({
|
|
2683
|
-
intent: plan?.intent ?? "",
|
|
2802
|
+
intent: decisionAt >= 0 ? "" : (plan?.intent ?? ""),
|
|
2684
2803
|
kind: h.kind,
|
|
2685
2804
|
stepIndex: h.position,
|
|
2686
2805
|
stepToolName: h.step?.toolName ?? undefined,
|
|
@@ -2715,13 +2834,17 @@ export class ReplayController {
|
|
|
2715
2834
|
totalSteps: plan?.totalSteps ?? state.stepsPlanned,
|
|
2716
2835
|
// A step that runs only under a test is marked so: the list must not
|
|
2717
2836
|
// read as steps this request needs (AC5).
|
|
2718
|
-
remainingTools: plan
|
|
2837
|
+
remainingTools: plan
|
|
2838
|
+
? plan.remainingFrom(handled ? h.position + 1 : h.position, decisionAt >= 0 ? decisionAt : undefined)
|
|
2839
|
+
: [],
|
|
2719
2840
|
params,
|
|
2720
2841
|
delivery: h.delivery,
|
|
2721
2842
|
whenText: h.whenText ?? null,
|
|
2722
2843
|
subTask: h.subTask ?? null,
|
|
2723
2844
|
// "None was needed" is the note's own wording for the end of the plan.
|
|
2724
2845
|
...(h.noneNeeded ? { stepIndex: plan?.totalSteps ?? state.stepsPlanned } : {}),
|
|
2846
|
+
// The request's own words for the choice (SB-18); never the service's.
|
|
2847
|
+
...(h.kind === "decision" ? { request: state.request ?? null } : {}),
|
|
2725
2848
|
});
|
|
2726
2849
|
}
|
|
2727
2850
|
/**
|
|
@@ -3228,17 +3351,10 @@ export class ReplayController {
|
|
|
3228
3351
|
this.upgrade(state, outcome);
|
|
3229
3352
|
// Only a plan that *finished* stops the clock (segmented.md R-MONEY-3). A
|
|
3230
3353
|
// hand-over or a divergence takes no mark: the agent's next tokens are the
|
|
3231
|
-
// consequence of this plan stopping, and belong to its bill.
|
|
3232
|
-
|
|
3233
|
-
|
|
3234
|
-
|
|
3235
|
-
state.retiredMark = state.markUsage?.();
|
|
3236
|
-
}
|
|
3237
|
-
catch {
|
|
3238
|
-
// An unreadable transcript leaves the mark undefined, and the window
|
|
3239
|
-
// falls back to the next arm or the end of the turn.
|
|
3240
|
-
}
|
|
3241
|
-
}
|
|
3354
|
+
// consequence of this plan stopping, and belong to its bill. (A decision
|
|
3355
|
+
// hand-over takes one of its own: `markRetired`.)
|
|
3356
|
+
if (completing)
|
|
3357
|
+
this.markRetired(state);
|
|
3242
3358
|
state.retired = true;
|
|
3243
3359
|
state.pinned.clear();
|
|
3244
3360
|
// A note naming a step to call next is void once the plan is: after a
|
|
@@ -3250,6 +3366,21 @@ export class ReplayController {
|
|
|
3250
3366
|
// A call waiting for a pin to thread has nothing left to wait for.
|
|
3251
3367
|
this.notifyPins(state);
|
|
3252
3368
|
}
|
|
3369
|
+
/**
|
|
3370
|
+
* Where the plan's work ended, for its bill: now, and the transcript's
|
|
3371
|
+
* watermark now. Taken at a completing retire, and at a decision hand-over
|
|
3372
|
+
* after a step ran (the service's segmentsbreakdown.md SB-26).
|
|
3373
|
+
*/
|
|
3374
|
+
markRetired(state) {
|
|
3375
|
+
state.retiredAt = Date.now();
|
|
3376
|
+
try {
|
|
3377
|
+
state.retiredMark = state.markUsage?.();
|
|
3378
|
+
}
|
|
3379
|
+
catch {
|
|
3380
|
+
// An unreadable transcript leaves the mark undefined, and the window
|
|
3381
|
+
// falls back to the next arm or the end of the turn.
|
|
3382
|
+
}
|
|
3383
|
+
}
|
|
3253
3384
|
}
|
|
3254
3385
|
/** Every target of a schema. A parameter with no kind is one (R-PARAM-3). */
|
|
3255
3386
|
function targetKeys(schema) {
|
|
@@ -156,6 +156,13 @@ export interface HandoverNoteInput {
|
|
|
156
156
|
* from there on are for, which this request is not (autocorrect.md AC5).
|
|
157
157
|
*/
|
|
158
158
|
whenText?: string | null;
|
|
159
|
+
/**
|
|
160
|
+
* For `decision`: the live request the plan serves, the user's own words.
|
|
161
|
+
* Its sentences with a condition word are quoted, so the agent sees the
|
|
162
|
+
* choice that is its to make (the service's segmentsbreakdown.md SB-18).
|
|
163
|
+
* Read for no other kind.
|
|
164
|
+
*/
|
|
165
|
+
request?: string | null;
|
|
159
166
|
}
|
|
160
167
|
export declare function buildHandoverNote(n: HandoverNoteInput): string;
|
|
161
168
|
/** "Step 2", "Steps 1–3", "Steps 1, 3 and 5–6": 1-based numbers in order, runs joined. */
|