@basein/runner 0.2.7 → 0.2.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/investigate.js +5 -1
- package/dist/control/server.js +7 -4
- package/dist/replay/controller.d.ts +24 -1
- package/dist/replay/controller.js +76 -20
- package/dist/replay/handover.js +5 -0
- package/dist/replay/plan.d.ts +2 -0
- package/dist/replay/plan.js +53 -6
- package/dist/replay/pricing.d.ts +1 -1
- package/dist/replay/pricing.js +12 -4
- package/dist/replay/tool-error.d.ts +15 -0
- package/dist/replay/tool-error.js +17 -0
- package/dist/replay/types.d.ts +48 -1
- package/docs/calculatedReplayGuide.md +2 -0
- package/package.json +1 -1
package/dist/bin/investigate.js
CHANGED
|
@@ -179,7 +179,11 @@ export const DECLINES = {
|
|
|
179
179
|
},
|
|
180
180
|
known_bad_first_step: {
|
|
181
181
|
cause: "the chain's first step is parked after repeated failures.",
|
|
182
|
-
fix: "
|
|
182
|
+
fix: "wait: the service retries it once an hour for a day, then repairs the plan from the next run of the prompt. To repair now: `bir scenario calc <runId> --force`.",
|
|
183
|
+
},
|
|
184
|
+
nondeterministic_first_step: {
|
|
185
|
+
cause: "the chain's first step needs a judgement the calculation could not write code for, so the plan cannot start.",
|
|
186
|
+
fix: "the agent does this task itself; a model step for such judgements is planned (plan-services.md W3.1).",
|
|
183
187
|
},
|
|
184
188
|
unusable_first_step: {
|
|
185
189
|
cause: "the first step calls a sub-task that is gone, switched off or stale.",
|
package/dist/control/server.js
CHANGED
|
@@ -944,11 +944,14 @@ export class ControlServer {
|
|
|
944
944
|
continue;
|
|
945
945
|
let position;
|
|
946
946
|
if (!state.plan) {
|
|
947
|
-
// Only a
|
|
948
|
-
//
|
|
949
|
-
// decline leaves the recording
|
|
950
|
-
|
|
947
|
+
// Only a first step the plan cannot start on — parked, or a judgement
|
|
948
|
+
// — means the turn is committed to doing this recording's work itself
|
|
949
|
+
// from position 0 (R-HIT-14); every other decline leaves the recording
|
|
950
|
+
// untouched.
|
|
951
|
+
if (state.declined === "known_bad_first_step" ||
|
|
952
|
+
state.declined === "nondeterministic_first_step") {
|
|
951
953
|
position = 0;
|
|
954
|
+
}
|
|
952
955
|
}
|
|
953
956
|
else if (state.handover) {
|
|
954
957
|
position = state.handover.stepIndex;
|
|
@@ -219,7 +219,7 @@ export interface ReplayState {
|
|
|
219
219
|
* Why a match did not arm. One code per gate of the ladder, so the caller can
|
|
220
220
|
* act on a decline without parsing the sentence that explains it.
|
|
221
221
|
*/
|
|
222
|
-
export type DeclineCode = "replay_disabled" | "not_ready" | "flatten_failed" | "similarity" | "known_bad_first_step" | "unusable_first_step" | "coverage" | "missing_target" | "no_derive_key";
|
|
222
|
+
export type DeclineCode = "replay_disabled" | "not_ready" | "flatten_failed" | "similarity" | "known_bad_first_step" | "nondeterministic_first_step" | "unusable_first_step" | "coverage" | "missing_target" | "no_derive_key";
|
|
223
223
|
/** What `PreToolUse` should do about this call. */
|
|
224
224
|
export type PreToolAction =
|
|
225
225
|
/** Pin the arguments and let the real tool run. */
|
|
@@ -417,6 +417,13 @@ export declare class ReplayController {
|
|
|
417
417
|
* scenario that did nothing has nothing to book, and is a baseline sample.
|
|
418
418
|
*/
|
|
419
419
|
private handOver;
|
|
420
|
+
/**
|
|
421
|
+
* Which kind of stop a plan's end is (plan-services.md D8). The stopping
|
|
422
|
+
* step's own served reason wins — the calculation's verdict is more exact
|
|
423
|
+
* than a counter — then a call the service could not answer, then a parked
|
|
424
|
+
* step.
|
|
425
|
+
*/
|
|
426
|
+
private stopKindOf;
|
|
420
427
|
private noteFor;
|
|
421
428
|
/**
|
|
422
429
|
* The executor handed to a plan: dispatch a step to the proxy that owns its
|
|
@@ -476,6 +483,22 @@ export interface RunMatchLike {
|
|
|
476
483
|
/** The frozen name, beside the runtime intent the plan runs on (R-INTENT-12). */
|
|
477
484
|
intentName?: string;
|
|
478
485
|
}
|
|
486
|
+
/**
|
|
487
|
+
* The old rule, for a service that sends no `stop`: the position of the first
|
|
488
|
+
* step whose failure count is above `maxStepFailures` (fallbk.md D3), or
|
|
489
|
+
* undefined when none is — including when the service sent no policy, which an
|
|
490
|
+
* older service never does.
|
|
491
|
+
*/
|
|
479
492
|
export declare function knownBadStepIndex(steps: readonly SerializedScenarioStep[], maxStepFailures: number | undefined): number | undefined;
|
|
493
|
+
/**
|
|
494
|
+
* Position of the first step the plan must stop in front of (plan-services.md
|
|
495
|
+
* D8), or undefined when there is none.
|
|
496
|
+
*
|
|
497
|
+
* A step that carries `stop` was decided by the service — `null` means run it,
|
|
498
|
+
* even when its count is past the limit: its retry is due. A step without the
|
|
499
|
+
* field came from an older service, and the old counter rule decides it. Both
|
|
500
|
+
* can sit in one chain, since a called segment is served on its own terms.
|
|
501
|
+
*/
|
|
502
|
+
export declare function stopIndex(steps: readonly SerializedScenarioStep[], maxStepFailures: number | undefined): number | undefined;
|
|
480
503
|
export {};
|
|
481
504
|
//# sourceMappingURL=controller.d.ts.map
|
|
@@ -204,20 +204,31 @@ export class ReplayController {
|
|
|
204
204
|
const entries = flat.entries;
|
|
205
205
|
if (entries.length === 0)
|
|
206
206
|
return decline("the chain has no runnable step", "not_ready");
|
|
207
|
-
// Gate 5 —
|
|
208
|
-
//
|
|
209
|
-
//
|
|
210
|
-
//
|
|
211
|
-
|
|
207
|
+
// Gate 5 — steps the plan must stop in front of (fallbk.md D3,
|
|
208
|
+
// plan-services.md D8): a step the service parked — failed more than its
|
|
209
|
+
// limit, retry not due — or one the calculation marked non-deterministic,
|
|
210
|
+
// on whichever scenario owns it (R-CALL-30). A call the service could not
|
|
211
|
+
// answer stops the plan the same way, and the earliest of them wins.
|
|
212
|
+
const badAt = stopIndex(entries.map((e) => e.step), match.fallback?.maxStepFailures);
|
|
212
213
|
const rawStop = [flat.stopAt, badAt].filter((n) => typeof n === "number");
|
|
213
214
|
// Clamped to the start of the outermost called frame it falls in: a plan
|
|
214
215
|
// never runs half a sub-task and leaves its result mapping unevaluated.
|
|
215
216
|
const stopAt = rawStop.length > 0 ? clampToFrameStart(entries, Math.min(...rawStop)) : undefined;
|
|
216
217
|
if (stopAt === 0) {
|
|
217
|
-
const
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
218
|
+
const unusable = flat.stopAt === 0 && flat.stopReason ? flat.stopReason : undefined;
|
|
219
|
+
// The step that stops the plan, before the clamp: the reason is its own.
|
|
220
|
+
const stopping = entries[Math.min(...rawStop)]?.step ?? entries[0].step;
|
|
221
|
+
const judgement = stopping.stop?.kind === "nondeterministic";
|
|
222
|
+
const why = unusable
|
|
223
|
+
? `its first step is a call that cannot run: ${unusable.reason}`
|
|
224
|
+
: judgement
|
|
225
|
+
? "its first step needs a judgement the plan cannot compute — the agent does this task"
|
|
226
|
+
: `its first step has failed ${stopping.failureCount ?? 0} times — the agent does this task`;
|
|
227
|
+
return decline(why, unusable
|
|
228
|
+
? "unusable_first_step"
|
|
229
|
+
: judgement
|
|
230
|
+
? "nondeterministic_first_step"
|
|
231
|
+
: "known_bad_first_step");
|
|
221
232
|
}
|
|
222
233
|
const planned = stopAt === undefined ? entries : entries.slice(0, stopAt);
|
|
223
234
|
// Gate 6 — tool coverage, over the steps that will actually run.
|
|
@@ -333,6 +344,8 @@ export class ReplayController {
|
|
|
333
344
|
// What this chain calls, and what answers each call right now (R-CALL-32).
|
|
334
345
|
calls: callLines(scenario),
|
|
335
346
|
unusable: flat.stopReason?.reason,
|
|
347
|
+
// Which steps the service told this plan to stop in front of, and why.
|
|
348
|
+
stops: stopLines(entries),
|
|
336
349
|
coverage: coverageOf(entries.map((e) => e.step), wrapped, this.opts.allowServers).join(","),
|
|
337
350
|
tools: entries.map((e) => e.step.toolName ?? "").join(","),
|
|
338
351
|
});
|
|
@@ -562,7 +575,7 @@ export class ReplayController {
|
|
|
562
575
|
// is a call the service could not answer (segmented.md R-CALL-29).
|
|
563
576
|
const parked = plan.stopStep();
|
|
564
577
|
this.handOver(state, {
|
|
565
|
-
kind: state.
|
|
578
|
+
kind: this.stopKindOf(state, plan.stopEntry()),
|
|
566
579
|
step: parked,
|
|
567
580
|
entry: plan.stopEntry(),
|
|
568
581
|
position: plan.stepCount,
|
|
@@ -639,7 +652,10 @@ export class ReplayController {
|
|
|
639
652
|
const position = stop.flatIndex ?? plan.stepCount;
|
|
640
653
|
const step = plan.allSteps()[position];
|
|
641
654
|
this.handOver(state, {
|
|
642
|
-
kind
|
|
655
|
+
// The plan only knows it stopped in front of its end; which kind of
|
|
656
|
+
// stop that is — parked, a judgement, a call that cannot run — is
|
|
657
|
+
// decided here, where the served reasons are.
|
|
658
|
+
kind: stop.kind === "known_bad_step" ? this.stopKindOf(state, plan.stopEntry()) : stop.kind,
|
|
643
659
|
step,
|
|
644
660
|
entry: plan.allEntries()[position],
|
|
645
661
|
position,
|
|
@@ -1070,7 +1086,7 @@ export class ReplayController {
|
|
|
1070
1086
|
let text = composed.text;
|
|
1071
1087
|
if (plan.stopsEarly()) {
|
|
1072
1088
|
const parked = plan.stopStep();
|
|
1073
|
-
const stopKind = state.
|
|
1089
|
+
const stopKind = this.stopKindOf(state, plan.stopEntry());
|
|
1074
1090
|
state.handover = {
|
|
1075
1091
|
kind: stopKind,
|
|
1076
1092
|
stepIndex: parked.stepIndex,
|
|
@@ -1132,9 +1148,22 @@ export class ReplayController {
|
|
|
1132
1148
|
? "a sub-task of this chain cannot run — the agent does that part itself"
|
|
1133
1149
|
: h.kind === "known_bad_step"
|
|
1134
1150
|
? "the next step has failed too often — the agent continues from here"
|
|
1135
|
-
:
|
|
1151
|
+
: h.kind === "nondeterministic_step"
|
|
1152
|
+
? "the next step needs a judgement the plan cannot compute — the agent makes it"
|
|
1153
|
+
: "a step broke — the agent continues from here",
|
|
1136
1154
|
});
|
|
1137
1155
|
}
|
|
1156
|
+
/**
|
|
1157
|
+
* Which kind of stop a plan's end is (plan-services.md D8). The stopping
|
|
1158
|
+
* step's own served reason wins — the calculation's verdict is more exact
|
|
1159
|
+
* than a counter — then a call the service could not answer, then a parked
|
|
1160
|
+
* step.
|
|
1161
|
+
*/
|
|
1162
|
+
stopKindOf(state, entry) {
|
|
1163
|
+
if (entry?.step.stop?.kind === "nondeterministic")
|
|
1164
|
+
return "nondeterministic_step";
|
|
1165
|
+
return state.flatStop ? "unusable_call" : "known_bad_step";
|
|
1166
|
+
}
|
|
1138
1167
|
noteFor(state, h) {
|
|
1139
1168
|
const plan = state.plan;
|
|
1140
1169
|
const executed = h.ranThisStep ? h.position + 1 : h.position;
|
|
@@ -1243,6 +1272,9 @@ export class ReplayController {
|
|
|
1243
1272
|
stage: info.stage ?? (status === "skipped" ? "tool_call" : undefined),
|
|
1244
1273
|
error,
|
|
1245
1274
|
durationMs: info.ms,
|
|
1275
|
+
// The tool is gone from its server: the service repairs the plan without
|
|
1276
|
+
// waiting out the grace (plan-services.md D8, kind 2).
|
|
1277
|
+
toolMissing: info.toolMissing || undefined,
|
|
1246
1278
|
// A verdict about a called segment's step is counted on that segment, at
|
|
1247
1279
|
// the revision it was served (R-CALL-13).
|
|
1248
1280
|
scenarioId: info.entry && info.entry.depth > 0 ? info.entry.scenarioId : undefined,
|
|
@@ -1340,11 +1372,6 @@ function targetKeys(schema) {
|
|
|
1340
1372
|
return [];
|
|
1341
1373
|
return Object.keys(schema).filter((k) => schema[k]?.kind !== "setting");
|
|
1342
1374
|
}
|
|
1343
|
-
/**
|
|
1344
|
-
* Position of the first step whose failure count is above `maxStepFailures`
|
|
1345
|
-
* (fallbk.md D3), or undefined when none is — including when the service sent
|
|
1346
|
-
* no policy, which an older service never does.
|
|
1347
|
-
*/
|
|
1348
1375
|
/**
|
|
1349
1376
|
* The index a hand-over reports, in the *caller's* chain (R-CALL-32).
|
|
1350
1377
|
*
|
|
@@ -1369,10 +1396,39 @@ function callLines(scenario) {
|
|
|
1369
1396
|
.map((s) => `${s.segmentId ?? "-"}:${s.unusable ?? s.resolution?.state ?? "resolved"}`)
|
|
1370
1397
|
.join(",");
|
|
1371
1398
|
}
|
|
1399
|
+
/** The served stops in a flat chain, for the `plan.armed` line (plan-services.md D8). */
|
|
1400
|
+
function stopLines(entries) {
|
|
1401
|
+
const lines = entries
|
|
1402
|
+
.map((e, i) => e.step.stop ? `${i}:${e.step.stop.kind}${e.step.stop.repairDue ? "/repair" : ""}` : undefined)
|
|
1403
|
+
.filter((s) => s !== undefined);
|
|
1404
|
+
return lines.length > 0 ? lines.join(",") : undefined;
|
|
1405
|
+
}
|
|
1406
|
+
/**
|
|
1407
|
+
* The old rule, for a service that sends no `stop`: the position of the first
|
|
1408
|
+
* step whose failure count is above `maxStepFailures` (fallbk.md D3), or
|
|
1409
|
+
* undefined when none is — including when the service sent no policy, which an
|
|
1410
|
+
* older service never does.
|
|
1411
|
+
*/
|
|
1372
1412
|
export function knownBadStepIndex(steps, maxStepFailures) {
|
|
1413
|
+
const i = steps.findIndex((s) => isKnownBad(s, maxStepFailures));
|
|
1414
|
+
return i < 0 ? undefined : i;
|
|
1415
|
+
}
|
|
1416
|
+
function isKnownBad(step, maxStepFailures) {
|
|
1373
1417
|
if (typeof maxStepFailures !== "number" || !Number.isFinite(maxStepFailures))
|
|
1374
|
-
return
|
|
1375
|
-
|
|
1418
|
+
return false;
|
|
1419
|
+
return (step.failureCount ?? 0) > maxStepFailures;
|
|
1420
|
+
}
|
|
1421
|
+
/**
|
|
1422
|
+
* Position of the first step the plan must stop in front of (plan-services.md
|
|
1423
|
+
* D8), or undefined when there is none.
|
|
1424
|
+
*
|
|
1425
|
+
* A step that carries `stop` was decided by the service — `null` means run it,
|
|
1426
|
+
* even when its count is past the limit: its retry is due. A step without the
|
|
1427
|
+
* field came from an older service, and the old counter rule decides it. Both
|
|
1428
|
+
* can sit in one chain, since a called segment is served on its own terms.
|
|
1429
|
+
*/
|
|
1430
|
+
export function stopIndex(steps, maxStepFailures) {
|
|
1431
|
+
const i = steps.findIndex((s) => s.stop !== undefined ? s.stop !== null : isKnownBad(s, maxStepFailures));
|
|
1376
1432
|
return i < 0 ? undefined : i;
|
|
1377
1433
|
}
|
|
1378
1434
|
//# sourceMappingURL=controller.js.map
|
package/dist/replay/handover.js
CHANGED
|
@@ -60,6 +60,11 @@ export function buildHandoverNote(n) {
|
|
|
60
60
|
: "";
|
|
61
61
|
lines.push(`Step ${human} of ${n.totalSteps}${tool.replace(/\)$/, `${owner})`)} did not run: it has failed${times} before, so the scenario stops in front of it.`);
|
|
62
62
|
}
|
|
63
|
+
else if (n.kind === "nondeterministic_step") {
|
|
64
|
+
// The calculation could not write code for this step's input: it is a
|
|
65
|
+
// judgement, and the agent is the one to make it (plan-services.md D8).
|
|
66
|
+
lines.push(`Step ${human} of ${n.totalSteps}${tool} did not run: its input needs a judgement the scenario cannot compute. Make that choice yourself and carry on.`);
|
|
67
|
+
}
|
|
63
68
|
else {
|
|
64
69
|
const error = n.error ? `: ${truncate(n.error, MAX_ERROR_CHARS)}` : "";
|
|
65
70
|
lines.push(`Step ${human} of ${n.totalSteps}${tool} failed${error}`);
|
package/dist/replay/plan.d.ts
CHANGED
|
@@ -65,6 +65,8 @@ export interface StepInfo {
|
|
|
65
65
|
error?: string;
|
|
66
66
|
derivedKeys?: string[];
|
|
67
67
|
ms: number;
|
|
68
|
+
/** The tool is gone from its server, read from the error (plan-services.md D8). */
|
|
69
|
+
toolMissing?: boolean;
|
|
68
70
|
}
|
|
69
71
|
export type StepObserver = (info: StepInfo) => void;
|
|
70
72
|
export interface ComposeResult {
|
package/dist/replay/plan.js
CHANGED
|
@@ -23,7 +23,7 @@
|
|
|
23
23
|
import { evalParamMapLogic, evalParamsLogic, evalResponseParamsLogic, evalResultMapLogic, evalToolInputLogic, evalToolOutputLogic, } from "./logic.js";
|
|
24
24
|
import { flatEntriesOf } from "./flatten.js";
|
|
25
25
|
import { assembleBundle, bundleInput, MAX_REPLAY_REASON } from "./bundle.js";
|
|
26
|
-
import { toolResultError } from "./tool-error.js";
|
|
26
|
+
import { isMissingToolError, toolResultError } from "./tool-error.js";
|
|
27
27
|
/**
|
|
28
28
|
* The recorded values for the *settings* a mapping did not name (R-CALL-21).
|
|
29
29
|
*
|
|
@@ -433,13 +433,29 @@ export class ScenarioReplayPlan {
|
|
|
433
433
|
: undefined) ?? (await recordedOutputFor?.(entry).catch(() => undefined));
|
|
434
434
|
if (!fallback) {
|
|
435
435
|
skipped += 1;
|
|
436
|
-
onStep?.({
|
|
436
|
+
onStep?.({
|
|
437
|
+
step,
|
|
438
|
+
entry,
|
|
439
|
+
input: computed,
|
|
440
|
+
outcome: "skipped",
|
|
441
|
+
error,
|
|
442
|
+
toolMissing: isMissingToolError(error) || undefined,
|
|
443
|
+
ms: Date.now() - startedAt,
|
|
444
|
+
});
|
|
437
445
|
continue;
|
|
438
446
|
}
|
|
439
447
|
response = fallback;
|
|
440
448
|
recorded = true;
|
|
441
449
|
recordedCount += 1;
|
|
442
|
-
onStep?.({
|
|
450
|
+
onStep?.({
|
|
451
|
+
step,
|
|
452
|
+
entry,
|
|
453
|
+
input: computed,
|
|
454
|
+
outcome: "recorded",
|
|
455
|
+
error,
|
|
456
|
+
toolMissing: isMissingToolError(error) || undefined,
|
|
457
|
+
ms: Date.now() - startedAt,
|
|
458
|
+
});
|
|
443
459
|
}
|
|
444
460
|
// Thread the output for later steps' inputs. A recorded output threads
|
|
445
461
|
// too: `toolOutputLogic` was authored against exactly this shape, and a
|
|
@@ -465,6 +481,7 @@ export class ScenarioReplayPlan {
|
|
|
465
481
|
outcome: error ? "failed" : "executed",
|
|
466
482
|
stage: error ? "tool_call" : undefined,
|
|
467
483
|
error,
|
|
484
|
+
toolMissing: isMissingToolError(error) || undefined,
|
|
468
485
|
derivedKeys,
|
|
469
486
|
ms: Date.now() - startedAt,
|
|
470
487
|
});
|
|
@@ -518,6 +535,9 @@ export class ScenarioReplayPlan {
|
|
|
518
535
|
}
|
|
519
536
|
const entry = this.entries[this.stepIndex];
|
|
520
537
|
const step = entry.step;
|
|
538
|
+
// Where this step sits in the flat list — what a stop reports. Taken now,
|
|
539
|
+
// because the cursor has moved on by the time some stops are written.
|
|
540
|
+
const flatIndex = this.stepIndex;
|
|
521
541
|
const startedAt = Date.now();
|
|
522
542
|
let computed;
|
|
523
543
|
try {
|
|
@@ -529,6 +549,7 @@ export class ScenarioReplayPlan {
|
|
|
529
549
|
stopped = {
|
|
530
550
|
kind: "step_failed",
|
|
531
551
|
stepIndex: step.stepIndex,
|
|
552
|
+
flatIndex,
|
|
532
553
|
stage: "tool_input_logic",
|
|
533
554
|
error: errText(err),
|
|
534
555
|
};
|
|
@@ -547,7 +568,15 @@ export class ScenarioReplayPlan {
|
|
|
547
568
|
: undefined) ?? (await recordedOutputFor?.(entry).catch(() => undefined));
|
|
548
569
|
if (!fallback) {
|
|
549
570
|
skipped += 1;
|
|
550
|
-
onStep?.({
|
|
571
|
+
onStep?.({
|
|
572
|
+
step,
|
|
573
|
+
entry,
|
|
574
|
+
input: computed,
|
|
575
|
+
outcome: "skipped",
|
|
576
|
+
error,
|
|
577
|
+
toolMissing: isMissingToolError(error) || undefined,
|
|
578
|
+
ms: Date.now() - startedAt,
|
|
579
|
+
});
|
|
551
580
|
// Advance past a step that cannot run, without threading anything.
|
|
552
581
|
// The frame is still left behind it: a sub-task whose last step could
|
|
553
582
|
// not run here still hands back whatever its earlier steps emitted.
|
|
@@ -558,7 +587,15 @@ export class ScenarioReplayPlan {
|
|
|
558
587
|
response = fallback;
|
|
559
588
|
recorded = true;
|
|
560
589
|
recordedCount += 1;
|
|
561
|
-
onStep?.({
|
|
590
|
+
onStep?.({
|
|
591
|
+
step,
|
|
592
|
+
entry,
|
|
593
|
+
input: computed,
|
|
594
|
+
outcome: "recorded",
|
|
595
|
+
error,
|
|
596
|
+
toolMissing: isMissingToolError(error) || undefined,
|
|
597
|
+
ms: Date.now() - startedAt,
|
|
598
|
+
});
|
|
562
599
|
}
|
|
563
600
|
if (stopOnFailure && !recorded) {
|
|
564
601
|
// Judged before the output logic, which was written against a success
|
|
@@ -573,6 +610,7 @@ export class ScenarioReplayPlan {
|
|
|
573
610
|
outcome: "failed",
|
|
574
611
|
stage: "tool_call",
|
|
575
612
|
error: toolError,
|
|
613
|
+
toolMissing: isMissingToolError(toolError) || undefined,
|
|
576
614
|
ms: Date.now() - startedAt,
|
|
577
615
|
});
|
|
578
616
|
entries.push({
|
|
@@ -585,6 +623,7 @@ export class ScenarioReplayPlan {
|
|
|
585
623
|
stopped = {
|
|
586
624
|
kind: "step_failed",
|
|
587
625
|
stepIndex: step.stepIndex,
|
|
626
|
+
flatIndex,
|
|
588
627
|
stage: "tool_call",
|
|
589
628
|
error: toolError,
|
|
590
629
|
};
|
|
@@ -608,6 +647,7 @@ export class ScenarioReplayPlan {
|
|
|
608
647
|
stopped = {
|
|
609
648
|
kind: "step_failed",
|
|
610
649
|
stepIndex: step.stepIndex,
|
|
650
|
+
flatIndex,
|
|
611
651
|
stage: "tool_output_logic",
|
|
612
652
|
error: errText(err),
|
|
613
653
|
};
|
|
@@ -627,6 +667,7 @@ export class ScenarioReplayPlan {
|
|
|
627
667
|
outcome: error ? "failed" : "executed",
|
|
628
668
|
stage: error ? "tool_call" : undefined,
|
|
629
669
|
error,
|
|
670
|
+
toolMissing: isMissingToolError(error) || undefined,
|
|
630
671
|
derivedKeys,
|
|
631
672
|
ms: Date.now() - startedAt,
|
|
632
673
|
});
|
|
@@ -640,7 +681,13 @@ export class ScenarioReplayPlan {
|
|
|
640
681
|
}
|
|
641
682
|
// Every planned step ran and the plan ends in front of a parked one.
|
|
642
683
|
if (!stopped && !partial && this.stopsEarly() && this.isDone()) {
|
|
643
|
-
|
|
684
|
+
// `known_bad_step` here means "in front of the plan's end": the
|
|
685
|
+
// controller reads the served reason and names the kind (D8).
|
|
686
|
+
stopped = {
|
|
687
|
+
kind: "known_bad_step",
|
|
688
|
+
stepIndex: this.entries[this.stopAt].stepIndex,
|
|
689
|
+
flatIndex: this.stopAt,
|
|
690
|
+
};
|
|
644
691
|
}
|
|
645
692
|
return {
|
|
646
693
|
text: assembleBundle(entries, maxChars, {
|
package/dist/replay/pricing.d.ts
CHANGED
|
@@ -17,7 +17,7 @@
|
|
|
17
17
|
* Bump the version — on **every** side — whenever a row changes.
|
|
18
18
|
*/
|
|
19
19
|
/** Identifies {@link MODEL_PRICING}. Must equal the service's constant. */
|
|
20
|
-
export declare const PRICING_VERSION = "2026-
|
|
20
|
+
export declare const PRICING_VERSION = "2026-09-24";
|
|
21
21
|
/**
|
|
22
22
|
* Tolerate provider prefixes (`anthropic/`, `us.anthropic.`), date snapshots and
|
|
23
23
|
* unseen version bumps by falling back to the longest matching family, so a new
|
package/dist/replay/pricing.js
CHANGED
|
@@ -17,24 +17,32 @@
|
|
|
17
17
|
* Bump the version — on **every** side — whenever a row changes.
|
|
18
18
|
*/
|
|
19
19
|
/** Identifies {@link MODEL_PRICING}. Must equal the service's constant. */
|
|
20
|
-
export const PRICING_VERSION = "2026-
|
|
21
|
-
/**
|
|
22
|
-
|
|
20
|
+
export const PRICING_VERSION = "2026-09-24";
|
|
21
|
+
/**
|
|
22
|
+
* A 5-minute cache write bills at 1.25x input. Cache reads bill at 0.1x input,
|
|
23
|
+
* except on models whose list price says otherwise (Opus 5.5, Fable 5.1), which
|
|
24
|
+
* pass their read price in dollars.
|
|
25
|
+
*/
|
|
26
|
+
const priced = (input, output, cacheRead = input * 0.1) => ({
|
|
23
27
|
input,
|
|
24
28
|
output,
|
|
25
|
-
cacheRead
|
|
29
|
+
cacheRead,
|
|
26
30
|
cacheWrite: input * 1.25,
|
|
27
31
|
});
|
|
28
32
|
const MODEL_PRICING = {
|
|
29
33
|
"claude-haiku-4-5": priced(1.0, 5.0),
|
|
30
34
|
"claude-haiku-4-5-20251001": priced(1.0, 5.0),
|
|
35
|
+
"claude-sonnet-4-5": priced(3.0, 15.0),
|
|
31
36
|
"claude-sonnet-4-6": priced(3.0, 15.0),
|
|
32
37
|
"claude-sonnet-5": priced(2.0, 10.0),
|
|
38
|
+
"claude-opus-4-5": priced(5.0, 25.0),
|
|
33
39
|
"claude-opus-4-6": priced(5.0, 25.0),
|
|
34
40
|
"claude-opus-4-7": priced(5.0, 25.0),
|
|
35
41
|
"claude-opus-4-8": priced(5.0, 25.0),
|
|
36
42
|
"claude-opus-5": priced(5.0, 25.0),
|
|
43
|
+
"claude-opus-5-5": priced(4.0, 20.0, 0.2),
|
|
37
44
|
"claude-fable-5": priced(10.0, 50.0),
|
|
45
|
+
"claude-fable-5-1": priced(10.0, 50.0, 0.25),
|
|
38
46
|
};
|
|
39
47
|
/**
|
|
40
48
|
* Tolerate provider prefixes (`anthropic/`, `us.anthropic.`), date snapshots and
|
|
@@ -20,4 +20,19 @@
|
|
|
20
20
|
* not. `serialized` is the proxy's own serialization — the whole result, as JSON.
|
|
21
21
|
*/
|
|
22
22
|
export declare function toolResultError(serialized: string): string | undefined;
|
|
23
|
+
/**
|
|
24
|
+
* What a server says when a tool no longer exists — the one failure a grace
|
|
25
|
+
* period cannot cure (plan-services.md D8, kind 2).
|
|
26
|
+
*
|
|
27
|
+
* Read from the text, because the proxy keeps only a JSON-RPC error's message
|
|
28
|
+
* and servers word it differently: the TypeScript SDK answers `Tool X not
|
|
29
|
+
* found`, the Python SDK `Unknown tool: X`, a hand-rolled server whatever it
|
|
30
|
+
* likes. Kept narrow on purpose — `not available` is what a server says during
|
|
31
|
+
* an outage, and a repair in the middle of one is the wrong answer. A miss here
|
|
32
|
+
* costs only the wait; a false hit costs one recording and one calculation,
|
|
33
|
+
* which the service caps.
|
|
34
|
+
*/
|
|
35
|
+
export declare const MISSING_TOOL_PATTERN: RegExp;
|
|
36
|
+
/** Whether an error text says the step's tool is gone from its server. */
|
|
37
|
+
export declare function isMissingToolError(text: string | undefined): boolean;
|
|
23
38
|
//# sourceMappingURL=tool-error.d.ts.map
|
|
@@ -35,6 +35,23 @@ export function toolResultError(serialized) {
|
|
|
35
35
|
return undefined;
|
|
36
36
|
return clip(firstText(result.content) ?? "the tool reported an error");
|
|
37
37
|
}
|
|
38
|
+
/**
|
|
39
|
+
* What a server says when a tool no longer exists — the one failure a grace
|
|
40
|
+
* period cannot cure (plan-services.md D8, kind 2).
|
|
41
|
+
*
|
|
42
|
+
* Read from the text, because the proxy keeps only a JSON-RPC error's message
|
|
43
|
+
* and servers word it differently: the TypeScript SDK answers `Tool X not
|
|
44
|
+
* found`, the Python SDK `Unknown tool: X`, a hand-rolled server whatever it
|
|
45
|
+
* likes. Kept narrow on purpose — `not available` is what a server says during
|
|
46
|
+
* an outage, and a repair in the middle of one is the wrong answer. A miss here
|
|
47
|
+
* costs only the wait; a false hit costs one recording and one calculation,
|
|
48
|
+
* which the service caps.
|
|
49
|
+
*/
|
|
50
|
+
export const MISSING_TOOL_PATTERN = /\b(unknown|no such|unsupported|unrecognized) tool\b|\btool\s+['"`]?[\w.:-]+['"`]?\s+(was\s+)?(not found|does not exist|is not registered)\b/i;
|
|
51
|
+
/** Whether an error text says the step's tool is gone from its server. */
|
|
52
|
+
export function isMissingToolError(text) {
|
|
53
|
+
return typeof text === "string" && MISSING_TOOL_PATTERN.test(text);
|
|
54
|
+
}
|
|
38
55
|
function firstText(content) {
|
|
39
56
|
if (!Array.isArray(content))
|
|
40
57
|
return undefined;
|
package/dist/replay/types.d.ts
CHANGED
|
@@ -37,6 +37,25 @@ export interface InlinedSegment {
|
|
|
37
37
|
}
|
|
38
38
|
/** Why a call cannot run right now (segmented.md 9.9). */
|
|
39
39
|
export type CallUnusable = "switched_off" | "not_ready" | "missing" | "stale" | "unresolved" | "depth" | "unsupported";
|
|
40
|
+
/**
|
|
41
|
+
* A reason the plan stops in front of a step, as the service decided it
|
|
42
|
+
* (plan-services.md D8). The service knows the grace period, the retry clock
|
|
43
|
+
* and the calculation's own verdict on the step; the runner only reads it.
|
|
44
|
+
*/
|
|
45
|
+
export interface StepStop {
|
|
46
|
+
/**
|
|
47
|
+
* `parked`: the step has failed too often and its retry is not due, or its
|
|
48
|
+
* grace has ended. `nondeterministic`: the calculation could not write code
|
|
49
|
+
* that computes this step's input — a judgement — so the agent makes it.
|
|
50
|
+
*/
|
|
51
|
+
kind: "parked" | "nondeterministic";
|
|
52
|
+
/** For `nondeterministic`: which check the step failed at calculation. */
|
|
53
|
+
why?: string;
|
|
54
|
+
/** For `parked` inside its grace: when the step may be tried again. */
|
|
55
|
+
retryAt?: string;
|
|
56
|
+
/** For `parked` past its grace, or whose tool is gone: a repair is due. */
|
|
57
|
+
repairDue?: boolean;
|
|
58
|
+
}
|
|
40
59
|
export interface SerializedScenarioStep {
|
|
41
60
|
stepIndex: number;
|
|
42
61
|
/**
|
|
@@ -60,6 +79,15 @@ export interface SerializedScenarioStep {
|
|
|
60
79
|
* model in front of it. Absent from an older service, which means never park.
|
|
61
80
|
*/
|
|
62
81
|
failureCount?: number;
|
|
82
|
+
/**
|
|
83
|
+
* Why the plan must stop in front of this step, decided by the service
|
|
84
|
+
* (plan-services.md D8). Present on every step — `null` when there is no
|
|
85
|
+
* reason — from a service that decides stops itself. A parked step whose
|
|
86
|
+
* retry is due arrives with `stop: null` and is tried. Absent altogether
|
|
87
|
+
* from an older service, where `failureCount` against
|
|
88
|
+
* `fallback.maxStepFailures` is the only rule.
|
|
89
|
+
*/
|
|
90
|
+
stop?: StepStop | null;
|
|
63
91
|
/**
|
|
64
92
|
* The literal output recorded for this step.
|
|
65
93
|
*
|
|
@@ -117,6 +145,13 @@ export declare function hasTargets(schema: ParamsSchema | null | undefined): boo
|
|
|
117
145
|
export interface SerializedScenario {
|
|
118
146
|
id: string;
|
|
119
147
|
runId: string;
|
|
148
|
+
/**
|
|
149
|
+
* The run the current chain was calculated from, when it is not `runId`: a
|
|
150
|
+
* plan repaired from a fresh recording (plan-services.md W2.4). A recorded
|
|
151
|
+
* output that stands in for a step comes from this run. Absent from an older
|
|
152
|
+
* service, and from a plan that was never repaired.
|
|
153
|
+
*/
|
|
154
|
+
sourceRunId?: string;
|
|
120
155
|
state: string;
|
|
121
156
|
intent: string;
|
|
122
157
|
parameters: unknown;
|
|
@@ -159,7 +194,13 @@ export type FallbackKind =
|
|
|
159
194
|
* gone, switched off, out of date, or nested too deep (segmented.md
|
|
160
195
|
* R-CALL-29). Like a known-bad step it counts nothing against any step.
|
|
161
196
|
*/
|
|
162
|
-
| "unusable_call"
|
|
197
|
+
| "unusable_call"
|
|
198
|
+
/**
|
|
199
|
+
* The plan stopped in front of a step the calculation marked
|
|
200
|
+
* non-deterministic: its input needs a judgement no code computes
|
|
201
|
+
* (plan-services.md D8, kind 3). Never tried, counts nothing.
|
|
202
|
+
*/
|
|
203
|
+
| "nondeterministic_step";
|
|
163
204
|
/**
|
|
164
205
|
* Upgrade order. `not_steered` is the floor — the control group, and the safe
|
|
165
206
|
* answer if a turn dies mid-flight — and each later decision point can only move
|
|
@@ -205,6 +246,12 @@ export interface ExecutionStepResult {
|
|
|
205
246
|
/** Message only — never a tool payload; this is rendered in a web page. */
|
|
206
247
|
error?: string;
|
|
207
248
|
durationMs?: number;
|
|
249
|
+
/**
|
|
250
|
+
* The step's tool no longer exists on its server, read from the error text
|
|
251
|
+
* (plan-services.md D8, kind 2). The service then repairs the plan without
|
|
252
|
+
* waiting out the grace period.
|
|
253
|
+
*/
|
|
254
|
+
toolMissing?: boolean;
|
|
208
255
|
/**
|
|
209
256
|
* The scenario this step belongs to, when it ran inside a called segment: a
|
|
210
257
|
* failure is counted where a repair would happen (segmented.md R-CALL-13).
|
|
@@ -390,6 +390,8 @@ first prompt that states the task plainly.
|
|
|
390
390
|
| `run_scenario` returns "no scenario is armed" | The model called it on a turn with no plan | Harmless. It happens when the model remembers the tool from an earlier turn |
|
|
391
391
|
| `execution.reported … 409 invalid_ticket` | The ticket was already redeemed, or belongs to another scenario | Harmless if `duplicate: true`. Otherwise a rollover raced `SessionEnd`; the server books once |
|
|
392
392
|
| `savedUsd` looks impossible | Pricing drift between the runner's table and the server's | Compare `PRICING_VERSION` on both sides. Design §11.3 documents a known live drift between two existing copies |
|
|
393
|
+
| The hand-over note says a step "did not run: its input needs a judgement" | The service marked the step non-deterministic when it calculated the plan: the generated logic could not compute its input from the prompt or from earlier outputs, so the recorded value was a copy (plan-services.md D8, kind 3) | Nothing on the runner. The agent makes that choice and finishes the task; a model step for such judgements is planned (W3.1). The journal's `plan.armed` line lists it under `stops=` |
|
|
394
|
+
| A parked step ran again after an hour, or a plan was recalculated by itself | The service's grace period (D8): a parked step is served with `stop: null` once an hour for a day and a success clears its count; if it keeps failing, or its tool is gone, the next repeat of the prompt is recorded in full and the plan is calculated again from it | Nothing: that is the repair. `bir investigate` names the kind (`step_retrying`, `step_parked … repair due`, `scenario_repairing`). To repair now by hand: `bir scenario calc <runId> --force` |
|
|
393
395
|
| `sessionCostUsd` grows every prompt in a session | The per-run usage watermark is missing or not taken | Design §11.4. Reports without a mark must carry `measured: false` |
|
|
394
396
|
| Everything works, nothing is saved | A replayed turn is deliberately not recorded (design §12) | Correct. The matched run stays canonical and its `iterations` is bumped |
|
|
395
397
|
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@basein/runner",
|
|
3
|
-
"version": "0.2.
|
|
3
|
+
"version": "0.2.8",
|
|
4
4
|
"description": "A recording MCP proxy: sits between any MCP client and its MCP servers, executes each call on the client's behalf, and records the run as a reusable BaseIn scenario.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "MIT",
|