@basein/runner 0.2.6 → 0.2.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/bir-hooks.js +5 -0
- package/dist/bin/bir.js +40 -1
- package/dist/bin/investigate.d.ts +195 -0
- package/dist/bin/investigate.js +580 -0
- package/dist/control/server.js +15 -4
- package/dist/record/recorder.d.ts +6 -0
- package/dist/record/remote-recorder.js +2 -0
- package/dist/replay/controller.d.ts +26 -1
- package/dist/replay/controller.js +78 -20
- package/dist/replay/handover.js +5 -0
- package/dist/replay/plan.d.ts +2 -0
- package/dist/replay/plan.js +53 -6
- package/dist/replay/pricing.d.ts +1 -1
- package/dist/replay/pricing.js +12 -4
- package/dist/replay/tool-error.d.ts +15 -0
- package/dist/replay/tool-error.js +17 -0
- package/dist/replay/types.d.ts +48 -1
- package/dist/util/journal.d.ts +38 -0
- package/dist/util/journal.js +97 -0
- package/dist/util/log.js +4 -0
- package/docs/calculatedReplayGuide.md +34 -0
- package/package.json +1 -1
|
@@ -219,7 +219,7 @@ export interface ReplayState {
|
|
|
219
219
|
* Why a match did not arm. One code per gate of the ladder, so the caller can
|
|
220
220
|
* act on a decline without parsing the sentence that explains it.
|
|
221
221
|
*/
|
|
222
|
-
export type DeclineCode = "replay_disabled" | "not_ready" | "flatten_failed" | "similarity" | "known_bad_first_step" | "unusable_first_step" | "coverage" | "missing_target" | "no_derive_key";
|
|
222
|
+
export type DeclineCode = "replay_disabled" | "not_ready" | "flatten_failed" | "similarity" | "known_bad_first_step" | "nondeterministic_first_step" | "unusable_first_step" | "coverage" | "missing_target" | "no_derive_key";
|
|
223
223
|
/** What `PreToolUse` should do about this call. */
|
|
224
224
|
export type PreToolAction =
|
|
225
225
|
/** Pin the arguments and let the real tool run. */
|
|
@@ -342,6 +342,8 @@ export declare class ReplayController {
|
|
|
342
342
|
prompt?: string;
|
|
343
343
|
/** Every plan state of this turn, this one included (R-MONEY-5). */
|
|
344
344
|
siblings?: readonly ReplayState[];
|
|
345
|
+
/** The control server's own run id for this turn — journal-only, never sent. */
|
|
346
|
+
runId?: string;
|
|
345
347
|
}): ExecutionReport | undefined;
|
|
346
348
|
/**
|
|
347
349
|
* The step a failed replay is fairly blamed on: the first whose own logic
|
|
@@ -415,6 +417,13 @@ export declare class ReplayController {
|
|
|
415
417
|
* scenario that did nothing has nothing to book, and is a baseline sample.
|
|
416
418
|
*/
|
|
417
419
|
private handOver;
|
|
420
|
+
/**
|
|
421
|
+
* Which kind of stop a plan's end is (plan-services.md D8). The stopping
|
|
422
|
+
* step's own served reason wins — the calculation's verdict is more exact
|
|
423
|
+
* than a counter — then a call the service could not answer, then a parked
|
|
424
|
+
* step.
|
|
425
|
+
*/
|
|
426
|
+
private stopKindOf;
|
|
418
427
|
private noteFor;
|
|
419
428
|
/**
|
|
420
429
|
* The executor handed to a plan: dispatch a step to the proxy that owns its
|
|
@@ -474,6 +483,22 @@ export interface RunMatchLike {
|
|
|
474
483
|
/** The frozen name, beside the runtime intent the plan runs on (R-INTENT-12). */
|
|
475
484
|
intentName?: string;
|
|
476
485
|
}
|
|
486
|
+
/**
|
|
487
|
+
* The old rule, for a service that sends no `stop`: the position of the first
|
|
488
|
+
* step whose failure count is above `maxStepFailures` (fallbk.md D3), or
|
|
489
|
+
* undefined when none is — including when the service sent no policy, which an
|
|
490
|
+
* older service never does.
|
|
491
|
+
*/
|
|
477
492
|
export declare function knownBadStepIndex(steps: readonly SerializedScenarioStep[], maxStepFailures: number | undefined): number | undefined;
|
|
493
|
+
/**
|
|
494
|
+
* Position of the first step the plan must stop in front of (plan-services.md
|
|
495
|
+
* D8), or undefined when there is none.
|
|
496
|
+
*
|
|
497
|
+
* A step that carries `stop` was decided by the service — `null` means run it,
|
|
498
|
+
* even when its count is past the limit: its retry is due. A step without the
|
|
499
|
+
* field came from an older service, and the old counter rule decides it. Both
|
|
500
|
+
* can sit in one chain, since a called segment is served on its own terms.
|
|
501
|
+
*/
|
|
502
|
+
export declare function stopIndex(steps: readonly SerializedScenarioStep[], maxStepFailures: number | undefined): number | undefined;
|
|
478
503
|
export {};
|
|
479
504
|
//# sourceMappingURL=controller.d.ts.map
|
|
@@ -146,6 +146,7 @@ export class ReplayController {
|
|
|
146
146
|
state.declined = code;
|
|
147
147
|
logLine("replay.decision", {
|
|
148
148
|
verdict: "no-steer",
|
|
149
|
+
code,
|
|
149
150
|
run: match.runId,
|
|
150
151
|
scenario: match.scenarioId ?? undefined,
|
|
151
152
|
similarity: match.similarity.toFixed(3),
|
|
@@ -203,20 +204,31 @@ export class ReplayController {
|
|
|
203
204
|
const entries = flat.entries;
|
|
204
205
|
if (entries.length === 0)
|
|
205
206
|
return decline("the chain has no runnable step", "not_ready");
|
|
206
|
-
// Gate 5 —
|
|
207
|
-
//
|
|
208
|
-
//
|
|
209
|
-
//
|
|
210
|
-
|
|
207
|
+
// Gate 5 — steps the plan must stop in front of (fallbk.md D3,
|
|
208
|
+
// plan-services.md D8): a step the service parked — failed more than its
|
|
209
|
+
// limit, retry not due — or one the calculation marked non-deterministic,
|
|
210
|
+
// on whichever scenario owns it (R-CALL-30). A call the service could not
|
|
211
|
+
// answer stops the plan the same way, and the earliest of them wins.
|
|
212
|
+
const badAt = stopIndex(entries.map((e) => e.step), match.fallback?.maxStepFailures);
|
|
211
213
|
const rawStop = [flat.stopAt, badAt].filter((n) => typeof n === "number");
|
|
212
214
|
// Clamped to the start of the outermost called frame it falls in: a plan
|
|
213
215
|
// never runs half a sub-task and leaves its result mapping unevaluated.
|
|
214
216
|
const stopAt = rawStop.length > 0 ? clampToFrameStart(entries, Math.min(...rawStop)) : undefined;
|
|
215
217
|
if (stopAt === 0) {
|
|
216
|
-
const
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
218
|
+
const unusable = flat.stopAt === 0 && flat.stopReason ? flat.stopReason : undefined;
|
|
219
|
+
// The step that stops the plan, before the clamp: the reason is its own.
|
|
220
|
+
const stopping = entries[Math.min(...rawStop)]?.step ?? entries[0].step;
|
|
221
|
+
const judgement = stopping.stop?.kind === "nondeterministic";
|
|
222
|
+
const why = unusable
|
|
223
|
+
? `its first step is a call that cannot run: ${unusable.reason}`
|
|
224
|
+
: judgement
|
|
225
|
+
? "its first step needs a judgement the plan cannot compute — the agent does this task"
|
|
226
|
+
: `its first step has failed ${stopping.failureCount ?? 0} times — the agent does this task`;
|
|
227
|
+
return decline(why, unusable
|
|
228
|
+
? "unusable_first_step"
|
|
229
|
+
: judgement
|
|
230
|
+
? "nondeterministic_first_step"
|
|
231
|
+
: "known_bad_first_step");
|
|
220
232
|
}
|
|
221
233
|
const planned = stopAt === undefined ? entries : entries.slice(0, stopAt);
|
|
222
234
|
// Gate 6 — tool coverage, over the steps that will actually run.
|
|
@@ -332,6 +344,8 @@ export class ReplayController {
|
|
|
332
344
|
// What this chain calls, and what answers each call right now (R-CALL-32).
|
|
333
345
|
calls: callLines(scenario),
|
|
334
346
|
unusable: flat.stopReason?.reason,
|
|
347
|
+
// Which steps the service told this plan to stop in front of, and why.
|
|
348
|
+
stops: stopLines(entries),
|
|
335
349
|
coverage: coverageOf(entries.map((e) => e.step), wrapped, this.opts.allowServers).join(","),
|
|
336
350
|
tools: entries.map((e) => e.step.toolName ?? "").join(","),
|
|
337
351
|
});
|
|
@@ -561,7 +575,7 @@ export class ReplayController {
|
|
|
561
575
|
// is a call the service could not answer (segmented.md R-CALL-29).
|
|
562
576
|
const parked = plan.stopStep();
|
|
563
577
|
this.handOver(state, {
|
|
564
|
-
kind: state.
|
|
578
|
+
kind: this.stopKindOf(state, plan.stopEntry()),
|
|
565
579
|
step: parked,
|
|
566
580
|
entry: plan.stopEntry(),
|
|
567
581
|
position: plan.stepCount,
|
|
@@ -638,7 +652,10 @@ export class ReplayController {
|
|
|
638
652
|
const position = stop.flatIndex ?? plan.stepCount;
|
|
639
653
|
const step = plan.allSteps()[position];
|
|
640
654
|
this.handOver(state, {
|
|
641
|
-
kind
|
|
655
|
+
// The plan only knows it stopped in front of its end; which kind of
|
|
656
|
+
// stop that is — parked, a judgement, a call that cannot run — is
|
|
657
|
+
// decided here, where the served reasons are.
|
|
658
|
+
kind: stop.kind === "known_bad_step" ? this.stopKindOf(state, plan.stopEntry()) : stop.kind,
|
|
642
659
|
step,
|
|
643
660
|
entry: plan.allEntries()[position],
|
|
644
661
|
position,
|
|
@@ -726,6 +743,7 @@ export class ReplayController {
|
|
|
726
743
|
const blame = this.blameStep(steps);
|
|
727
744
|
return {
|
|
728
745
|
scenarioId: state.scenarioId,
|
|
746
|
+
runId: d.runId,
|
|
729
747
|
ticket: state.ticket,
|
|
730
748
|
outcome: state.outcome,
|
|
731
749
|
deriveCostUsd: state.deriveCostUsd,
|
|
@@ -1068,7 +1086,7 @@ export class ReplayController {
|
|
|
1068
1086
|
let text = composed.text;
|
|
1069
1087
|
if (plan.stopsEarly()) {
|
|
1070
1088
|
const parked = plan.stopStep();
|
|
1071
|
-
const stopKind = state.
|
|
1089
|
+
const stopKind = this.stopKindOf(state, plan.stopEntry());
|
|
1072
1090
|
state.handover = {
|
|
1073
1091
|
kind: stopKind,
|
|
1074
1092
|
stepIndex: parked.stepIndex,
|
|
@@ -1130,9 +1148,22 @@ export class ReplayController {
|
|
|
1130
1148
|
? "a sub-task of this chain cannot run — the agent does that part itself"
|
|
1131
1149
|
: h.kind === "known_bad_step"
|
|
1132
1150
|
? "the next step has failed too often — the agent continues from here"
|
|
1133
|
-
:
|
|
1151
|
+
: h.kind === "nondeterministic_step"
|
|
1152
|
+
? "the next step needs a judgement the plan cannot compute — the agent makes it"
|
|
1153
|
+
: "a step broke — the agent continues from here",
|
|
1134
1154
|
});
|
|
1135
1155
|
}
|
|
1156
|
+
/**
|
|
1157
|
+
* Which kind of stop a plan's end is (plan-services.md D8). The stopping
|
|
1158
|
+
* step's own served reason wins — the calculation's verdict is more exact
|
|
1159
|
+
* than a counter — then a call the service could not answer, then a parked
|
|
1160
|
+
* step.
|
|
1161
|
+
*/
|
|
1162
|
+
stopKindOf(state, entry) {
|
|
1163
|
+
if (entry?.step.stop?.kind === "nondeterministic")
|
|
1164
|
+
return "nondeterministic_step";
|
|
1165
|
+
return state.flatStop ? "unusable_call" : "known_bad_step";
|
|
1166
|
+
}
|
|
1136
1167
|
noteFor(state, h) {
|
|
1137
1168
|
const plan = state.plan;
|
|
1138
1169
|
const executed = h.ranThisStep ? h.position + 1 : h.position;
|
|
@@ -1241,6 +1272,9 @@ export class ReplayController {
|
|
|
1241
1272
|
stage: info.stage ?? (status === "skipped" ? "tool_call" : undefined),
|
|
1242
1273
|
error,
|
|
1243
1274
|
durationMs: info.ms,
|
|
1275
|
+
// The tool is gone from its server: the service repairs the plan without
|
|
1276
|
+
// waiting out the grace (plan-services.md D8, kind 2).
|
|
1277
|
+
toolMissing: info.toolMissing || undefined,
|
|
1244
1278
|
// A verdict about a called segment's step is counted on that segment, at
|
|
1245
1279
|
// the revision it was served (R-CALL-13).
|
|
1246
1280
|
scenarioId: info.entry && info.entry.depth > 0 ? info.entry.scenarioId : undefined,
|
|
@@ -1338,11 +1372,6 @@ function targetKeys(schema) {
|
|
|
1338
1372
|
return [];
|
|
1339
1373
|
return Object.keys(schema).filter((k) => schema[k]?.kind !== "setting");
|
|
1340
1374
|
}
|
|
1341
|
-
/**
|
|
1342
|
-
* Position of the first step whose failure count is above `maxStepFailures`
|
|
1343
|
-
* (fallbk.md D3), or undefined when none is — including when the service sent
|
|
1344
|
-
* no policy, which an older service never does.
|
|
1345
|
-
*/
|
|
1346
1375
|
/**
|
|
1347
1376
|
* The index a hand-over reports, in the *caller's* chain (R-CALL-32).
|
|
1348
1377
|
*
|
|
@@ -1367,10 +1396,39 @@ function callLines(scenario) {
|
|
|
1367
1396
|
.map((s) => `${s.segmentId ?? "-"}:${s.unusable ?? s.resolution?.state ?? "resolved"}`)
|
|
1368
1397
|
.join(",");
|
|
1369
1398
|
}
|
|
1399
|
+
/** The served stops in a flat chain, for the `plan.armed` line (plan-services.md D8). */
|
|
1400
|
+
function stopLines(entries) {
|
|
1401
|
+
const lines = entries
|
|
1402
|
+
.map((e, i) => e.step.stop ? `${i}:${e.step.stop.kind}${e.step.stop.repairDue ? "/repair" : ""}` : undefined)
|
|
1403
|
+
.filter((s) => s !== undefined);
|
|
1404
|
+
return lines.length > 0 ? lines.join(",") : undefined;
|
|
1405
|
+
}
|
|
1406
|
+
/**
|
|
1407
|
+
* The old rule, for a service that sends no `stop`: the position of the first
|
|
1408
|
+
* step whose failure count is above `maxStepFailures` (fallbk.md D3), or
|
|
1409
|
+
* undefined when none is — including when the service sent no policy, which an
|
|
1410
|
+
* older service never does.
|
|
1411
|
+
*/
|
|
1370
1412
|
export function knownBadStepIndex(steps, maxStepFailures) {
|
|
1413
|
+
const i = steps.findIndex((s) => isKnownBad(s, maxStepFailures));
|
|
1414
|
+
return i < 0 ? undefined : i;
|
|
1415
|
+
}
|
|
1416
|
+
function isKnownBad(step, maxStepFailures) {
|
|
1371
1417
|
if (typeof maxStepFailures !== "number" || !Number.isFinite(maxStepFailures))
|
|
1372
|
-
return
|
|
1373
|
-
|
|
1418
|
+
return false;
|
|
1419
|
+
return (step.failureCount ?? 0) > maxStepFailures;
|
|
1420
|
+
}
|
|
1421
|
+
/**
|
|
1422
|
+
* Position of the first step the plan must stop in front of (plan-services.md
|
|
1423
|
+
* D8), or undefined when there is none.
|
|
1424
|
+
*
|
|
1425
|
+
* A step that carries `stop` was decided by the service — `null` means run it,
|
|
1426
|
+
* even when its count is past the limit: its retry is due. A step without the
|
|
1427
|
+
* field came from an older service, and the old counter rule decides it. Both
|
|
1428
|
+
* can sit in one chain, since a called segment is served on its own terms.
|
|
1429
|
+
*/
|
|
1430
|
+
export function stopIndex(steps, maxStepFailures) {
|
|
1431
|
+
const i = steps.findIndex((s) => s.stop !== undefined ? s.stop !== null : isKnownBad(s, maxStepFailures));
|
|
1374
1432
|
return i < 0 ? undefined : i;
|
|
1375
1433
|
}
|
|
1376
1434
|
//# sourceMappingURL=controller.js.map
|
package/dist/replay/handover.js
CHANGED
|
@@ -60,6 +60,11 @@ export function buildHandoverNote(n) {
|
|
|
60
60
|
: "";
|
|
61
61
|
lines.push(`Step ${human} of ${n.totalSteps}${tool.replace(/\)$/, `${owner})`)} did not run: it has failed${times} before, so the scenario stops in front of it.`);
|
|
62
62
|
}
|
|
63
|
+
else if (n.kind === "nondeterministic_step") {
|
|
64
|
+
// The calculation could not write code for this step's input: it is a
|
|
65
|
+
// judgement, and the agent is the one to make it (plan-services.md D8).
|
|
66
|
+
lines.push(`Step ${human} of ${n.totalSteps}${tool} did not run: its input needs a judgement the scenario cannot compute. Make that choice yourself and carry on.`);
|
|
67
|
+
}
|
|
63
68
|
else {
|
|
64
69
|
const error = n.error ? `: ${truncate(n.error, MAX_ERROR_CHARS)}` : "";
|
|
65
70
|
lines.push(`Step ${human} of ${n.totalSteps}${tool} failed${error}`);
|
package/dist/replay/plan.d.ts
CHANGED
|
@@ -65,6 +65,8 @@ export interface StepInfo {
|
|
|
65
65
|
error?: string;
|
|
66
66
|
derivedKeys?: string[];
|
|
67
67
|
ms: number;
|
|
68
|
+
/** The tool is gone from its server, read from the error (plan-services.md D8). */
|
|
69
|
+
toolMissing?: boolean;
|
|
68
70
|
}
|
|
69
71
|
export type StepObserver = (info: StepInfo) => void;
|
|
70
72
|
export interface ComposeResult {
|
package/dist/replay/plan.js
CHANGED
|
@@ -23,7 +23,7 @@
|
|
|
23
23
|
import { evalParamMapLogic, evalParamsLogic, evalResponseParamsLogic, evalResultMapLogic, evalToolInputLogic, evalToolOutputLogic, } from "./logic.js";
|
|
24
24
|
import { flatEntriesOf } from "./flatten.js";
|
|
25
25
|
import { assembleBundle, bundleInput, MAX_REPLAY_REASON } from "./bundle.js";
|
|
26
|
-
import { toolResultError } from "./tool-error.js";
|
|
26
|
+
import { isMissingToolError, toolResultError } from "./tool-error.js";
|
|
27
27
|
/**
|
|
28
28
|
* The recorded values for the *settings* a mapping did not name (R-CALL-21).
|
|
29
29
|
*
|
|
@@ -433,13 +433,29 @@ export class ScenarioReplayPlan {
|
|
|
433
433
|
: undefined) ?? (await recordedOutputFor?.(entry).catch(() => undefined));
|
|
434
434
|
if (!fallback) {
|
|
435
435
|
skipped += 1;
|
|
436
|
-
onStep?.({
|
|
436
|
+
onStep?.({
|
|
437
|
+
step,
|
|
438
|
+
entry,
|
|
439
|
+
input: computed,
|
|
440
|
+
outcome: "skipped",
|
|
441
|
+
error,
|
|
442
|
+
toolMissing: isMissingToolError(error) || undefined,
|
|
443
|
+
ms: Date.now() - startedAt,
|
|
444
|
+
});
|
|
437
445
|
continue;
|
|
438
446
|
}
|
|
439
447
|
response = fallback;
|
|
440
448
|
recorded = true;
|
|
441
449
|
recordedCount += 1;
|
|
442
|
-
onStep?.({
|
|
450
|
+
onStep?.({
|
|
451
|
+
step,
|
|
452
|
+
entry,
|
|
453
|
+
input: computed,
|
|
454
|
+
outcome: "recorded",
|
|
455
|
+
error,
|
|
456
|
+
toolMissing: isMissingToolError(error) || undefined,
|
|
457
|
+
ms: Date.now() - startedAt,
|
|
458
|
+
});
|
|
443
459
|
}
|
|
444
460
|
// Thread the output for later steps' inputs. A recorded output threads
|
|
445
461
|
// too: `toolOutputLogic` was authored against exactly this shape, and a
|
|
@@ -465,6 +481,7 @@ export class ScenarioReplayPlan {
|
|
|
465
481
|
outcome: error ? "failed" : "executed",
|
|
466
482
|
stage: error ? "tool_call" : undefined,
|
|
467
483
|
error,
|
|
484
|
+
toolMissing: isMissingToolError(error) || undefined,
|
|
468
485
|
derivedKeys,
|
|
469
486
|
ms: Date.now() - startedAt,
|
|
470
487
|
});
|
|
@@ -518,6 +535,9 @@ export class ScenarioReplayPlan {
|
|
|
518
535
|
}
|
|
519
536
|
const entry = this.entries[this.stepIndex];
|
|
520
537
|
const step = entry.step;
|
|
538
|
+
// Where this step sits in the flat list — what a stop reports. Taken now,
|
|
539
|
+
// because the cursor has moved on by the time some stops are written.
|
|
540
|
+
const flatIndex = this.stepIndex;
|
|
521
541
|
const startedAt = Date.now();
|
|
522
542
|
let computed;
|
|
523
543
|
try {
|
|
@@ -529,6 +549,7 @@ export class ScenarioReplayPlan {
|
|
|
529
549
|
stopped = {
|
|
530
550
|
kind: "step_failed",
|
|
531
551
|
stepIndex: step.stepIndex,
|
|
552
|
+
flatIndex,
|
|
532
553
|
stage: "tool_input_logic",
|
|
533
554
|
error: errText(err),
|
|
534
555
|
};
|
|
@@ -547,7 +568,15 @@ export class ScenarioReplayPlan {
|
|
|
547
568
|
: undefined) ?? (await recordedOutputFor?.(entry).catch(() => undefined));
|
|
548
569
|
if (!fallback) {
|
|
549
570
|
skipped += 1;
|
|
550
|
-
onStep?.({
|
|
571
|
+
onStep?.({
|
|
572
|
+
step,
|
|
573
|
+
entry,
|
|
574
|
+
input: computed,
|
|
575
|
+
outcome: "skipped",
|
|
576
|
+
error,
|
|
577
|
+
toolMissing: isMissingToolError(error) || undefined,
|
|
578
|
+
ms: Date.now() - startedAt,
|
|
579
|
+
});
|
|
551
580
|
// Advance past a step that cannot run, without threading anything.
|
|
552
581
|
// The frame is still left behind it: a sub-task whose last step could
|
|
553
582
|
// not run here still hands back whatever its earlier steps emitted.
|
|
@@ -558,7 +587,15 @@ export class ScenarioReplayPlan {
|
|
|
558
587
|
response = fallback;
|
|
559
588
|
recorded = true;
|
|
560
589
|
recordedCount += 1;
|
|
561
|
-
onStep?.({
|
|
590
|
+
onStep?.({
|
|
591
|
+
step,
|
|
592
|
+
entry,
|
|
593
|
+
input: computed,
|
|
594
|
+
outcome: "recorded",
|
|
595
|
+
error,
|
|
596
|
+
toolMissing: isMissingToolError(error) || undefined,
|
|
597
|
+
ms: Date.now() - startedAt,
|
|
598
|
+
});
|
|
562
599
|
}
|
|
563
600
|
if (stopOnFailure && !recorded) {
|
|
564
601
|
// Judged before the output logic, which was written against a success
|
|
@@ -573,6 +610,7 @@ export class ScenarioReplayPlan {
|
|
|
573
610
|
outcome: "failed",
|
|
574
611
|
stage: "tool_call",
|
|
575
612
|
error: toolError,
|
|
613
|
+
toolMissing: isMissingToolError(toolError) || undefined,
|
|
576
614
|
ms: Date.now() - startedAt,
|
|
577
615
|
});
|
|
578
616
|
entries.push({
|
|
@@ -585,6 +623,7 @@ export class ScenarioReplayPlan {
|
|
|
585
623
|
stopped = {
|
|
586
624
|
kind: "step_failed",
|
|
587
625
|
stepIndex: step.stepIndex,
|
|
626
|
+
flatIndex,
|
|
588
627
|
stage: "tool_call",
|
|
589
628
|
error: toolError,
|
|
590
629
|
};
|
|
@@ -608,6 +647,7 @@ export class ScenarioReplayPlan {
|
|
|
608
647
|
stopped = {
|
|
609
648
|
kind: "step_failed",
|
|
610
649
|
stepIndex: step.stepIndex,
|
|
650
|
+
flatIndex,
|
|
611
651
|
stage: "tool_output_logic",
|
|
612
652
|
error: errText(err),
|
|
613
653
|
};
|
|
@@ -627,6 +667,7 @@ export class ScenarioReplayPlan {
|
|
|
627
667
|
outcome: error ? "failed" : "executed",
|
|
628
668
|
stage: error ? "tool_call" : undefined,
|
|
629
669
|
error,
|
|
670
|
+
toolMissing: isMissingToolError(error) || undefined,
|
|
630
671
|
derivedKeys,
|
|
631
672
|
ms: Date.now() - startedAt,
|
|
632
673
|
});
|
|
@@ -640,7 +681,13 @@ export class ScenarioReplayPlan {
|
|
|
640
681
|
}
|
|
641
682
|
// Every planned step ran and the plan ends in front of a parked one.
|
|
642
683
|
if (!stopped && !partial && this.stopsEarly() && this.isDone()) {
|
|
643
|
-
|
|
684
|
+
// `known_bad_step` here means "in front of the plan's end": the
|
|
685
|
+
// controller reads the served reason and names the kind (D8).
|
|
686
|
+
stopped = {
|
|
687
|
+
kind: "known_bad_step",
|
|
688
|
+
stepIndex: this.entries[this.stopAt].stepIndex,
|
|
689
|
+
flatIndex: this.stopAt,
|
|
690
|
+
};
|
|
644
691
|
}
|
|
645
692
|
return {
|
|
646
693
|
text: assembleBundle(entries, maxChars, {
|
package/dist/replay/pricing.d.ts
CHANGED
|
@@ -17,7 +17,7 @@
|
|
|
17
17
|
* Bump the version — on **every** side — whenever a row changes.
|
|
18
18
|
*/
|
|
19
19
|
/** Identifies {@link MODEL_PRICING}. Must equal the service's constant. */
|
|
20
|
-
export declare const PRICING_VERSION = "2026-
|
|
20
|
+
export declare const PRICING_VERSION = "2026-09-24";
|
|
21
21
|
/**
|
|
22
22
|
* Tolerate provider prefixes (`anthropic/`, `us.anthropic.`), date snapshots and
|
|
23
23
|
* unseen version bumps by falling back to the longest matching family, so a new
|
package/dist/replay/pricing.js
CHANGED
|
@@ -17,24 +17,32 @@
|
|
|
17
17
|
* Bump the version — on **every** side — whenever a row changes.
|
|
18
18
|
*/
|
|
19
19
|
/** Identifies {@link MODEL_PRICING}. Must equal the service's constant. */
|
|
20
|
-
export const PRICING_VERSION = "2026-
|
|
21
|
-
/**
|
|
22
|
-
|
|
20
|
+
export const PRICING_VERSION = "2026-09-24";
|
|
21
|
+
/**
|
|
22
|
+
* A 5-minute cache write bills at 1.25x input. Cache reads bill at 0.1x input,
|
|
23
|
+
* except on models whose list price says otherwise (Opus 5.5, Fable 5.1), which
|
|
24
|
+
* pass their read price in dollars.
|
|
25
|
+
*/
|
|
26
|
+
const priced = (input, output, cacheRead = input * 0.1) => ({
|
|
23
27
|
input,
|
|
24
28
|
output,
|
|
25
|
-
cacheRead
|
|
29
|
+
cacheRead,
|
|
26
30
|
cacheWrite: input * 1.25,
|
|
27
31
|
});
|
|
28
32
|
const MODEL_PRICING = {
|
|
29
33
|
"claude-haiku-4-5": priced(1.0, 5.0),
|
|
30
34
|
"claude-haiku-4-5-20251001": priced(1.0, 5.0),
|
|
35
|
+
"claude-sonnet-4-5": priced(3.0, 15.0),
|
|
31
36
|
"claude-sonnet-4-6": priced(3.0, 15.0),
|
|
32
37
|
"claude-sonnet-5": priced(2.0, 10.0),
|
|
38
|
+
"claude-opus-4-5": priced(5.0, 25.0),
|
|
33
39
|
"claude-opus-4-6": priced(5.0, 25.0),
|
|
34
40
|
"claude-opus-4-7": priced(5.0, 25.0),
|
|
35
41
|
"claude-opus-4-8": priced(5.0, 25.0),
|
|
36
42
|
"claude-opus-5": priced(5.0, 25.0),
|
|
43
|
+
"claude-opus-5-5": priced(4.0, 20.0, 0.2),
|
|
37
44
|
"claude-fable-5": priced(10.0, 50.0),
|
|
45
|
+
"claude-fable-5-1": priced(10.0, 50.0, 0.25),
|
|
38
46
|
};
|
|
39
47
|
/**
|
|
40
48
|
* Tolerate provider prefixes (`anthropic/`, `us.anthropic.`), date snapshots and
|
|
@@ -20,4 +20,19 @@
|
|
|
20
20
|
* not. `serialized` is the proxy's own serialization — the whole result, as JSON.
|
|
21
21
|
*/
|
|
22
22
|
export declare function toolResultError(serialized: string): string | undefined;
|
|
23
|
+
/**
|
|
24
|
+
* What a server says when a tool no longer exists — the one failure a grace
|
|
25
|
+
* period cannot cure (plan-services.md D8, kind 2).
|
|
26
|
+
*
|
|
27
|
+
* Read from the text, because the proxy keeps only a JSON-RPC error's message
|
|
28
|
+
* and servers word it differently: the TypeScript SDK answers `Tool X not
|
|
29
|
+
* found`, the Python SDK `Unknown tool: X`, a hand-rolled server whatever it
|
|
30
|
+
* likes. Kept narrow on purpose — `not available` is what a server says during
|
|
31
|
+
* an outage, and a repair in the middle of one is the wrong answer. A miss here
|
|
32
|
+
* costs only the wait; a false hit costs one recording and one calculation,
|
|
33
|
+
* which the service caps.
|
|
34
|
+
*/
|
|
35
|
+
export declare const MISSING_TOOL_PATTERN: RegExp;
|
|
36
|
+
/** Whether an error text says the step's tool is gone from its server. */
|
|
37
|
+
export declare function isMissingToolError(text: string | undefined): boolean;
|
|
23
38
|
//# sourceMappingURL=tool-error.d.ts.map
|
|
@@ -35,6 +35,23 @@ export function toolResultError(serialized) {
|
|
|
35
35
|
return undefined;
|
|
36
36
|
return clip(firstText(result.content) ?? "the tool reported an error");
|
|
37
37
|
}
|
|
38
|
+
/**
|
|
39
|
+
* What a server says when a tool no longer exists — the one failure a grace
|
|
40
|
+
* period cannot cure (plan-services.md D8, kind 2).
|
|
41
|
+
*
|
|
42
|
+
* Read from the text, because the proxy keeps only a JSON-RPC error's message
|
|
43
|
+
* and servers word it differently: the TypeScript SDK answers `Tool X not
|
|
44
|
+
* found`, the Python SDK `Unknown tool: X`, a hand-rolled server whatever it
|
|
45
|
+
* likes. Kept narrow on purpose — `not available` is what a server says during
|
|
46
|
+
* an outage, and a repair in the middle of one is the wrong answer. A miss here
|
|
47
|
+
* costs only the wait; a false hit costs one recording and one calculation,
|
|
48
|
+
* which the service caps.
|
|
49
|
+
*/
|
|
50
|
+
export const MISSING_TOOL_PATTERN = /\b(unknown|no such|unsupported|unrecognized) tool\b|\btool\s+['"`]?[\w.:-]+['"`]?\s+(was\s+)?(not found|does not exist|is not registered)\b/i;
|
|
51
|
+
/** Whether an error text says the step's tool is gone from its server. */
|
|
52
|
+
export function isMissingToolError(text) {
|
|
53
|
+
return typeof text === "string" && MISSING_TOOL_PATTERN.test(text);
|
|
54
|
+
}
|
|
38
55
|
function firstText(content) {
|
|
39
56
|
if (!Array.isArray(content))
|
|
40
57
|
return undefined;
|
package/dist/replay/types.d.ts
CHANGED
|
@@ -37,6 +37,25 @@ export interface InlinedSegment {
|
|
|
37
37
|
}
|
|
38
38
|
/** Why a call cannot run right now (segmented.md 9.9). */
|
|
39
39
|
export type CallUnusable = "switched_off" | "not_ready" | "missing" | "stale" | "unresolved" | "depth" | "unsupported";
|
|
40
|
+
/**
|
|
41
|
+
* A reason the plan stops in front of a step, as the service decided it
|
|
42
|
+
* (plan-services.md D8). The service knows the grace period, the retry clock
|
|
43
|
+
* and the calculation's own verdict on the step; the runner only reads it.
|
|
44
|
+
*/
|
|
45
|
+
export interface StepStop {
|
|
46
|
+
/**
|
|
47
|
+
* `parked`: the step has failed too often and its retry is not due, or its
|
|
48
|
+
* grace has ended. `nondeterministic`: the calculation could not write code
|
|
49
|
+
* that computes this step's input — a judgement — so the agent makes it.
|
|
50
|
+
*/
|
|
51
|
+
kind: "parked" | "nondeterministic";
|
|
52
|
+
/** For `nondeterministic`: which check the step failed at calculation. */
|
|
53
|
+
why?: string;
|
|
54
|
+
/** For `parked` inside its grace: when the step may be tried again. */
|
|
55
|
+
retryAt?: string;
|
|
56
|
+
/** For `parked` past its grace, or whose tool is gone: a repair is due. */
|
|
57
|
+
repairDue?: boolean;
|
|
58
|
+
}
|
|
40
59
|
export interface SerializedScenarioStep {
|
|
41
60
|
stepIndex: number;
|
|
42
61
|
/**
|
|
@@ -60,6 +79,15 @@ export interface SerializedScenarioStep {
|
|
|
60
79
|
* model in front of it. Absent from an older service, which means never park.
|
|
61
80
|
*/
|
|
62
81
|
failureCount?: number;
|
|
82
|
+
/**
|
|
83
|
+
* Why the plan must stop in front of this step, decided by the service
|
|
84
|
+
* (plan-services.md D8). Present on every step — `null` when there is no
|
|
85
|
+
* reason — from a service that decides stops itself. A parked step whose
|
|
86
|
+
* retry is due arrives with `stop: null` and is tried. Absent altogether
|
|
87
|
+
* from an older service, where `failureCount` against
|
|
88
|
+
* `fallback.maxStepFailures` is the only rule.
|
|
89
|
+
*/
|
|
90
|
+
stop?: StepStop | null;
|
|
63
91
|
/**
|
|
64
92
|
* The literal output recorded for this step.
|
|
65
93
|
*
|
|
@@ -117,6 +145,13 @@ export declare function hasTargets(schema: ParamsSchema | null | undefined): boo
|
|
|
117
145
|
export interface SerializedScenario {
|
|
118
146
|
id: string;
|
|
119
147
|
runId: string;
|
|
148
|
+
/**
|
|
149
|
+
* The run the current chain was calculated from, when it is not `runId`: a
|
|
150
|
+
* plan repaired from a fresh recording (plan-services.md W2.4). A recorded
|
|
151
|
+
* output that stands in for a step comes from this run. Absent from an older
|
|
152
|
+
* service, and from a plan that was never repaired.
|
|
153
|
+
*/
|
|
154
|
+
sourceRunId?: string;
|
|
120
155
|
state: string;
|
|
121
156
|
intent: string;
|
|
122
157
|
parameters: unknown;
|
|
@@ -159,7 +194,13 @@ export type FallbackKind =
|
|
|
159
194
|
* gone, switched off, out of date, or nested too deep (segmented.md
|
|
160
195
|
* R-CALL-29). Like a known-bad step it counts nothing against any step.
|
|
161
196
|
*/
|
|
162
|
-
| "unusable_call"
|
|
197
|
+
| "unusable_call"
|
|
198
|
+
/**
|
|
199
|
+
* The plan stopped in front of a step the calculation marked
|
|
200
|
+
* non-deterministic: its input needs a judgement no code computes
|
|
201
|
+
* (plan-services.md D8, kind 3). Never tried, counts nothing.
|
|
202
|
+
*/
|
|
203
|
+
| "nondeterministic_step";
|
|
163
204
|
/**
|
|
164
205
|
* Upgrade order. `not_steered` is the floor — the control group, and the safe
|
|
165
206
|
* answer if a turn dies mid-flight — and each later decision point can only move
|
|
@@ -205,6 +246,12 @@ export interface ExecutionStepResult {
|
|
|
205
246
|
/** Message only — never a tool payload; this is rendered in a web page. */
|
|
206
247
|
error?: string;
|
|
207
248
|
durationMs?: number;
|
|
249
|
+
/**
|
|
250
|
+
* The step's tool no longer exists on its server, read from the error text
|
|
251
|
+
* (plan-services.md D8, kind 2). The service then repairs the plan without
|
|
252
|
+
* waiting out the grace period.
|
|
253
|
+
*/
|
|
254
|
+
toolMissing?: boolean;
|
|
208
255
|
/**
|
|
209
256
|
* The scenario this step belongs to, when it ran inside a called segment: a
|
|
210
257
|
* failure is counted where a repair would happen (segmented.md R-CALL-13).
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* journal — the audit lines that explain a turn, kept on disk for
|
|
3
|
+
* `bir investigate` (docs/calculatedReplayGuide.md §9.1).
|
|
4
|
+
*
|
|
5
|
+
* `logLine` prints to stderr and is gone when the terminal scrolls. The
|
|
6
|
+
* questions people ask afterwards — why did this prompt not run its scenario,
|
|
7
|
+
* which gate declined, what mode did the plan arm in, what did the execution
|
|
8
|
+
* report — are answered by exactly those lines. So `bir-hooks` opens a journal
|
|
9
|
+
* for its directory and every audit line whose event is in {@link isJournaled}
|
|
10
|
+
* lands in it too, as one JSON object per line with the same fields.
|
|
11
|
+
*
|
|
12
|
+
* One file per directory, keyed like the discovery file, so two projects on
|
|
13
|
+
* one machine never mix. Rotated at {@link JOURNAL_MAX_BYTES} by keeping the
|
|
14
|
+
* newest half. Written best-effort and synchronously: a full disk must never be
|
|
15
|
+
* the reason a hook fails (design §1, rule 4), and a hook's answer must not
|
|
16
|
+
* race the line that explains it.
|
|
17
|
+
*/
|
|
18
|
+
export interface JournalEntry {
|
|
19
|
+
/** ISO timestamp, written by the journal, not by the caller. */
|
|
20
|
+
at: string;
|
|
21
|
+
event: string;
|
|
22
|
+
[field: string]: unknown;
|
|
23
|
+
}
|
|
24
|
+
export declare function isJournaled(event: string): boolean;
|
|
25
|
+
export declare const JOURNAL_MAX_BYTES: number;
|
|
26
|
+
export declare function journalPath(cwd: string): string;
|
|
27
|
+
/** Start journaling this directory's audit lines. Returns the file, for the startup log. */
|
|
28
|
+
export declare function openJournal(cwd: string): string;
|
|
29
|
+
export declare function closeJournal(): void;
|
|
30
|
+
/**
|
|
31
|
+
* Append one entry, if a journal is open and the event is one we keep.
|
|
32
|
+
* Called by `logLine` for every audit line, and directly for the few facts
|
|
33
|
+
* that belong in the journal but not on stderr (a prompt preview).
|
|
34
|
+
*/
|
|
35
|
+
export declare function journal(event: string, fields?: Record<string, unknown>): void;
|
|
36
|
+
/** Every entry in a journal file, oldest first. A malformed line is skipped, not fatal. */
|
|
37
|
+
export declare function readJournal(path: string): JournalEntry[];
|
|
38
|
+
//# sourceMappingURL=journal.d.ts.map
|