@basein/runner 0.3.11 → 0.3.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/bir.js +4 -0
- package/dist/bin/investigate.js +5 -0
- package/dist/control/server.d.ts +32 -1
- package/dist/control/server.js +96 -33
- package/dist/record/recorder.d.ts +25 -0
- package/dist/record/recorder.js +3 -0
- package/dist/record/remote-recorder.d.ts +30 -5
- package/dist/record/remote-recorder.js +51 -6
- package/dist/record/switchable-recorder.js +2 -0
- package/dist/replay/controller.d.ts +18 -1
- package/dist/replay/controller.js +30 -12
- package/dist/replay/derive.d.ts +12 -0
- package/dist/replay/derive.js +25 -0
- package/dist/replay/types.d.ts +7 -0
- package/docs/calculatedReplay.md +18 -5
- package/package.json +1 -1
- package/production.md +1 -1
package/dist/bin/bir.js
CHANGED
|
@@ -68,6 +68,7 @@ import { SCENARIO_EDIT_SUBCOMMANDS, scenarioEditCommand } from "./scenario-edit.
|
|
|
68
68
|
import { loadCredentials } from "../auth/client.js";
|
|
69
69
|
import { packageVersion } from "../util/version.js";
|
|
70
70
|
import { runnerOutdated, serviceHeaders } from "../util/service-headers.js";
|
|
71
|
+
import { localDay } from "../replay/derive.js";
|
|
71
72
|
const VERSION = packageVersion();
|
|
72
73
|
/**
|
|
73
74
|
* The npx SessionStart hook's own budget, passed as `ensure --budget-ms <n>`
|
|
@@ -3630,6 +3631,9 @@ async function replayCommand(args) {
|
|
|
3630
3631
|
if (args.dry) {
|
|
3631
3632
|
const { status, body } = await service("POST", `/scenarios/${scenarioId}/replay`, {
|
|
3632
3633
|
prompt: args.prompt,
|
|
3634
|
+
// The day on this machine, as a session's reading sends it: a dry
|
|
3635
|
+
// replay works a value like "tomorrow" out from the same day.
|
|
3636
|
+
today: localDay(),
|
|
3633
3637
|
});
|
|
3634
3638
|
if (status !== 200) {
|
|
3635
3639
|
out(`Dry replay failed (HTTP ${status}): ${JSON.stringify(body)}`);
|
package/dist/bin/investigate.js
CHANGED
|
@@ -217,6 +217,11 @@ export const DECLINES = {
|
|
|
217
217
|
cause: "the plan was not ready before the hook that would have announced it had to answer: the prompt hook waits 7.5 s in all, match included, and a tool call that starts a plan mid-task waits 1.5 s. What arming waits for is the reading of the prompt's values (the derivation). A plan that was not announced is never run.",
|
|
218
218
|
fix: "run the prompt again. If it keeps happening the reading is slow: the recorder's log (`bir status` names the file) shows when its `replay.derived` line landed, and a dry replay shows what the service's reading took, under `reading` (`bir scenario replay <scnId> --prompt \"…\" --dry --json`). BIR_HOOK_DERIVE_BUDGET_MS does not help here, because the hook's own limit was reached first. To read values on this machine instead of asking the service, set BIR_DERIVE_API_KEY where Claude Code starts.",
|
|
219
219
|
},
|
|
220
|
+
// Nor is this one: the match itself was late, so nothing was armed at all.
|
|
221
|
+
match_late: {
|
|
222
|
+
cause: "the service's answer to the match landed too late: after the prompt hook had stopped waiting for it (3 s, BIR_MATCH_BUDGET_MS); while prompts were not waiting on the match at all — after one slow answer they stop waiting for a few minutes (`replay.match_gate state=holding`); or with the hook not waiting from the start, because a failure to reach the service was still on record (a `service.degraded` line, perhaps hours before, and no `service.recovered` until after this prompt) or the match request's first attempt failed. The agent did the task itself.",
|
|
223
|
+
fix: "nothing in the prompt has to change: run it again a few minutes later. A `service.recovered` line just after this turn's `run.start` means an old failure was still on record, and the next prompt waits again. If it keeps happening without one, the service answers matches slowly: `bir doctor` shows its state, and `/health.matchGate` when prompts wait again.",
|
|
224
|
+
},
|
|
220
225
|
runner_update_required: {
|
|
221
226
|
cause: "the service matched the prompt and handed over no plan: this runner is older than its plans need.",
|
|
222
227
|
fix: "update the runner (`bir update`), then run `bir setup` in each project that uses it; `bir doctor` says when the service is satisfied.",
|
package/dist/control/server.d.ts
CHANGED
|
@@ -582,6 +582,12 @@ export declare class ControlServer {
|
|
|
582
582
|
* plan. Memoized, because `ensureRun` is reached from five different hooks and
|
|
583
583
|
* arming twice would run the scenario twice.
|
|
584
584
|
*
|
|
585
|
+
* Two answers are followed. `pending` is what the prompt hook may wait for,
|
|
586
|
+
* and the recorder answers it null at once while a failure is on record or
|
|
587
|
+
* once the create's first attempt failed. `settled` is the service's own
|
|
588
|
+
* answer, whenever it lands: the hit that stops the recording, and a ticket
|
|
589
|
+
* the turn owes a report even when nobody waited for it.
|
|
590
|
+
*
|
|
585
591
|
* Resolves with the steering directive to inject, or undefined.
|
|
586
592
|
*/
|
|
587
593
|
private watchForMatch;
|
|
@@ -593,6 +599,11 @@ export declare class ControlServer {
|
|
|
593
599
|
* match. A match that misses its budget, or a plan still deriving when the
|
|
594
600
|
* budget runs out, is never delivered: the turn runs as an ordinary one, and
|
|
595
601
|
* one `replay.decision` line says why (docs/calculatedReplay.md §10).
|
|
602
|
+
*
|
|
603
|
+
* `pending` is what the prompt may wait for; `settled` is the service's own
|
|
604
|
+
* answer, whenever it lands (see {@link watchForMatch}). Every way of going
|
|
605
|
+
* ahead without a match follows `settled`, so a hand-out nobody waited for
|
|
606
|
+
* still has its ticket reported ({@link followLateMatch}).
|
|
596
607
|
*/
|
|
597
608
|
private armFromMatch;
|
|
598
609
|
/**
|
|
@@ -607,6 +618,25 @@ export declare class ControlServer {
|
|
|
607
618
|
* not several, so the hold doubles at most once per hold (on its probe).
|
|
608
619
|
*/
|
|
609
620
|
private noteMatchLatency;
|
|
621
|
+
/**
|
|
622
|
+
* The prompt went ahead without the match: when the service's own answer
|
|
623
|
+
* lands, a hit is a hand-out nobody used. It is said once, as `match_late`
|
|
624
|
+
* with `why`, and its ticket is carried to the turn's report
|
|
625
|
+
* ({@link noteLateMatch}). A miss says nothing: the service has no ticket out.
|
|
626
|
+
*/
|
|
627
|
+
private followLateMatch;
|
|
628
|
+
/**
|
|
629
|
+
* A match that landed after its prompt had been answered, or while prompts
|
|
630
|
+
* were not waiting on it. Nothing is armed for it, and nothing derived. But
|
|
631
|
+
* the service issued a ticket with that hand-out, and without a report under
|
|
632
|
+
* it the console can only say the turn never reported — for a turn the agent
|
|
633
|
+
* simply did itself. So the turn carries the ticket to its report, as a
|
|
634
|
+
* decline with its own code, like a plan that was only late ({@link shelve}).
|
|
635
|
+
*
|
|
636
|
+
* Only a hand-out with a ticket: a match the service handed nothing for has
|
|
637
|
+
* its reason on the service already. And only while the turn is open.
|
|
638
|
+
*/
|
|
639
|
+
private noteLateMatch;
|
|
610
640
|
/**
|
|
611
641
|
* {@link armFromMatch} with replay off: nothing can be announced, and no hook
|
|
612
642
|
* waits for the answer (`onPrompt` has already returned). So there is no
|
|
@@ -614,7 +644,8 @@ export declare class ControlServer {
|
|
|
614
644
|
* had, and must not mark the service slow (that would open the process-wide
|
|
615
645
|
* cool-down, and `/health` would report a degradation the user never had).
|
|
616
646
|
* A hit still books its decline — the ledger's baseline sample — whenever
|
|
617
|
-
* it lands, as long as the turn is still open to carry it.
|
|
647
|
+
* it lands, as long as the turn is still open to carry it. So `settled` is
|
|
648
|
+
* the service's own answer, not the one a hook may have been given at once.
|
|
618
649
|
*/
|
|
619
650
|
private noteDisabledMatch;
|
|
620
651
|
/**
|
package/dist/control/server.js
CHANGED
|
@@ -36,7 +36,7 @@ import { isHousekeepingCall } from "../record/housekeeping.js";
|
|
|
36
36
|
import { redact, redactText } from "../record/redact.js";
|
|
37
37
|
import { DEFAULT_MAX_STRING_BYTES, serializeCapped, truncateString, truncateStrings, } from "../record/truncate.js";
|
|
38
38
|
import { StepQueue } from "../record/queue.js";
|
|
39
|
-
import { NullRecorder, isIntentMatcher, isMatchAware, isRecorderHealthReporter, isRunCreationAware, isScenarioReporter, } from "../record/recorder.js";
|
|
39
|
+
import { NullRecorder, isIntentMatcher, isMatchAware, isRecorderHealthReporter, isRunCreationAware, isScenarioReporter, isServiceProber, } from "../record/recorder.js";
|
|
40
40
|
import { ReplayController, POLL_HOLD_MS, } from "../replay/controller.js";
|
|
41
41
|
import { calculateCostUsd, hasModelPricing } from "../replay/pricing.js";
|
|
42
42
|
import { logDetail, logLine, logOnce, errText } from "../util/log.js";
|
|
@@ -1640,6 +1640,12 @@ export class ControlServer {
|
|
|
1640
1640
|
* plan. Memoized, because `ensureRun` is reached from five different hooks and
|
|
1641
1641
|
* arming twice would run the scenario twice.
|
|
1642
1642
|
*
|
|
1643
|
+
* Two answers are followed. `pending` is what the prompt hook may wait for,
|
|
1644
|
+
* and the recorder answers it null at once while a failure is on record or
|
|
1645
|
+
* once the create's first attempt failed. `settled` is the service's own
|
|
1646
|
+
* answer, whenever it lands: the hit that stops the recording, and a ticket
|
|
1647
|
+
* the turn owes a report even when nobody waited for it.
|
|
1648
|
+
*
|
|
1643
1649
|
* Resolves with the steering directive to inject, or undefined.
|
|
1644
1650
|
*/
|
|
1645
1651
|
watchForMatch(session, run, prompt = "") {
|
|
@@ -1650,7 +1656,10 @@ export class ControlServer {
|
|
|
1650
1656
|
const runId = run.runId;
|
|
1651
1657
|
const startedAt = Date.now();
|
|
1652
1658
|
const pending = this.recorder.getMatch(runId).catch(() => null);
|
|
1653
|
-
|
|
1659
|
+
const settled = this.recorder.settledMatch
|
|
1660
|
+
? this.recorder.settledMatch(runId).catch(() => null)
|
|
1661
|
+
: pending;
|
|
1662
|
+
void settled.then((match) => {
|
|
1654
1663
|
if (!match || run.runId !== runId)
|
|
1655
1664
|
return;
|
|
1656
1665
|
run.recording = false;
|
|
@@ -1662,7 +1671,7 @@ export class ControlServer {
|
|
|
1662
1671
|
why: "similar prompt — the service kept its own run; not recording this one",
|
|
1663
1672
|
});
|
|
1664
1673
|
});
|
|
1665
|
-
run.matchWatch = this.armFromMatch(session, run, pending, prompt, startedAt).catch((err) => {
|
|
1674
|
+
run.matchWatch = this.armFromMatch(session, run, pending, settled, prompt, startedAt).catch((err) => {
|
|
1666
1675
|
// Replay is an optimisation; a broken one must never cost a turn.
|
|
1667
1676
|
logLine("replay.arm_failed", { run: run.runId, error: errText(err) });
|
|
1668
1677
|
return undefined;
|
|
@@ -1677,12 +1686,17 @@ export class ControlServer {
|
|
|
1677
1686
|
* match. A match that misses its budget, or a plan still deriving when the
|
|
1678
1687
|
* budget runs out, is never delivered: the turn runs as an ordinary one, and
|
|
1679
1688
|
* one `replay.decision` line says why (docs/calculatedReplay.md §10).
|
|
1689
|
+
*
|
|
1690
|
+
* `pending` is what the prompt may wait for; `settled` is the service's own
|
|
1691
|
+
* answer, whenever it lands (see {@link watchForMatch}). Every way of going
|
|
1692
|
+
* ahead without a match follows `settled`, so a hand-out nobody waited for
|
|
1693
|
+
* still has its ticket reported ({@link followLateMatch}).
|
|
1680
1694
|
*/
|
|
1681
|
-
async armFromMatch(session, run, pending, prompt, startedAt) {
|
|
1695
|
+
async armFromMatch(session, run, pending, settled, prompt, startedAt) {
|
|
1682
1696
|
// Off for the recorder, or — in a session of another host — not asked
|
|
1683
1697
|
// for by this run's adapter: either way nothing can be announced.
|
|
1684
1698
|
if (!this.replay.enabled || this.recordsOnly(session, run)) {
|
|
1685
|
-
return this.noteDisabledMatch(session, run,
|
|
1699
|
+
return this.noteDisabledMatch(session, run, settled, prompt);
|
|
1686
1700
|
}
|
|
1687
1701
|
const promptWaitMs = this.promptWaitMs();
|
|
1688
1702
|
const matchBudget = Math.min(this.replay.budgets.matchMs, promptWaitMs);
|
|
@@ -1699,18 +1713,7 @@ export class ControlServer {
|
|
|
1699
1713
|
run: run.runId,
|
|
1700
1714
|
why: "the service has been answering matches slowly — not waited on; this turn runs normally",
|
|
1701
1715
|
});
|
|
1702
|
-
|
|
1703
|
-
if (!late)
|
|
1704
|
-
return;
|
|
1705
|
-
logLine("replay.decision", {
|
|
1706
|
-
verdict: "no-steer",
|
|
1707
|
-
code: "match_late",
|
|
1708
|
-
run: run.runId,
|
|
1709
|
-
scenario: late.scenarioId ?? undefined,
|
|
1710
|
-
ms: Date.now() - startedAt,
|
|
1711
|
-
why: "the match landed while prompts were not waiting on it — not steered, and not recorded",
|
|
1712
|
-
});
|
|
1713
|
-
});
|
|
1716
|
+
this.followLateMatch(session, run, settled, startedAt, "the match landed while prompts were not waiting on it — not steered, and not recorded");
|
|
1714
1717
|
return undefined;
|
|
1715
1718
|
}
|
|
1716
1719
|
const cooling = !this.service.canWait();
|
|
@@ -1734,23 +1737,18 @@ export class ControlServer {
|
|
|
1734
1737
|
? "the service is failing — not waited on; this turn runs normally"
|
|
1735
1738
|
: "the match did not answer inside its budget; this turn runs normally",
|
|
1736
1739
|
});
|
|
1737
|
-
|
|
1738
|
-
if (!late)
|
|
1739
|
-
return;
|
|
1740
|
-
logLine("replay.decision", {
|
|
1741
|
-
verdict: "no-steer",
|
|
1742
|
-
code: "match_late",
|
|
1743
|
-
run: run.runId,
|
|
1744
|
-
scenario: late.scenarioId ?? undefined,
|
|
1745
|
-
ms: Date.now() - startedAt,
|
|
1746
|
-
why: "the match landed after the prompt was answered — not steered, and not recorded",
|
|
1747
|
-
});
|
|
1748
|
-
});
|
|
1740
|
+
this.followLateMatch(session, run, settled, startedAt, "the match landed after the prompt was answered — not steered, and not recorded");
|
|
1749
1741
|
}
|
|
1750
1742
|
return undefined;
|
|
1751
1743
|
}
|
|
1752
|
-
if (!match)
|
|
1744
|
+
if (!match) {
|
|
1745
|
+
// "No match" — which may have come without waiting: the recorder answers
|
|
1746
|
+
// the prompt so at once while a service failure is on record, however
|
|
1747
|
+
// old, and once the create's first attempt failed. The service may have
|
|
1748
|
+
// matched the prompt all the same, and issued a ticket.
|
|
1749
|
+
this.followLateMatch(session, run, settled, startedAt, "the match landed after the prompt had gone ahead without it (a service failure was still on record, or the create's first attempt failed) — not steered, and not recorded");
|
|
1753
1750
|
return undefined;
|
|
1751
|
+
}
|
|
1754
1752
|
// Awaited: a scenario with a target derives before the directive goes
|
|
1755
1753
|
// out, so a plan that cannot find its target is never delivered
|
|
1756
1754
|
// (segmented.md R-PARAM-4). Inside the prompt's own budget.
|
|
@@ -1869,6 +1867,45 @@ export class ControlServer {
|
|
|
1869
1867
|
why: "the match answered fast again — prompts wait on it as before",
|
|
1870
1868
|
});
|
|
1871
1869
|
}
|
|
1870
|
+
/**
|
|
1871
|
+
* The prompt went ahead without the match: when the service's own answer
|
|
1872
|
+
* lands, a hit is a hand-out nobody used. It is said once, as `match_late`
|
|
1873
|
+
* with `why`, and its ticket is carried to the turn's report
|
|
1874
|
+
* ({@link noteLateMatch}). A miss says nothing: the service has no ticket out.
|
|
1875
|
+
*/
|
|
1876
|
+
followLateMatch(session, run, settled, startedAt, why) {
|
|
1877
|
+
void settled.then((late) => {
|
|
1878
|
+
if (!late)
|
|
1879
|
+
return;
|
|
1880
|
+
logLine("replay.decision", {
|
|
1881
|
+
verdict: "no-steer",
|
|
1882
|
+
code: "match_late",
|
|
1883
|
+
run: run.runId,
|
|
1884
|
+
scenario: late.scenarioId ?? undefined,
|
|
1885
|
+
ms: Date.now() - startedAt,
|
|
1886
|
+
why,
|
|
1887
|
+
});
|
|
1888
|
+
this.noteLateMatch(session, run, late);
|
|
1889
|
+
});
|
|
1890
|
+
}
|
|
1891
|
+
/**
|
|
1892
|
+
* A match that landed after its prompt had been answered, or while prompts
|
|
1893
|
+
* were not waiting on it. Nothing is armed for it, and nothing derived. But
|
|
1894
|
+
* the service issued a ticket with that hand-out, and without a report under
|
|
1895
|
+
* it the console can only say the turn never reported — for a turn the agent
|
|
1896
|
+
* simply did itself. So the turn carries the ticket to its report, as a
|
|
1897
|
+
* decline with its own code, like a plan that was only late ({@link shelve}).
|
|
1898
|
+
*
|
|
1899
|
+
* Only a hand-out with a ticket: a match the service handed nothing for has
|
|
1900
|
+
* its reason on the service already. And only while the turn is open.
|
|
1901
|
+
*/
|
|
1902
|
+
noteLateMatch(session, run, match) {
|
|
1903
|
+
if (run.finished || run.lateMatch || !match.scenarioId || !match.executionTicket)
|
|
1904
|
+
return;
|
|
1905
|
+
const state = this.replay.lateMatch(match, this.hostContext(session, run));
|
|
1906
|
+
state.sessionId = session.sessionId;
|
|
1907
|
+
run.lateMatch = state;
|
|
1908
|
+
}
|
|
1872
1909
|
/**
|
|
1873
1910
|
* {@link armFromMatch} with replay off: nothing can be announced, and no hook
|
|
1874
1911
|
* waits for the answer (`onPrompt` has already returned). So there is no
|
|
@@ -1876,10 +1913,11 @@ export class ControlServer {
|
|
|
1876
1913
|
* had, and must not mark the service slow (that would open the process-wide
|
|
1877
1914
|
* cool-down, and `/health` would report a degradation the user never had).
|
|
1878
1915
|
* A hit still books its decline — the ledger's baseline sample — whenever
|
|
1879
|
-
* it lands, as long as the turn is still open to carry it.
|
|
1916
|
+
* it lands, as long as the turn is still open to carry it. So `settled` is
|
|
1917
|
+
* the service's own answer, not the one a hook may have been given at once.
|
|
1880
1918
|
*/
|
|
1881
|
-
async noteDisabledMatch(session, run,
|
|
1882
|
-
const match = await
|
|
1919
|
+
async noteDisabledMatch(session, run, settled, prompt) {
|
|
1920
|
+
const match = await settled;
|
|
1883
1921
|
if (!match || run.finished)
|
|
1884
1922
|
return undefined;
|
|
1885
1923
|
// Declined at its first gate, before derivation: no transcript is read for it.
|
|
@@ -2436,6 +2474,25 @@ export class ControlServer {
|
|
|
2436
2474
|
}
|
|
2437
2475
|
this.queue.push(() => recorder.reportExecution(report));
|
|
2438
2476
|
});
|
|
2477
|
+
// The prompt match that landed too late (see `noteLateMatch`). Not when a
|
|
2478
|
+
// plan of the same scenario armed later in the turn: that one has a ticket
|
|
2479
|
+
// of its own, and says what happened. The turn's cost is a sample of the
|
|
2480
|
+
// task's usual cost only when no plan ran in it at all.
|
|
2481
|
+
const late = run.lateMatch;
|
|
2482
|
+
if (late && !states.some((s) => s.scenarioId === late.scenarioId)) {
|
|
2483
|
+
const alone = states.length === 0;
|
|
2484
|
+
const report = this.replay.buildReport(late, {
|
|
2485
|
+
sessionCostUsd: alone ? cost.usd : 0,
|
|
2486
|
+
measured: alone && cost.measured,
|
|
2487
|
+
durationMs,
|
|
2488
|
+
prompt: run.input || undefined,
|
|
2489
|
+
siblings: [...states, late],
|
|
2490
|
+
runId: run.runId,
|
|
2491
|
+
cutShort: cutShort || undefined,
|
|
2492
|
+
});
|
|
2493
|
+
if (report)
|
|
2494
|
+
this.queue.push(() => recorder.reportExecution(report));
|
|
2495
|
+
}
|
|
2439
2496
|
}
|
|
2440
2497
|
/**
|
|
2441
2498
|
* Seal the session's run now, and drain it to the service in the background.
|
|
@@ -2513,6 +2570,12 @@ export class ControlServer {
|
|
|
2513
2570
|
cwd: session.cwd,
|
|
2514
2571
|
why: "run deferred to prompt",
|
|
2515
2572
|
});
|
|
2573
|
+
// The first prompt is seconds away, and its match is not waited for while
|
|
2574
|
+
// a service failure is on record — one from hours ago included, when the
|
|
2575
|
+
// machine slept or went offline and nothing was sent since. Find out now,
|
|
2576
|
+
// in the background, whether the service answers again.
|
|
2577
|
+
if (isServiceProber(this.recorder))
|
|
2578
|
+
this.recorder.probeIfFailing();
|
|
2516
2579
|
return {};
|
|
2517
2580
|
}
|
|
2518
2581
|
async onPrompt(payload) {
|
|
@@ -180,6 +180,16 @@ export interface RunMatch {
|
|
|
180
180
|
/** Optional capability: `null` means "fresh run, keep recording". */
|
|
181
181
|
export interface MatchAware {
|
|
182
182
|
getMatch(runId: string): Promise<RunMatch | null>;
|
|
183
|
+
/**
|
|
184
|
+
* The create call's own answer, whenever it lands. `getMatch` is what a hook
|
|
185
|
+
* may wait for, and a recorder may answer it null without waiting — while
|
|
186
|
+
* the service has a failure on record, or once the create's first attempt
|
|
187
|
+
* failed. The service can still match the prompt then, hand out the scenario
|
|
188
|
+
* and issue a ticket the turn owes a report; this is how the turn finds out.
|
|
189
|
+
* Never for a hook to wait on. Absent where `getMatch` always waits for the
|
|
190
|
+
* create's answer: it is the same answer then.
|
|
191
|
+
*/
|
|
192
|
+
settledMatch?(runId: string): Promise<RunMatch | null>;
|
|
183
193
|
}
|
|
184
194
|
export declare function isMatchAware(r: Recorder): r is Recorder & MatchAware;
|
|
185
195
|
/**
|
|
@@ -201,6 +211,21 @@ export interface RunCreationAware {
|
|
|
201
211
|
runCreated(runId: string): Promise<RunCreation>;
|
|
202
212
|
}
|
|
203
213
|
export declare function isRunCreationAware(r: Recorder): r is Recorder & RunCreationAware;
|
|
214
|
+
/**
|
|
215
|
+
* Optional capability: a prompt is on its way, so the service verdict it will
|
|
216
|
+
* be decided by should be current.
|
|
217
|
+
*
|
|
218
|
+
* A hook never waits on a service with a failure on record, and is never the
|
|
219
|
+
* request that finds out whether it answers again. After hours with nothing
|
|
220
|
+
* sent — a laptop asleep, a network that dropped — the failure is still on
|
|
221
|
+
* record at the next prompt, whose match is then not waited for however fast
|
|
222
|
+
* the service answers. Called when a session starts, before its first prompt:
|
|
223
|
+
* it checks in the background, and sends nothing while no failure is on record.
|
|
224
|
+
*/
|
|
225
|
+
export interface ServiceProber {
|
|
226
|
+
probeIfFailing(): void;
|
|
227
|
+
}
|
|
228
|
+
export declare function isServiceProber(r: Recorder): r is Recorder & ServiceProber;
|
|
204
229
|
/** What a ReAct iteration sends to find a scenario by its intent (fallbk.md §Runner 4). */
|
|
205
230
|
export interface IntentMatchRequest {
|
|
206
231
|
/**
|
package/dist/record/recorder.js
CHANGED
|
@@ -14,6 +14,9 @@ export function isMatchAware(r) {
|
|
|
14
14
|
export function isRunCreationAware(r) {
|
|
15
15
|
return typeof r.runCreated === "function";
|
|
16
16
|
}
|
|
17
|
+
export function isServiceProber(r) {
|
|
18
|
+
return typeof r.probeIfFailing === "function";
|
|
19
|
+
}
|
|
17
20
|
export function isIntentMatcher(r) {
|
|
18
21
|
return typeof r.matchIntent === "function";
|
|
19
22
|
}
|
|
@@ -43,8 +43,10 @@
|
|
|
43
43
|
* SIMILAR-PROMPT HITS. `POST /recordings/runs` may answer `{ id: null, matched }`,
|
|
44
44
|
* meaning the server recognised the prompt and did **not** create a run. Every
|
|
45
45
|
* later step post for that id would then 404, so callers must stop recording —
|
|
46
|
-
* {@link RemoteRecorder.
|
|
47
|
-
*
|
|
46
|
+
* {@link RemoteRecorder.settledMatch} is how they find out, whenever the answer
|
|
47
|
+
* lands ({@link RemoteRecorder.getMatch} is only what a hook may wait for) — and
|
|
48
|
+
* this recorder skips them itself too, for the caller that stopped waiting
|
|
49
|
+
* before the answer came.
|
|
48
50
|
*/
|
|
49
51
|
import { type AuthSession } from "../auth/client.js";
|
|
50
52
|
import { type ServiceHealth } from "../util/service-health.js";
|
|
@@ -187,11 +189,34 @@ export declare class RemoteRecorder implements Recorder, RecorderHealthReporter
|
|
|
187
189
|
matchIntent(req: IntentMatchRequest, opts?: IntentMatchOptions): Promise<IntentMatchAnswer | null>;
|
|
188
190
|
/**
|
|
189
191
|
* The similar-prompt match for a run's create call (null = fresh run). Like
|
|
190
|
-
* {@link runCreated}, answers null at once while
|
|
191
|
-
*
|
|
192
|
-
*
|
|
192
|
+
* {@link runCreated}, answers null at once while a failure is on record —
|
|
193
|
+
* in a cool-down, and after it until an answer clears it, however long ago
|
|
194
|
+
* it was — and as soon as the create's first attempt fails: the first prompt
|
|
195
|
+
* of an outage is not held for the retry's sleep (up to 2 s behind a
|
|
196
|
+
* Retry-After). The create goes on, and its own answer is
|
|
197
|
+
* {@link settledMatch}'s.
|
|
193
198
|
*/
|
|
194
199
|
getMatch(runId: string): Promise<RunMatch | null>;
|
|
200
|
+
/**
|
|
201
|
+
* The create's own match (null = fresh run), whenever it lands — after the
|
|
202
|
+
* cool-down and the retry, if it took them — however {@link getMatch}
|
|
203
|
+
* answered the hook. A hook told "no match" at once, because a failure was
|
|
204
|
+
* on record or the first attempt failed, may be about a prompt the service
|
|
205
|
+
* matched all the same: it then handed out the scenario and issued a ticket,
|
|
206
|
+
* and the turn owes that ticket a report. Never for a hook to wait on. Null
|
|
207
|
+
* for an id this recorder never opened.
|
|
208
|
+
*/
|
|
209
|
+
settledMatch(runId: string): Promise<RunMatch | null>;
|
|
210
|
+
/**
|
|
211
|
+
* A prompt is on its way (a session just started). While a failure is on
|
|
212
|
+
* record no hook waits on the service, and none is the request that finds
|
|
213
|
+
* out whether it answers again ({@link mayHotWait}). After hours with
|
|
214
|
+
* nothing sent, the failure is still on record at the first prompt, and its
|
|
215
|
+
* match is not waited for however fast the service answers — so find out
|
|
216
|
+
* now: one background `GET /health`, once the cool-down is over. Nothing
|
|
217
|
+
* while no failure is on record, during a cool-down, or with a probe out.
|
|
218
|
+
*/
|
|
219
|
+
probeIfFailing(): void;
|
|
195
220
|
recordToolSelected(runId: string, stepIndex: number, data: {
|
|
196
221
|
toolName: string;
|
|
197
222
|
toolInput: string;
|
|
@@ -43,8 +43,10 @@
|
|
|
43
43
|
* SIMILAR-PROMPT HITS. `POST /recordings/runs` may answer `{ id: null, matched }`,
|
|
44
44
|
* meaning the server recognised the prompt and did **not** create a run. Every
|
|
45
45
|
* later step post for that id would then 404, so callers must stop recording —
|
|
46
|
-
* {@link RemoteRecorder.
|
|
47
|
-
*
|
|
46
|
+
* {@link RemoteRecorder.settledMatch} is how they find out, whenever the answer
|
|
47
|
+
* lands ({@link RemoteRecorder.getMatch} is only what a hook may wait for) — and
|
|
48
|
+
* this recorder skips them itself too, for the caller that stopped waiting
|
|
49
|
+
* before the answer came.
|
|
48
50
|
*/
|
|
49
51
|
import { randomUUID } from "node:crypto";
|
|
50
52
|
import { statSync } from "node:fs";
|
|
@@ -479,7 +481,8 @@ export class RemoteRecorder {
|
|
|
479
481
|
}),
|
|
480
482
|
// The id is client-generated, so the service can tell a repeat. A
|
|
481
483
|
// hook waiting on this answer is let go before the retry: a match
|
|
482
|
-
// found by the retry
|
|
484
|
+
// found by the retry can no longer steer the turn, and only its
|
|
485
|
+
// ticket's report is left to it (`settledMatch`).
|
|
483
486
|
{ retryTransient: true, onRetry: () => entry.release() });
|
|
484
487
|
const b = body && typeof body === "object"
|
|
485
488
|
? body
|
|
@@ -579,9 +582,12 @@ export class RemoteRecorder {
|
|
|
579
582
|
}
|
|
580
583
|
/**
|
|
581
584
|
* The similar-prompt match for a run's create call (null = fresh run). Like
|
|
582
|
-
* {@link runCreated}, answers null at once while
|
|
583
|
-
*
|
|
584
|
-
*
|
|
585
|
+
* {@link runCreated}, answers null at once while a failure is on record —
|
|
586
|
+
* in a cool-down, and after it until an answer clears it, however long ago
|
|
587
|
+
* it was — and as soon as the create's first attempt fails: the first prompt
|
|
588
|
+
* of an outage is not held for the retry's sleep (up to 2 s behind a
|
|
589
|
+
* Retry-After). The create goes on, and its own answer is
|
|
590
|
+
* {@link settledMatch}'s.
|
|
585
591
|
*/
|
|
586
592
|
async getMatch(runId) {
|
|
587
593
|
const entry = this.runs.get(runId);
|
|
@@ -595,6 +601,39 @@ export class RemoteRecorder {
|
|
|
595
601
|
await entry.hot;
|
|
596
602
|
return entry.settled?.match ?? null;
|
|
597
603
|
}
|
|
604
|
+
/**
|
|
605
|
+
* The create's own match (null = fresh run), whenever it lands — after the
|
|
606
|
+
* cool-down and the retry, if it took them — however {@link getMatch}
|
|
607
|
+
* answered the hook. A hook told "no match" at once, because a failure was
|
|
608
|
+
* on record or the first attempt failed, may be about a prompt the service
|
|
609
|
+
* matched all the same: it then handed out the scenario and issued a ticket,
|
|
610
|
+
* and the turn owes that ticket a report. Never for a hook to wait on. Null
|
|
611
|
+
* for an id this recorder never opened.
|
|
612
|
+
*/
|
|
613
|
+
async settledMatch(runId) {
|
|
614
|
+
const entry = this.runs.get(runId);
|
|
615
|
+
if (!entry)
|
|
616
|
+
return null;
|
|
617
|
+
entry.lastUsed = Date.now();
|
|
618
|
+
if (entry.settled)
|
|
619
|
+
return entry.settled.match;
|
|
620
|
+
return (await entry.answer) ?? null;
|
|
621
|
+
}
|
|
622
|
+
/**
|
|
623
|
+
* A prompt is on its way (a session just started). While a failure is on
|
|
624
|
+
* record no hook waits on the service, and none is the request that finds
|
|
625
|
+
* out whether it answers again ({@link mayHotWait}). After hours with
|
|
626
|
+
* nothing sent, the failure is still on record at the first prompt, and its
|
|
627
|
+
* match is not waited for however fast the service answers — so find out
|
|
628
|
+
* now: one background `GET /health`, once the cool-down is over. Nothing
|
|
629
|
+
* while no failure is on record, during a cool-down, or with a probe out.
|
|
630
|
+
*/
|
|
631
|
+
probeIfFailing() {
|
|
632
|
+
this.checkCredentials();
|
|
633
|
+
if (this.state !== "recording" || this.healthyNow())
|
|
634
|
+
return;
|
|
635
|
+
this.probeSoon();
|
|
636
|
+
}
|
|
598
637
|
recordToolSelected(runId, stepIndex, data) {
|
|
599
638
|
const id = "step_" + randomUUID();
|
|
600
639
|
const toolInput = pend(data.toolInput, DEFAULT_MAX_PAYLOAD_BYTES, true);
|
|
@@ -796,6 +835,7 @@ export class RemoteRecorder {
|
|
|
796
835
|
newEntry() {
|
|
797
836
|
let resolveHot;
|
|
798
837
|
let resolveCreated;
|
|
838
|
+
let resolveAnswer;
|
|
799
839
|
const entry = {
|
|
800
840
|
userId: this.session.user.id,
|
|
801
841
|
hot: new Promise((r) => {
|
|
@@ -804,6 +844,9 @@ export class RemoteRecorder {
|
|
|
804
844
|
created: new Promise((r) => {
|
|
805
845
|
resolveCreated = r;
|
|
806
846
|
}),
|
|
847
|
+
answer: new Promise((r) => {
|
|
848
|
+
resolveAnswer = r;
|
|
849
|
+
}),
|
|
807
850
|
settle: (match, created) => {
|
|
808
851
|
if (entry.settled)
|
|
809
852
|
return;
|
|
@@ -811,9 +854,11 @@ export class RemoteRecorder {
|
|
|
811
854
|
entry.settledAt = Date.now();
|
|
812
855
|
resolveHot();
|
|
813
856
|
resolveCreated(created);
|
|
857
|
+
resolveAnswer(match);
|
|
814
858
|
// Callers that are waiting hold the promises themselves.
|
|
815
859
|
entry.hot = undefined;
|
|
816
860
|
entry.created = undefined;
|
|
861
|
+
entry.answer = undefined;
|
|
817
862
|
},
|
|
818
863
|
release: () => {
|
|
819
864
|
if (entry.settled || entry.released)
|
|
@@ -46,10 +46,12 @@ import { NullRecorder, } from "./recorder.js";
|
|
|
46
46
|
*/
|
|
47
47
|
const NEUTRAL = {
|
|
48
48
|
getMatch: () => Promise.resolve(null),
|
|
49
|
+
settledMatch: () => Promise.resolve(null),
|
|
49
50
|
runCreated: () => Promise.resolve({ created: true }),
|
|
50
51
|
matchIntent: () => Promise.resolve(null),
|
|
51
52
|
reportExecution: () => undefined,
|
|
52
53
|
backfillContext: () => undefined,
|
|
54
|
+
probeIfFailing: () => undefined,
|
|
53
55
|
};
|
|
54
56
|
/**
|
|
55
57
|
* `RemoteRecorder`'s private helpers. They are methods at runtime (TypeScript's
|
|
@@ -514,7 +514,13 @@ export type DeclineCode = "replay_disabled" | "not_ready" | "flatten_failed" | "
|
|
|
514
514
|
* (`arm_unannounced`). Set by the control server, where a prompt's wait
|
|
515
515
|
* ends, so the report can say why a matched turn was not steered.
|
|
516
516
|
*/
|
|
517
|
-
| "arm_timeout" | "arm_unannounced"
|
|
517
|
+
| "arm_timeout" | "arm_unannounced"
|
|
518
|
+
/**
|
|
519
|
+
* Nothing was armed at all: the service's answer to the match landed after
|
|
520
|
+
* the prompt had been answered, or while prompts were not waiting on it
|
|
521
|
+
* (the match gate). Set by the control server, like the two above.
|
|
522
|
+
*/
|
|
523
|
+
| "match_late";
|
|
518
524
|
/** What `PreToolUse` should do about this call. */
|
|
519
525
|
export type PreToolAction =
|
|
520
526
|
/** Pin the arguments and let the real tool run. */
|
|
@@ -587,6 +593,17 @@ export declare class ReplayController {
|
|
|
587
593
|
* covers both "no match" and "too slow", which are the same thing from here.
|
|
588
594
|
*/
|
|
589
595
|
awaitMatch(pending: Promise<unknown>): Promise<RunMatchLike | null>;
|
|
596
|
+
/** A matched state before any gate has judged it: nothing planned, nothing run. */
|
|
597
|
+
private blankState;
|
|
598
|
+
/**
|
|
599
|
+
* The state of a prompt match that landed too late to be used
|
|
600
|
+
* (`match_late`). No gate is run and nothing is derived — a plan could no
|
|
601
|
+
* longer be announced, and a reading nobody waits for is money spent on
|
|
602
|
+
* nothing. It exists for the one thing the hand-out still owes: its ticket's
|
|
603
|
+
* report, with the reason, so the service does not show a turn the agent did
|
|
604
|
+
* itself as one that never reported.
|
|
605
|
+
*/
|
|
606
|
+
lateMatch(match: RunMatchLike, host?: HostContext): ReplayState;
|
|
590
607
|
/**
|
|
591
608
|
* Run the gate ladder and, if every gate passes, arm a plan.
|
|
592
609
|
*
|
|
@@ -305,18 +305,9 @@ export class ReplayController {
|
|
|
305
305
|
clearTimeout(timer);
|
|
306
306
|
}
|
|
307
307
|
}
|
|
308
|
-
/**
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
* Always returns a state — a declined match still has a ticket to redeem and a
|
|
312
|
-
* baseline sample to contribute, and losing that is how a savings ledger ends
|
|
313
|
-
* up with a denominator nobody measured.
|
|
314
|
-
*
|
|
315
|
-
* `host` is given for a session whose host is not Claude Code, and is kept
|
|
316
|
-
* on the state (see {@link HostContext}).
|
|
317
|
-
*/
|
|
318
|
-
async arm(match, prompt, wrapped, armedBy = "prompt", ctx = {}, host) {
|
|
319
|
-
const state = {
|
|
308
|
+
/** A matched state before any gate has judged it: nothing planned, nothing run. */
|
|
309
|
+
blankState(match, wrapped, armedBy, host) {
|
|
310
|
+
return {
|
|
320
311
|
scenarioId: match.scenarioId,
|
|
321
312
|
ticket: match.executionTicket,
|
|
322
313
|
similarity: match.similarity,
|
|
@@ -345,6 +336,33 @@ export class ReplayController {
|
|
|
345
336
|
verified: match.verified,
|
|
346
337
|
sharedWith: [],
|
|
347
338
|
};
|
|
339
|
+
}
|
|
340
|
+
/**
|
|
341
|
+
* The state of a prompt match that landed too late to be used
|
|
342
|
+
* (`match_late`). No gate is run and nothing is derived — a plan could no
|
|
343
|
+
* longer be announced, and a reading nobody waits for is money spent on
|
|
344
|
+
* nothing. It exists for the one thing the hand-out still owes: its ticket's
|
|
345
|
+
* report, with the reason, so the service does not show a turn the agent did
|
|
346
|
+
* itself as one that never reported.
|
|
347
|
+
*/
|
|
348
|
+
lateMatch(match, host) {
|
|
349
|
+
const state = this.blankState(match, new Set(), "prompt", host);
|
|
350
|
+
state.declined = "match_late";
|
|
351
|
+
state.retired = true;
|
|
352
|
+
return state;
|
|
353
|
+
}
|
|
354
|
+
/**
|
|
355
|
+
* Run the gate ladder and, if every gate passes, arm a plan.
|
|
356
|
+
*
|
|
357
|
+
* Always returns a state — a declined match still has a ticket to redeem and a
|
|
358
|
+
* baseline sample to contribute, and losing that is how a savings ledger ends
|
|
359
|
+
* up with a denominator nobody measured.
|
|
360
|
+
*
|
|
361
|
+
* `host` is given for a session whose host is not Claude Code, and is kept
|
|
362
|
+
* on the state (see {@link HostContext}).
|
|
363
|
+
*/
|
|
364
|
+
async arm(match, prompt, wrapped, armedBy = "prompt", ctx = {}, host) {
|
|
365
|
+
const state = this.blankState(match, wrapped, armedBy, host);
|
|
348
366
|
const isSegment = state.kind === "segment";
|
|
349
367
|
const decline = (why, code) => {
|
|
350
368
|
state.declined = code;
|
package/dist/replay/derive.d.ts
CHANGED
|
@@ -60,6 +60,8 @@ export interface DeriveOptions extends DeriveContext {
|
|
|
60
60
|
prompt: string;
|
|
61
61
|
intent: string;
|
|
62
62
|
paramsObject: ParamsSchema | null;
|
|
63
|
+
/** The day on this machine. Injected by tests; defaults to {@link localDay}. */
|
|
64
|
+
today?: string;
|
|
63
65
|
/** An override that keeps the reading on this machine. Absent → the service. */
|
|
64
66
|
apiKey?: string;
|
|
65
67
|
/** Used when there is no `apiKey`. Absent too → recorded samples. */
|
|
@@ -90,6 +92,16 @@ export interface DeriveResult {
|
|
|
90
92
|
/** Settings filled from their recorded sample (R-PARAM-2). */
|
|
91
93
|
sampled: string[];
|
|
92
94
|
}
|
|
95
|
+
/**
|
|
96
|
+
* The day on this machine, `YYYY-MM-DD`, in its own time zone.
|
|
97
|
+
*
|
|
98
|
+
* Sent to the service with every reading, because a prompt says "tomorrow" and
|
|
99
|
+
* the date is in no prompt: the service works such a value out, and it must
|
|
100
|
+
* count from the user's day. Its own day is the wrong one — a server on UTC is
|
|
101
|
+
* a day ahead of a user in Seattle for a third of the working day. Local on
|
|
102
|
+
* purpose, then, and never `toISOString`, which is UTC.
|
|
103
|
+
*/
|
|
104
|
+
export declare function localDay(now?: Date): string;
|
|
93
105
|
/**
|
|
94
106
|
* First balanced `{…}` in a text blob. Models wrap JSON in prose and fences more
|
|
95
107
|
* often than they emit it bare, and a failed parse here costs a whole replay.
|
package/dist/replay/derive.js
CHANGED
|
@@ -75,6 +75,19 @@ function serviceOk(startedAt) {
|
|
|
75
75
|
const ok = serviceHealth.ok.bind(serviceHealth);
|
|
76
76
|
ok(startedAt);
|
|
77
77
|
}
|
|
78
|
+
/**
|
|
79
|
+
* The day on this machine, `YYYY-MM-DD`, in its own time zone.
|
|
80
|
+
*
|
|
81
|
+
* Sent to the service with every reading, because a prompt says "tomorrow" and
|
|
82
|
+
* the date is in no prompt: the service works such a value out, and it must
|
|
83
|
+
* count from the user's day. Its own day is the wrong one — a server on UTC is
|
|
84
|
+
* a day ahead of a user in Seattle for a third of the working day. Local on
|
|
85
|
+
* purpose, then, and never `toISOString`, which is UTC.
|
|
86
|
+
*/
|
|
87
|
+
export function localDay(now = new Date()) {
|
|
88
|
+
const two = (n) => String(n).padStart(2, "0");
|
|
89
|
+
return `${now.getFullYear()}-${two(now.getMonth() + 1)}-${two(now.getDate())}`;
|
|
90
|
+
}
|
|
78
91
|
/** Every parameter at its recorded sample value. The zero-cost fallback. */
|
|
79
92
|
function sampleValues(schema) {
|
|
80
93
|
const out = {};
|
|
@@ -251,6 +264,15 @@ export async function deriveParameters(opts) {
|
|
|
251
264
|
const missing = [];
|
|
252
265
|
const sampled = [];
|
|
253
266
|
for (const key of keys) {
|
|
267
|
+
// A value worked out from the day ("tomorrow") cannot be read here: this
|
|
268
|
+
// copy of the text does not tell the model the day, and what comes back
|
|
269
|
+
// for such a value is then the recorded date or a guess. The service's
|
|
270
|
+
// reading does work it out; with a key of its own the value is missing.
|
|
271
|
+
if (isTarget(schema, key) && schema[key].from === "today") {
|
|
272
|
+
parsed[key] = null;
|
|
273
|
+
missing.push(key);
|
|
274
|
+
continue;
|
|
275
|
+
}
|
|
254
276
|
if (parsed[key] !== null && parsed[key] !== undefined)
|
|
255
277
|
continue;
|
|
256
278
|
if (isTarget(schema, key)) {
|
|
@@ -318,6 +340,9 @@ async function deriveViaService(opts, service, schema, keys) {
|
|
|
318
340
|
prompt: outgoingPrompt(opts.prompt),
|
|
319
341
|
liveCall: context.liveCall,
|
|
320
342
|
recentResults: context.recentResults,
|
|
343
|
+
// What "tomorrow" is counted from. A service that does not know the
|
|
344
|
+
// field ignores it.
|
|
345
|
+
today: opts.today ?? localDay(),
|
|
321
346
|
}),
|
|
322
347
|
signal: opts.signal ?? timeoutSignal(),
|
|
323
348
|
});
|
package/dist/replay/types.d.ts
CHANGED
|
@@ -138,6 +138,13 @@ export type ParamsSchema = Record<string, {
|
|
|
138
138
|
sampleValue: unknown;
|
|
139
139
|
description: string;
|
|
140
140
|
kind?: ParameterKind;
|
|
141
|
+
/**
|
|
142
|
+
* Where the recorded value came from, when the service calculated the plan
|
|
143
|
+
* beside its request. Only `today` is read here: a value worked out from
|
|
144
|
+
* the day the request is made on ("tomorrow"), which the service works out
|
|
145
|
+
* again from the day this runner sends (replay/derive.ts, `localDay`).
|
|
146
|
+
*/
|
|
147
|
+
from?: string;
|
|
141
148
|
}>;
|
|
142
149
|
/** Whether this schema has anything the runner may not guess (R-PARAM-4). */
|
|
143
150
|
export declare function hasTargets(schema: ParamsSchema | null | undefined): boolean;
|
package/docs/calculatedReplay.md
CHANGED
|
@@ -194,7 +194,7 @@ Every gate that declines logs **one** `replay.decision` line carrying the reason
|
|
|
194
194
|
|
|
195
195
|
| # | Gate | Declines when | Outcome floor |
|
|
196
196
|
|---|---|---|---|
|
|
197
|
-
| 1 | **A match arrived** | `getMatch()` resolved `null`, or the budget expired (§10) | (no report — nothing matched) |
|
|
197
|
+
| 1 | **A match arrived** | `getMatch()` resolved `null`, or the budget expired (§10) | (no report — nothing matched; unless the create's own answer is a hit after all: `match_late`, below) |
|
|
198
198
|
| 2 | **Replay enabled** | `BIR_REPLAY === "0"`, or, with it unset, the project's stored switch is off (`bir replay off`, or a `bir install` without `--replay`). In a session of another host, also when the run's adapter asked for no plan with its prompt (`host_records_only`, §7.4) | `not_steered` |
|
|
199
199
|
| 3 | **Scenario ready** | `scenario == null`, `state !== "ready"`, or `steps.length === 0` | `not_steered` |
|
|
200
200
|
| 4 | ~~**Similarity**~~ | Retired: a match is the decision to use the scenario (below) | — |
|
|
@@ -203,9 +203,15 @@ Every gate that declines logs **one** `replay.decision` line carrying the reason
|
|
|
203
203
|
Past the table, a plan can still be refused before the model hears of it, each refusal with its own
|
|
204
204
|
`replay.decision` code. The match is not waited on at all while the service is cooling down after a
|
|
205
205
|
failure (`service_unhealthy`), or while prompts are holding off a service that answered matches
|
|
206
|
-
slowly (`match_slow`, the match gate, §10).
|
|
206
|
+
slowly (`match_slow`, the match gate, §10). Nor after the cool-down, while the failure is still on
|
|
207
|
+
record and no answer has cleared it: the recorder answers the prompt "no match" at once, and the
|
|
208
|
+
create goes on without it. A match or a plan that is not ready
|
|
207
209
|
inside the prompt's budget (§10) is dropped (`match_timeout`, `arm_timeout`), and one that lands
|
|
208
|
-
after the prompt was answered is retired unannounced (`match_late`, `arm_unannounced`).
|
|
210
|
+
after the prompt was answered is retired unannounced (`match_late`, `arm_unannounced`). The turn's
|
|
211
|
+
report carries that code as its decline, `match_late` included: nothing is armed or derived for a
|
|
212
|
+
late match, but the ticket of its hand-out is still reported, so the service knows why the turn was
|
|
213
|
+
not steered. That holds for a match the prompt never waited for, too: the turn follows the create's
|
|
214
|
+
own answer (`settledMatch`), not the "no match" the prompt was given. A `direct`
|
|
209
215
|
plan is declined when no running `bir-scenario` has said it serves `run_scenario`
|
|
210
216
|
(`no_delivery_tool`, §6.3). In plan mode (`permission_mode: "plan"`) nothing is announced at all.
|
|
211
217
|
In a session of another host (§7.4) a plan that armed is taken back, before any step of it runs,
|
|
@@ -915,7 +921,13 @@ minutes, while its steps keep starting. `bir replay` is not held to the ceiling.
|
|
|
915
921
|
`BIR_MATCH_SLOW_MS` (**2000**), or not at all, also makes prompts stop waiting on the match for 5
|
|
916
922
|
minutes, doubling up to 30 while the service stays slow; a fast match ends the hold
|
|
917
923
|
(`replay.match_gate`). A healthy service answers in 300–450 ms, so past 2 s waiting would add
|
|
918
|
-
that much to every prompt. The match is still asked meanwhile, so a hit still stops the recording.
|
|
924
|
+
that much to every prompt. The match is still asked meanwhile, so a hit still stops the recording.
|
|
925
|
+
**A failure on record:** a prompt never waits on a service whose last request failed, until an
|
|
926
|
+
answer clears that failure, and a prompt is never the request that finds out. After hours with
|
|
927
|
+
nothing sent (a machine asleep, a network that dropped) the failure is still on record, so a
|
|
928
|
+
session's start checks it in the background (`GET /health`) before the first prompt arrives. A match
|
|
929
|
+
that lands after a prompt went ahead without it is `match_late`, and its ticket is reported as the
|
|
930
|
+
turn's decline. Derivation budget expired →
|
|
919
931
|
settings take their recorded `sampleValue`s and targets stay `null`, never `{}`; a plan that needs a
|
|
920
932
|
missing target, or whose `paramsLogic` throws, hands over instead of running (in steer mode the
|
|
921
933
|
note replaces the call, in direct mode `run_scenario` answers with it). The derivation request
|
|
@@ -1356,8 +1368,9 @@ fail because of BaseInstRunner.**
|
|
|
1356
1368
|
|
|
1357
1369
|
| Failure | Behaviour |
|
|
1358
1370
|
|---|---|
|
|
1359
|
-
| `getMatch()` slower than the match budget | No plan. Ordinary turn. One `replay.decision` line (`match_timeout`); a match that lands later is `match_late` and never steers. A slow match also makes prompts stop waiting on it for a while (`replay.match_gate`, §10) |
|
|
1371
|
+
| `getMatch()` slower than the match budget | No plan. Ordinary turn. One `replay.decision` line (`match_timeout`); a match that lands later is `match_late` and never steers; while the turn is open it is reported to the service as that turn's decline. A slow match also makes prompts stop waiting on it for a while (`replay.match_gate`, §10) |
|
|
1360
1372
|
| The service is cooling down after a failure | Nothing waits on it: no match wait (`service_unhealthy`), no derivation wait, no intent probe. Ordinary turn |
|
|
1373
|
+
| A failure is still on record after the cool-down, with nothing sent since | No match wait and no intent probe: the prompt is answered "no match" at once and the create goes on. A hit that lands is `match_late`, and is reported as the turn's decline while the turn is open. A session's start checks the service in the background (`GET /health`), so an old failure does not decide its first prompt |
|
|
1361
1374
|
| No derivation key here (`BIR_DERIVE_API_KEY`; the background recorder ignores `ANTHROPIC_API_KEY` unless `BIR_USE_ANTHROPIC_API_KEY=1`) | The service derives instead (segmented.md R-PARAM-5). Signed out as well: settings take their recorded `sampleValue`s, a scenario with a target declines `no_derive_key`, `deriveCostUsd: 0` |
|
|
1362
1375
|
| The service declines, is slow, or cannot be reached | The same, with its reason carried into the `replay.derived` and `replay.decision` lines. Never a throw |
|
|
1363
1376
|
| Derivation call fails or times out | Settings take their samples and targets stay `null`, never `{}`; a plan that needs a missing target hands over instead of running. Plus a `replay.derive_failed` or `replay.derive_timeout` line |
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@basein/runner",
|
|
3
|
-
"version": "0.3.
|
|
3
|
+
"version": "0.3.12",
|
|
4
4
|
"description": "A recording MCP proxy: sits between any MCP client and its MCP servers, executes each call on the client's behalf, and records the run as a reusable BaseIn scenario.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "MIT",
|
package/production.md
CHANGED
|
@@ -909,7 +909,7 @@ Lines read `[bir] <ISO time> <event> key=value …`, with each field clipped at
|
|
|
909
909
|
|
|
910
910
|
| Topic | Lines and what they mean |
|
|
911
911
|
|---|---|
|
|
912
|
-
| Service health | `service.degraded` (first failure: reason, cool-down), `service.recovered`. `replay.decision code=service_unhealthy` (prompt did not wait: breaker open), `code=match_timeout` (prompt waited its budget), `code=match_late` (the answer landed after the prompt), `code=match_slow` (verbose: gate holding). `replay.match_gate state=holding\|open`. |
|
|
912
|
+
| Service health | `service.degraded` (first failure: reason, cool-down), `service.recovered`. `replay.decision code=service_unhealthy` (prompt did not wait: breaker open), `code=match_timeout` (prompt waited its budget), `code=match_late` (the answer landed after the prompt, including one the prompt never waited for because a failure was still on record), `code=match_slow` (verbose: gate holding). `replay.match_gate state=holding\|open`. |
|
|
913
913
|
| Recording backlog | `recorder.backlog_full` / `recorder.backlog_recovered` (oldest steps dropped). `recorder.run_incomplete` (a run lost steps). `recorder.run_abandoned` (`reason=service_down`: a create given up after a minute of cool-down). `recorder.flush_deferred`, `recorder.flush_incomplete`, `recorder.send_failed` (once per run), `recorder.matched_after_retry`, `run.lossy`. |
|
|
914
914
|
| Sign-in | `auth.ok`, `auth.settling` (plain-http address being checked for https), `auth.refresh_deferred`, `auth.refresh_refused`, `auth.recovered`. `recorder.enabled`, `recorder.signed_out`, `recorder.disabled`, `recorder.paused` / `recorder.resumed`, `recorder.account_changed`, `recorder.report_dropped`, `derive.api_key_ignored`. |
|
|
915
915
|
| Hooks | `control.listening` (port, `recorder=<state>`, `correlation=`). `control.correlation_allow` (auto-approval opted in). `control.hook_deadline`, `control.hook_unauthorized` (a token from another install), `control.hook_unknown`, `control.hook_oversize`, `control.hook_event_mismatch`, `control.route_failed`, `control.drain_abandoned`. |
|