@basein/runner 0.2.6 → 0.2.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/bir-hooks.js +5 -0
- package/dist/bin/bir.js +40 -1
- package/dist/bin/investigate.d.ts +195 -0
- package/dist/bin/investigate.js +576 -0
- package/dist/control/server.js +8 -0
- package/dist/record/recorder.d.ts +6 -0
- package/dist/record/remote-recorder.js +2 -0
- package/dist/replay/controller.d.ts +2 -0
- package/dist/replay/controller.js +2 -0
- package/dist/util/journal.d.ts +38 -0
- package/dist/util/journal.js +97 -0
- package/dist/util/log.js +4 -0
- package/docs/calculatedReplayGuide.md +32 -0
- package/package.json +1 -1
package/dist/bin/bir-hooks.js
CHANGED
|
@@ -71,6 +71,7 @@ import { readSidecar } from "../config/generate.js";
|
|
|
71
71
|
import { resolveServers } from "../config/resolve.js";
|
|
72
72
|
import { isWrapped } from "../config/generate.js";
|
|
73
73
|
import { logLine, errText } from "../util/log.js";
|
|
74
|
+
import { openJournal } from "../util/journal.js";
|
|
74
75
|
function parsePort(value) {
|
|
75
76
|
const n = Number(value);
|
|
76
77
|
return Number.isInteger(n) && n > 0 && n < 65536 ? n : DEFAULT_CONTROL_PORT;
|
|
@@ -207,6 +208,9 @@ async function main() {
|
|
|
207
208
|
}
|
|
208
209
|
const cwd = process.cwd();
|
|
209
210
|
const sidecar = readSidecar();
|
|
211
|
+
// Open before anything logs: the journal is what `bir investigate` reads
|
|
212
|
+
// later, and the startup lines are part of the story.
|
|
213
|
+
const journalFile = openJournal(cwd);
|
|
210
214
|
const auth = await buildRecorder();
|
|
211
215
|
const server = new ControlServer({
|
|
212
216
|
recorder: auth.recorder,
|
|
@@ -234,6 +238,7 @@ async function main() {
|
|
|
234
238
|
url: address.url,
|
|
235
239
|
cwd,
|
|
236
240
|
discovery: discoveryFile,
|
|
241
|
+
journal: journalFile,
|
|
237
242
|
machine: hostname(),
|
|
238
243
|
wrapped: wrappedServersFor(cwd).join(",") || "(none)",
|
|
239
244
|
});
|
package/dist/bin/bir.js
CHANGED
|
@@ -31,6 +31,9 @@ import { readGenericServers, setGenericServerEntry } from "../config/adapters/ge
|
|
|
31
31
|
import { AUTH_URL_HINT, DeviceFlowAborted, DeviceFlowUnsupported, authenticate, deviceLogin, describeAuthService, legacyPasswordLogin, tokenLogin, logout, normalizeAuthUrl, resolveAuthUrl, } from "../auth/client.js";
|
|
32
32
|
import { DEFAULT_CONTROL_PORT } from "../control/server.js";
|
|
33
33
|
import { errText } from "../util/log.js";
|
|
34
|
+
import { journalPath, readJournal } from "../util/journal.js";
|
|
35
|
+
import { investigateCommand } from "./investigate.js";
|
|
36
|
+
import { loadCredentials } from "../auth/client.js";
|
|
34
37
|
import { packageVersion } from "../util/version.js";
|
|
35
38
|
const VERSION = packageVersion();
|
|
36
39
|
const out = (line = "") => {
|
|
@@ -54,6 +57,12 @@ Commands:
|
|
|
54
57
|
scenario replay <scnId> --prompt "…" [--dry]
|
|
55
58
|
replay --scenario <scnId> --prompt "…" [--dry]
|
|
56
59
|
|
|
60
|
+
investigate [<id>] why the newest turn here (or run_/scn_/sexec_ <id>) did what
|
|
61
|
+
it did, what it cost, what to fix — journal + service
|
|
62
|
+
investigate list [--limit <n>] recent turns in this directory and their verdicts
|
|
63
|
+
investigate executions [--limit <n>] [--user <email|id>]
|
|
64
|
+
the ledger, newest first (--user: admins only)
|
|
65
|
+
|
|
57
66
|
Options:
|
|
58
67
|
--config <path> an explicit { "mcpServers": … } file, for clients with no adapter
|
|
59
68
|
--server <name> restrict install/uninstall to one server (repeatable)
|
|
@@ -68,7 +77,9 @@ Options:
|
|
|
68
77
|
--dry replay against recorded outputs only; run no real tools
|
|
69
78
|
--no-browser login: print the link and code, open nothing (SSH, headless)
|
|
70
79
|
--token <value> login: redeem a one-time setup token from the console (no browser)
|
|
71
|
-
--password login: use the old email/password prompt (deprecated)
|
|
80
|
+
--password login: use the old email/password prompt (deprecated)
|
|
81
|
+
--user <ref> investigate executions: another account, by email or id (admin)
|
|
82
|
+
--limit <n> investigate: how many turns / executions to show`);
|
|
72
83
|
process.exit(code);
|
|
73
84
|
}
|
|
74
85
|
function parseArgs(argv) {
|
|
@@ -146,6 +157,18 @@ function parseArgs(argv) {
|
|
|
146
157
|
case "--json":
|
|
147
158
|
args.json = true;
|
|
148
159
|
break;
|
|
160
|
+
case "--user":
|
|
161
|
+
args.user = argv[++i];
|
|
162
|
+
if (!args.user)
|
|
163
|
+
usage(2);
|
|
164
|
+
break;
|
|
165
|
+
case "--limit": {
|
|
166
|
+
const n = Number(argv[++i]);
|
|
167
|
+
if (!Number.isInteger(n) || n <= 0)
|
|
168
|
+
usage(2);
|
|
169
|
+
args.limit = n;
|
|
170
|
+
break;
|
|
171
|
+
}
|
|
149
172
|
default:
|
|
150
173
|
if (arg.startsWith("-"))
|
|
151
174
|
usage(2);
|
|
@@ -933,6 +956,22 @@ async function main() {
|
|
|
933
956
|
return scenarioCommand(args);
|
|
934
957
|
case "replay":
|
|
935
958
|
return replayCommand(args);
|
|
959
|
+
case "investigate":
|
|
960
|
+
return investigateCommand({ positionals: args.positionals, json: args.json, user: args.user, limit: args.limit }, {
|
|
961
|
+
cwd: process.cwd(),
|
|
962
|
+
out,
|
|
963
|
+
journal: (cwd) => readJournal(journalPath(cwd)),
|
|
964
|
+
journalPath,
|
|
965
|
+
viewerEmail: loadCredentials()?.user?.email,
|
|
966
|
+
service: async (path) => {
|
|
967
|
+
try {
|
|
968
|
+
return await service("GET", path);
|
|
969
|
+
}
|
|
970
|
+
catch (err) {
|
|
971
|
+
return { error: errText(err) };
|
|
972
|
+
}
|
|
973
|
+
},
|
|
974
|
+
});
|
|
936
975
|
case "login": {
|
|
937
976
|
// Settle the URL the way every other command does, then refuse to go on
|
|
938
977
|
// when what is there is not the service. "Signed in" must mean the service
|
|
@@ -0,0 +1,195 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `bir investigate` — why did this turn do what it did, and what did it cost?
|
|
3
|
+
* (docs/calculatedReplayGuide.md §9.1)
|
|
4
|
+
*
|
|
5
|
+
* Two sources, merged:
|
|
6
|
+
*
|
|
7
|
+
* 1. The **journal** this directory's `bir-hooks` keeps (util/journal.ts):
|
|
8
|
+
* the run boundaries, the match, every gate decision, the plan's mode and
|
|
9
|
+
* coverage, the replay outcome and the execution report. This is the only
|
|
10
|
+
* record of a *matched* turn, because the service deliberately keeps no
|
|
11
|
+
* run for one.
|
|
12
|
+
* 2. The **service**'s `GET /investigate/:id`: the recording, its scenario,
|
|
13
|
+
* the baseline, the execution ledger and the service's own findings.
|
|
14
|
+
* Owner-only; an admin may look at anyone's.
|
|
15
|
+
*
|
|
16
|
+
* Each side diagnoses what only it can see. The journal knows which gate
|
|
17
|
+
* declined and what mode armed; the service knows the baseline, the ledger and
|
|
18
|
+
* whether a run ever became a recording. Both lists are printed, worst first.
|
|
19
|
+
*/
|
|
20
|
+
import type { JournalEntry } from "../util/journal.js";
|
|
21
|
+
export interface Turn {
|
|
22
|
+
runId: string;
|
|
23
|
+
sessionId?: string;
|
|
24
|
+
startedAt: string;
|
|
25
|
+
finishedAt?: string;
|
|
26
|
+
prompt?: string;
|
|
27
|
+
matched?: {
|
|
28
|
+
runId: string;
|
|
29
|
+
scenarioId?: string;
|
|
30
|
+
similarity?: number;
|
|
31
|
+
};
|
|
32
|
+
decision?: {
|
|
33
|
+
verdict: string;
|
|
34
|
+
code?: string;
|
|
35
|
+
why?: string;
|
|
36
|
+
threshold?: number;
|
|
37
|
+
};
|
|
38
|
+
plan?: {
|
|
39
|
+
mode: string;
|
|
40
|
+
steps?: number;
|
|
41
|
+
coverage?: string[];
|
|
42
|
+
tools?: string[];
|
|
43
|
+
scenarioId?: string;
|
|
44
|
+
by?: string;
|
|
45
|
+
};
|
|
46
|
+
derived?: {
|
|
47
|
+
params?: number;
|
|
48
|
+
costUsd?: number;
|
|
49
|
+
source?: string;
|
|
50
|
+
};
|
|
51
|
+
done?: {
|
|
52
|
+
outcome: string;
|
|
53
|
+
steps?: string;
|
|
54
|
+
ms?: number;
|
|
55
|
+
};
|
|
56
|
+
reported?: {
|
|
57
|
+
outcome: string;
|
|
58
|
+
derive: number;
|
|
59
|
+
session: number;
|
|
60
|
+
fallback: number;
|
|
61
|
+
savedUsd?: number;
|
|
62
|
+
measured?: boolean;
|
|
63
|
+
ticket?: string;
|
|
64
|
+
scenarioId?: string;
|
|
65
|
+
declined?: string;
|
|
66
|
+
};
|
|
67
|
+
finish?: {
|
|
68
|
+
steps?: number;
|
|
69
|
+
durationMs?: number;
|
|
70
|
+
costUsd?: number;
|
|
71
|
+
measured?: boolean;
|
|
72
|
+
recorded?: boolean;
|
|
73
|
+
};
|
|
74
|
+
/** `replay.*_failed`, `replay.arm_failed`, `replay.diverge`, `replay.handover` … */
|
|
75
|
+
incidents: JournalEntry[];
|
|
76
|
+
events: JournalEntry[];
|
|
77
|
+
}
|
|
78
|
+
/**
|
|
79
|
+
* Group journal entries into turns. A turn opens at `run.start` and closes at
|
|
80
|
+
* `run.finish`; an entry joins the turn whose local run id it names, else the
|
|
81
|
+
* turn whose *matched* run id it names (the replay lines log the service's run),
|
|
82
|
+
* else the latest turn — which is where an older build's `execution.reported`,
|
|
83
|
+
* arriving after `run.finish` with no run id, belongs.
|
|
84
|
+
*/
|
|
85
|
+
export declare function turnsFromJournal(entries: readonly JournalEntry[]): Turn[];
|
|
86
|
+
/** The turn an id names: its own run, the run it matched, its scenario or its ticket. */
|
|
87
|
+
export declare function findTurn(turns: readonly Turn[], id: string): Turn | undefined;
|
|
88
|
+
export interface Finding {
|
|
89
|
+
code: string;
|
|
90
|
+
severity: "problem" | "warn" | "info";
|
|
91
|
+
title: string;
|
|
92
|
+
cause: string;
|
|
93
|
+
fix: string;
|
|
94
|
+
evidence?: Record<string, unknown>;
|
|
95
|
+
}
|
|
96
|
+
/** The runner's own gate vocabulary, with the cause and the fix (guide §9). */
|
|
97
|
+
export declare const DECLINES: Record<string, {
|
|
98
|
+
cause: string;
|
|
99
|
+
fix: string;
|
|
100
|
+
}>;
|
|
101
|
+
export declare const LOW_SAVING_PCT = 0.3;
|
|
102
|
+
export declare function localFindings(turn: Turn): Finding[];
|
|
103
|
+
/** `saved ÷ (saved + cost)`: the baseline is recoverable from the report alone. */
|
|
104
|
+
export declare function savedPct(turn: Turn): number | null;
|
|
105
|
+
export type ServiceReply = {
|
|
106
|
+
status: number;
|
|
107
|
+
body: unknown;
|
|
108
|
+
} | {
|
|
109
|
+
error: string;
|
|
110
|
+
};
|
|
111
|
+
export interface InvestigateDeps {
|
|
112
|
+
cwd: string;
|
|
113
|
+
out: (line?: string) => void;
|
|
114
|
+
service: (path: string) => Promise<ServiceReply>;
|
|
115
|
+
journal: (cwd: string) => JournalEntry[];
|
|
116
|
+
journalPath: (cwd: string) => string;
|
|
117
|
+
/** The signed-in account, so "owner you" can be told from "owner <email>". */
|
|
118
|
+
viewerEmail?: string;
|
|
119
|
+
}
|
|
120
|
+
export interface InvestigateArgs {
|
|
121
|
+
positionals: string[];
|
|
122
|
+
json: boolean;
|
|
123
|
+
user?: string;
|
|
124
|
+
limit?: number;
|
|
125
|
+
}
|
|
126
|
+
interface ServiceInvestigation {
|
|
127
|
+
subject?: {
|
|
128
|
+
kind: string;
|
|
129
|
+
id: string;
|
|
130
|
+
};
|
|
131
|
+
owner?: {
|
|
132
|
+
id: string;
|
|
133
|
+
email: string | null;
|
|
134
|
+
};
|
|
135
|
+
run?: {
|
|
136
|
+
id: string;
|
|
137
|
+
input: string;
|
|
138
|
+
title: string | null;
|
|
139
|
+
finishedAt: string | null;
|
|
140
|
+
actionCount: number;
|
|
141
|
+
tools: string[];
|
|
142
|
+
originalCostUsd: number | null;
|
|
143
|
+
iterations: number;
|
|
144
|
+
isRecording: boolean;
|
|
145
|
+
embedded: boolean;
|
|
146
|
+
duplicateOf: string | null;
|
|
147
|
+
} | null;
|
|
148
|
+
scenario?: {
|
|
149
|
+
id: string;
|
|
150
|
+
state: string;
|
|
151
|
+
steps: Array<{
|
|
152
|
+
stepIndex: number;
|
|
153
|
+
toolName: string | null;
|
|
154
|
+
failureCount: number;
|
|
155
|
+
}>;
|
|
156
|
+
baseline: {
|
|
157
|
+
costUsd: number | null;
|
|
158
|
+
samples: number;
|
|
159
|
+
method: string | null;
|
|
160
|
+
};
|
|
161
|
+
lastDecline: {
|
|
162
|
+
reason: string;
|
|
163
|
+
at: string | null;
|
|
164
|
+
} | null;
|
|
165
|
+
hitCount: number | null;
|
|
166
|
+
disabledAt: string | null;
|
|
167
|
+
error: string | null;
|
|
168
|
+
} | null;
|
|
169
|
+
executions?: Array<{
|
|
170
|
+
id: string;
|
|
171
|
+
createdAt: string;
|
|
172
|
+
outcome: string;
|
|
173
|
+
costUsd: number;
|
|
174
|
+
deriveCostUsd: number | null;
|
|
175
|
+
sessionCostUsd: number | null;
|
|
176
|
+
fallbackCostUsd: number | null;
|
|
177
|
+
baselineCostUsd: number | null;
|
|
178
|
+
savedUsd: number | null;
|
|
179
|
+
savedPct: number | null;
|
|
180
|
+
measured: boolean;
|
|
181
|
+
durationMs: number | null;
|
|
182
|
+
steps: Array<{
|
|
183
|
+
stepIndex: number;
|
|
184
|
+
toolName?: string;
|
|
185
|
+
status: string;
|
|
186
|
+
error?: string;
|
|
187
|
+
}> | null;
|
|
188
|
+
}>;
|
|
189
|
+
findings?: Finding[];
|
|
190
|
+
}
|
|
191
|
+
export declare function renderTurn(out: (l?: string) => void, turn: Turn): void;
|
|
192
|
+
export declare function renderService(out: (l?: string) => void, s: ServiceInvestigation, viewerIsOwner: boolean): void;
|
|
193
|
+
export declare function investigateCommand(args: InvestigateArgs, deps: InvestigateDeps): Promise<number>;
|
|
194
|
+
export {};
|
|
195
|
+
//# sourceMappingURL=investigate.d.ts.map
|
|
@@ -0,0 +1,576 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `bir investigate` — why did this turn do what it did, and what did it cost?
|
|
3
|
+
* (docs/calculatedReplayGuide.md §9.1)
|
|
4
|
+
*
|
|
5
|
+
* Two sources, merged:
|
|
6
|
+
*
|
|
7
|
+
* 1. The **journal** this directory's `bir-hooks` keeps (util/journal.ts):
|
|
8
|
+
* the run boundaries, the match, every gate decision, the plan's mode and
|
|
9
|
+
* coverage, the replay outcome and the execution report. This is the only
|
|
10
|
+
* record of a *matched* turn, because the service deliberately keeps no
|
|
11
|
+
* run for one.
|
|
12
|
+
* 2. The **service**'s `GET /investigate/:id`: the recording, its scenario,
|
|
13
|
+
* the baseline, the execution ledger and the service's own findings.
|
|
14
|
+
* Owner-only; an admin may look at anyone's.
|
|
15
|
+
*
|
|
16
|
+
* Each side diagnoses what only it can see. The journal knows which gate
|
|
17
|
+
* declined and what mode armed; the service knows the baseline, the ledger and
|
|
18
|
+
* whether a run ever became a recording. Both lists are printed, worst first.
|
|
19
|
+
*/
|
|
20
|
+
const num = (v) => {
|
|
21
|
+
if (v === undefined || v === null || v === "")
|
|
22
|
+
return undefined;
|
|
23
|
+
const n = Number(v);
|
|
24
|
+
return Number.isFinite(n) ? n : undefined;
|
|
25
|
+
};
|
|
26
|
+
const str = (v) => (v === undefined || v === null ? undefined : String(v));
|
|
27
|
+
const bool = (v) => v === true || v === "true" ? true : v === false || v === "false" ? false : undefined;
|
|
28
|
+
const list = (v) => {
|
|
29
|
+
const s = str(v);
|
|
30
|
+
return s ? s.split(",").filter(Boolean) : undefined;
|
|
31
|
+
};
|
|
32
|
+
const INCIDENTS = new Set([
|
|
33
|
+
"replay.arm_failed",
|
|
34
|
+
"replay.derive_failed",
|
|
35
|
+
"replay.compose_failed",
|
|
36
|
+
"replay.flatten_failed",
|
|
37
|
+
"replay.thread_failed",
|
|
38
|
+
"replay.step_failed",
|
|
39
|
+
"replay.failed",
|
|
40
|
+
"replay.diverge",
|
|
41
|
+
"replay.handover",
|
|
42
|
+
"replay.thread_fallback",
|
|
43
|
+
"run.lossy",
|
|
44
|
+
"run.unmeasured",
|
|
45
|
+
"run.prompt_late",
|
|
46
|
+
]);
|
|
47
|
+
/**
|
|
48
|
+
* Group journal entries into turns. A turn opens at `run.start` and closes at
|
|
49
|
+
* `run.finish`; an entry joins the turn whose local run id it names, else the
|
|
50
|
+
* turn whose *matched* run id it names (the replay lines log the service's run),
|
|
51
|
+
* else the latest turn — which is where an older build's `execution.reported`,
|
|
52
|
+
* arriving after `run.finish` with no run id, belongs.
|
|
53
|
+
*/
|
|
54
|
+
export function turnsFromJournal(entries) {
|
|
55
|
+
const turns = [];
|
|
56
|
+
const owner = (e) => {
|
|
57
|
+
const run = str(e.run);
|
|
58
|
+
// The newest turn that is, or matched, this run. Newest wins: once a
|
|
59
|
+
// recording has been matched, `plan.armed run=<its id>` is about the
|
|
60
|
+
// matching turn, not about the recording's own (long finished) turn.
|
|
61
|
+
if (run) {
|
|
62
|
+
for (let i = turns.length - 1; i >= 0; i -= 1) {
|
|
63
|
+
if (turns[i].runId === run || turns[i].matched?.runId === run)
|
|
64
|
+
return turns[i];
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
if (e.event === "execution.reported") {
|
|
68
|
+
for (let i = turns.length - 1; i >= 0; i -= 1) {
|
|
69
|
+
if (turns[i].plan && !turns[i].reported)
|
|
70
|
+
return turns[i];
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
return turns[turns.length - 1];
|
|
74
|
+
};
|
|
75
|
+
for (const e of entries) {
|
|
76
|
+
if (e.event === "run.start") {
|
|
77
|
+
const runId = str(e.run) ?? `unknown-${turns.length}`;
|
|
78
|
+
const turn = { runId, sessionId: str(e.sess), startedAt: e.at, incidents: [], events: [e] };
|
|
79
|
+
turns.push(turn);
|
|
80
|
+
continue;
|
|
81
|
+
}
|
|
82
|
+
const turn = owner(e);
|
|
83
|
+
if (!turn)
|
|
84
|
+
continue;
|
|
85
|
+
turn.events.push(e);
|
|
86
|
+
switch (e.event) {
|
|
87
|
+
case "run.prompt":
|
|
88
|
+
turn.prompt = str(e.prompt);
|
|
89
|
+
break;
|
|
90
|
+
case "run.matched":
|
|
91
|
+
turn.matched = { runId: str(e.matchedRun) ?? "", scenarioId: str(e.scenario), similarity: num(e.similarity) };
|
|
92
|
+
break;
|
|
93
|
+
case "replay.decision":
|
|
94
|
+
turn.decision = {
|
|
95
|
+
verdict: str(e.verdict) ?? "",
|
|
96
|
+
code: str(e.code),
|
|
97
|
+
why: str(e.why),
|
|
98
|
+
threshold: num(e.threshold),
|
|
99
|
+
};
|
|
100
|
+
break;
|
|
101
|
+
case "plan.armed":
|
|
102
|
+
turn.plan = {
|
|
103
|
+
mode: str(e.mode) ?? "",
|
|
104
|
+
steps: num(e.steps),
|
|
105
|
+
coverage: list(e.coverage),
|
|
106
|
+
tools: list(e.tools),
|
|
107
|
+
scenarioId: str(e.scenario),
|
|
108
|
+
by: str(e.by),
|
|
109
|
+
};
|
|
110
|
+
break;
|
|
111
|
+
case "replay.derived":
|
|
112
|
+
turn.derived = { params: num(e.params), costUsd: num(e.costUsd), source: str(e.source) };
|
|
113
|
+
break;
|
|
114
|
+
case "replay.done":
|
|
115
|
+
turn.done = { outcome: str(e.outcome) ?? "", steps: str(e.steps), ms: num(e.ms) };
|
|
116
|
+
break;
|
|
117
|
+
case "execution.reported":
|
|
118
|
+
turn.reported = {
|
|
119
|
+
outcome: str(e.outcome) ?? "",
|
|
120
|
+
derive: num(e.derive) ?? 0,
|
|
121
|
+
session: num(e.session) ?? 0,
|
|
122
|
+
fallback: num(e.fallback) ?? 0,
|
|
123
|
+
savedUsd: num(e.savedUsd),
|
|
124
|
+
measured: bool(e.measured),
|
|
125
|
+
ticket: str(e.ticket),
|
|
126
|
+
scenarioId: str(e.scenario),
|
|
127
|
+
declined: str(e.declined),
|
|
128
|
+
};
|
|
129
|
+
break;
|
|
130
|
+
case "run.finish":
|
|
131
|
+
turn.finishedAt = e.at;
|
|
132
|
+
turn.finish = {
|
|
133
|
+
steps: num(e.steps),
|
|
134
|
+
durationMs: num(e.durationMs),
|
|
135
|
+
costUsd: num(e.costUsd),
|
|
136
|
+
measured: bool(e.measured),
|
|
137
|
+
recorded: bool(e.recorded),
|
|
138
|
+
};
|
|
139
|
+
break;
|
|
140
|
+
default:
|
|
141
|
+
if (INCIDENTS.has(e.event))
|
|
142
|
+
turn.incidents.push(e);
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
return turns;
|
|
146
|
+
}
|
|
147
|
+
/** The turn an id names: its own run, the run it matched, its scenario or its ticket. */
|
|
148
|
+
export function findTurn(turns, id) {
|
|
149
|
+
for (let i = turns.length - 1; i >= 0; i -= 1) {
|
|
150
|
+
const t = turns[i];
|
|
151
|
+
if (t.runId === id ||
|
|
152
|
+
t.matched?.runId === id ||
|
|
153
|
+
t.matched?.scenarioId === id ||
|
|
154
|
+
t.plan?.scenarioId === id ||
|
|
155
|
+
t.reported?.scenarioId === id ||
|
|
156
|
+
t.reported?.ticket === id) {
|
|
157
|
+
return t;
|
|
158
|
+
}
|
|
159
|
+
}
|
|
160
|
+
return undefined;
|
|
161
|
+
}
|
|
162
|
+
/** The runner's own gate vocabulary, with the cause and the fix (guide §9). */
|
|
163
|
+
export const DECLINES = {
|
|
164
|
+
replay_disabled: {
|
|
165
|
+
cause: "calculated replay was off in this bir-hooks (BIR_REPLAY=0), or a sub-task was handed out with BIR_SEGMENT_ARM off.",
|
|
166
|
+
fix: "start bir-hooks with BIR_REPLAY=1 (the default); for sub-tasks set BIR_SEGMENT_ARM=1.",
|
|
167
|
+
},
|
|
168
|
+
not_ready: {
|
|
169
|
+
cause: "the matched recording had no ready scenario at that moment.",
|
|
170
|
+
fix: "wait for the calculation, or start one with `bir scenario calc <runId>`, then run the prompt again.",
|
|
171
|
+
},
|
|
172
|
+
similarity: {
|
|
173
|
+
cause: "the prompt was a hit on the recording but below the steering threshold (BIR_REPLAY_MIN_SIMILARITY, default 0.92).",
|
|
174
|
+
fix: "phrase the prompt closer to the recorded one, or lower BIR_REPLAY_MIN_SIMILARITY.",
|
|
175
|
+
},
|
|
176
|
+
coverage: {
|
|
177
|
+
cause: "no step of the scenario could run anywhere: its tools are neither wrapped MCP servers registered in this session nor built-ins.",
|
|
178
|
+
fix: "wrap the servers the scenario uses (`bir install --server <name>`) and start a fresh session; `bir doctor` lists what is registered.",
|
|
179
|
+
},
|
|
180
|
+
known_bad_first_step: {
|
|
181
|
+
cause: "the chain's first step is parked after repeated failures.",
|
|
182
|
+
fix: "recalculate the scenario (`bir scenario calc <runId> --force`).",
|
|
183
|
+
},
|
|
184
|
+
unusable_first_step: {
|
|
185
|
+
cause: "the first step calls a sub-task that is gone, switched off or stale.",
|
|
186
|
+
fix: "enable or recalculate that sub-task, or recalculate this scenario.",
|
|
187
|
+
},
|
|
188
|
+
missing_target: {
|
|
189
|
+
cause: "the prompt names a target the runner could not find where the recording had it.",
|
|
190
|
+
fix: "run from the recording's directory, or name the target as the recording did.",
|
|
191
|
+
},
|
|
192
|
+
no_derive_key: {
|
|
193
|
+
cause: "the scenario has parameters and the runner had no way to derive them: no service session and no ANTHROPIC_API_KEY.",
|
|
194
|
+
fix: "set BIR_AUTH_URL and `bir login` in the terminal that starts bir-hooks, or export ANTHROPIC_API_KEY there.",
|
|
195
|
+
},
|
|
196
|
+
flatten_failed: {
|
|
197
|
+
cause: "the chain could not be flattened — a sub-task it calls no longer resolves.",
|
|
198
|
+
fix: "recalculate the scenario.",
|
|
199
|
+
},
|
|
200
|
+
};
|
|
201
|
+
export const LOW_SAVING_PCT = 0.3;
|
|
202
|
+
const usd = (v) => `$${v.toFixed(4)}`;
|
|
203
|
+
const pct = (v) => `${(v * 100).toFixed(1)}%`;
|
|
204
|
+
export function localFindings(turn) {
|
|
205
|
+
const out = [];
|
|
206
|
+
if (!turn.matched) {
|
|
207
|
+
out.push({
|
|
208
|
+
code: "no_match",
|
|
209
|
+
severity: "info",
|
|
210
|
+
title: "This turn matched no recording, so the agent did the task itself.",
|
|
211
|
+
cause: "The service found no recording whose prompt is similar enough (its SIMILARITY_THRESHOLD). A run that is below the recording threshold is never embedded and can never be matched.",
|
|
212
|
+
fix: "If a recording of this task exists, run the prompt closer to its wording. The service section below says whether this turn became a recording.",
|
|
213
|
+
});
|
|
214
|
+
}
|
|
215
|
+
else if (turn.decision?.verdict === "no-steer") {
|
|
216
|
+
const code = turn.decision.code ?? "";
|
|
217
|
+
const known = DECLINES[code];
|
|
218
|
+
out.push({
|
|
219
|
+
code: `declined_${code || "unknown"}`,
|
|
220
|
+
severity: code === "not_ready" && !turn.matched.scenarioId ? "info" : "problem",
|
|
221
|
+
title: `Matched ${turn.matched.runId} but declined to steer: ${turn.decision.why ?? code}.`,
|
|
222
|
+
cause: known?.cause ?? "the runner's log line carries the reason text.",
|
|
223
|
+
fix: known?.fix ??
|
|
224
|
+
(turn.matched.scenarioId
|
|
225
|
+
? "read the `replay.decision` line above."
|
|
226
|
+
: "the matched recording has no scenario yet — calculate it, or let the service pick it up after enough hits."),
|
|
227
|
+
evidence: { code, similarity: turn.matched.similarity, threshold: turn.decision.threshold },
|
|
228
|
+
});
|
|
229
|
+
}
|
|
230
|
+
else if (!turn.plan) {
|
|
231
|
+
out.push({
|
|
232
|
+
code: "not_armed",
|
|
233
|
+
severity: "warn",
|
|
234
|
+
title: `Matched ${turn.matched.runId} but no plan was armed.`,
|
|
235
|
+
cause: "The match arrived but arming failed or timed out before the prompt hook answered (`replay.arm_failed`, or the match budget).",
|
|
236
|
+
fix: "Check the incidents below and `bir doctor`; the service section says whether the scenario is ready.",
|
|
237
|
+
});
|
|
238
|
+
}
|
|
239
|
+
if (turn.plan) {
|
|
240
|
+
const coverage = turn.plan.coverage ?? [];
|
|
241
|
+
const tools = turn.plan.tools ?? [];
|
|
242
|
+
const live = coverage
|
|
243
|
+
.map((c, i) => (c === "live" ? (tools[i] ?? `step ${i}`) : undefined))
|
|
244
|
+
.filter((t) => Boolean(t));
|
|
245
|
+
if (turn.plan.mode === "steer" && live.length > 0) {
|
|
246
|
+
const lowSaving = savedPct(turn) != null && savedPct(turn) < LOW_SAVING_PCT;
|
|
247
|
+
out.push({
|
|
248
|
+
code: "steer_mode",
|
|
249
|
+
severity: lowSaving ? "warn" : "info",
|
|
250
|
+
title: `The plan armed in steer mode: ${live.length} of ${coverage.length} steps are reachable only inside the session (${[...new Set(live)].join(", ")}).`,
|
|
251
|
+
cause: "In steer mode the runner pins each call's inputs and auto-approves it, but the model still emits every tool call, reads every result and writes the answer — so the whole live turn is still paid for. Direct mode, where the model makes no tool call, needs every step to be a wrapped MCP server tool.",
|
|
252
|
+
fix: "Put the data those tools reach behind an MCP server, wrap it (`bir install --server <name>`), and record the prompt again so its steps are MCP calls.",
|
|
253
|
+
evidence: { coverage, tools },
|
|
254
|
+
});
|
|
255
|
+
}
|
|
256
|
+
if (turn.plan.mode === "direct") {
|
|
257
|
+
out.push({
|
|
258
|
+
code: "direct_mode",
|
|
259
|
+
severity: "info",
|
|
260
|
+
title: `The plan armed in direct mode: ${turn.plan.steps ?? coverage.length} steps ran on the proxies, no model in the loop.`,
|
|
261
|
+
cause: "Every step is a wrapped MCP tool this session has a proxy for.",
|
|
262
|
+
fix: "",
|
|
263
|
+
});
|
|
264
|
+
}
|
|
265
|
+
}
|
|
266
|
+
for (const i of turn.incidents) {
|
|
267
|
+
const fields = Object.fromEntries(Object.entries(i).filter(([k]) => k !== "at" && k !== "event"));
|
|
268
|
+
out.push({
|
|
269
|
+
code: i.event.replace(/\./g, "_"),
|
|
270
|
+
severity: i.event.endsWith("_failed") || i.event === "replay.failed" ? "problem" : "warn",
|
|
271
|
+
title: `${i.event}${i.error ? `: ${String(i.error)}` : ""}`,
|
|
272
|
+
cause: Object.entries(fields)
|
|
273
|
+
.map(([k, v]) => `${k}=${String(v)}`)
|
|
274
|
+
.join(" "),
|
|
275
|
+
fix: i.event === "run.unmeasured"
|
|
276
|
+
? "the turn's cost could not be read from the transcript, so its report is unmeasured; run bir-hooks where the Claude Code transcript is readable."
|
|
277
|
+
: i.event === "replay.diverge" || i.event === "replay.handover"
|
|
278
|
+
? "the model left the script or the runner handed the turn back; the service section shows what the fallback cost."
|
|
279
|
+
: "see the guide's troubleshooting table for this line.",
|
|
280
|
+
});
|
|
281
|
+
}
|
|
282
|
+
const r = turn.reported;
|
|
283
|
+
if (r) {
|
|
284
|
+
if (r.measured === false) {
|
|
285
|
+
out.push({
|
|
286
|
+
code: "unmeasured",
|
|
287
|
+
severity: "warn",
|
|
288
|
+
title: "The execution report is unmeasured: the session's tokens could not be read.",
|
|
289
|
+
cause: "Without the transcript delta the report carries the derivation cost alone, and `baseline − cost` overstates the saving.",
|
|
290
|
+
fix: "Run bir-hooks on the machine and user that own the Claude Code transcript; `run.unmeasured` names what was missing.",
|
|
291
|
+
});
|
|
292
|
+
}
|
|
293
|
+
const p = savedPct(turn);
|
|
294
|
+
const cost = r.derive + r.session + r.fallback;
|
|
295
|
+
if (r.savedUsd != null && r.savedUsd < 0) {
|
|
296
|
+
out.push({
|
|
297
|
+
code: "saving_negative",
|
|
298
|
+
severity: "problem",
|
|
299
|
+
title: `This replay cost more than the agent: ${usd(cost)} against a baseline of ${usd(r.savedUsd + cost)}.`,
|
|
300
|
+
cause: `derive ${usd(r.derive)}, session ${usd(r.session)}, fallback ${usd(r.fallback)}.`,
|
|
301
|
+
fix: "The service findings below name the dominant cost and the fix.",
|
|
302
|
+
evidence: { ...r, costUsd: cost },
|
|
303
|
+
});
|
|
304
|
+
}
|
|
305
|
+
else if (p != null && p < LOW_SAVING_PCT && r.outcome === "steered_full") {
|
|
306
|
+
out.push({
|
|
307
|
+
code: "saving_low",
|
|
308
|
+
severity: "warn",
|
|
309
|
+
title: `Saved ${pct(p)}: ${usd(r.savedUsd ?? 0)} of a ${usd((r.savedUsd ?? 0) + cost)} baseline.`,
|
|
310
|
+
cause: `The live session cost ${usd(r.session)} of that baseline${turn.plan?.mode === "steer" ? " — steer mode pays for the whole turn." : "."}`,
|
|
311
|
+
fix: turn.plan?.mode === "steer"
|
|
312
|
+
? "Direct mode is the fix: wrap the servers so every step is an MCP call (see steer_mode)."
|
|
313
|
+
: "Compare the baseline with a few unsteered runs (BIR_REPLAY=0); a single cheap source run makes any replay look poor.",
|
|
314
|
+
evidence: { ...r, costUsd: cost, savedPct: p },
|
|
315
|
+
});
|
|
316
|
+
}
|
|
317
|
+
}
|
|
318
|
+
else if (turn.plan && turn.finishedAt) {
|
|
319
|
+
out.push({
|
|
320
|
+
code: "not_reported",
|
|
321
|
+
severity: "warn",
|
|
322
|
+
title: "A plan armed but no execution report was sent.",
|
|
323
|
+
cause: "The turn ended without `execution.reported`: the report failed to send, or the ticket was missing (an older service).",
|
|
324
|
+
fix: "Look for `recorder.send_failed` in the bir-hooks log; the saving of this turn is not on the ledger.",
|
|
325
|
+
});
|
|
326
|
+
}
|
|
327
|
+
const rank = { problem: 0, warn: 1, info: 2 };
|
|
328
|
+
return out
|
|
329
|
+
.map((f, i) => ({ f, i }))
|
|
330
|
+
.sort((a, b) => rank[a.f.severity] - rank[b.f.severity] || a.i - b.i)
|
|
331
|
+
.map(({ f }) => f);
|
|
332
|
+
}
|
|
333
|
+
/** `saved ÷ (saved + cost)`: the baseline is recoverable from the report alone. */
|
|
334
|
+
export function savedPct(turn) {
|
|
335
|
+
const r = turn.reported;
|
|
336
|
+
if (!r || r.savedUsd == null)
|
|
337
|
+
return null;
|
|
338
|
+
const baseline = r.savedUsd + r.derive + r.session + r.fallback;
|
|
339
|
+
return baseline > 0 ? r.savedUsd / baseline : null;
|
|
340
|
+
}
|
|
341
|
+
const short = (id) => (id ? id.slice(0, 13) : "—");
|
|
342
|
+
const money = (v) => (v == null ? "—" : usd(v));
|
|
343
|
+
const when = (iso) => (iso ? iso.replace("T", " ").replace(/\.\d+Z$/, "Z") : "—");
|
|
344
|
+
function renderFindings(out, findings, heading) {
|
|
345
|
+
if (findings.length === 0)
|
|
346
|
+
return;
|
|
347
|
+
out();
|
|
348
|
+
out(heading);
|
|
349
|
+
findings.forEach((f, i) => {
|
|
350
|
+
out(` ${i + 1}. [${f.severity}] ${f.title}`);
|
|
351
|
+
if (f.cause)
|
|
352
|
+
out(` why: ${f.cause}`);
|
|
353
|
+
if (f.fix)
|
|
354
|
+
out(` fix: ${f.fix}`);
|
|
355
|
+
});
|
|
356
|
+
}
|
|
357
|
+
export function renderTurn(out, turn) {
|
|
358
|
+
out(`Turn ${turn.runId} ${when(turn.startedAt)}${turn.sessionId ? ` session ${turn.sessionId.slice(0, 8)}` : ""}`);
|
|
359
|
+
if (turn.prompt)
|
|
360
|
+
out(` prompt ${turn.prompt.replace(/\s+/g, " ").slice(0, 110)}`);
|
|
361
|
+
if (turn.matched) {
|
|
362
|
+
out(` matched ${turn.matched.runId}${turn.matched.similarity != null ? ` (similarity ${turn.matched.similarity.toFixed(3)})` : ""}${turn.matched.scenarioId ? ` → scenario ${turn.matched.scenarioId}` : " → no scenario"}`);
|
|
363
|
+
}
|
|
364
|
+
else {
|
|
365
|
+
out(" matched no recording — the agent ran the task; this turn was recorded");
|
|
366
|
+
}
|
|
367
|
+
if (turn.decision?.verdict === "no-steer") {
|
|
368
|
+
out(` decision declined${turn.decision.code ? ` (${turn.decision.code})` : ""}: ${turn.decision.why ?? ""}`);
|
|
369
|
+
}
|
|
370
|
+
if (turn.plan) {
|
|
371
|
+
const cov = turn.plan.coverage?.map((c, i) => `${turn.plan?.tools?.[i] ?? "?"}:${c}`).join(" ");
|
|
372
|
+
out(` plan ${turn.plan.mode}${turn.plan.by ? ` (by ${turn.plan.by})` : ""}, ${turn.plan.steps ?? "?"} steps${cov ? ` ${cov}` : ""}`);
|
|
373
|
+
}
|
|
374
|
+
if (turn.derived) {
|
|
375
|
+
out(` derived ${turn.derived.params ?? "?"} params by ${turn.derived.source ?? "?"} ${money(turn.derived.costUsd)}`);
|
|
376
|
+
}
|
|
377
|
+
if (turn.done) {
|
|
378
|
+
out(` replay ${turn.done.outcome}${turn.done.steps ? ` ${turn.done.steps} steps` : ""}${turn.done.ms != null ? ` in ${(turn.done.ms / 1000).toFixed(1)} s` : ""}`);
|
|
379
|
+
}
|
|
380
|
+
if (turn.reported) {
|
|
381
|
+
const r = turn.reported;
|
|
382
|
+
const p = savedPct(turn);
|
|
383
|
+
const cost = r.derive + r.session + r.fallback;
|
|
384
|
+
out(` reported ${r.outcome}${r.declined ? ` (${r.declined})` : ""} derive ${usd(r.derive)} session ${usd(r.session)} fallback ${usd(r.fallback)}` +
|
|
385
|
+
(r.savedUsd != null
|
|
386
|
+
? ` → saved ${usd(r.savedUsd)}${p != null ? ` (${pct(p)} of ${usd(r.savedUsd + cost)})` : ""}`
|
|
387
|
+
: "") +
|
|
388
|
+
(r.measured === false ? " UNMEASURED" : ""));
|
|
389
|
+
}
|
|
390
|
+
if (turn.finish) {
|
|
391
|
+
const f = turn.finish;
|
|
392
|
+
out(` finished ${f.steps ?? "?"} steps, ${f.durationMs != null ? `${(f.durationMs / 1000).toFixed(1)} s` : "?"}${f.costUsd != null ? `, cost ${usd(f.costUsd)}` : f.measured === false ? ", cost unmeasured" : ""}, ${f.recorded ? "recorded" : "not recorded"}`);
|
|
393
|
+
}
|
|
394
|
+
else {
|
|
395
|
+
out(" finished (not yet — the turn is still open, or the session ended without its Stop hook)");
|
|
396
|
+
}
|
|
397
|
+
}
|
|
398
|
+
export function renderService(out, s, viewerIsOwner) {
|
|
399
|
+
out();
|
|
400
|
+
out(`From the service${s.subject ? ` (${s.subject.kind} ${s.subject.id})` : ""}${s.owner ? `, owner ${viewerIsOwner ? "you" : (s.owner.email ?? s.owner.id)}` : ""}:`);
|
|
401
|
+
if (s.run) {
|
|
402
|
+
const r = s.run;
|
|
403
|
+
out(` recording ${r.id} ${r.isRecording ? "listed" : "below threshold"}${r.embedded ? ", embedded" : ", NOT embedded"} ${r.actionCount} tool calls (${r.tools.join(", ") || "none"}) cost ${money(r.originalCostUsd)} hits ${r.iterations}${r.duplicateOf ? ` duplicate of ${r.duplicateOf}` : ""}`);
|
|
404
|
+
if (r.title || r.input)
|
|
405
|
+
out(` "${(r.title ?? r.input).replace(/\s+/g, " ").slice(0, 100)}"`);
|
|
406
|
+
}
|
|
407
|
+
else {
|
|
408
|
+
out(" recording none");
|
|
409
|
+
}
|
|
410
|
+
if (s.scenario) {
|
|
411
|
+
const sc = s.scenario;
|
|
412
|
+
out(` scenario ${sc.id} ${sc.state}${sc.disabledAt ? " (switched off)" : ""} ${sc.steps.length} steps (${[...new Set(sc.steps.map((x) => x.toolName ?? "?"))].join(", ")}) baseline ${money(sc.baseline.costUsd)} from ${sc.baseline.samples} sample(s)${sc.hitCount != null ? ` hits ${sc.hitCount}` : ""}`);
|
|
413
|
+
if (sc.lastDecline)
|
|
414
|
+
out(` last decline ${sc.lastDecline.reason} at ${when(sc.lastDecline.at ?? undefined)}`);
|
|
415
|
+
if (sc.error)
|
|
416
|
+
out(` error ${sc.error.slice(0, 160)}`);
|
|
417
|
+
}
|
|
418
|
+
else {
|
|
419
|
+
out(" scenario none");
|
|
420
|
+
}
|
|
421
|
+
const ex = s.executions ?? [];
|
|
422
|
+
if (ex.length > 0) {
|
|
423
|
+
out(` executions (${ex.length}, newest first)`);
|
|
424
|
+
for (const e of ex.slice(0, 10)) {
|
|
425
|
+
out(` ${when(e.createdAt)} ${short(e.id)} ${e.outcome.padEnd(13)} cost ${usd(e.costUsd)} saved ${money(e.savedUsd)}${e.savedPct != null ? ` (${pct(e.savedPct)})` : ""}${e.measured ? "" : " unmeasured"}`);
|
|
426
|
+
const failed = e.steps?.filter((st) => st.status === "failed") ?? [];
|
|
427
|
+
for (const st of failed)
|
|
428
|
+
out(` step ${st.stepIndex} ${st.toolName ?? ""} failed${st.error ? `: ${st.error.slice(0, 120)}` : ""}`);
|
|
429
|
+
}
|
|
430
|
+
}
|
|
431
|
+
renderFindings(out, s.findings ?? [], "Service findings");
|
|
432
|
+
}
|
|
433
|
+
// ── the command ───────────────────────────────────────────────────────────────
|
|
434
|
+
async function fetchInvestigation(deps, id, limit) {
|
|
435
|
+
const reply = await deps.service(`/investigate/${id}${limit ? `?limit=${limit}` : ""}`);
|
|
436
|
+
if ("error" in reply)
|
|
437
|
+
return { note: `service not consulted: ${reply.error}` };
|
|
438
|
+
if (reply.status === 404)
|
|
439
|
+
return { note: `the service has nothing for ${id} (not yours, or never reached it)` };
|
|
440
|
+
if (reply.status !== 200)
|
|
441
|
+
return { note: `service answered HTTP ${reply.status}` };
|
|
442
|
+
return { data: reply.body };
|
|
443
|
+
}
|
|
444
|
+
/** Which id the service should be asked about for a turn. */
|
|
445
|
+
function serviceIdFor(turn) {
|
|
446
|
+
return (turn.reported?.ticket ??
|
|
447
|
+
turn.reported?.scenarioId ??
|
|
448
|
+
turn.plan?.scenarioId ??
|
|
449
|
+
turn.matched?.scenarioId ??
|
|
450
|
+
turn.matched?.runId ??
|
|
451
|
+
turn.runId);
|
|
452
|
+
}
|
|
453
|
+
export async function investigateCommand(args, deps) {
|
|
454
|
+
const sub = args.positionals[0];
|
|
455
|
+
const { out } = deps;
|
|
456
|
+
if (sub === "executions") {
|
|
457
|
+
const params = new URLSearchParams();
|
|
458
|
+
if (args.limit)
|
|
459
|
+
params.set("limit", String(args.limit));
|
|
460
|
+
if (args.user)
|
|
461
|
+
params.set("user", args.user);
|
|
462
|
+
const reply = await deps.service(`/investigate/executions${params.size ? `?${params}` : ""}`);
|
|
463
|
+
if ("error" in reply) {
|
|
464
|
+
out(`Could not reach the service: ${reply.error}`);
|
|
465
|
+
return 1;
|
|
466
|
+
}
|
|
467
|
+
if (reply.status === 403) {
|
|
468
|
+
out("Only an admin may look at another account's executions.");
|
|
469
|
+
return 1;
|
|
470
|
+
}
|
|
471
|
+
if (reply.status === 404) {
|
|
472
|
+
out(`No such account: ${args.user}`);
|
|
473
|
+
return 1;
|
|
474
|
+
}
|
|
475
|
+
if (reply.status !== 200) {
|
|
476
|
+
out(`Could not list executions (HTTP ${reply.status}).`);
|
|
477
|
+
return 1;
|
|
478
|
+
}
|
|
479
|
+
const body = reply.body;
|
|
480
|
+
if (args.json) {
|
|
481
|
+
out(JSON.stringify(body, null, 2));
|
|
482
|
+
return 0;
|
|
483
|
+
}
|
|
484
|
+
const rows = body.executions ?? [];
|
|
485
|
+
out(`Executions of ${body.owner?.email ?? body.owner?.id ?? "you"} (${rows.length}, newest first)`);
|
|
486
|
+
if (rows.length === 0)
|
|
487
|
+
out(" none yet");
|
|
488
|
+
for (const e of rows) {
|
|
489
|
+
out(` ${when(e.createdAt)} ${e.id} ${e.outcome.padEnd(13)} cost ${usd(e.costUsd)} saved ${money(e.savedUsd)}${e.savedPct != null ? ` (${pct(e.savedPct)})` : ""}${e.measured ? "" : " unmeasured"}`);
|
|
490
|
+
out(` scenario ${e.scenarioId}${e.runId ? ` run ${e.runId}` : ""}${e.intent ? ` "${e.intent.slice(0, 60)}"` : ""}`);
|
|
491
|
+
}
|
|
492
|
+
out();
|
|
493
|
+
out("Investigate one with `bir investigate <sexec_…>`.");
|
|
494
|
+
return 0;
|
|
495
|
+
}
|
|
496
|
+
const entries = deps.journal(deps.cwd);
|
|
497
|
+
const turns = turnsFromJournal(entries);
|
|
498
|
+
if (sub === "list") {
|
|
499
|
+
const limit = args.limit ?? 20;
|
|
500
|
+
const recent = turns.slice(-limit).reverse();
|
|
501
|
+
if (args.json) {
|
|
502
|
+
out(JSON.stringify(recent, null, 2));
|
|
503
|
+
return 0;
|
|
504
|
+
}
|
|
505
|
+
if (recent.length === 0) {
|
|
506
|
+
out(`No turns in the journal for this directory (${deps.journalPath(deps.cwd)}).`);
|
|
507
|
+
out("Start `bir-hooks` here and run a prompt; every turn is journaled from then on.");
|
|
508
|
+
return 0;
|
|
509
|
+
}
|
|
510
|
+
out(`Recent turns in ${deps.cwd} (newest first)`);
|
|
511
|
+
for (const t of recent) {
|
|
512
|
+
const verdict = !t.matched
|
|
513
|
+
? "recorded"
|
|
514
|
+
: t.decision?.verdict === "no-steer"
|
|
515
|
+
? `declined ${t.decision.code ?? ""}`.trim()
|
|
516
|
+
: t.plan
|
|
517
|
+
? `${t.plan.mode} ${t.done?.outcome ?? t.reported?.outcome ?? "armed"}`
|
|
518
|
+
: "matched";
|
|
519
|
+
const p = savedPct(t);
|
|
520
|
+
const saved = t.reported?.savedUsd != null ? ` saved ${usd(t.reported.savedUsd)}${p != null ? ` (${pct(p)})` : ""}` : "";
|
|
521
|
+
out(` ${when(t.startedAt)} ${t.runId} ${verdict}${saved}`);
|
|
522
|
+
if (t.prompt)
|
|
523
|
+
out(` "${t.prompt.replace(/\s+/g, " ").slice(0, 90)}"`);
|
|
524
|
+
}
|
|
525
|
+
out();
|
|
526
|
+
out("Investigate one with `bir investigate <run_…>`; the newest with `bir investigate`.");
|
|
527
|
+
return 0;
|
|
528
|
+
}
|
|
529
|
+
// One subject: an id, or the newest turn.
|
|
530
|
+
const id = sub;
|
|
531
|
+
let turn;
|
|
532
|
+
if (id) {
|
|
533
|
+
turn = findTurn(turns, id);
|
|
534
|
+
}
|
|
535
|
+
else {
|
|
536
|
+
turn = turns[turns.length - 1];
|
|
537
|
+
if (!turn) {
|
|
538
|
+
out(`No turns in the journal for this directory (${deps.journalPath(deps.cwd)}).`);
|
|
539
|
+
out("Start `bir-hooks` here and run a prompt, or name an id: `bir investigate <run_|scn_|sexec_ id>`.");
|
|
540
|
+
return 1;
|
|
541
|
+
}
|
|
542
|
+
}
|
|
543
|
+
if (id && !turn && !/^(run_|scn_|sexec_)/.test(id)) {
|
|
544
|
+
out(`Not an id this command knows: ${id}. Expected run_…, scn_… or sexec_…, or a subcommand (list, executions).`);
|
|
545
|
+
return 2;
|
|
546
|
+
}
|
|
547
|
+
const findings = turn ? localFindings(turn) : [];
|
|
548
|
+
const serviceId = turn ? serviceIdFor(turn) : id;
|
|
549
|
+
const service = await fetchInvestigation(deps, serviceId, args.limit);
|
|
550
|
+
// A matched turn's own run id is unknown to the service; when the scenario
|
|
551
|
+
// lookup fails, the matched run is still worth a try.
|
|
552
|
+
const fallbackId = turn && service.note && turn.matched?.runId && serviceId !== turn.matched.runId ? turn.matched.runId : undefined;
|
|
553
|
+
const second = fallbackId ? await fetchInvestigation(deps, fallbackId, args.limit) : undefined;
|
|
554
|
+
const chosen = service.data ? service : second?.data ? second : service;
|
|
555
|
+
if (args.json) {
|
|
556
|
+
out(JSON.stringify({ turn: turn ?? null, findings, service: chosen.data ?? null, note: chosen.note ?? null }, null, 2));
|
|
557
|
+
return 0;
|
|
558
|
+
}
|
|
559
|
+
if (turn) {
|
|
560
|
+
renderTurn(out, turn);
|
|
561
|
+
renderFindings(out, findings, "Runner findings");
|
|
562
|
+
}
|
|
563
|
+
else {
|
|
564
|
+
out(`Nothing in this directory's journal for ${id}; asking the service.`);
|
|
565
|
+
}
|
|
566
|
+
if (chosen.data) {
|
|
567
|
+
const viewerIsOwner = Boolean(deps.viewerEmail) && chosen.data.owner?.email === deps.viewerEmail;
|
|
568
|
+
renderService(out, chosen.data, viewerIsOwner);
|
|
569
|
+
}
|
|
570
|
+
else {
|
|
571
|
+
out();
|
|
572
|
+
out(`Service: ${chosen.note ?? "no data"}`);
|
|
573
|
+
}
|
|
574
|
+
return 0;
|
|
575
|
+
}
|
|
576
|
+
//# sourceMappingURL=investigate.js.map
|
package/dist/control/server.js
CHANGED
|
@@ -36,6 +36,7 @@ import { NullRecorder, isIntentMatcher, isMatchAware, isRunCreationAware, isScen
|
|
|
36
36
|
import { ReplayController, POLL_HOLD_MS, } from "../replay/controller.js";
|
|
37
37
|
import { calculateCostUsd } from "../replay/pricing.js";
|
|
38
38
|
import { logDetail, logLine, errText } from "../util/log.js";
|
|
39
|
+
import { journal } from "../util/journal.js";
|
|
39
40
|
import { packageVersion } from "../util/version.js";
|
|
40
41
|
/** How long a `/tool/post` waits for the proxy's own report before recording its own view. */
|
|
41
42
|
const PROXY_REPORT_GRACE_MS = 1_500;
|
|
@@ -388,6 +389,10 @@ export class ControlServer {
|
|
|
388
389
|
};
|
|
389
390
|
session.run = run;
|
|
390
391
|
logLine("run.start", { run: runId, sess: session.sessionId, tier: "bound" });
|
|
392
|
+
// Journal-only: the prompt is what `bir investigate` lists a turn by. Never
|
|
393
|
+
// on stderr, and never more than a preview.
|
|
394
|
+
if (input)
|
|
395
|
+
journal("run.prompt", { run: runId, prompt: input.slice(0, 200) });
|
|
391
396
|
// Started here so the round trip overlaps whatever the caller does next;
|
|
392
397
|
// `onPrompt` awaits the same memoized promise under its own budget.
|
|
393
398
|
void this.watchForMatch(session, run);
|
|
@@ -521,6 +526,8 @@ export class ControlServer {
|
|
|
521
526
|
run: run.runId,
|
|
522
527
|
steps: run.ordering.next,
|
|
523
528
|
durationMs,
|
|
529
|
+
costUsd: cost.measured ? cost.usd.toFixed(4) : undefined,
|
|
530
|
+
measured: cost.measured,
|
|
524
531
|
lossy: this.lossy,
|
|
525
532
|
recorded: run.recording,
|
|
526
533
|
});
|
|
@@ -603,6 +610,7 @@ export class ControlServer {
|
|
|
603
610
|
durationMs: Math.max(0, windowMs),
|
|
604
611
|
prompt: run.input || undefined,
|
|
605
612
|
siblings: states,
|
|
613
|
+
runId: run.runId,
|
|
606
614
|
});
|
|
607
615
|
if (!report)
|
|
608
616
|
return;
|
|
@@ -216,6 +216,12 @@ export declare function isIntentMatcher(r: Recorder): r is Recorder & IntentMatc
|
|
|
216
216
|
*/
|
|
217
217
|
export interface ExecutionReport {
|
|
218
218
|
scenarioId: string;
|
|
219
|
+
/**
|
|
220
|
+
* The control server's run id for the turn this report closes. Journal-only:
|
|
221
|
+
* it lets `bir investigate` join the report to the turn's other lines. Never
|
|
222
|
+
* sent to the service, which has no row for a matched turn.
|
|
223
|
+
*/
|
|
224
|
+
runId?: string;
|
|
219
225
|
/** The match's claim token. It *is* the execution row's id, so a doubled report books once. */
|
|
220
226
|
ticket?: string;
|
|
221
227
|
outcome: "steered_full" | "diverged" | "not_steered" | "failed" | "fell_back";
|
|
@@ -219,7 +219,9 @@ export class RemoteRecorder {
|
|
|
219
219
|
const r = (body ?? {});
|
|
220
220
|
const failed = report.steps?.filter((s) => s.status === "failed").length ?? 0;
|
|
221
221
|
logLine("execution.reported", {
|
|
222
|
+
run: report.runId,
|
|
222
223
|
scenario: report.scenarioId,
|
|
224
|
+
ticket: report.ticket,
|
|
223
225
|
outcome: report.outcome,
|
|
224
226
|
derive: report.deriveCostUsd.toFixed(4),
|
|
225
227
|
session: report.sessionCostUsd.toFixed(4),
|
|
@@ -342,6 +342,8 @@ export declare class ReplayController {
|
|
|
342
342
|
prompt?: string;
|
|
343
343
|
/** Every plan state of this turn, this one included (R-MONEY-5). */
|
|
344
344
|
siblings?: readonly ReplayState[];
|
|
345
|
+
/** The control server's own run id for this turn — journal-only, never sent. */
|
|
346
|
+
runId?: string;
|
|
345
347
|
}): ExecutionReport | undefined;
|
|
346
348
|
/**
|
|
347
349
|
* The step a failed replay is fairly blamed on: the first whose own logic
|
|
@@ -146,6 +146,7 @@ export class ReplayController {
|
|
|
146
146
|
state.declined = code;
|
|
147
147
|
logLine("replay.decision", {
|
|
148
148
|
verdict: "no-steer",
|
|
149
|
+
code,
|
|
149
150
|
run: match.runId,
|
|
150
151
|
scenario: match.scenarioId ?? undefined,
|
|
151
152
|
similarity: match.similarity.toFixed(3),
|
|
@@ -726,6 +727,7 @@ export class ReplayController {
|
|
|
726
727
|
const blame = this.blameStep(steps);
|
|
727
728
|
return {
|
|
728
729
|
scenarioId: state.scenarioId,
|
|
730
|
+
runId: d.runId,
|
|
729
731
|
ticket: state.ticket,
|
|
730
732
|
outcome: state.outcome,
|
|
731
733
|
deriveCostUsd: state.deriveCostUsd,
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* journal — the audit lines that explain a turn, kept on disk for
|
|
3
|
+
* `bir investigate` (docs/calculatedReplayGuide.md §9.1).
|
|
4
|
+
*
|
|
5
|
+
* `logLine` prints to stderr and is gone when the terminal scrolls. The
|
|
6
|
+
* questions people ask afterwards — why did this prompt not run its scenario,
|
|
7
|
+
* which gate declined, what mode did the plan arm in, what did the execution
|
|
8
|
+
* report — are answered by exactly those lines. So `bir-hooks` opens a journal
|
|
9
|
+
* for its directory and every audit line whose event is in {@link isJournaled}
|
|
10
|
+
* lands in it too, as one JSON object per line with the same fields.
|
|
11
|
+
*
|
|
12
|
+
* One file per directory, keyed like the discovery file, so two projects on
|
|
13
|
+
* one machine never mix. Rotated at {@link JOURNAL_MAX_BYTES} by keeping the
|
|
14
|
+
* newest half. Written best-effort and synchronously: a full disk must never be
|
|
15
|
+
* the reason a hook fails (design §1, rule 4), and a hook's answer must not
|
|
16
|
+
* race the line that explains it.
|
|
17
|
+
*/
|
|
18
|
+
export interface JournalEntry {
|
|
19
|
+
/** ISO timestamp, written by the journal, not by the caller. */
|
|
20
|
+
at: string;
|
|
21
|
+
event: string;
|
|
22
|
+
[field: string]: unknown;
|
|
23
|
+
}
|
|
24
|
+
export declare function isJournaled(event: string): boolean;
|
|
25
|
+
export declare const JOURNAL_MAX_BYTES: number;
|
|
26
|
+
export declare function journalPath(cwd: string): string;
|
|
27
|
+
/** Start journaling this directory's audit lines. Returns the file, for the startup log. */
|
|
28
|
+
export declare function openJournal(cwd: string): string;
|
|
29
|
+
export declare function closeJournal(): void;
|
|
30
|
+
/**
|
|
31
|
+
* Append one entry, if a journal is open and the event is one we keep.
|
|
32
|
+
* Called by `logLine` for every audit line, and directly for the few facts
|
|
33
|
+
* that belong in the journal but not on stderr (a prompt preview).
|
|
34
|
+
*/
|
|
35
|
+
export declare function journal(event: string, fields?: Record<string, unknown>): void;
|
|
36
|
+
/** Every entry in a journal file, oldest first. A malformed line is skipped, not fatal. */
|
|
37
|
+
export declare function readJournal(path: string): JournalEntry[];
|
|
38
|
+
//# sourceMappingURL=journal.d.ts.map
|
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* journal — the audit lines that explain a turn, kept on disk for
|
|
3
|
+
* `bir investigate` (docs/calculatedReplayGuide.md §9.1).
|
|
4
|
+
*
|
|
5
|
+
* `logLine` prints to stderr and is gone when the terminal scrolls. The
|
|
6
|
+
* questions people ask afterwards — why did this prompt not run its scenario,
|
|
7
|
+
* which gate declined, what mode did the plan arm in, what did the execution
|
|
8
|
+
* report — are answered by exactly those lines. So `bir-hooks` opens a journal
|
|
9
|
+
* for its directory and every audit line whose event is in {@link isJournaled}
|
|
10
|
+
* lands in it too, as one JSON object per line with the same fields.
|
|
11
|
+
*
|
|
12
|
+
* One file per directory, keyed like the discovery file, so two projects on
|
|
13
|
+
* one machine never mix. Rotated at {@link JOURNAL_MAX_BYTES} by keeping the
|
|
14
|
+
* newest half. Written best-effort and synchronously: a full disk must never be
|
|
15
|
+
* the reason a hook fails (design §1, rule 4), and a hook's answer must not
|
|
16
|
+
* race the line that explains it.
|
|
17
|
+
*/
|
|
18
|
+
import { appendFileSync, existsSync, readFileSync, statSync, writeFileSync } from "node:fs";
|
|
19
|
+
import { dirname, join } from "node:path";
|
|
20
|
+
import { controlDir, controlKey, ensureDir } from "../control/paths.js";
|
|
21
|
+
/** Events worth keeping: the run boundaries, every replay decision, the money. */
|
|
22
|
+
const JOURNALED_PREFIXES = ["run.", "replay.", "plan.", "execution."];
|
|
23
|
+
const JOURNALED_EVENTS = new Set(["session.start", "control.listening", "proxy.registered"]);
|
|
24
|
+
export function isJournaled(event) {
|
|
25
|
+
return JOURNALED_EVENTS.has(event) || JOURNALED_PREFIXES.some((p) => event.startsWith(p));
|
|
26
|
+
}
|
|
27
|
+
export const JOURNAL_MAX_BYTES = 4 * 1024 * 1024;
|
|
28
|
+
export function journalPath(cwd) {
|
|
29
|
+
return join(controlDir(), "journal", `${controlKey(cwd)}.jsonl`);
|
|
30
|
+
}
|
|
31
|
+
let current;
|
|
32
|
+
/** Start journaling this directory's audit lines. Returns the file, for the startup log. */
|
|
33
|
+
export function openJournal(cwd) {
|
|
34
|
+
const path = journalPath(cwd);
|
|
35
|
+
ensureDir(dirname(path));
|
|
36
|
+
current = path;
|
|
37
|
+
return path;
|
|
38
|
+
}
|
|
39
|
+
export function closeJournal() {
|
|
40
|
+
current = undefined;
|
|
41
|
+
}
|
|
42
|
+
/**
|
|
43
|
+
* Append one entry, if a journal is open and the event is one we keep.
|
|
44
|
+
* Called by `logLine` for every audit line, and directly for the few facts
|
|
45
|
+
* that belong in the journal but not on stderr (a prompt preview).
|
|
46
|
+
*/
|
|
47
|
+
export function journal(event, fields = {}) {
|
|
48
|
+
if (!current || !isJournaled(event))
|
|
49
|
+
return;
|
|
50
|
+
const entry = { at: new Date().toISOString(), event };
|
|
51
|
+
for (const [key, value] of Object.entries(fields)) {
|
|
52
|
+
if (value === undefined || value === null)
|
|
53
|
+
continue;
|
|
54
|
+
entry[key] = value;
|
|
55
|
+
}
|
|
56
|
+
try {
|
|
57
|
+
appendFileSync(current, `${JSON.stringify(entry)}\n`);
|
|
58
|
+
rotate(current);
|
|
59
|
+
}
|
|
60
|
+
catch {
|
|
61
|
+
/* best effort — never the reason a hook fails */
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
function rotate(path) {
|
|
65
|
+
let size = 0;
|
|
66
|
+
try {
|
|
67
|
+
size = statSync(path).size;
|
|
68
|
+
}
|
|
69
|
+
catch {
|
|
70
|
+
return;
|
|
71
|
+
}
|
|
72
|
+
if (size <= JOURNAL_MAX_BYTES)
|
|
73
|
+
return;
|
|
74
|
+
const lines = readFileSync(path, "utf8").split("\n").filter(Boolean);
|
|
75
|
+
const keep = lines.slice(Math.floor(lines.length / 2));
|
|
76
|
+
writeFileSync(path, keep.length ? `${keep.join("\n")}\n` : "");
|
|
77
|
+
}
|
|
78
|
+
/** Every entry in a journal file, oldest first. A malformed line is skipped, not fatal. */
|
|
79
|
+
export function readJournal(path) {
|
|
80
|
+
if (!existsSync(path))
|
|
81
|
+
return [];
|
|
82
|
+
const out = [];
|
|
83
|
+
for (const line of readFileSync(path, "utf8").split("\n")) {
|
|
84
|
+
if (!line.trim())
|
|
85
|
+
continue;
|
|
86
|
+
try {
|
|
87
|
+
const parsed = JSON.parse(line);
|
|
88
|
+
if (typeof parsed.event === "string" && typeof parsed.at === "string")
|
|
89
|
+
out.push(parsed);
|
|
90
|
+
}
|
|
91
|
+
catch {
|
|
92
|
+
/* a torn last line from a crash, or a hand edit — ignore it */
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
return out;
|
|
96
|
+
}
|
|
97
|
+
//# sourceMappingURL=journal.js.map
|
package/dist/util/log.js
CHANGED
|
@@ -18,8 +18,12 @@ export const VERBOSE = process.env.BIR_VERBOSE === "1";
|
|
|
18
18
|
* session: the audit log is the only record of what the recorder decided.
|
|
19
19
|
*/
|
|
20
20
|
export const QUIET = process.env.BIR_QUIET === "1";
|
|
21
|
+
import { journal } from "./journal.js";
|
|
21
22
|
/** Print one audit line: `[bir] <iso ts> <event> key=value …`. */
|
|
22
23
|
export function logLine(event, fields = {}) {
|
|
24
|
+
// The journal keeps the decision lines for `bir investigate` (journal.ts).
|
|
25
|
+
// Before the QUIET check on purpose: silence is for the terminal, not the record.
|
|
26
|
+
journal(event, fields);
|
|
23
27
|
if (QUIET)
|
|
24
28
|
return;
|
|
25
29
|
const pairs = [];
|
|
@@ -395,6 +395,38 @@ first prompt that states the task plainly.
|
|
|
395
395
|
|
|
396
396
|
---
|
|
397
397
|
|
|
398
|
+
### 9.1 `bir investigate` — the table above, applied for you
|
|
399
|
+
|
|
400
|
+
Every audit line that explains a turn is also kept in a **journal**, one JSON
|
|
401
|
+
object per line, under `~/.baseinstrunner/control/journal/` (one file per
|
|
402
|
+
directory, rotated at 4 MB). `bir-hooks` prints its path at start. Ask the
|
|
403
|
+
journal and the service together:
|
|
404
|
+
|
|
405
|
+
```
|
|
406
|
+
bir investigate # the newest turn in this directory
|
|
407
|
+
bir investigate <run_|scn_|sexec_ id> # that run, scenario or execution
|
|
408
|
+
bir investigate list [--limit n] # recent turns and their verdicts
|
|
409
|
+
bir investigate executions [--limit n] [--user <email|id>] # the ledger
|
|
410
|
+
```
|
|
411
|
+
|
|
412
|
+
The output is the turn as the journal tells it (matched what, which gate
|
|
413
|
+
declined and why, what mode armed with which coverage, what the execution
|
|
414
|
+
reported), then **runner findings**, then the service's view (the recording,
|
|
415
|
+
the scenario, the baseline and its sample count, the executions with their
|
|
416
|
+
saved $ and %) and **service findings**. Every finding is *problem → cause →
|
|
417
|
+
fix*. The three questions it answers:
|
|
418
|
+
|
|
419
|
+
| Question | Where the answer comes from |
|
|
420
|
+
|---|---|
|
|
421
|
+
| Why did this turn not run its calculated scenario? | `replay.decision code=…` in the journal, mapped to the cause and the fix; and the service: is the run a recording, is it embedded, how many hits, is the scenario ready / failed / switched off, which step is parked |
|
|
422
|
+
| Why did it save so little, or cost more? | the execution's cost breakdown (derive, session, fallback) against the baseline; steer mode with built-in steps is the usual answer, a single-sample baseline the second |
|
|
423
|
+
| What happened, step by step? | the turn's lines in order, the per-step verdicts on the execution, and `replay.*_failed`, `replay.diverge`, `replay.handover` as incidents |
|
|
424
|
+
|
|
425
|
+
You see your own data. An admin of the service sees everyone's and may pass
|
|
426
|
+
`--user` to `executions`. `--json` on any form prints the raw merge.
|
|
427
|
+
|
|
428
|
+
---
|
|
429
|
+
|
|
398
430
|
## 10. What is built
|
|
399
431
|
|
|
400
432
|
All of it. Phases R0–R7 of [calculatedReplay.md](calculatedReplay.md) §17 are implemented and
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@basein/runner",
|
|
3
|
-
"version": "0.2.
|
|
3
|
+
"version": "0.2.7",
|
|
4
4
|
"description": "A recording MCP proxy: sits between any MCP client and its MCP servers, executes each call on the client's behalf, and records the run as a reusable BaseIn scenario.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "MIT",
|