@basein/runner 0.3.17 → 0.3.18
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/bir-scenario.d.ts +5 -1
- package/dist/bin/bir-scenario.js +155 -5
- package/dist/bin/bir.js +2 -2
- package/dist/bin/scenario-edit.d.ts +7 -0
- package/dist/bin/scenario-edit.js +9 -2
- package/dist/bin/scenario-liveness.d.ts +12 -1
- package/dist/bin/scenario-liveness.js +8 -4
- package/dist/bin/setup.js +3 -3
- package/dist/control/client.d.ts +5 -0
- package/dist/control/client.js +7 -0
- package/dist/control/ensure-hook.d.ts +14 -3
- package/dist/control/ensure-hook.js +18 -5
- package/dist/control/server.d.ts +165 -2
- package/dist/control/server.js +719 -21
- package/dist/control/transcript.d.ts +19 -0
- package/dist/control/transcript.js +57 -0
- package/dist/proxy/intercept.js +13 -0
- package/dist/proxy/session.d.ts +23 -0
- package/dist/proxy/session.js +71 -3
- package/dist/record/recorder.d.ts +146 -0
- package/dist/record/recorder.js +9 -0
- package/dist/record/remote-recorder.d.ts +59 -1
- package/dist/record/remote-recorder.js +297 -20
- package/dist/record/switchable-recorder.js +6 -0
- package/dist/replay/bundle.d.ts +8 -0
- package/dist/replay/bundle.js +20 -6
- package/dist/replay/controller.d.ts +33 -1
- package/dist/replay/controller.js +137 -12
- package/dist/replay/derive.d.ts +15 -0
- package/dist/replay/derive.js +5 -0
- package/dist/replay/plan.d.ts +90 -1
- package/dist/replay/plan.js +232 -10
- package/dist/replay/source-run.d.ts +2 -2
- package/dist/replay/source-run.js +3 -3
- package/dist/replay/targets.d.ts +40 -1
- package/dist/replay/targets.js +105 -3
- package/dist/replay/types.d.ts +6 -5
- package/package.json +1 -1
|
@@ -54,8 +54,8 @@ import { credentialsPath, describeBody, isAuthResponse, issuedFor, loadCredentia
|
|
|
54
54
|
import { errText, logDetail, logEvery, logLine } from "../util/log.js";
|
|
55
55
|
import { serviceHeaders } from "../util/service-headers.js";
|
|
56
56
|
import { isUnhealthyStatus, serviceHealth, serviceTimeoutMs, timeoutSignal, } from "../util/service-health.js";
|
|
57
|
-
import { redactJsonText, redactText } from "./redact.js";
|
|
58
|
-
import { capJsonText, capText, DEFAULT_MAX_PAYLOAD_BYTES, preCut, REPORT_PAYLOAD_MAX_BYTES, } from "./truncate.js";
|
|
57
|
+
import { redact, redactJsonText, redactText } from "./redact.js";
|
|
58
|
+
import { capJsonText, capText, DEFAULT_MAX_PAYLOAD_BYTES, preCut, REPORT_PAYLOAD_MAX_BYTES, truncateString, } from "./truncate.js";
|
|
59
59
|
/**
|
|
60
60
|
* Caps for the free text this recorder sends, in UTF-8 bytes. Tool payloads
|
|
61
61
|
* arrive capped at 64 KiB by their producers; text never passed through one,
|
|
@@ -149,6 +149,8 @@ const AUTH_RECHECK_MS = 1_500;
|
|
|
149
149
|
const AUTH_REJECTED_RETRY_MS = 10 * 60_000;
|
|
150
150
|
/** Execution reports ride their own lane: a matched turn has no run on the service. */
|
|
151
151
|
const EXECUTIONS = "__executions__";
|
|
152
|
+
/** Tool lists too (the service's autocorrect.md AC12): they belong to no run. */
|
|
153
|
+
const TOOL_LISTS = "__tool_lists__";
|
|
152
154
|
/** A send that did not succeed, with the status the service answered, if it answered. */
|
|
153
155
|
export class ServiceError extends Error {
|
|
154
156
|
status;
|
|
@@ -207,10 +209,14 @@ function pendStep(step) {
|
|
|
207
209
|
* input, then the outputs, each in the order the steps ran. A payload that
|
|
208
210
|
* does not fit is left out; its step keeps its verdict, and the costs the
|
|
209
211
|
* report exists for are never what gives way.
|
|
212
|
+
*
|
|
213
|
+
* `extra` rides outside that budget: an answered turn's whole calls
|
|
214
|
+
* ({@link answerCallsBody}), which hold themselves to a budget of their own.
|
|
215
|
+
* Counted inside it, they would leave no room for a single step's payload.
|
|
210
216
|
*/
|
|
211
|
-
function reportBody(fields, steps) {
|
|
217
|
+
function reportBody(fields, steps, extra = {}) {
|
|
212
218
|
if (!steps)
|
|
213
|
-
return JSON.stringify(fields);
|
|
219
|
+
return JSON.stringify({ ...fields, ...extra });
|
|
214
220
|
const sent = steps.map(({ step }) => ({
|
|
215
221
|
...step,
|
|
216
222
|
error: typeof step.error === "string"
|
|
@@ -243,7 +249,125 @@ function reportBody(fields, steps) {
|
|
|
243
249
|
add(i, "toolInput");
|
|
244
250
|
for (const i of all)
|
|
245
251
|
add(i, "toolOutput");
|
|
246
|
-
return JSON.stringify({ ...fields, steps: sent });
|
|
252
|
+
return JSON.stringify({ ...fields, steps: sent, ...extra });
|
|
253
|
+
}
|
|
254
|
+
/** What one answered turn's call may carry of its output, in UTF-8 bytes (review item 12). */
|
|
255
|
+
export const ANSWER_CALL_OUTPUT_MAX_BYTES = 64 * 1024;
|
|
256
|
+
/** All of one report's answer calls together, serialized, in UTF-8 bytes. */
|
|
257
|
+
export const ANSWER_CALLS_MAX_BYTES = 600_000;
|
|
258
|
+
/** Below this, an output is not cut further to make room: the calls are left out instead. */
|
|
259
|
+
const ANSWER_CALL_OUTPUT_MIN_BYTES = 1_024;
|
|
260
|
+
/** At most this many calls go: a plan keeps no more verdicts than this either. */
|
|
261
|
+
const ANSWER_CALLS_MAX = 200;
|
|
262
|
+
const utf8Bytes = (text) => Buffer.byteLength(text, "utf8");
|
|
263
|
+
/** One output as it goes: redacted — by its keys when it is JSON — and cut at `max` bytes with the marker. */
|
|
264
|
+
function answerOutput(text, max) {
|
|
265
|
+
let clean;
|
|
266
|
+
try {
|
|
267
|
+
JSON.parse(text);
|
|
268
|
+
clean = redactJsonText(text);
|
|
269
|
+
}
|
|
270
|
+
catch {
|
|
271
|
+
clean = redactText(text);
|
|
272
|
+
}
|
|
273
|
+
return truncateString(clean, max);
|
|
274
|
+
}
|
|
275
|
+
/**
|
|
276
|
+
* An answered turn's calls as they are sent (the service's review of
|
|
277
|
+
* calcscenarioautocorrect.md, item 12): each input redacted again, each output
|
|
278
|
+
* redacted and cut at {@link ANSWER_CALL_OUTPUT_MAX_BYTES}, and all of them
|
|
279
|
+
* held to `maxBytes` serialized by cutting the largest outputs first — every
|
|
280
|
+
* output above one common length is cut to it, the longest length that fits.
|
|
281
|
+
* The cut is the recordings' own, "… [truncated N of M bytes]", so a reader
|
|
282
|
+
* can tell a short answer from a cut one. Undefined when there is nothing to
|
|
283
|
+
* send, or when even outputs of {@link ANSWER_CALL_OUTPUT_MIN_BYTES} would not
|
|
284
|
+
* fit (inputs that large): the service then reads the report's cut copies,
|
|
285
|
+
* as it did before, rather than half a list.
|
|
286
|
+
*/
|
|
287
|
+
export function answerCallsBody(calls, maxBytes = ANSWER_CALLS_MAX_BYTES) {
|
|
288
|
+
if (!calls || calls.length === 0)
|
|
289
|
+
return undefined;
|
|
290
|
+
const base = calls.slice(0, ANSWER_CALLS_MAX).map((c) => {
|
|
291
|
+
const input = redact(c.input);
|
|
292
|
+
return {
|
|
293
|
+
stepIndex: c.stepIndex,
|
|
294
|
+
// The service takes 200 characters of a name, and refuses the whole list past it.
|
|
295
|
+
toolName: String(c.toolName ?? "").slice(0, 200),
|
|
296
|
+
input: (input && typeof input === "object" && !Array.isArray(input) ? input : {}),
|
|
297
|
+
output: answerOutput(typeof c.output === "string" ? c.output : "", ANSWER_CALL_OUTPUT_MAX_BYTES),
|
|
298
|
+
...(c.isError === true ? { isError: true } : {}),
|
|
299
|
+
};
|
|
300
|
+
});
|
|
301
|
+
const sized = (cap) => {
|
|
302
|
+
const cut = base.map((c) => (utf8Bytes(c.output) > cap ? { ...c, output: truncateString(c.output, cap) } : c));
|
|
303
|
+
return { calls: cut, bytes: utf8Bytes(JSON.stringify(cut)) };
|
|
304
|
+
};
|
|
305
|
+
const whole = sized(ANSWER_CALL_OUTPUT_MAX_BYTES);
|
|
306
|
+
if (whole.bytes <= maxBytes)
|
|
307
|
+
return whole.calls;
|
|
308
|
+
if (sized(ANSWER_CALL_OUTPUT_MIN_BYTES).bytes > maxBytes)
|
|
309
|
+
return undefined;
|
|
310
|
+
// The longest common length that fits, found by halving to within a
|
|
311
|
+
// kilobyte: each try is one serialization — six or so in all — and the
|
|
312
|
+
// measure is the real one, escapes and markers in.
|
|
313
|
+
let lo = ANSWER_CALL_OUTPUT_MIN_BYTES;
|
|
314
|
+
let hi = ANSWER_CALL_OUTPUT_MAX_BYTES;
|
|
315
|
+
while (hi - lo > 1_024) {
|
|
316
|
+
const mid = Math.floor((lo + hi) / 2);
|
|
317
|
+
if (sized(mid).bytes <= maxBytes)
|
|
318
|
+
lo = mid;
|
|
319
|
+
else
|
|
320
|
+
hi = mid;
|
|
321
|
+
}
|
|
322
|
+
return sized(lo).calls;
|
|
323
|
+
}
|
|
324
|
+
/** The day a finish says, checked: a line of at most 400 characters, a `YYYY-MM-DD`; undefined when neither. */
|
|
325
|
+
function agentDayBody(day) {
|
|
326
|
+
if (!day)
|
|
327
|
+
return undefined;
|
|
328
|
+
const said = typeof day.said === "string" ? redactText(day.said).replace(/\s+/g, " ").trim().slice(0, 400) : "";
|
|
329
|
+
const local = typeof day.local === "string" && /^\d{4}-\d{2}-\d{2}$/.test(day.local) ? day.local : "";
|
|
330
|
+
if (!said && !local)
|
|
331
|
+
return undefined;
|
|
332
|
+
return { ...(said ? { said } : {}), ...(local ? { local } : {}) };
|
|
333
|
+
}
|
|
334
|
+
const ATTENTION_REASONS = new Set([
|
|
335
|
+
"checking",
|
|
336
|
+
"rejected",
|
|
337
|
+
"rejected_again",
|
|
338
|
+
"verifying",
|
|
339
|
+
"verify_failed",
|
|
340
|
+
"verify_stale",
|
|
341
|
+
]);
|
|
342
|
+
/** The plans of a `GET /scenarios/attention` answer, read off the wire: at most five, each checked. */
|
|
343
|
+
function attentionPlans(body) {
|
|
344
|
+
const raw = body?.plans;
|
|
345
|
+
if (!Array.isArray(raw))
|
|
346
|
+
return [];
|
|
347
|
+
const plans = [];
|
|
348
|
+
for (const item of raw) {
|
|
349
|
+
if (plans.length >= 5)
|
|
350
|
+
break;
|
|
351
|
+
if (!item || typeof item !== "object")
|
|
352
|
+
continue;
|
|
353
|
+
const p = item;
|
|
354
|
+
if (typeof p.scenarioId !== "string" || !p.scenarioId || typeof p.url !== "string")
|
|
355
|
+
continue;
|
|
356
|
+
// Not held — only a correction waits — the service says neither since
|
|
357
|
+
// when nor why.
|
|
358
|
+
const heldSince = typeof p.heldSince === "string" && p.heldSince ? p.heldSince.slice(0, 40) : null;
|
|
359
|
+
plans.push({
|
|
360
|
+
scenarioId: p.scenarioId.slice(0, 100),
|
|
361
|
+
title: typeof p.title === "string" ? p.title.slice(0, 500) : "",
|
|
362
|
+
heldSince,
|
|
363
|
+
reason: heldSince
|
|
364
|
+
? (typeof p.reason === "string" && ATTENTION_REASONS.has(p.reason) ? p.reason : "checking")
|
|
365
|
+
: null,
|
|
366
|
+
correction: p.correction === "proposed" ? "proposed" : null,
|
|
367
|
+
url: p.url.slice(0, 1_000),
|
|
368
|
+
});
|
|
369
|
+
}
|
|
370
|
+
return plans;
|
|
247
371
|
}
|
|
248
372
|
/** A tool error is JSON when a producer serialized it, and prose when it did not. */
|
|
249
373
|
function pendError(text) {
|
|
@@ -676,9 +800,10 @@ export class RemoteRecorder {
|
|
|
676
800
|
toolName: data.toolName,
|
|
677
801
|
toolOutput: out(toolOutput),
|
|
678
802
|
toolError: out(toolError),
|
|
679
|
-
// Omitted, not nulled, when absent
|
|
680
|
-
//
|
|
681
|
-
//
|
|
803
|
+
// Omitted, not nulled, when absent. A call's id rides on both of its
|
|
804
|
+
// rows, and `isError` on the answer of one that failed (review item
|
|
805
|
+
// 7); a plan's own call recorded into a redo carries `byPlan` too
|
|
806
|
+
// (the service's autocorrect.md AC2).
|
|
682
807
|
metadata: data.metadata,
|
|
683
808
|
}));
|
|
684
809
|
return id;
|
|
@@ -707,15 +832,20 @@ export class RemoteRecorder {
|
|
|
707
832
|
// user's data, so it is cleaned and cut like every other text sent.
|
|
708
833
|
const error = pend(metrics?.error, TEXT_CAPS.reportError);
|
|
709
834
|
const asked = metrics?.incomplete;
|
|
835
|
+
// The day its agent was told it is (review item 1). Cleaned here, like
|
|
836
|
+
// every text sent: the line comes out of a system prompt.
|
|
837
|
+
const agentDay = agentDayBody(metrics?.agentDay);
|
|
710
838
|
this.enqueueRun(runId, "finish", sizeOf(output, error), () =>
|
|
711
|
-
// `error` and `
|
|
712
|
-
// finish of an ordinary run is the body it has always
|
|
839
|
+
// `error`, `incomplete` and `agentDay` are omitted, not nulled, when
|
|
840
|
+
// absent, so the finish of an ordinary run is the body it has always
|
|
841
|
+
// been. An older service drops `agentDay`.
|
|
713
842
|
this.post(`/recordings/runs/${runId}/finish`, {
|
|
714
843
|
finalOutput: out(output),
|
|
715
844
|
error: out(error),
|
|
716
845
|
originalCostUsd: metrics?.costUsd,
|
|
717
846
|
originalMs: metrics?.durationMs,
|
|
718
847
|
incomplete: asked ? this.incompleteOf(runId, asked.lossy) : undefined,
|
|
848
|
+
agentDay,
|
|
719
849
|
}));
|
|
720
850
|
}
|
|
721
851
|
/**
|
|
@@ -727,6 +857,128 @@ export class RemoteRecorder {
|
|
|
727
857
|
const lostSteps = this.runs.get(runId)?.lostSteps ?? 0;
|
|
728
858
|
return lossy || lostSteps > 0 ? { lostSteps, lossy } : undefined;
|
|
729
859
|
}
|
|
860
|
+
/**
|
|
861
|
+
* One wrapped server's tool list (the service's autocorrect.md AC12): the
|
|
862
|
+
* names, descriptions and input schemas a correction from an agent's answer
|
|
863
|
+
* writes a step with. On its own lane, and never counted as a recording's
|
|
864
|
+
* lost send. Quiet towards the service's health: a list the service refuses
|
|
865
|
+
* (an older one's 404, the route's own rate limit, a fault of that route)
|
|
866
|
+
* opens no breaker the hooks wait on. Resolves true when the service took it.
|
|
867
|
+
*
|
|
868
|
+
* `project` is the list's project (review item 16): two projects with a
|
|
869
|
+
* server of one name keep a list each. An older service drops the field.
|
|
870
|
+
*/
|
|
871
|
+
reportTools(serverName, tools, project) {
|
|
872
|
+
return new Promise((resolve) => {
|
|
873
|
+
const body = JSON.stringify(project ? { serverName, project, tools } : { serverName, tools });
|
|
874
|
+
this.enqueue({
|
|
875
|
+
lane: TOOL_LISTS,
|
|
876
|
+
kind: "tools",
|
|
877
|
+
bytes: body.length + 256,
|
|
878
|
+
send: async () => {
|
|
879
|
+
try {
|
|
880
|
+
await this.request("POST", "/recordings/tools", body, { quiet: true });
|
|
881
|
+
logDetail("recorder.tools_sent", { server: serverName, tools: tools.length });
|
|
882
|
+
resolve(true);
|
|
883
|
+
}
|
|
884
|
+
catch (err) {
|
|
885
|
+
if (err instanceof AccountChanged)
|
|
886
|
+
throw err;
|
|
887
|
+
logEvery(`recorder.tools_failed:${serverName}`, 10 * 60_000, "recorder.tools_failed", {
|
|
888
|
+
server: serverName,
|
|
889
|
+
why: err instanceof ServiceError && err.status === 404
|
|
890
|
+
? "the service does not take tool lists yet"
|
|
891
|
+
: "the tool list was not sent; it is sent again later",
|
|
892
|
+
error: errText(err),
|
|
893
|
+
});
|
|
894
|
+
resolve(false);
|
|
895
|
+
}
|
|
896
|
+
},
|
|
897
|
+
abandon: () => resolve(false),
|
|
898
|
+
});
|
|
899
|
+
});
|
|
900
|
+
}
|
|
901
|
+
/** The account this recorder sends for, as signed in now. */
|
|
902
|
+
accountId() {
|
|
903
|
+
return this.session.user.id;
|
|
904
|
+
}
|
|
905
|
+
/**
|
|
906
|
+
* The plans of this account that wait for their owner (`GET
|
|
907
|
+
* /scenarios/attention`; the service's review of calcscenarioautocorrect.md,
|
|
908
|
+
* item 8): what the line a session start shows is made of. Asked on a
|
|
909
|
+
* session start's path, so it is bounded (1.5 s by default), asked only
|
|
910
|
+
* while the service is known to answer, and quiet towards its health: a
|
|
911
|
+
* slow or refusing answer here opens no breaker the hooks wait on. Null on
|
|
912
|
+
* any failure — an older service's 404 included — and never a throw.
|
|
913
|
+
*/
|
|
914
|
+
async scenarioAttention(opts = {}) {
|
|
915
|
+
try {
|
|
916
|
+
if (!this.mayHotWait())
|
|
917
|
+
return null;
|
|
918
|
+
const body = await this.request("GET", "/scenarios/attention", undefined, {
|
|
919
|
+
timeoutMs: opts.timeoutMs ?? 1_500,
|
|
920
|
+
quiet: true,
|
|
921
|
+
});
|
|
922
|
+
return attentionPlans(body);
|
|
923
|
+
}
|
|
924
|
+
catch (err) {
|
|
925
|
+
logEvery("recorder.attention_failed", 10 * 60_000, "recorder.attention_failed", {
|
|
926
|
+
why: err instanceof ServiceError && err.status === 404
|
|
927
|
+
? "the service does not list plans that need their owner yet"
|
|
928
|
+
: "the plans that need their owner were not read; a session start says nothing of them",
|
|
929
|
+
error: errText(err),
|
|
930
|
+
});
|
|
931
|
+
return null;
|
|
932
|
+
}
|
|
933
|
+
}
|
|
934
|
+
/**
|
|
935
|
+
* The agent said a replay's results do not answer its request (`POST
|
|
936
|
+
* /scenarios/:scenarioId/dispute`; the service's review of
|
|
937
|
+
* calcscenarioautocorrect.md, item 19). `executionId` is the turn's
|
|
938
|
+
* execution ticket — the id its report books under at the end of the turn,
|
|
939
|
+
* so while the turn runs the service has the ticket and not yet the row.
|
|
940
|
+
* Awaited — the agent waits for one sentence back — bounded (10 s by
|
|
941
|
+
* default), and quiet towards the service's health. Never throws.
|
|
942
|
+
*/
|
|
943
|
+
async disputeScenario(scenarioId, report, opts = {}) {
|
|
944
|
+
if (!/^scn_[\w-]+$/.test(scenarioId))
|
|
945
|
+
return { ok: false, why: "no_plan" };
|
|
946
|
+
this.checkCredentials();
|
|
947
|
+
if (this.state !== "recording")
|
|
948
|
+
return { ok: false, why: "signed_out" };
|
|
949
|
+
// The agent's own words: they may quote what a tool returned.
|
|
950
|
+
const reason = redactText(String(report.reason ?? "")).trim().slice(0, 2_000);
|
|
951
|
+
try {
|
|
952
|
+
const body = await this.request("POST", `/scenarios/${scenarioId}/dispute`, JSON.stringify({
|
|
953
|
+
reason,
|
|
954
|
+
by: "agent",
|
|
955
|
+
...(report.executionId ? { executionId: report.executionId } : {}),
|
|
956
|
+
}), { timeoutMs: opts.timeoutMs ?? 10_000, quiet: true });
|
|
957
|
+
const b = (body && typeof body === "object" ? body : {});
|
|
958
|
+
const answer = {
|
|
959
|
+
ok: b.ok === true,
|
|
960
|
+
held: b.held === true,
|
|
961
|
+
...(typeof b.why === "string" && b.why ? { why: b.why.slice(0, 200) } : {}),
|
|
962
|
+
};
|
|
963
|
+
logLine("recorder.disputed", {
|
|
964
|
+
scenario: scenarioId,
|
|
965
|
+
execution: report.executionId,
|
|
966
|
+
ok: answer.ok,
|
|
967
|
+
held: answer.held,
|
|
968
|
+
why: answer.why,
|
|
969
|
+
});
|
|
970
|
+
return answer;
|
|
971
|
+
}
|
|
972
|
+
catch (err) {
|
|
973
|
+
const status = err instanceof ServiceError ? err.status : undefined;
|
|
974
|
+
logLine("recorder.dispute_failed", {
|
|
975
|
+
scenario: scenarioId,
|
|
976
|
+
why: status === 404 ? "the service does not take reports that a replay was wrong yet" : "the report was not sent",
|
|
977
|
+
error: errText(err),
|
|
978
|
+
});
|
|
979
|
+
return { ok: false, why: status === 404 ? "unsupported" : "service_error" };
|
|
980
|
+
}
|
|
981
|
+
}
|
|
730
982
|
/**
|
|
731
983
|
* Report what a matched turn cost (docs/calculatedReplay.md §11.2).
|
|
732
984
|
*
|
|
@@ -746,11 +998,28 @@ export class RemoteRecorder {
|
|
|
746
998
|
const error = pend(report.error, TEXT_CAPS.reportError);
|
|
747
999
|
const answer = pend(report.answer, TEXT_CAPS.reportAnswer);
|
|
748
1000
|
const steps = report.steps?.map(pendStep);
|
|
1001
|
+
// The plan's whole calls go only beside an answer (review item 12): the
|
|
1002
|
+
// service keeps them only while it keeps the answer.
|
|
1003
|
+
const calls = answer && report.answerCalls?.length ? report.answerCalls : undefined;
|
|
1004
|
+
const callBytes = calls
|
|
1005
|
+
? calls.reduce((n, c) => n + c.output.length + 256, 0) + JSON.stringify(calls.map((c) => c.input)).length
|
|
1006
|
+
: 0;
|
|
749
1007
|
this.enqueue({
|
|
750
1008
|
lane: EXECUTIONS,
|
|
751
1009
|
kind: "report",
|
|
752
|
-
bytes: sizeOf(prompt, error, answer, ...(steps ?? []).flatMap((s) => [s.toolInput, s.toolOutput])) +
|
|
1010
|
+
bytes: sizeOf(prompt, error, answer, ...(steps ?? []).flatMap((s) => [s.toolInput, s.toolOutput])) +
|
|
1011
|
+
Math.min(callBytes, ANSWER_CALLS_MAX_BYTES + 64 * 1024) +
|
|
1012
|
+
1024,
|
|
753
1013
|
send: async () => {
|
|
1014
|
+
// Shaped here, off the hook's path: the cuts serialize the calls a few times.
|
|
1015
|
+
const answerCalls = calls ? answerCallsBody(calls) : undefined;
|
|
1016
|
+
if (calls && !answerCalls) {
|
|
1017
|
+
logLine("execution.answer_calls_left_out", {
|
|
1018
|
+
scenario: report.scenarioId,
|
|
1019
|
+
calls: calls.length,
|
|
1020
|
+
why: "the plan's calls did not fit the report even cut short — the service reads the report's own copies",
|
|
1021
|
+
});
|
|
1022
|
+
}
|
|
754
1023
|
// A redo whose run the service refused to create is no run of its:
|
|
755
1024
|
// never named (the service's autocorrect.md AC1), even when the
|
|
756
1025
|
// refusal came after the report was built.
|
|
@@ -795,7 +1064,10 @@ export class RemoteRecorder {
|
|
|
795
1064
|
// ...or only answered: its answer, capped and redacted like any
|
|
796
1065
|
// text (AC11). Additive, like `redoneBy`.
|
|
797
1066
|
answer: out(answer),
|
|
798
|
-
}, steps
|
|
1067
|
+
}, steps,
|
|
1068
|
+
// Beside the answer, the plan's calls whole (review item 12),
|
|
1069
|
+
// outside the steps' budget. Additive: an older service drops them.
|
|
1070
|
+
answerCalls ? { answerCalls } : {}), { retryTransient: Boolean(report.ticket) });
|
|
799
1071
|
const r = (body ?? {});
|
|
800
1072
|
const failed = report.steps?.filter((s) => s.status === "failed").length ?? 0;
|
|
801
1073
|
logLine("execution.reported", {
|
|
@@ -817,6 +1089,7 @@ export class RemoteRecorder {
|
|
|
817
1089
|
// `bir investigate` prints it beside the outcome: "redone by run_…".
|
|
818
1090
|
redoneBy: report.redoneBy?.runId,
|
|
819
1091
|
ownCalls: report.redoneBy?.ownCalls,
|
|
1092
|
+
answerCalls: answerCalls?.length,
|
|
820
1093
|
});
|
|
821
1094
|
},
|
|
822
1095
|
});
|
|
@@ -1130,8 +1403,9 @@ export class RemoteRecorder {
|
|
|
1130
1403
|
catch {
|
|
1131
1404
|
/* never */
|
|
1132
1405
|
}
|
|
1133
|
-
// A matched turn records nothing by design: nothing was lost.
|
|
1134
|
-
|
|
1406
|
+
// A matched turn records nothing by design: nothing was lost. Nor is a
|
|
1407
|
+
// tool list a recording's send (AC12): it is sent again later.
|
|
1408
|
+
if (why !== "not_created" && job.kind !== "tools") {
|
|
1135
1409
|
this.droppedSends += 1;
|
|
1136
1410
|
if (run) {
|
|
1137
1411
|
run.dropped += 1;
|
|
@@ -1453,6 +1727,9 @@ export class RemoteRecorder {
|
|
|
1453
1727
|
async request(method, path, body, opts = {}) {
|
|
1454
1728
|
let retried401 = false;
|
|
1455
1729
|
let retriedTransient = false;
|
|
1730
|
+
const health = opts.quiet
|
|
1731
|
+
? { fail: (_reason) => undefined, ok: (_started) => undefined }
|
|
1732
|
+
: { fail: (reason) => this.health.fail(reason), ok: (started) => this.health.ok(started) };
|
|
1456
1733
|
const user = this.session.user.id;
|
|
1457
1734
|
for (let attempt = 0;; attempt++) {
|
|
1458
1735
|
if (attempt > 0 && !opts.anonymous && this.session.user.id !== user) {
|
|
@@ -1487,13 +1764,13 @@ export class RemoteRecorder {
|
|
|
1487
1764
|
await sleep(jitterMs());
|
|
1488
1765
|
continue;
|
|
1489
1766
|
}
|
|
1490
|
-
|
|
1767
|
+
health.fail(reason);
|
|
1491
1768
|
throw new ServiceError(`${method} ${path}: ${reason}`);
|
|
1492
1769
|
}
|
|
1493
1770
|
if (res.status === 401 && !opts.anonymous) {
|
|
1494
1771
|
// Release the socket now rather than at GC, then renew — once.
|
|
1495
1772
|
await res.body?.cancel().catch(() => undefined);
|
|
1496
|
-
|
|
1773
|
+
health.ok(started);
|
|
1497
1774
|
if (retried401)
|
|
1498
1775
|
throw new ServiceError(`HTTP 401 ${method} ${path}`, 401);
|
|
1499
1776
|
retried401 = true;
|
|
@@ -1523,7 +1800,7 @@ export class RemoteRecorder {
|
|
|
1523
1800
|
if (opts.signal?.aborted)
|
|
1524
1801
|
throw err;
|
|
1525
1802
|
const reason = signal.aborted ? `timed out after ${timeoutMs} ms reading the answer` : reasonOf(err);
|
|
1526
|
-
|
|
1803
|
+
health.fail(reason);
|
|
1527
1804
|
throw new ServiceError(`${method} ${path}: ${reason}`, res.status);
|
|
1528
1805
|
}
|
|
1529
1806
|
if (isUnhealthyStatus(res.status)) {
|
|
@@ -1533,19 +1810,19 @@ export class RemoteRecorder {
|
|
|
1533
1810
|
await sleep(retryDelayMs(res));
|
|
1534
1811
|
continue;
|
|
1535
1812
|
}
|
|
1536
|
-
|
|
1813
|
+
health.fail(`HTTP ${res.status}`);
|
|
1537
1814
|
throw new ServiceError(`HTTP ${res.status} ${method} ${path}${detail(text, type)}`, res.status);
|
|
1538
1815
|
}
|
|
1539
1816
|
// A 2xx web page is a captive portal or a website, and a 403 one a CDN's
|
|
1540
1817
|
// challenge — not the API answering. An empty or plain 2xx is: only the
|
|
1541
1818
|
// body decides, not a `text/html` a framework put on `res.send("ok")`.
|
|
1542
1819
|
if ((res.ok || res.status === 403) && isWebPage(text, type)) {
|
|
1543
|
-
|
|
1820
|
+
health.fail("html");
|
|
1544
1821
|
throw new ServiceError(`HTTP ${res.status} ${method} ${path} — the answer was a web page, not the BaseIn API`, res.status);
|
|
1545
1822
|
}
|
|
1546
1823
|
// Anything else is the service answering, including a 4xx — an older
|
|
1547
1824
|
// service's 404 must not look like an outage.
|
|
1548
|
-
|
|
1825
|
+
health.ok(started);
|
|
1549
1826
|
if (!res.ok) {
|
|
1550
1827
|
if (res.status === 409 && retriedTransient)
|
|
1551
1828
|
return undefined; // the first attempt landed
|
|
@@ -52,6 +52,12 @@ const NEUTRAL = {
|
|
|
52
52
|
reportExecution: () => undefined,
|
|
53
53
|
backfillContext: () => undefined,
|
|
54
54
|
probeIfFailing: () => undefined,
|
|
55
|
+
// A tool list (the service's autocorrect.md AC12): not taken, so sent again later.
|
|
56
|
+
reportTools: () => Promise.resolve(false),
|
|
57
|
+
accountId: () => undefined,
|
|
58
|
+
// Signed out: no plan to list for a session start, no report taken (review items 8, 19).
|
|
59
|
+
scenarioAttention: () => Promise.resolve(null),
|
|
60
|
+
disputeScenario: () => Promise.resolve({ ok: false, why: "signed_out" }),
|
|
55
61
|
};
|
|
56
62
|
/**
|
|
57
63
|
* `RemoteRecorder`'s private helpers. They are methods at runtime (TypeScript's
|
package/dist/replay/bundle.d.ts
CHANGED
|
@@ -55,6 +55,14 @@ export interface AssembleOptions {
|
|
|
55
55
|
* feed the agent's own task, which continues. `handover` wins over this.
|
|
56
56
|
*/
|
|
57
57
|
subTask?: boolean;
|
|
58
|
+
/**
|
|
59
|
+
* `bir`'s report tool, by the name the agent calls it, when the session has
|
|
60
|
+
* it (`mcp__bir__report_wrong_result`): the words that forbid every tool
|
|
61
|
+
* call make one exception for it ({@link withReportException}). Only the
|
|
62
|
+
* live and mixed bundles forbid tools; a hand-over and a sub-task already
|
|
63
|
+
* leave the agent its tools.
|
|
64
|
+
*/
|
|
65
|
+
reportTool?: string;
|
|
58
66
|
}
|
|
59
67
|
/**
|
|
60
68
|
* Compose the bundle. `maxChars` is a hard ceiling on the returned string.
|
package/dist/replay/bundle.js
CHANGED
|
@@ -70,6 +70,24 @@ const MIXED_HEADER = "[BaseInstRunner calculated replay] A known-good tool seque
|
|
|
70
70
|
"because their tool cannot be executed here — treat them as possibly out of date. " +
|
|
71
71
|
"Use them to answer the user's request now — do NOT call any tools.";
|
|
72
72
|
const MIXED_FOOTER = "\n\nAnswer the user's request using the results above. Do not call any tools.";
|
|
73
|
+
/**
|
|
74
|
+
* The one exception the "do NOT call any tools" lines make, when the session
|
|
75
|
+
* has `bir`'s own report tool (the service's review of
|
|
76
|
+
* calcscenarioautocorrect.md, item 19). A plan that took the wrong half of an
|
|
77
|
+
* "if … otherwise" is caught only when the agent says so, and these lines
|
|
78
|
+
* told it not even to check: an agent that sees the results do not answer
|
|
79
|
+
* the request may now say so, in one call that undoes nothing, and then tell
|
|
80
|
+
* the user. Without the tool the lines stay exactly as they were.
|
|
81
|
+
*/
|
|
82
|
+
function withReportException(text, tool) {
|
|
83
|
+
if (!tool)
|
|
84
|
+
return text;
|
|
85
|
+
const except = `do NOT call any tools, except: if these results do not answer the request (for example, ` +
|
|
86
|
+
`the wrong branch of an if/otherwise was taken), call ${tool} with why, then tell the user.`;
|
|
87
|
+
return text
|
|
88
|
+
.replace("do NOT call any tools.", except)
|
|
89
|
+
.replace("Do not call any tools.", `Do not call any tools (except ${tool}, as said above).`);
|
|
90
|
+
}
|
|
73
91
|
/**
|
|
74
92
|
* The scenario stopped before the task was done (fallbk.md), so the model must
|
|
75
93
|
* NOT be told to answer without tools — the rest of the work is its own. The
|
|
@@ -108,16 +126,12 @@ export function assembleBundle(entries, maxChars, opts = {}) {
|
|
|
108
126
|
? HANDOVER_HEADER
|
|
109
127
|
: opts.subTask
|
|
110
128
|
? SUBTASK_HEADER
|
|
111
|
-
: anyRecorded
|
|
112
|
-
? MIXED_HEADER
|
|
113
|
-
: LIVE_HEADER;
|
|
129
|
+
: withReportException(anyRecorded ? MIXED_HEADER : LIVE_HEADER, opts.reportTool);
|
|
114
130
|
const footer = opts.handover
|
|
115
131
|
? HANDOVER_FOOTER
|
|
116
132
|
: opts.subTask
|
|
117
133
|
? SUBTASK_FOOTER
|
|
118
|
-
: anyRecorded
|
|
119
|
-
? MIXED_FOOTER
|
|
120
|
-
: LIVE_FOOTER;
|
|
134
|
+
: withReportException(anyRecorded ? MIXED_FOOTER : LIVE_FOOTER, opts.reportTool);
|
|
121
135
|
const fixedCost = header.length +
|
|
122
136
|
footer.length +
|
|
123
137
|
entries.reduce((sum, e) => sum +
|
|
@@ -13,7 +13,7 @@
|
|
|
13
13
|
*/
|
|
14
14
|
import type { ExecutionReport, MatchKey, MatchSegment, MatchVerdict } from "../record/recorder.js";
|
|
15
15
|
import { type ReplayMode, type StepReach } from "./coverage.js";
|
|
16
|
-
import { deriveParameters, type DeriveContext } from "./derive.js";
|
|
16
|
+
import { deriveParameters, type AgentDay, type DeriveContext, type DeriveResult } from "./derive.js";
|
|
17
17
|
import { ProxyWorkQueue } from "./executor.js";
|
|
18
18
|
import { ScenarioReplayPlan } from "./plan.js";
|
|
19
19
|
import { type HostSinks } from "./safety.js";
|
|
@@ -22,6 +22,13 @@ import { type FlatStop } from "./flatten.js";
|
|
|
22
22
|
import { type ExecutionOutcome, type ExecutionStepResult, type FallbackKind, type SerializedScenario, type SerializedScenarioStep } from "./types.js";
|
|
23
23
|
/** The first-party tool a `direct` plan is delivered through (§6.3). */
|
|
24
24
|
export declare const DIRECT_TOOL_NAME = "mcp__bir__run_scenario";
|
|
25
|
+
/**
|
|
26
|
+
* `bir`'s own tool for saying a replay's results do not answer the request
|
|
27
|
+
* (bin/bir-scenario.ts; the service's review of calcscenarioautocorrect.md,
|
|
28
|
+
* item 19): the one call a finished bundle lets the agent make, where the
|
|
29
|
+
* session has it ({@link ReplayState.reportTool}).
|
|
30
|
+
*/
|
|
31
|
+
export declare const REPORT_TOOL_NAME = "mcp__bir__report_wrong_result";
|
|
25
32
|
/** How long a `/proxy/poll` is held open before it answers empty. */
|
|
26
33
|
export declare const POLL_HOLD_MS = 25000;
|
|
27
34
|
/**
|
|
@@ -501,6 +508,25 @@ export interface ReplayState {
|
|
|
501
508
|
* is waiting on the service, and obeys its breaker (util/service-health.ts).
|
|
502
509
|
*/
|
|
503
510
|
deriveOnService?: boolean;
|
|
511
|
+
/** The day the plan's values were read for (issue 1212): the agent's own, or this machine's. */
|
|
512
|
+
deriveDay?: string;
|
|
513
|
+
/** The line that told the agent its day, as the reading was sent it. */
|
|
514
|
+
deriveLine?: string;
|
|
515
|
+
/** The agent's own day, read again when the plan starts. */
|
|
516
|
+
agentDayNow?: () => AgentDay | undefined;
|
|
517
|
+
/** Read the plan's values again for another day; set when the plan armed. */
|
|
518
|
+
rederive?: (day: AgentDay) => Promise<DeriveResult>;
|
|
519
|
+
/** What the second reading left unnamed, once it has answered. */
|
|
520
|
+
rederivedMissing?: string[];
|
|
521
|
+
/**
|
|
522
|
+
* Whether this plan's session has `bir`'s report tool ({@link
|
|
523
|
+
* REPORT_TOOL_NAME}) right now, asked when a bundle is laid out: set by the
|
|
524
|
+
* control server, from what the session's `bir` server said of itself. A
|
|
525
|
+
* bundle that forbids every tool call then makes one exception for it
|
|
526
|
+
* (bundle.ts). Absent — `bir replay`, another host, an older server — the
|
|
527
|
+
* words stay as they were.
|
|
528
|
+
*/
|
|
529
|
+
reportTool?: () => boolean;
|
|
504
530
|
/**
|
|
505
531
|
* Set when the session's host is not Claude Code (see {@link HostContext}).
|
|
506
532
|
* Such a plan is always `steer` with every step `live`, and nothing here
|
|
@@ -1105,6 +1131,12 @@ export declare class ReplayController {
|
|
|
1105
1131
|
* cut off by its clock (`uncertain`), a step that ran and broke (`failed`).
|
|
1106
1132
|
*/
|
|
1107
1133
|
private executeBuiltin;
|
|
1134
|
+
/**
|
|
1135
|
+
* The report tool a bundle may name, when this plan's session has it
|
|
1136
|
+
* ({@link ReplayState.reportTool}); undefined leaves the bundle's words as
|
|
1137
|
+
* they always were. Never for another host: its adapter reads no bundle.
|
|
1138
|
+
*/
|
|
1139
|
+
private reportToolFor;
|
|
1108
1140
|
/** Where recorded outputs come from, when they may stand in at all. */
|
|
1109
1141
|
private recordedSource;
|
|
1110
1142
|
/**
|