@koda-sl/baker-bridge 0.155.0-dev.25961ab20 → 0.155.0-dev.d0f8a7fd4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +12 -0
- package/dist/hono/agent.d.ts.map +1 -1
- package/dist/hono/agent.js +6 -2
- package/dist/hono/agent.js.map +1 -1
- package/dist/hono/bridge-job.d.ts +14 -0
- package/dist/hono/bridge-job.d.ts.map +1 -1
- package/dist/hono/bridge-job.js +41 -2
- package/dist/hono/bridge-job.js.map +1 -1
- package/dist/hono/bridge-job.test.js +85 -0
- package/dist/hono/bridge-job.test.js.map +1 -1
- package/dist/hono/convex.d.ts +2 -1
- package/dist/hono/convex.d.ts.map +1 -1
- package/dist/hono/convex.js +4 -0
- package/dist/hono/convex.js.map +1 -1
- package/dist/hono/main-sync.d.ts +7 -0
- package/dist/hono/main-sync.d.ts.map +1 -1
- package/dist/hono/main-sync.js +39 -0
- package/dist/hono/main-sync.js.map +1 -1
- package/dist/hono/main-sync.test.js +29 -1
- package/dist/hono/main-sync.test.js.map +1 -1
- package/dist/hono/relay.d.ts +10 -1
- package/dist/hono/relay.d.ts.map +1 -1
- package/dist/hono/relay.js +4 -0
- package/dist/hono/relay.js.map +1 -1
- package/dist/hono/relay.test.js +11 -0
- package/dist/hono/relay.test.js.map +1 -1
- package/dist/hono/served-model.d.ts +23 -0
- package/dist/hono/served-model.d.ts.map +1 -0
- package/dist/hono/served-model.js +82 -0
- package/dist/hono/served-model.js.map +1 -0
- package/dist/hono/served-model.test.d.ts +2 -0
- package/dist/hono/served-model.test.d.ts.map +1 -0
- package/dist/hono/served-model.test.js +29 -0
- package/dist/hono/served-model.test.js.map +1 -0
- package/dist/hono/session-continuity.integration.test.js +1 -0
- package/dist/hono/session-continuity.integration.test.js.map +1 -1
- package/dist/hono/session-usage-baseline.d.ts +1 -0
- package/dist/hono/session-usage-baseline.d.ts.map +1 -1
- package/dist/hono/session-usage-baseline.js +17 -4
- package/dist/hono/session-usage-baseline.js.map +1 -1
- package/dist/review/digest.d.ts +45 -1
- package/dist/review/digest.d.ts.map +1 -1
- package/dist/review/digest.js +385 -63
- package/dist/review/digest.js.map +1 -1
- package/dist/review/digest.test.js +235 -1
- package/dist/review/digest.test.js.map +1 -1
- package/dist/review/evidence.d.ts.map +1 -1
- package/dist/review/evidence.js +1 -1
- package/dist/review/evidence.js.map +1 -1
- package/package.json +1 -1
package/dist/review/digest.js
CHANGED
|
@@ -15,7 +15,9 @@
|
|
|
15
15
|
* it could NOT find, so missing evidence is a fact before the model starts
|
|
16
16
|
* rather than a silence it fills in.
|
|
17
17
|
*/
|
|
18
|
+
import { createHash } from "node:crypto";
|
|
18
19
|
import { imageSize } from "image-size";
|
|
20
|
+
import { parse } from "unbash";
|
|
19
21
|
import { languageMismatch, marketerWords, messageLanguage } from "./reply-language.js";
|
|
20
22
|
/** Parse a `.jsonl` body, dropping lines that are not JSON rather than failing. */
|
|
21
23
|
export function parseTranscript(body) {
|
|
@@ -289,6 +291,49 @@ function clip(text, max = TRUNCATE) {
|
|
|
289
291
|
function currentTurn(acc) {
|
|
290
292
|
return acc.turns[acc.turns.length - 1];
|
|
291
293
|
}
|
|
294
|
+
/** One conversation inside a thread: the main agent, or one subagent. */
|
|
295
|
+
function conversationKey(threadId, agentId) {
|
|
296
|
+
return `${threadId ?? ""}\u0000${agentId ?? ""}`;
|
|
297
|
+
}
|
|
298
|
+
function isSubagentLine(line) {
|
|
299
|
+
return line.isSidechain === true || line.isSubagent === true;
|
|
300
|
+
}
|
|
301
|
+
/**
|
|
302
|
+
* The turn a line belongs to.
|
|
303
|
+
*
|
|
304
|
+
* A subagent's own file is read after the whole session file it was spawned
|
|
305
|
+
* in, so the turn open by then is that session's last one. Charged to it, a
|
|
306
|
+
* fan-out in turn 1 was listed under the re-render turn 2 — its refusals, its
|
|
307
|
+
* denials and its parent's closing recap as their intent. A subagent belongs to
|
|
308
|
+
* the last turn of its thread that began before the prompt that started it, or
|
|
309
|
+
* before the later prompt that continued it.
|
|
310
|
+
*/
|
|
311
|
+
function turnIndexOf(acc, line) {
|
|
312
|
+
const current = currentTurn(acc)?.index;
|
|
313
|
+
if (!isSubagentLine(line)) {
|
|
314
|
+
return current;
|
|
315
|
+
}
|
|
316
|
+
const key = line.agentId === undefined ? undefined : conversationKey(line.threadId, line.agentId);
|
|
317
|
+
const known = key === undefined ? undefined : acc.subagentTurns.get(key);
|
|
318
|
+
if (known !== undefined) {
|
|
319
|
+
return known;
|
|
320
|
+
}
|
|
321
|
+
const at = stampOf(line.timestamp);
|
|
322
|
+
if (at === undefined) {
|
|
323
|
+
return current;
|
|
324
|
+
}
|
|
325
|
+
let spawned;
|
|
326
|
+
for (const turn of acc.turns) {
|
|
327
|
+
const began = stampOf(turn.timestamp);
|
|
328
|
+
if (turn.threadId === line.threadId && began !== undefined && began <= at) {
|
|
329
|
+
spawned = turn.index;
|
|
330
|
+
}
|
|
331
|
+
}
|
|
332
|
+
if (key !== undefined && spawned !== undefined) {
|
|
333
|
+
acc.subagentTurns.set(key, spawned);
|
|
334
|
+
}
|
|
335
|
+
return spawned ?? current;
|
|
336
|
+
}
|
|
292
337
|
/**
|
|
293
338
|
* Close the open turn at the latest line that still belongs to it.
|
|
294
339
|
*
|
|
@@ -302,7 +347,8 @@ function currentTurn(acc) {
|
|
|
302
347
|
*/
|
|
303
348
|
function noteTurnEnd(acc, line) {
|
|
304
349
|
const at = stampOf(line.timestamp);
|
|
305
|
-
const
|
|
350
|
+
const index = turnIndexOf(acc, line);
|
|
351
|
+
const turn = index === undefined ? undefined : acc.turns[index];
|
|
306
352
|
if (at === undefined || turn === undefined) {
|
|
307
353
|
return;
|
|
308
354
|
}
|
|
@@ -420,26 +466,38 @@ export function namedGuard(message) {
|
|
|
420
466
|
}
|
|
421
467
|
return undefined;
|
|
422
468
|
}
|
|
423
|
-
/**
|
|
424
|
-
|
|
469
|
+
/**
|
|
470
|
+
* Ledger a failure: its class, the guard row, and the turn's denial count when a
|
|
471
|
+
* guard refused it. A call can fail more than once — each `baker` call one
|
|
472
|
+
* command line ran — and its own class and message are its first failure's.
|
|
473
|
+
*/
|
|
474
|
+
function absorbFailure(acc, call, message, failureClass = classifyFailure(message)) {
|
|
425
475
|
const turn = acc.turns[call.turn];
|
|
426
|
-
|
|
427
|
-
call.failureClass
|
|
428
|
-
if (turn &&
|
|
476
|
+
call.isError = true;
|
|
477
|
+
call.failureClass ??= failureClass;
|
|
478
|
+
if (turn && failureClass === "refused") {
|
|
429
479
|
turn.denials += 1;
|
|
430
480
|
}
|
|
431
|
-
acc.failureMessages.
|
|
481
|
+
if (!acc.failureMessages.has(call.toolUseId)) {
|
|
482
|
+
acc.failureMessages.set(call.toolUseId, clip(message, 400));
|
|
483
|
+
}
|
|
432
484
|
const guard = namedGuard(message);
|
|
485
|
+
// A subagent's words, never the turn's: the turn's are its parent's.
|
|
486
|
+
const intent = call.isSidechain
|
|
487
|
+
? acc.subagentWords.get(conversationKey(call.threadId, call.agentId))
|
|
488
|
+
: turn?.assistantText || undefined;
|
|
433
489
|
acc.guards.push({
|
|
434
490
|
turn: call.turn,
|
|
435
491
|
...(call.threadId === undefined ? {} : { threadId: call.threadId }),
|
|
436
492
|
toolUseId: call.toolUseId,
|
|
437
493
|
tool: call.tool,
|
|
438
494
|
attempted: call.summary,
|
|
495
|
+
inputKey: call.inputKey,
|
|
439
496
|
message: clip(message, 400),
|
|
440
|
-
refusal:
|
|
497
|
+
refusal: failureClass === "refused",
|
|
441
498
|
...(guard === undefined ? {} : { guard }),
|
|
442
|
-
...(
|
|
499
|
+
...(intent ? { intent: clip(intent, 300) } : {}),
|
|
500
|
+
...(call.isSidechain && call.agentId !== undefined ? { agentId: call.agentId } : {}),
|
|
443
501
|
repeats: 1,
|
|
444
502
|
});
|
|
445
503
|
}
|
|
@@ -457,12 +515,11 @@ function absorbToolResults(acc, line) {
|
|
|
457
515
|
const seen = measureToolResult(block.content);
|
|
458
516
|
call.resultBytes = seen.bytes;
|
|
459
517
|
call.resultTokens = seen.tokens;
|
|
460
|
-
call.isError = block.is_error === true;
|
|
461
518
|
if (line.timestamp !== undefined) {
|
|
462
519
|
acc.answeredAt.set(call.toolUseId, line.timestamp);
|
|
463
520
|
}
|
|
464
|
-
if (
|
|
465
|
-
absorbFailure(acc, call, block);
|
|
521
|
+
if (block.is_error === true) {
|
|
522
|
+
absorbFailure(acc, call, typeof block.content === "string" ? block.content : String(block.content ?? ""));
|
|
466
523
|
}
|
|
467
524
|
}
|
|
468
525
|
}
|
|
@@ -490,6 +547,11 @@ function absorbUserLine(acc, line) {
|
|
|
490
547
|
absorbToolResults(acc, line);
|
|
491
548
|
const spoken = raw.replace(SYSTEM_REMINDER, "").trim();
|
|
492
549
|
const isToolResult = blocks(line).some((block) => block.type === "tool_result");
|
|
550
|
+
if (isSubagentLine(line) && !isToolResult && spoken.length > 0) {
|
|
551
|
+
// A prompt to a subagent: its spawn, or a later turn continuing it (`SendMessage`
|
|
552
|
+
// keeps the id), so the turn it belongs to is read again from this line.
|
|
553
|
+
acc.subagentTurns.delete(conversationKey(line.threadId, line.agentId));
|
|
554
|
+
}
|
|
493
555
|
if (line.isSidechain || line.isSubagent || isToolResult || spoken.length === 0) {
|
|
494
556
|
return;
|
|
495
557
|
}
|
|
@@ -509,6 +571,7 @@ function absorbUserLine(acc, line) {
|
|
|
509
571
|
outputTokens: 0,
|
|
510
572
|
cacheReadTokens: 0,
|
|
511
573
|
cacheWriteTokens: 0,
|
|
574
|
+
subagent: { inputTokens: 0, outputTokens: 0, cacheReadTokens: 0, cacheWriteTokens: 0 },
|
|
512
575
|
denials: 0,
|
|
513
576
|
compacted: false,
|
|
514
577
|
});
|
|
@@ -535,21 +598,25 @@ function absorbRequestUsage(acc, line, turnAt) {
|
|
|
535
598
|
acc.countedRequests.add(requestId);
|
|
536
599
|
}
|
|
537
600
|
const usage = line.message?.usage ?? {};
|
|
601
|
+
const subagent = isSubagentLine(line);
|
|
538
602
|
const request = {
|
|
539
603
|
turn: turnAt,
|
|
540
604
|
...(line.message?.model === undefined ? {} : { model: line.message.model }),
|
|
605
|
+
subagent,
|
|
606
|
+
...(subagent && line.agentId !== undefined ? { agentId: line.agentId } : {}),
|
|
541
607
|
inputTokens: usage.input_tokens ?? 0,
|
|
542
608
|
outputTokens: usage.output_tokens ?? 0,
|
|
543
609
|
cacheReadTokens: usage.cache_read_input_tokens ?? 0,
|
|
544
610
|
cacheWriteTokens: usage.cache_creation_input_tokens ?? 0,
|
|
545
611
|
};
|
|
546
612
|
acc.requests.push(request);
|
|
547
|
-
const turn =
|
|
613
|
+
const turn = acc.turns[turnAt];
|
|
548
614
|
if (turn) {
|
|
549
|
-
turn.
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
615
|
+
const usage = subagent ? turn.subagent : turn;
|
|
616
|
+
usage.inputTokens += request.inputTokens;
|
|
617
|
+
usage.outputTokens += request.outputTokens;
|
|
618
|
+
usage.cacheReadTokens += request.cacheReadTokens;
|
|
619
|
+
usage.cacheWriteTokens += request.cacheWriteTokens;
|
|
553
620
|
}
|
|
554
621
|
}
|
|
555
622
|
function buildToolCall(input) {
|
|
@@ -561,6 +628,10 @@ function buildToolCall(input) {
|
|
|
561
628
|
toolUseId: input.block.id ?? `${input.turnAt}:${input.index}`,
|
|
562
629
|
tool,
|
|
563
630
|
summary: clip(summarizeInput(tool, input.block.input), 400),
|
|
631
|
+
inputKey: createHash("sha256")
|
|
632
|
+
.update(JSON.stringify(input.block.input ?? {}))
|
|
633
|
+
.digest("hex")
|
|
634
|
+
.slice(0, 16),
|
|
564
635
|
...(bakerVerb ? { bakerVerb } : {}),
|
|
565
636
|
resultBytes: 0,
|
|
566
637
|
resultTokens: 0,
|
|
@@ -581,6 +652,12 @@ function userFacingMessage(turn, text, marketerLanguage) {
|
|
|
581
652
|
...(mismatch === undefined ? {} : { languageMismatch: mismatch }),
|
|
582
653
|
};
|
|
583
654
|
}
|
|
655
|
+
/** A Bash call's whole command, which placing a failed `baker` call on it reads. */
|
|
656
|
+
function rememberCommand(acc, call, use) {
|
|
657
|
+
if (call.tool === "Bash" && typeof use.input?.command === "string") {
|
|
658
|
+
acc.commands.set(call.toolUseId, use.input.command);
|
|
659
|
+
}
|
|
660
|
+
}
|
|
584
661
|
/** Ledger each tool call a response asked for, under the response it came in. */
|
|
585
662
|
function absorbToolUses(acc, line, turnAt, isSidechain) {
|
|
586
663
|
const uses = blocks(line).filter((block) => block.type === "tool_use");
|
|
@@ -597,6 +674,7 @@ function absorbToolUses(acc, line, turnAt, isSidechain) {
|
|
|
597
674
|
});
|
|
598
675
|
acc.toolCalls.push(call);
|
|
599
676
|
acc.pending.set(call.toolUseId, call);
|
|
677
|
+
rememberCommand(acc, call, use);
|
|
600
678
|
if (requestId !== undefined) {
|
|
601
679
|
acc.batches.set(requestId, [...(acc.batches.get(requestId) ?? []), call]);
|
|
602
680
|
}
|
|
@@ -608,12 +686,16 @@ function absorbToolUses(acc, line, turnAt, isSidechain) {
|
|
|
608
686
|
return uses.length;
|
|
609
687
|
}
|
|
610
688
|
function absorbAssistantLine(acc, line) {
|
|
611
|
-
const turnAt =
|
|
689
|
+
const turnAt = turnIndexOf(acc, line) ?? 0;
|
|
612
690
|
absorbRequestUsage(acc, line, turnAt);
|
|
613
|
-
const isSidechain = line
|
|
691
|
+
const isSidechain = isSubagentLine(line);
|
|
614
692
|
const uses = absorbToolUses(acc, line, turnAt, isSidechain);
|
|
615
693
|
const text = textOf(line);
|
|
616
|
-
if (text.trim().length === 0
|
|
694
|
+
if (text.trim().length === 0) {
|
|
695
|
+
return;
|
|
696
|
+
}
|
|
697
|
+
if (isSidechain) {
|
|
698
|
+
acc.subagentWords.set(conversationKey(line.threadId, line.agentId), text);
|
|
617
699
|
return;
|
|
618
700
|
}
|
|
619
701
|
const turn = currentTurn(acc);
|
|
@@ -667,6 +749,122 @@ function timeCalls(acc) {
|
|
|
667
749
|
call.durationMs = took;
|
|
668
750
|
}
|
|
669
751
|
}
|
|
752
|
+
return startOf;
|
|
753
|
+
}
|
|
754
|
+
/** Whether the CLI says an invocation failed: a non-zero exit, or an `ok: false` envelope it printed. */
|
|
755
|
+
function cliFailed(invocation) {
|
|
756
|
+
return (invocation.exitCode !== undefined && invocation.exitCode !== 0) || invocation.error !== undefined;
|
|
757
|
+
}
|
|
758
|
+
/**
|
|
759
|
+
* What the CLI said about a failed invocation, and nothing the agent typed: the
|
|
760
|
+
* class is read from this, so `--title "Fix checkout timeout"` is not a timeout.
|
|
761
|
+
*/
|
|
762
|
+
function cliErrorOf(invocation) {
|
|
763
|
+
const exit = invocation.exitCode === undefined ? "failed" : `exited ${invocation.exitCode}`;
|
|
764
|
+
return invocation.error === undefined ? exit : `${invocation.error.code}: ${invocation.error.message} — ${exit}`;
|
|
765
|
+
}
|
|
766
|
+
/** The error first, so clipping a long command never takes the code with it. */
|
|
767
|
+
function describeCliFailure(invocation) {
|
|
768
|
+
return invocation.error === undefined
|
|
769
|
+
? `\`baker ${invocation.argv.join(" ")}\` ${cliErrorOf(invocation)}`
|
|
770
|
+
: `${cliErrorOf(invocation)} — \`baker ${invocation.argv.join(" ")}\``;
|
|
771
|
+
}
|
|
772
|
+
/**
|
|
773
|
+
* The arguments of every `baker` command a shell command line runs, read the way
|
|
774
|
+
* the shell reads it: dequoted, inside pipes, lists, loops and substitutions,
|
|
775
|
+
* never inside a heredoc body or a quoted string.
|
|
776
|
+
*/
|
|
777
|
+
function bakerArgvsOf(command) {
|
|
778
|
+
const script = parse(command);
|
|
779
|
+
if (script.errors && script.errors.length > 0) {
|
|
780
|
+
return [];
|
|
781
|
+
}
|
|
782
|
+
const argvs = [];
|
|
783
|
+
// Serialised because unbash materialises a word's parts only in `toJSON`.
|
|
784
|
+
JSON.stringify(script, (_key, value) => {
|
|
785
|
+
const node = value;
|
|
786
|
+
if (node?.type === "Command" && typeof node.name?.value === "string" && /(?:^|\/)baker$/.test(node.name.value)) {
|
|
787
|
+
argvs.push((node.suffix ?? []).map((word) => String(word.value ?? "")));
|
|
788
|
+
}
|
|
789
|
+
return value;
|
|
790
|
+
});
|
|
791
|
+
return argvs;
|
|
792
|
+
}
|
|
793
|
+
/** A word the shell expands before `baker` sees it, and so matches whatever argument it became. */
|
|
794
|
+
const EXPANDED_WORD = /[$*?[`]/;
|
|
795
|
+
/** Whether a command's own `baker` words are the ones an invocation received. */
|
|
796
|
+
function ranInvocation(words, argv) {
|
|
797
|
+
return words.length === argv.length && words.every((word, index) => word === argv[index] || EXPANDED_WORD.test(word));
|
|
798
|
+
}
|
|
799
|
+
/**
|
|
800
|
+
* The Bash call a `baker` invocation ran inside: the one running when it
|
|
801
|
+
* started whose command runs exactly that `baker` call.
|
|
802
|
+
*
|
|
803
|
+
* Nothing else places it. Subagents run side by side, so two running calls can
|
|
804
|
+
* share a platform (`baker ads meta campaigns`, `baker ads meta ads pause`), and
|
|
805
|
+
* a watcher or a hook calls `baker` while an unrelated command runs; a failure
|
|
806
|
+
* pinned on whichever call was running reports a success as failed and the
|
|
807
|
+
* failed call as a success. When no running call ran it, or more than one did,
|
|
808
|
+
* it is left unplaced.
|
|
809
|
+
*/
|
|
810
|
+
function callRunning(invocation, acc, startOf, argvsOf) {
|
|
811
|
+
const at = stampOf(invocation.startedAt);
|
|
812
|
+
if (at === undefined) {
|
|
813
|
+
return undefined;
|
|
814
|
+
}
|
|
815
|
+
const ran = acc.toolCalls.filter((call) => {
|
|
816
|
+
const from = stampOf(startOf.get(call.toolUseId));
|
|
817
|
+
const to = stampOf(acc.answeredAt.get(call.toolUseId));
|
|
818
|
+
return (call.tool === "Bash" &&
|
|
819
|
+
from !== undefined &&
|
|
820
|
+
to !== undefined &&
|
|
821
|
+
from <= at &&
|
|
822
|
+
at <= to &&
|
|
823
|
+
argvsOf(call).some((words) => ranInvocation(words, invocation.argv)));
|
|
824
|
+
});
|
|
825
|
+
return ran.length === 1 ? ran[0] : undefined;
|
|
826
|
+
}
|
|
827
|
+
/**
|
|
828
|
+
* Count a `baker` call that failed as a failure, whatever its shell returned.
|
|
829
|
+
*
|
|
830
|
+
* A Bash result is an error only when the whole command line exited non-zero,
|
|
831
|
+
* and `| head`, `> file` or a later `; echo` make that zero. Read from the tool
|
|
832
|
+
* result alone, a Session whose every refused or timed-out `baker` call was
|
|
833
|
+
* piped reported "none of them failed" — while the CLI's own record said why
|
|
834
|
+
* each one did. A failure no call can be placed on is returned, never dropped.
|
|
835
|
+
*/
|
|
836
|
+
function absorbCliFailures(acc, cli, startOf) {
|
|
837
|
+
const unplaced = [];
|
|
838
|
+
const failedIn = new Map();
|
|
839
|
+
const argvs = new Map();
|
|
840
|
+
const argvsOf = (call) => {
|
|
841
|
+
const known = argvs.get(call.toolUseId) ?? bakerArgvsOf(acc.commands.get(call.toolUseId) ?? "");
|
|
842
|
+
argvs.set(call.toolUseId, known);
|
|
843
|
+
return known;
|
|
844
|
+
};
|
|
845
|
+
for (const invocation of cli.filter(cliFailed)) {
|
|
846
|
+
const call = callRunning(invocation, acc, startOf, argvsOf);
|
|
847
|
+
if (call === undefined) {
|
|
848
|
+
unplaced.push(invocation);
|
|
849
|
+
}
|
|
850
|
+
else if (!call.isError) {
|
|
851
|
+
failedIn.set(call, [...(failedIn.get(call) ?? []), invocation]);
|
|
852
|
+
}
|
|
853
|
+
}
|
|
854
|
+
for (const [call, invocations] of failedIn) {
|
|
855
|
+
const failures = invocations.map((invocation) => ({
|
|
856
|
+
failureClass: classifyFailure(cliErrorOf(invocation)),
|
|
857
|
+
message: clip(describeCliFailure(invocation), 400),
|
|
858
|
+
durationMs: invocation.durationMs ?? 0,
|
|
859
|
+
}));
|
|
860
|
+
for (const failure of failures) {
|
|
861
|
+
absorbFailure(acc, call, failure.message, failure.failureClass);
|
|
862
|
+
}
|
|
863
|
+
acc.cliFailures.set(call.toolUseId, failures);
|
|
864
|
+
}
|
|
865
|
+
const order = new Map(acc.toolCalls.map((call, index) => [call.toolUseId, index]));
|
|
866
|
+
acc.guards.sort((a, b) => (order.get(a.toolUseId) ?? 0) - (order.get(b.toolUseId) ?? 0));
|
|
867
|
+
return unplaced;
|
|
670
868
|
}
|
|
671
869
|
/** The latest of some ISO timestamps, or nothing when none is given. */
|
|
672
870
|
function latest(stamps) {
|
|
@@ -698,7 +896,7 @@ function attributeGuardFollowUps(guards, calls) {
|
|
|
698
896
|
.find((call) => call.threadId === refused.threadId &&
|
|
699
897
|
call.isSidechain === refused.isSidechain &&
|
|
700
898
|
call.agentId === refused.agentId &&
|
|
701
|
-
call.
|
|
899
|
+
call.inputKey !== refused.inputKey);
|
|
702
900
|
if (after) {
|
|
703
901
|
guard.nextAction = `${after.tool}: ${clip(after.summary, 160)}`;
|
|
704
902
|
}
|
|
@@ -715,15 +913,15 @@ function attributeGuardFollowUps(guards, calls) {
|
|
|
715
913
|
function countGuardRepeats(guards) {
|
|
716
914
|
const counts = new Map();
|
|
717
915
|
for (const guard of guards) {
|
|
718
|
-
const key = `${guard.tool}\u0000${guard.
|
|
916
|
+
const key = `${guard.tool}\u0000${guard.inputKey}\u0000${guard.guard ?? guard.message}`;
|
|
719
917
|
counts.set(key, (counts.get(key) ?? 0) + 1);
|
|
720
918
|
}
|
|
721
919
|
for (const guard of guards) {
|
|
722
|
-
const key = `${guard.tool}\u0000${guard.
|
|
920
|
+
const key = `${guard.tool}\u0000${guard.inputKey}\u0000${guard.guard ?? guard.message}`;
|
|
723
921
|
guard.repeats = counts.get(key) ?? 1;
|
|
724
922
|
}
|
|
725
923
|
}
|
|
726
|
-
export function buildTranscriptDigest(lines) {
|
|
924
|
+
export function buildTranscriptDigest(lines, cli = []) {
|
|
727
925
|
const acc = {
|
|
728
926
|
turns: [],
|
|
729
927
|
toolCalls: [],
|
|
@@ -739,6 +937,10 @@ export function buildTranscriptDigest(lines) {
|
|
|
739
937
|
marketerLanguage: new Map(),
|
|
740
938
|
batches: new Map(),
|
|
741
939
|
answeredAt: new Map(),
|
|
940
|
+
subagentTurns: new Map(),
|
|
941
|
+
subagentWords: new Map(),
|
|
942
|
+
commands: new Map(),
|
|
943
|
+
cliFailures: new Map(),
|
|
742
944
|
};
|
|
743
945
|
for (const line of lines) {
|
|
744
946
|
const openTurns = acc.turns.length;
|
|
@@ -770,7 +972,8 @@ export function buildTranscriptDigest(lines) {
|
|
|
770
972
|
call.failureClass = "timeout";
|
|
771
973
|
acc.failureMessages.set(call.toolUseId, "No result was ever recorded for this call.");
|
|
772
974
|
}
|
|
773
|
-
timeCalls(acc);
|
|
975
|
+
const startOf = timeCalls(acc);
|
|
976
|
+
const unplacedCliFailures = absorbCliFailures(acc, cli, startOf);
|
|
774
977
|
attributeGuardFollowUps(acc.guards, acc.toolCalls);
|
|
775
978
|
countGuardRepeats(acc.guards);
|
|
776
979
|
const threadStamps = stampsByThread(lines);
|
|
@@ -789,7 +992,8 @@ export function buildTranscriptDigest(lines) {
|
|
|
789
992
|
.filter((call) => call.durationMs !== undefined)
|
|
790
993
|
.sort((a, b) => (b.durationMs ?? 0) - (a.durationMs ?? 0))
|
|
791
994
|
.slice(0, 15),
|
|
792
|
-
failures: aggregateFailures(acc.toolCalls, acc.failureMessages),
|
|
995
|
+
failures: aggregateFailures(acc.toolCalls, acc.failureMessages, acc.cliFailures),
|
|
996
|
+
unplacedCliFailures: unplacedCliFailures.map(describeCliFailure),
|
|
793
997
|
flaky: findFlakyCalls(acc.toolCalls),
|
|
794
998
|
introspectionCalls: acc.toolCalls.filter((call) => call.bakerVerb?.startsWith("baker schema") || /--help\b/.test(call.summary)),
|
|
795
999
|
hooks: [...acc.hooks.values()].sort((a, b) => b.fired - a.fired),
|
|
@@ -809,30 +1013,37 @@ export function buildTranscriptDigest(lines) {
|
|
|
809
1013
|
* a bad moment, eleven of them at the same bound is a default set for a network
|
|
810
1014
|
* this sandbox does not have.
|
|
811
1015
|
*/
|
|
812
|
-
function
|
|
1016
|
+
function ownFailure(call, failureClass, messages) {
|
|
1017
|
+
return { failureClass, message: messages.get(call.toolUseId) ?? "", durationMs: call.durationMs ?? 0 };
|
|
1018
|
+
}
|
|
1019
|
+
function aggregateFailures(calls, messages, cliFailures) {
|
|
813
1020
|
const map = new Map();
|
|
814
1021
|
for (const call of calls) {
|
|
815
1022
|
if (!call.isError || call.failureClass === undefined) {
|
|
816
1023
|
continue;
|
|
817
1024
|
}
|
|
818
|
-
|
|
819
|
-
const
|
|
820
|
-
|
|
821
|
-
|
|
822
|
-
|
|
823
|
-
|
|
824
|
-
|
|
825
|
-
|
|
826
|
-
|
|
827
|
-
|
|
828
|
-
|
|
829
|
-
|
|
830
|
-
entry.
|
|
831
|
-
|
|
832
|
-
|
|
833
|
-
|
|
1025
|
+
// A command line that ran several failed `baker` calls counts each of them.
|
|
1026
|
+
const failures = cliFailures.get(call.toolUseId) ?? [ownFailure(call, call.failureClass, messages)];
|
|
1027
|
+
for (const failure of failures) {
|
|
1028
|
+
const key = `${failure.failureClass}::${call.tool}`;
|
|
1029
|
+
const entry = map.get(key) ?? {
|
|
1030
|
+
failureClass: failure.failureClass,
|
|
1031
|
+
tool: call.tool,
|
|
1032
|
+
calls: 0,
|
|
1033
|
+
totalMs: 0,
|
|
1034
|
+
example: failure.message,
|
|
1035
|
+
turns: [],
|
|
1036
|
+
};
|
|
1037
|
+
entry.calls += 1;
|
|
1038
|
+
entry.totalMs += failure.durationMs;
|
|
1039
|
+
if (entry.example.length === 0) {
|
|
1040
|
+
entry.example = failure.message;
|
|
1041
|
+
}
|
|
1042
|
+
if (!entry.turns.includes(call.turn)) {
|
|
1043
|
+
entry.turns.push(call.turn);
|
|
1044
|
+
}
|
|
1045
|
+
map.set(key, entry);
|
|
834
1046
|
}
|
|
835
|
-
map.set(key, entry);
|
|
836
1047
|
}
|
|
837
1048
|
return [...map.values()].sort((a, b) => b.calls - a.calls);
|
|
838
1049
|
}
|
|
@@ -847,7 +1058,7 @@ function aggregateFailures(calls, messages) {
|
|
|
847
1058
|
function findFlakyCalls(calls) {
|
|
848
1059
|
const groups = new Map();
|
|
849
1060
|
for (const call of calls) {
|
|
850
|
-
const key = `${call.tool}\u0000${call.
|
|
1061
|
+
const key = `${call.tool}\u0000${call.inputKey}`;
|
|
851
1062
|
groups.set(key, [...(groups.get(key) ?? []), call]);
|
|
852
1063
|
}
|
|
853
1064
|
const out = [];
|
|
@@ -877,7 +1088,18 @@ function findFlakyCalls(calls) {
|
|
|
877
1088
|
* and this is what says where in the Session to look.
|
|
878
1089
|
*/
|
|
879
1090
|
function emptyThread(threadId) {
|
|
880
|
-
return {
|
|
1091
|
+
return {
|
|
1092
|
+
threadId,
|
|
1093
|
+
turns: 0,
|
|
1094
|
+
toolCalls: 0,
|
|
1095
|
+
resultBytes: 0,
|
|
1096
|
+
resultTokens: 0,
|
|
1097
|
+
subagentToolCalls: 0,
|
|
1098
|
+
subagentResultTokens: 0,
|
|
1099
|
+
errors: 0,
|
|
1100
|
+
denials: 0,
|
|
1101
|
+
workingMs: 0,
|
|
1102
|
+
};
|
|
881
1103
|
}
|
|
882
1104
|
/** Milliseconds parsed off an ISO timestamp, or nothing. */
|
|
883
1105
|
function stampOf(value) {
|
|
@@ -914,6 +1136,8 @@ function aggregateByThread(turns, calls, stamps) {
|
|
|
914
1136
|
entry.toolCalls += 1;
|
|
915
1137
|
entry.resultBytes += call.resultBytes;
|
|
916
1138
|
entry.resultTokens += call.resultTokens;
|
|
1139
|
+
entry.subagentToolCalls += call.isSidechain ? 1 : 0;
|
|
1140
|
+
entry.subagentResultTokens += call.isSidechain ? call.resultTokens : 0;
|
|
917
1141
|
entry.errors += call.isError ? 1 : 0;
|
|
918
1142
|
}
|
|
919
1143
|
for (const [threadId, values] of stamps) {
|
|
@@ -930,14 +1154,18 @@ function aggregateByTool(calls) {
|
|
|
930
1154
|
const entry = map.get(call.tool) ?? {
|
|
931
1155
|
tool: call.tool,
|
|
932
1156
|
calls: 0,
|
|
1157
|
+
subagentCalls: 0,
|
|
933
1158
|
totalBytes: 0,
|
|
934
1159
|
totalTokens: 0,
|
|
1160
|
+
subagentTokens: 0,
|
|
935
1161
|
maxBytes: 0,
|
|
936
1162
|
errors: 0,
|
|
937
1163
|
};
|
|
938
1164
|
entry.calls += 1;
|
|
1165
|
+
entry.subagentCalls += call.isSidechain ? 1 : 0;
|
|
939
1166
|
entry.totalBytes += call.resultBytes;
|
|
940
1167
|
entry.totalTokens += call.resultTokens;
|
|
1168
|
+
entry.subagentTokens += call.isSidechain ? call.resultTokens : 0;
|
|
941
1169
|
entry.maxBytes = Math.max(entry.maxBytes, call.resultBytes);
|
|
942
1170
|
entry.errors += call.isError ? 1 : 0;
|
|
943
1171
|
map.set(call.tool, entry);
|
|
@@ -974,7 +1202,7 @@ function aggregateByVerb(calls) {
|
|
|
974
1202
|
function findRepeats(calls) {
|
|
975
1203
|
const seen = new Map();
|
|
976
1204
|
for (const call of calls) {
|
|
977
|
-
const key = `${call.tool}\u0000${call.
|
|
1205
|
+
const key = `${call.tool}\u0000${call.inputKey}`;
|
|
978
1206
|
const entry = seen.get(key);
|
|
979
1207
|
if (entry) {
|
|
980
1208
|
entry.extra += 1;
|
|
@@ -1029,6 +1257,9 @@ function absorbInvocation(entry, line) {
|
|
|
1029
1257
|
return;
|
|
1030
1258
|
}
|
|
1031
1259
|
entry.argv = line.argv;
|
|
1260
|
+
if (line.ts !== undefined) {
|
|
1261
|
+
entry.startedAt = line.ts;
|
|
1262
|
+
}
|
|
1032
1263
|
const verb = bakerVerbOf(`baker ${line.argv.join(" ")}`);
|
|
1033
1264
|
if (verb) {
|
|
1034
1265
|
entry.verb = verb;
|
|
@@ -1053,14 +1284,38 @@ function absorbExit(entry, line) {
|
|
|
1053
1284
|
entry.durationMs = line.durationMs;
|
|
1054
1285
|
}
|
|
1055
1286
|
}
|
|
1287
|
+
/** What it printed, and the error it reported when the text is an `ok: false` envelope. */
|
|
1288
|
+
function absorbOutput(entry, line) {
|
|
1289
|
+
if (typeof line.text !== "string") {
|
|
1290
|
+
return;
|
|
1291
|
+
}
|
|
1292
|
+
entry.printedBytes += Buffer.byteLength(line.text);
|
|
1293
|
+
const error = printedError(line.text);
|
|
1294
|
+
if (error !== undefined && entry.error === undefined) {
|
|
1295
|
+
entry.error = error;
|
|
1296
|
+
}
|
|
1297
|
+
}
|
|
1298
|
+
function printedError(text) {
|
|
1299
|
+
let parsed;
|
|
1300
|
+
try {
|
|
1301
|
+
parsed = JSON.parse(text);
|
|
1302
|
+
}
|
|
1303
|
+
catch {
|
|
1304
|
+
return undefined;
|
|
1305
|
+
}
|
|
1306
|
+
if (typeof parsed !== "object" || parsed === null || parsed.ok !== false) {
|
|
1307
|
+
return undefined;
|
|
1308
|
+
}
|
|
1309
|
+
const error = parsed.error;
|
|
1310
|
+
return {
|
|
1311
|
+
code: typeof error?.code === "string" ? error.code : "ERROR",
|
|
1312
|
+
message: typeof error?.message === "string" ? error.message : "",
|
|
1313
|
+
};
|
|
1314
|
+
}
|
|
1056
1315
|
const DEBUG_LINE_HANDLERS = {
|
|
1057
1316
|
invocation: absorbInvocation,
|
|
1058
1317
|
http: absorbHttpCall,
|
|
1059
|
-
output:
|
|
1060
|
-
if (typeof line.text === "string") {
|
|
1061
|
-
entry.printedBytes += Buffer.byteLength(line.text);
|
|
1062
|
-
}
|
|
1063
|
-
},
|
|
1318
|
+
output: absorbOutput,
|
|
1064
1319
|
exit: absorbExit,
|
|
1065
1320
|
};
|
|
1066
1321
|
/**
|
|
@@ -1141,12 +1396,25 @@ export function renderTurns(digest) {
|
|
|
1141
1396
|
"",
|
|
1142
1397
|
digest.threads.length === 0
|
|
1143
1398
|
? "No thread could be attributed — the transcripts carried no thread id."
|
|
1144
|
-
: table([
|
|
1399
|
+
: table([
|
|
1400
|
+
"thread",
|
|
1401
|
+
"turns",
|
|
1402
|
+
"tool calls",
|
|
1403
|
+
"subagent calls",
|
|
1404
|
+
"result bytes",
|
|
1405
|
+
"~main result tokens",
|
|
1406
|
+
"~subagent result tokens",
|
|
1407
|
+
"errors",
|
|
1408
|
+
"denials",
|
|
1409
|
+
"running",
|
|
1410
|
+
], digest.threads.map((thread) => [
|
|
1145
1411
|
thread.threadId,
|
|
1146
1412
|
thread.turns,
|
|
1147
1413
|
thread.toolCalls,
|
|
1414
|
+
thread.subagentToolCalls,
|
|
1148
1415
|
thread.resultBytes,
|
|
1149
|
-
thread.resultTokens,
|
|
1416
|
+
thread.resultTokens - thread.subagentResultTokens,
|
|
1417
|
+
thread.subagentResultTokens,
|
|
1150
1418
|
thread.errors,
|
|
1151
1419
|
thread.denials,
|
|
1152
1420
|
`${Math.round(thread.workingMs / 1000)}s`,
|
|
@@ -1154,7 +1422,25 @@ export function renderTurns(digest) {
|
|
|
1154
1422
|
"",
|
|
1155
1423
|
"## Every turn",
|
|
1156
1424
|
"",
|
|
1157
|
-
|
|
1425
|
+
"`in`, `out` and `cache read` are the main agent's context; `subagent` columns",
|
|
1426
|
+
"are what that turn's subagents spent, which never reached it.",
|
|
1427
|
+
"",
|
|
1428
|
+
table([
|
|
1429
|
+
"#",
|
|
1430
|
+
"thread",
|
|
1431
|
+
"at",
|
|
1432
|
+
"user",
|
|
1433
|
+
"assistant",
|
|
1434
|
+
"tools",
|
|
1435
|
+
"in",
|
|
1436
|
+
"out",
|
|
1437
|
+
"cache read",
|
|
1438
|
+
"subagent in",
|
|
1439
|
+
"subagent out",
|
|
1440
|
+
"subagent cache read",
|
|
1441
|
+
"denials",
|
|
1442
|
+
"compacted",
|
|
1443
|
+
], digest.turns.map((turn) => [
|
|
1158
1444
|
turn.index,
|
|
1159
1445
|
turn.threadId ?? "",
|
|
1160
1446
|
turn.timestamp ?? "",
|
|
@@ -1164,23 +1450,36 @@ export function renderTurns(digest) {
|
|
|
1164
1450
|
turn.inputTokens,
|
|
1165
1451
|
turn.outputTokens,
|
|
1166
1452
|
turn.cacheReadTokens,
|
|
1453
|
+
turn.subagent.inputTokens,
|
|
1454
|
+
turn.subagent.outputTokens,
|
|
1455
|
+
turn.subagent.cacheReadTokens,
|
|
1167
1456
|
turn.denials,
|
|
1168
1457
|
turn.compacted ? "yes" : "",
|
|
1169
1458
|
])),
|
|
1170
1459
|
"",
|
|
1171
1460
|
].join("\n");
|
|
1172
1461
|
}
|
|
1462
|
+
/** Who made a call: the main agent, or the subagent by its id. */
|
|
1463
|
+
function agentOf(call) {
|
|
1464
|
+
return call.isSidechain ? `subagent ${call.agentId ?? "(unnamed)"}` : "main";
|
|
1465
|
+
}
|
|
1173
1466
|
export function renderTools(digest) {
|
|
1174
1467
|
const sections = [
|
|
1175
1468
|
"# Tool calls",
|
|
1176
1469
|
"",
|
|
1177
1470
|
"## Per tool",
|
|
1178
1471
|
"",
|
|
1179
|
-
|
|
1472
|
+
"`main` is what the agent the marketer talks to called; `subagent` ran in a",
|
|
1473
|
+
"subagent, and its results never reached the main agent's context.",
|
|
1474
|
+
"",
|
|
1475
|
+
table(["tool", "calls", "main", "subagent", "total bytes", "~main tokens", "~subagent tokens", "max bytes", "errors"], digest.byTool.map((entry) => [
|
|
1180
1476
|
entry.tool,
|
|
1181
1477
|
entry.calls,
|
|
1478
|
+
entry.calls - entry.subagentCalls,
|
|
1479
|
+
entry.subagentCalls,
|
|
1182
1480
|
entry.totalBytes,
|
|
1183
|
-
entry.totalTokens,
|
|
1481
|
+
entry.totalTokens - entry.subagentTokens,
|
|
1482
|
+
entry.subagentTokens,
|
|
1184
1483
|
entry.maxBytes,
|
|
1185
1484
|
entry.errors,
|
|
1186
1485
|
])),
|
|
@@ -1208,7 +1507,14 @@ export function renderTools(digest) {
|
|
|
1208
1507
|
"",
|
|
1209
1508
|
"## Ten largest single results (by what the model received)",
|
|
1210
1509
|
"",
|
|
1211
|
-
table(["turn", "tool", "input", "bytes", "~tokens"], digest.largestResults.map((call) => [
|
|
1510
|
+
table(["turn", "agent", "tool", "input", "bytes", "~tokens"], digest.largestResults.map((call) => [
|
|
1511
|
+
call.turn,
|
|
1512
|
+
agentOf(call),
|
|
1513
|
+
call.tool,
|
|
1514
|
+
call.summary,
|
|
1515
|
+
call.resultBytes,
|
|
1516
|
+
call.resultTokens,
|
|
1517
|
+
])),
|
|
1212
1518
|
"",
|
|
1213
1519
|
"## Introspection calls (`baker schema`, `--help`)",
|
|
1214
1520
|
"",
|
|
@@ -1258,6 +1564,17 @@ export function renderFailures(digest) {
|
|
|
1258
1564
|
clip(entry.example, 160),
|
|
1259
1565
|
])),
|
|
1260
1566
|
"",
|
|
1567
|
+
...(digest.unplacedCliFailures.length === 0
|
|
1568
|
+
? []
|
|
1569
|
+
: [
|
|
1570
|
+
"## Failed `baker` calls no tool call can be placed on",
|
|
1571
|
+
"",
|
|
1572
|
+
"The CLI recorded these as failed, and when each started no loaded transcript",
|
|
1573
|
+
"had exactly one running Bash call whose command ran that `baker` call.",
|
|
1574
|
+
"",
|
|
1575
|
+
...digest.unplacedCliFailures.map((failure) => `- ${clip(failure, 300)}`),
|
|
1576
|
+
"",
|
|
1577
|
+
]),
|
|
1261
1578
|
"## The same call, two answers",
|
|
1262
1579
|
"",
|
|
1263
1580
|
"Identical tool and input, one attempt failing and another succeeding, with",
|
|
@@ -1279,8 +1596,9 @@ export function renderFailures(digest) {
|
|
|
1279
1596
|
"",
|
|
1280
1597
|
digest.slowest.length === 0
|
|
1281
1598
|
? "No call carried both a request and a result timestamp, so nothing here can be timed."
|
|
1282
|
-
: table(["turn", "tool", "took", "failed", "input"], digest.slowest.map((call) => [
|
|
1599
|
+
: table(["turn", "agent", "tool", "took", "failed", "input"], digest.slowest.map((call) => [
|
|
1283
1600
|
call.turn,
|
|
1601
|
+
agentOf(call),
|
|
1284
1602
|
call.tool,
|
|
1285
1603
|
secs(call.durationMs ?? 0),
|
|
1286
1604
|
call.isError ? (call.failureClass ?? "yes") : "",
|
|
@@ -1304,6 +1622,9 @@ export function renderHooks(digest) {
|
|
|
1304
1622
|
"",
|
|
1305
1623
|
].join("\n");
|
|
1306
1624
|
}
|
|
1625
|
+
function guardAgent(guard) {
|
|
1626
|
+
return guard.agentId === undefined ? "" : ` · subagent ${guard.agentId}`;
|
|
1627
|
+
}
|
|
1307
1628
|
export function renderGuards(digest) {
|
|
1308
1629
|
const refusals = digest.guards.filter((guard) => guard.refusal);
|
|
1309
1630
|
const errors = digest.guards.filter((guard) => !guard.refusal);
|
|
@@ -1323,13 +1644,14 @@ export function renderGuards(digest) {
|
|
|
1323
1644
|
: [
|
|
1324
1645
|
`${refusals.length} refusals.`,
|
|
1325
1646
|
"",
|
|
1326
|
-
"`Intent` is the agent's own words
|
|
1327
|
-
"
|
|
1647
|
+
"`Intent` is the refused agent's own words — the main agent's in that",
|
|
1648
|
+
"turn, or the subagent's own when a subagent was refused: what it thought",
|
|
1649
|
+
"it was doing. It is the transcript's only answer to why it went there;",
|
|
1328
1650
|
"the rest of the answer is in whatever doc, skill, hint or example",
|
|
1329
1651
|
"pointed it that way, which is yours to go and open.",
|
|
1330
1652
|
"",
|
|
1331
1653
|
...refusals.map((guard) => [
|
|
1332
|
-
`### Turn ${guard.turn} · ${guard.tool}${guard.guard ? ` · ${guard.guard}` : ""}${guard.repeats > 1 ? ` · refused ${guard.repeats}× this Session` : ""}`,
|
|
1654
|
+
`### Turn ${guard.turn}${guardAgent(guard)} · ${guard.tool}${guard.guard ? ` · ${guard.guard}` : ""}${guard.repeats > 1 ? ` · refused ${guard.repeats}× this Session` : ""}`,
|
|
1333
1655
|
"",
|
|
1334
1656
|
`Attempted: \`${guard.attempted}\``,
|
|
1335
1657
|
"",
|
|
@@ -1348,7 +1670,7 @@ export function renderGuards(digest) {
|
|
|
1348
1670
|
? "No tool returned an error."
|
|
1349
1671
|
: errors
|
|
1350
1672
|
.map((guard) => [
|
|
1351
|
-
`### Turn ${guard.turn} · ${guard.tool}${guard.repeats > 1 ? ` · ${guard.repeats}×` : ""}`,
|
|
1673
|
+
`### Turn ${guard.turn}${guardAgent(guard)} · ${guard.tool}${guard.repeats > 1 ? ` · ${guard.repeats}×` : ""}`,
|
|
1352
1674
|
"",
|
|
1353
1675
|
`Attempted: \`${guard.attempted}\``,
|
|
1354
1676
|
"",
|