agentfootprint 1.3.0 → 1.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,33 +1,35 @@
1
1
  /**
2
2
  * ExplainRecorder — collects grounding evidence + evaluation context during traversal.
3
3
  *
4
- * Captures:
5
- * - Sources (tool results ground truth)
6
- * - Claims (LLM output to verify)
7
- * - Decisions (tool calls — what the LLM chose to do)
8
- * - Context (system prompt, tool descriptions, messages — what the LLM had)
4
+ * Data is structured per-iteration for evaluation:
5
+ * iterations[0] = { context (what LLM had), decisions (tools chosen), sources (results), claim? }
6
+ * iterations[1] = { context (updated with tool results), decisions, sources, claim (final answer) }
9
7
  *
10
- * Everything for evaluation in one recorder: "what did it have?" + "what did it produce?"
8
+ * An evaluator walks iterations: for each one that has a claim, check if it's
9
+ * grounded in that iteration's sources + all prior sources.
11
10
  *
12
11
  * Usage:
13
12
  * const explain = new ExplainRecorder();
14
13
  * agent.recorder(explain);
15
14
  * await agent.run('Check order');
16
15
  *
17
- * explain.explain(); // { sources, claims, decisions, context, summary }
16
+ * const report = explain.explain();
17
+ * report.iterations; // per-iteration evaluation units
18
+ * report.sources; // flat convenience (all sources)
19
+ * report.claims; // flat convenience (all claims)
20
+ * report.context; // last context snapshot
18
21
  */
19
22
  export class ExplainRecorder {
20
23
  id;
21
- sources = [];
22
- claims = [];
23
- decisions = [];
24
+ // Per-iteration accumulation
25
+ iterations = [];
26
+ currentIterationDecisions = [];
27
+ currentIterationSources = [];
28
+ // Tracking state
24
29
  currentTurn = 0;
25
30
  input;
26
- lastModel;
27
- lastIteration = 0;
28
- lastSystemPrompt;
29
- lastToolDescriptions;
30
- lastMessages;
31
+ currentIteration = -1;
32
+ currentContext = {};
31
33
  constructor(id = 'explain-recorder') {
32
34
  this.id = id;
33
35
  }
@@ -36,83 +38,105 @@ export class ExplainRecorder {
36
38
  this.input = event.message;
37
39
  }
38
40
  onLLMCall(event) {
39
- this.lastModel = event.model;
40
- this.lastIteration = event.loopIteration;
41
- // Capture evaluation context from the LLM call event
42
- if (event.systemPrompt)
43
- this.lastSystemPrompt = event.systemPrompt;
44
- if (event.toolDescriptions)
45
- this.lastToolDescriptions = event.toolDescriptions;
46
- if (event.messages)
47
- this.lastMessages = event.messages;
41
+ // Flush previous iteration (if any) — it had no claim (tool-calling response)
42
+ if (this.currentIteration >= 0) {
43
+ this.flushIteration(null);
44
+ }
45
+ this.currentIteration = event.loopIteration;
46
+ // Snapshot context for THIS iteration
47
+ this.currentContext = {
48
+ input: this.input,
49
+ systemPrompt: event.systemPrompt ?? this.currentContext.systemPrompt,
50
+ availableTools: event.toolDescriptions
51
+ ? [...event.toolDescriptions]
52
+ : this.currentContext.availableTools,
53
+ messages: event.messages ? [...event.messages] : this.currentContext.messages,
54
+ model: event.model,
55
+ };
48
56
  }
49
57
  onToolCall(event) {
50
- this.sources.push({
58
+ const source = {
51
59
  toolName: event.toolName,
52
60
  args: { ...event.args },
53
61
  result: String(event.result.content),
54
62
  turnNumber: this.currentTurn,
55
- });
56
- this.decisions.push({
63
+ };
64
+ this.currentIterationSources.push(source);
65
+ const decision = {
57
66
  toolName: event.toolName,
58
67
  args: { ...event.args },
59
68
  latencyMs: event.latencyMs,
60
- });
69
+ };
70
+ this.currentIterationDecisions.push(decision);
61
71
  }
62
72
  onTurnComplete(event) {
63
- this.claims.push({
73
+ // Final iteration has a claim
74
+ const claim = {
64
75
  content: event.content,
65
- model: this.lastModel,
66
- iteration: this.lastIteration,
76
+ model: this.currentContext.model,
77
+ iteration: this.currentIteration,
78
+ };
79
+ this.flushIteration(claim);
80
+ }
81
+ /** Flush current iteration data into the iterations array. */
82
+ flushIteration(claim) {
83
+ this.iterations.push({
84
+ iteration: this.currentIteration,
85
+ context: { ...this.currentContext },
86
+ decisions: this.currentIterationDecisions.map((d) => ({ ...d, args: { ...d.args } })),
87
+ sources: [...this.currentIterationSources],
88
+ claim,
67
89
  });
90
+ this.currentIterationDecisions = [];
91
+ this.currentIterationSources = [];
68
92
  }
69
- /** Tool results the ground truth data. */
93
+ // ── Flat convenience accessors (backward compatible) ────
70
94
  getSources() {
71
- return [...this.sources];
95
+ return this.iterations.flatMap((it) => it.sources);
72
96
  }
73
- /** LLM responses — what it claimed. */
74
97
  getClaims() {
75
- return [...this.claims];
98
+ return this.iterations.filter((it) => it.claim).map((it) => it.claim);
76
99
  }
77
- /** Tool call decisions — what the LLM chose to do. */
78
100
  getDecisions() {
79
- return this.decisions.map((d) => ({ ...d, args: { ...d.args } }));
101
+ return this.iterations.flatMap((it) => it.decisions.map((d) => ({ ...d, args: { ...d.args } })));
80
102
  }
81
- /** What the LLM had when it made decisions — for evaluation. */
82
103
  getContext() {
83
- return {
84
- input: this.input,
85
- systemPrompt: this.lastSystemPrompt,
86
- availableTools: this.lastToolDescriptions ? [...this.lastToolDescriptions] : undefined,
87
- messages: this.lastMessages ? [...this.lastMessages] : undefined,
88
- model: this.lastModel,
89
- };
104
+ return { ...this.currentContext };
105
+ }
106
+ /** Per-iteration evaluation units. */
107
+ getIterations() {
108
+ return this.iterations.map((it) => ({
109
+ ...it,
110
+ context: { ...it.context },
111
+ decisions: it.decisions.map((d) => ({ ...d, args: { ...d.args } })),
112
+ sources: [...it.sources],
113
+ }));
90
114
  }
91
- /** Structured explanation — everything needed for evaluation. */
115
+ /** Structured explanation — flat + per-iteration, everything for evaluation. */
92
116
  explain() {
93
- const toolNames = [...new Set(this.decisions.map((d) => d.toolName))];
94
- const summary = this.sources.length === 0
117
+ const allDecisions = this.getDecisions();
118
+ const toolNames = [...new Set(allDecisions.map((d) => d.toolName))];
119
+ const allSources = this.getSources();
120
+ const summary = allSources.length === 0
95
121
  ? `Agent responded directly without calling tools.`
96
- : `Agent called ${toolNames.join(', ')} (${this.sources.length} call${this.sources.length > 1 ? 's' : ''}), then responded based on the results.`;
122
+ : `Agent called ${toolNames.join(', ')} (${allSources.length} call${allSources.length > 1 ? 's' : ''}), then responded based on the results.`;
97
123
  return {
98
- sources: this.getSources(),
124
+ iterations: this.getIterations(),
125
+ sources: allSources,
99
126
  claims: this.getClaims(),
100
- decisions: this.getDecisions(),
127
+ decisions: allDecisions,
101
128
  context: this.getContext(),
102
129
  summary,
103
130
  };
104
131
  }
105
132
  clear() {
106
- this.sources = [];
107
- this.claims = [];
108
- this.decisions = [];
133
+ this.iterations = [];
134
+ this.currentIterationDecisions = [];
135
+ this.currentIterationSources = [];
109
136
  this.currentTurn = 0;
110
137
  this.input = undefined;
111
- this.lastModel = undefined;
112
- this.lastIteration = 0;
113
- this.lastSystemPrompt = undefined;
114
- this.lastToolDescriptions = undefined;
115
- this.lastMessages = undefined;
138
+ this.currentIteration = -1;
139
+ this.currentContext = {};
116
140
  }
117
141
  }
118
142
  //# sourceMappingURL=ExplainRecorder.js.map
@@ -1 +1 @@
1
- {"version":3,"file":"ExplainRecorder.js","sourceRoot":"","sources":["../../../../src/recorders/v2/ExplainRecorder.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AAwDH,MAAM,OAAO,eAAe;IACjB,EAAE,CAAS;IACZ,OAAO,GAAiB,EAAE,CAAC;IAC3B,MAAM,GAAe,EAAE,CAAC;IACxB,SAAS,GAAoB,EAAE,CAAC;IAChC,WAAW,GAAG,CAAC,CAAC;IAChB,KAAK,CAAU;IACf,SAAS,CAAU;IACnB,aAAa,GAAG,CAAC,CAAC;IAClB,gBAAgB,CAAU;IAC1B,oBAAoB,CAAwD;IAC5E,YAAY,CAAqD;IAEzE,YAAY,EAAE,GAAG,kBAAkB;QACjC,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC;IACf,CAAC;IAED,WAAW,CAAC,KAAqB;QAC/B,IAAI,CAAC,WAAW,GAAG,KAAK,CAAC,UAAU,CAAC;QACpC,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC,OAAO,CAAC;IAC7B,CAAC;IAED,SAAS,CAAC,KAAmB;QAC3B,IAAI,CAAC,SAAS,GAAG,KAAK,CAAC,KAAK,CAAC;QAC7B,IAAI,CAAC,aAAa,GAAG,KAAK,CAAC,aAAa,CAAC;QACzC,qDAAqD;QACrD,IAAI,KAAK,CAAC,YAAY;YAAE,IAAI,CAAC,gBAAgB,GAAG,KAAK,CAAC,YAAY,CAAC;QACnE,IAAI,KAAK,CAAC,gBAAgB;YAAE,IAAI,CAAC,oBAAoB,GAAG,KAAK,CAAC,gBAAgB,CAAC;QAC/E,IAAI,KAAK,CAAC,QAAQ;YAAE,IAAI,CAAC,YAAY,GAAG,KAAK,CAAC,QAAQ,CAAC;IACzD,CAAC;IAED,UAAU,CAAC,KAAoB;QAC7B,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC;YAChB,QAAQ,EAAE,KAAK,CAAC,QAAQ;YACxB,IAAI,EAAE,EAAE,GAAG,KAAK,CAAC,IAAI,EAAE;YACvB,MAAM,EAAE,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,OAAO,CAAC;YACpC,UAAU,EAAE,IAAI,CAAC,WAAW;SAC7B,CAAC,CAAC;QAEH,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC;YAClB,QAAQ,EAAE,KAAK,CAAC,QAAQ;YACxB,IAAI,EAAE,EAAE,GAAG,KAAK,CAAC,IAAI,EAAE;YACvB,SAAS,EAAE,KAAK,CAAC,SAAS;SAC3B,CAAC,CAAC;IACL,CAAC;IAED,cAAc,CAAC,KAAwB;QACrC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC;YACf,OAAO,EAAE,KAAK,CAAC,OAAO;YACtB,KAAK,EAAE,IAAI,CAAC,SAAS;YACrB,SAAS,EAAE,IAAI,CAAC,aAAa;SAC9B,CAAC,CAAC;IACL,CAAC;IAED,4CAA4C;IAC5C,UAAU;QACR,OAAO,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,CAAC;IAC3B,CAAC;IAED,uCAAuC;IACvC,SAAS;QACP,OAAO,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC;IAC1B,CAAC;IAED,sDAAsD;IACtD,YAAY;QACV,OAAO,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,IAAI,EAAE,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,CAAC;IACpE,CAAC;IAED,gEAAgE;IAChE,UAAU;QACR,OAAO;YACL,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,YAAY,EAAE,IAAI,CAAC,gBAAgB;YACnC,cAAc,EAAE,IAAI,CAAC,oBAAoB,CAAC,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,oBAAoB,CAAC,CAAC,CAAC,CAAC,SAAS;YACtF,QAAQ,EAAE,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,SAAS;YAChE,KAAK,EAAE,IAAI,CAAC,SAAS;SACtB,CAAC;IACJ,CAAC;IAED,iEAAiE;IACjE,OAAO;QACL,MAAM,SAAS,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;QACtE,MAAM,OAAO,GACX,IAAI,CAAC,OAAO,CAAC,MAAM,KAAK,CAAC;YACvB,CAAC,CAAC,iDAAiD;YACnD,CAAC,CAAC,gBAAgB,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,IAAI,CAAC,OAAO,CAAC,MAAM,QAC1D,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAClC,yCAAyC,CAAC;QAEhD,OAAO;YACL,OAAO,EAAE,IAAI,CAAC,UAAU,EAAE;YAC1B,MAAM,EAAE,IAAI,CAAC,SAAS,EAAE;YACxB,SAAS,EAAE,IAAI,CAAC,YAAY,EAAE;YAC9B,OAAO,EAAE,IAAI,CAAC,UAAU,EAAE;YAC1B,OAAO;SACR,CAAC;IACJ,CAAC;IAED,KAAK;QACH,IAAI,CAAC,OAAO,GAAG,EAAE,CAAC;QAClB,IAAI,CAAC,MAAM,GAAG,EAAE,CAAC;QACjB,IAAI,CAAC,SAAS,GAAG,EAAE,CAAC;QACpB,IAAI,CAAC,WAAW,GAAG,CAAC,CAAC;QACrB,IAAI,CAAC,KAAK,GAAG,SAAS,CAAC;QACvB,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,aAAa,GAAG,CAAC,CAAC;QACvB,IAAI,CAAC,gBAAgB,GAAG,SAAS,CAAC;QAClC,IAAI,CAAC,oBAAoB,GAAG,SAAS,CAAC;QACtC,IAAI,CAAC,YAAY,GAAG,SAAS,CAAC;IAChC,CAAC;CACF"}
1
+ {"version":3,"file":"ExplainRecorder.js","sourceRoot":"","sources":["../../../../src/recorders/v2/ExplainRecorder.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AA2EH,MAAM,OAAO,eAAe;IACjB,EAAE,CAAS;IAEpB,6BAA6B;IACrB,UAAU,GAAoB,EAAE,CAAC;IACjC,yBAAyB,GAAoB,EAAE,CAAC;IAChD,uBAAuB,GAAiB,EAAE,CAAC;IAEnD,iBAAiB;IACT,WAAW,GAAG,CAAC,CAAC;IAChB,KAAK,CAAU;IACf,gBAAgB,GAAG,CAAC,CAAC,CAAC;IACtB,cAAc,GAAe,EAAE,CAAC;IAExC,YAAY,EAAE,GAAG,kBAAkB;QACjC,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC;IACf,CAAC;IAED,WAAW,CAAC,KAAqB;QAC/B,IAAI,CAAC,WAAW,GAAG,KAAK,CAAC,UAAU,CAAC;QACpC,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC,OAAO,CAAC;IAC7B,CAAC;IAED,SAAS,CAAC,KAAmB;QAC3B,8EAA8E;QAC9E,IAAI,IAAI,CAAC,gBAAgB,IAAI,CAAC,EAAE,CAAC;YAC/B,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC;QAC5B,CAAC;QAED,IAAI,CAAC,gBAAgB,GAAG,KAAK,CAAC,aAAa,CAAC;QAE5C,sCAAsC;QACtC,IAAI,CAAC,cAAc,GAAG;YACpB,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,YAAY,EAAE,KAAK,CAAC,YAAY,IAAI,IAAI,CAAC,cAAc,CAAC,YAAY;YACpE,cAAc,EAAE,KAAK,CAAC,gBAAgB;gBACpC,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC,gBAAgB,CAAC;gBAC7B,CAAC,CAAC,IAAI,CAAC,cAAc,CAAC,cAAc;YACtC,QAAQ,EAAE,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,cAAc,CAAC,QAAQ;YAC7E,KAAK,EAAE,KAAK,CAAC,KAAK;SACnB,CAAC;IACJ,CAAC;IAED,UAAU,CAAC,KAAoB;QAC7B,MAAM,MAAM,GAAe;YACzB,QAAQ,EAAE,KAAK,CAAC,QAAQ;YACxB,IAAI,EAAE,EAAE,GAAG,KAAK,CAAC,IAAI,EAAE;YACvB,MAAM,EAAE,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,OAAO,CAAC;YACpC,UAAU,EAAE,IAAI,CAAC,WAAW;SAC7B,CAAC;QACF,IAAI,CAAC,uBAAuB,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAE1C,MAAM,QAAQ,GAAkB;YAC9B,QAAQ,EAAE,KAAK,CAAC,QAAQ;YACxB,IAAI,EAAE,EAAE,GAAG,KAAK,CAAC,IAAI,EAAE;YACvB,SAAS,EAAE,KAAK,CAAC,SAAS;SAC3B,CAAC;QACF,IAAI,CAAC,yBAAyB,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IAChD,CAAC;IAED,cAAc,CAAC,KAAwB;QACrC,8BAA8B;QAC9B,MAAM,KAAK,GAAa;YACtB,OAAO,EAAE,KAAK,CAAC,OAAO;YACtB,KAAK,EAAE,IAAI,CAAC,cAAc,CAAC,KAAK;YAChC,SAAS,EAAE,IAAI,CAAC,gBAAgB;SACjC,CAAC;QACF,IAAI,CAAC,cAAc,CAAC,KAAK,CAAC,CAAC;IAC7B,CAAC;IAED,8DAA8D;IACtD,cAAc,CAAC,KAAsB;QAC3C,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;YACnB,SAAS,EAAE,IAAI,CAAC,gBAAgB;YAChC,OAAO,EAAE,EAAE,GAAG,IAAI,CAAC,cAAc,EAAE;YACnC,SAAS,EAAE,IAAI,CAAC,yBAAyB,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,IAAI,EAAE,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;YACrF,OAAO,EAAE,CAAC,GAAG,IAAI,CAAC,uBAAuB,CAAC;YAC1C,KAAK;SACN,CAAC,CAAC;QACH,IAAI,CAAC,yBAAyB,GAAG,EAAE,CAAC;QACpC,IAAI,CAAC,uBAAuB,GAAG,EAAE,CAAC;IACpC,CAAC;IAED,2DAA2D;IAE3D,UAAU;QACR,OAAO,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,OAAO,CAAC,CAAC;IACrD,CAAC;IAED,SAAS;QACP,OAAO,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,KAAM,CAAC,CAAC;IACzE,CAAC;IAED,YAAY;QACV,OAAO,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,EAAE,EAAE,EAAE,CACpC,EAAE,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,IAAI,EAAE,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,CACzD,CAAC;IACJ,CAAC;IAED,UAAU;QACR,OAAO,EAAE,GAAG,IAAI,CAAC,cAAc,EAAE,CAAC;IACpC,CAAC;IAED,sCAAsC;IACtC,aAAa;QACX,OAAO,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC;YAClC,GAAG,EAAE;YACL,OAAO,EAAE,EAAE,GAAG,EAAE,CAAC,OAAO,EAAE;YAC1B,SAAS,EAAE,EAAE,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,IAAI,EAAE,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;YACnE,OAAO,EAAE,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC;SACzB,CAAC,CAAC,CAAC;IACN,CAAC;IAED,gFAAgF;IAChF,OAAO;QACL,MAAM,YAAY,GAAG,IAAI,CAAC,YAAY,EAAE,CAAC;QACzC,MAAM,SAAS,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;QACpE,MAAM,UAAU,GAAG,IAAI,CAAC,UAAU,EAAE,CAAC;QACrC,MAAM,OAAO,GACX,UAAU,CAAC,MAAM,KAAK,CAAC;YACrB,CAAC,CAAC,iDAAiD;YACnD,CAAC,CAAC,gBAAgB,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,UAAU,CAAC,MAAM,QACxD,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAChC,yCAAyC,CAAC;QAEhD,OAAO;YACL,UAAU,EAAE,IAAI,CAAC,aAAa,EAAE;YAChC,OAAO,EAAE,UAAU;YACnB,MAAM,EAAE,IAAI,CAAC,SAAS,EAAE;YACxB,SAAS,EAAE,YAAY;YACvB,OAAO,EAAE,IAAI,CAAC,UAAU,EAAE;YAC1B,OAAO;SACR,CAAC;IACJ,CAAC;IAED,KAAK;QACH,IAAI,CAAC,UAAU,GAAG,EAAE,CAAC;QACrB,IAAI,CAAC,yBAAyB,GAAG,EAAE,CAAC;QACpC,IAAI,CAAC,uBAAuB,GAAG,EAAE,CAAC;QAClC,IAAI,CAAC,WAAW,GAAG,CAAC,CAAC;QACrB,IAAI,CAAC,KAAK,GAAG,SAAS,CAAC;QACvB,IAAI,CAAC,gBAAgB,GAAG,CAAC,CAAC,CAAC;QAC3B,IAAI,CAAC,cAAc,GAAG,EAAE,CAAC;IAC3B,CAAC;CACF"}
@@ -2,35 +2,37 @@
2
2
  /**
3
3
  * ExplainRecorder — collects grounding evidence + evaluation context during traversal.
4
4
  *
5
- * Captures:
6
- * - Sources (tool results ground truth)
7
- * - Claims (LLM output to verify)
8
- * - Decisions (tool calls — what the LLM chose to do)
9
- * - Context (system prompt, tool descriptions, messages — what the LLM had)
5
+ * Data is structured per-iteration for evaluation:
6
+ * iterations[0] = { context (what LLM had), decisions (tools chosen), sources (results), claim? }
7
+ * iterations[1] = { context (updated with tool results), decisions, sources, claim (final answer) }
10
8
  *
11
- * Everything for evaluation in one recorder: "what did it have?" + "what did it produce?"
9
+ * An evaluator walks iterations: for each one that has a claim, check if it's
10
+ * grounded in that iteration's sources + all prior sources.
12
11
  *
13
12
  * Usage:
14
13
  * const explain = new ExplainRecorder();
15
14
  * agent.recorder(explain);
16
15
  * await agent.run('Check order');
17
16
  *
18
- * explain.explain(); // { sources, claims, decisions, context, summary }
17
+ * const report = explain.explain();
18
+ * report.iterations; // per-iteration evaluation units
19
+ * report.sources; // flat convenience (all sources)
20
+ * report.claims; // flat convenience (all claims)
21
+ * report.context; // last context snapshot
19
22
  */
20
23
  Object.defineProperty(exports, "__esModule", { value: true });
21
24
  exports.ExplainRecorder = void 0;
22
25
  class ExplainRecorder {
23
26
  id;
24
- sources = [];
25
- claims = [];
26
- decisions = [];
27
+ // Per-iteration accumulation
28
+ iterations = [];
29
+ currentIterationDecisions = [];
30
+ currentIterationSources = [];
31
+ // Tracking state
27
32
  currentTurn = 0;
28
33
  input;
29
- lastModel;
30
- lastIteration = 0;
31
- lastSystemPrompt;
32
- lastToolDescriptions;
33
- lastMessages;
34
+ currentIteration = -1;
35
+ currentContext = {};
34
36
  constructor(id = 'explain-recorder') {
35
37
  this.id = id;
36
38
  }
@@ -39,83 +41,105 @@ class ExplainRecorder {
39
41
  this.input = event.message;
40
42
  }
41
43
  onLLMCall(event) {
42
- this.lastModel = event.model;
43
- this.lastIteration = event.loopIteration;
44
- // Capture evaluation context from the LLM call event
45
- if (event.systemPrompt)
46
- this.lastSystemPrompt = event.systemPrompt;
47
- if (event.toolDescriptions)
48
- this.lastToolDescriptions = event.toolDescriptions;
49
- if (event.messages)
50
- this.lastMessages = event.messages;
44
+ // Flush previous iteration (if any) — it had no claim (tool-calling response)
45
+ if (this.currentIteration >= 0) {
46
+ this.flushIteration(null);
47
+ }
48
+ this.currentIteration = event.loopIteration;
49
+ // Snapshot context for THIS iteration
50
+ this.currentContext = {
51
+ input: this.input,
52
+ systemPrompt: event.systemPrompt ?? this.currentContext.systemPrompt,
53
+ availableTools: event.toolDescriptions
54
+ ? [...event.toolDescriptions]
55
+ : this.currentContext.availableTools,
56
+ messages: event.messages ? [...event.messages] : this.currentContext.messages,
57
+ model: event.model,
58
+ };
51
59
  }
52
60
  onToolCall(event) {
53
- this.sources.push({
61
+ const source = {
54
62
  toolName: event.toolName,
55
63
  args: { ...event.args },
56
64
  result: String(event.result.content),
57
65
  turnNumber: this.currentTurn,
58
- });
59
- this.decisions.push({
66
+ };
67
+ this.currentIterationSources.push(source);
68
+ const decision = {
60
69
  toolName: event.toolName,
61
70
  args: { ...event.args },
62
71
  latencyMs: event.latencyMs,
63
- });
72
+ };
73
+ this.currentIterationDecisions.push(decision);
64
74
  }
65
75
  onTurnComplete(event) {
66
- this.claims.push({
76
+ // Final iteration has a claim
77
+ const claim = {
67
78
  content: event.content,
68
- model: this.lastModel,
69
- iteration: this.lastIteration,
79
+ model: this.currentContext.model,
80
+ iteration: this.currentIteration,
81
+ };
82
+ this.flushIteration(claim);
83
+ }
84
+ /** Flush current iteration data into the iterations array. */
85
+ flushIteration(claim) {
86
+ this.iterations.push({
87
+ iteration: this.currentIteration,
88
+ context: { ...this.currentContext },
89
+ decisions: this.currentIterationDecisions.map((d) => ({ ...d, args: { ...d.args } })),
90
+ sources: [...this.currentIterationSources],
91
+ claim,
70
92
  });
93
+ this.currentIterationDecisions = [];
94
+ this.currentIterationSources = [];
71
95
  }
72
- /** Tool results the ground truth data. */
96
+ // ── Flat convenience accessors (backward compatible) ────
73
97
  getSources() {
74
- return [...this.sources];
98
+ return this.iterations.flatMap((it) => it.sources);
75
99
  }
76
- /** LLM responses — what it claimed. */
77
100
  getClaims() {
78
- return [...this.claims];
101
+ return this.iterations.filter((it) => it.claim).map((it) => it.claim);
79
102
  }
80
- /** Tool call decisions — what the LLM chose to do. */
81
103
  getDecisions() {
82
- return this.decisions.map((d) => ({ ...d, args: { ...d.args } }));
104
+ return this.iterations.flatMap((it) => it.decisions.map((d) => ({ ...d, args: { ...d.args } })));
83
105
  }
84
- /** What the LLM had when it made decisions — for evaluation. */
85
106
  getContext() {
86
- return {
87
- input: this.input,
88
- systemPrompt: this.lastSystemPrompt,
89
- availableTools: this.lastToolDescriptions ? [...this.lastToolDescriptions] : undefined,
90
- messages: this.lastMessages ? [...this.lastMessages] : undefined,
91
- model: this.lastModel,
92
- };
107
+ return { ...this.currentContext };
108
+ }
109
+ /** Per-iteration evaluation units. */
110
+ getIterations() {
111
+ return this.iterations.map((it) => ({
112
+ ...it,
113
+ context: { ...it.context },
114
+ decisions: it.decisions.map((d) => ({ ...d, args: { ...d.args } })),
115
+ sources: [...it.sources],
116
+ }));
93
117
  }
94
- /** Structured explanation — everything needed for evaluation. */
118
+ /** Structured explanation — flat + per-iteration, everything for evaluation. */
95
119
  explain() {
96
- const toolNames = [...new Set(this.decisions.map((d) => d.toolName))];
97
- const summary = this.sources.length === 0
120
+ const allDecisions = this.getDecisions();
121
+ const toolNames = [...new Set(allDecisions.map((d) => d.toolName))];
122
+ const allSources = this.getSources();
123
+ const summary = allSources.length === 0
98
124
  ? `Agent responded directly without calling tools.`
99
- : `Agent called ${toolNames.join(', ')} (${this.sources.length} call${this.sources.length > 1 ? 's' : ''}), then responded based on the results.`;
125
+ : `Agent called ${toolNames.join(', ')} (${allSources.length} call${allSources.length > 1 ? 's' : ''}), then responded based on the results.`;
100
126
  return {
101
- sources: this.getSources(),
127
+ iterations: this.getIterations(),
128
+ sources: allSources,
102
129
  claims: this.getClaims(),
103
- decisions: this.getDecisions(),
130
+ decisions: allDecisions,
104
131
  context: this.getContext(),
105
132
  summary,
106
133
  };
107
134
  }
108
135
  clear() {
109
- this.sources = [];
110
- this.claims = [];
111
- this.decisions = [];
136
+ this.iterations = [];
137
+ this.currentIterationDecisions = [];
138
+ this.currentIterationSources = [];
112
139
  this.currentTurn = 0;
113
140
  this.input = undefined;
114
- this.lastModel = undefined;
115
- this.lastIteration = 0;
116
- this.lastSystemPrompt = undefined;
117
- this.lastToolDescriptions = undefined;
118
- this.lastMessages = undefined;
141
+ this.currentIteration = -1;
142
+ this.currentContext = {};
119
143
  }
120
144
  }
121
145
  exports.ExplainRecorder = ExplainRecorder;
@@ -1 +1 @@
1
- {"version":3,"file":"ExplainRecorder.js","sourceRoot":"","sources":["../../../src/recorders/v2/ExplainRecorder.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;;;;;;;;GAiBG;;;AAwDH,MAAa,eAAe;IACjB,EAAE,CAAS;IACZ,OAAO,GAAiB,EAAE,CAAC;IAC3B,MAAM,GAAe,EAAE,CAAC;IACxB,SAAS,GAAoB,EAAE,CAAC;IAChC,WAAW,GAAG,CAAC,CAAC;IAChB,KAAK,CAAU;IACf,SAAS,CAAU;IACnB,aAAa,GAAG,CAAC,CAAC;IAClB,gBAAgB,CAAU;IAC1B,oBAAoB,CAAwD;IAC5E,YAAY,CAAqD;IAEzE,YAAY,EAAE,GAAG,kBAAkB;QACjC,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC;IACf,CAAC;IAED,WAAW,CAAC,KAAqB;QAC/B,IAAI,CAAC,WAAW,GAAG,KAAK,CAAC,UAAU,CAAC;QACpC,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC,OAAO,CAAC;IAC7B,CAAC;IAED,SAAS,CAAC,KAAmB;QAC3B,IAAI,CAAC,SAAS,GAAG,KAAK,CAAC,KAAK,CAAC;QAC7B,IAAI,CAAC,aAAa,GAAG,KAAK,CAAC,aAAa,CAAC;QACzC,qDAAqD;QACrD,IAAI,KAAK,CAAC,YAAY;YAAE,IAAI,CAAC,gBAAgB,GAAG,KAAK,CAAC,YAAY,CAAC;QACnE,IAAI,KAAK,CAAC,gBAAgB;YAAE,IAAI,CAAC,oBAAoB,GAAG,KAAK,CAAC,gBAAgB,CAAC;QAC/E,IAAI,KAAK,CAAC,QAAQ;YAAE,IAAI,CAAC,YAAY,GAAG,KAAK,CAAC,QAAQ,CAAC;IACzD,CAAC;IAED,UAAU,CAAC,KAAoB;QAC7B,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC;YAChB,QAAQ,EAAE,KAAK,CAAC,QAAQ;YACxB,IAAI,EAAE,EAAE,GAAG,KAAK,CAAC,IAAI,EAAE;YACvB,MAAM,EAAE,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,OAAO,CAAC;YACpC,UAAU,EAAE,IAAI,CAAC,WAAW;SAC7B,CAAC,CAAC;QAEH,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC;YAClB,QAAQ,EAAE,KAAK,CAAC,QAAQ;YACxB,IAAI,EAAE,EAAE,GAAG,KAAK,CAAC,IAAI,EAAE;YACvB,SAAS,EAAE,KAAK,CAAC,SAAS;SAC3B,CAAC,CAAC;IACL,CAAC;IAED,cAAc,CAAC,KAAwB;QACrC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC;YACf,OAAO,EAAE,KAAK,CAAC,OAAO;YACtB,KAAK,EAAE,IAAI,CAAC,SAAS;YACrB,SAAS,EAAE,IAAI,CAAC,aAAa;SAC9B,CAAC,CAAC;IACL,CAAC;IAED,4CAA4C;IAC5C,UAAU;QACR,OAAO,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,CAAC;IAC3B,CAAC;IAED,uCAAuC;IACvC,SAAS;QACP,OAAO,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC;IAC1B,CAAC;IAED,sDAAsD;IACtD,YAAY;QACV,OAAO,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,IAAI,EAAE,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,CAAC;IACpE,CAAC;IAED,gEAAgE;IAChE,UAAU;QACR,OAAO;YACL,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,YAAY,EAAE,IAAI,CAAC,gBAAgB;YACnC,cAAc,EAAE,IAAI,CAAC,oBAAoB,CAAC,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,oBAAoB,CAAC,CAAC,CAAC,CAAC,SAAS;YACtF,QAAQ,EAAE,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,SAAS;YAChE,KAAK,EAAE,IAAI,CAAC,SAAS;SACtB,CAAC;IACJ,CAAC;IAED,iEAAiE;IACjE,OAAO;QACL,MAAM,SAAS,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;QACtE,MAAM,OAAO,GACX,IAAI,CAAC,OAAO,CAAC,MAAM,KAAK,CAAC;YACvB,CAAC,CAAC,iDAAiD;YACnD,CAAC,CAAC,gBAAgB,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,IAAI,CAAC,OAAO,CAAC,MAAM,QAC1D,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAClC,yCAAyC,CAAC;QAEhD,OAAO;YACL,OAAO,EAAE,IAAI,CAAC,UAAU,EAAE;YAC1B,MAAM,EAAE,IAAI,CAAC,SAAS,EAAE;YACxB,SAAS,EAAE,IAAI,CAAC,YAAY,EAAE;YAC9B,OAAO,EAAE,IAAI,CAAC,UAAU,EAAE;YAC1B,OAAO;SACR,CAAC;IACJ,CAAC;IAED,KAAK;QACH,IAAI,CAAC,OAAO,GAAG,EAAE,CAAC;QAClB,IAAI,CAAC,MAAM,GAAG,EAAE,CAAC;QACjB,IAAI,CAAC,SAAS,GAAG,EAAE,CAAC;QACpB,IAAI,CAAC,WAAW,GAAG,CAAC,CAAC;QACrB,IAAI,CAAC,KAAK,GAAG,SAAS,CAAC;QACvB,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,aAAa,GAAG,CAAC,CAAC;QACvB,IAAI,CAAC,gBAAgB,GAAG,SAAS,CAAC;QAClC,IAAI,CAAC,oBAAoB,GAAG,SAAS,CAAC;QACtC,IAAI,CAAC,YAAY,GAAG,SAAS,CAAC;IAChC,CAAC;CACF;AA/GD,0CA+GC"}
1
+ {"version":3,"file":"ExplainRecorder.js","sourceRoot":"","sources":["../../../src/recorders/v2/ExplainRecorder.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;;;AA2EH,MAAa,eAAe;IACjB,EAAE,CAAS;IAEpB,6BAA6B;IACrB,UAAU,GAAoB,EAAE,CAAC;IACjC,yBAAyB,GAAoB,EAAE,CAAC;IAChD,uBAAuB,GAAiB,EAAE,CAAC;IAEnD,iBAAiB;IACT,WAAW,GAAG,CAAC,CAAC;IAChB,KAAK,CAAU;IACf,gBAAgB,GAAG,CAAC,CAAC,CAAC;IACtB,cAAc,GAAe,EAAE,CAAC;IAExC,YAAY,EAAE,GAAG,kBAAkB;QACjC,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC;IACf,CAAC;IAED,WAAW,CAAC,KAAqB;QAC/B,IAAI,CAAC,WAAW,GAAG,KAAK,CAAC,UAAU,CAAC;QACpC,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC,OAAO,CAAC;IAC7B,CAAC;IAED,SAAS,CAAC,KAAmB;QAC3B,8EAA8E;QAC9E,IAAI,IAAI,CAAC,gBAAgB,IAAI,CAAC,EAAE,CAAC;YAC/B,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC;QAC5B,CAAC;QAED,IAAI,CAAC,gBAAgB,GAAG,KAAK,CAAC,aAAa,CAAC;QAE5C,sCAAsC;QACtC,IAAI,CAAC,cAAc,GAAG;YACpB,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,YAAY,EAAE,KAAK,CAAC,YAAY,IAAI,IAAI,CAAC,cAAc,CAAC,YAAY;YACpE,cAAc,EAAE,KAAK,CAAC,gBAAgB;gBACpC,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC,gBAAgB,CAAC;gBAC7B,CAAC,CAAC,IAAI,CAAC,cAAc,CAAC,cAAc;YACtC,QAAQ,EAAE,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,cAAc,CAAC,QAAQ;YAC7E,KAAK,EAAE,KAAK,CAAC,KAAK;SACnB,CAAC;IACJ,CAAC;IAED,UAAU,CAAC,KAAoB;QAC7B,MAAM,MAAM,GAAe;YACzB,QAAQ,EAAE,KAAK,CAAC,QAAQ;YACxB,IAAI,EAAE,EAAE,GAAG,KAAK,CAAC,IAAI,EAAE;YACvB,MAAM,EAAE,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,OAAO,CAAC;YACpC,UAAU,EAAE,IAAI,CAAC,WAAW;SAC7B,CAAC;QACF,IAAI,CAAC,uBAAuB,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAE1C,MAAM,QAAQ,GAAkB;YAC9B,QAAQ,EAAE,KAAK,CAAC,QAAQ;YACxB,IAAI,EAAE,EAAE,GAAG,KAAK,CAAC,IAAI,EAAE;YACvB,SAAS,EAAE,KAAK,CAAC,SAAS;SAC3B,CAAC;QACF,IAAI,CAAC,yBAAyB,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IAChD,CAAC;IAED,cAAc,CAAC,KAAwB;QACrC,8BAA8B;QAC9B,MAAM,KAAK,GAAa;YACtB,OAAO,EAAE,KAAK,CAAC,OAAO;YACtB,KAAK,EAAE,IAAI,CAAC,cAAc,CAAC,KAAK;YAChC,SAAS,EAAE,IAAI,CAAC,gBAAgB;SACjC,CAAC;QACF,IAAI,CAAC,cAAc,CAAC,KAAK,CAAC,CAAC;IAC7B,CAAC;IAED,8DAA8D;IACtD,cAAc,CAAC,KAAsB;QAC3C,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;YACnB,SAAS,EAAE,IAAI,CAAC,gBAAgB;YAChC,OAAO,EAAE,EAAE,GAAG,IAAI,CAAC,cAAc,EAAE;YACnC,SAAS,EAAE,IAAI,CAAC,yBAAyB,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,IAAI,EAAE,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;YACrF,OAAO,EAAE,CAAC,GAAG,IAAI,CAAC,uBAAuB,CAAC;YAC1C,KAAK;SACN,CAAC,CAAC;QACH,IAAI,CAAC,yBAAyB,GAAG,EAAE,CAAC;QACpC,IAAI,CAAC,uBAAuB,GAAG,EAAE,CAAC;IACpC,CAAC;IAED,2DAA2D;IAE3D,UAAU;QACR,OAAO,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,OAAO,CAAC,CAAC;IACrD,CAAC;IAED,SAAS;QACP,OAAO,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,KAAM,CAAC,CAAC;IACzE,CAAC;IAED,YAAY;QACV,OAAO,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,EAAE,EAAE,EAAE,CACpC,EAAE,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,IAAI,EAAE,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,CACzD,CAAC;IACJ,CAAC;IAED,UAAU;QACR,OAAO,EAAE,GAAG,IAAI,CAAC,cAAc,EAAE,CAAC;IACpC,CAAC;IAED,sCAAsC;IACtC,aAAa;QACX,OAAO,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC;YAClC,GAAG,EAAE;YACL,OAAO,EAAE,EAAE,GAAG,EAAE,CAAC,OAAO,EAAE;YAC1B,SAAS,EAAE,EAAE,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,IAAI,EAAE,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;YACnE,OAAO,EAAE,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC;SACzB,CAAC,CAAC,CAAC;IACN,CAAC;IAED,gFAAgF;IAChF,OAAO;QACL,MAAM,YAAY,GAAG,IAAI,CAAC,YAAY,EAAE,CAAC;QACzC,MAAM,SAAS,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;QACpE,MAAM,UAAU,GAAG,IAAI,CAAC,UAAU,EAAE,CAAC;QACrC,MAAM,OAAO,GACX,UAAU,CAAC,MAAM,KAAK,CAAC;YACrB,CAAC,CAAC,iDAAiD;YACnD,CAAC,CAAC,gBAAgB,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,UAAU,CAAC,MAAM,QACxD,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAChC,yCAAyC,CAAC;QAEhD,OAAO;YACL,UAAU,EAAE,IAAI,CAAC,aAAa,EAAE;YAChC,OAAO,EAAE,UAAU;YACnB,MAAM,EAAE,IAAI,CAAC,SAAS,EAAE;YACxB,SAAS,EAAE,YAAY;YACvB,OAAO,EAAE,IAAI,CAAC,UAAU,EAAE;YAC1B,OAAO;SACR,CAAC;IACJ,CAAC;IAED,KAAK;QACH,IAAI,CAAC,UAAU,GAAG,EAAE,CAAC;QACrB,IAAI,CAAC,yBAAyB,GAAG,EAAE,CAAC;QACpC,IAAI,CAAC,uBAAuB,GAAG,EAAE,CAAC;QAClC,IAAI,CAAC,WAAW,GAAG,CAAC,CAAC;QACrB,IAAI,CAAC,KAAK,GAAG,SAAS,CAAC;QACvB,IAAI,CAAC,gBAAgB,GAAG,CAAC,CAAC,CAAC;QAC3B,IAAI,CAAC,cAAc,GAAG,EAAE,CAAC;IAC3B,CAAC;CACF;AAhJD,0CAgJC"}
@@ -21,5 +21,5 @@
21
21
  export { createAgentRenderer } from './lib/narrative';
22
22
  export type { AgentRendererOptions } from './lib/narrative';
23
23
  export { ExplainRecorder } from './recorders/v2/ExplainRecorder';
24
- export type { ToolSource, LLMClaim, AgentDecision, LLMContext, Explanation, } from './recorders/v2/ExplainRecorder';
24
+ export type { ToolSource, LLMClaim, AgentDecision, LLMContext, EvalIteration, Explanation, } from './recorders/v2/ExplainRecorder';
25
25
  //# sourceMappingURL=explain.barrel.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"explain.barrel.d.ts","sourceRoot":"","sources":["../../src/explain.barrel.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;GAmBG;AAEH,OAAO,EAAE,mBAAmB,EAAE,MAAM,iBAAiB,CAAC;AACtD,YAAY,EAAE,oBAAoB,EAAE,MAAM,iBAAiB,CAAC;AAG5D,OAAO,EAAE,eAAe,EAAE,MAAM,gCAAgC,CAAC;AACjE,YAAY,EACV,UAAU,EACV,QAAQ,EACR,aAAa,EACb,UAAU,EACV,WAAW,GACZ,MAAM,gCAAgC,CAAC"}
1
+ {"version":3,"file":"explain.barrel.d.ts","sourceRoot":"","sources":["../../src/explain.barrel.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;GAmBG;AAEH,OAAO,EAAE,mBAAmB,EAAE,MAAM,iBAAiB,CAAC;AACtD,YAAY,EAAE,oBAAoB,EAAE,MAAM,iBAAiB,CAAC;AAG5D,OAAO,EAAE,eAAe,EAAE,MAAM,gCAAgC,CAAC;AACjE,YAAY,EACV,UAAU,EACV,QAAQ,EACR,aAAa,EACb,UAAU,EACV,aAAa,EACb,WAAW,GACZ,MAAM,gCAAgC,CAAC"}
@@ -1,20 +1,23 @@
1
1
  /**
2
2
  * ExplainRecorder — collects grounding evidence + evaluation context during traversal.
3
3
  *
4
- * Captures:
5
- * - Sources (tool results ground truth)
6
- * - Claims (LLM output to verify)
7
- * - Decisions (tool calls — what the LLM chose to do)
8
- * - Context (system prompt, tool descriptions, messages — what the LLM had)
4
+ * Data is structured per-iteration for evaluation:
5
+ * iterations[0] = { context (what LLM had), decisions (tools chosen), sources (results), claim? }
6
+ * iterations[1] = { context (updated with tool results), decisions, sources, claim (final answer) }
9
7
  *
10
- * Everything for evaluation in one recorder: "what did it have?" + "what did it produce?"
8
+ * An evaluator walks iterations: for each one that has a claim, check if it's
9
+ * grounded in that iteration's sources + all prior sources.
11
10
  *
12
11
  * Usage:
13
12
  * const explain = new ExplainRecorder();
14
13
  * agent.recorder(explain);
15
14
  * await agent.run('Check order');
16
15
  *
17
- * explain.explain(); // { sources, claims, decisions, context, summary }
16
+ * const report = explain.explain();
17
+ * report.iterations; // per-iteration evaluation units
18
+ * report.sources; // flat convenience (all sources)
19
+ * report.claims; // flat convenience (all claims)
20
+ * report.context; // last context snapshot
18
21
  */
19
22
  import type { AgentRecorder, LLMCallEvent, ToolCallEvent, TurnStartEvent, TurnCompleteEvent } from '../../core';
20
23
  /** A source of truth — data returned by a tool. */
@@ -55,41 +58,56 @@ export interface LLMContext {
55
58
  /** Model used. */
56
59
  readonly model?: string;
57
60
  }
58
- /** Structured explanation of an agent's executioneverything needed for evaluation. */
61
+ /** One iteration of the agent loopa self-contained evaluation unit. */
62
+ export interface EvalIteration {
63
+ /** Loop iteration number (0-based). */
64
+ readonly iteration: number;
65
+ /** What the LLM had THIS iteration (context changes each loop — messages grow). */
66
+ readonly context: LLMContext;
67
+ /** Tool calls the LLM chose to make (empty if final response). */
68
+ readonly decisions: readonly AgentDecision[];
69
+ /** Tool results returned (empty if no tools called). */
70
+ readonly sources: readonly ToolSource[];
71
+ /** LLM's response — null for tool-calling iterations, string for final answer. */
72
+ readonly claim: LLMClaim | null;
73
+ }
74
+ /** Structured explanation — everything needed for evaluation. */
59
75
  export interface Explanation {
76
+ /** Per-iteration evaluation units — the connected data shape. */
77
+ readonly iterations: readonly EvalIteration[];
78
+ /** Flat convenience: all sources across all iterations. */
60
79
  readonly sources: readonly ToolSource[];
80
+ /** Flat convenience: all claims across all iterations. */
61
81
  readonly claims: readonly LLMClaim[];
82
+ /** Flat convenience: all decisions across all iterations. */
62
83
  readonly decisions: readonly AgentDecision[];
63
- /** What the LLM had when it made decisions. */
84
+ /** Last context snapshot (same as iterations[last].context). */
64
85
  readonly context: LLMContext;
65
86
  readonly summary: string;
66
87
  }
67
88
  export declare class ExplainRecorder implements AgentRecorder {
68
89
  readonly id: string;
69
- private sources;
70
- private claims;
71
- private decisions;
90
+ private iterations;
91
+ private currentIterationDecisions;
92
+ private currentIterationSources;
72
93
  private currentTurn;
73
94
  private input?;
74
- private lastModel?;
75
- private lastIteration;
76
- private lastSystemPrompt?;
77
- private lastToolDescriptions?;
78
- private lastMessages?;
95
+ private currentIteration;
96
+ private currentContext;
79
97
  constructor(id?: string);
80
98
  onTurnStart(event: TurnStartEvent): void;
81
99
  onLLMCall(event: LLMCallEvent): void;
82
100
  onToolCall(event: ToolCallEvent): void;
83
101
  onTurnComplete(event: TurnCompleteEvent): void;
84
- /** Tool results the ground truth data. */
102
+ /** Flush current iteration data into the iterations array. */
103
+ private flushIteration;
85
104
  getSources(): readonly ToolSource[];
86
- /** LLM responses — what it claimed. */
87
105
  getClaims(): readonly LLMClaim[];
88
- /** Tool call decisions — what the LLM chose to do. */
89
106
  getDecisions(): readonly AgentDecision[];
90
- /** What the LLM had when it made decisions — for evaluation. */
91
107
  getContext(): LLMContext;
92
- /** Structured explanation — everything needed for evaluation. */
108
+ /** Per-iteration evaluation units. */
109
+ getIterations(): readonly EvalIteration[];
110
+ /** Structured explanation — flat + per-iteration, everything for evaluation. */
93
111
  explain(): Explanation;
94
112
  clear(): void;
95
113
  }
@@ -1 +1 @@
1
- {"version":3,"file":"ExplainRecorder.d.ts","sourceRoot":"","sources":["../../../../src/recorders/v2/ExplainRecorder.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AAEH,OAAO,KAAK,EACV,aAAa,EACb,YAAY,EACZ,aAAa,EACb,cAAc,EACd,iBAAiB,EAClB,MAAM,YAAY,CAAC;AAEpB,mDAAmD;AACnD,MAAM,WAAW,UAAU;IACzB,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACvC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,CAAC;CAC9B;AAED,+CAA+C;AAC/C,MAAM,WAAW,QAAQ;IACvB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;CAC5B;AAED,2CAA2C;AAC3C,MAAM,WAAW,aAAa;IAC5B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACvC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;CAC5B;AAED,sEAAsE;AACtE,MAAM,WAAW,UAAU;IACzB,0CAA0C;IAC1C,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,CAAC;IACxB,sCAAsC;IACtC,QAAQ,CAAC,YAAY,CAAC,EAAE,MAAM,CAAC;IAC/B,qEAAqE;IACrE,QAAQ,CAAC,cAAc,CAAC,EAAE,aAAa,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,WAAW,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAC/E,uDAAuD;IACvD,QAAQ,CAAC,QAAQ,CAAC,EAAE,aAAa,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,OAAO,CAAA;KAAE,CAAC,CAAC;IACtE,kBAAkB;IAClB,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,CAAC;CACzB;AAED,yFAAyF;AACzF,MAAM,WAAW,WAAW;IAC1B,QAAQ,CAAC,OAAO,EAAE,SAAS,UAAU,EAAE,CAAC;IACxC,QAAQ,CAAC,MAAM,EAAE,SAAS,QAAQ,EAAE,CAAC;IACrC,QAAQ,CAAC,SAAS,EAAE,SAAS,aAAa,EAAE,CAAC;IAC7C,+CAA+C;IAC/C,QAAQ,CAAC,OAAO,EAAE,UAAU,CAAC;IAC7B,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;CAC1B;AAED,qBAAa,eAAgB,YAAW,aAAa;IACnD,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,OAAO,CAAC,OAAO,CAAoB;IACnC,OAAO,CAAC,MAAM,CAAkB;IAChC,OAAO,CAAC,SAAS,CAAuB;IACxC,OAAO,CAAC,WAAW,CAAK;IACxB,OAAO,CAAC,KAAK,CAAC,CAAS;IACvB,OAAO,CAAC,SAAS,CAAC,CAAS;IAC3B,OAAO,CAAC,aAAa,CAAK;IAC1B,OAAO,CAAC,gBAAgB,CAAC,CAAS;IAClC,OAAO,CAAC,oBAAoB,CAAC,CAAuD;IACpF,OAAO,CAAC,YAAY,CAAC,CAAoD;gBAE7D,EAAE,SAAqB;IAInC,WAAW,CAAC,KAAK,EAAE,cAAc,GAAG,IAAI;IAKxC,SAAS,CAAC,KAAK,EAAE,YAAY,GAAG,IAAI;IASpC,UAAU,CAAC,KAAK,EAAE,aAAa,GAAG,IAAI;IAetC,cAAc,CAAC,KAAK,EAAE,iBAAiB,GAAG,IAAI;IAQ9C,4CAA4C;IAC5C,UAAU,IAAI,SAAS,UAAU,EAAE;IAInC,uCAAuC;IACvC,SAAS,IAAI,SAAS,QAAQ,EAAE;IAIhC,sDAAsD;IACtD,YAAY,IAAI,SAAS,aAAa,EAAE;IAIxC,gEAAgE;IAChE,UAAU,IAAI,UAAU;IAUxB,iEAAiE;IACjE,OAAO,IAAI,WAAW;IAkBtB,KAAK,IAAI,IAAI;CAYd"}
1
+ {"version":3,"file":"ExplainRecorder.d.ts","sourceRoot":"","sources":["../../../../src/recorders/v2/ExplainRecorder.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AAEH,OAAO,KAAK,EACV,aAAa,EACb,YAAY,EACZ,aAAa,EACb,cAAc,EACd,iBAAiB,EAClB,MAAM,YAAY,CAAC;AAEpB,mDAAmD;AACnD,MAAM,WAAW,UAAU;IACzB,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACvC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,CAAC;CAC9B;AAED,+CAA+C;AAC/C,MAAM,WAAW,QAAQ;IACvB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;CAC5B;AAED,2CAA2C;AAC3C,MAAM,WAAW,aAAa;IAC5B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACvC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;CAC5B;AAED,sEAAsE;AACtE,MAAM,WAAW,UAAU;IACzB,0CAA0C;IAC1C,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,CAAC;IACxB,sCAAsC;IACtC,QAAQ,CAAC,YAAY,CAAC,EAAE,MAAM,CAAC;IAC/B,qEAAqE;IACrE,QAAQ,CAAC,cAAc,CAAC,EAAE,aAAa,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,WAAW,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAC/E,uDAAuD;IACvD,QAAQ,CAAC,QAAQ,CAAC,EAAE,aAAa,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,OAAO,CAAA;KAAE,CAAC,CAAC;IACtE,kBAAkB;IAClB,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,CAAC;CACzB;AAED,0EAA0E;AAC1E,MAAM,WAAW,aAAa;IAC5B,uCAAuC;IACvC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,mFAAmF;IACnF,QAAQ,CAAC,OAAO,EAAE,UAAU,CAAC;IAC7B,kEAAkE;IAClE,QAAQ,CAAC,SAAS,EAAE,SAAS,aAAa,EAAE,CAAC;IAC7C,wDAAwD;IACxD,QAAQ,CAAC,OAAO,EAAE,SAAS,UAAU,EAAE,CAAC;IACxC,kFAAkF;IAClF,QAAQ,CAAC,KAAK,EAAE,QAAQ,GAAG,IAAI,CAAC;CACjC;AAED,iEAAiE;AACjE,MAAM,WAAW,WAAW;IAC1B,iEAAiE;IACjE,QAAQ,CAAC,UAAU,EAAE,SAAS,aAAa,EAAE,CAAC;IAC9C,2DAA2D;IAC3D,QAAQ,CAAC,OAAO,EAAE,SAAS,UAAU,EAAE,CAAC;IACxC,0DAA0D;IAC1D,QAAQ,CAAC,MAAM,EAAE,SAAS,QAAQ,EAAE,CAAC;IACrC,6DAA6D;IAC7D,QAAQ,CAAC,SAAS,EAAE,SAAS,aAAa,EAAE,CAAC;IAC7C,gEAAgE;IAChE,QAAQ,CAAC,OAAO,EAAE,UAAU,CAAC;IAC7B,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;CAC1B;AAED,qBAAa,eAAgB,YAAW,aAAa;IACnD,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IAGpB,OAAO,CAAC,UAAU,CAAuB;IACzC,OAAO,CAAC,yBAAyB,CAAuB;IACxD,OAAO,CAAC,uBAAuB,CAAoB;IAGnD,OAAO,CAAC,WAAW,CAAK;IACxB,OAAO,CAAC,KAAK,CAAC,CAAS;IACvB,OAAO,CAAC,gBAAgB,CAAM;IAC9B,OAAO,CAAC,cAAc,CAAkB;gBAE5B,EAAE,SAAqB;IAInC,WAAW,CAAC,KAAK,EAAE,cAAc,GAAG,IAAI;IAKxC,SAAS,CAAC,KAAK,EAAE,YAAY,GAAG,IAAI;IAoBpC,UAAU,CAAC,KAAK,EAAE,aAAa,GAAG,IAAI;IAiBtC,cAAc,CAAC,KAAK,EAAE,iBAAiB,GAAG,IAAI;IAU9C,8DAA8D;IAC9D,OAAO,CAAC,cAAc;IActB,UAAU,IAAI,SAAS,UAAU,EAAE;IAInC,SAAS,IAAI,SAAS,QAAQ,EAAE;IAIhC,YAAY,IAAI,SAAS,aAAa,EAAE;IAMxC,UAAU,IAAI,UAAU;IAIxB,sCAAsC;IACtC,aAAa,IAAI,SAAS,aAAa,EAAE;IASzC,gFAAgF;IAChF,OAAO,IAAI,WAAW;IAqBtB,KAAK,IAAI,IAAI;CASd"}
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "agentfootprint",
3
- "version": "1.3.0",
3
+ "version": "1.4.0",
4
4
  "description": "The explainable agent framework — build AI agents you can explain, audit, and trust. Built on footprintjs.",
5
5
  "license": "MIT",
6
6
  "author": "Sanjay Krishna Anbalagan",