@gaunt-sloth/core 2.0.0-alpha.22 → 2.0.0-alpha.23
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/core/runStats.d.ts +14 -3
- package/dist/core/runStats.js +48 -3
- package/dist/core/runStats.js.map +1 -1
- package/dist/core/types.d.ts +22 -0
- package/package.json +1 -1
package/dist/core/runStats.d.ts
CHANGED
|
@@ -10,7 +10,14 @@
|
|
|
10
10
|
* message chunks / `ToolMessage`s as they arrive), across providers whose message shapes vary.
|
|
11
11
|
* Nothing here may throw into a run — a missing/odd field just means that datum is skipped.
|
|
12
12
|
*/
|
|
13
|
-
import type { GthRunStats } from '#src/core/types.js';
|
|
13
|
+
import type { GthRunStats, GthToolResult } from '#src/core/types.js';
|
|
14
|
+
/**
|
|
15
|
+
* BATCH-21 — cap on a captured tool-result `content` (characters). Keeps a giant payload (a whole
|
|
16
|
+
* file read, a long shell log) from bloating run stats; anything longer is truncated to this
|
|
17
|
+
* length. Sized so realistic structured payloads (the `gth eval` tool-result-assertion use case)
|
|
18
|
+
* survive intact.
|
|
19
|
+
*/
|
|
20
|
+
export declare const TOOL_RESULT_CONTENT_CAP = 8192;
|
|
14
21
|
/** Mutable tally behind {@link finalizeRunStats}; see {@link createRunStatsAccumulator}. */
|
|
15
22
|
export interface RunStatsAccumulator {
|
|
16
23
|
/** Running sum of input/prompt tokens. */
|
|
@@ -21,15 +28,19 @@ export interface RunStatsAccumulator {
|
|
|
21
28
|
sawUsage: boolean;
|
|
22
29
|
/** Deduplicated set of invoked tool names. */
|
|
23
30
|
tools: Set<string>;
|
|
31
|
+
/** BATCH-21 — one record per executed tool result (`ToolMessage`), in arrival order, un-deduped. */
|
|
32
|
+
toolResults: GthToolResult[];
|
|
24
33
|
}
|
|
25
34
|
/** A fresh, empty accumulator. */
|
|
26
35
|
export declare function createRunStatsAccumulator(): RunStatsAccumulator;
|
|
27
36
|
/**
|
|
28
37
|
* Fold one LangChain message (or message chunk) into the accumulator. Fail-soft: any unexpected
|
|
29
38
|
* shape is swallowed so a run is never affected. Harvests, when present:
|
|
30
|
-
* - `usage_metadata.input_tokens` / `.output_tokens` (summed; marks `sawUsage`),
|
|
39
|
+
* - `usage_metadata.input_tokens` / `.output_tokens` (summed; marks `sawUsage`),
|
|
31
40
|
* - tool names from an AIMessage's requested `tool_calls[].name` AND from a `ToolMessage`'s own
|
|
32
|
-
* `.name` (the executed tool), so both "requested" and "executed" tools are captured
|
|
41
|
+
* `.name` (the executed tool), so both "requested" and "executed" tools are captured, and
|
|
42
|
+
* - (BATCH-21) a per-`ToolMessage` result record — `name` + `isError` (from `.status`) + capped
|
|
43
|
+
* `content` — into `acc.toolResults`, so tool-RESULT assertions can grade what a tool returned.
|
|
33
44
|
*/
|
|
34
45
|
export declare function accumulateMessage(acc: RunStatsAccumulator, message: unknown): void;
|
|
35
46
|
/** Freeze the accumulator into the public {@link GthRunStats}. Tokens omitted unless observed. */
|
package/dist/core/runStats.js
CHANGED
|
@@ -1,13 +1,47 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* BATCH-21 — cap on a captured tool-result `content` (characters). Keeps a giant payload (a whole
|
|
3
|
+
* file read, a long shell log) from bloating run stats; anything longer is truncated to this
|
|
4
|
+
* length. Sized so realistic structured payloads (the `gth eval` tool-result-assertion use case)
|
|
5
|
+
* survive intact.
|
|
6
|
+
*/
|
|
7
|
+
export const TOOL_RESULT_CONTENT_CAP = 8192;
|
|
1
8
|
/** A fresh, empty accumulator. */
|
|
2
9
|
export function createRunStatsAccumulator() {
|
|
3
|
-
return { input: 0, output: 0, sawUsage: false, tools: new Set() };
|
|
10
|
+
return { input: 0, output: 0, sawUsage: false, tools: new Set(), toolResults: [] };
|
|
11
|
+
}
|
|
12
|
+
/**
|
|
13
|
+
* BATCH-21 — derive a tool result's text payload from a `ToolMessage.content`, fail-soft. A string
|
|
14
|
+
* passes through; anything else non-`undefined` is JSON-stringified (the same derivation the
|
|
15
|
+
* `tool_result` stream event uses in `GthAbstractAgent`); the result is capped at
|
|
16
|
+
* {@link TOOL_RESULT_CONTENT_CAP}. Returns `undefined` (payload omitted) when nothing textual can
|
|
17
|
+
* be derived — never throws.
|
|
18
|
+
*/
|
|
19
|
+
function toolResultContentText(content) {
|
|
20
|
+
try {
|
|
21
|
+
let text;
|
|
22
|
+
if (typeof content === 'string') {
|
|
23
|
+
text = content;
|
|
24
|
+
}
|
|
25
|
+
else if (content !== undefined) {
|
|
26
|
+
text = JSON.stringify(content);
|
|
27
|
+
}
|
|
28
|
+
if (text === undefined)
|
|
29
|
+
return undefined;
|
|
30
|
+
return text.length > TOOL_RESULT_CONTENT_CAP ? text.slice(0, TOOL_RESULT_CONTENT_CAP) : text;
|
|
31
|
+
}
|
|
32
|
+
catch {
|
|
33
|
+
/* fail-soft: an unstringifiable payload just means no content is recorded */
|
|
34
|
+
return undefined;
|
|
35
|
+
}
|
|
4
36
|
}
|
|
5
37
|
/**
|
|
6
38
|
* Fold one LangChain message (or message chunk) into the accumulator. Fail-soft: any unexpected
|
|
7
39
|
* shape is swallowed so a run is never affected. Harvests, when present:
|
|
8
|
-
* - `usage_metadata.input_tokens` / `.output_tokens` (summed; marks `sawUsage`),
|
|
40
|
+
* - `usage_metadata.input_tokens` / `.output_tokens` (summed; marks `sawUsage`),
|
|
9
41
|
* - tool names from an AIMessage's requested `tool_calls[].name` AND from a `ToolMessage`'s own
|
|
10
|
-
* `.name` (the executed tool), so both "requested" and "executed" tools are captured
|
|
42
|
+
* `.name` (the executed tool), so both "requested" and "executed" tools are captured, and
|
|
43
|
+
* - (BATCH-21) a per-`ToolMessage` result record — `name` + `isError` (from `.status`) + capped
|
|
44
|
+
* `content` — into `acc.toolResults`, so tool-RESULT assertions can grade what a tool returned.
|
|
11
45
|
*/
|
|
12
46
|
export function accumulateMessage(acc, message) {
|
|
13
47
|
try {
|
|
@@ -39,6 +73,16 @@ export function accumulateMessage(acc, message) {
|
|
|
39
73
|
const type = typeof m.getType === 'function' ? m.getType() : m._getType?.();
|
|
40
74
|
if (type === 'tool' && typeof m.name === 'string' && m.name.length > 0) {
|
|
41
75
|
acc.tools.add(m.name);
|
|
76
|
+
// BATCH-21 — capture the RESULT record too (same capture site, same fail-soft discipline):
|
|
77
|
+
// `.status === 'error'` is LangChain's real tool-error signal, `.content` the returned
|
|
78
|
+
// payload (capped; omitted when no text can be derived). One record per ToolMessage, in
|
|
79
|
+
// arrival order — deliberately NOT deduplicated, unlike the name set above.
|
|
80
|
+
const content = toolResultContentText(m.content);
|
|
81
|
+
acc.toolResults.push({
|
|
82
|
+
name: m.name,
|
|
83
|
+
isError: m.status === 'error',
|
|
84
|
+
...(content !== undefined ? { content } : {}),
|
|
85
|
+
});
|
|
42
86
|
}
|
|
43
87
|
}
|
|
44
88
|
catch {
|
|
@@ -51,6 +95,7 @@ export function finalizeRunStats(acc) {
|
|
|
51
95
|
tokensInput: acc.sawUsage ? acc.input : undefined,
|
|
52
96
|
tokensOutput: acc.sawUsage ? acc.output : undefined,
|
|
53
97
|
tools: [...acc.tools],
|
|
98
|
+
toolResults: [...acc.toolResults],
|
|
54
99
|
};
|
|
55
100
|
}
|
|
56
101
|
/**
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"runStats.js","sourceRoot":"","sources":["../../src/core/runStats.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"runStats.js","sourceRoot":"","sources":["../../src/core/runStats.ts"],"names":[],"mappings":"AAcA;;;;;GAKG;AACH,MAAM,CAAC,MAAM,uBAAuB,GAAG,IAAI,CAAC;AAgB5C,kCAAkC;AAClC,MAAM,UAAU,yBAAyB;IACvC,OAAO,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,QAAQ,EAAE,KAAK,EAAE,KAAK,EAAE,IAAI,GAAG,EAAU,EAAE,WAAW,EAAE,EAAE,EAAE,CAAC;AAC7F,CAAC;AAED;;;;;;GAMG;AACH,SAAS,qBAAqB,CAAC,OAAgB;IAC7C,IAAI,CAAC;QACH,IAAI,IAAwB,CAAC;QAC7B,IAAI,OAAO,OAAO,KAAK,QAAQ,EAAE,CAAC;YAChC,IAAI,GAAG,OAAO,CAAC;QACjB,CAAC;aAAM,IAAI,OAAO,KAAK,SAAS,EAAE,CAAC;YACjC,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;QACjC,CAAC;QACD,IAAI,IAAI,KAAK,SAAS;YAAE,OAAO,SAAS,CAAC;QACzC,OAAO,IAAI,CAAC,MAAM,GAAG,uBAAuB,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,uBAAuB,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAC/F,CAAC;IAAC,MAAM,CAAC;QACP,6EAA6E;QAC7E,OAAO,SAAS,CAAC;IACnB,CAAC;AACH,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,UAAU,iBAAiB,CAAC,GAAwB,EAAE,OAAgB;IAC1E,IAAI,CAAC;QACH,8DAA8D;QAC9D,MAAM,CAAC,GAAG,OAAc,CAAC;QACzB,IAAI,CAAC,CAAC,IAAI,OAAO,CAAC,KAAK,QAAQ;YAAE,OAAO;QAExC,MAAM,KAAK,GAAG,CAAC,CAAC,cAAc,CAAC;QAC/B,IAAI,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;YACvC,GAAG,CAAC,QAAQ,GAAG,IAAI,CAAC;YACpB,IAAI,OAAO,KAAK,CAAC,YAAY,KAAK,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,YAAY,CAAC,EAAE,CAAC;gBAClF,GAAG,CAAC,KAAK,IAAI,KAAK,CAAC,YAAY,CAAC;YAClC,CAAC;YACD,IAAI,OAAO,KAAK,CAAC,aAAa,KAAK,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,aAAa,CAAC,EAAE,CAAC;gBACpF,GAAG,CAAC,MAAM,IAAI,KAAK,CAAC,aAAa,CAAC;YACpC,CAAC;QACH,CAAC;QAED,uFAAuF;QACvF,0FAA0F;QAC1F,MAAM,SAAS,GAAG,CAAC,CAAC,UAAU,CAAC;QAC/B,IAAI,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,CAAC;YAC7B,KAAK,MAAM,EAAE,IAAI,SAAS,EAAE,CAAC;gBAC3B,MAAM,IAAI,GAAG,EAAE,EAAE,IAAI,CAAC;gBACtB,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC;oBAAE,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YACvE,CAAC;QACH,CAAC;QAED,wFAAwF;QACxF,MAAM,IAAI,GAAY,OAAO,CAAC,CAAC,OAAO,KAAK,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC;QACrF,IAAI,IAAI,KAAK,MAAM,IAAI,OAAO,CAAC,CAAC,IAAI,KAAK,QAAQ,IAAI,CAAC,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACvE,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;YACtB,2FAA2F;YAC3F,uFAAuF;YACvF,wFAAwF;YACxF,4EAA4E;YAC5E,MAAM,OAAO,GAAG,qBAAqB,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;YACjD,GAAG,CAAC,WAAW,CAAC,IAAI,CAAC;gBACnB,IAAI,EAAE,CAAC,CAAC,IAAI;gBACZ,OAAO,EAAE,CAAC,CAAC,MAAM,KAAK,OAAO;gBAC7B,GAAG,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;aAC9C,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAAC,MAAM,CAAC;QACP,qDAAqD;IACvD,CAAC;AACH,CAAC;AAED,kGAAkG;AAClG,MAAM,UAAU,gBAAgB,CAAC,GAAwB;IACvD,OAAO;QACL,WAAW,EAAE,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS;QACjD,YAAY,EAAE,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS;QACnD,KAAK,EAAE,CAAC,GAAG,GAAG,CAAC,KAAK,CAAC;QACrB,WAAW,EAAE,CAAC,GAAG,GAAG,CAAC,WAAW,CAAC;KAClC,CAAC;AACJ,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,eAAe,CAAC,QAAiB;IAC/C,MAAM,GAAG,GAAG,yBAAyB,EAAE,CAAC;IACxC,IAAI,CAAC;QACH,IAAI,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAC;YAC5B,KAAK,MAAM,CAAC,IAAI,QAAQ;gBAAE,iBAAiB,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC;QACtD,CAAC;IACH,CAAC;IAAC,MAAM,CAAC;QACP,eAAe;IACjB,CAAC;IACD,OAAO,gBAAgB,CAAC,GAAG,CAAC,CAAC;AAC/B,CAAC"}
|
package/dist/core/types.d.ts
CHANGED
|
@@ -38,6 +38,28 @@ export interface GthRunStats {
|
|
|
38
38
|
tokensOutput?: number;
|
|
39
39
|
/** Names of tools invoked during the run (deduplicated); empty when no tools were used. */
|
|
40
40
|
tools: string[];
|
|
41
|
+
/**
|
|
42
|
+
* BATCH-21 — one record per executed tool result (`ToolMessage`) observed during the run, in
|
|
43
|
+
* arrival order and NOT deduplicated (a tool called twice yields two records), so `gth eval`'s
|
|
44
|
+
* tool-RESULT assertions (`must_error` / `tool_result_json_path`) can grade what a tool
|
|
45
|
+
* *returned*, not just that it was called. Optional (additive): producers that predate the field
|
|
46
|
+
* simply omit it; {@link runStats.js finalizeRunStats} always sets it.
|
|
47
|
+
*/
|
|
48
|
+
toolResults?: GthToolResult[];
|
|
49
|
+
}
|
|
50
|
+
/**
|
|
51
|
+
* BATCH-21 — one executed tool call's result, harvested from its `ToolMessage` by the GS2-16
|
|
52
|
+
* run-stats accumulator (`core/runStats.ts`). Fail-soft like everything else there: `content` is
|
|
53
|
+
* omitted when no text payload could be derived, and is size-capped
|
|
54
|
+
* ({@link runStats.js TOOL_RESULT_CONTENT_CAP}) so a giant payload can't bloat run stats.
|
|
55
|
+
*/
|
|
56
|
+
export interface GthToolResult {
|
|
57
|
+
/** The tool that produced the result (`ToolMessage.name`). */
|
|
58
|
+
name: string;
|
|
59
|
+
/** `true` iff the result carried LangChain's real error signal (`ToolMessage.status === 'error'`). */
|
|
60
|
+
isError: boolean;
|
|
61
|
+
/** The result payload as text (a non-string payload is JSON-stringified), capped in length. */
|
|
62
|
+
content?: string;
|
|
41
63
|
}
|
|
42
64
|
/**
|
|
43
65
|
* Typed events emitted by the agent's {@link GthAgentInterface#streamWithEvents} path.
|