@gaunt-sloth/batch 2.0.0-beta.11 → 2.0.0-beta.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/toolResultChecks.d.ts +3 -0
- package/dist/toolResultChecks.js +11 -1
- package/dist/toolResultChecks.js.map +1 -1
- package/dist/types.d.ts +11 -0
- package/dist/types.js.map +1 -1
- package/package.json +3 -3
|
@@ -21,6 +21,9 @@ import type { ToolResultRecord } from '#src/types.js';
|
|
|
21
21
|
* `equals` deep-equals / `contains` substring-matches the resolved value. A non-JSON (or absent)
|
|
22
22
|
* payload is a deterministic per-result FAIL, never a throw — note capture caps payloads
|
|
23
23
|
* (`TOOL_RESULT_CONTENT_CAP`), so a truncated over-cap payload also fails as non-JSON.
|
|
24
|
+
* (BATCH-43) The payload it reads is the result's `errorPayload` when capture recovered one — an
|
|
25
|
+
* errored MCP tool's own error body, freed from the adapter's prose prefix — and otherwise the
|
|
26
|
+
* observed `content`, which is left exactly as the model saw it in either case.
|
|
24
27
|
*
|
|
25
28
|
* The runner only ever calls this with expectation blocks a `gth-agent` suite produced — the suite
|
|
26
29
|
* parser rejects tool-result assertions against `ag-ui`/`adk-agent` targets, whose wire carries no
|
package/dist/toolResultChecks.js
CHANGED
|
@@ -22,6 +22,9 @@ import { resolveJsonPath } from '#src/deterministicChecks.js';
|
|
|
22
22
|
* `equals` deep-equals / `contains` substring-matches the resolved value. A non-JSON (or absent)
|
|
23
23
|
* payload is a deterministic per-result FAIL, never a throw — note capture caps payloads
|
|
24
24
|
* (`TOOL_RESULT_CONTENT_CAP`), so a truncated over-cap payload also fails as non-JSON.
|
|
25
|
+
* (BATCH-43) The payload it reads is the result's `errorPayload` when capture recovered one — an
|
|
26
|
+
* errored MCP tool's own error body, freed from the adapter's prose prefix — and otherwise the
|
|
27
|
+
* observed `content`, which is left exactly as the model saw it in either case.
|
|
25
28
|
*
|
|
26
29
|
* The runner only ever calls this with expectation blocks a `gth-agent` suite produced — the suite
|
|
27
30
|
* parser rejects tool-result assertions against `ag-ui`/`adk-agent` targets, whose wire carries no
|
|
@@ -67,9 +70,16 @@ function checkToolResultJsonPath(toolResults, check) {
|
|
|
67
70
|
/** Evaluate ONE tool result against ONE check: `undefined` = satisfied, else the failure reason.
|
|
68
71
|
* Deterministic and throw-free: a non-JSON/absent payload is a reason, not an exception. */
|
|
69
72
|
function evaluateResultAgainstCheck(result, check) {
|
|
73
|
+
// BATCH-43 — grade the recovered MCP error body when capture produced one, else the observed
|
|
74
|
+
// payload exactly as before. An errored MCP tool's `content` is the adapter's prose-prefixed
|
|
75
|
+
// message and never parses; `errorPayload` is the server's own body with that prefix removed,
|
|
76
|
+
// and it is only ever recorded when it parses, so this branch cannot introduce a new failure —
|
|
77
|
+
// when it is absent (any non-MCP result, or an MCP error whose body is prose) the reason below
|
|
78
|
+
// is the same deterministic one the check has always given.
|
|
79
|
+
const payload = result.errorPayload ?? result.content ?? '';
|
|
70
80
|
let root;
|
|
71
81
|
try {
|
|
72
|
-
root = JSON.parse(
|
|
82
|
+
root = JSON.parse(payload.trim());
|
|
73
83
|
}
|
|
74
84
|
catch {
|
|
75
85
|
return 'result payload is not JSON';
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"toolResultChecks.js","sourceRoot":"","sources":["../src/toolResultChecks.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,MAAM,WAAW,CAAC;AAE9C,OAAO,EAAE,sBAAsB,EAAE,MAAM,yCAAyC,CAAC;AAEjF,OAAO,EAAE,eAAe,EAAE,MAAM,6BAA6B,CAAC;AAI9D
|
|
1
|
+
{"version":3,"file":"toolResultChecks.js","sourceRoot":"","sources":["../src/toolResultChecks.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,MAAM,WAAW,CAAC;AAE9C,OAAO,EAAE,sBAAsB,EAAE,MAAM,yCAAyC,CAAC;AAEjF,OAAO,EAAE,eAAe,EAAE,MAAM,6BAA6B,CAAC;AAI9D;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6BG;AACH,MAAM,UAAU,mBAAmB,CACjC,WAA+B,EAC/B,WAAsE;IAEtE,MAAM,QAAQ,GAAa,EAAE,CAAC;IAE9B,KAAK,MAAM,OAAO,IAAI,WAAW,CAAC,SAAS,EAAE,CAAC;QAC5C,MAAM,cAAc,GAAG,WAAW,CAAC,IAAI,CACrC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,IAAI,sBAAsB,CAAC,MAAM,CAAC,IAAI,EAAE,OAAO,CAAC,CAC3E,CAAC;QACF,IAAI,CAAC,cAAc,EAAE,CAAC;YACpB,QAAQ,CAAC,IAAI,CAAC,SAAS,OAAO,2BAA2B,CAAC,CAAC;QAC7D,CAAC;IACH,CAAC;IAED,KAAK,MAAM,KAAK,IAAI,WAAW,CAAC,kBAAkB,EAAE,CAAC;QACnD,MAAM,OAAO,GAAG,uBAAuB,CAAC,WAAW,EAAE,KAAK,CAAC,CAAC;QAC5D,IAAI,OAAO,KAAK,SAAS,EAAE,CAAC;YAC1B,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QACzB,CAAC;IACH,CAAC;IAED,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED;;0CAE0C;AAC1C,SAAS,uBAAuB,CAC9B,WAA+B,EAC/B,KAA8B;IAE9B,MAAM,KAAK,GAAG,0BAA0B,KAAK,CAAC,IAAI,YAAY,KAAK,CAAC,IAAI,IAAI,CAAC;IAC7E,MAAM,QAAQ,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,sBAAsB,CAAC,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC;IACjG,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC1B,OAAO,GAAG,KAAK,kCAAkC,CAAC;IACpD,CAAC;IAED,8FAA8F;IAC9F,mFAAmF;IACnF,MAAM,OAAO,GAAG,IAAI,GAAG,EAAU,CAAC;IAClC,KAAK,MAAM,MAAM,IAAI,QAAQ,EAAE,CAAC;QAC9B,MAAM,MAAM,GAAG,0BAA0B,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;QACzD,IAAI,MAAM,KAAK,SAAS;YAAE,OAAO,SAAS,CAAC;QAC3C,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;IACtB,CAAC;IACD,OAAO,GAAG,KAAK,KAAK,CAAC,GAAG,OAAO,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;AAChD,CAAC;AAED;4FAC4F;AAC5F,SAAS,0BAA0B,CACjC,MAAwB,EACxB,KAA8B;IAE9B,6FAA6F;IAC7F,6FAA6F;IAC7F,8FAA8F;IAC9F,+FAA+F;IAC/F,+FAA+F;IAC/F,4DAA4D;IAC5D,MAAM,OAAO,GAAG,MAAM,CAAC,YAAY,IAAI,MAAM,CAAC,OAAO,IAAI,EAAE,CAAC;IAC5D,IAAI,IAAa,CAAC;IAClB,IAAI,CAAC;QACH,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC;IACpC,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,4BAA4B,CAAC;IACtC,CAAC;IAED,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,GAAG,eAAe,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;IAC3D,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,OAAO,sBAAsB,CAAC;IAChC,CAAC;IAED,IAAI,KAAK,CAAC,QAAQ,KAAK,SAAS,EAAE,CAAC;QACjC,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;YAC9B,OAAO,MAAM,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,qCAAqC,CAAC;QAC1E,CAAC;QACD,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,QAAQ,CAAC,EAAE,CAAC;YACpC,OAAO,qBAAqB,KAAK,CAAC,QAAQ,GAAG,CAAC;QAChD,CAAC;QACD,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,8FAA8F;IAC9F,8FAA8F;IAC9F,qBAAqB;IACrB,IAAI,QAAQ,IAAI,KAAK,EAAE,CAAC;QACtB,IAAI,CAAC,iBAAiB,CAAC,KAAK,EAAE,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC;YAC5C,OAAO,MAAM,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,cAAc,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC;QACjF,CAAC;QACD,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,qFAAqF;IACrF,OAAO,SAAS,CAAC;AACnB,CAAC"}
|
package/dist/types.d.ts
CHANGED
|
@@ -47,6 +47,17 @@ export interface ToolResultRecord {
|
|
|
47
47
|
isError: boolean;
|
|
48
48
|
/** The result payload as text (non-string payloads JSON-stringified), size-capped at capture. */
|
|
49
49
|
content?: string;
|
|
50
|
+
/**
|
|
51
|
+
* BATCH-43 — an errored MCP tool's own error body, recovered at capture and carried BESIDE
|
|
52
|
+
* {@link content}, which stays exactly what the model observed. `tool_result_json_path` grades
|
|
53
|
+
* this when it is present; when it is absent the check reads {@link content} and behaves as it
|
|
54
|
+
* always has. Present implies it parses as JSON, so a check that finds it never has to ask.
|
|
55
|
+
*
|
|
56
|
+
* Absent for everything that is not an errored MCP call, and absent too for an MCP error whose
|
|
57
|
+
* body is prose rather than JSON — a server that puts its detail only in MCP `structuredContent`
|
|
58
|
+
* is not gradable at all, because the adapter discards that before any of our code runs.
|
|
59
|
+
*/
|
|
60
|
+
errorPayload?: string;
|
|
50
61
|
}
|
|
51
62
|
/** What one attempt at running a cell through the shared single-shot runtime produced. */
|
|
52
63
|
export interface CellRunOutcome {
|
package/dist/types.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types.js","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"types.js","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AA6IA;;;;;;GAMG;AACH,MAAM,CAAC,MAAM,wBAAwB,GAAG,CAAC,CAAC;AAE1C;;;;GAIG;AACH,MAAM,CAAC,MAAM,4BAA4B,GAAG,CAAC,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@gaunt-sloth/batch",
|
|
3
|
-
"version": "2.0.0-beta.
|
|
3
|
+
"version": "2.0.0-beta.12",
|
|
4
4
|
"description": "Batch matrix runtime for Gaunt Sloth (BATCH-1): run a prompt-executable over a matrix of models and/or content-bound inputs",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"author": "Andrew Kondratev",
|
|
@@ -37,8 +37,8 @@
|
|
|
37
37
|
"@langchain/core": "^1.2.11",
|
|
38
38
|
"yaml": "^2.9.1",
|
|
39
39
|
"zod": "^4.6.3",
|
|
40
|
-
"@gaunt-sloth/
|
|
41
|
-
"@gaunt-sloth/
|
|
40
|
+
"@gaunt-sloth/core": "2.0.0-beta.12",
|
|
41
|
+
"@gaunt-sloth/agent": "2.0.0-beta.12"
|
|
42
42
|
},
|
|
43
43
|
"files": [
|
|
44
44
|
"./dist/*"
|