@tracedojo/sdk 0.1.2 → 0.1.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +114 -74
- package/dist/cli.js +2 -2
- package/dist/demo.d.ts +48 -0
- package/dist/matrix-cli.d.ts +24 -0
- package/dist/matrix.d.ts +24 -0
- package/dist/schema.d.ts +72 -0
- package/dist/schema.js +62 -0
- package/dist/upload.d.ts +8 -0
- package/dist/upload.js +14 -0
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -1,96 +1,136 @@
|
|
|
1
|
-
# TraceDojo
|
|
1
|
+
# TraceDojo
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
**Chaos testing for AI agent tools.**
|
|
4
4
|
|
|
5
|
-
|
|
6
|
-
|
|
5
|
+
The booking succeeded. The response timed out. Your agent booked it again.
|
|
6
|
+
|
|
7
|
+
TraceDojo makes your agent's tools fail on purpose — lost confirmations,
|
|
8
|
+
duplicate deliveries, stale reads, injected instructions — and then checks what
|
|
9
|
+
actually changed in your application's state, not what the agent claimed.
|
|
10
|
+
|
|
11
|
+
[](https://www.npmjs.com/package/@tracedojo/sdk)
|
|
12
|
+
[](https://github.com/syedarman1/tracedojo-sdk/blob/main/packages/sdk/LICENSE)
|
|
13
|
+
|
|
14
|
+

|
|
15
|
+
|
|
16
|
+
## Try it in 30 seconds
|
|
17
|
+
|
|
18
|
+
No API key, no account. Node.js 22.23.1 or later.
|
|
7
19
|
|
|
8
20
|
```sh
|
|
9
|
-
npx @
|
|
21
|
+
npx tracedojo@0.1.2 demo
|
|
10
22
|
```
|
|
11
23
|
|
|
12
|
-
|
|
24
|
+
```text
|
|
25
|
+
A booking succeeded. Its confirmation timed out. The agent retried.
|
|
26
|
+
Before: failed — 2 committed writes; duplicate booking.
|
|
27
|
+
After: passed — 1 committed write; the retry reused the original booking.
|
|
28
|
+
Both no-fault controls passed. Scripted example; no model calls.
|
|
29
|
+
```
|
|
30
|
+
|
|
31
|
+
## What it catches
|
|
32
|
+
|
|
33
|
+
| Fault | What happens to the tool call | Typical agent bug |
|
|
34
|
+
| ------------------------------------------------------------------ | ------------------------------------------------- | -------------------------------------------- |
|
|
35
|
+
| `timeout_after` | The write commits, then the response is lost | Retry with a new key → duplicate record |
|
|
36
|
+
| `duplicate_delivery` | The same write is delivered twice | No idempotency → double charge or email |
|
|
37
|
+
| `stale_read` | A later read returns an earlier snapshot | Agent gives up, or "repairs" with a rewrite |
|
|
38
|
+
| `prompt_injection` | Tool output is replaced with attacker-chosen data | Agent calls a tool it must never call |
|
|
39
|
+
| `timeout_before`, `unavailable`, `malformed`, `permission_revoked` | The call fails in a controlled way | Wrong outcome, false "done", unsafe fallback |
|
|
40
|
+
|
|
41
|
+
Every scenario runs against a passing no-fault control first, so a failure means
|
|
42
|
+
the fault caused it. Faults that are never reached are reported as
|
|
43
|
+
inconclusive, not as passes.
|
|
44
|
+
|
|
45
|
+
## How it works
|
|
46
|
+
|
|
47
|
+
1. **Simulate your tools** in plain JavaScript against an in-memory state.
|
|
48
|
+
2. **Route your agent's tool calls** through `context.call()`. Bring any agent: scripted,
|
|
49
|
+
OpenAI, Claude, or LangChain JS (`@tracedojo/sdk/langchain`).
|
|
50
|
+
3. **Declare outcomes** on the resulting state in `workflow.json`:
|
|
51
|
+
|
|
52
|
+
```json
|
|
53
|
+
{
|
|
54
|
+
"checks": [
|
|
55
|
+
{
|
|
56
|
+
"id": "one-booking",
|
|
57
|
+
"type": "count",
|
|
58
|
+
"path": "/bookings",
|
|
59
|
+
"expected": 1
|
|
60
|
+
},
|
|
61
|
+
{ "id": "other-customer", "type": "unchanged", "path": "/otherCustomer" },
|
|
62
|
+
{
|
|
63
|
+
"id": "done",
|
|
64
|
+
"type": "equals",
|
|
65
|
+
"source": "completion",
|
|
66
|
+
"path": "/status",
|
|
67
|
+
"expected": "completed"
|
|
68
|
+
}
|
|
69
|
+
],
|
|
70
|
+
"scenarios": [
|
|
71
|
+
{
|
|
72
|
+
"id": "lost-confirmation",
|
|
73
|
+
"fault": { "type": "timeout_after", "tool": "reserve_slot" }
|
|
74
|
+
}
|
|
75
|
+
]
|
|
76
|
+
}
|
|
77
|
+
```
|
|
78
|
+
|
|
79
|
+
Then:
|
|
80
|
+
|
|
81
|
+
```sh
|
|
82
|
+
npm install --save-dev --save-exact @tracedojo/sdk@0.1.3
|
|
83
|
+
npx tracedojo init # starter adapter + workflow
|
|
84
|
+
npx tracedojo test --trials 3
|
|
85
|
+
```
|
|
86
|
+
|
|
87
|
+
Ten starter worlds are included — calendar, payments, refunds, CRM, email, files,
|
|
88
|
+
SQL, inventory, subscriptions, webhooks: `npx tracedojo init --template refunds`.
|
|
89
|
+
Or generate a fault matrix from your declared tools: `npx tracedojo matrix`.
|
|
90
|
+
|
|
91
|
+
## Gate pull requests
|
|
13
92
|
|
|
14
93
|
```sh
|
|
15
|
-
npm install --save-dev @tracedojo/sdk
|
|
16
94
|
npx tracedojo init --ci
|
|
17
|
-
npx tracedojo test
|
|
18
95
|
```
|
|
19
96
|
|
|
20
|
-
|
|
97
|
+
Generates a GitHub workflow that runs the suite on every PR, compares it against
|
|
98
|
+
the base commit, fails on new regressions, and keeps a single up-to-date PR
|
|
99
|
+
comment. No model credentials required for scripted agents. A standalone
|
|
100
|
+
[GitHub Action](https://github.com/marketplace/actions/tracedojo-agent-test) is
|
|
101
|
+
also available.
|
|
21
102
|
|
|
22
|
-
|
|
23
|
-
Transactions remain serialized and late results cannot commit after a trial stops.
|
|
24
|
-
Try `npx tracedojo init --virtual-time` for simulated service delays and retry
|
|
25
|
-
backoff. Use `clock.now()` / `await clock.sleep(ms)` in tools and agents. Reports
|
|
26
|
-
separate virtual elapsed time from actual runtime; native timers are unchanged.
|
|
103
|
+
## Start from a production trace
|
|
27
104
|
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
105
|
+
```sh
|
|
106
|
+
npx tracedojo import-langsmith trace.json --out imported-dojo
|
|
107
|
+
```
|
|
31
108
|
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
The comment script runs from that trusted branch; agent tests have read-only
|
|
35
|
-
permissions. Reports last 30 days, and missing baseline evidence blocks the gate.
|
|
109
|
+
Generates a runnable playback adapter with observed tool signatures and review
|
|
110
|
+
TODOs — a starting point for a real test, not a test of your agent by itself.
|
|
36
111
|
|
|
37
|
-
|
|
38
|
-
`npx tracedojo init --template refunds --out dojo`. Available worlds: calendar,
|
|
39
|
-
payments, refunds, CRM (`crm`), email, files, SQL (`sql`), inventory, subscriptions,
|
|
40
|
-
and webhooks. Each includes five faults, a reference agent, state checks, and a
|
|
41
|
-
documented retry bug. These are synthetic fixtures, not full service emulators.
|
|
112
|
+
## Real-model evidence
|
|
42
113
|
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
114
|
+
We ran our model-backed booking agent through the suite. One of three stale-read
|
|
115
|
+
trials stopped and reported itself blocked even though the booking existed. A
|
|
116
|
+
one-paragraph prompt change was followed by three passing trials under the same
|
|
117
|
+
faults. Small sample, synthetic data, full evidence downloadable:
|
|
118
|
+
[read the case study](https://tracedojo.com/docs/booking-case-study).
|
|
48
119
|
|
|
49
|
-
|
|
50
|
-
`duplicate_delivery` fault requires a write tool and preserves two transaction
|
|
51
|
-
records for one agent call. The fixture demonstrates service-side idempotency.
|
|
120
|
+
## Honest limits
|
|
52
121
|
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
122
|
+
TraceDojo tests behavior inside a simulation you define. A pass is evidence about
|
|
123
|
+
the tested scenario, not a guarantee of production reliability. Adapters are
|
|
124
|
+
trusted code, not a sandbox. Read the full
|
|
125
|
+
[limitations](https://tracedojo.com/docs/limitations) and
|
|
126
|
+
[security boundaries](https://tracedojo.com/docs/security).
|
|
57
127
|
|
|
58
|
-
|
|
128
|
+
## Docs and community
|
|
59
129
|
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
130
|
+
- [Setup guide](https://tracedojo.com/docs) · [Demos](https://tracedojo.com/demos) ·
|
|
131
|
+
[Full CLI reference](https://github.com/syedarman1/tracedojo-sdk/blob/main/docs/WORKFLOWS.md)
|
|
132
|
+
- Optional hosted dashboard for saving and comparing reports: [tracedojo.com](https://tracedojo.com)
|
|
133
|
+
- Bugs and feature requests: [open an issue](https://github.com/syedarman1/tracedojo-sdk/issues)
|
|
64
134
|
|
|
65
|
-
|
|
66
|
-
Review the editable matrix before running it: task, state, and outcome checks come
|
|
67
|
-
from your workflow, with matching scenario-specific checks preserved. The config
|
|
68
|
-
may omit `scenarios` entirely. Writes get six faults including lost confirmation
|
|
69
|
-
and duplicate delivery; reads get timeout, unavailability, and malformed output.
|
|
70
|
-
Use `--tools reserve_slot --faults timeout_after,duplicate_delivery` to narrow it.
|
|
71
|
-
Stale reads are opt-in (`--faults stale_read`); injection payloads require manual
|
|
72
|
-
review. Each tool gets a separate bounded suite and normal dashboard-compatible
|
|
73
|
-
report. `index.json` lists results and missing coverage; it is not a report import.
|
|
74
|
-
Unreached tools remain incomplete while other suites run. Interruptions and failed
|
|
75
|
-
controls stop remaining suites. This generates fault schedules, not business rules.
|
|
76
|
-
|
|
77
|
-
Start from a LangSmith JSON export with
|
|
78
|
-
`npx tracedojo import-langsmith trace.json --out imported-dojo`. This generates a
|
|
79
|
-
runnable playback adapter, observed signatures, and TODOs. Run it with
|
|
80
|
-
`npx tracedojo test --config imported-dojo/workflow.json --adapter imported-dojo/adapter.mjs`.
|
|
81
|
-
A successful, unambiguous trace passes its control and fails an injected timeout.
|
|
82
|
-
This demonstrates interrupted playback; connect your agent and define independent
|
|
83
|
-
state checks before treating it as an agent test. Use `--draft-only` to retain the
|
|
84
|
-
JSON-only import format.
|
|
85
|
-
Replace the task, in-memory tools, and outcome checks with your application's
|
|
86
|
-
behavior, and route your agent's tool calls through the SDK. LangChain JavaScript
|
|
87
|
-
tools are supported through `@tracedojo/sdk/langchain`.
|
|
88
|
-
|
|
89
|
-
The SDK, compiled distribution, documentation in this package, and starter
|
|
90
|
-
templates are licensed under [Apache-2.0](LICENSE). The hosted dashboard is
|
|
135
|
+
The SDK, launcher, and starter templates are Apache-2.0. The hosted dashboard is
|
|
91
136
|
proprietary and is not included in this package.
|
|
92
|
-
|
|
93
|
-
[Setup guide](https://tracedojo.com/docs)
|
|
94
|
-
· [Security](https://tracedojo.com/docs/security)
|
|
95
|
-
· [Limitations](https://tracedojo.com/docs/limitations)
|
|
96
|
-
· [Dashboard](https://tracedojo.com)
|
package/dist/cli.js
CHANGED
|
@@ -10,7 +10,7 @@ import { runWorkflow } from './index.js';
|
|
|
10
10
|
import { boundedJson, parseWorkflow, workflowExitCode, commitShaSchema, } from './schema.js';
|
|
11
11
|
import { sanitizeWorkflowBatch } from './privacy.js';
|
|
12
12
|
import { importLangSmithTrace } from './langsmith.js';
|
|
13
|
-
import { uploadWorkflowReports } from './upload.js';
|
|
13
|
+
import { uploadMessage, uploadWorkflowReports } from './upload.js';
|
|
14
14
|
import { createStarter } from './starter.js';
|
|
15
15
|
import { workflowSummary } from './summary.js';
|
|
16
16
|
import { runDemo } from './demo.js';
|
|
@@ -166,7 +166,7 @@ async function main() {
|
|
|
166
166
|
project: values.project ?? process.env.TRACEDOJO_PROJECT_ID ?? '',
|
|
167
167
|
token: process.env.TRACEDOJO_UPLOAD_TOKEN ?? '',
|
|
168
168
|
});
|
|
169
|
-
console.log(
|
|
169
|
+
console.log(uploadMessage(receipt));
|
|
170
170
|
return;
|
|
171
171
|
}
|
|
172
172
|
if (command === 'compare') {
|
package/dist/demo.d.ts
CHANGED
|
@@ -43,10 +43,22 @@ export declare function runDemo(directory: string): Promise<{
|
|
|
43
43
|
initialState: Record<string, import("zod").JSONType>;
|
|
44
44
|
finalState: Record<string, import("zod").JSONType>;
|
|
45
45
|
definitions: ({
|
|
46
|
+
id: string;
|
|
47
|
+
label: string;
|
|
48
|
+
type: "tool_calls";
|
|
49
|
+
tool: string;
|
|
50
|
+
min?: number | undefined;
|
|
51
|
+
max?: number | undefined;
|
|
52
|
+
} | {
|
|
46
53
|
id: string;
|
|
47
54
|
label: string;
|
|
48
55
|
type: "forbidden_tool";
|
|
49
56
|
tool: string;
|
|
57
|
+
} | {
|
|
58
|
+
id: string;
|
|
59
|
+
label: string;
|
|
60
|
+
type: "tool_order";
|
|
61
|
+
tools: string[];
|
|
50
62
|
} | {
|
|
51
63
|
id: string;
|
|
52
64
|
label: string;
|
|
@@ -156,10 +168,22 @@ export declare function runDemo(directory: string): Promise<{
|
|
|
156
168
|
initialState: Record<string, import("zod").JSONType>;
|
|
157
169
|
finalState: Record<string, import("zod").JSONType>;
|
|
158
170
|
definitions: ({
|
|
171
|
+
id: string;
|
|
172
|
+
label: string;
|
|
173
|
+
type: "tool_calls";
|
|
174
|
+
tool: string;
|
|
175
|
+
min?: number | undefined;
|
|
176
|
+
max?: number | undefined;
|
|
177
|
+
} | {
|
|
159
178
|
id: string;
|
|
160
179
|
label: string;
|
|
161
180
|
type: "forbidden_tool";
|
|
162
181
|
tool: string;
|
|
182
|
+
} | {
|
|
183
|
+
id: string;
|
|
184
|
+
label: string;
|
|
185
|
+
type: "tool_order";
|
|
186
|
+
tools: string[];
|
|
163
187
|
} | {
|
|
164
188
|
id: string;
|
|
165
189
|
label: string;
|
|
@@ -274,10 +298,22 @@ export declare function runDemo(directory: string): Promise<{
|
|
|
274
298
|
initialState: Record<string, import("zod").JSONType>;
|
|
275
299
|
finalState: Record<string, import("zod").JSONType>;
|
|
276
300
|
definitions: ({
|
|
301
|
+
id: string;
|
|
302
|
+
label: string;
|
|
303
|
+
type: "tool_calls";
|
|
304
|
+
tool: string;
|
|
305
|
+
min?: number | undefined;
|
|
306
|
+
max?: number | undefined;
|
|
307
|
+
} | {
|
|
277
308
|
id: string;
|
|
278
309
|
label: string;
|
|
279
310
|
type: "forbidden_tool";
|
|
280
311
|
tool: string;
|
|
312
|
+
} | {
|
|
313
|
+
id: string;
|
|
314
|
+
label: string;
|
|
315
|
+
type: "tool_order";
|
|
316
|
+
tools: string[];
|
|
281
317
|
} | {
|
|
282
318
|
id: string;
|
|
283
319
|
label: string;
|
|
@@ -387,10 +423,22 @@ export declare function runDemo(directory: string): Promise<{
|
|
|
387
423
|
initialState: Record<string, import("zod").JSONType>;
|
|
388
424
|
finalState: Record<string, import("zod").JSONType>;
|
|
389
425
|
definitions: ({
|
|
426
|
+
id: string;
|
|
427
|
+
label: string;
|
|
428
|
+
type: "tool_calls";
|
|
429
|
+
tool: string;
|
|
430
|
+
min?: number | undefined;
|
|
431
|
+
max?: number | undefined;
|
|
432
|
+
} | {
|
|
390
433
|
id: string;
|
|
391
434
|
label: string;
|
|
392
435
|
type: "forbidden_tool";
|
|
393
436
|
tool: string;
|
|
437
|
+
} | {
|
|
438
|
+
id: string;
|
|
439
|
+
label: string;
|
|
440
|
+
type: "tool_order";
|
|
441
|
+
tools: string[];
|
|
394
442
|
} | {
|
|
395
443
|
id: string;
|
|
396
444
|
label: string;
|
package/dist/matrix-cli.d.ts
CHANGED
|
@@ -12,10 +12,22 @@ export declare function writeFaultMatrix(input: unknown, adapterPath: string, de
|
|
|
12
12
|
task: string;
|
|
13
13
|
initialState: Record<string, import("zod").JSONType>;
|
|
14
14
|
checks: ({
|
|
15
|
+
id: string;
|
|
16
|
+
label: string;
|
|
17
|
+
type: "tool_calls";
|
|
18
|
+
tool: string;
|
|
19
|
+
min?: number | undefined;
|
|
20
|
+
max?: number | undefined;
|
|
21
|
+
} | {
|
|
15
22
|
id: string;
|
|
16
23
|
label: string;
|
|
17
24
|
type: "forbidden_tool";
|
|
18
25
|
tool: string;
|
|
26
|
+
} | {
|
|
27
|
+
id: string;
|
|
28
|
+
label: string;
|
|
29
|
+
type: "tool_order";
|
|
30
|
+
tools: string[];
|
|
19
31
|
} | {
|
|
20
32
|
id: string;
|
|
21
33
|
label: string;
|
|
@@ -53,10 +65,22 @@ export declare function writeFaultMatrix(input: unknown, adapterPath: string, de
|
|
|
53
65
|
type: "none";
|
|
54
66
|
};
|
|
55
67
|
checks?: ({
|
|
68
|
+
id: string;
|
|
69
|
+
label: string;
|
|
70
|
+
type: "tool_calls";
|
|
71
|
+
tool: string;
|
|
72
|
+
min?: number | undefined;
|
|
73
|
+
max?: number | undefined;
|
|
74
|
+
} | {
|
|
56
75
|
id: string;
|
|
57
76
|
label: string;
|
|
58
77
|
type: "forbidden_tool";
|
|
59
78
|
tool: string;
|
|
79
|
+
} | {
|
|
80
|
+
id: string;
|
|
81
|
+
label: string;
|
|
82
|
+
type: "tool_order";
|
|
83
|
+
tools: string[];
|
|
60
84
|
} | {
|
|
61
85
|
id: string;
|
|
62
86
|
label: string;
|
package/dist/matrix.d.ts
CHANGED
|
@@ -48,6 +48,18 @@ export declare const workflowMatrixSchema: z.ZodObject<{
|
|
|
48
48
|
label: z.ZodString;
|
|
49
49
|
type: z.ZodLiteral<"forbidden_tool">;
|
|
50
50
|
tool: z.ZodString;
|
|
51
|
+
}, z.core.$strict>, z.ZodObject<{
|
|
52
|
+
id: z.ZodString;
|
|
53
|
+
label: z.ZodString;
|
|
54
|
+
type: z.ZodLiteral<"tool_calls">;
|
|
55
|
+
tool: z.ZodString;
|
|
56
|
+
min: z.ZodOptional<z.ZodNumber>;
|
|
57
|
+
max: z.ZodOptional<z.ZodNumber>;
|
|
58
|
+
}, z.core.$strict>, z.ZodObject<{
|
|
59
|
+
id: z.ZodString;
|
|
60
|
+
label: z.ZodString;
|
|
61
|
+
type: z.ZodLiteral<"tool_order">;
|
|
62
|
+
tools: z.ZodArray<z.ZodString>;
|
|
51
63
|
}, z.core.$strict>, z.ZodObject<{
|
|
52
64
|
id: z.ZodString;
|
|
53
65
|
label: z.ZodString;
|
|
@@ -100,6 +112,18 @@ export declare const workflowMatrixSchema: z.ZodObject<{
|
|
|
100
112
|
label: z.ZodString;
|
|
101
113
|
type: z.ZodLiteral<"forbidden_tool">;
|
|
102
114
|
tool: z.ZodString;
|
|
115
|
+
}, z.core.$strict>, z.ZodObject<{
|
|
116
|
+
id: z.ZodString;
|
|
117
|
+
label: z.ZodString;
|
|
118
|
+
type: z.ZodLiteral<"tool_calls">;
|
|
119
|
+
tool: z.ZodString;
|
|
120
|
+
min: z.ZodOptional<z.ZodNumber>;
|
|
121
|
+
max: z.ZodOptional<z.ZodNumber>;
|
|
122
|
+
}, z.core.$strict>, z.ZodObject<{
|
|
123
|
+
id: z.ZodString;
|
|
124
|
+
label: z.ZodString;
|
|
125
|
+
type: z.ZodLiteral<"tool_order">;
|
|
126
|
+
tools: z.ZodArray<z.ZodString>;
|
|
103
127
|
}, z.core.$strict>, z.ZodObject<{
|
|
104
128
|
id: z.ZodString;
|
|
105
129
|
label: z.ZodString;
|
package/dist/schema.d.ts
CHANGED
|
@@ -39,6 +39,18 @@ export declare const checkSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
|
39
39
|
label: z.ZodString;
|
|
40
40
|
type: z.ZodLiteral<"forbidden_tool">;
|
|
41
41
|
tool: z.ZodString;
|
|
42
|
+
}, z.core.$strict>, z.ZodObject<{
|
|
43
|
+
id: z.ZodString;
|
|
44
|
+
label: z.ZodString;
|
|
45
|
+
type: z.ZodLiteral<"tool_calls">;
|
|
46
|
+
tool: z.ZodString;
|
|
47
|
+
min: z.ZodOptional<z.ZodNumber>;
|
|
48
|
+
max: z.ZodOptional<z.ZodNumber>;
|
|
49
|
+
}, z.core.$strict>, z.ZodObject<{
|
|
50
|
+
id: z.ZodString;
|
|
51
|
+
label: z.ZodString;
|
|
52
|
+
type: z.ZodLiteral<"tool_order">;
|
|
53
|
+
tools: z.ZodArray<z.ZodString>;
|
|
42
54
|
}, z.core.$strict>, z.ZodObject<{
|
|
43
55
|
id: z.ZodString;
|
|
44
56
|
label: z.ZodString;
|
|
@@ -78,6 +90,18 @@ export declare const workflowSchema: z.ZodObject<{
|
|
|
78
90
|
label: z.ZodString;
|
|
79
91
|
type: z.ZodLiteral<"forbidden_tool">;
|
|
80
92
|
tool: z.ZodString;
|
|
93
|
+
}, z.core.$strict>, z.ZodObject<{
|
|
94
|
+
id: z.ZodString;
|
|
95
|
+
label: z.ZodString;
|
|
96
|
+
type: z.ZodLiteral<"tool_calls">;
|
|
97
|
+
tool: z.ZodString;
|
|
98
|
+
min: z.ZodOptional<z.ZodNumber>;
|
|
99
|
+
max: z.ZodOptional<z.ZodNumber>;
|
|
100
|
+
}, z.core.$strict>, z.ZodObject<{
|
|
101
|
+
id: z.ZodString;
|
|
102
|
+
label: z.ZodString;
|
|
103
|
+
type: z.ZodLiteral<"tool_order">;
|
|
104
|
+
tools: z.ZodArray<z.ZodString>;
|
|
81
105
|
}, z.core.$strict>, z.ZodObject<{
|
|
82
106
|
id: z.ZodString;
|
|
83
107
|
label: z.ZodString;
|
|
@@ -130,6 +154,18 @@ export declare const workflowSchema: z.ZodObject<{
|
|
|
130
154
|
label: z.ZodString;
|
|
131
155
|
type: z.ZodLiteral<"forbidden_tool">;
|
|
132
156
|
tool: z.ZodString;
|
|
157
|
+
}, z.core.$strict>, z.ZodObject<{
|
|
158
|
+
id: z.ZodString;
|
|
159
|
+
label: z.ZodString;
|
|
160
|
+
type: z.ZodLiteral<"tool_calls">;
|
|
161
|
+
tool: z.ZodString;
|
|
162
|
+
min: z.ZodOptional<z.ZodNumber>;
|
|
163
|
+
max: z.ZodOptional<z.ZodNumber>;
|
|
164
|
+
}, z.core.$strict>, z.ZodObject<{
|
|
165
|
+
id: z.ZodString;
|
|
166
|
+
label: z.ZodString;
|
|
167
|
+
type: z.ZodLiteral<"tool_order">;
|
|
168
|
+
tools: z.ZodArray<z.ZodString>;
|
|
133
169
|
}, z.core.$strict>, z.ZodObject<{
|
|
134
170
|
id: z.ZodString;
|
|
135
171
|
label: z.ZodString;
|
|
@@ -315,6 +351,18 @@ export declare const workflowReportSchema: z.ZodObject<{
|
|
|
315
351
|
label: z.ZodString;
|
|
316
352
|
type: z.ZodLiteral<"forbidden_tool">;
|
|
317
353
|
tool: z.ZodString;
|
|
354
|
+
}, z.core.$strict>, z.ZodObject<{
|
|
355
|
+
id: z.ZodString;
|
|
356
|
+
label: z.ZodString;
|
|
357
|
+
type: z.ZodLiteral<"tool_calls">;
|
|
358
|
+
tool: z.ZodString;
|
|
359
|
+
min: z.ZodOptional<z.ZodNumber>;
|
|
360
|
+
max: z.ZodOptional<z.ZodNumber>;
|
|
361
|
+
}, z.core.$strict>, z.ZodObject<{
|
|
362
|
+
id: z.ZodString;
|
|
363
|
+
label: z.ZodString;
|
|
364
|
+
type: z.ZodLiteral<"tool_order">;
|
|
365
|
+
tools: z.ZodArray<z.ZodString>;
|
|
318
366
|
}, z.core.$strict>, z.ZodObject<{
|
|
319
367
|
id: z.ZodString;
|
|
320
368
|
label: z.ZodString;
|
|
@@ -477,6 +525,18 @@ export declare const workflowBatchSchema: z.ZodObject<{
|
|
|
477
525
|
label: z.ZodString;
|
|
478
526
|
type: z.ZodLiteral<"forbidden_tool">;
|
|
479
527
|
tool: z.ZodString;
|
|
528
|
+
}, z.core.$strict>, z.ZodObject<{
|
|
529
|
+
id: z.ZodString;
|
|
530
|
+
label: z.ZodString;
|
|
531
|
+
type: z.ZodLiteral<"tool_calls">;
|
|
532
|
+
tool: z.ZodString;
|
|
533
|
+
min: z.ZodOptional<z.ZodNumber>;
|
|
534
|
+
max: z.ZodOptional<z.ZodNumber>;
|
|
535
|
+
}, z.core.$strict>, z.ZodObject<{
|
|
536
|
+
id: z.ZodString;
|
|
537
|
+
label: z.ZodString;
|
|
538
|
+
type: z.ZodLiteral<"tool_order">;
|
|
539
|
+
tools: z.ZodArray<z.ZodString>;
|
|
480
540
|
}, z.core.$strict>, z.ZodObject<{
|
|
481
541
|
id: z.ZodString;
|
|
482
542
|
label: z.ZodString;
|
|
@@ -636,6 +696,18 @@ export declare const workflowBatchSchema: z.ZodObject<{
|
|
|
636
696
|
label: z.ZodString;
|
|
637
697
|
type: z.ZodLiteral<"forbidden_tool">;
|
|
638
698
|
tool: z.ZodString;
|
|
699
|
+
}, z.core.$strict>, z.ZodObject<{
|
|
700
|
+
id: z.ZodString;
|
|
701
|
+
label: z.ZodString;
|
|
702
|
+
type: z.ZodLiteral<"tool_calls">;
|
|
703
|
+
tool: z.ZodString;
|
|
704
|
+
min: z.ZodOptional<z.ZodNumber>;
|
|
705
|
+
max: z.ZodOptional<z.ZodNumber>;
|
|
706
|
+
}, z.core.$strict>, z.ZodObject<{
|
|
707
|
+
id: z.ZodString;
|
|
708
|
+
label: z.ZodString;
|
|
709
|
+
type: z.ZodLiteral<"tool_order">;
|
|
710
|
+
tools: z.ZodArray<z.ZodString>;
|
|
639
711
|
}, z.core.$strict>, z.ZodObject<{
|
|
640
712
|
id: z.ZodString;
|
|
641
713
|
label: z.ZodString;
|
package/dist/schema.js
CHANGED
|
@@ -93,6 +93,23 @@ export const checkSchema = z.discriminatedUnion('type', [
|
|
|
93
93
|
type: z.literal('forbidden_tool'),
|
|
94
94
|
tool: name,
|
|
95
95
|
}),
|
|
96
|
+
z
|
|
97
|
+
.strictObject({
|
|
98
|
+
id: name,
|
|
99
|
+
label: text,
|
|
100
|
+
type: z.literal('tool_calls'),
|
|
101
|
+
tool: name,
|
|
102
|
+
min: z.number().int().min(0).max(1000).optional(),
|
|
103
|
+
max: z.number().int().min(0).max(1000).optional(),
|
|
104
|
+
})
|
|
105
|
+
.refine((c) => (c.min !== undefined || c.max !== undefined) &&
|
|
106
|
+
(c.min === undefined || c.max === undefined || c.min <= c.max), 'Set min, max, or both, with min no greater than max.'),
|
|
107
|
+
z.strictObject({
|
|
108
|
+
id: name,
|
|
109
|
+
label: text,
|
|
110
|
+
type: z.literal('tool_order'),
|
|
111
|
+
tools: z.array(name).min(2).max(20),
|
|
112
|
+
}),
|
|
96
113
|
z.strictObject({
|
|
97
114
|
id: name,
|
|
98
115
|
label: text,
|
|
@@ -233,6 +250,51 @@ export function evaluateChecks(definitions, initial, final, completion, events)
|
|
|
233
250
|
evidenceIds: attempts.map((event) => event.id),
|
|
234
251
|
};
|
|
235
252
|
}
|
|
253
|
+
if (check.type === 'tool_calls') {
|
|
254
|
+
const calls = events.filter((event) => event.kind === 'tool_call' && event.tool === check.tool);
|
|
255
|
+
const n = calls.length;
|
|
256
|
+
const passed = (check.min === undefined || n >= check.min) &&
|
|
257
|
+
(check.max === undefined || n <= check.max);
|
|
258
|
+
const range = check.min === undefined
|
|
259
|
+
? `at most ${check.max}`
|
|
260
|
+
: check.max === undefined
|
|
261
|
+
? `at least ${check.min}`
|
|
262
|
+
: check.min === check.max
|
|
263
|
+
? `exactly ${check.min}`
|
|
264
|
+
: `between ${check.min} and ${check.max}`;
|
|
265
|
+
return {
|
|
266
|
+
id: check.id,
|
|
267
|
+
label: check.label,
|
|
268
|
+
passed,
|
|
269
|
+
detail: `The agent called ${check.tool} ${n} time${n === 1 ? '' : 's'}; expected ${range}.`,
|
|
270
|
+
actual: n,
|
|
271
|
+
expected: {
|
|
272
|
+
...(check.min !== undefined ? { min: check.min } : {}),
|
|
273
|
+
...(check.max !== undefined ? { max: check.max } : {}),
|
|
274
|
+
},
|
|
275
|
+
evidenceIds: calls.map((event) => event.id),
|
|
276
|
+
};
|
|
277
|
+
}
|
|
278
|
+
if (check.type === 'tool_order') {
|
|
279
|
+
const calls = events.filter((event) => event.kind === 'tool_call');
|
|
280
|
+
const matched = [];
|
|
281
|
+
for (const event of calls)
|
|
282
|
+
if (matched.length < check.tools.length &&
|
|
283
|
+
event.tool === check.tools[matched.length])
|
|
284
|
+
matched.push(event);
|
|
285
|
+
const passed = matched.length === check.tools.length;
|
|
286
|
+
return {
|
|
287
|
+
id: check.id,
|
|
288
|
+
label: check.label,
|
|
289
|
+
passed,
|
|
290
|
+
detail: passed
|
|
291
|
+
? 'The agent called these tools in the expected order.'
|
|
292
|
+
: `No ${check.tools[matched.length]} call followed ${matched.length ? check.tools[matched.length - 1] : 'the start'} in the expected order.`,
|
|
293
|
+
actual: calls.map((event) => event.tool ?? null),
|
|
294
|
+
expected: check.tools,
|
|
295
|
+
evidenceIds: matched.map((event) => event.id),
|
|
296
|
+
};
|
|
297
|
+
}
|
|
236
298
|
const prior = at(initial, check.path);
|
|
237
299
|
const observed = at(check.type === 'equals' && check.source === 'completion'
|
|
238
300
|
? completion
|
package/dist/upload.d.ts
CHANGED
|
@@ -4,5 +4,13 @@ export declare function uploadWorkflowReports(input: unknown, options: {
|
|
|
4
4
|
token: string;
|
|
5
5
|
}, send?: typeof fetch): Promise<{
|
|
6
6
|
count: number;
|
|
7
|
+
inserted: number;
|
|
8
|
+
duplicates: number;
|
|
7
9
|
url: string;
|
|
8
10
|
}>;
|
|
11
|
+
/** Report what the server stored, not how many reports the file held. */
|
|
12
|
+
export declare function uploadMessage(receipt: {
|
|
13
|
+
inserted: number;
|
|
14
|
+
duplicates: number;
|
|
15
|
+
url: string;
|
|
16
|
+
}): string;
|
package/dist/upload.js
CHANGED
|
@@ -18,6 +18,8 @@ export async function uploadWorkflowReports(input, options, send = fetch) {
|
|
|
18
18
|
throw new Error('Use an HTTPS origin, or HTTP on localhost for development.');
|
|
19
19
|
const reports = parseWorkflowReports(input);
|
|
20
20
|
const endpoint = new URL(`/api/projects/${options.project}/runs`, origin);
|
|
21
|
+
let inserted = 0;
|
|
22
|
+
let duplicates = 0;
|
|
21
23
|
for (let start = 0; start < reports.length; start += 21) {
|
|
22
24
|
const chunk = reports.slice(start, start + 21);
|
|
23
25
|
const body = JSON.stringify({
|
|
@@ -63,11 +65,23 @@ export async function uploadWorkflowReports(input, options, send = fetch) {
|
|
|
63
65
|
receipt.data.runIds.length !== chunk.length ||
|
|
64
66
|
receipt.data.runIds.some((id, i) => id !== chunk[i].id))
|
|
65
67
|
throw new Error('Unexpected upload receipt. The local report is preserved; retry safely.');
|
|
68
|
+
inserted += receipt.data.inserted;
|
|
69
|
+
duplicates += receipt.data.duplicates;
|
|
66
70
|
break;
|
|
67
71
|
}
|
|
68
72
|
}
|
|
69
73
|
return {
|
|
70
74
|
count: reports.length,
|
|
75
|
+
inserted,
|
|
76
|
+
duplicates,
|
|
71
77
|
url: new URL(`/projects/${options.project}/workflows`, origin).href,
|
|
72
78
|
};
|
|
73
79
|
}
|
|
80
|
+
/** Report what the server stored, not how many reports the file held. */
|
|
81
|
+
export function uploadMessage(receipt) {
|
|
82
|
+
const stored = `${receipt.inserted} new ${receipt.inserted === 1 ? 'report' : 'reports'}`;
|
|
83
|
+
const counts = receipt.duplicates
|
|
84
|
+
? `${stored}, ${receipt.duplicates} already stored`
|
|
85
|
+
: stored;
|
|
86
|
+
return `Uploaded ${counts}. ${receipt.url}`;
|
|
87
|
+
}
|