@tracedojo/sdk 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +202 -0
- package/NOTICE +5 -0
- package/README.md +96 -0
- package/dist/cli.d.ts +2 -0
- package/dist/cli.js +356 -0
- package/dist/clock.d.ts +41 -0
- package/dist/clock.js +118 -0
- package/dist/compare.d.ts +20 -0
- package/dist/compare.js +147 -0
- package/dist/demo.d.ts +465 -0
- package/dist/demo.js +45 -0
- package/dist/index.d.ts +35 -0
- package/dist/index.js +515 -0
- package/dist/langchain.d.ts +9 -0
- package/dist/langchain.js +17 -0
- package/dist/langsmith-starter.d.ts +4 -0
- package/dist/langsmith-starter.js +77 -0
- package/dist/langsmith.d.ts +30 -0
- package/dist/langsmith.js +148 -0
- package/dist/matrix-cli.d.ts +107 -0
- package/dist/matrix-cli.js +154 -0
- package/dist/matrix.d.ts +138 -0
- package/dist/matrix.js +148 -0
- package/dist/privacy.d.ts +5 -0
- package/dist/privacy.js +51 -0
- package/dist/reports.d.ts +2 -0
- package/dist/reports.js +38 -0
- package/dist/schema.d.ts +715 -0
- package/dist/schema.js +433 -0
- package/dist/starter.d.ts +1 -0
- package/dist/starter.js +64 -0
- package/dist/summary.d.ts +2 -0
- package/dist/summary.js +50 -0
- package/dist/trends.d.ts +25 -0
- package/dist/trends.js +164 -0
- package/dist/upload.d.ts +8 -0
- package/dist/upload.js +73 -0
- package/dist/worlds.d.ts +9 -0
- package/dist/worlds.js +279 -0
- package/package.json +85 -0
- package/templates/README.md +65 -0
- package/templates/adapter.mjs +88 -0
- package/templates/ci-comment.yml +40 -0
- package/templates/ci.yml +100 -0
- package/templates/clock-adapter.mjs +42 -0
- package/templates/comment.cjs +209 -0
- package/templates/faults/duplicate-delivery/README.md +19 -0
- package/templates/faults/duplicate-delivery/adapter.mjs +40 -0
- package/templates/faults/duplicate-delivery/workflow.json +31 -0
- package/templates/faults/prompt-injection/README.md +17 -0
- package/templates/faults/prompt-injection/adapter.mjs +36 -0
- package/templates/faults/prompt-injection/workflow.json +43 -0
- package/templates/faults/stale-read/README.md +21 -0
- package/templates/faults/stale-read/adapter.mjs +59 -0
- package/templates/faults/stale-read/workflow.json +33 -0
- package/templates/langsmith-adapter.mjs +83 -0
- package/templates/workflow.json +106 -0
- package/templates/world-adapter.mjs +67 -0
package/dist/index.js
ADDED
|
@@ -0,0 +1,515 @@
|
|
|
1
|
+
// SPDX-License-Identifier: Apache-2.0
|
|
2
|
+
// Copyright 2026 Syed Arman
|
|
3
|
+
import { createHash, randomUUID } from 'node:crypto';
|
|
4
|
+
import { z } from 'zod';
|
|
5
|
+
import { boundedJson, completionSchema, commitShaSchema, evaluateChecks, parseWorkflow, resultSchema, stableJson, workflowBatchSchema, workflowReportSchema, } from './schema.js';
|
|
6
|
+
export * from './schema.js';
|
|
7
|
+
import { TrialClock } from './clock.js';
|
|
8
|
+
const metadata = z.strictObject({
|
|
9
|
+
id: z.string().regex(/^[a-zA-Z0-9][a-zA-Z0-9_-]{0,99}$/),
|
|
10
|
+
version: z.string().trim().min(1).max(2000),
|
|
11
|
+
});
|
|
12
|
+
const failure = (code, message, retryable = false) => ({ ok: false, error: { code, message, retryable } });
|
|
13
|
+
class Stop extends Error {
|
|
14
|
+
reason;
|
|
15
|
+
constructor(reason) {
|
|
16
|
+
super(reason);
|
|
17
|
+
this.reason = reason;
|
|
18
|
+
}
|
|
19
|
+
}
|
|
20
|
+
async function trial(workflow, adapter, agent, scenario, fingerprint, execution, signal) {
|
|
21
|
+
const start = performance.now();
|
|
22
|
+
const initialState = structuredClone(workflow.initialState);
|
|
23
|
+
let state = structuredClone(initialState);
|
|
24
|
+
const events = [];
|
|
25
|
+
const controller = new AbortController();
|
|
26
|
+
let issue;
|
|
27
|
+
let completion;
|
|
28
|
+
let closed = false;
|
|
29
|
+
let pending = 0;
|
|
30
|
+
let calls = 0;
|
|
31
|
+
let admitted = 0;
|
|
32
|
+
let occurrences = 0;
|
|
33
|
+
let triggerCount = 0;
|
|
34
|
+
let evidenceBytes = 0;
|
|
35
|
+
let fatal;
|
|
36
|
+
let rejectClock;
|
|
37
|
+
const clockFailure = new Promise((_, reject) => {
|
|
38
|
+
rejectClock = reject;
|
|
39
|
+
});
|
|
40
|
+
const clock = new TrialClock(workflow.clock, controller.signal, (reason) => {
|
|
41
|
+
fatal ??= new Stop(reason);
|
|
42
|
+
rejectClock(fatal);
|
|
43
|
+
});
|
|
44
|
+
const revoked = new Set();
|
|
45
|
+
const patterns = new Map();
|
|
46
|
+
const readSnapshots = new Map();
|
|
47
|
+
const fault = scenario.fault;
|
|
48
|
+
const emit = (event) => {
|
|
49
|
+
boundedJson(event, 100000);
|
|
50
|
+
evidenceBytes += Buffer.byteLength(JSON.stringify(event));
|
|
51
|
+
if (events.length >= 990 || evidenceBytes > 600000)
|
|
52
|
+
throw new Stop('evidence_limit');
|
|
53
|
+
events.push(structuredClone({
|
|
54
|
+
...event,
|
|
55
|
+
id: `evt_${events.length + 1}`,
|
|
56
|
+
sequence: events.length + 1,
|
|
57
|
+
...(workflow.clock ? { timeMs: clock.api.now() } : {}),
|
|
58
|
+
}));
|
|
59
|
+
};
|
|
60
|
+
const inject = (tool, callId, extra = {}) => {
|
|
61
|
+
emit({
|
|
62
|
+
kind: 'fault',
|
|
63
|
+
visibility: 'evaluator',
|
|
64
|
+
tool,
|
|
65
|
+
callId,
|
|
66
|
+
detail: `Injected ${fault.type}.`,
|
|
67
|
+
payload: {
|
|
68
|
+
...extra,
|
|
69
|
+
occurrence: occurrences,
|
|
70
|
+
triggerCount: triggerCount + 1,
|
|
71
|
+
},
|
|
72
|
+
});
|
|
73
|
+
triggerCount++;
|
|
74
|
+
};
|
|
75
|
+
emit({ kind: 'request', visibility: 'agent', detail: workflow.task });
|
|
76
|
+
const guard = () => {
|
|
77
|
+
if (closed ||
|
|
78
|
+
controller.signal.aborted ||
|
|
79
|
+
performance.now() - start >= workflow.limits.timeoutMs)
|
|
80
|
+
throw new Stop(signal?.aborted ? 'cancelled' : 'timed_out');
|
|
81
|
+
if (fatal)
|
|
82
|
+
throw fatal;
|
|
83
|
+
};
|
|
84
|
+
const execute = async (tool, input) => {
|
|
85
|
+
guard();
|
|
86
|
+
if (calls >= workflow.limits.maxToolCalls)
|
|
87
|
+
throw new Stop('call_limit');
|
|
88
|
+
boundedJson(input, 16000);
|
|
89
|
+
if (!input ||
|
|
90
|
+
Array.isArray(input) ||
|
|
91
|
+
typeof input !== 'object' ||
|
|
92
|
+
typeof tool !== 'string')
|
|
93
|
+
throw new Stop('tool_contract');
|
|
94
|
+
const args = structuredClone(input);
|
|
95
|
+
const key = stableJson([tool, args]);
|
|
96
|
+
const count = (patterns.get(key) ?? 0) + 1;
|
|
97
|
+
if (count > workflow.limits.maxRepeatedCalls)
|
|
98
|
+
throw new Stop('repeated_calls');
|
|
99
|
+
patterns.set(key, count);
|
|
100
|
+
const callId = `call_${++calls}`;
|
|
101
|
+
emit({
|
|
102
|
+
kind: 'tool_call',
|
|
103
|
+
visibility: 'agent',
|
|
104
|
+
tool,
|
|
105
|
+
callId,
|
|
106
|
+
detail: `Call ${calls}`,
|
|
107
|
+
payload: args,
|
|
108
|
+
});
|
|
109
|
+
const definition = Object.hasOwn(adapter.tools, tool)
|
|
110
|
+
? adapter.tools[tool]
|
|
111
|
+
: undefined;
|
|
112
|
+
const matches = fault.type !== 'none' && tool === fault.tool;
|
|
113
|
+
if (matches)
|
|
114
|
+
occurrences++;
|
|
115
|
+
const target = matches &&
|
|
116
|
+
occurrences >= fault.occurrence &&
|
|
117
|
+
occurrences < fault.occurrence + fault.repeat;
|
|
118
|
+
let response;
|
|
119
|
+
if (!definition)
|
|
120
|
+
response = failure('UNKNOWN_TOOL', 'This tool is not registered in the simulated workflow.');
|
|
121
|
+
else if (target &&
|
|
122
|
+
(fault.type === 'timeout_before' || fault.type === 'unavailable')) {
|
|
123
|
+
inject(tool, callId);
|
|
124
|
+
response =
|
|
125
|
+
fault.type === 'timeout_before'
|
|
126
|
+
? failure('TIMEOUT', 'The request timed out. Its outcome could not be confirmed.', true)
|
|
127
|
+
: failure('SERVICE_UNAVAILABLE', 'The service is temporarily unavailable.', true);
|
|
128
|
+
}
|
|
129
|
+
else {
|
|
130
|
+
if (target && fault.type === 'permission_revoked') {
|
|
131
|
+
revoked.add(tool);
|
|
132
|
+
inject(tool, callId);
|
|
133
|
+
}
|
|
134
|
+
if (revoked.has(tool))
|
|
135
|
+
response = failure('PERMISSION_DENIED', 'Write access to this tool has been revoked.');
|
|
136
|
+
else {
|
|
137
|
+
const deliver = async (delivery) => {
|
|
138
|
+
guard();
|
|
139
|
+
if (target && fault.type === 'duplicate_delivery')
|
|
140
|
+
emit({
|
|
141
|
+
kind: 'delivery',
|
|
142
|
+
visibility: 'evaluator',
|
|
143
|
+
tool,
|
|
144
|
+
callId,
|
|
145
|
+
detail: `Simulated write delivery ${delivery}.`,
|
|
146
|
+
payload: { delivery, args },
|
|
147
|
+
});
|
|
148
|
+
const draft = structuredClone(state);
|
|
149
|
+
try {
|
|
150
|
+
const raw = await definition.execute(structuredClone(args), draft, {
|
|
151
|
+
signal: controller.signal,
|
|
152
|
+
clock: clock.api,
|
|
153
|
+
});
|
|
154
|
+
// A late result cannot commit after cancellation, a deadline, or completion.
|
|
155
|
+
guard();
|
|
156
|
+
boundedJson(raw, 16000);
|
|
157
|
+
boundedJson(draft, 100000);
|
|
158
|
+
const original = resultSchema.parse(raw);
|
|
159
|
+
const changed = stableJson(state) !== stableJson(draft);
|
|
160
|
+
if (changed && (definition.kind === 'read' || !original.ok))
|
|
161
|
+
throw new Stop('tool_contract');
|
|
162
|
+
if (changed) {
|
|
163
|
+
// Reserve evidence before committing; an evidence overflow cannot hide a write.
|
|
164
|
+
emit({
|
|
165
|
+
kind: 'mutation',
|
|
166
|
+
visibility: 'evaluator',
|
|
167
|
+
tool,
|
|
168
|
+
callId,
|
|
169
|
+
detail: 'Committed in the simulated tool. Hidden from the agent.',
|
|
170
|
+
payload: { before: state, after: draft },
|
|
171
|
+
});
|
|
172
|
+
state = structuredClone(draft);
|
|
173
|
+
}
|
|
174
|
+
return structuredClone(original);
|
|
175
|
+
}
|
|
176
|
+
catch (error) {
|
|
177
|
+
if (fatal)
|
|
178
|
+
throw fatal;
|
|
179
|
+
guard();
|
|
180
|
+
throw error instanceof Stop ? error : new Stop('tool_contract');
|
|
181
|
+
}
|
|
182
|
+
};
|
|
183
|
+
const original = await deliver(1);
|
|
184
|
+
if (target && fault.type === 'duplicate_delivery') {
|
|
185
|
+
emit({
|
|
186
|
+
kind: 'tool_result',
|
|
187
|
+
visibility: 'evaluator',
|
|
188
|
+
tool,
|
|
189
|
+
callId,
|
|
190
|
+
detail: 'First delivery response; this is the response returned to the agent.',
|
|
191
|
+
payload: original,
|
|
192
|
+
});
|
|
193
|
+
inject(tool, callId);
|
|
194
|
+
const duplicate = await deliver(2);
|
|
195
|
+
emit({
|
|
196
|
+
kind: 'tool_result',
|
|
197
|
+
visibility: 'evaluator',
|
|
198
|
+
tool,
|
|
199
|
+
callId,
|
|
200
|
+
detail: 'Duplicate delivery response, hidden from the agent.',
|
|
201
|
+
payload: duplicate,
|
|
202
|
+
});
|
|
203
|
+
}
|
|
204
|
+
response = original;
|
|
205
|
+
if (matches && fault.type === 'stale_read' && original.ok) {
|
|
206
|
+
const snapshot = readSnapshots.get(key);
|
|
207
|
+
if (target &&
|
|
208
|
+
snapshot &&
|
|
209
|
+
stableJson(snapshot.result) !== stableJson(original)) {
|
|
210
|
+
emit({
|
|
211
|
+
kind: 'tool_result',
|
|
212
|
+
visibility: 'evaluator',
|
|
213
|
+
tool,
|
|
214
|
+
callId,
|
|
215
|
+
detail: 'Fresh read response, hidden from the agent.',
|
|
216
|
+
payload: original,
|
|
217
|
+
});
|
|
218
|
+
inject(tool, callId, { sourceCallId: snapshot.callId });
|
|
219
|
+
response = structuredClone(snapshot.result);
|
|
220
|
+
}
|
|
221
|
+
// Pin the first successful read of these inputs for this trial only.
|
|
222
|
+
if (!snapshot)
|
|
223
|
+
readSnapshots.set(key, {
|
|
224
|
+
result: structuredClone(original),
|
|
225
|
+
callId,
|
|
226
|
+
});
|
|
227
|
+
}
|
|
228
|
+
if (target &&
|
|
229
|
+
((fault.type === 'timeout_after' && original.ok) ||
|
|
230
|
+
(fault.type === 'prompt_injection' && original.ok) ||
|
|
231
|
+
fault.type === 'malformed')) {
|
|
232
|
+
emit({
|
|
233
|
+
kind: 'tool_result',
|
|
234
|
+
visibility: 'evaluator',
|
|
235
|
+
tool,
|
|
236
|
+
callId,
|
|
237
|
+
detail: 'Original tool response, hidden from the agent.',
|
|
238
|
+
payload: original,
|
|
239
|
+
});
|
|
240
|
+
inject(tool, callId);
|
|
241
|
+
response =
|
|
242
|
+
fault.type === 'timeout_after'
|
|
243
|
+
? failure('TIMEOUT', 'The request timed out. Its outcome could not be confirmed.', true)
|
|
244
|
+
: fault.type === 'prompt_injection'
|
|
245
|
+
? { ok: true, data: structuredClone(fault.replacement) }
|
|
246
|
+
: '{"ok":true,"data":';
|
|
247
|
+
}
|
|
248
|
+
}
|
|
249
|
+
}
|
|
250
|
+
emit({
|
|
251
|
+
kind: 'tool_result',
|
|
252
|
+
visibility: 'agent',
|
|
253
|
+
tool,
|
|
254
|
+
callId,
|
|
255
|
+
detail: 'Response delivered to the agent.',
|
|
256
|
+
payload: response,
|
|
257
|
+
});
|
|
258
|
+
return structuredClone(response);
|
|
259
|
+
};
|
|
260
|
+
// Serialize whole transactions, including awaits. Each draft starts from the
|
|
261
|
+
// previous committed state; overlapping agent calls cannot lose each other's writes.
|
|
262
|
+
let tail = Promise.resolve();
|
|
263
|
+
const call = (tool, args) => {
|
|
264
|
+
pending++;
|
|
265
|
+
const previous = tail;
|
|
266
|
+
const promise = (async () => {
|
|
267
|
+
try {
|
|
268
|
+
guard();
|
|
269
|
+
if (++admitted > workflow.limits.maxToolCalls)
|
|
270
|
+
throw new Stop('call_limit');
|
|
271
|
+
boundedJson(args, 16000);
|
|
272
|
+
const capturedArgs = structuredClone(args);
|
|
273
|
+
await previous;
|
|
274
|
+
const release = clock.hold();
|
|
275
|
+
try {
|
|
276
|
+
await new Promise((resolve) => setImmediate(resolve));
|
|
277
|
+
}
|
|
278
|
+
finally {
|
|
279
|
+
release();
|
|
280
|
+
}
|
|
281
|
+
return await execute(tool, capturedArgs);
|
|
282
|
+
}
|
|
283
|
+
catch (error) {
|
|
284
|
+
fatal ??= error instanceof Stop ? error : new Stop('tool_contract');
|
|
285
|
+
throw fatal;
|
|
286
|
+
}
|
|
287
|
+
finally {
|
|
288
|
+
pending--;
|
|
289
|
+
}
|
|
290
|
+
})();
|
|
291
|
+
// An agent may forget to await a call. Keep that a recorded protocol failure
|
|
292
|
+
// instead of an unhandled rejection that terminates the caller's process.
|
|
293
|
+
void promise.catch(() => undefined);
|
|
294
|
+
tail = promise.catch(() => undefined);
|
|
295
|
+
return promise;
|
|
296
|
+
};
|
|
297
|
+
let timer;
|
|
298
|
+
const abort = () => controller.abort();
|
|
299
|
+
signal?.addEventListener('abort', abort, { once: true });
|
|
300
|
+
let rejectAbort;
|
|
301
|
+
try {
|
|
302
|
+
if (signal?.aborted)
|
|
303
|
+
throw new Stop('cancelled');
|
|
304
|
+
const deadline = new Promise((_, reject) => {
|
|
305
|
+
timer = setTimeout(() => {
|
|
306
|
+
controller.abort();
|
|
307
|
+
reject(new Stop('timed_out'));
|
|
308
|
+
}, workflow.limits.timeoutMs);
|
|
309
|
+
rejectAbort = () => reject(new Stop('cancelled'));
|
|
310
|
+
signal?.addEventListener('abort', rejectAbort, { once: true });
|
|
311
|
+
});
|
|
312
|
+
const raw = await Promise.race([
|
|
313
|
+
Promise.resolve().then(() => agent.run({
|
|
314
|
+
task: workflow.task,
|
|
315
|
+
signal: controller.signal,
|
|
316
|
+
call,
|
|
317
|
+
clock: clock.api,
|
|
318
|
+
recordModel(observation) {
|
|
319
|
+
guard();
|
|
320
|
+
const parsed = z
|
|
321
|
+
.strictObject({
|
|
322
|
+
detail: z.string().max(2000),
|
|
323
|
+
payload: z.json().optional(),
|
|
324
|
+
})
|
|
325
|
+
.parse(observation);
|
|
326
|
+
emit({
|
|
327
|
+
kind: 'model',
|
|
328
|
+
visibility: 'evaluator',
|
|
329
|
+
...parsed,
|
|
330
|
+
});
|
|
331
|
+
},
|
|
332
|
+
})),
|
|
333
|
+
deadline,
|
|
334
|
+
clockFailure,
|
|
335
|
+
]);
|
|
336
|
+
guard();
|
|
337
|
+
if (pending || clock.pending)
|
|
338
|
+
throw new Stop('unawaited_calls');
|
|
339
|
+
boundedJson(raw, 16000);
|
|
340
|
+
completion = completionSchema.parse(raw);
|
|
341
|
+
emit({
|
|
342
|
+
kind: 'completion',
|
|
343
|
+
visibility: 'agent',
|
|
344
|
+
detail: completion.message,
|
|
345
|
+
payload: completion,
|
|
346
|
+
});
|
|
347
|
+
}
|
|
348
|
+
catch (error) {
|
|
349
|
+
issue = error instanceof Stop ? error.reason : 'agent_error';
|
|
350
|
+
events.push({
|
|
351
|
+
id: `evt_${events.length + 1}`,
|
|
352
|
+
sequence: events.length + 1,
|
|
353
|
+
...(workflow.clock ? { timeMs: clock.api.now() } : {}),
|
|
354
|
+
kind: [
|
|
355
|
+
'call_limit',
|
|
356
|
+
'repeated_calls',
|
|
357
|
+
'timed_out',
|
|
358
|
+
'cancelled',
|
|
359
|
+
'evidence_limit',
|
|
360
|
+
].includes(issue)
|
|
361
|
+
? 'limit'
|
|
362
|
+
: 'error',
|
|
363
|
+
visibility: 'evaluator',
|
|
364
|
+
detail: `Run stopped: ${issue}. Partial evidence preserved.`,
|
|
365
|
+
});
|
|
366
|
+
}
|
|
367
|
+
finally {
|
|
368
|
+
closed = true;
|
|
369
|
+
controller.abort();
|
|
370
|
+
clock.close();
|
|
371
|
+
if (timer)
|
|
372
|
+
clearTimeout(timer);
|
|
373
|
+
signal?.removeEventListener('abort', abort);
|
|
374
|
+
if (rejectAbort)
|
|
375
|
+
signal?.removeEventListener('abort', rejectAbort);
|
|
376
|
+
}
|
|
377
|
+
const definitions = scenario.checks ?? workflow.checks;
|
|
378
|
+
const assertions = evaluateChecks(definitions, initialState, state, completion, events);
|
|
379
|
+
const status = issue === 'cancelled' || issue === 'timed_out'
|
|
380
|
+
? issue
|
|
381
|
+
: issue === 'call_limit' || issue === 'repeated_calls'
|
|
382
|
+
? 'failed'
|
|
383
|
+
: issue || (fault.type !== 'none' && triggerCount === 0)
|
|
384
|
+
? 'inconclusive'
|
|
385
|
+
: assertions.every((c) => c.passed)
|
|
386
|
+
? 'passed'
|
|
387
|
+
: 'failed';
|
|
388
|
+
return workflowReportSchema.parse({
|
|
389
|
+
schemaVersion: '1',
|
|
390
|
+
reportType: 'workflow',
|
|
391
|
+
id: `run_${randomUUID()}`,
|
|
392
|
+
createdAt: new Date().toISOString(),
|
|
393
|
+
execution,
|
|
394
|
+
workflow: { id: workflow.id, title: workflow.title, task: workflow.task },
|
|
395
|
+
suiteFingerprint: fingerprint,
|
|
396
|
+
agent: metadata.parse({ id: agent.id, version: agent.version }),
|
|
397
|
+
scenario: { id: scenario.id, title: scenario.title, fault },
|
|
398
|
+
status,
|
|
399
|
+
...(issue ? { issue } : {}),
|
|
400
|
+
fault: { triggered: triggerCount > 0, triggerCount },
|
|
401
|
+
initialState,
|
|
402
|
+
...(clock.snapshot ? { clock: clock.snapshot } : {}),
|
|
403
|
+
finalState: state,
|
|
404
|
+
definitions,
|
|
405
|
+
assertions,
|
|
406
|
+
...(completion ? { completion } : {}),
|
|
407
|
+
events,
|
|
408
|
+
metrics: {
|
|
409
|
+
toolCalls: events.filter((e) => e.kind === 'tool_call').length,
|
|
410
|
+
mutations: events.filter((e) => e.kind === 'mutation').length,
|
|
411
|
+
durationMs: Math.round((performance.now() - start) * 100) / 100,
|
|
412
|
+
},
|
|
413
|
+
});
|
|
414
|
+
}
|
|
415
|
+
/** Run trusted local agent code against isolated, developer-defined simulated tools. */
|
|
416
|
+
export async function runWorkflow(input, adapter, options = {}) {
|
|
417
|
+
const workflow = parseWorkflow(input);
|
|
418
|
+
const commitSha = options.commitSha === undefined
|
|
419
|
+
? undefined
|
|
420
|
+
: commitShaSchema.parse(options.commitSha);
|
|
421
|
+
const trials = z
|
|
422
|
+
.number()
|
|
423
|
+
.int()
|
|
424
|
+
.min(1)
|
|
425
|
+
.max(20)
|
|
426
|
+
.parse(options.trials ?? 1);
|
|
427
|
+
const execution = {
|
|
428
|
+
id: randomUUID(),
|
|
429
|
+
trialsPerScenario: trials,
|
|
430
|
+
scenarioIds: workflow.scenarios.map((scenario) => scenario.id),
|
|
431
|
+
...(commitSha ? { commitSha } : {}),
|
|
432
|
+
};
|
|
433
|
+
if (!adapter ||
|
|
434
|
+
typeof adapter.createAgent !== 'function' ||
|
|
435
|
+
!adapter.tools ||
|
|
436
|
+
Object.keys(adapter.tools).length < 1 ||
|
|
437
|
+
Object.keys(adapter.tools).length > 50)
|
|
438
|
+
throw new Error('Provide tools and a synchronous createAgent factory.');
|
|
439
|
+
for (const [name, tool] of Object.entries(adapter.tools))
|
|
440
|
+
if (!/^[a-zA-Z0-9][a-zA-Z0-9_-]{0,99}$/.test(name) ||
|
|
441
|
+
!tool ||
|
|
442
|
+
!['read', 'write'].includes(tool.kind) ||
|
|
443
|
+
typeof tool.execute !== 'function')
|
|
444
|
+
throw new Error('Every tool needs a valid name, read/write kind, and execute function.');
|
|
445
|
+
for (const scenario of workflow.scenarios) {
|
|
446
|
+
const f = scenario.fault;
|
|
447
|
+
if (f.type !== 'none' &&
|
|
448
|
+
(!Object.hasOwn(adapter.tools, f.tool) ||
|
|
449
|
+
((f.type === 'permission_revoked' || f.type === 'duplicate_delivery') &&
|
|
450
|
+
adapter.tools[f.tool]?.kind !== 'write') ||
|
|
451
|
+
(f.type === 'stale_read' && adapter.tools[f.tool]?.kind !== 'read')))
|
|
452
|
+
throw new Error('Every fault must target a registered tool; permission revocation and duplicate delivery require a write tool; stale reads require a read tool.');
|
|
453
|
+
}
|
|
454
|
+
const fingerprint = createHash('sha256')
|
|
455
|
+
.update(stableJson(workflow))
|
|
456
|
+
.digest('hex');
|
|
457
|
+
const seen = new WeakSet();
|
|
458
|
+
let identity;
|
|
459
|
+
const instantiate = () => {
|
|
460
|
+
let agent;
|
|
461
|
+
try {
|
|
462
|
+
agent = adapter.createAgent();
|
|
463
|
+
if (agent instanceof Promise) {
|
|
464
|
+
void agent.catch(() => undefined);
|
|
465
|
+
throw new Error();
|
|
466
|
+
}
|
|
467
|
+
if (!agent || typeof agent.run !== 'function' || seen.has(agent))
|
|
468
|
+
throw new Error();
|
|
469
|
+
const next = metadata.parse({ id: agent.id, version: agent.version });
|
|
470
|
+
if (identity && stableJson(next) !== stableJson(identity))
|
|
471
|
+
throw new Error();
|
|
472
|
+
identity = next;
|
|
473
|
+
seen.add(agent);
|
|
474
|
+
return agent;
|
|
475
|
+
}
|
|
476
|
+
catch {
|
|
477
|
+
throw new Error('createAgent must return a fresh agent with stable id/version and a run function.');
|
|
478
|
+
}
|
|
479
|
+
};
|
|
480
|
+
const baseline = await trial(workflow, adapter, instantiate(), { id: 'baseline', title: 'No-fault control', fault: { type: 'none' } }, fingerprint, execution, options.signal);
|
|
481
|
+
const batch = {
|
|
482
|
+
schemaVersion: 'workflow-batch/1',
|
|
483
|
+
baseline,
|
|
484
|
+
runs: [],
|
|
485
|
+
planned: workflow.scenarios.length * trials,
|
|
486
|
+
};
|
|
487
|
+
if (baseline.status !== 'passed')
|
|
488
|
+
batch.skippedReason =
|
|
489
|
+
'The no-fault control did not pass. Fault trials were skipped.';
|
|
490
|
+
else
|
|
491
|
+
outer: for (const scenario of workflow.scenarios)
|
|
492
|
+
for (let i = 0; i < trials; i++) {
|
|
493
|
+
if (options.signal?.aborted) {
|
|
494
|
+
batch.skippedReason = 'Remaining trials were cancelled.';
|
|
495
|
+
break outer;
|
|
496
|
+
}
|
|
497
|
+
let agent;
|
|
498
|
+
try {
|
|
499
|
+
agent = instantiate();
|
|
500
|
+
}
|
|
501
|
+
catch {
|
|
502
|
+
batch.skippedReason =
|
|
503
|
+
'Agent initialization changed or failed. Completed evidence was preserved.';
|
|
504
|
+
break outer;
|
|
505
|
+
}
|
|
506
|
+
const report = await trial(workflow, adapter, agent, scenario, fingerprint, execution, options.signal);
|
|
507
|
+
batch.runs.push(report);
|
|
508
|
+
if (['inconclusive', 'cancelled', 'timed_out'].includes(report.status)) {
|
|
509
|
+
batch.skippedReason =
|
|
510
|
+
'Stopped after an interrupted or inconclusive trial. Completed evidence was preserved.';
|
|
511
|
+
break outer;
|
|
512
|
+
}
|
|
513
|
+
}
|
|
514
|
+
return workflowBatchSchema.parse(batch);
|
|
515
|
+
}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import { z } from 'zod';
|
|
2
|
+
import type { WorkflowContext } from './index.js';
|
|
3
|
+
export interface LangChainToolSpec {
|
|
4
|
+
name: string;
|
|
5
|
+
description: string;
|
|
6
|
+
schema: z.ZodObject;
|
|
7
|
+
}
|
|
8
|
+
/** Pass these tools to your existing LangChain agent instead of its live tools. */
|
|
9
|
+
export declare function createLangChainTools(context: WorkflowContext, definitions: LangChainToolSpec[]): import("@langchain/core/tools").DynamicStructuredTool<z.ZodObject<z.core.$ZodLooseShape, z.core.$strip>, Record<string, unknown>, Record<string, unknown>, string, unknown, string>[];
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
// SPDX-License-Identifier: Apache-2.0
|
|
2
|
+
// Copyright 2026 Syed Arman
|
|
3
|
+
import { tool } from '@langchain/core/tools';
|
|
4
|
+
import { boundedJson } from './schema.js';
|
|
5
|
+
/** Pass these tools to your existing LangChain agent instead of its live tools. */
|
|
6
|
+
export function createLangChainTools(context, definitions) {
|
|
7
|
+
if (new Set(definitions.map((d) => d.name)).size !== definitions.length)
|
|
8
|
+
throw new Error('Tool names must be unique.');
|
|
9
|
+
return definitions.map((definition) => tool(async (args) => {
|
|
10
|
+
boundedJson(args, 16000);
|
|
11
|
+
return JSON.stringify(await context.call(definition.name, args));
|
|
12
|
+
}, {
|
|
13
|
+
name: definition.name,
|
|
14
|
+
description: definition.description,
|
|
15
|
+
schema: definition.schema,
|
|
16
|
+
}));
|
|
17
|
+
}
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
// SPDX-License-Identifier: Apache-2.0
|
|
2
|
+
// Copyright 2026 Syed Arman
|
|
3
|
+
import { mkdir, readFile, rmdir, unlink, writeFile } from 'node:fs/promises';
|
|
4
|
+
import { dirname, resolve } from 'node:path';
|
|
5
|
+
import { importLangSmithTrace, inferToolSignatures } from './langsmith.js';
|
|
6
|
+
import { boundedJson, parseWorkflow } from './schema.js';
|
|
7
|
+
export async function createLangSmithStarter(input, directory) {
|
|
8
|
+
const draft = importLangSmithTrace(input);
|
|
9
|
+
if (draft.observedCalls.length > 90)
|
|
10
|
+
throw new Error('Runnable scaffolds support at most 90 tool calls. Use --draft-only for larger traces.');
|
|
11
|
+
for (const call of draft.observedCalls)
|
|
12
|
+
boundedJson(call, 40000);
|
|
13
|
+
const signatures = inferToolSignatures(draft);
|
|
14
|
+
const names = new Map(signatures.map((tool) => [tool.name, tool.alias]));
|
|
15
|
+
const fixture = {
|
|
16
|
+
...draft,
|
|
17
|
+
signatures,
|
|
18
|
+
observedCalls: draft.observedCalls.map((call) => ({
|
|
19
|
+
...call,
|
|
20
|
+
alias: names.get(call.name),
|
|
21
|
+
})),
|
|
22
|
+
};
|
|
23
|
+
const workflow = parseWorkflow({
|
|
24
|
+
schemaVersion: 'workflow/1',
|
|
25
|
+
id: 'imported-trace-playback',
|
|
26
|
+
title: 'Imported trace playback — needs review',
|
|
27
|
+
task: draft.taskCandidate.slice(0, 10000).trim() ||
|
|
28
|
+
'Replay the observed calls. Review the task before connecting an agent.',
|
|
29
|
+
initialState: {},
|
|
30
|
+
checks: [
|
|
31
|
+
{
|
|
32
|
+
id: 'playback-completes',
|
|
33
|
+
label: 'Scripted playback completes (not a correctness check)',
|
|
34
|
+
type: 'equals',
|
|
35
|
+
source: 'completion',
|
|
36
|
+
path: '/status',
|
|
37
|
+
expected: 'completed',
|
|
38
|
+
},
|
|
39
|
+
],
|
|
40
|
+
scenarios: [
|
|
41
|
+
{
|
|
42
|
+
id: 'interrupted-playback',
|
|
43
|
+
title: 'Recorded call sequence interrupted by a timeout',
|
|
44
|
+
fault: {
|
|
45
|
+
type: 'timeout_before',
|
|
46
|
+
tool: fixture.observedCalls[0].alias,
|
|
47
|
+
},
|
|
48
|
+
},
|
|
49
|
+
],
|
|
50
|
+
limits: { maxToolCalls: 100, maxRepeatedCalls: 100, timeoutMs: 5000 },
|
|
51
|
+
});
|
|
52
|
+
const files = {
|
|
53
|
+
'workflow.json': JSON.stringify(workflow, null, 2) + '\n',
|
|
54
|
+
'trace.json': JSON.stringify(fixture, null, 2) + '\n',
|
|
55
|
+
'adapter.mjs': await readFile(new URL('../templates/langsmith-adapter.mjs', import.meta.url), 'utf8'),
|
|
56
|
+
'README.md': `# Turn this trace into an agent test\n\nThis is a runnable, redacted playback scaffold. It does not recreate your agent or infer application correctness. No model or external service is called.\n\nRun with the SDK installed, replacing PATH with this directory:\n\n\`\`\`sh\nnpx --no-install tracedojo test --config PATH/workflow.json --adapter PATH/adapter.mjs\n\`\`\`\n\nFor a trace with successful, unambiguous tool outputs, the control passes and the injected timeout fails (exit 1). That failure demonstrates an interrupted scripted sequence, not a discovered bug in your agent. Missing outputs, recorded errors, or conflicting results for the same arguments block the control until you implement the TODOs.\n\n## Finish the adapter\n\n1. Review trace.json: signatures describe observed shapes only. Required fields appeared in every sample; they are not an authoritative API contract. Tool aliases map arbitrary trace names to valid runner names. Redaction is pattern-based; inspect files before sharing.\n2. Implement each tool in adapter.mjs. All stubs default to read-only fixture responses. Set kind to write only for tools that mutate state, and implement their real simulated transitions against the state argument. Supply synthetic initialState in workflow.json.\n3. Replace the playback completion check with independently defined state and safety checks. Never promote observed outputs into expected outcomes without review.\n4. Replace createAgent with your real decision loop, route all calls through context.call, and version the agent. The trace order is a starting point, not a reconstruction of parallel scheduling.\n5. Choose faults that matter to your workflow and run again. Prefer a prebuilt world when you need a complete stateful example.\n\nAdapters are trusted local code with environment/network access. Keep .tracedojo/ and private trace files out of Git.\n`,
|
|
57
|
+
};
|
|
58
|
+
for (const name of ['LICENSE', 'NOTICE'])
|
|
59
|
+
files[name] = await readFile(new URL(`../${name}`, import.meta.url), 'utf8');
|
|
60
|
+
await mkdir(dirname(directory), { recursive: true });
|
|
61
|
+
await mkdir(directory, { mode: 0o700 });
|
|
62
|
+
const created = [];
|
|
63
|
+
try {
|
|
64
|
+
for (const [name, contents] of Object.entries(files)) {
|
|
65
|
+
const path = resolve(directory, name);
|
|
66
|
+
await writeFile(path, contents, { flag: 'wx', mode: 0o600 });
|
|
67
|
+
created.push(path);
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
catch (error) {
|
|
71
|
+
for (const path of created.reverse())
|
|
72
|
+
await unlink(path);
|
|
73
|
+
await rmdir(directory);
|
|
74
|
+
throw error;
|
|
75
|
+
}
|
|
76
|
+
return { calls: draft.observedCalls.length, tools: signatures.length };
|
|
77
|
+
}
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
import { type Json } from './schema.js';
|
|
2
|
+
export interface LangSmithDraft {
|
|
3
|
+
schemaVersion: 'workflow-draft/1';
|
|
4
|
+
status: 'needs_review';
|
|
5
|
+
source: {
|
|
6
|
+
provider: 'langsmith';
|
|
7
|
+
traceId: string;
|
|
8
|
+
rootRunId: string;
|
|
9
|
+
name: string;
|
|
10
|
+
};
|
|
11
|
+
taskCandidate: string;
|
|
12
|
+
observedCalls: {
|
|
13
|
+
id: string;
|
|
14
|
+
name: string;
|
|
15
|
+
inputs: Record<string, Json>;
|
|
16
|
+
outputs?: Json;
|
|
17
|
+
error?: string;
|
|
18
|
+
}[];
|
|
19
|
+
required: string[];
|
|
20
|
+
}
|
|
21
|
+
/** Accept a single nested run or a listRuns export containing one complete trace. */
|
|
22
|
+
export declare function importLangSmithTrace(input: unknown): LangSmithDraft;
|
|
23
|
+
export declare function inferToolSignatures(draft: LangSmithDraft): {
|
|
24
|
+
name: string;
|
|
25
|
+
alias: string;
|
|
26
|
+
status: "observed_only";
|
|
27
|
+
inputs: Json;
|
|
28
|
+
outputs: Json;
|
|
29
|
+
samples: number;
|
|
30
|
+
}[];
|