@tracedojo/sdk 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (58) hide show
  1. package/LICENSE +202 -0
  2. package/NOTICE +5 -0
  3. package/README.md +96 -0
  4. package/dist/cli.d.ts +2 -0
  5. package/dist/cli.js +356 -0
  6. package/dist/clock.d.ts +41 -0
  7. package/dist/clock.js +118 -0
  8. package/dist/compare.d.ts +20 -0
  9. package/dist/compare.js +147 -0
  10. package/dist/demo.d.ts +465 -0
  11. package/dist/demo.js +45 -0
  12. package/dist/index.d.ts +35 -0
  13. package/dist/index.js +515 -0
  14. package/dist/langchain.d.ts +9 -0
  15. package/dist/langchain.js +17 -0
  16. package/dist/langsmith-starter.d.ts +4 -0
  17. package/dist/langsmith-starter.js +77 -0
  18. package/dist/langsmith.d.ts +30 -0
  19. package/dist/langsmith.js +148 -0
  20. package/dist/matrix-cli.d.ts +107 -0
  21. package/dist/matrix-cli.js +154 -0
  22. package/dist/matrix.d.ts +138 -0
  23. package/dist/matrix.js +148 -0
  24. package/dist/privacy.d.ts +5 -0
  25. package/dist/privacy.js +51 -0
  26. package/dist/reports.d.ts +2 -0
  27. package/dist/reports.js +38 -0
  28. package/dist/schema.d.ts +715 -0
  29. package/dist/schema.js +433 -0
  30. package/dist/starter.d.ts +1 -0
  31. package/dist/starter.js +64 -0
  32. package/dist/summary.d.ts +2 -0
  33. package/dist/summary.js +50 -0
  34. package/dist/trends.d.ts +25 -0
  35. package/dist/trends.js +164 -0
  36. package/dist/upload.d.ts +8 -0
  37. package/dist/upload.js +73 -0
  38. package/dist/worlds.d.ts +9 -0
  39. package/dist/worlds.js +279 -0
  40. package/package.json +85 -0
  41. package/templates/README.md +65 -0
  42. package/templates/adapter.mjs +88 -0
  43. package/templates/ci-comment.yml +40 -0
  44. package/templates/ci.yml +100 -0
  45. package/templates/clock-adapter.mjs +42 -0
  46. package/templates/comment.cjs +209 -0
  47. package/templates/faults/duplicate-delivery/README.md +19 -0
  48. package/templates/faults/duplicate-delivery/adapter.mjs +40 -0
  49. package/templates/faults/duplicate-delivery/workflow.json +31 -0
  50. package/templates/faults/prompt-injection/README.md +17 -0
  51. package/templates/faults/prompt-injection/adapter.mjs +36 -0
  52. package/templates/faults/prompt-injection/workflow.json +43 -0
  53. package/templates/faults/stale-read/README.md +21 -0
  54. package/templates/faults/stale-read/adapter.mjs +59 -0
  55. package/templates/faults/stale-read/workflow.json +33 -0
  56. package/templates/langsmith-adapter.mjs +83 -0
  57. package/templates/workflow.json +106 -0
  58. package/templates/world-adapter.mjs +67 -0
package/dist/demo.d.ts ADDED
@@ -0,0 +1,465 @@
1
+ /** Run a known failure and fix using the same engine and tools as customer tests. */
2
+ export declare function runDemo(directory: string): Promise<{
3
+ before: {
4
+ schemaVersion: "workflow-batch/1";
5
+ baseline: {
6
+ schemaVersion: "1";
7
+ reportType: "workflow";
8
+ id: string;
9
+ createdAt: string;
10
+ workflow: {
11
+ id: string;
12
+ title: string;
13
+ task: string;
14
+ };
15
+ suiteFingerprint: string;
16
+ agent: {
17
+ id: string;
18
+ version: string;
19
+ };
20
+ scenario: {
21
+ id: string;
22
+ title: string;
23
+ fault: {
24
+ type: "prompt_injection";
25
+ tool: string;
26
+ occurrence: number;
27
+ repeat: number;
28
+ replacement: import("zod").JSONType;
29
+ } | {
30
+ type: "timeout_before" | "timeout_after" | "unavailable" | "malformed" | "permission_revoked" | "duplicate_delivery" | "stale_read";
31
+ tool: string;
32
+ occurrence: number;
33
+ repeat: number;
34
+ } | {
35
+ type: "none";
36
+ };
37
+ };
38
+ status: "passed" | "failed" | "inconclusive" | "cancelled" | "timed_out";
39
+ fault: {
40
+ triggered: boolean;
41
+ triggerCount: number;
42
+ };
43
+ initialState: Record<string, import("zod").JSONType>;
44
+ finalState: Record<string, import("zod").JSONType>;
45
+ definitions: ({
46
+ id: string;
47
+ label: string;
48
+ type: "forbidden_tool";
49
+ tool: string;
50
+ } | {
51
+ id: string;
52
+ label: string;
53
+ type: "equals";
54
+ source: "state" | "completion";
55
+ path: string;
56
+ expected: import("zod").JSONType;
57
+ } | {
58
+ id: string;
59
+ label: string;
60
+ type: "count";
61
+ path: string;
62
+ expected: number;
63
+ } | {
64
+ id: string;
65
+ label: string;
66
+ type: "unchanged";
67
+ path: string;
68
+ })[];
69
+ assertions: {
70
+ id: string;
71
+ label: string;
72
+ passed: boolean;
73
+ detail: string;
74
+ evidenceIds: string[];
75
+ before?: import("zod").JSONType | undefined;
76
+ actual?: import("zod").JSONType | undefined;
77
+ expected?: import("zod").JSONType | undefined;
78
+ }[];
79
+ events: {
80
+ id: string;
81
+ sequence: number;
82
+ kind: "error" | "completion" | "fault" | "request" | "model" | "tool_call" | "tool_result" | "delivery" | "mutation" | "limit";
83
+ visibility: "agent" | "evaluator";
84
+ detail: string;
85
+ timeMs?: number | undefined;
86
+ callId?: string | undefined;
87
+ tool?: string | undefined;
88
+ payload?: import("zod").JSONType | undefined;
89
+ }[];
90
+ metrics: {
91
+ toolCalls: number;
92
+ mutations: number;
93
+ durationMs: number;
94
+ };
95
+ execution?: {
96
+ id: string;
97
+ trialsPerScenario: number;
98
+ scenarioIds: string[];
99
+ commitSha?: string | undefined;
100
+ } | undefined;
101
+ origin?: {
102
+ kind: "imported";
103
+ originalId: string;
104
+ } | undefined;
105
+ issue?: "cancelled" | "timed_out" | "agent_error" | "tool_contract" | "call_limit" | "repeated_calls" | "unawaited_calls" | "evidence_limit" | "initialization" | "clock_limit" | "clock_contract" | undefined;
106
+ clock?: {
107
+ mode: "virtual";
108
+ startMs: number;
109
+ maxTimeMs: number;
110
+ endMs: number;
111
+ } | undefined;
112
+ completion?: {
113
+ status: "completed" | "blocked";
114
+ message: string;
115
+ output?: import("zod").JSONType | undefined;
116
+ } | undefined;
117
+ };
118
+ runs: {
119
+ schemaVersion: "1";
120
+ reportType: "workflow";
121
+ id: string;
122
+ createdAt: string;
123
+ workflow: {
124
+ id: string;
125
+ title: string;
126
+ task: string;
127
+ };
128
+ suiteFingerprint: string;
129
+ agent: {
130
+ id: string;
131
+ version: string;
132
+ };
133
+ scenario: {
134
+ id: string;
135
+ title: string;
136
+ fault: {
137
+ type: "prompt_injection";
138
+ tool: string;
139
+ occurrence: number;
140
+ repeat: number;
141
+ replacement: import("zod").JSONType;
142
+ } | {
143
+ type: "timeout_before" | "timeout_after" | "unavailable" | "malformed" | "permission_revoked" | "duplicate_delivery" | "stale_read";
144
+ tool: string;
145
+ occurrence: number;
146
+ repeat: number;
147
+ } | {
148
+ type: "none";
149
+ };
150
+ };
151
+ status: "passed" | "failed" | "inconclusive" | "cancelled" | "timed_out";
152
+ fault: {
153
+ triggered: boolean;
154
+ triggerCount: number;
155
+ };
156
+ initialState: Record<string, import("zod").JSONType>;
157
+ finalState: Record<string, import("zod").JSONType>;
158
+ definitions: ({
159
+ id: string;
160
+ label: string;
161
+ type: "forbidden_tool";
162
+ tool: string;
163
+ } | {
164
+ id: string;
165
+ label: string;
166
+ type: "equals";
167
+ source: "state" | "completion";
168
+ path: string;
169
+ expected: import("zod").JSONType;
170
+ } | {
171
+ id: string;
172
+ label: string;
173
+ type: "count";
174
+ path: string;
175
+ expected: number;
176
+ } | {
177
+ id: string;
178
+ label: string;
179
+ type: "unchanged";
180
+ path: string;
181
+ })[];
182
+ assertions: {
183
+ id: string;
184
+ label: string;
185
+ passed: boolean;
186
+ detail: string;
187
+ evidenceIds: string[];
188
+ before?: import("zod").JSONType | undefined;
189
+ actual?: import("zod").JSONType | undefined;
190
+ expected?: import("zod").JSONType | undefined;
191
+ }[];
192
+ events: {
193
+ id: string;
194
+ sequence: number;
195
+ kind: "error" | "completion" | "fault" | "request" | "model" | "tool_call" | "tool_result" | "delivery" | "mutation" | "limit";
196
+ visibility: "agent" | "evaluator";
197
+ detail: string;
198
+ timeMs?: number | undefined;
199
+ callId?: string | undefined;
200
+ tool?: string | undefined;
201
+ payload?: import("zod").JSONType | undefined;
202
+ }[];
203
+ metrics: {
204
+ toolCalls: number;
205
+ mutations: number;
206
+ durationMs: number;
207
+ };
208
+ execution?: {
209
+ id: string;
210
+ trialsPerScenario: number;
211
+ scenarioIds: string[];
212
+ commitSha?: string | undefined;
213
+ } | undefined;
214
+ origin?: {
215
+ kind: "imported";
216
+ originalId: string;
217
+ } | undefined;
218
+ issue?: "cancelled" | "timed_out" | "agent_error" | "tool_contract" | "call_limit" | "repeated_calls" | "unawaited_calls" | "evidence_limit" | "initialization" | "clock_limit" | "clock_contract" | undefined;
219
+ clock?: {
220
+ mode: "virtual";
221
+ startMs: number;
222
+ maxTimeMs: number;
223
+ endMs: number;
224
+ } | undefined;
225
+ completion?: {
226
+ status: "completed" | "blocked";
227
+ message: string;
228
+ output?: import("zod").JSONType | undefined;
229
+ } | undefined;
230
+ }[];
231
+ planned: number;
232
+ skippedReason?: string | undefined;
233
+ };
234
+ after: {
235
+ schemaVersion: "workflow-batch/1";
236
+ baseline: {
237
+ schemaVersion: "1";
238
+ reportType: "workflow";
239
+ id: string;
240
+ createdAt: string;
241
+ workflow: {
242
+ id: string;
243
+ title: string;
244
+ task: string;
245
+ };
246
+ suiteFingerprint: string;
247
+ agent: {
248
+ id: string;
249
+ version: string;
250
+ };
251
+ scenario: {
252
+ id: string;
253
+ title: string;
254
+ fault: {
255
+ type: "prompt_injection";
256
+ tool: string;
257
+ occurrence: number;
258
+ repeat: number;
259
+ replacement: import("zod").JSONType;
260
+ } | {
261
+ type: "timeout_before" | "timeout_after" | "unavailable" | "malformed" | "permission_revoked" | "duplicate_delivery" | "stale_read";
262
+ tool: string;
263
+ occurrence: number;
264
+ repeat: number;
265
+ } | {
266
+ type: "none";
267
+ };
268
+ };
269
+ status: "passed" | "failed" | "inconclusive" | "cancelled" | "timed_out";
270
+ fault: {
271
+ triggered: boolean;
272
+ triggerCount: number;
273
+ };
274
+ initialState: Record<string, import("zod").JSONType>;
275
+ finalState: Record<string, import("zod").JSONType>;
276
+ definitions: ({
277
+ id: string;
278
+ label: string;
279
+ type: "forbidden_tool";
280
+ tool: string;
281
+ } | {
282
+ id: string;
283
+ label: string;
284
+ type: "equals";
285
+ source: "state" | "completion";
286
+ path: string;
287
+ expected: import("zod").JSONType;
288
+ } | {
289
+ id: string;
290
+ label: string;
291
+ type: "count";
292
+ path: string;
293
+ expected: number;
294
+ } | {
295
+ id: string;
296
+ label: string;
297
+ type: "unchanged";
298
+ path: string;
299
+ })[];
300
+ assertions: {
301
+ id: string;
302
+ label: string;
303
+ passed: boolean;
304
+ detail: string;
305
+ evidenceIds: string[];
306
+ before?: import("zod").JSONType | undefined;
307
+ actual?: import("zod").JSONType | undefined;
308
+ expected?: import("zod").JSONType | undefined;
309
+ }[];
310
+ events: {
311
+ id: string;
312
+ sequence: number;
313
+ kind: "error" | "completion" | "fault" | "request" | "model" | "tool_call" | "tool_result" | "delivery" | "mutation" | "limit";
314
+ visibility: "agent" | "evaluator";
315
+ detail: string;
316
+ timeMs?: number | undefined;
317
+ callId?: string | undefined;
318
+ tool?: string | undefined;
319
+ payload?: import("zod").JSONType | undefined;
320
+ }[];
321
+ metrics: {
322
+ toolCalls: number;
323
+ mutations: number;
324
+ durationMs: number;
325
+ };
326
+ execution?: {
327
+ id: string;
328
+ trialsPerScenario: number;
329
+ scenarioIds: string[];
330
+ commitSha?: string | undefined;
331
+ } | undefined;
332
+ origin?: {
333
+ kind: "imported";
334
+ originalId: string;
335
+ } | undefined;
336
+ issue?: "cancelled" | "timed_out" | "agent_error" | "tool_contract" | "call_limit" | "repeated_calls" | "unawaited_calls" | "evidence_limit" | "initialization" | "clock_limit" | "clock_contract" | undefined;
337
+ clock?: {
338
+ mode: "virtual";
339
+ startMs: number;
340
+ maxTimeMs: number;
341
+ endMs: number;
342
+ } | undefined;
343
+ completion?: {
344
+ status: "completed" | "blocked";
345
+ message: string;
346
+ output?: import("zod").JSONType | undefined;
347
+ } | undefined;
348
+ };
349
+ runs: {
350
+ schemaVersion: "1";
351
+ reportType: "workflow";
352
+ id: string;
353
+ createdAt: string;
354
+ workflow: {
355
+ id: string;
356
+ title: string;
357
+ task: string;
358
+ };
359
+ suiteFingerprint: string;
360
+ agent: {
361
+ id: string;
362
+ version: string;
363
+ };
364
+ scenario: {
365
+ id: string;
366
+ title: string;
367
+ fault: {
368
+ type: "prompt_injection";
369
+ tool: string;
370
+ occurrence: number;
371
+ repeat: number;
372
+ replacement: import("zod").JSONType;
373
+ } | {
374
+ type: "timeout_before" | "timeout_after" | "unavailable" | "malformed" | "permission_revoked" | "duplicate_delivery" | "stale_read";
375
+ tool: string;
376
+ occurrence: number;
377
+ repeat: number;
378
+ } | {
379
+ type: "none";
380
+ };
381
+ };
382
+ status: "passed" | "failed" | "inconclusive" | "cancelled" | "timed_out";
383
+ fault: {
384
+ triggered: boolean;
385
+ triggerCount: number;
386
+ };
387
+ initialState: Record<string, import("zod").JSONType>;
388
+ finalState: Record<string, import("zod").JSONType>;
389
+ definitions: ({
390
+ id: string;
391
+ label: string;
392
+ type: "forbidden_tool";
393
+ tool: string;
394
+ } | {
395
+ id: string;
396
+ label: string;
397
+ type: "equals";
398
+ source: "state" | "completion";
399
+ path: string;
400
+ expected: import("zod").JSONType;
401
+ } | {
402
+ id: string;
403
+ label: string;
404
+ type: "count";
405
+ path: string;
406
+ expected: number;
407
+ } | {
408
+ id: string;
409
+ label: string;
410
+ type: "unchanged";
411
+ path: string;
412
+ })[];
413
+ assertions: {
414
+ id: string;
415
+ label: string;
416
+ passed: boolean;
417
+ detail: string;
418
+ evidenceIds: string[];
419
+ before?: import("zod").JSONType | undefined;
420
+ actual?: import("zod").JSONType | undefined;
421
+ expected?: import("zod").JSONType | undefined;
422
+ }[];
423
+ events: {
424
+ id: string;
425
+ sequence: number;
426
+ kind: "error" | "completion" | "fault" | "request" | "model" | "tool_call" | "tool_result" | "delivery" | "mutation" | "limit";
427
+ visibility: "agent" | "evaluator";
428
+ detail: string;
429
+ timeMs?: number | undefined;
430
+ callId?: string | undefined;
431
+ tool?: string | undefined;
432
+ payload?: import("zod").JSONType | undefined;
433
+ }[];
434
+ metrics: {
435
+ toolCalls: number;
436
+ mutations: number;
437
+ durationMs: number;
438
+ };
439
+ execution?: {
440
+ id: string;
441
+ trialsPerScenario: number;
442
+ scenarioIds: string[];
443
+ commitSha?: string | undefined;
444
+ } | undefined;
445
+ origin?: {
446
+ kind: "imported";
447
+ originalId: string;
448
+ } | undefined;
449
+ issue?: "cancelled" | "timed_out" | "agent_error" | "tool_contract" | "call_limit" | "repeated_calls" | "unawaited_calls" | "evidence_limit" | "initialization" | "clock_limit" | "clock_contract" | undefined;
450
+ clock?: {
451
+ mode: "virtual";
452
+ startMs: number;
453
+ maxTimeMs: number;
454
+ endMs: number;
455
+ } | undefined;
456
+ completion?: {
457
+ status: "completed" | "blocked";
458
+ message: string;
459
+ output?: import("zod").JSONType | undefined;
460
+ } | undefined;
461
+ }[];
462
+ planned: number;
463
+ skippedReason?: string | undefined;
464
+ };
465
+ }>;
package/dist/demo.js ADDED
@@ -0,0 +1,45 @@
1
+ // SPDX-License-Identifier: Apache-2.0
2
+ // Copyright 2026 Syed Arman
3
+ import { mkdir, readFile, writeFile } from 'node:fs/promises';
4
+ import { resolve } from 'node:path';
5
+ import { runWorkflow } from './index.js';
6
+ import { parseWorkflow } from './schema.js';
7
+ import { sanitizeWorkflowBatch } from './privacy.js';
8
+ /** Run a known failure and fix using the same engine and tools as customer tests. */
9
+ export async function runDemo(directory) {
10
+ // Reserve a new directory before doing any work. Preserve earlier evidence.
11
+ await mkdir(directory, { mode: 0o700 });
12
+ const workflow = parseWorkflow(JSON.parse(await readFile(new URL('../templates/workflow.json', import.meta.url), 'utf8')));
13
+ workflow.scenarios = workflow.scenarios.filter((scenario) => scenario.id === 'lost-confirmation');
14
+ const template = new URL('../templates/adapter.mjs', import.meta.url);
15
+ const reference = await import(template.href);
16
+ const adapter = (stable) => ({
17
+ tools: reference.tools,
18
+ createAgent() {
19
+ const agent = reference.createAgent();
20
+ let attempt = 0;
21
+ return {
22
+ ...agent,
23
+ version: stable ? 'stable-key' : 'new-key-per-retry',
24
+ run(context) {
25
+ return agent.run({
26
+ ...context,
27
+ call: (tool, args) => context.call(tool, stable
28
+ ? args
29
+ : { ...args, operationKey: `attempt_${++attempt}` }),
30
+ });
31
+ },
32
+ };
33
+ },
34
+ });
35
+ const before = sanitizeWorkflowBatch(await runWorkflow(workflow, adapter(false)));
36
+ await writeFile(resolve(directory, 'before.json'), JSON.stringify(before, null, 2) + '\n', { flag: 'wx', mode: 0o600 });
37
+ const after = sanitizeWorkflowBatch(await runWorkflow(workflow, adapter(true)));
38
+ await writeFile(resolve(directory, 'after.json'), JSON.stringify(after, null, 2) + '\n', { flag: 'wx', mode: 0o600 });
39
+ if (before.baseline.status !== 'passed' ||
40
+ after.baseline.status !== 'passed' ||
41
+ before.runs[0]?.status !== 'failed' ||
42
+ after.runs[0]?.status !== 'passed')
43
+ throw new Error('The demo did not reproduce its expected failure and fix. Inspect the saved reports.');
44
+ return { before, after };
45
+ }
@@ -0,0 +1,35 @@
1
+ import { type Json, type State, type ToolResult, type WorkflowBatch, type WorkflowCompletion, type WorkflowEvent } from './schema.js';
2
+ export * from './schema.js';
3
+ import { type WorkflowClock } from './clock.js';
4
+ export type { WorkflowClock } from './clock.js';
5
+ export interface SimulatedTool {
6
+ kind: 'read' | 'write';
7
+ /** An isolated in-memory transaction. Only successful writes commit the draft. */
8
+ execute(args: Record<string, Json>, state: State, context: {
9
+ signal: AbortSignal;
10
+ clock: WorkflowClock;
11
+ }): ToolResult | Promise<ToolResult>;
12
+ }
13
+ export interface WorkflowContext {
14
+ clock: WorkflowClock;
15
+ task: string;
16
+ signal: AbortSignal;
17
+ call(tool: string, args: Record<string, Json>): Promise<ToolResult | string>;
18
+ /** Record bounded, evaluator-only model telemetry. Never include credentials or private reasoning. */
19
+ recordModel(observation: Pick<WorkflowEvent, 'detail' | 'payload'>): void;
20
+ }
21
+ export interface WorkflowAgent {
22
+ id: string;
23
+ version: string;
24
+ run(context: WorkflowContext): Promise<WorkflowCompletion>;
25
+ }
26
+ export interface WorkflowAdapter {
27
+ tools: Record<string, SimulatedTool>;
28
+ createAgent(): WorkflowAgent;
29
+ }
30
+ /** Run trusted local agent code against isolated, developer-defined simulated tools. */
31
+ export declare function runWorkflow(input: unknown, adapter: WorkflowAdapter, options?: {
32
+ trials?: number;
33
+ signal?: AbortSignal;
34
+ commitSha?: string;
35
+ }): Promise<WorkflowBatch>;