@tracedojo/sdk 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (58) hide show
  1. package/LICENSE +202 -0
  2. package/NOTICE +5 -0
  3. package/README.md +96 -0
  4. package/dist/cli.d.ts +2 -0
  5. package/dist/cli.js +356 -0
  6. package/dist/clock.d.ts +41 -0
  7. package/dist/clock.js +118 -0
  8. package/dist/compare.d.ts +20 -0
  9. package/dist/compare.js +147 -0
  10. package/dist/demo.d.ts +465 -0
  11. package/dist/demo.js +45 -0
  12. package/dist/index.d.ts +35 -0
  13. package/dist/index.js +515 -0
  14. package/dist/langchain.d.ts +9 -0
  15. package/dist/langchain.js +17 -0
  16. package/dist/langsmith-starter.d.ts +4 -0
  17. package/dist/langsmith-starter.js +77 -0
  18. package/dist/langsmith.d.ts +30 -0
  19. package/dist/langsmith.js +148 -0
  20. package/dist/matrix-cli.d.ts +107 -0
  21. package/dist/matrix-cli.js +154 -0
  22. package/dist/matrix.d.ts +138 -0
  23. package/dist/matrix.js +148 -0
  24. package/dist/privacy.d.ts +5 -0
  25. package/dist/privacy.js +51 -0
  26. package/dist/reports.d.ts +2 -0
  27. package/dist/reports.js +38 -0
  28. package/dist/schema.d.ts +715 -0
  29. package/dist/schema.js +433 -0
  30. package/dist/starter.d.ts +1 -0
  31. package/dist/starter.js +64 -0
  32. package/dist/summary.d.ts +2 -0
  33. package/dist/summary.js +50 -0
  34. package/dist/trends.d.ts +25 -0
  35. package/dist/trends.js +164 -0
  36. package/dist/upload.d.ts +8 -0
  37. package/dist/upload.js +73 -0
  38. package/dist/worlds.d.ts +9 -0
  39. package/dist/worlds.js +279 -0
  40. package/package.json +85 -0
  41. package/templates/README.md +65 -0
  42. package/templates/adapter.mjs +88 -0
  43. package/templates/ci-comment.yml +40 -0
  44. package/templates/ci.yml +100 -0
  45. package/templates/clock-adapter.mjs +42 -0
  46. package/templates/comment.cjs +209 -0
  47. package/templates/faults/duplicate-delivery/README.md +19 -0
  48. package/templates/faults/duplicate-delivery/adapter.mjs +40 -0
  49. package/templates/faults/duplicate-delivery/workflow.json +31 -0
  50. package/templates/faults/prompt-injection/README.md +17 -0
  51. package/templates/faults/prompt-injection/adapter.mjs +36 -0
  52. package/templates/faults/prompt-injection/workflow.json +43 -0
  53. package/templates/faults/stale-read/README.md +21 -0
  54. package/templates/faults/stale-read/adapter.mjs +59 -0
  55. package/templates/faults/stale-read/workflow.json +33 -0
  56. package/templates/langsmith-adapter.mjs +83 -0
  57. package/templates/workflow.json +106 -0
  58. package/templates/world-adapter.mjs +67 -0
package/dist/schema.js ADDED
@@ -0,0 +1,433 @@
1
+ // SPDX-License-Identifier: Apache-2.0
2
+ // Copyright 2026 Syed Arman
3
+ import { z } from 'zod';
4
+ const time = z.number().int().min(0).max(8_000_000_100_000_000);
5
+ export const virtualClockSchema = z.strictObject({
6
+ mode: z.literal('virtual'),
7
+ startMs: z.number().int().min(0).max(8_000_000_000_000_000).default(0),
8
+ maxTimeMs: z.number().int().min(1).max(86400000).default(60000),
9
+ });
10
+ const name = z.string().regex(/^[a-zA-Z0-9][a-zA-Z0-9_-]{0,99}$/);
11
+ const text = z.string().trim().min(1).max(2000);
12
+ const object = z.record(z.string(), z.json());
13
+ const pointer = z
14
+ .string()
15
+ .max(500)
16
+ .regex(/^(?:\/(?:[^~]|~[01])*)*$/);
17
+ /** Validate before recursive schemas, cloning, or hashing untrusted JSON. */
18
+ export function boundedJson(value, maxBytes = 5_000_000) {
19
+ const seen = new WeakSet();
20
+ let nodes = 0;
21
+ function walk(item, depth) {
22
+ if (++nodes > 100000 || depth > 30)
23
+ throw new Error('JSON is too deeply nested or too large.');
24
+ if (item === null || typeof item === 'string' || typeof item === 'boolean')
25
+ return;
26
+ if (typeof item === 'number' && Number.isFinite(item))
27
+ return;
28
+ if (!item || typeof item !== 'object' || seen.has(item))
29
+ throw new Error('Expected finite, acyclic JSON data.');
30
+ if (!Array.isArray(item) &&
31
+ ![Object.prototype, null].includes(Object.getPrototypeOf(item)))
32
+ throw new Error('Expected plain JSON objects.');
33
+ seen.add(item);
34
+ for (const child of Object.values(item))
35
+ walk(child, depth + 1);
36
+ seen.delete(item);
37
+ }
38
+ walk(value, 0);
39
+ if (new TextEncoder().encode(JSON.stringify(value)).length > maxBytes)
40
+ throw new Error('JSON exceeds the size limit.');
41
+ }
42
+ export function stableJson(value) {
43
+ if (Array.isArray(value))
44
+ return `[${value.map(stableJson).join(',')}]`;
45
+ if (value && typeof value === 'object')
46
+ return `{${Object.entries(value)
47
+ .sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0))
48
+ .map(([k, v]) => `${JSON.stringify(k)}:${stableJson(v)}`)
49
+ .join(',')}}`;
50
+ return JSON.stringify(value);
51
+ }
52
+ export const workflowFaultSchema = z.discriminatedUnion('type', [
53
+ z.strictObject({ type: z.literal('none') }),
54
+ z
55
+ .strictObject({
56
+ type: z.literal('prompt_injection'),
57
+ tool: name,
58
+ occurrence: z.number().int().min(1).max(100).default(1),
59
+ repeat: z.number().int().min(1).max(10).default(1),
60
+ replacement: z.json().refine((value) => {
61
+ try {
62
+ boundedJson(value, 16000);
63
+ return true;
64
+ }
65
+ catch {
66
+ return false;
67
+ }
68
+ }, 'Injected data exceeds the size or depth limit.'),
69
+ })
70
+ .refine((f) => f.occurrence + f.repeat - 1 <= 100, 'Invalid fault schedule.'),
71
+ z
72
+ .strictObject({
73
+ type: z.enum([
74
+ 'timeout_before',
75
+ 'timeout_after',
76
+ 'unavailable',
77
+ 'malformed',
78
+ 'permission_revoked',
79
+ 'duplicate_delivery',
80
+ 'stale_read',
81
+ ]),
82
+ tool: name,
83
+ occurrence: z.number().int().min(1).max(100).default(1),
84
+ repeat: z.number().int().min(1).max(10).default(1),
85
+ })
86
+ .refine((f) => f.occurrence + f.repeat - 1 <= 100 &&
87
+ (f.type !== 'permission_revoked' || f.repeat === 1), 'Invalid fault schedule.'),
88
+ ]);
89
+ export const checkSchema = z.discriminatedUnion('type', [
90
+ z.strictObject({
91
+ id: name,
92
+ label: text,
93
+ type: z.literal('forbidden_tool'),
94
+ tool: name,
95
+ }),
96
+ z.strictObject({
97
+ id: name,
98
+ label: text,
99
+ type: z.literal('equals'),
100
+ source: z.enum(['state', 'completion']).default('state'),
101
+ path: pointer,
102
+ expected: z.json(),
103
+ }),
104
+ z.strictObject({
105
+ id: name,
106
+ label: text,
107
+ type: z.literal('count'),
108
+ path: pointer,
109
+ expected: z.number().int().nonnegative(),
110
+ }),
111
+ z.strictObject({
112
+ id: name,
113
+ label: text,
114
+ type: z.literal('unchanged'),
115
+ path: pointer,
116
+ }),
117
+ ]);
118
+ const checks = z
119
+ .array(checkSchema)
120
+ .min(1)
121
+ .max(50)
122
+ .refine((items) => new Set(items.map((c) => c.id)).size === items.length, 'Check IDs must be unique.');
123
+ export const workflowSchema = z
124
+ .strictObject({
125
+ schemaVersion: z.literal('workflow/1'),
126
+ id: name,
127
+ title: text,
128
+ task: z.string().trim().min(1).max(10000),
129
+ initialState: object,
130
+ clock: virtualClockSchema.optional(),
131
+ checks,
132
+ scenarios: z
133
+ .array(z.strictObject({
134
+ id: name,
135
+ title: text,
136
+ fault: workflowFaultSchema,
137
+ checks: checks.optional(),
138
+ }))
139
+ .min(1)
140
+ .max(10),
141
+ limits: z
142
+ .strictObject({
143
+ maxToolCalls: z.number().int().min(1).max(100).default(20),
144
+ maxRepeatedCalls: z.number().int().min(1).max(100).default(5),
145
+ timeoutMs: z.number().int().min(1).max(60000).default(5000),
146
+ })
147
+ .default({ maxToolCalls: 20, maxRepeatedCalls: 5, timeoutMs: 5000 }),
148
+ })
149
+ .refine((w) => new Set(w.scenarios.map((s) => s.id)).size === w.scenarios.length &&
150
+ w.scenarios.every((s) => s.id !== 'baseline' && s.fault.type !== 'none'), 'Scenario IDs must be unique and cannot use the reserved baseline.');
151
+ export function parseWorkflow(input) {
152
+ boundedJson(input, 500000);
153
+ return workflowSchema.parse(input);
154
+ }
155
+ export const completionSchema = z.strictObject({
156
+ status: z.enum(['completed', 'blocked']),
157
+ message: z.string().max(10000),
158
+ output: z.json().optional(),
159
+ });
160
+ export const resultSchema = z.discriminatedUnion('ok', [
161
+ z.strictObject({ ok: z.literal(true), data: z.json() }),
162
+ z.strictObject({
163
+ ok: z.literal(false),
164
+ error: z.strictObject({
165
+ code: name,
166
+ message: text,
167
+ retryable: z.boolean(),
168
+ }),
169
+ }),
170
+ ]);
171
+ export const eventSchema = z.strictObject({
172
+ timeMs: time.optional(),
173
+ id: name,
174
+ sequence: z.number().int().positive(),
175
+ kind: z.enum([
176
+ 'request',
177
+ 'model',
178
+ 'tool_call',
179
+ 'tool_result',
180
+ 'delivery',
181
+ 'mutation',
182
+ 'fault',
183
+ 'completion',
184
+ 'error',
185
+ 'limit',
186
+ ]),
187
+ visibility: z.enum(['agent', 'evaluator']),
188
+ detail: z.string().max(10000),
189
+ callId: name.optional(),
190
+ tool: name.optional(),
191
+ payload: z.json().optional(),
192
+ });
193
+ const checkResultSchema = z.strictObject({
194
+ id: name,
195
+ label: text,
196
+ passed: z.boolean(),
197
+ detail: text,
198
+ before: z.json().optional(),
199
+ actual: z.json().optional(),
200
+ expected: z.json().optional(),
201
+ evidenceIds: z.array(name),
202
+ });
203
+ function at(value, path) {
204
+ let current = value;
205
+ if (path !== '')
206
+ for (const key of path
207
+ .slice(1)
208
+ .split('/')
209
+ .map((p) => p.replace(/~1/g, '/').replace(/~0/g, '~'))) {
210
+ if (!current ||
211
+ typeof current !== 'object' ||
212
+ !Object.hasOwn(current, key))
213
+ return { found: false };
214
+ current = current[key];
215
+ }
216
+ return current === undefined
217
+ ? { found: false }
218
+ : { found: true, value: current };
219
+ }
220
+ export function evaluateChecks(definitions, initial, final, completion, events) {
221
+ return definitions.map((check) => {
222
+ if (check.type === 'forbidden_tool') {
223
+ const attempts = events.filter((event) => event.kind === 'tool_call' && event.tool === check.tool);
224
+ return {
225
+ id: check.id,
226
+ label: check.label,
227
+ passed: attempts.length === 0,
228
+ detail: attempts.length === 0
229
+ ? 'No forbidden tool call was attempted.'
230
+ : 'The agent attempted a forbidden tool call.',
231
+ actual: attempts.length,
232
+ expected: 0,
233
+ evidenceIds: attempts.map((event) => event.id),
234
+ };
235
+ }
236
+ const prior = at(initial, check.path);
237
+ const observed = at(check.type === 'equals' && check.source === 'completion'
238
+ ? completion
239
+ : final, check.path);
240
+ const expected = check.type === 'unchanged' ? prior.value : check.expected;
241
+ const actual = check.type === 'count'
242
+ ? Array.isArray(observed.value)
243
+ ? observed.value.length
244
+ : undefined
245
+ : observed.value;
246
+ const passed = observed.found &&
247
+ actual !== undefined &&
248
+ expected !== undefined &&
249
+ stableJson(actual) === stableJson(expected);
250
+ return {
251
+ id: check.id,
252
+ label: check.label,
253
+ passed,
254
+ detail: !observed.found
255
+ ? `Missing value at ${check.path || '/'}.`
256
+ : check.type === 'count' && !Array.isArray(observed.value)
257
+ ? `Expected an array at ${check.path || '/'}.`
258
+ : passed
259
+ ? 'The observed outcome matches this check.'
260
+ : 'The observed outcome does not match this check.',
261
+ ...(prior.found ? { before: prior.value } : {}),
262
+ ...(actual !== undefined ? { actual } : {}),
263
+ ...(expected !== undefined ? { expected } : {}),
264
+ evidenceIds: events
265
+ .filter((e) => check.type === 'equals' && check.source === 'completion'
266
+ ? e.kind === 'completion'
267
+ : e.kind === 'mutation')
268
+ .map((e) => e.id),
269
+ };
270
+ });
271
+ }
272
+ export const commitShaSchema = z.string().regex(/^[a-f0-9]{40}$/);
273
+ export const executionSchema = z.strictObject({
274
+ id: z.uuid(),
275
+ commitSha: commitShaSchema.optional(),
276
+ trialsPerScenario: z.number().int().min(1).max(20),
277
+ scenarioIds: z
278
+ .array(name)
279
+ .min(1)
280
+ .max(10)
281
+ .refine((ids) => new Set(ids).size === ids.length && !ids.includes('baseline'), 'Invalid planned scenario IDs.'),
282
+ });
283
+ export const workflowReportSchema = z
284
+ .strictObject({
285
+ schemaVersion: z.literal('1'),
286
+ reportType: z.literal('workflow'),
287
+ id: z.string().regex(/^run_[a-f0-9-]{36}$/),
288
+ createdAt: z.iso.datetime(),
289
+ execution: executionSchema.optional(),
290
+ origin: z
291
+ .strictObject({
292
+ kind: z.literal('imported'),
293
+ originalId: z.string().max(100),
294
+ })
295
+ .optional(),
296
+ workflow: z.strictObject({
297
+ id: name,
298
+ title: text,
299
+ task: z.string().max(10000),
300
+ }),
301
+ suiteFingerprint: z.string().regex(/^[a-f0-9]{64}$/),
302
+ agent: z.strictObject({ id: name, version: text }),
303
+ scenario: z.strictObject({
304
+ id: name,
305
+ title: text,
306
+ fault: workflowFaultSchema,
307
+ }),
308
+ status: z.enum([
309
+ 'passed',
310
+ 'failed',
311
+ 'inconclusive',
312
+ 'cancelled',
313
+ 'timed_out',
314
+ ]),
315
+ issue: z
316
+ .enum([
317
+ 'agent_error',
318
+ 'tool_contract',
319
+ 'call_limit',
320
+ 'repeated_calls',
321
+ 'cancelled',
322
+ 'timed_out',
323
+ 'unawaited_calls',
324
+ 'evidence_limit',
325
+ 'initialization',
326
+ 'clock_limit',
327
+ 'clock_contract',
328
+ ])
329
+ .optional(),
330
+ fault: z.strictObject({
331
+ triggered: z.boolean(),
332
+ triggerCount: z.number().int().min(0).max(10),
333
+ }),
334
+ initialState: object,
335
+ clock: virtualClockSchema.extend({ endMs: time }).optional(),
336
+ finalState: object,
337
+ definitions: checks,
338
+ assertions: z.array(checkResultSchema).min(1).max(50),
339
+ completion: completionSchema.optional(),
340
+ events: z.array(eventSchema).min(1).max(1000),
341
+ metrics: z.strictObject({
342
+ toolCalls: z.number().int().min(0).max(100),
343
+ mutations: z.number().int().min(0).max(110),
344
+ durationMs: z.number().nonnegative(),
345
+ }),
346
+ })
347
+ .superRefine((report, ctx) => {
348
+ const validClock = report.clock
349
+ ? report.clock.endMs >= report.clock.startMs &&
350
+ report.clock.endMs - report.clock.startMs <= report.clock.maxTimeMs &&
351
+ report.events[0]?.timeMs === report.clock.startMs &&
352
+ report.events.at(-1)?.timeMs === report.clock.endMs &&
353
+ report.events.every((event, index) => event.timeMs !== undefined &&
354
+ event.timeMs >=
355
+ (index
356
+ ? report.events[index - 1].timeMs
357
+ : report.clock.startMs) &&
358
+ event.timeMs <= report.clock.endMs)
359
+ : report.events.every((event) => event.timeMs === undefined);
360
+ const faultEvents = report.events.filter((e) => e.kind === 'fault');
361
+ const validEvents = report.events.every((e, i) => e.sequence === i + 1 && e.id === `evt_${i + 1}`);
362
+ const assertions = evaluateChecks(report.definitions, report.initialState, report.finalState, report.completion, report.events);
363
+ const expectedStatus = report.issue === 'cancelled' || report.issue === 'timed_out'
364
+ ? report.issue
365
+ : report.issue === 'call_limit' || report.issue === 'repeated_calls'
366
+ ? 'failed'
367
+ : report.issue ||
368
+ (report.scenario.fault.type !== 'none' && !report.fault.triggered)
369
+ ? 'inconclusive'
370
+ : assertions.every((a) => a.passed)
371
+ ? 'passed'
372
+ : 'failed';
373
+ if (!validEvents ||
374
+ (report.execution &&
375
+ report.scenario.id !== 'baseline' &&
376
+ !report.execution.scenarioIds.includes(report.scenario.id)) ||
377
+ !validClock ||
378
+ stableJson(assertions) !== stableJson(report.assertions) ||
379
+ expectedStatus !== report.status ||
380
+ report.fault.triggerCount !== faultEvents.length ||
381
+ report.fault.triggered !== faultEvents.length > 0 ||
382
+ (report.scenario.fault.type === 'none' && report.fault.triggered) ||
383
+ report.metrics.toolCalls !==
384
+ report.events.filter((e) => e.kind === 'tool_call').length ||
385
+ report.metrics.mutations !==
386
+ report.events.filter((e) => e.kind === 'mutation').length ||
387
+ (!report.issue && !report.completion))
388
+ ctx.addIssue({
389
+ code: 'custom',
390
+ message: 'Report outcomes contradict their recorded evidence.',
391
+ });
392
+ });
393
+ export const workflowBatchSchema = z
394
+ .strictObject({
395
+ schemaVersion: z.literal('workflow-batch/1'),
396
+ baseline: workflowReportSchema,
397
+ runs: z.array(workflowReportSchema).max(200),
398
+ planned: z.number().int().min(1).max(200),
399
+ skippedReason: text.optional(),
400
+ })
401
+ .superRefine((batch, ctx) => {
402
+ if (batch.baseline.scenario.fault.type !== 'none' ||
403
+ (batch.baseline.execution &&
404
+ batch.planned !==
405
+ batch.baseline.execution.scenarioIds.length *
406
+ batch.baseline.execution.trialsPerScenario) ||
407
+ batch.runs.length > batch.planned ||
408
+ (batch.runs.length < batch.planned && !batch.skippedReason) ||
409
+ (batch.baseline.status !== 'passed' && batch.runs.length > 0) ||
410
+ batch.runs.some((r) => r.suiteFingerprint !== batch.baseline.suiteFingerprint ||
411
+ stableJson(r.execution ?? null) !==
412
+ stableJson(batch.baseline.execution ?? null) ||
413
+ stableJson(r.agent) !== stableJson(batch.baseline.agent) ||
414
+ stableJson(r.clock ? [r.clock.startMs, r.clock.maxTimeMs] : null) !==
415
+ stableJson(batch.baseline.clock
416
+ ? [batch.baseline.clock.startMs, batch.baseline.clock.maxTimeMs]
417
+ : null) ||
418
+ r.scenario.fault.type === 'none') ||
419
+ new Set([batch.baseline.id, ...batch.runs.map((r) => r.id)]).size !==
420
+ batch.runs.length + 1)
421
+ ctx.addIssue({
422
+ code: 'custom',
423
+ message: 'Batch evidence is inconsistent.',
424
+ });
425
+ });
426
+ export function workflowExitCode(batch) {
427
+ if (batch.skippedReason ||
428
+ [batch.baseline, ...batch.runs].some((r) => ['inconclusive', 'timed_out', 'cancelled'].includes(r.status)))
429
+ return 3;
430
+ return [batch.baseline, ...batch.runs].some((r) => r.status === 'failed')
431
+ ? 1
432
+ : 0;
433
+ }
@@ -0,0 +1 @@
1
+ export declare function createStarter(directory: string, ci: boolean, template?: string, virtualTime?: boolean): Promise<void>;
@@ -0,0 +1,64 @@
1
+ // SPDX-License-Identifier: Apache-2.0
2
+ // Copyright 2026 Syed Arman
3
+ import { mkdir, readFile, rmdir, unlink, writeFile } from 'node:fs/promises';
4
+ import { dirname, isAbsolute, relative, resolve, sep } from 'node:path';
5
+ import { worldFiles } from './worlds.js';
6
+ export async function createStarter(directory, ci, template = 'calendar', virtualTime = false) {
7
+ const files = await worldFiles(template);
8
+ if (virtualTime) {
9
+ const workflow = JSON.parse(files['workflow.json']);
10
+ workflow.clock = { mode: 'virtual', startMs: 0, maxTimeMs: 60000 };
11
+ files['workflow.json'] = JSON.stringify(workflow, null, 2) + '\n';
12
+ files['reference.mjs'] = files['adapter.mjs'];
13
+ files['adapter.mjs'] = await readFile(new URL('../templates/clock-adapter.mjs', import.meta.url), 'utf8');
14
+ files['README.md'] =
15
+ files['README.md'].replace('In `adapter.mjs`', 'In `reference.mjs`').replace('in the reference agent', 'in `reference.mjs`') +
16
+ '\n## Virtual time\n\nThe original tool/agent implementation is in `reference.mjs`. `adapter.mjs` wraps it with 25 ms of simulated service delay and 1000 ms of retry backoff. Run the same test command above. Each trial starts at virtual time zero. Reports include `clock.startMs`, `clock.endMs`, and event `timeMs`; `metrics.durationMs` remains actual runtime. Change the synthetic tool behavior and retry key in `reference.mjs`.\n\nUse the supplied `clock.now()` and `await clock.sleep(ms)` in custom logic. Date.now(), native timers, and external services are not virtualized. Whole tool transactions remain serialized. The clock has a per-trial time budget and a 1000-sleep cap; the real wall-clock deadline remains active.\n';
17
+ }
18
+ const configPath = relative(process.cwd(), resolve(directory, 'workflow.json'));
19
+ if (ci &&
20
+ (isAbsolute(configPath) ||
21
+ configPath.startsWith(`..${sep}`) ||
22
+ !/^[a-zA-Z0-9_./ -]+$/.test(configPath)))
23
+ throw new Error('With --ci, choose a folder inside this repository using letters, numbers, spaces, dashes, or underscores.');
24
+ const created = [];
25
+ const ciFile = resolve('.github/workflows/tracedojo.yml');
26
+ await mkdir(dirname(directory), { recursive: true });
27
+ await mkdir(directory, { mode: 0o700 });
28
+ try {
29
+ for (const file of [...Object.keys(files), 'LICENSE', 'NOTICE']) {
30
+ const destination = resolve(directory, file);
31
+ await writeFile(destination, files[file] ??
32
+ (await readFile(new URL(file === 'LICENSE' || file === 'NOTICE'
33
+ ? `../${file}`
34
+ : `../templates/${file}`, import.meta.url))), { flag: 'wx', mode: 0o600 });
35
+ created.push(destination);
36
+ }
37
+ if (ci) {
38
+ const template = await readFile(new URL('../templates/ci.yml', import.meta.url), 'utf8');
39
+ await mkdir(dirname(ciFile), { recursive: true });
40
+ await writeFile(ciFile, template
41
+ .replace('TRACEDOJO_CONFIG_TEMPLATE', JSON.stringify(configPath.split(sep).join('/')))
42
+ .replace('TRACEDOJO_ADAPTER_TEMPLATE', JSON.stringify(relative(process.cwd(), resolve(directory, 'adapter.mjs'))
43
+ .split(sep)
44
+ .join('/'))), { flag: 'wx', mode: 0o600 });
45
+ created.push(ciFile);
46
+ for (const [source, path] of [
47
+ ['ci-comment.yml', '.github/workflows/tracedojo-comment.yml'],
48
+ ['comment.cjs', '.github/tracedojo/comment.cjs'],
49
+ ]) {
50
+ const destination = resolve(path);
51
+ await mkdir(dirname(destination), { recursive: true });
52
+ await writeFile(destination, await readFile(new URL(`../templates/${source}`, import.meta.url)), { flag: 'wx', mode: 0o600 });
53
+ created.push(destination);
54
+ }
55
+ }
56
+ }
57
+ catch (error) {
58
+ // Only remove files created by this attempt. Never replace an existing setup.
59
+ for (const file of created.reverse())
60
+ await unlink(file);
61
+ await rmdir(directory);
62
+ throw error;
63
+ }
64
+ }
@@ -0,0 +1,2 @@
1
+ import { type WorkflowBatch } from './schema.js';
2
+ export declare function workflowSummary(batch: WorkflowBatch): string;
@@ -0,0 +1,50 @@
1
+ // SPDX-License-Identifier: Apache-2.0
2
+ // Copyright 2026 Syed Arman
3
+ import { workflowExitCode } from './schema.js';
4
+ // Report labels are user data, including in Markdown rendered by CI providers.
5
+ function escape(value) {
6
+ return value.replace(/[&<>|`*_[\]\\\r\n]/g, (char) => char === '\r' || char === '\n' ? ' ' : `&#${char.charCodeAt(0)};`);
7
+ }
8
+ export function workflowSummary(batch) {
9
+ const status = ['Passed', 'Failed', '', 'Incomplete'][workflowExitCode(batch)];
10
+ const lines = [
11
+ `## TraceDojo: ${status}`,
12
+ '',
13
+ `${escape(batch.baseline.workflow.title)} · ${escape(batch.baseline.agent.id)} (${escape(batch.baseline.agent.version)})`,
14
+ '',
15
+ `No-fault control: **${batch.baseline.status}**. Fault trials recorded: **${batch.runs.length}/${batch.planned}**.`,
16
+ '',
17
+ '| Scenario | Passed | Failed | Incomplete | Fault reached |',
18
+ '| --- | ---: | ---: | ---: | ---: |',
19
+ ];
20
+ const scenarios = new Map();
21
+ for (const run of batch.runs) {
22
+ const group = scenarios.get(run.scenario.id) ?? [];
23
+ group.push(run);
24
+ scenarios.set(run.scenario.id, group);
25
+ }
26
+ for (const group of scenarios.values()) {
27
+ const passed = group.filter((r) => r.status === 'passed').length;
28
+ const failed = group.filter((r) => r.status === 'failed').length;
29
+ const triggered = group.filter((r) => r.fault.triggered).length;
30
+ lines.push(`| ${escape(group[0].scenario.title)} | ${passed} | ${failed} | ${group.length - passed - failed} | ${triggered}/${group.length} |`);
31
+ }
32
+ if (batch.skippedReason)
33
+ lines.push('', escape(batch.skippedReason));
34
+ const problems = [batch.baseline, ...batch.runs].filter((run) => run.status !== 'passed');
35
+ if (problems.length)
36
+ lines.push('', '### Investigate', '');
37
+ for (const run of problems.slice(0, 10)) {
38
+ lines.push(`- ${escape(run.scenario.title)}: **${run.status}** (${run.id}).`);
39
+ if (run.issue)
40
+ lines.push(` - Execution issue: ${run.issue}.`);
41
+ if (run.scenario.fault.type !== 'none' && !run.fault.triggered)
42
+ lines.push(' - The configured fault was not reached; recovery is untested.');
43
+ for (const check of run.assertions.filter((c) => !c.passed))
44
+ lines.push(` - ${escape(check.label)}: ${escape(check.detail)}`);
45
+ }
46
+ if (problems.length > 10)
47
+ lines.push(`- ${problems.length - 10} more results need review in the JSON report.`);
48
+ lines.push('', 'Open the JSON report in TraceDojo to inspect tool calls, committed changes, and assertion evidence.', 'Counts describe these trials only; they are not a guarantee of production reliability.', '');
49
+ return lines.join('\n');
50
+ }
@@ -0,0 +1,25 @@
1
+ export interface TrendPoint {
2
+ commitSha: string;
3
+ recordedAt: string;
4
+ executions: number;
5
+ planned: number;
6
+ passed: number;
7
+ failed: number;
8
+ incomplete: number;
9
+ passRate: number | null;
10
+ reason?: string;
11
+ reportIds: string[];
12
+ }
13
+ export interface ScenarioTrend {
14
+ id: string;
15
+ title: string;
16
+ workflow: string;
17
+ agentId: string;
18
+ suiteFingerprint: string;
19
+ points: TrendPoint[];
20
+ }
21
+ /** Descriptive rates over supplied evidence only; never infer missing trials as passes. */
22
+ export declare function workflowTrends(input: unknown[]): {
23
+ series: ScenarioTrend[];
24
+ unlabelled: number;
25
+ };