@tracedojo/sdk 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (58) hide show
  1. package/LICENSE +202 -0
  2. package/NOTICE +5 -0
  3. package/README.md +96 -0
  4. package/dist/cli.d.ts +2 -0
  5. package/dist/cli.js +356 -0
  6. package/dist/clock.d.ts +41 -0
  7. package/dist/clock.js +118 -0
  8. package/dist/compare.d.ts +20 -0
  9. package/dist/compare.js +147 -0
  10. package/dist/demo.d.ts +465 -0
  11. package/dist/demo.js +45 -0
  12. package/dist/index.d.ts +35 -0
  13. package/dist/index.js +515 -0
  14. package/dist/langchain.d.ts +9 -0
  15. package/dist/langchain.js +17 -0
  16. package/dist/langsmith-starter.d.ts +4 -0
  17. package/dist/langsmith-starter.js +77 -0
  18. package/dist/langsmith.d.ts +30 -0
  19. package/dist/langsmith.js +148 -0
  20. package/dist/matrix-cli.d.ts +107 -0
  21. package/dist/matrix-cli.js +154 -0
  22. package/dist/matrix.d.ts +138 -0
  23. package/dist/matrix.js +148 -0
  24. package/dist/privacy.d.ts +5 -0
  25. package/dist/privacy.js +51 -0
  26. package/dist/reports.d.ts +2 -0
  27. package/dist/reports.js +38 -0
  28. package/dist/schema.d.ts +715 -0
  29. package/dist/schema.js +433 -0
  30. package/dist/starter.d.ts +1 -0
  31. package/dist/starter.js +64 -0
  32. package/dist/summary.d.ts +2 -0
  33. package/dist/summary.js +50 -0
  34. package/dist/trends.d.ts +25 -0
  35. package/dist/trends.js +164 -0
  36. package/dist/upload.d.ts +8 -0
  37. package/dist/upload.js +73 -0
  38. package/dist/worlds.d.ts +9 -0
  39. package/dist/worlds.js +279 -0
  40. package/package.json +85 -0
  41. package/templates/README.md +65 -0
  42. package/templates/adapter.mjs +88 -0
  43. package/templates/ci-comment.yml +40 -0
  44. package/templates/ci.yml +100 -0
  45. package/templates/clock-adapter.mjs +42 -0
  46. package/templates/comment.cjs +209 -0
  47. package/templates/faults/duplicate-delivery/README.md +19 -0
  48. package/templates/faults/duplicate-delivery/adapter.mjs +40 -0
  49. package/templates/faults/duplicate-delivery/workflow.json +31 -0
  50. package/templates/faults/prompt-injection/README.md +17 -0
  51. package/templates/faults/prompt-injection/adapter.mjs +36 -0
  52. package/templates/faults/prompt-injection/workflow.json +43 -0
  53. package/templates/faults/stale-read/README.md +21 -0
  54. package/templates/faults/stale-read/adapter.mjs +59 -0
  55. package/templates/faults/stale-read/workflow.json +33 -0
  56. package/templates/langsmith-adapter.mjs +83 -0
  57. package/templates/workflow.json +106 -0
  58. package/templates/world-adapter.mjs +67 -0
@@ -0,0 +1,41 @@
1
+ export interface WorkflowClock {
2
+ /** Epoch milliseconds in real mode; configured start plus elapsed time in virtual mode. */
3
+ now(): number;
4
+ /** Awaitable delay; virtual sleeps advance simulated time without waiting in real time. */
5
+ sleep(ms: number): Promise<void>;
6
+ }
7
+ type Fault = 'timed_out' | 'clock_limit' | 'clock_contract';
8
+ /** Per-trial scheduler. Only the frozen API is exposed to agents and tools. */
9
+ export declare class TrialClock {
10
+ private readonly config;
11
+ private readonly signal;
12
+ private readonly onFault;
13
+ readonly api: WorkflowClock;
14
+ private elapsed;
15
+ private sleeps;
16
+ private holds;
17
+ private closed;
18
+ private waiters;
19
+ private pump?;
20
+ private readonly abort;
21
+ constructor(config: {
22
+ mode: 'virtual';
23
+ startMs: number;
24
+ maxTimeMs: number;
25
+ } | undefined, signal: AbortSignal, onFault: (reason: Fault) => void);
26
+ get pending(): number;
27
+ /** Let a ready runner transaction start before advancing to a later timer. */
28
+ hold(): () => void;
29
+ get snapshot(): {
30
+ endMs: number;
31
+ mode: "virtual";
32
+ startMs: number;
33
+ maxTimeMs: number;
34
+ } | undefined;
35
+ private now;
36
+ private sleep;
37
+ private fail;
38
+ private schedule;
39
+ close(): void;
40
+ }
41
+ export {};
package/dist/clock.js ADDED
@@ -0,0 +1,118 @@
1
+ // SPDX-License-Identifier: Apache-2.0
2
+ // Copyright 2026 Syed Arman
3
+ /** Per-trial scheduler. Only the frozen API is exposed to agents and tools. */
4
+ export class TrialClock {
5
+ config;
6
+ signal;
7
+ onFault;
8
+ api;
9
+ elapsed = 0;
10
+ sleeps = 0;
11
+ holds = 0;
12
+ closed = false;
13
+ waiters = new Set();
14
+ pump;
15
+ abort;
16
+ constructor(config, signal, onFault) {
17
+ this.config = config;
18
+ this.signal = signal;
19
+ this.onFault = onFault;
20
+ this.api = Object.freeze({
21
+ now: () => this.now(),
22
+ sleep: (ms) => this.sleep(ms),
23
+ });
24
+ this.abort = () => this.close();
25
+ signal.addEventListener('abort', this.abort, { once: true });
26
+ }
27
+ get pending() {
28
+ return this.waiters.size;
29
+ }
30
+ /** Let a ready runner transaction start before advancing to a later timer. */
31
+ hold() {
32
+ this.holds++;
33
+ return () => {
34
+ this.holds--;
35
+ };
36
+ }
37
+ get snapshot() {
38
+ return this.config ? { ...this.config, endMs: this.now() } : undefined;
39
+ }
40
+ now() {
41
+ return this.config ? this.config.startMs + this.elapsed : Date.now();
42
+ }
43
+ sleep(ms) {
44
+ const promise = new Promise((resolve, reject) => {
45
+ if (this.closed || this.signal.aborted) {
46
+ reject(new Error('Trial clock is closed.'));
47
+ return;
48
+ }
49
+ if (!Number.isSafeInteger(ms) || ms < 0 || ms > 86400000) {
50
+ this.fail('clock_contract');
51
+ reject(new Error('Delay must be an integer from 0 to 86400000.'));
52
+ return;
53
+ }
54
+ if (++this.sleeps > 1000) {
55
+ this.fail('clock_limit');
56
+ reject(new Error('Trial sleep limit exceeded.'));
57
+ return;
58
+ }
59
+ const waiter = { due: this.elapsed + ms, resolve, reject };
60
+ this.waiters.add(waiter);
61
+ if (this.config)
62
+ this.schedule();
63
+ else
64
+ waiter.timer = setTimeout(() => {
65
+ this.waiters.delete(waiter);
66
+ resolve();
67
+ }, ms);
68
+ });
69
+ // Abandoned work must not create process-level unhandled rejections on shutdown.
70
+ void promise.catch(() => undefined);
71
+ return promise;
72
+ }
73
+ fail(reason) {
74
+ this.onFault(reason);
75
+ this.close();
76
+ }
77
+ schedule() {
78
+ if (this.pump || this.closed || !this.waiters.size)
79
+ return;
80
+ this.pump = setImmediate(() => {
81
+ this.pump = undefined;
82
+ if (this.closed)
83
+ return;
84
+ if (this.holds) {
85
+ this.schedule();
86
+ return;
87
+ }
88
+ const due = Math.min(...[...this.waiters].map((w) => w.due));
89
+ if (due > this.config.maxTimeMs) {
90
+ this.elapsed = this.config.maxTimeMs;
91
+ this.fail('timed_out');
92
+ return;
93
+ }
94
+ this.elapsed = due;
95
+ for (const waiter of this.waiters)
96
+ if (waiter.due === due) {
97
+ this.waiters.delete(waiter);
98
+ waiter.resolve();
99
+ }
100
+ // Yield so resumed code can register its next timer before time moves again.
101
+ this.schedule();
102
+ });
103
+ }
104
+ close() {
105
+ if (this.closed)
106
+ return;
107
+ this.closed = true;
108
+ this.signal.removeEventListener('abort', this.abort);
109
+ if (this.pump)
110
+ clearImmediate(this.pump);
111
+ for (const waiter of this.waiters) {
112
+ if (waiter.timer)
113
+ clearTimeout(waiter.timer);
114
+ waiter.reject(new Error('Trial clock is closed.'));
115
+ }
116
+ this.waiters.clear();
117
+ }
118
+ }
@@ -0,0 +1,20 @@
1
+ export interface ScenarioComparison {
2
+ id: string;
3
+ title: string;
4
+ trials: number;
5
+ baseFailures: number;
6
+ candidateFailures: number;
7
+ regressedChecks: string[];
8
+ status: 'regressed' | 'improved' | 'unchanged';
9
+ }
10
+ export interface WorkflowComparison {
11
+ schemaVersion: 'workflow-comparison/1';
12
+ status: 'no_regressions' | 'regressed' | 'incomparable';
13
+ reason: string;
14
+ scenarios: ScenarioComparison[];
15
+ }
16
+ export declare function incompleteComparison(reason: string): WorkflowComparison;
17
+ /** Compare matched observations, never treat missing or changed evidence as a pass. */
18
+ export declare function compareWorkflowBatches(baseInput: unknown, candidateInput: unknown): WorkflowComparison;
19
+ export declare function comparisonExitCode(result: WorkflowComparison): 0 | 1 | 3;
20
+ export declare function comparisonSummary(result: WorkflowComparison): string;
@@ -0,0 +1,147 @@
1
+ // SPDX-License-Identifier: Apache-2.0
2
+ // Copyright 2026 Syed Arman
3
+ import { boundedJson, stableJson, workflowBatchSchema, } from './schema.js';
4
+ export function incompleteComparison(reason) {
5
+ return {
6
+ schemaVersion: 'workflow-comparison/1',
7
+ status: 'incomparable',
8
+ reason,
9
+ scenarios: [],
10
+ };
11
+ }
12
+ function group(runs) {
13
+ const groups = new Map();
14
+ for (const run of runs)
15
+ groups.set(run.scenario.id, [...(groups.get(run.scenario.id) ?? []), run]);
16
+ return groups;
17
+ }
18
+ function conditions(run) {
19
+ return stableJson({
20
+ workflow: run.workflow,
21
+ initialState: run.initialState,
22
+ scenario: run.scenario,
23
+ definitions: run.definitions,
24
+ clock: run.clock
25
+ ? {
26
+ mode: run.clock.mode,
27
+ startMs: run.clock.startMs,
28
+ maxTimeMs: run.clock.maxTimeMs,
29
+ }
30
+ : null,
31
+ });
32
+ }
33
+ function failureCounts(runs) {
34
+ const counts = new Map();
35
+ for (const run of runs) {
36
+ for (const check of run.assertions)
37
+ if (!check.passed)
38
+ counts.set(check.id, (counts.get(check.id) ?? 0) + 1);
39
+ if (run.issue)
40
+ counts.set(`execution:${run.issue}`, (counts.get(`execution:${run.issue}`) ?? 0) + 1);
41
+ }
42
+ return counts;
43
+ }
44
+ /** Compare matched observations, never treat missing or changed evidence as a pass. */
45
+ export function compareWorkflowBatches(baseInput, candidateInput) {
46
+ if (baseInput === undefined || candidateInput === undefined)
47
+ return incompleteComparison('A baseline or candidate report is missing. Run both before gating regressions.');
48
+ boundedJson(baseInput);
49
+ boundedJson(candidateInput);
50
+ const base = workflowBatchSchema.parse(baseInput);
51
+ const candidate = workflowBatchSchema.parse(candidateInput);
52
+ if (base.baseline.suiteFingerprint !== candidate.baseline.suiteFingerprint ||
53
+ base.baseline.agent.id !== candidate.baseline.agent.id ||
54
+ conditions(base.baseline) !== conditions(candidate.baseline))
55
+ return incompleteComparison('Test conditions or agent identity changed. Review the suite and establish a matching baseline.');
56
+ if (base.baseline.status !== 'passed')
57
+ return incompleteComparison('The baseline no-fault control did not pass. Establish a working baseline first.');
58
+ if (candidate.baseline.status === 'failed')
59
+ return {
60
+ schemaVersion: 'workflow-comparison/1',
61
+ status: 'regressed',
62
+ reason: 'The candidate no-fault control regressed; its fault trials were skipped.',
63
+ scenarios: [
64
+ {
65
+ id: 'baseline',
66
+ title: 'No-fault control',
67
+ trials: 1,
68
+ baseFailures: 0,
69
+ candidateFailures: 1,
70
+ regressedChecks: [...failureCounts([candidate.baseline]).keys()],
71
+ status: 'regressed',
72
+ },
73
+ ],
74
+ };
75
+ const complete = (batch) => !batch.skippedReason &&
76
+ batch.runs.length === batch.planned &&
77
+ batch.baseline.status === 'passed' &&
78
+ batch.runs.every((run) => ['passed', 'failed'].includes(run.status) && run.fault.triggered);
79
+ if (!complete(base) || !complete(candidate))
80
+ return incompleteComparison('Trials are missing, interrupted, or did not reach their configured fault. Recovery is untested.');
81
+ const left = group(base.runs);
82
+ const right = group(candidate.runs);
83
+ if (base.planned !== candidate.planned ||
84
+ left.size !== right.size ||
85
+ [...left].some(([id, runs]) => right.get(id)?.length !== runs.length))
86
+ return incompleteComparison('Scenario sets or trial counts differ. Run matching scenarios with equal trial counts.');
87
+ const scenarios = [];
88
+ for (const [id, before] of left) {
89
+ const after = right.get(id);
90
+ if ([...before, ...after].some((run) => conditions(run) !== conditions(before[0])))
91
+ return incompleteComparison('Recorded scenario conditions differ despite the suite fingerprint. Review the evidence.');
92
+ const a = failureCounts(before);
93
+ const b = failureCounts(after);
94
+ const regressedChecks = [...b]
95
+ .filter(([key, count]) => count > (a.get(key) ?? 0))
96
+ .map(([key]) => key);
97
+ const baseFailures = before.filter((run) => run.status === 'failed').length;
98
+ const candidateFailures = after.filter((run) => run.status === 'failed').length;
99
+ const regressed = candidateFailures > baseFailures || regressedChecks.length > 0;
100
+ const improved = candidateFailures < baseFailures ||
101
+ [...a].some(([key, count]) => count > (b.get(key) ?? 0));
102
+ scenarios.push({
103
+ id,
104
+ title: before[0].scenario.title,
105
+ trials: before.length,
106
+ baseFailures,
107
+ candidateFailures,
108
+ regressedChecks,
109
+ status: regressed ? 'regressed' : improved ? 'improved' : 'unchanged',
110
+ });
111
+ }
112
+ const regressions = scenarios.filter((s) => s.status === 'regressed').length;
113
+ return {
114
+ schemaVersion: 'workflow-comparison/1',
115
+ status: regressions ? 'regressed' : 'no_regressions',
116
+ reason: regressions
117
+ ? `${regressions} scenario(s) regressed against the baseline.`
118
+ : 'No new failures in matched trials. Existing failures may remain.',
119
+ scenarios,
120
+ };
121
+ }
122
+ export function comparisonExitCode(result) {
123
+ return result.status === 'incomparable'
124
+ ? 3
125
+ : result.status === 'regressed'
126
+ ? 1
127
+ : 0;
128
+ }
129
+ const escape = (value) => value.replace(/[&<>|`*_[\]\\\r\n@]/g, (char) => char === '\r' || char === '\n' ? ' ' : `&#${char.charCodeAt(0)};`);
130
+ export function comparisonSummary(result) {
131
+ return [
132
+ `## TraceDojo: ${result.status === 'regressed' ? 'Regressions found' : result.status === 'incomparable' ? 'Comparison incomplete' : 'No regressions'}`,
133
+ '',
134
+ escape(result.reason),
135
+ '',
136
+ '| Scenario | Baseline failures | Candidate failures | Trials | Change |',
137
+ '| --- | ---: | ---: | ---: | --- |',
138
+ ...result.scenarios.map((s) => `| ${escape(s.title)} | ${s.baseFailures} | ${s.candidateFailures} | ${s.trials} | ${s.status} |`),
139
+ '',
140
+ ...result.scenarios
141
+ .filter((s) => s.regressedChecks.length)
142
+ .map((s) => `- ${escape(s.title)}: increased failures for ${s.regressedChecks.map(escape).join(', ')}.`),
143
+ '',
144
+ 'Counts describe these observations, not statistical significance or a guarantee of reliability. Tool implementation equivalence requires review.',
145
+ '',
146
+ ].join('\n');
147
+ }