@tracedojo/sdk 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (58) hide show
  1. package/LICENSE +202 -0
  2. package/NOTICE +5 -0
  3. package/README.md +96 -0
  4. package/dist/cli.d.ts +2 -0
  5. package/dist/cli.js +356 -0
  6. package/dist/clock.d.ts +41 -0
  7. package/dist/clock.js +118 -0
  8. package/dist/compare.d.ts +20 -0
  9. package/dist/compare.js +147 -0
  10. package/dist/demo.d.ts +465 -0
  11. package/dist/demo.js +45 -0
  12. package/dist/index.d.ts +35 -0
  13. package/dist/index.js +515 -0
  14. package/dist/langchain.d.ts +9 -0
  15. package/dist/langchain.js +17 -0
  16. package/dist/langsmith-starter.d.ts +4 -0
  17. package/dist/langsmith-starter.js +77 -0
  18. package/dist/langsmith.d.ts +30 -0
  19. package/dist/langsmith.js +148 -0
  20. package/dist/matrix-cli.d.ts +107 -0
  21. package/dist/matrix-cli.js +154 -0
  22. package/dist/matrix.d.ts +138 -0
  23. package/dist/matrix.js +148 -0
  24. package/dist/privacy.d.ts +5 -0
  25. package/dist/privacy.js +51 -0
  26. package/dist/reports.d.ts +2 -0
  27. package/dist/reports.js +38 -0
  28. package/dist/schema.d.ts +715 -0
  29. package/dist/schema.js +433 -0
  30. package/dist/starter.d.ts +1 -0
  31. package/dist/starter.js +64 -0
  32. package/dist/summary.d.ts +2 -0
  33. package/dist/summary.js +50 -0
  34. package/dist/trends.d.ts +25 -0
  35. package/dist/trends.js +164 -0
  36. package/dist/upload.d.ts +8 -0
  37. package/dist/upload.js +73 -0
  38. package/dist/worlds.d.ts +9 -0
  39. package/dist/worlds.js +279 -0
  40. package/package.json +85 -0
  41. package/templates/README.md +65 -0
  42. package/templates/adapter.mjs +88 -0
  43. package/templates/ci-comment.yml +40 -0
  44. package/templates/ci.yml +100 -0
  45. package/templates/clock-adapter.mjs +42 -0
  46. package/templates/comment.cjs +209 -0
  47. package/templates/faults/duplicate-delivery/README.md +19 -0
  48. package/templates/faults/duplicate-delivery/adapter.mjs +40 -0
  49. package/templates/faults/duplicate-delivery/workflow.json +31 -0
  50. package/templates/faults/prompt-injection/README.md +17 -0
  51. package/templates/faults/prompt-injection/adapter.mjs +36 -0
  52. package/templates/faults/prompt-injection/workflow.json +43 -0
  53. package/templates/faults/stale-read/README.md +21 -0
  54. package/templates/faults/stale-read/adapter.mjs +59 -0
  55. package/templates/faults/stale-read/workflow.json +33 -0
  56. package/templates/langsmith-adapter.mjs +83 -0
  57. package/templates/workflow.json +106 -0
  58. package/templates/world-adapter.mjs +67 -0
package/dist/matrix.js ADDED
@@ -0,0 +1,148 @@
1
+ // SPDX-License-Identifier: Apache-2.0
2
+ // Copyright 2026 Syed Arman
3
+ import { createHash } from 'node:crypto';
4
+ import { z } from 'zod';
5
+ import { boundedJson, parseWorkflow, stableJson, workflowSchema, } from './schema.js';
6
+ const name = z.string().regex(/^[a-zA-Z0-9][a-zA-Z0-9_-]{0,99}$/);
7
+ const inventorySchema = z
8
+ .array(z.strictObject({ name, kind: z.enum(['read', 'write']) }))
9
+ .min(1)
10
+ .max(50)
11
+ .refine((tools) => new Set(tools.map((tool) => tool.name)).size === tools.length, 'Tool names must be unique.');
12
+ const faultTypes = [
13
+ 'timeout_after',
14
+ 'duplicate_delivery',
15
+ 'timeout_before',
16
+ 'unavailable',
17
+ 'malformed',
18
+ 'permission_revoked',
19
+ 'stale_read',
20
+ ];
21
+ const optionsSchema = z.strictObject({
22
+ tools: z.array(name).min(1).max(50).optional(),
23
+ faults: z.array(z.enum(faultTypes)).min(1).max(7).optional(),
24
+ });
25
+ export function parseMatrixOptions(input) {
26
+ boundedJson(input, 20000);
27
+ return optionsSchema.parse(input);
28
+ }
29
+ export function parseMatrixDefinition(input) {
30
+ boundedJson(input, 500000);
31
+ const definition = input && typeof input === 'object' && !Array.isArray(input)
32
+ ? input
33
+ : {};
34
+ return parseWorkflow({
35
+ ...definition,
36
+ ...(Object.hasOwn(definition, 'scenarios')
37
+ ? {}
38
+ : {
39
+ scenarios: [
40
+ {
41
+ id: 'matrix-placeholder',
42
+ title: 'Matrix placeholder',
43
+ fault: { type: 'timeout_before', tool: 'matrix_tool' },
44
+ },
45
+ ],
46
+ }),
47
+ });
48
+ }
49
+ const applicable = (kind, fault) => kind === 'write'
50
+ ? fault !== 'stale_read'
51
+ : ['timeout_before', 'unavailable', 'malformed', 'stale_read'].includes(fault);
52
+ export const workflowMatrixSchema = z
53
+ .strictObject({
54
+ schemaVersion: z.literal('workflow-matrix/1'),
55
+ suites: z
56
+ .array(z.strictObject({
57
+ id: name,
58
+ tool: name,
59
+ kind: z.enum(['read', 'write']),
60
+ workflow: workflowSchema,
61
+ }))
62
+ .min(1)
63
+ .max(50),
64
+ })
65
+ .superRefine((matrix, ctx) => {
66
+ if (new Set(matrix.suites.map((suite) => suite.id)).size !==
67
+ matrix.suites.length ||
68
+ new Set(matrix.suites.map((suite) => suite.tool)).size !==
69
+ matrix.suites.length ||
70
+ matrix.suites.some((suite) => suite.workflow.scenarios.some(({ fault }) => fault.type === 'none' ||
71
+ fault.tool !== suite.tool ||
72
+ (['duplicate_delivery', 'permission_revoked'].includes(fault.type) &&
73
+ suite.kind !== 'write') ||
74
+ (fault.type === 'stale_read' && suite.kind !== 'read'))))
75
+ ctx.addIssue({
76
+ code: 'custom',
77
+ message: 'Matrix suites require unique IDs and tools with applicable faults.',
78
+ });
79
+ });
80
+ export function parseWorkflowMatrix(input) {
81
+ boundedJson(input);
82
+ const matrix = workflowMatrixSchema.parse(input);
83
+ for (const suite of matrix.suites)
84
+ boundedJson(suite.workflow, 500000);
85
+ return matrix;
86
+ }
87
+ /** Generate editable scenarios from declarations, without executing tools or an agent. */
88
+ export function generateFaultMatrix(input, inventory, options = {}) {
89
+ boundedJson(input, 500000);
90
+ boundedJson(inventory, 20000);
91
+ boundedJson(options, 20000);
92
+ const tools = inventorySchema
93
+ .parse(inventory)
94
+ .sort((a, b) => (a.name < b.name ? -1 : a.name > b.name ? 1 : 0));
95
+ const selected = parseMatrixOptions(options);
96
+ if (selected.tools &&
97
+ (new Set(selected.tools).size !== selected.tools.length ||
98
+ selected.tools.some((tool) => !tools.some((entry) => entry.name === tool))))
99
+ throw new Error('Select unique, registered tool names.');
100
+ if (selected.faults &&
101
+ new Set(selected.faults).size !== selected.faults.length)
102
+ throw new Error('Select unique fault types.');
103
+ // A task/state/check definition may omit scenarios entirely. Existing scenarios
104
+ // are still validated, and their exact-match check overrides are preserved.
105
+ const workflow = parseMatrixDefinition(input);
106
+ const originals = Object.hasOwn(input, 'scenarios')
107
+ ? workflow.scenarios
108
+ : [];
109
+ const faults = faultTypes.filter((type) => selected.faults ? selected.faults.includes(type) : type !== 'stale_read');
110
+ const suites = [];
111
+ for (const tool of tools.filter((tool) => !selected.tools || selected.tools.includes(tool.name))) {
112
+ const scenarios = [];
113
+ for (const type of faults.filter((type) => applicable(tool.kind, type))) {
114
+ const fault = {
115
+ type,
116
+ tool: tool.name,
117
+ occurrence: type === 'stale_read' ? 2 : 1,
118
+ repeat: 1,
119
+ };
120
+ const matches = originals.filter((scenario) => stableJson(scenario.fault) === stableJson(fault));
121
+ if (new Set(matches.map((scenario) => stableJson(scenario.checks ?? workflow.checks))).size > 1)
122
+ throw new Error('Matching scenarios have conflicting checks. Keep one set of expectations before generating a matrix.');
123
+ scenarios.push({
124
+ id: `matrix-${type}`,
125
+ title: `${tool.name}: ${type.replaceAll('_', ' ')}`,
126
+ fault,
127
+ ...(matches[0]?.checks
128
+ ? { checks: structuredClone(matches[0].checks) }
129
+ : {}),
130
+ });
131
+ }
132
+ if (!scenarios.length)
133
+ continue;
134
+ const digest = createHash('sha256')
135
+ .update(tool.name)
136
+ .digest('hex')
137
+ .slice(0, 12);
138
+ suites.push({
139
+ id: `${tool.name.slice(0, 60)}-${digest}`,
140
+ tool: tool.name,
141
+ kind: tool.kind,
142
+ workflow: parseWorkflow({ ...structuredClone(workflow), scenarios }),
143
+ });
144
+ }
145
+ if (!suites.length)
146
+ throw new Error('No selected fault applies to the selected tools.');
147
+ return parseWorkflowMatrix({ schemaVersion: 'workflow-matrix/1', suites });
148
+ }
@@ -0,0 +1,5 @@
1
+ import { type Json, type WorkflowBatch, type WorkflowReport } from './schema.js';
2
+ /** Pattern-based redaction, not comprehensive personal-data detection. */
3
+ export declare function sanitizeJson(input: unknown): Json;
4
+ export declare function sanitizeWorkflowReport(input: unknown): WorkflowReport;
5
+ export declare function sanitizeWorkflowBatch(input: unknown): WorkflowBatch;
@@ -0,0 +1,51 @@
1
+ // SPDX-License-Identifier: Apache-2.0
2
+ // Copyright 2026 Syed Arman
3
+ import { boundedJson, workflowBatchSchema, workflowReportSchema, } from './schema.js';
4
+ function scrub(value) {
5
+ return value
6
+ .replace(/\b(?:td_upload_[a-f0-9]{64}|sb_secret_[A-Za-z0-9_-]+|sk-(?:proj-)?[A-Za-z0-9_-]{16,}|gh[pousr]_[A-Za-z0-9]{20,}|github_pat_[A-Za-z0-9_]{20,})\b/g, '[REDACTED]')
7
+ .replace(/\bBearer\s+[A-Za-z0-9._~+/=-]+/gi, 'Bearer [REDACTED]')
8
+ .replace(/\b(password|api[_-]?key|access[_-]?token|refresh[_-]?token|secret)\s*[:=]\s*(?:"[^"\n]*"|'[^'\n]*'|[^\s,;}]+)/gi, '$1="[REDACTED]"')
9
+ .replace(/[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}/gi, (address) => /@(?:[a-z0-9.-]+\.)?(?:example|invalid)$/i.test(address)
10
+ ? address
11
+ : 'redacted@private.invalid');
12
+ }
13
+ /** Pattern-based redaction, not comprehensive personal-data detection. */
14
+ export function sanitizeJson(input) {
15
+ boundedJson(input);
16
+ const walk = (value) => {
17
+ if (typeof value === 'string')
18
+ return scrub(value);
19
+ if (Array.isArray(value))
20
+ return value.map(walk);
21
+ if (value && typeof value === 'object') {
22
+ const entries = Object.entries(value).map(([key, item]) => [
23
+ scrub(key),
24
+ /authorization|password|secret|api[_-]?key|access[_-]?token|refresh[_-]?token/i.test(key)
25
+ ? '[REDACTED]'
26
+ : walk(item),
27
+ ]);
28
+ if (new Set(entries.map(([key]) => key)).size !== entries.length)
29
+ throw new Error('Redaction would merge distinct fields. Remove private fields before export.');
30
+ return Object.fromEntries(entries);
31
+ }
32
+ return value;
33
+ };
34
+ return walk(input);
35
+ }
36
+ export function sanitizeWorkflowReport(input) {
37
+ boundedJson(input);
38
+ const report = workflowReportSchema.parse(input);
39
+ // Revalidate after scrubbing. If distinct secrets collapse into a false match,
40
+ // fail closed rather than silently changing a recorded verdict.
41
+ return workflowReportSchema.parse(sanitizeJson(report));
42
+ }
43
+ export function sanitizeWorkflowBatch(input) {
44
+ boundedJson(input);
45
+ const batch = workflowBatchSchema.parse(input);
46
+ return workflowBatchSchema.parse({
47
+ ...sanitizeJson(batch),
48
+ baseline: sanitizeWorkflowReport(batch.baseline),
49
+ runs: batch.runs.map(sanitizeWorkflowReport),
50
+ });
51
+ }
@@ -0,0 +1,2 @@
1
+ import { type WorkflowReport } from './schema.js';
2
+ export declare function parseWorkflowReports(input: unknown): WorkflowReport[];
@@ -0,0 +1,38 @@
1
+ // SPDX-License-Identifier: Apache-2.0
2
+ // Copyright 2026 Syed Arman
3
+ import { createHash } from 'node:crypto';
4
+ import { z } from 'zod';
5
+ import { boundedJson, stableJson, workflowBatchSchema, workflowReportSchema, } from './schema.js';
6
+ import { sanitizeWorkflowReport } from './privacy.js';
7
+ export function parseWorkflowReports(input) {
8
+ boundedJson(input);
9
+ const bundle = input &&
10
+ typeof input === 'object' &&
11
+ 'schemaVersion' in input &&
12
+ input.schemaVersion === 'workflow-upload/1'
13
+ ? z
14
+ .strictObject({
15
+ schemaVersion: z.literal('workflow-upload/1'),
16
+ reports: z.array(workflowReportSchema).min(1).max(21),
17
+ })
18
+ .parse(input).reports
19
+ : undefined;
20
+ const single = workflowReportSchema.safeParse(input);
21
+ const batch = single.success || bundle ? undefined : workflowBatchSchema.parse(input);
22
+ return (bundle ??
23
+ (single.success ? [single.data] : [batch.baseline, ...batch.runs])).map((report) => {
24
+ const { origin, ...data } = report;
25
+ const originalId = origin?.originalId ?? report.id;
26
+ const sanitized = sanitizeWorkflowReport({ ...data, id: originalId });
27
+ const digest = createHash('sha256')
28
+ .update(stableJson(sanitized))
29
+ .digest('hex')
30
+ .slice(0, 32);
31
+ const id = `run_${digest.slice(0, 8)}-${digest.slice(8, 12)}-${digest.slice(12, 16)}-${digest.slice(16, 20)}-${digest.slice(20)}`;
32
+ return workflowReportSchema.parse({
33
+ ...sanitized,
34
+ id,
35
+ origin: { kind: 'imported', originalId },
36
+ });
37
+ });
38
+ }