@tracedojo/sdk 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +202 -0
- package/NOTICE +5 -0
- package/README.md +96 -0
- package/dist/cli.d.ts +2 -0
- package/dist/cli.js +356 -0
- package/dist/clock.d.ts +41 -0
- package/dist/clock.js +118 -0
- package/dist/compare.d.ts +20 -0
- package/dist/compare.js +147 -0
- package/dist/demo.d.ts +465 -0
- package/dist/demo.js +45 -0
- package/dist/index.d.ts +35 -0
- package/dist/index.js +515 -0
- package/dist/langchain.d.ts +9 -0
- package/dist/langchain.js +17 -0
- package/dist/langsmith-starter.d.ts +4 -0
- package/dist/langsmith-starter.js +77 -0
- package/dist/langsmith.d.ts +30 -0
- package/dist/langsmith.js +148 -0
- package/dist/matrix-cli.d.ts +107 -0
- package/dist/matrix-cli.js +154 -0
- package/dist/matrix.d.ts +138 -0
- package/dist/matrix.js +148 -0
- package/dist/privacy.d.ts +5 -0
- package/dist/privacy.js +51 -0
- package/dist/reports.d.ts +2 -0
- package/dist/reports.js +38 -0
- package/dist/schema.d.ts +715 -0
- package/dist/schema.js +433 -0
- package/dist/starter.d.ts +1 -0
- package/dist/starter.js +64 -0
- package/dist/summary.d.ts +2 -0
- package/dist/summary.js +50 -0
- package/dist/trends.d.ts +25 -0
- package/dist/trends.js +164 -0
- package/dist/upload.d.ts +8 -0
- package/dist/upload.js +73 -0
- package/dist/worlds.d.ts +9 -0
- package/dist/worlds.js +279 -0
- package/package.json +85 -0
- package/templates/README.md +65 -0
- package/templates/adapter.mjs +88 -0
- package/templates/ci-comment.yml +40 -0
- package/templates/ci.yml +100 -0
- package/templates/clock-adapter.mjs +42 -0
- package/templates/comment.cjs +209 -0
- package/templates/faults/duplicate-delivery/README.md +19 -0
- package/templates/faults/duplicate-delivery/adapter.mjs +40 -0
- package/templates/faults/duplicate-delivery/workflow.json +31 -0
- package/templates/faults/prompt-injection/README.md +17 -0
- package/templates/faults/prompt-injection/adapter.mjs +36 -0
- package/templates/faults/prompt-injection/workflow.json +43 -0
- package/templates/faults/stale-read/README.md +21 -0
- package/templates/faults/stale-read/adapter.mjs +59 -0
- package/templates/faults/stale-read/workflow.json +33 -0
- package/templates/langsmith-adapter.mjs +83 -0
- package/templates/workflow.json +106 -0
- package/templates/world-adapter.mjs +67 -0
package/dist/matrix.js
ADDED
|
@@ -0,0 +1,148 @@
|
|
|
1
|
+
// SPDX-License-Identifier: Apache-2.0
|
|
2
|
+
// Copyright 2026 Syed Arman
|
|
3
|
+
import { createHash } from 'node:crypto';
|
|
4
|
+
import { z } from 'zod';
|
|
5
|
+
import { boundedJson, parseWorkflow, stableJson, workflowSchema, } from './schema.js';
|
|
6
|
+
const name = z.string().regex(/^[a-zA-Z0-9][a-zA-Z0-9_-]{0,99}$/);
|
|
7
|
+
const inventorySchema = z
|
|
8
|
+
.array(z.strictObject({ name, kind: z.enum(['read', 'write']) }))
|
|
9
|
+
.min(1)
|
|
10
|
+
.max(50)
|
|
11
|
+
.refine((tools) => new Set(tools.map((tool) => tool.name)).size === tools.length, 'Tool names must be unique.');
|
|
12
|
+
const faultTypes = [
|
|
13
|
+
'timeout_after',
|
|
14
|
+
'duplicate_delivery',
|
|
15
|
+
'timeout_before',
|
|
16
|
+
'unavailable',
|
|
17
|
+
'malformed',
|
|
18
|
+
'permission_revoked',
|
|
19
|
+
'stale_read',
|
|
20
|
+
];
|
|
21
|
+
const optionsSchema = z.strictObject({
|
|
22
|
+
tools: z.array(name).min(1).max(50).optional(),
|
|
23
|
+
faults: z.array(z.enum(faultTypes)).min(1).max(7).optional(),
|
|
24
|
+
});
|
|
25
|
+
export function parseMatrixOptions(input) {
|
|
26
|
+
boundedJson(input, 20000);
|
|
27
|
+
return optionsSchema.parse(input);
|
|
28
|
+
}
|
|
29
|
+
export function parseMatrixDefinition(input) {
|
|
30
|
+
boundedJson(input, 500000);
|
|
31
|
+
const definition = input && typeof input === 'object' && !Array.isArray(input)
|
|
32
|
+
? input
|
|
33
|
+
: {};
|
|
34
|
+
return parseWorkflow({
|
|
35
|
+
...definition,
|
|
36
|
+
...(Object.hasOwn(definition, 'scenarios')
|
|
37
|
+
? {}
|
|
38
|
+
: {
|
|
39
|
+
scenarios: [
|
|
40
|
+
{
|
|
41
|
+
id: 'matrix-placeholder',
|
|
42
|
+
title: 'Matrix placeholder',
|
|
43
|
+
fault: { type: 'timeout_before', tool: 'matrix_tool' },
|
|
44
|
+
},
|
|
45
|
+
],
|
|
46
|
+
}),
|
|
47
|
+
});
|
|
48
|
+
}
|
|
49
|
+
const applicable = (kind, fault) => kind === 'write'
|
|
50
|
+
? fault !== 'stale_read'
|
|
51
|
+
: ['timeout_before', 'unavailable', 'malformed', 'stale_read'].includes(fault);
|
|
52
|
+
export const workflowMatrixSchema = z
|
|
53
|
+
.strictObject({
|
|
54
|
+
schemaVersion: z.literal('workflow-matrix/1'),
|
|
55
|
+
suites: z
|
|
56
|
+
.array(z.strictObject({
|
|
57
|
+
id: name,
|
|
58
|
+
tool: name,
|
|
59
|
+
kind: z.enum(['read', 'write']),
|
|
60
|
+
workflow: workflowSchema,
|
|
61
|
+
}))
|
|
62
|
+
.min(1)
|
|
63
|
+
.max(50),
|
|
64
|
+
})
|
|
65
|
+
.superRefine((matrix, ctx) => {
|
|
66
|
+
if (new Set(matrix.suites.map((suite) => suite.id)).size !==
|
|
67
|
+
matrix.suites.length ||
|
|
68
|
+
new Set(matrix.suites.map((suite) => suite.tool)).size !==
|
|
69
|
+
matrix.suites.length ||
|
|
70
|
+
matrix.suites.some((suite) => suite.workflow.scenarios.some(({ fault }) => fault.type === 'none' ||
|
|
71
|
+
fault.tool !== suite.tool ||
|
|
72
|
+
(['duplicate_delivery', 'permission_revoked'].includes(fault.type) &&
|
|
73
|
+
suite.kind !== 'write') ||
|
|
74
|
+
(fault.type === 'stale_read' && suite.kind !== 'read'))))
|
|
75
|
+
ctx.addIssue({
|
|
76
|
+
code: 'custom',
|
|
77
|
+
message: 'Matrix suites require unique IDs and tools with applicable faults.',
|
|
78
|
+
});
|
|
79
|
+
});
|
|
80
|
+
export function parseWorkflowMatrix(input) {
|
|
81
|
+
boundedJson(input);
|
|
82
|
+
const matrix = workflowMatrixSchema.parse(input);
|
|
83
|
+
for (const suite of matrix.suites)
|
|
84
|
+
boundedJson(suite.workflow, 500000);
|
|
85
|
+
return matrix;
|
|
86
|
+
}
|
|
87
|
+
/** Generate editable scenarios from declarations, without executing tools or an agent. */
|
|
88
|
+
export function generateFaultMatrix(input, inventory, options = {}) {
|
|
89
|
+
boundedJson(input, 500000);
|
|
90
|
+
boundedJson(inventory, 20000);
|
|
91
|
+
boundedJson(options, 20000);
|
|
92
|
+
const tools = inventorySchema
|
|
93
|
+
.parse(inventory)
|
|
94
|
+
.sort((a, b) => (a.name < b.name ? -1 : a.name > b.name ? 1 : 0));
|
|
95
|
+
const selected = parseMatrixOptions(options);
|
|
96
|
+
if (selected.tools &&
|
|
97
|
+
(new Set(selected.tools).size !== selected.tools.length ||
|
|
98
|
+
selected.tools.some((tool) => !tools.some((entry) => entry.name === tool))))
|
|
99
|
+
throw new Error('Select unique, registered tool names.');
|
|
100
|
+
if (selected.faults &&
|
|
101
|
+
new Set(selected.faults).size !== selected.faults.length)
|
|
102
|
+
throw new Error('Select unique fault types.');
|
|
103
|
+
// A task/state/check definition may omit scenarios entirely. Existing scenarios
|
|
104
|
+
// are still validated, and their exact-match check overrides are preserved.
|
|
105
|
+
const workflow = parseMatrixDefinition(input);
|
|
106
|
+
const originals = Object.hasOwn(input, 'scenarios')
|
|
107
|
+
? workflow.scenarios
|
|
108
|
+
: [];
|
|
109
|
+
const faults = faultTypes.filter((type) => selected.faults ? selected.faults.includes(type) : type !== 'stale_read');
|
|
110
|
+
const suites = [];
|
|
111
|
+
for (const tool of tools.filter((tool) => !selected.tools || selected.tools.includes(tool.name))) {
|
|
112
|
+
const scenarios = [];
|
|
113
|
+
for (const type of faults.filter((type) => applicable(tool.kind, type))) {
|
|
114
|
+
const fault = {
|
|
115
|
+
type,
|
|
116
|
+
tool: tool.name,
|
|
117
|
+
occurrence: type === 'stale_read' ? 2 : 1,
|
|
118
|
+
repeat: 1,
|
|
119
|
+
};
|
|
120
|
+
const matches = originals.filter((scenario) => stableJson(scenario.fault) === stableJson(fault));
|
|
121
|
+
if (new Set(matches.map((scenario) => stableJson(scenario.checks ?? workflow.checks))).size > 1)
|
|
122
|
+
throw new Error('Matching scenarios have conflicting checks. Keep one set of expectations before generating a matrix.');
|
|
123
|
+
scenarios.push({
|
|
124
|
+
id: `matrix-${type}`,
|
|
125
|
+
title: `${tool.name}: ${type.replaceAll('_', ' ')}`,
|
|
126
|
+
fault,
|
|
127
|
+
...(matches[0]?.checks
|
|
128
|
+
? { checks: structuredClone(matches[0].checks) }
|
|
129
|
+
: {}),
|
|
130
|
+
});
|
|
131
|
+
}
|
|
132
|
+
if (!scenarios.length)
|
|
133
|
+
continue;
|
|
134
|
+
const digest = createHash('sha256')
|
|
135
|
+
.update(tool.name)
|
|
136
|
+
.digest('hex')
|
|
137
|
+
.slice(0, 12);
|
|
138
|
+
suites.push({
|
|
139
|
+
id: `${tool.name.slice(0, 60)}-${digest}`,
|
|
140
|
+
tool: tool.name,
|
|
141
|
+
kind: tool.kind,
|
|
142
|
+
workflow: parseWorkflow({ ...structuredClone(workflow), scenarios }),
|
|
143
|
+
});
|
|
144
|
+
}
|
|
145
|
+
if (!suites.length)
|
|
146
|
+
throw new Error('No selected fault applies to the selected tools.');
|
|
147
|
+
return parseWorkflowMatrix({ schemaVersion: 'workflow-matrix/1', suites });
|
|
148
|
+
}
|
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
import { type Json, type WorkflowBatch, type WorkflowReport } from './schema.js';
|
|
2
|
+
/** Pattern-based redaction, not comprehensive personal-data detection. */
|
|
3
|
+
export declare function sanitizeJson(input: unknown): Json;
|
|
4
|
+
export declare function sanitizeWorkflowReport(input: unknown): WorkflowReport;
|
|
5
|
+
export declare function sanitizeWorkflowBatch(input: unknown): WorkflowBatch;
|
package/dist/privacy.js
ADDED
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
// SPDX-License-Identifier: Apache-2.0
|
|
2
|
+
// Copyright 2026 Syed Arman
|
|
3
|
+
import { boundedJson, workflowBatchSchema, workflowReportSchema, } from './schema.js';
|
|
4
|
+
function scrub(value) {
|
|
5
|
+
return value
|
|
6
|
+
.replace(/\b(?:td_upload_[a-f0-9]{64}|sb_secret_[A-Za-z0-9_-]+|sk-(?:proj-)?[A-Za-z0-9_-]{16,}|gh[pousr]_[A-Za-z0-9]{20,}|github_pat_[A-Za-z0-9_]{20,})\b/g, '[REDACTED]')
|
|
7
|
+
.replace(/\bBearer\s+[A-Za-z0-9._~+/=-]+/gi, 'Bearer [REDACTED]')
|
|
8
|
+
.replace(/\b(password|api[_-]?key|access[_-]?token|refresh[_-]?token|secret)\s*[:=]\s*(?:"[^"\n]*"|'[^'\n]*'|[^\s,;}]+)/gi, '$1="[REDACTED]"')
|
|
9
|
+
.replace(/[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}/gi, (address) => /@(?:[a-z0-9.-]+\.)?(?:example|invalid)$/i.test(address)
|
|
10
|
+
? address
|
|
11
|
+
: 'redacted@private.invalid');
|
|
12
|
+
}
|
|
13
|
+
/** Pattern-based redaction, not comprehensive personal-data detection. */
|
|
14
|
+
export function sanitizeJson(input) {
|
|
15
|
+
boundedJson(input);
|
|
16
|
+
const walk = (value) => {
|
|
17
|
+
if (typeof value === 'string')
|
|
18
|
+
return scrub(value);
|
|
19
|
+
if (Array.isArray(value))
|
|
20
|
+
return value.map(walk);
|
|
21
|
+
if (value && typeof value === 'object') {
|
|
22
|
+
const entries = Object.entries(value).map(([key, item]) => [
|
|
23
|
+
scrub(key),
|
|
24
|
+
/authorization|password|secret|api[_-]?key|access[_-]?token|refresh[_-]?token/i.test(key)
|
|
25
|
+
? '[REDACTED]'
|
|
26
|
+
: walk(item),
|
|
27
|
+
]);
|
|
28
|
+
if (new Set(entries.map(([key]) => key)).size !== entries.length)
|
|
29
|
+
throw new Error('Redaction would merge distinct fields. Remove private fields before export.');
|
|
30
|
+
return Object.fromEntries(entries);
|
|
31
|
+
}
|
|
32
|
+
return value;
|
|
33
|
+
};
|
|
34
|
+
return walk(input);
|
|
35
|
+
}
|
|
36
|
+
export function sanitizeWorkflowReport(input) {
|
|
37
|
+
boundedJson(input);
|
|
38
|
+
const report = workflowReportSchema.parse(input);
|
|
39
|
+
// Revalidate after scrubbing. If distinct secrets collapse into a false match,
|
|
40
|
+
// fail closed rather than silently changing a recorded verdict.
|
|
41
|
+
return workflowReportSchema.parse(sanitizeJson(report));
|
|
42
|
+
}
|
|
43
|
+
export function sanitizeWorkflowBatch(input) {
|
|
44
|
+
boundedJson(input);
|
|
45
|
+
const batch = workflowBatchSchema.parse(input);
|
|
46
|
+
return workflowBatchSchema.parse({
|
|
47
|
+
...sanitizeJson(batch),
|
|
48
|
+
baseline: sanitizeWorkflowReport(batch.baseline),
|
|
49
|
+
runs: batch.runs.map(sanitizeWorkflowReport),
|
|
50
|
+
});
|
|
51
|
+
}
|
package/dist/reports.js
ADDED
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
// SPDX-License-Identifier: Apache-2.0
|
|
2
|
+
// Copyright 2026 Syed Arman
|
|
3
|
+
import { createHash } from 'node:crypto';
|
|
4
|
+
import { z } from 'zod';
|
|
5
|
+
import { boundedJson, stableJson, workflowBatchSchema, workflowReportSchema, } from './schema.js';
|
|
6
|
+
import { sanitizeWorkflowReport } from './privacy.js';
|
|
7
|
+
export function parseWorkflowReports(input) {
|
|
8
|
+
boundedJson(input);
|
|
9
|
+
const bundle = input &&
|
|
10
|
+
typeof input === 'object' &&
|
|
11
|
+
'schemaVersion' in input &&
|
|
12
|
+
input.schemaVersion === 'workflow-upload/1'
|
|
13
|
+
? z
|
|
14
|
+
.strictObject({
|
|
15
|
+
schemaVersion: z.literal('workflow-upload/1'),
|
|
16
|
+
reports: z.array(workflowReportSchema).min(1).max(21),
|
|
17
|
+
})
|
|
18
|
+
.parse(input).reports
|
|
19
|
+
: undefined;
|
|
20
|
+
const single = workflowReportSchema.safeParse(input);
|
|
21
|
+
const batch = single.success || bundle ? undefined : workflowBatchSchema.parse(input);
|
|
22
|
+
return (bundle ??
|
|
23
|
+
(single.success ? [single.data] : [batch.baseline, ...batch.runs])).map((report) => {
|
|
24
|
+
const { origin, ...data } = report;
|
|
25
|
+
const originalId = origin?.originalId ?? report.id;
|
|
26
|
+
const sanitized = sanitizeWorkflowReport({ ...data, id: originalId });
|
|
27
|
+
const digest = createHash('sha256')
|
|
28
|
+
.update(stableJson(sanitized))
|
|
29
|
+
.digest('hex')
|
|
30
|
+
.slice(0, 32);
|
|
31
|
+
const id = `run_${digest.slice(0, 8)}-${digest.slice(8, 12)}-${digest.slice(12, 16)}-${digest.slice(16, 20)}-${digest.slice(20)}`;
|
|
32
|
+
return workflowReportSchema.parse({
|
|
33
|
+
...sanitized,
|
|
34
|
+
id,
|
|
35
|
+
origin: { kind: 'imported', originalId },
|
|
36
|
+
});
|
|
37
|
+
});
|
|
38
|
+
}
|