@tracedojo/sdk 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +202 -0
- package/NOTICE +5 -0
- package/README.md +96 -0
- package/dist/cli.d.ts +2 -0
- package/dist/cli.js +356 -0
- package/dist/clock.d.ts +41 -0
- package/dist/clock.js +118 -0
- package/dist/compare.d.ts +20 -0
- package/dist/compare.js +147 -0
- package/dist/demo.d.ts +465 -0
- package/dist/demo.js +45 -0
- package/dist/index.d.ts +35 -0
- package/dist/index.js +515 -0
- package/dist/langchain.d.ts +9 -0
- package/dist/langchain.js +17 -0
- package/dist/langsmith-starter.d.ts +4 -0
- package/dist/langsmith-starter.js +77 -0
- package/dist/langsmith.d.ts +30 -0
- package/dist/langsmith.js +148 -0
- package/dist/matrix-cli.d.ts +107 -0
- package/dist/matrix-cli.js +154 -0
- package/dist/matrix.d.ts +138 -0
- package/dist/matrix.js +148 -0
- package/dist/privacy.d.ts +5 -0
- package/dist/privacy.js +51 -0
- package/dist/reports.d.ts +2 -0
- package/dist/reports.js +38 -0
- package/dist/schema.d.ts +715 -0
- package/dist/schema.js +433 -0
- package/dist/starter.d.ts +1 -0
- package/dist/starter.js +64 -0
- package/dist/summary.d.ts +2 -0
- package/dist/summary.js +50 -0
- package/dist/trends.d.ts +25 -0
- package/dist/trends.js +164 -0
- package/dist/upload.d.ts +8 -0
- package/dist/upload.js +73 -0
- package/dist/worlds.d.ts +9 -0
- package/dist/worlds.js +279 -0
- package/package.json +85 -0
- package/templates/README.md +65 -0
- package/templates/adapter.mjs +88 -0
- package/templates/ci-comment.yml +40 -0
- package/templates/ci.yml +100 -0
- package/templates/clock-adapter.mjs +42 -0
- package/templates/comment.cjs +209 -0
- package/templates/faults/duplicate-delivery/README.md +19 -0
- package/templates/faults/duplicate-delivery/adapter.mjs +40 -0
- package/templates/faults/duplicate-delivery/workflow.json +31 -0
- package/templates/faults/prompt-injection/README.md +17 -0
- package/templates/faults/prompt-injection/adapter.mjs +36 -0
- package/templates/faults/prompt-injection/workflow.json +43 -0
- package/templates/faults/stale-read/README.md +21 -0
- package/templates/faults/stale-read/adapter.mjs +59 -0
- package/templates/faults/stale-read/workflow.json +33 -0
- package/templates/langsmith-adapter.mjs +83 -0
- package/templates/workflow.json +106 -0
- package/templates/world-adapter.mjs +67 -0
|
@@ -0,0 +1,148 @@
|
|
|
1
|
+
// SPDX-License-Identifier: Apache-2.0
|
|
2
|
+
// Copyright 2026 Syed Arman
|
|
3
|
+
import { z } from 'zod';
|
|
4
|
+
import { boundedJson } from './schema.js';
|
|
5
|
+
import { sanitizeJson } from './privacy.js';
|
|
6
|
+
const runSchema = z.object({
|
|
7
|
+
id: z.string().min(1).max(100),
|
|
8
|
+
name: z.string().min(1).max(200),
|
|
9
|
+
run_type: z.string().max(50),
|
|
10
|
+
parent_run_id: z.string().max(100).nullable().optional(),
|
|
11
|
+
trace_id: z.string().max(100).optional(),
|
|
12
|
+
start_time: z.iso.datetime({ offset: true }).optional(),
|
|
13
|
+
inputs: z.record(z.string(), z.json()).default({}),
|
|
14
|
+
outputs: z.json().optional(),
|
|
15
|
+
error: z.string().nullable().optional(),
|
|
16
|
+
child_runs: z.array(z.unknown()).optional(),
|
|
17
|
+
});
|
|
18
|
+
/** Accept a single nested run or a listRuns export containing one complete trace. */
|
|
19
|
+
export function importLangSmithTrace(input) {
|
|
20
|
+
boundedJson(input);
|
|
21
|
+
const runs = [];
|
|
22
|
+
const visit = (value) => {
|
|
23
|
+
if (runs.length >= 500)
|
|
24
|
+
throw new Error('Import at most 500 spans from one trace.');
|
|
25
|
+
const run = runSchema.parse(value);
|
|
26
|
+
runs.push(run);
|
|
27
|
+
for (const child of run.child_runs ?? [])
|
|
28
|
+
visit(child);
|
|
29
|
+
};
|
|
30
|
+
const entries = Array.isArray(input)
|
|
31
|
+
? input
|
|
32
|
+
: input && typeof input === 'object' && 'runs' in input
|
|
33
|
+
? input.runs
|
|
34
|
+
: [input];
|
|
35
|
+
if (!Array.isArray(entries) || entries.length === 0)
|
|
36
|
+
throw new Error('Choose a LangSmith JSON export containing one trace.');
|
|
37
|
+
entries.forEach(visit);
|
|
38
|
+
const ids = new Set(runs.map((r) => r.id));
|
|
39
|
+
if (ids.size !== runs.length)
|
|
40
|
+
throw new Error('The export contains duplicate spans. Export one complete trace.');
|
|
41
|
+
const roots = runs.filter((r) => !r.parent_run_id);
|
|
42
|
+
if (roots.length !== 1)
|
|
43
|
+
throw new Error('Export one complete trace with exactly one root run.');
|
|
44
|
+
const root = roots[0];
|
|
45
|
+
const traceId = root.trace_id ?? root.id;
|
|
46
|
+
const byId = new Map(runs.map((r) => [r.id, r]));
|
|
47
|
+
for (const run of runs) {
|
|
48
|
+
if (run.trace_id && run.trace_id !== traceId)
|
|
49
|
+
throw new Error('The export mixes different traces.');
|
|
50
|
+
const visited = new Set();
|
|
51
|
+
let current = run;
|
|
52
|
+
while (current.parent_run_id) {
|
|
53
|
+
if (visited.has(current.id))
|
|
54
|
+
throw new Error('The trace contains a parent cycle.');
|
|
55
|
+
visited.add(current.id);
|
|
56
|
+
const parent = byId.get(current.parent_run_id);
|
|
57
|
+
if (!parent)
|
|
58
|
+
throw new Error('The trace is incomplete: a parent span is missing.');
|
|
59
|
+
current = parent;
|
|
60
|
+
}
|
|
61
|
+
if (current.id !== root.id)
|
|
62
|
+
throw new Error('The export contains disconnected spans.');
|
|
63
|
+
}
|
|
64
|
+
const calls = runs.filter((r) => r.run_type === 'tool');
|
|
65
|
+
// With missing timestamps, retain export order instead of ordering by opaque IDs.
|
|
66
|
+
if (calls.every((call) => call.start_time))
|
|
67
|
+
calls.sort((a, b) => Date.parse(a.start_time) - Date.parse(b.start_time));
|
|
68
|
+
if (!calls.length)
|
|
69
|
+
throw new Error('No tool calls found. Export a complete agent trace including tool spans.');
|
|
70
|
+
const candidate = [
|
|
71
|
+
root.inputs.task,
|
|
72
|
+
root.inputs.input,
|
|
73
|
+
root.inputs.question,
|
|
74
|
+
].find((v) => typeof v === 'string');
|
|
75
|
+
const draft = {
|
|
76
|
+
schemaVersion: 'workflow-draft/1',
|
|
77
|
+
status: 'needs_review',
|
|
78
|
+
source: {
|
|
79
|
+
provider: 'langsmith',
|
|
80
|
+
traceId,
|
|
81
|
+
rootRunId: root.id,
|
|
82
|
+
name: root.name,
|
|
83
|
+
},
|
|
84
|
+
taskCandidate: typeof candidate === 'string' ? candidate : '',
|
|
85
|
+
observedCalls: calls.map((r) => ({
|
|
86
|
+
id: r.id,
|
|
87
|
+
name: r.name,
|
|
88
|
+
inputs: r.inputs,
|
|
89
|
+
...(r.outputs !== undefined ? { outputs: r.outputs } : {}),
|
|
90
|
+
...(r.error ? { error: r.error } : {}),
|
|
91
|
+
})),
|
|
92
|
+
required: [
|
|
93
|
+
'Review the task and tool inputs; recorded outputs are observations, not expected outcomes.',
|
|
94
|
+
'Supply starting state and isolated simulated tool implementations.',
|
|
95
|
+
'Define outcome checks and choose which tool failure to inject.',
|
|
96
|
+
],
|
|
97
|
+
};
|
|
98
|
+
const sanitized = sanitizeJson(draft);
|
|
99
|
+
if (new Set(calls.map((call) => call.name)).size !==
|
|
100
|
+
new Set(sanitized.observedCalls.map((call) => call.name)).size)
|
|
101
|
+
throw new Error('Redaction would merge distinct tool names. Rename them before export.');
|
|
102
|
+
return sanitized;
|
|
103
|
+
}
|
|
104
|
+
/** Observed JSON shapes only; these do not prove a tool's complete API contract. */
|
|
105
|
+
function observedShape(values) {
|
|
106
|
+
if (!values.length)
|
|
107
|
+
return {};
|
|
108
|
+
const groups = new Map();
|
|
109
|
+
for (const value of values) {
|
|
110
|
+
const type = value === null ? 'null' : Array.isArray(value) ? 'array' : typeof value;
|
|
111
|
+
groups.set(type, [...(groups.get(type) ?? []), value]);
|
|
112
|
+
}
|
|
113
|
+
if (groups.size > 1)
|
|
114
|
+
return { anyOf: [...groups.values()].map(observedShape) };
|
|
115
|
+
const type = [...groups.keys()][0];
|
|
116
|
+
if (type === 'object') {
|
|
117
|
+
const objects = values;
|
|
118
|
+
const keys = [...new Set(objects.flatMap(Object.keys))].sort();
|
|
119
|
+
return {
|
|
120
|
+
type,
|
|
121
|
+
properties: Object.fromEntries(keys.map((key) => [
|
|
122
|
+
key,
|
|
123
|
+
observedShape(objects
|
|
124
|
+
.filter((obj) => Object.hasOwn(obj, key))
|
|
125
|
+
.map((obj) => obj[key])),
|
|
126
|
+
])),
|
|
127
|
+
required: keys.filter((key) => objects.every((obj) => Object.hasOwn(obj, key))),
|
|
128
|
+
};
|
|
129
|
+
}
|
|
130
|
+
if (type === 'array')
|
|
131
|
+
return { type, items: observedShape(values.flat()) };
|
|
132
|
+
return { type };
|
|
133
|
+
}
|
|
134
|
+
export function inferToolSignatures(draft) {
|
|
135
|
+
return [...new Set(draft.observedCalls.map((call) => call.name))].map((name, index) => {
|
|
136
|
+
const calls = draft.observedCalls.filter((call) => call.name === name);
|
|
137
|
+
return {
|
|
138
|
+
name,
|
|
139
|
+
alias: `tool_${index + 1}`,
|
|
140
|
+
status: 'observed_only',
|
|
141
|
+
inputs: observedShape(calls.map((call) => call.inputs)),
|
|
142
|
+
outputs: observedShape(calls
|
|
143
|
+
.filter((call) => call.outputs !== undefined)
|
|
144
|
+
.map((call) => call.outputs)),
|
|
145
|
+
samples: calls.length,
|
|
146
|
+
};
|
|
147
|
+
});
|
|
148
|
+
}
|
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
import { type MatrixOptions } from './matrix.js';
|
|
2
|
+
export declare function writeFaultMatrix(input: unknown, adapterPath: string, destination: string, options: MatrixOptions): Promise<{
|
|
3
|
+
schemaVersion: "workflow-matrix/1";
|
|
4
|
+
suites: {
|
|
5
|
+
id: string;
|
|
6
|
+
tool: string;
|
|
7
|
+
kind: "read" | "write";
|
|
8
|
+
workflow: {
|
|
9
|
+
schemaVersion: "workflow/1";
|
|
10
|
+
id: string;
|
|
11
|
+
title: string;
|
|
12
|
+
task: string;
|
|
13
|
+
initialState: Record<string, import("zod").JSONType>;
|
|
14
|
+
checks: ({
|
|
15
|
+
id: string;
|
|
16
|
+
label: string;
|
|
17
|
+
type: "forbidden_tool";
|
|
18
|
+
tool: string;
|
|
19
|
+
} | {
|
|
20
|
+
id: string;
|
|
21
|
+
label: string;
|
|
22
|
+
type: "equals";
|
|
23
|
+
source: "state" | "completion";
|
|
24
|
+
path: string;
|
|
25
|
+
expected: import("zod").JSONType;
|
|
26
|
+
} | {
|
|
27
|
+
id: string;
|
|
28
|
+
label: string;
|
|
29
|
+
type: "count";
|
|
30
|
+
path: string;
|
|
31
|
+
expected: number;
|
|
32
|
+
} | {
|
|
33
|
+
id: string;
|
|
34
|
+
label: string;
|
|
35
|
+
type: "unchanged";
|
|
36
|
+
path: string;
|
|
37
|
+
})[];
|
|
38
|
+
scenarios: {
|
|
39
|
+
id: string;
|
|
40
|
+
title: string;
|
|
41
|
+
fault: {
|
|
42
|
+
type: "prompt_injection";
|
|
43
|
+
tool: string;
|
|
44
|
+
occurrence: number;
|
|
45
|
+
repeat: number;
|
|
46
|
+
replacement: import("zod").JSONType;
|
|
47
|
+
} | {
|
|
48
|
+
type: "timeout_before" | "timeout_after" | "unavailable" | "malformed" | "permission_revoked" | "duplicate_delivery" | "stale_read";
|
|
49
|
+
tool: string;
|
|
50
|
+
occurrence: number;
|
|
51
|
+
repeat: number;
|
|
52
|
+
} | {
|
|
53
|
+
type: "none";
|
|
54
|
+
};
|
|
55
|
+
checks?: ({
|
|
56
|
+
id: string;
|
|
57
|
+
label: string;
|
|
58
|
+
type: "forbidden_tool";
|
|
59
|
+
tool: string;
|
|
60
|
+
} | {
|
|
61
|
+
id: string;
|
|
62
|
+
label: string;
|
|
63
|
+
type: "equals";
|
|
64
|
+
source: "state" | "completion";
|
|
65
|
+
path: string;
|
|
66
|
+
expected: import("zod").JSONType;
|
|
67
|
+
} | {
|
|
68
|
+
id: string;
|
|
69
|
+
label: string;
|
|
70
|
+
type: "count";
|
|
71
|
+
path: string;
|
|
72
|
+
expected: number;
|
|
73
|
+
} | {
|
|
74
|
+
id: string;
|
|
75
|
+
label: string;
|
|
76
|
+
type: "unchanged";
|
|
77
|
+
path: string;
|
|
78
|
+
})[] | undefined;
|
|
79
|
+
}[];
|
|
80
|
+
limits: {
|
|
81
|
+
maxToolCalls: number;
|
|
82
|
+
maxRepeatedCalls: number;
|
|
83
|
+
timeoutMs: number;
|
|
84
|
+
};
|
|
85
|
+
clock?: {
|
|
86
|
+
mode: "virtual";
|
|
87
|
+
startMs: number;
|
|
88
|
+
maxTimeMs: number;
|
|
89
|
+
} | undefined;
|
|
90
|
+
};
|
|
91
|
+
}[];
|
|
92
|
+
}>;
|
|
93
|
+
export declare function testFaultMatrix(input: unknown, adapterPath: string, destination: string, trials: number, signal?: AbortSignal, commitSha?: string): Promise<{
|
|
94
|
+
reason?: string | undefined;
|
|
95
|
+
schemaVersion: string;
|
|
96
|
+
status: string;
|
|
97
|
+
exitCode: number;
|
|
98
|
+
planned: number;
|
|
99
|
+
completed: number;
|
|
100
|
+
suites: {
|
|
101
|
+
suite: string;
|
|
102
|
+
tool: string;
|
|
103
|
+
report: string;
|
|
104
|
+
status: string;
|
|
105
|
+
exitCode: number;
|
|
106
|
+
}[];
|
|
107
|
+
}>;
|
|
@@ -0,0 +1,154 @@
|
|
|
1
|
+
// SPDX-License-Identifier: Apache-2.0
|
|
2
|
+
// Copyright 2026 Syed Arman
|
|
3
|
+
import { mkdir, open, rmdir, unlink } from 'node:fs/promises';
|
|
4
|
+
import { dirname, join } from 'node:path';
|
|
5
|
+
import { pathToFileURL } from 'node:url';
|
|
6
|
+
import { runWorkflow } from './index.js';
|
|
7
|
+
import { generateFaultMatrix, parseMatrixDefinition, parseMatrixOptions, parseWorkflowMatrix, } from './matrix.js';
|
|
8
|
+
import { sanitizeWorkflowBatch } from './privacy.js';
|
|
9
|
+
import { workflowExitCode, commitShaSchema } from './schema.js';
|
|
10
|
+
async function loadAdapter(path) {
|
|
11
|
+
try {
|
|
12
|
+
return await import(pathToFileURL(path).href);
|
|
13
|
+
}
|
|
14
|
+
catch {
|
|
15
|
+
throw new Error('Could not load the trusted local adapter. Check its syntax and dependencies.');
|
|
16
|
+
}
|
|
17
|
+
}
|
|
18
|
+
export async function writeFaultMatrix(input, adapterPath, destination, options) {
|
|
19
|
+
parseMatrixDefinition(input);
|
|
20
|
+
parseMatrixOptions(options);
|
|
21
|
+
await mkdir(dirname(destination), { recursive: true });
|
|
22
|
+
const file = await open(destination, 'wx', 0o600);
|
|
23
|
+
let saved = false;
|
|
24
|
+
try {
|
|
25
|
+
// Reserve the output before importing code. Generation never calls factories,
|
|
26
|
+
// agents or tool handlers, but module initialization is still trusted code.
|
|
27
|
+
const adapter = await loadAdapter(adapterPath);
|
|
28
|
+
if (!adapter.tools ||
|
|
29
|
+
typeof adapter.tools !== 'object' ||
|
|
30
|
+
Array.isArray(adapter.tools))
|
|
31
|
+
throw new Error('The adapter must export a tools object with read/write declarations.');
|
|
32
|
+
const matrix = generateFaultMatrix(input, Object.entries(adapter.tools).map(([name, tool]) => ({
|
|
33
|
+
name,
|
|
34
|
+
kind: tool?.kind,
|
|
35
|
+
})), options);
|
|
36
|
+
const serialized = JSON.stringify(matrix, null, 2) + '\n';
|
|
37
|
+
if (Buffer.byteLength(serialized) > 5_000_000)
|
|
38
|
+
throw new Error('Generated matrix exceeds 5 MB. Select fewer tools or reduce fixture data.');
|
|
39
|
+
await file.writeFile(serialized);
|
|
40
|
+
saved = true;
|
|
41
|
+
return matrix;
|
|
42
|
+
}
|
|
43
|
+
finally {
|
|
44
|
+
await file.close();
|
|
45
|
+
if (!saved)
|
|
46
|
+
await unlink(destination);
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
export async function testFaultMatrix(input, adapterPath, destination, trials, signal, commitSha) {
|
|
50
|
+
const matrix = parseWorkflowMatrix(input);
|
|
51
|
+
if (commitSha !== undefined)
|
|
52
|
+
commitShaSchema.parse(commitSha);
|
|
53
|
+
if (!Number.isInteger(trials) || trials < 1 || trials > 20)
|
|
54
|
+
throw new Error('--trials must be from 1 to 20.');
|
|
55
|
+
await mkdir(dirname(destination), { recursive: true });
|
|
56
|
+
await mkdir(destination, { mode: 0o700 });
|
|
57
|
+
const files = new Map();
|
|
58
|
+
const saved = new Set();
|
|
59
|
+
const rows = matrix.suites.map((suite) => ({
|
|
60
|
+
suite: suite.id,
|
|
61
|
+
tool: suite.tool,
|
|
62
|
+
report: `report-${suite.id}.json`,
|
|
63
|
+
status: 'not_run',
|
|
64
|
+
exitCode: 3,
|
|
65
|
+
}));
|
|
66
|
+
let reason;
|
|
67
|
+
try {
|
|
68
|
+
for (const filename of ['index.json', ...rows.map((row) => row.report)])
|
|
69
|
+
files.set(filename, await open(join(destination, filename), 'wx', 0o600));
|
|
70
|
+
const adapter = await loadAdapter(adapterPath);
|
|
71
|
+
// Validate all declarations before the first model call. A changed tool kind
|
|
72
|
+
// requires regenerating/reviewing the matrix rather than silently changing it.
|
|
73
|
+
if (typeof adapter.createAgent !== 'function' ||
|
|
74
|
+
matrix.suites.some((suite) => !Object.hasOwn(adapter.tools ?? {}, suite.tool) ||
|
|
75
|
+
adapter.tools[suite.tool]?.kind !== suite.kind ||
|
|
76
|
+
typeof adapter.tools[suite.tool]?.execute !== 'function'))
|
|
77
|
+
throw new Error('Matrix tools do not match the adapter. Review its declarations and regenerate the matrix.');
|
|
78
|
+
for (const [index, suite] of matrix.suites.entries()) {
|
|
79
|
+
if (signal?.aborted) {
|
|
80
|
+
reason = 'Cancelled before the next suite.';
|
|
81
|
+
break;
|
|
82
|
+
}
|
|
83
|
+
try {
|
|
84
|
+
const batch = sanitizeWorkflowBatch(await runWorkflow(suite.workflow, adapter, {
|
|
85
|
+
trials,
|
|
86
|
+
signal,
|
|
87
|
+
commitSha,
|
|
88
|
+
}));
|
|
89
|
+
const row = rows[index];
|
|
90
|
+
await files
|
|
91
|
+
.get(row.report)
|
|
92
|
+
.writeFile(JSON.stringify(batch, null, 2) + '\n');
|
|
93
|
+
saved.add(row.report);
|
|
94
|
+
row.exitCode = workflowExitCode(batch);
|
|
95
|
+
row.status =
|
|
96
|
+
row.exitCode === 0
|
|
97
|
+
? 'passed'
|
|
98
|
+
: row.exitCode === 1
|
|
99
|
+
? 'failed'
|
|
100
|
+
: 'incomplete';
|
|
101
|
+
if (batch.baseline.status !== 'passed' ||
|
|
102
|
+
batch.runs.some((run) => run.status === 'cancelled' ||
|
|
103
|
+
run.status === 'timed_out' ||
|
|
104
|
+
(run.status === 'inconclusive' && run.issue))) {
|
|
105
|
+
reason =
|
|
106
|
+
'Stopped after a failed control or interrupted suite; remaining suites were not run.';
|
|
107
|
+
break;
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
catch {
|
|
111
|
+
rows[index].status = 'error';
|
|
112
|
+
// Keep completed suites and describe missing evidence without printing
|
|
113
|
+
// adapter errors, which may contain secrets or private model responses.
|
|
114
|
+
reason =
|
|
115
|
+
'A suite could not produce valid evidence; remaining suites were not run.';
|
|
116
|
+
break;
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
const exitCode = rows.some((row) => row.exitCode === 3)
|
|
120
|
+
? 3
|
|
121
|
+
: rows.some((row) => row.exitCode === 1)
|
|
122
|
+
? 1
|
|
123
|
+
: 0;
|
|
124
|
+
const result = {
|
|
125
|
+
schemaVersion: 'workflow-matrix-result/1',
|
|
126
|
+
status: exitCode === 0 ? 'passed' : exitCode === 1 ? 'failed' : 'incomplete',
|
|
127
|
+
exitCode,
|
|
128
|
+
planned: rows.length,
|
|
129
|
+
completed: saved.size,
|
|
130
|
+
suites: rows,
|
|
131
|
+
...(reason
|
|
132
|
+
? { reason }
|
|
133
|
+
: exitCode === 3
|
|
134
|
+
? {
|
|
135
|
+
reason: 'Some fault scenarios were not reached. Incomplete coverage is not a passing result.',
|
|
136
|
+
}
|
|
137
|
+
: {}),
|
|
138
|
+
};
|
|
139
|
+
await files
|
|
140
|
+
.get('index.json')
|
|
141
|
+
.writeFile(JSON.stringify(result, null, 2) + '\n');
|
|
142
|
+
saved.add('index.json');
|
|
143
|
+
return result;
|
|
144
|
+
}
|
|
145
|
+
finally {
|
|
146
|
+
for (const [filename, file] of files) {
|
|
147
|
+
await file.close();
|
|
148
|
+
if (!saved.has(filename))
|
|
149
|
+
await unlink(join(destination, filename));
|
|
150
|
+
}
|
|
151
|
+
if (!saved.size)
|
|
152
|
+
await rmdir(destination);
|
|
153
|
+
}
|
|
154
|
+
}
|
package/dist/matrix.d.ts
ADDED
|
@@ -0,0 +1,138 @@
|
|
|
1
|
+
import { z } from 'zod';
|
|
2
|
+
import { type Workflow } from './schema.js';
|
|
3
|
+
declare const inventorySchema: z.ZodArray<z.ZodObject<{
|
|
4
|
+
name: z.ZodString;
|
|
5
|
+
kind: z.ZodEnum<{
|
|
6
|
+
read: "read";
|
|
7
|
+
write: "write";
|
|
8
|
+
}>;
|
|
9
|
+
}, z.core.$strict>>;
|
|
10
|
+
export type MatrixTool = z.infer<typeof inventorySchema>[number];
|
|
11
|
+
declare const optionsSchema: z.ZodObject<{
|
|
12
|
+
tools: z.ZodOptional<z.ZodArray<z.ZodString>>;
|
|
13
|
+
faults: z.ZodOptional<z.ZodArray<z.ZodEnum<{
|
|
14
|
+
timeout_before: "timeout_before";
|
|
15
|
+
timeout_after: "timeout_after";
|
|
16
|
+
unavailable: "unavailable";
|
|
17
|
+
malformed: "malformed";
|
|
18
|
+
permission_revoked: "permission_revoked";
|
|
19
|
+
duplicate_delivery: "duplicate_delivery";
|
|
20
|
+
stale_read: "stale_read";
|
|
21
|
+
}>>>;
|
|
22
|
+
}, z.core.$strict>;
|
|
23
|
+
export type MatrixOptions = z.infer<typeof optionsSchema>;
|
|
24
|
+
export declare function parseMatrixOptions(input: unknown): MatrixOptions;
|
|
25
|
+
export declare function parseMatrixDefinition(input: unknown): Workflow;
|
|
26
|
+
export declare const workflowMatrixSchema: z.ZodObject<{
|
|
27
|
+
schemaVersion: z.ZodLiteral<"workflow-matrix/1">;
|
|
28
|
+
suites: z.ZodArray<z.ZodObject<{
|
|
29
|
+
id: z.ZodString;
|
|
30
|
+
tool: z.ZodString;
|
|
31
|
+
kind: z.ZodEnum<{
|
|
32
|
+
read: "read";
|
|
33
|
+
write: "write";
|
|
34
|
+
}>;
|
|
35
|
+
workflow: z.ZodObject<{
|
|
36
|
+
schemaVersion: z.ZodLiteral<"workflow/1">;
|
|
37
|
+
id: z.ZodString;
|
|
38
|
+
title: z.ZodString;
|
|
39
|
+
task: z.ZodString;
|
|
40
|
+
initialState: z.ZodRecord<z.ZodString, z.ZodJSONSchema>;
|
|
41
|
+
clock: z.ZodOptional<z.ZodObject<{
|
|
42
|
+
mode: z.ZodLiteral<"virtual">;
|
|
43
|
+
startMs: z.ZodDefault<z.ZodNumber>;
|
|
44
|
+
maxTimeMs: z.ZodDefault<z.ZodNumber>;
|
|
45
|
+
}, z.core.$strict>>;
|
|
46
|
+
checks: z.ZodArray<z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
47
|
+
id: z.ZodString;
|
|
48
|
+
label: z.ZodString;
|
|
49
|
+
type: z.ZodLiteral<"forbidden_tool">;
|
|
50
|
+
tool: z.ZodString;
|
|
51
|
+
}, z.core.$strict>, z.ZodObject<{
|
|
52
|
+
id: z.ZodString;
|
|
53
|
+
label: z.ZodString;
|
|
54
|
+
type: z.ZodLiteral<"equals">;
|
|
55
|
+
source: z.ZodDefault<z.ZodEnum<{
|
|
56
|
+
state: "state";
|
|
57
|
+
completion: "completion";
|
|
58
|
+
}>>;
|
|
59
|
+
path: z.ZodString;
|
|
60
|
+
expected: z.ZodJSONSchema;
|
|
61
|
+
}, z.core.$strict>, z.ZodObject<{
|
|
62
|
+
id: z.ZodString;
|
|
63
|
+
label: z.ZodString;
|
|
64
|
+
type: z.ZodLiteral<"count">;
|
|
65
|
+
path: z.ZodString;
|
|
66
|
+
expected: z.ZodNumber;
|
|
67
|
+
}, z.core.$strict>, z.ZodObject<{
|
|
68
|
+
id: z.ZodString;
|
|
69
|
+
label: z.ZodString;
|
|
70
|
+
type: z.ZodLiteral<"unchanged">;
|
|
71
|
+
path: z.ZodString;
|
|
72
|
+
}, z.core.$strict>], "type">>;
|
|
73
|
+
scenarios: z.ZodArray<z.ZodObject<{
|
|
74
|
+
id: z.ZodString;
|
|
75
|
+
title: z.ZodString;
|
|
76
|
+
fault: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
77
|
+
type: z.ZodLiteral<"none">;
|
|
78
|
+
}, z.core.$strict>, z.ZodObject<{
|
|
79
|
+
type: z.ZodLiteral<"prompt_injection">;
|
|
80
|
+
tool: z.ZodString;
|
|
81
|
+
occurrence: z.ZodDefault<z.ZodNumber>;
|
|
82
|
+
repeat: z.ZodDefault<z.ZodNumber>;
|
|
83
|
+
replacement: z.ZodJSONSchema;
|
|
84
|
+
}, z.core.$strict>, z.ZodObject<{
|
|
85
|
+
type: z.ZodEnum<{
|
|
86
|
+
timeout_before: "timeout_before";
|
|
87
|
+
timeout_after: "timeout_after";
|
|
88
|
+
unavailable: "unavailable";
|
|
89
|
+
malformed: "malformed";
|
|
90
|
+
permission_revoked: "permission_revoked";
|
|
91
|
+
duplicate_delivery: "duplicate_delivery";
|
|
92
|
+
stale_read: "stale_read";
|
|
93
|
+
}>;
|
|
94
|
+
tool: z.ZodString;
|
|
95
|
+
occurrence: z.ZodDefault<z.ZodNumber>;
|
|
96
|
+
repeat: z.ZodDefault<z.ZodNumber>;
|
|
97
|
+
}, z.core.$strict>], "type">;
|
|
98
|
+
checks: z.ZodOptional<z.ZodArray<z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
99
|
+
id: z.ZodString;
|
|
100
|
+
label: z.ZodString;
|
|
101
|
+
type: z.ZodLiteral<"forbidden_tool">;
|
|
102
|
+
tool: z.ZodString;
|
|
103
|
+
}, z.core.$strict>, z.ZodObject<{
|
|
104
|
+
id: z.ZodString;
|
|
105
|
+
label: z.ZodString;
|
|
106
|
+
type: z.ZodLiteral<"equals">;
|
|
107
|
+
source: z.ZodDefault<z.ZodEnum<{
|
|
108
|
+
state: "state";
|
|
109
|
+
completion: "completion";
|
|
110
|
+
}>>;
|
|
111
|
+
path: z.ZodString;
|
|
112
|
+
expected: z.ZodJSONSchema;
|
|
113
|
+
}, z.core.$strict>, z.ZodObject<{
|
|
114
|
+
id: z.ZodString;
|
|
115
|
+
label: z.ZodString;
|
|
116
|
+
type: z.ZodLiteral<"count">;
|
|
117
|
+
path: z.ZodString;
|
|
118
|
+
expected: z.ZodNumber;
|
|
119
|
+
}, z.core.$strict>, z.ZodObject<{
|
|
120
|
+
id: z.ZodString;
|
|
121
|
+
label: z.ZodString;
|
|
122
|
+
type: z.ZodLiteral<"unchanged">;
|
|
123
|
+
path: z.ZodString;
|
|
124
|
+
}, z.core.$strict>], "type">>>;
|
|
125
|
+
}, z.core.$strict>>;
|
|
126
|
+
limits: z.ZodDefault<z.ZodObject<{
|
|
127
|
+
maxToolCalls: z.ZodDefault<z.ZodNumber>;
|
|
128
|
+
maxRepeatedCalls: z.ZodDefault<z.ZodNumber>;
|
|
129
|
+
timeoutMs: z.ZodDefault<z.ZodNumber>;
|
|
130
|
+
}, z.core.$strict>>;
|
|
131
|
+
}, z.core.$strict>;
|
|
132
|
+
}, z.core.$strict>>;
|
|
133
|
+
}, z.core.$strict>;
|
|
134
|
+
export type WorkflowMatrix = z.infer<typeof workflowMatrixSchema>;
|
|
135
|
+
export declare function parseWorkflowMatrix(input: unknown): WorkflowMatrix;
|
|
136
|
+
/** Generate editable scenarios from declarations, without executing tools or an agent. */
|
|
137
|
+
export declare function generateFaultMatrix(input: unknown, inventory: unknown, options?: MatrixOptions): WorkflowMatrix;
|
|
138
|
+
export {};
|