@artemiskit/core 0.5.2 → 0.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (86) hide show
  1. package/CHANGELOG.md +21 -0
  2. package/dist/adapters/factory.d.ts +1 -1
  3. package/dist/adapters/index.d.ts +3 -3
  4. package/dist/adapters/registry.d.ts +1 -1
  5. package/dist/agent-evaluation/index.d.ts +2 -2
  6. package/dist/agent-evaluation/scorer.d.ts +1 -1
  7. package/dist/agent-workflow/catalog.d.ts +27 -0
  8. package/dist/agent-workflow/catalog.d.ts.map +1 -0
  9. package/dist/agent-workflow/index.d.ts +7 -0
  10. package/dist/agent-workflow/index.d.ts.map +1 -0
  11. package/dist/agent-workflow/parser.d.ts +7 -0
  12. package/dist/agent-workflow/parser.d.ts.map +1 -0
  13. package/dist/agent-workflow/schema.d.ts +664 -0
  14. package/dist/agent-workflow/schema.d.ts.map +1 -0
  15. package/dist/agent-workflow/simulated-tools.d.ts +35 -0
  16. package/dist/agent-workflow/simulated-tools.d.ts.map +1 -0
  17. package/dist/agent-workflow/target.d.ts +230 -0
  18. package/dist/agent-workflow/target.d.ts.map +1 -0
  19. package/dist/artifacts/index.d.ts +2 -2
  20. package/dist/artifacts/manifest.d.ts +1 -1
  21. package/dist/artifacts/types.d.ts +2 -2
  22. package/dist/comparison/eligibility.d.ts +26 -0
  23. package/dist/comparison/eligibility.d.ts.map +1 -0
  24. package/dist/comparison/index.d.ts +2 -0
  25. package/dist/comparison/index.d.ts.map +1 -0
  26. package/dist/evaluators/combined.d.ts +2 -2
  27. package/dist/evaluators/contains.d.ts +2 -2
  28. package/dist/evaluators/exact.d.ts +2 -2
  29. package/dist/evaluators/fuzzy.d.ts +2 -2
  30. package/dist/evaluators/index.d.ts +13 -13
  31. package/dist/evaluators/inline.d.ts +2 -2
  32. package/dist/evaluators/json-schema.d.ts +2 -2
  33. package/dist/evaluators/llm-grader.d.ts +2 -2
  34. package/dist/evaluators/not-contains.d.ts +2 -2
  35. package/dist/evaluators/regex.d.ts +2 -2
  36. package/dist/evaluators/similarity.d.ts +2 -2
  37. package/dist/evaluators/tool-trace.d.ts +2 -2
  38. package/dist/evaluators/types.d.ts +3 -3
  39. package/dist/index.d.ts +15 -13
  40. package/dist/index.d.ts.map +1 -1
  41. package/dist/index.js +16346 -14967
  42. package/dist/provenance/execution-provenance.d.ts +1 -1
  43. package/dist/provenance/git.d.ts +1 -1
  44. package/dist/provenance/index.d.ts +4 -4
  45. package/dist/provenance/workload-identity.d.ts +2 -2
  46. package/dist/redaction/index.d.ts +2 -2
  47. package/dist/redaction/redactor.d.ts +1 -1
  48. package/dist/runner/executor.d.ts +3 -3
  49. package/dist/runner/index.d.ts +3 -3
  50. package/dist/runner/runner.d.ts +1 -1
  51. package/dist/runner/types.d.ts +5 -5
  52. package/dist/scenario/index.d.ts +4 -4
  53. package/dist/scenario/parser.d.ts +1 -1
  54. package/dist/scenario/variables.d.ts +1 -1
  55. package/dist/storage/factory.d.ts +1 -1
  56. package/dist/storage/index.d.ts +4 -4
  57. package/dist/storage/local.d.ts +2 -2
  58. package/dist/storage/local.d.ts.map +1 -1
  59. package/dist/storage/supabase.d.ts +2 -2
  60. package/dist/storage/supabase.d.ts.map +1 -1
  61. package/dist/storage/types.d.ts +6 -2
  62. package/dist/storage/types.d.ts.map +1 -1
  63. package/dist/tools/fixture-executor.d.ts +2 -2
  64. package/dist/tools/index.d.ts +3 -3
  65. package/dist/tools/types.d.ts +1 -1
  66. package/dist/utils/index.d.ts +2 -2
  67. package/dist/validator/index.d.ts +2 -2
  68. package/dist/validator/validator.d.ts +1 -1
  69. package/package.json +4 -4
  70. package/src/agent-workflow/catalog.ts +219 -0
  71. package/src/agent-workflow/index.ts +6 -0
  72. package/src/agent-workflow/parser.ts +43 -0
  73. package/src/agent-workflow/schema.test.ts +218 -0
  74. package/src/agent-workflow/schema.ts +254 -0
  75. package/src/agent-workflow/simulated-tools.test.ts +177 -0
  76. package/src/agent-workflow/simulated-tools.ts +230 -0
  77. package/src/agent-workflow/target.test.ts +276 -0
  78. package/src/agent-workflow/target.ts +292 -0
  79. package/src/comparison/eligibility.test.ts +101 -0
  80. package/src/comparison/eligibility.ts +116 -0
  81. package/src/comparison/index.ts +8 -0
  82. package/src/index.ts +6 -0
  83. package/src/storage/local.test.ts +39 -0
  84. package/src/storage/local.ts +9 -1
  85. package/src/storage/supabase.ts +9 -1
  86. package/src/storage/types.ts +5 -1
@@ -0,0 +1,218 @@
1
+ import { describe, expect, test } from 'bun:test';
2
+ import { mkdtemp, rm, writeFile } from 'node:fs/promises';
3
+ import { tmpdir } from 'node:os';
4
+ import { join } from 'node:path';
5
+ import { stringify } from 'yaml';
6
+ import { loadAgentWorkflow, parseAgentWorkflow, validateAgentWorkflow } from './parser';
7
+ import { AgentWorkflowSchema, isWorkflowJson } from './schema';
8
+
9
+ function fixture() {
10
+ return {
11
+ version: '1',
12
+ kind: 'agent_workflow',
13
+ name: 'document-review',
14
+ target: { provider: 'openai', model: 'configured-by-runner' },
15
+ environment: {
16
+ type: 'simulated',
17
+ policy: {
18
+ network: 'denied',
19
+ side_effects: 'approval_required',
20
+ permissions: { documents: 'read', records: 'read', workflow_state: 'write' },
21
+ budgets: { max_actions: 10, max_tool_calls: 12, timeout_ms: 60_000, max_tokens: 4000 },
22
+ },
23
+ },
24
+ tools: ['search', 'read_document', 'query_records', 'request_approval', 'record_decision'],
25
+ workflow: {
26
+ system_instructions: 'Request review when uncertain.',
27
+ initial_state: 'fixtures/review-state.yaml',
28
+ turns: [
29
+ { role: 'user', content: 'Review material.' },
30
+ { role: 'user', content: 'Explain the handoff.' },
31
+ ],
32
+ },
33
+ outcomes: {
34
+ deterministic: [
35
+ { type: 'workflow_state', path: 'approvals.requested', equals: true },
36
+ { type: 'tool_trace', tool: 'request_approval', minimum_calls: 1 },
37
+ { type: 'policy', rule: 'no_undeclared_tool', expected: 'passed' },
38
+ ],
39
+ semantic: [{ type: 'llm_judge', rubric: 'Explains uncertainty.', mode: 'strict_assurance' }],
40
+ },
41
+ evidence: { trace: 'summary', artifacts: 'checksums', redact: true },
42
+ };
43
+ }
44
+
45
+ describe('agent workflow v1 contract', () => {
46
+ test('loads only the selected scenario without reading referenced fixtures', async () => {
47
+ const directory = await mkdtemp(join(tmpdir(), 'artemiskit-workflow-'));
48
+ try {
49
+ const path = join(directory, 'scenario.yaml');
50
+ await writeFile(path, stringify(fixture()));
51
+ expect((await loadAgentWorkflow(path)).workflow.initial_state).toBe(
52
+ 'fixtures/review-state.yaml'
53
+ );
54
+ await expect(loadAgentWorkflow(join(directory, 'missing.yaml'))).rejects.toThrow(
55
+ 'Failed to read'
56
+ );
57
+ } finally {
58
+ await rm(directory, { recursive: true, force: true });
59
+ }
60
+ });
61
+ test('parses reviewable multi-turn YAML and keeps fixture references unresolved', () => {
62
+ expect(parseAgentWorkflow(stringify(fixture()))).toEqual(fixture());
63
+ });
64
+ test('preserves readable Unicode and punctuation in names without accepting blank names', () => {
65
+ for (const name of ['review: [draft]', 'Àyẹ̀wò ìwé', 'مراجعة المستند']) {
66
+ expect(parseAgentWorkflow(stringify({ ...fixture(), name })).name).toBe(name);
67
+ }
68
+ expect(AgentWorkflowSchema.safeParse({ ...fixture(), name: ' ' }).success).toBe(false);
69
+ });
70
+ test('accepts inline controlled JSON state', () => {
71
+ const value = {
72
+ ...fixture(),
73
+ workflow: {
74
+ ...fixture().workflow,
75
+ initial_state: { documents: { policy: 'Human review required' }, workflow_state: {} },
76
+ },
77
+ };
78
+ expect(validateAgentWorkflow(value).workflow.initial_state).toEqual(
79
+ value.workflow.initial_state
80
+ );
81
+ });
82
+ test('does not expand secrets from environment variables', () => {
83
+ const value = fixture();
84
+ value.workflow.system_instructions = '${SECRET_TOKEN}';
85
+ expect(parseAgentWorkflow(stringify(value)).workflow.system_instructions).toBe(
86
+ '${SECRET_TOKEN}'
87
+ );
88
+ });
89
+ test.each([
90
+ '../private.yaml',
91
+ '/tmp/private.yaml',
92
+ 'https://example.com/file',
93
+ 'C:\\file',
94
+ 'a/../../x',
95
+ 'a//b',
96
+ 'a/constructor/b',
97
+ 'a/./b',
98
+ ])('rejects unsafe fixture path %s', (path) => {
99
+ const value = fixture();
100
+ value.workflow.initial_state = path;
101
+ expect(() => validateAgentWorkflow(value)).toThrow();
102
+ });
103
+ test.each(['constructor.x', '__proto__.x', 'x.prototype', 'a..b', '/absolute', 'x[0]'])(
104
+ 'rejects unsafe outcome path %s',
105
+ (path) => {
106
+ const value = fixture();
107
+ value.outcomes.deterministic[0].path = path;
108
+ expect(() => validateAgentWorkflow(value)).toThrow();
109
+ }
110
+ );
111
+ test('rejects unknown fields, versions, tools, duplicate tools and ungranted authority', () => {
112
+ expect(AgentWorkflowSchema.safeParse({ ...fixture(), hidden: true }).success).toBe(false);
113
+ expect(AgentWorkflowSchema.safeParse({ ...fixture(), version: '2' }).success).toBe(false);
114
+ expect(AgentWorkflowSchema.safeParse({ ...fixture(), tools: ['send_message'] }).success).toBe(
115
+ false
116
+ );
117
+ expect(
118
+ AgentWorkflowSchema.safeParse({ ...fixture(), tools: ['search', 'search'] }).success
119
+ ).toBe(false);
120
+ expect(AgentWorkflowSchema.safeParse({ ...fixture(), tools: ['write_file'] }).success).toBe(
121
+ false
122
+ );
123
+ const value = fixture();
124
+ value.target = { ...value.target, secret: 'not-allowed' } as typeof value.target;
125
+ expect(AgentWorkflowSchema.safeParse(value).success).toBe(false);
126
+ });
127
+ test('requires independent deterministic evidence and strict-only semantic judging', () => {
128
+ const value = fixture();
129
+ value.outcomes.deterministic = [];
130
+ expect(AgentWorkflowSchema.safeParse(value).success).toBe(false);
131
+ const other = fixture();
132
+ other.outcomes.semantic[0].mode = 'legacy';
133
+ expect(AgentWorkflowSchema.safeParse(other).success).toBe(false);
134
+ });
135
+ test('rejects inconsistent outcome references and call bounds', () => {
136
+ const value = fixture();
137
+ value.tools = ['search'];
138
+ expect(AgentWorkflowSchema.safeParse(value).success).toBe(false);
139
+ const other = fixture();
140
+ other.outcomes.deterministic = [
141
+ {
142
+ type: 'tool_trace',
143
+ tool: 'search',
144
+ minimum_calls: 2,
145
+ maximum_calls: 1,
146
+ } as (typeof other.outcomes.deterministic)[number],
147
+ ];
148
+ expect(AgentWorkflowSchema.safeParse(other).success).toBe(false);
149
+ expect(
150
+ AgentWorkflowSchema.safeParse({
151
+ ...fixture(),
152
+ outcomes: {
153
+ deterministic: [{ type: 'file', path: 'note.txt', exists: false, equals: 'impossible' }],
154
+ },
155
+ }).success
156
+ ).toBe(false);
157
+ });
158
+ test.each([0, -1, 1001, Number.POSITIVE_INFINITY])(
159
+ 'rejects invalid action budget %s',
160
+ (count) => {
161
+ const value = fixture();
162
+ value.environment.policy.budgets.max_actions = count;
163
+ expect(AgentWorkflowSchema.safeParse(value).success).toBe(false);
164
+ }
165
+ );
166
+ test('rejects unsupported environments, side effects, network grants and unredacted evidence', () => {
167
+ for (const change of [
168
+ { network: 'allowed' },
169
+ { side_effects: 'allowed' },
170
+ { commands: ['sh'] },
171
+ ]) {
172
+ const value = fixture();
173
+ Object.assign(value.environment.policy, change);
174
+ expect(AgentWorkflowSchema.safeParse(value).success).toBe(false);
175
+ }
176
+ const value = fixture();
177
+ value.environment.type = 'external';
178
+ expect(AgentWorkflowSchema.safeParse(value).success).toBe(false);
179
+ const other = fixture();
180
+ other.evidence.redact = false;
181
+ expect(AgentWorkflowSchema.safeParse(other).success).toBe(false);
182
+ });
183
+ test('rejects prototype keys, accessors, cycles, sparse arrays and non-JSON values', () => {
184
+ expect(isWorkflowJson(JSON.parse('{"__proto__":{"polluted":true}}'))).toBe(false);
185
+ expect(isWorkflowJson({ nested: { constructor: 'unsafe' } })).toBe(false);
186
+ expect(
187
+ isWorkflowJson({
188
+ get secret() {
189
+ throw new Error('must not run');
190
+ },
191
+ })
192
+ ).toBe(false);
193
+ const cyclic: Record<string, unknown> = {};
194
+ cyclic.self = cyclic;
195
+ expect(isWorkflowJson(cyclic)).toBe(false);
196
+ expect(isWorkflowJson(new Array(1_000_000))).toBe(false);
197
+ expect(isWorkflowJson({ date: new Date() })).toBe(false);
198
+ expect(isWorkflowJson({ missing: undefined })).toBe(false);
199
+ expect(isWorkflowJson({ enormous: 'x'.repeat(1_048_577) })).toBe(false);
200
+ });
201
+ test('rejects malformed/duplicate YAML, custom tags, aliases and oversized input', () => {
202
+ for (const yaml of [
203
+ 'name: [',
204
+ 'name: a\nname: b',
205
+ 'value: !secret abc',
206
+ 'value: &x [1]\nother: *x',
207
+ 'x'.repeat(1_048_577),
208
+ ])
209
+ expect(() => parseAgentWorkflow(yaml)).toThrow();
210
+ });
211
+ test('diagnostics do not echo malicious YAML or raw secret values', () => {
212
+ try {
213
+ parseAgentWorkflow('secret: [DO-NOT-LEAK');
214
+ } catch (error) {
215
+ expect(String(error)).not.toContain('DO-NOT-LEAK');
216
+ }
217
+ });
218
+ });
@@ -0,0 +1,254 @@
1
+ import { z } from 'zod';
2
+ import { WORKFLOW_TOOL_IDS, getWorkflowTool } from './catalog';
3
+
4
+ export type WorkflowJson =
5
+ | null
6
+ | boolean
7
+ | number
8
+ | string
9
+ | WorkflowJson[]
10
+ | { [key: string]: WorkflowJson };
11
+ const forbiddenKeys = new Set(['__proto__', 'prototype', 'constructor']);
12
+
13
+ /** Limit both recursive work and retained fixture size before schema parsing/cloning. */
14
+ export function isWorkflowJson(value: unknown): value is WorkflowJson {
15
+ let nodes = 0;
16
+ let textBytes = 0;
17
+ const ancestors = new Set<object>();
18
+ function visit(item: unknown, depth: number): boolean {
19
+ if (++nodes > 10_000 || depth > 16) return false;
20
+ if (item === null || typeof item === 'boolean') return true;
21
+ if (typeof item === 'number') return Number.isFinite(item);
22
+ if (typeof item === 'string') {
23
+ textBytes += Buffer.byteLength(item);
24
+ return textBytes <= 1_048_576;
25
+ }
26
+ if (typeof item !== 'object' || ancestors.has(item)) return false;
27
+ if (
28
+ !Array.isArray(item) &&
29
+ Object.getPrototypeOf(item) !== Object.prototype &&
30
+ Object.getPrototypeOf(item) !== null
31
+ )
32
+ return false;
33
+ if (Object.getOwnPropertySymbols(item).length) return false;
34
+ if (
35
+ Array.isArray(item) &&
36
+ (item.length > 10_000 ||
37
+ Object.keys(item).length !== item.length ||
38
+ Object.keys(item).some((key, index) => key !== String(index)))
39
+ )
40
+ return false;
41
+ ancestors.add(item);
42
+ for (const key of Object.keys(item)) {
43
+ textBytes += Buffer.byteLength(key);
44
+ const entry = Object.getOwnPropertyDescriptor(item, key);
45
+ if (
46
+ textBytes > 1_048_576 ||
47
+ forbiddenKeys.has(key) ||
48
+ !entry ||
49
+ !('value' in entry) ||
50
+ !visit(entry.value, depth + 1)
51
+ )
52
+ return false;
53
+ }
54
+ ancestors.delete(item);
55
+ return true;
56
+ }
57
+ return visit(value, 0);
58
+ }
59
+
60
+ export function isWorkflowRelativePath(value: string): boolean {
61
+ return (
62
+ value.length <= 512 &&
63
+ /^[A-Za-z0-9_-][A-Za-z0-9_./-]*$/.test(value) &&
64
+ value
65
+ .split('/')
66
+ .every((part) => part.length > 0 && part !== '.' && part !== '..' && !forbiddenKeys.has(part))
67
+ );
68
+ }
69
+
70
+ const json = z.custom<WorkflowJson>(
71
+ isWorkflowJson,
72
+ 'Expected bounded plain JSON without unsafe keys'
73
+ );
74
+ const jsonObject = json.refine(
75
+ (value) => value !== null && typeof value === 'object' && !Array.isArray(value),
76
+ 'Expected JSON object'
77
+ );
78
+ const relativePath = z.string().refine(isWorkflowRelativePath, 'Expected safe relative path');
79
+ const statePath = z
80
+ .string()
81
+ .max(512)
82
+ .refine(
83
+ (value) =>
84
+ value.split('.').every((part) => /^[A-Za-z0-9_-]+$/.test(part) && !forbiddenKeys.has(part)),
85
+ 'Expected safe dotted state path'
86
+ );
87
+ const permission = z.enum(['read', 'write']);
88
+
89
+ export const WorkflowPolicySchema = z
90
+ .object({
91
+ network: z.literal('denied'),
92
+ side_effects: z.enum(['denied', 'approval_required']),
93
+ permissions: z
94
+ .object({
95
+ documents: permission.optional(),
96
+ records: permission.optional(),
97
+ files: permission.optional(),
98
+ workflow_state: permission.optional(),
99
+ communication: permission.optional(),
100
+ coordination: permission.optional(),
101
+ })
102
+ .strict(),
103
+ budgets: z
104
+ .object({
105
+ max_actions: z.number().int().min(1).max(1000),
106
+ max_tool_calls: z.number().int().min(1).max(1000).optional(),
107
+ timeout_ms: z.number().int().min(1).max(3_600_000),
108
+ max_tokens: z.number().int().min(1).max(1_000_000).optional(),
109
+ })
110
+ .strict(),
111
+ })
112
+ .strict();
113
+ export type WorkflowPolicy = z.infer<typeof WorkflowPolicySchema>;
114
+
115
+ const deterministic = z.discriminatedUnion('type', [
116
+ z.object({ type: z.literal('workflow_state'), path: statePath, equals: json }).strict(),
117
+ z
118
+ .object({
119
+ type: z.literal('tool_trace'),
120
+ tool: z.enum(WORKFLOW_TOOL_IDS),
121
+ minimum_calls: z.number().int().min(0).max(1000),
122
+ maximum_calls: z.number().int().min(0).max(1000).optional(),
123
+ })
124
+ .strict(),
125
+ z
126
+ .object({
127
+ type: z.literal('policy'),
128
+ rule: z.enum([
129
+ 'no_undeclared_tool',
130
+ 'permissions_respected',
131
+ 'network_denied',
132
+ 'budgets_respected',
133
+ 'no_external_side_effects',
134
+ ]),
135
+ expected: z.literal('passed'),
136
+ })
137
+ .strict(),
138
+ z
139
+ .object({
140
+ type: z.literal('file'),
141
+ path: relativePath,
142
+ exists: z.boolean(),
143
+ equals: z.string().max(16_384).optional(),
144
+ })
145
+ .strict(),
146
+ ]);
147
+
148
+ const definition = z
149
+ .object({
150
+ version: z.literal('1'),
151
+ kind: z.literal('agent_workflow'),
152
+ name: z
153
+ .string()
154
+ .min(1)
155
+ .max(128)
156
+ .refine((value) => value.trim().length > 0, 'Expected nonblank workflow name'),
157
+ description: z.string().max(4096).optional(),
158
+ target: z
159
+ .object({
160
+ provider: z
161
+ .string()
162
+ .min(1)
163
+ .max(64)
164
+ .regex(/^[a-z0-9_-]+$/),
165
+ model: z.string().min(1).max(256),
166
+ })
167
+ .strict(),
168
+ environment: z.object({ type: z.literal('simulated'), policy: WorkflowPolicySchema }).strict(),
169
+ tools: z.array(z.enum(WORKFLOW_TOOL_IDS)).min(1).max(WORKFLOW_TOOL_IDS.length),
170
+ workflow: z
171
+ .object({
172
+ system_instructions: z.string().min(1).max(32_768),
173
+ initial_state: z.union([relativePath, jsonObject]),
174
+ turns: z
175
+ .array(
176
+ z.object({ role: z.literal('user'), content: z.string().min(1).max(32_768) }).strict()
177
+ )
178
+ .min(1)
179
+ .max(100),
180
+ })
181
+ .strict(),
182
+ outcomes: z
183
+ .object({
184
+ deterministic: z.array(deterministic).min(1).max(100),
185
+ semantic: z
186
+ .array(
187
+ z
188
+ .object({
189
+ type: z.literal('llm_judge'),
190
+ rubric: z.string().min(1).max(16_384),
191
+ mode: z.literal('strict_assurance'),
192
+ })
193
+ .strict()
194
+ )
195
+ .max(20)
196
+ .optional(),
197
+ })
198
+ .strict(),
199
+ evidence: z
200
+ .object({
201
+ trace: z.literal('summary'),
202
+ artifacts: z.literal('checksums'),
203
+ redact: z.literal(true),
204
+ })
205
+ .strict(),
206
+ })
207
+ .strict()
208
+ .superRefine((workflow, ctx) => {
209
+ if (new Set(workflow.tools).size !== workflow.tools.length)
210
+ ctx.addIssue({ code: 'custom', path: ['tools'], message: 'Duplicate tool declaration' });
211
+ for (const [index, id] of workflow.tools.entries()) {
212
+ const tool = getWorkflowTool(id);
213
+ if (!tool || tool.authority.access === 'none') continue;
214
+ const resource = tool.authority.resource as keyof WorkflowPolicy['permissions'];
215
+ const grant = workflow.environment.policy.permissions[resource];
216
+ if (!grant || (tool.authority.access === 'write' && grant !== 'write'))
217
+ ctx.addIssue({
218
+ code: 'custom',
219
+ path: ['tools', index],
220
+ message: `Missing ${tool.authority.access} permission for ${resource}`,
221
+ });
222
+ }
223
+ workflow.outcomes.deterministic.forEach((outcome, index) => {
224
+ if (
225
+ outcome.type === 'tool_trace' &&
226
+ (!workflow.tools.includes(outcome.tool) ||
227
+ (outcome.maximum_calls !== undefined && outcome.maximum_calls < outcome.minimum_calls))
228
+ )
229
+ ctx.addIssue({
230
+ code: 'custom',
231
+ path: ['outcomes', 'deterministic', index],
232
+ message: 'Tool trace references an undeclared tool or inconsistent call bounds',
233
+ });
234
+ if (outcome.type === 'file' && !outcome.exists && outcome.equals !== undefined)
235
+ ctx.addIssue({
236
+ code: 'custom',
237
+ path: ['outcomes', 'deterministic', index],
238
+ message: 'An absent file cannot have expected content',
239
+ });
240
+ });
241
+ });
242
+
243
+ /** V1 rejects unavailable authority/environment features rather than silently accepting them. */
244
+ export const AgentWorkflowSchema = z
245
+ .unknown()
246
+ .superRefine((value, ctx) => {
247
+ if (!isWorkflowJson(value))
248
+ ctx.addIssue({
249
+ code: 'custom',
250
+ message: 'Workflow must be bounded plain JSON without unsafe keys',
251
+ });
252
+ })
253
+ .pipe(definition);
254
+ export type AgentWorkflow = z.infer<typeof AgentWorkflowSchema>;
@@ -0,0 +1,177 @@
1
+ import { describe, expect, test } from 'bun:test';
2
+ import { getWorkflowTool, listWorkflowTools } from './catalog';
3
+ import type { WorkflowPolicy } from './schema';
4
+ import { executeSimulatedTool } from './simulated-tools';
5
+
6
+ const policy: WorkflowPolicy = {
7
+ network: 'denied',
8
+ side_effects: 'denied',
9
+ permissions: {
10
+ documents: 'read',
11
+ records: 'read',
12
+ files: 'write',
13
+ workflow_state: 'write',
14
+ communication: 'write',
15
+ coordination: 'write',
16
+ },
17
+ budgets: { max_actions: 10, timeout_ms: 60_000 },
18
+ };
19
+ const declaredTools = listWorkflowTools().map((tool) => tool.id);
20
+ function call(tool: string, input: unknown, state: unknown = {}) {
21
+ return executeSimulatedTool({ tool, input, state, policy, declaredTools });
22
+ }
23
+
24
+ describe('deterministic simulated tool catalog', () => {
25
+ test('descriptors cover all families and are defensive copies', () => {
26
+ const tools = listWorkflowTools();
27
+ expect(new Set(tools.map((tool) => tool.family)).size).toBe(9);
28
+ for (const tool of tools) {
29
+ expect(tool.version).toBe('1');
30
+ expect(tool.inputSchema).toBeDefined();
31
+ expect(tool.outputSchema).toBeDefined();
32
+ expect(tool.authority.network).toBe('denied');
33
+ expect(tool.failureModes.length).toBeGreaterThan(0);
34
+ }
35
+ tools[0].authority.network = 'allowed' as 'denied';
36
+ expect(getWorkflowTool('search')?.authority.network).toBe('denied');
37
+ expect(getWorkflowTool('send_message')).toBeUndefined();
38
+ });
39
+ test('search/document reads and structured records use only declared state', () => {
40
+ const state = {
41
+ documents: { a: 'Policy REVIEW', b: 'unrelated' },
42
+ records: { orders: [{ id: 1, approved: false }] },
43
+ };
44
+ const search = call('search', { query: 'review' }, state);
45
+ expect(search.status === 'succeeded' && search.output).toEqual({ matches: [{ id: 'a' }] });
46
+ const read = call('read_document', { id: 'a' }, state);
47
+ expect(read.status === 'succeeded' && read.output).toEqual({
48
+ id: 'a',
49
+ content: 'Policy REVIEW',
50
+ });
51
+ const records = call('query_records', { collection: 'orders' }, state);
52
+ expect(records.status === 'succeeded' && records.output).toEqual({
53
+ records: [{ id: 1, approved: false }],
54
+ });
55
+ });
56
+ test('distinguishes an empty document from a missing document', () => {
57
+ expect(call('read_document', { id: 'empty' }, { documents: { empty: '' } })).toMatchObject({
58
+ status: 'succeeded',
59
+ output: { id: 'empty', content: '' },
60
+ });
61
+ expect(call('read_document', { id: 'missing' }, { documents: { empty: '' } })).toMatchObject({
62
+ status: 'failed',
63
+ code: 'not_found',
64
+ });
65
+ });
66
+ test('file writes are isolated from fixtures and independent calls', () => {
67
+ const state = { files: { 'notes/a.txt': 'before' } };
68
+ const first = call('write_file', { path: 'notes/a.txt', content: 'after' }, state);
69
+ expect(state.files['notes/a.txt']).toBe('before');
70
+ expect(first.status === 'succeeded' && first.state.files).toEqual({ 'notes/a.txt': 'after' });
71
+ const read = call('read_file', { path: 'notes/a.txt' }, state);
72
+ expect(read.status === 'succeeded' && read.output).toEqual({ content: 'before' });
73
+ });
74
+ test.each(['../x', '/etc/passwd', 'a/../../x', 'a//b', 'a/constructor/b', 'C:\\private'])(
75
+ 'denies unsafe file input %s',
76
+ (path) => {
77
+ expect(call('write_file', { path, content: 'unsafe' }).status).toBe('invalid');
78
+ }
79
+ );
80
+ test('calculator uses explicit finite operations and rejects executable expressions', () => {
81
+ const result = call('calculator', { operation: 'multiply', a: 6, b: 7 });
82
+ expect(result.status === 'succeeded' && result.output).toEqual({ value: 42 });
83
+ expect(call('calculator', { operation: 'divide', a: 1, b: 0 }).status).toBe('failed');
84
+ expect(call('calculator', { operation: 'eval', a: 1, b: 2 }).status).toBe('invalid');
85
+ expect(call('calculator', { operation: 'add', a: '1', b: 2 }).status).toBe('invalid');
86
+ expect(call('calculator', { operation: 'add', a: Number.NaN, b: 2 }).status).toBe('invalid');
87
+ });
88
+ test('approval requests remain pending and decisions cannot change policy', () => {
89
+ const state = { workflow_state: {} };
90
+ const result = call('request_approval', { reason: 'Needs human review' }, state);
91
+ expect(result.status === 'succeeded' && result.output).toEqual({
92
+ requested: true,
93
+ status: 'pending',
94
+ });
95
+ expect(state.workflow_state).toEqual({});
96
+ const decision = call('record_decision', { decision: 'Propose review' });
97
+ expect(decision.status === 'succeeded' && decision.state.workflow_state).toEqual({
98
+ decision: 'Propose review',
99
+ });
100
+ expect(call('request_approval', { reason: 'review', approved: true }).status).toBe('invalid');
101
+ });
102
+ test('communication and delegation create only pending local records', () => {
103
+ const draft = call('draft_message', { recipient: 'reviewer', body: 'Review please' });
104
+ expect(draft.status === 'succeeded' && draft.output).toEqual({
105
+ id: 'draft-1',
106
+ status: 'draft',
107
+ });
108
+ const task = call('delegate_task', { task: 'Inspect policy' });
109
+ expect(task.status === 'succeeded' && task.output).toEqual({ id: 'task-1', status: 'pending' });
110
+ if (task.status !== 'succeeded') throw new Error('Expected task');
111
+ const status = call('get_task_status', { id: 'task-1' }, task.state);
112
+ expect(status.status === 'succeeded' && status.output).toEqual({
113
+ id: 'task-1',
114
+ status: 'pending',
115
+ });
116
+ expect(call('send_message', { recipient: 'reviewer' }).status).toBe('denied');
117
+ });
118
+ test('enforces declared tools and permissions at execution, regardless of prior schema validation', () => {
119
+ expect(
120
+ executeSimulatedTool({
121
+ tool: 'search',
122
+ input: { query: 'a' },
123
+ state: {},
124
+ policy,
125
+ declaredTools: [],
126
+ }).status
127
+ ).toBe('denied');
128
+ expect(
129
+ executeSimulatedTool({
130
+ tool: 'write_file',
131
+ input: { path: 'x', content: 'x' },
132
+ state: {},
133
+ policy: { ...policy, permissions: { files: 'read' } },
134
+ declaredTools,
135
+ }).status
136
+ ).toBe('denied');
137
+ expect(
138
+ executeSimulatedTool({
139
+ tool: 'search',
140
+ input: { query: 'a' },
141
+ state: {},
142
+ policy: { ...policy, network: 'allowed' } as unknown as WorkflowPolicy,
143
+ declaredTools,
144
+ }).status
145
+ ).toBe('denied');
146
+ });
147
+ test('handles unavailable and malformed fixture data without exposing raw input in evidence', () => {
148
+ expect(call('read_document', { id: 'missing' }).status).toBe('failed');
149
+ expect(
150
+ call('query_records', { collection: 'orders' }, { records: { orders: ['invalid'] } }).status
151
+ ).toBe('failed');
152
+ expect(call('read_document', { id: 'constructor' }).status).toBe('invalid');
153
+ expect(
154
+ call('read_document', { id: 'a' }, JSON.parse('{"__proto__":{"secret":"x"}}')).status
155
+ ).toBe('invalid');
156
+ const result = call('draft_message', { recipient: 'SECRET_RECIPIENT', body: 'SECRET_BODY' });
157
+ expect(JSON.stringify(result.evidence)).not.toContain('SECRET');
158
+ const unknown = call('SECRET_TOOL', {});
159
+ expect(JSON.stringify(unknown.evidence)).not.toContain('SECRET');
160
+ });
161
+ test('output and state have no mutable aliases and empty reads do not mutate state', () => {
162
+ const result = call('get_workflow_state', {}, { workflow_state: { approved: false } });
163
+ if (result.status !== 'succeeded') throw new Error('Expected state');
164
+ (result.output as { state: { approved: boolean } }).state.approved = true;
165
+ expect(result.state.workflow_state).toEqual({ approved: false });
166
+ const empty = call('get_workflow_state', {});
167
+ expect(empty.status === 'succeeded' && empty.state).toEqual({});
168
+ });
169
+ test('bounds oversized output and fixture state', () => {
170
+ expect(
171
+ call('read_document', { id: 'a' }, { documents: { a: 'x'.repeat(20_000) } }).status
172
+ ).toBe('failed');
173
+ expect(call('search', { query: 'x' }, { documents: { a: 'x'.repeat(1_048_577) } }).status).toBe(
174
+ 'invalid'
175
+ );
176
+ });
177
+ });