@artemiskit/core 0.5.3 → 0.6.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +28 -0
- package/README.md +12 -0
- package/dist/adapters/factory.d.ts +1 -1
- package/dist/adapters/index.d.ts +3 -3
- package/dist/adapters/registry.d.ts +1 -1
- package/dist/adapters/types.d.ts +7 -0
- package/dist/adapters/types.d.ts.map +1 -1
- package/dist/agent-evaluation/index.d.ts +2 -2
- package/dist/agent-evaluation/scorer.d.ts +1 -1
- package/dist/agent-workflow/catalog.d.ts +28 -0
- package/dist/agent-workflow/catalog.d.ts.map +1 -0
- package/dist/agent-workflow/environment.d.ts +47 -0
- package/dist/agent-workflow/environment.d.ts.map +1 -0
- package/dist/agent-workflow/index.d.ts +10 -0
- package/dist/agent-workflow/index.d.ts.map +1 -0
- package/dist/agent-workflow/parser.d.ts +7 -0
- package/dist/agent-workflow/parser.d.ts.map +1 -0
- package/dist/agent-workflow/sandbox-fixtures/qualify.d.ts +2 -0
- package/dist/agent-workflow/sandbox-fixtures/qualify.d.ts.map +1 -0
- package/dist/agent-workflow/sandbox.d.ts +12 -0
- package/dist/agent-workflow/sandbox.d.ts.map +1 -0
- package/dist/agent-workflow/schema.d.ts +774 -0
- package/dist/agent-workflow/schema.d.ts.map +1 -0
- package/dist/agent-workflow/session.d.ts +113 -0
- package/dist/agent-workflow/session.d.ts.map +1 -0
- package/dist/agent-workflow/simulated-tools.d.ts +35 -0
- package/dist/agent-workflow/simulated-tools.d.ts.map +1 -0
- package/dist/agent-workflow/target.d.ts +243 -0
- package/dist/agent-workflow/target.d.ts.map +1 -0
- package/dist/artifacts/index.d.ts +2 -2
- package/dist/artifacts/manifest.d.ts +1 -1
- package/dist/artifacts/types.d.ts +2 -2
- package/dist/comparison/eligibility.d.ts +1 -1
- package/dist/comparison/index.d.ts +1 -1
- package/dist/evaluators/combined.d.ts +2 -2
- package/dist/evaluators/contains.d.ts +2 -2
- package/dist/evaluators/exact.d.ts +2 -2
- package/dist/evaluators/fuzzy.d.ts +2 -2
- package/dist/evaluators/index.d.ts +13 -13
- package/dist/evaluators/inline.d.ts +2 -2
- package/dist/evaluators/json-schema.d.ts +2 -2
- package/dist/evaluators/llm-grader.d.ts +2 -2
- package/dist/evaluators/not-contains.d.ts +2 -2
- package/dist/evaluators/regex.d.ts +2 -2
- package/dist/evaluators/similarity.d.ts +2 -2
- package/dist/evaluators/tool-trace.d.ts +2 -2
- package/dist/evaluators/types.d.ts +3 -3
- package/dist/index.d.ts +15 -14
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +17827 -15174
- package/dist/provenance/execution-provenance.d.ts +1 -1
- package/dist/provenance/git.d.ts +1 -1
- package/dist/provenance/index.d.ts +4 -4
- package/dist/provenance/workload-identity.d.ts +2 -2
- package/dist/redaction/index.d.ts +2 -2
- package/dist/redaction/redactor.d.ts +1 -1
- package/dist/runner/executor.d.ts +3 -3
- package/dist/runner/index.d.ts +3 -3
- package/dist/runner/runner.d.ts +1 -1
- package/dist/runner/types.d.ts +5 -5
- package/dist/scenario/index.d.ts +4 -4
- package/dist/scenario/parser.d.ts +1 -1
- package/dist/scenario/variables.d.ts +1 -1
- package/dist/storage/factory.d.ts +1 -1
- package/dist/storage/index.d.ts +4 -4
- package/dist/storage/local.d.ts +2 -2
- package/dist/storage/supabase.d.ts +2 -2
- package/dist/storage/types.d.ts +2 -2
- package/dist/tools/fixture-executor.d.ts +2 -2
- package/dist/tools/index.d.ts +3 -3
- package/dist/tools/types.d.ts +1 -1
- package/dist/utils/index.d.ts +2 -2
- package/dist/validator/index.d.ts +2 -2
- package/dist/validator/validator.d.ts +1 -1
- package/package.json +4 -4
- package/src/adapters/types.ts +7 -0
- package/src/agent-workflow/catalog.ts +220 -0
- package/src/agent-workflow/environment.ts +207 -0
- package/src/agent-workflow/index.ts +10 -0
- package/src/agent-workflow/parser.ts +43 -0
- package/src/agent-workflow/sandbox-fixtures/qualify.ts +305 -0
- package/src/agent-workflow/sandbox.test.ts +117 -0
- package/src/agent-workflow/sandbox.ts +438 -0
- package/src/agent-workflow/schema.test.ts +218 -0
- package/src/agent-workflow/schema.ts +270 -0
- package/src/agent-workflow/session.test.ts +629 -0
- package/src/agent-workflow/session.ts +1119 -0
- package/src/agent-workflow/simulated-tools.test.ts +177 -0
- package/src/agent-workflow/simulated-tools.ts +230 -0
- package/src/agent-workflow/target.test.ts +280 -0
- package/src/agent-workflow/target.ts +357 -0
- package/src/index.ts +3 -0
|
@@ -0,0 +1,177 @@
|
|
|
1
|
+
import { describe, expect, test } from 'bun:test';
|
|
2
|
+
import { getWorkflowTool, listWorkflowTools } from './catalog';
|
|
3
|
+
import type { WorkflowPolicy } from './schema';
|
|
4
|
+
import { executeSimulatedTool } from './simulated-tools';
|
|
5
|
+
|
|
6
|
+
const policy: WorkflowPolicy = {
|
|
7
|
+
network: 'denied',
|
|
8
|
+
side_effects: 'denied',
|
|
9
|
+
permissions: {
|
|
10
|
+
documents: 'read',
|
|
11
|
+
records: 'read',
|
|
12
|
+
files: 'write',
|
|
13
|
+
workflow_state: 'write',
|
|
14
|
+
communication: 'write',
|
|
15
|
+
coordination: 'write',
|
|
16
|
+
},
|
|
17
|
+
budgets: { max_actions: 10, timeout_ms: 60_000 },
|
|
18
|
+
};
|
|
19
|
+
const declaredTools = listWorkflowTools().map((tool) => tool.id);
|
|
20
|
+
function call(tool: string, input: unknown, state: unknown = {}) {
|
|
21
|
+
return executeSimulatedTool({ tool, input, state, policy, declaredTools });
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
describe('deterministic simulated tool catalog', () => {
|
|
25
|
+
test('descriptors cover all families and are defensive copies', () => {
|
|
26
|
+
const tools = listWorkflowTools();
|
|
27
|
+
expect(new Set(tools.map((tool) => tool.family)).size).toBe(9);
|
|
28
|
+
for (const tool of tools) {
|
|
29
|
+
expect(tool.version).toBe('1');
|
|
30
|
+
expect(tool.inputSchema).toBeDefined();
|
|
31
|
+
expect(tool.outputSchema).toBeDefined();
|
|
32
|
+
expect(tool.authority.network).toBe('denied');
|
|
33
|
+
expect(tool.failureModes.length).toBeGreaterThan(0);
|
|
34
|
+
}
|
|
35
|
+
tools[0].authority.network = 'allowed' as 'denied';
|
|
36
|
+
expect(getWorkflowTool('search')?.authority.network).toBe('denied');
|
|
37
|
+
expect(getWorkflowTool('send_message')).toBeUndefined();
|
|
38
|
+
});
|
|
39
|
+
test('search/document reads and structured records use only declared state', () => {
|
|
40
|
+
const state = {
|
|
41
|
+
documents: { a: 'Policy REVIEW', b: 'unrelated' },
|
|
42
|
+
records: { orders: [{ id: 1, approved: false }] },
|
|
43
|
+
};
|
|
44
|
+
const search = call('search', { query: 'review' }, state);
|
|
45
|
+
expect(search.status === 'succeeded' && search.output).toEqual({ matches: [{ id: 'a' }] });
|
|
46
|
+
const read = call('read_document', { id: 'a' }, state);
|
|
47
|
+
expect(read.status === 'succeeded' && read.output).toEqual({
|
|
48
|
+
id: 'a',
|
|
49
|
+
content: 'Policy REVIEW',
|
|
50
|
+
});
|
|
51
|
+
const records = call('query_records', { collection: 'orders' }, state);
|
|
52
|
+
expect(records.status === 'succeeded' && records.output).toEqual({
|
|
53
|
+
records: [{ id: 1, approved: false }],
|
|
54
|
+
});
|
|
55
|
+
});
|
|
56
|
+
test('distinguishes an empty document from a missing document', () => {
|
|
57
|
+
expect(call('read_document', { id: 'empty' }, { documents: { empty: '' } })).toMatchObject({
|
|
58
|
+
status: 'succeeded',
|
|
59
|
+
output: { id: 'empty', content: '' },
|
|
60
|
+
});
|
|
61
|
+
expect(call('read_document', { id: 'missing' }, { documents: { empty: '' } })).toMatchObject({
|
|
62
|
+
status: 'failed',
|
|
63
|
+
code: 'not_found',
|
|
64
|
+
});
|
|
65
|
+
});
|
|
66
|
+
test('file writes are isolated from fixtures and independent calls', () => {
|
|
67
|
+
const state = { files: { 'notes/a.txt': 'before' } };
|
|
68
|
+
const first = call('write_file', { path: 'notes/a.txt', content: 'after' }, state);
|
|
69
|
+
expect(state.files['notes/a.txt']).toBe('before');
|
|
70
|
+
expect(first.status === 'succeeded' && first.state.files).toEqual({ 'notes/a.txt': 'after' });
|
|
71
|
+
const read = call('read_file', { path: 'notes/a.txt' }, state);
|
|
72
|
+
expect(read.status === 'succeeded' && read.output).toEqual({ content: 'before' });
|
|
73
|
+
});
|
|
74
|
+
test.each(['../x', '/etc/passwd', 'a/../../x', 'a//b', 'a/constructor/b', 'C:\\private'])(
|
|
75
|
+
'denies unsafe file input %s',
|
|
76
|
+
(path) => {
|
|
77
|
+
expect(call('write_file', { path, content: 'unsafe' }).status).toBe('invalid');
|
|
78
|
+
}
|
|
79
|
+
);
|
|
80
|
+
test('calculator uses explicit finite operations and rejects executable expressions', () => {
|
|
81
|
+
const result = call('calculator', { operation: 'multiply', a: 6, b: 7 });
|
|
82
|
+
expect(result.status === 'succeeded' && result.output).toEqual({ value: 42 });
|
|
83
|
+
expect(call('calculator', { operation: 'divide', a: 1, b: 0 }).status).toBe('failed');
|
|
84
|
+
expect(call('calculator', { operation: 'eval', a: 1, b: 2 }).status).toBe('invalid');
|
|
85
|
+
expect(call('calculator', { operation: 'add', a: '1', b: 2 }).status).toBe('invalid');
|
|
86
|
+
expect(call('calculator', { operation: 'add', a: Number.NaN, b: 2 }).status).toBe('invalid');
|
|
87
|
+
});
|
|
88
|
+
test('approval requests remain pending and decisions cannot change policy', () => {
|
|
89
|
+
const state = { workflow_state: {} };
|
|
90
|
+
const result = call('request_approval', { reason: 'Needs human review' }, state);
|
|
91
|
+
expect(result.status === 'succeeded' && result.output).toEqual({
|
|
92
|
+
requested: true,
|
|
93
|
+
status: 'pending',
|
|
94
|
+
});
|
|
95
|
+
expect(state.workflow_state).toEqual({});
|
|
96
|
+
const decision = call('record_decision', { decision: 'Propose review' });
|
|
97
|
+
expect(decision.status === 'succeeded' && decision.state.workflow_state).toEqual({
|
|
98
|
+
decision: 'Propose review',
|
|
99
|
+
});
|
|
100
|
+
expect(call('request_approval', { reason: 'review', approved: true }).status).toBe('invalid');
|
|
101
|
+
});
|
|
102
|
+
test('communication and delegation create only pending local records', () => {
|
|
103
|
+
const draft = call('draft_message', { recipient: 'reviewer', body: 'Review please' });
|
|
104
|
+
expect(draft.status === 'succeeded' && draft.output).toEqual({
|
|
105
|
+
id: 'draft-1',
|
|
106
|
+
status: 'draft',
|
|
107
|
+
});
|
|
108
|
+
const task = call('delegate_task', { task: 'Inspect policy' });
|
|
109
|
+
expect(task.status === 'succeeded' && task.output).toEqual({ id: 'task-1', status: 'pending' });
|
|
110
|
+
if (task.status !== 'succeeded') throw new Error('Expected task');
|
|
111
|
+
const status = call('get_task_status', { id: 'task-1' }, task.state);
|
|
112
|
+
expect(status.status === 'succeeded' && status.output).toEqual({
|
|
113
|
+
id: 'task-1',
|
|
114
|
+
status: 'pending',
|
|
115
|
+
});
|
|
116
|
+
expect(call('send_message', { recipient: 'reviewer' }).status).toBe('denied');
|
|
117
|
+
});
|
|
118
|
+
test('enforces declared tools and permissions at execution, regardless of prior schema validation', () => {
|
|
119
|
+
expect(
|
|
120
|
+
executeSimulatedTool({
|
|
121
|
+
tool: 'search',
|
|
122
|
+
input: { query: 'a' },
|
|
123
|
+
state: {},
|
|
124
|
+
policy,
|
|
125
|
+
declaredTools: [],
|
|
126
|
+
}).status
|
|
127
|
+
).toBe('denied');
|
|
128
|
+
expect(
|
|
129
|
+
executeSimulatedTool({
|
|
130
|
+
tool: 'write_file',
|
|
131
|
+
input: { path: 'x', content: 'x' },
|
|
132
|
+
state: {},
|
|
133
|
+
policy: { ...policy, permissions: { files: 'read' } },
|
|
134
|
+
declaredTools,
|
|
135
|
+
}).status
|
|
136
|
+
).toBe('denied');
|
|
137
|
+
expect(
|
|
138
|
+
executeSimulatedTool({
|
|
139
|
+
tool: 'search',
|
|
140
|
+
input: { query: 'a' },
|
|
141
|
+
state: {},
|
|
142
|
+
policy: { ...policy, network: 'allowed' } as unknown as WorkflowPolicy,
|
|
143
|
+
declaredTools,
|
|
144
|
+
}).status
|
|
145
|
+
).toBe('denied');
|
|
146
|
+
});
|
|
147
|
+
test('handles unavailable and malformed fixture data without exposing raw input in evidence', () => {
|
|
148
|
+
expect(call('read_document', { id: 'missing' }).status).toBe('failed');
|
|
149
|
+
expect(
|
|
150
|
+
call('query_records', { collection: 'orders' }, { records: { orders: ['invalid'] } }).status
|
|
151
|
+
).toBe('failed');
|
|
152
|
+
expect(call('read_document', { id: 'constructor' }).status).toBe('invalid');
|
|
153
|
+
expect(
|
|
154
|
+
call('read_document', { id: 'a' }, JSON.parse('{"__proto__":{"secret":"x"}}')).status
|
|
155
|
+
).toBe('invalid');
|
|
156
|
+
const result = call('draft_message', { recipient: 'SECRET_RECIPIENT', body: 'SECRET_BODY' });
|
|
157
|
+
expect(JSON.stringify(result.evidence)).not.toContain('SECRET');
|
|
158
|
+
const unknown = call('SECRET_TOOL', {});
|
|
159
|
+
expect(JSON.stringify(unknown.evidence)).not.toContain('SECRET');
|
|
160
|
+
});
|
|
161
|
+
test('output and state have no mutable aliases and empty reads do not mutate state', () => {
|
|
162
|
+
const result = call('get_workflow_state', {}, { workflow_state: { approved: false } });
|
|
163
|
+
if (result.status !== 'succeeded') throw new Error('Expected state');
|
|
164
|
+
(result.output as { state: { approved: boolean } }).state.approved = true;
|
|
165
|
+
expect(result.state.workflow_state).toEqual({ approved: false });
|
|
166
|
+
const empty = call('get_workflow_state', {});
|
|
167
|
+
expect(empty.status === 'succeeded' && empty.state).toEqual({});
|
|
168
|
+
});
|
|
169
|
+
test('bounds oversized output and fixture state', () => {
|
|
170
|
+
expect(
|
|
171
|
+
call('read_document', { id: 'a' }, { documents: { a: 'x'.repeat(20_000) } }).status
|
|
172
|
+
).toBe('failed');
|
|
173
|
+
expect(call('search', { query: 'x' }, { documents: { a: 'x'.repeat(1_048_577) } }).status).toBe(
|
|
174
|
+
'invalid'
|
|
175
|
+
);
|
|
176
|
+
});
|
|
177
|
+
});
|
|
@@ -0,0 +1,230 @@
|
|
|
1
|
+
import Ajv from 'ajv';
|
|
2
|
+
import { getWorkflowTool, listWorkflowTools } from './catalog';
|
|
3
|
+
import {
|
|
4
|
+
type WorkflowJson,
|
|
5
|
+
type WorkflowPolicy,
|
|
6
|
+
WorkflowPolicySchema,
|
|
7
|
+
isWorkflowJson,
|
|
8
|
+
isWorkflowRelativePath,
|
|
9
|
+
} from './schema';
|
|
10
|
+
|
|
11
|
+
type JsonObject = { [key: string]: WorkflowJson };
|
|
12
|
+
type ToolFailureCode =
|
|
13
|
+
| 'undeclared_tool'
|
|
14
|
+
| 'permission_denied'
|
|
15
|
+
| 'invalid_input'
|
|
16
|
+
| 'invalid_state'
|
|
17
|
+
| 'not_found'
|
|
18
|
+
| 'output_limit'
|
|
19
|
+
| 'invalid_policy'
|
|
20
|
+
| 'tool_error';
|
|
21
|
+
export interface SimulatedToolEvidence {
|
|
22
|
+
tool: string;
|
|
23
|
+
version: '1';
|
|
24
|
+
status: 'succeeded' | 'denied' | 'invalid' | 'failed';
|
|
25
|
+
code?: ToolFailureCode;
|
|
26
|
+
}
|
|
27
|
+
export type SimulatedToolResult =
|
|
28
|
+
| {
|
|
29
|
+
status: 'succeeded';
|
|
30
|
+
output: WorkflowJson;
|
|
31
|
+
state: JsonObject;
|
|
32
|
+
evidence: SimulatedToolEvidence;
|
|
33
|
+
}
|
|
34
|
+
| {
|
|
35
|
+
status: 'denied' | 'invalid' | 'failed';
|
|
36
|
+
code: ToolFailureCode;
|
|
37
|
+
evidence: SimulatedToolEvidence;
|
|
38
|
+
};
|
|
39
|
+
|
|
40
|
+
const ajv = new Ajv({ allErrors: false, strict: true });
|
|
41
|
+
const validators = new Map(
|
|
42
|
+
listWorkflowTools().map((tool) => [
|
|
43
|
+
tool.id,
|
|
44
|
+
{ input: ajv.compile(tool.inputSchema), output: ajv.compile(tool.outputSchema) },
|
|
45
|
+
])
|
|
46
|
+
);
|
|
47
|
+
|
|
48
|
+
function isObject(value: unknown): value is JsonObject {
|
|
49
|
+
return value !== null && typeof value === 'object' && !Array.isArray(value);
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
class SimulatedFailure extends Error {
|
|
53
|
+
constructor(readonly code: ToolFailureCode) {
|
|
54
|
+
super(code);
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
function objectAt(state: JsonObject, key: string, create = false): JsonObject {
|
|
59
|
+
const value = state[key];
|
|
60
|
+
if (value === undefined) {
|
|
61
|
+
const empty = {};
|
|
62
|
+
if (create) state[key] = empty;
|
|
63
|
+
return empty;
|
|
64
|
+
}
|
|
65
|
+
if (!isObject(value)) throw new SimulatedFailure('invalid_state');
|
|
66
|
+
return value;
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
function nextId(collection: JsonObject, prefix: string): string {
|
|
70
|
+
let index = Object.keys(collection).length + 1;
|
|
71
|
+
while (Object.hasOwn(collection, `${prefix}-${index}`)) index++;
|
|
72
|
+
return `${prefix}-${index}`;
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
/**
|
|
76
|
+
* One deterministic simulated call, with isolated input/output state and metadata-only evidence.
|
|
77
|
+
* Budgets are declarative here: a future environment runner owns action counts and deadlines.
|
|
78
|
+
* `side_effects` governs external effects; declared local state writes never authorize real effects.
|
|
79
|
+
*/
|
|
80
|
+
export function executeSimulatedTool(request: {
|
|
81
|
+
tool: string;
|
|
82
|
+
input: unknown;
|
|
83
|
+
state: unknown;
|
|
84
|
+
policy: WorkflowPolicy;
|
|
85
|
+
declaredTools: readonly string[];
|
|
86
|
+
}): SimulatedToolResult {
|
|
87
|
+
const tool = getWorkflowTool(request.tool);
|
|
88
|
+
const evidenceTool = tool?.id ?? 'unknown';
|
|
89
|
+
function failure(
|
|
90
|
+
status: 'denied' | 'invalid' | 'failed',
|
|
91
|
+
code: ToolFailureCode
|
|
92
|
+
): SimulatedToolResult {
|
|
93
|
+
return { status, code, evidence: { tool: evidenceTool, version: '1', status, code } };
|
|
94
|
+
}
|
|
95
|
+
if (!tool || !request.declaredTools.includes(tool.id))
|
|
96
|
+
return failure('denied', 'undeclared_tool');
|
|
97
|
+
if (!isWorkflowJson(request.policy)) return failure('denied', 'invalid_policy');
|
|
98
|
+
const policy = WorkflowPolicySchema.safeParse(request.policy);
|
|
99
|
+
if (!policy.success) return failure('denied', 'invalid_policy');
|
|
100
|
+
if (tool.authority.access !== 'none') {
|
|
101
|
+
const resource = tool.authority.resource as keyof WorkflowPolicy['permissions'];
|
|
102
|
+
const grant = policy.data.permissions[resource];
|
|
103
|
+
if (!grant || (tool.authority.access === 'write' && grant !== 'write'))
|
|
104
|
+
return failure('denied', 'permission_denied');
|
|
105
|
+
}
|
|
106
|
+
const validator = validators.get(tool.id);
|
|
107
|
+
if (
|
|
108
|
+
!validator ||
|
|
109
|
+
!isWorkflowJson(request.input) ||
|
|
110
|
+
!isObject(request.input) ||
|
|
111
|
+
!validator.input(request.input)
|
|
112
|
+
)
|
|
113
|
+
return failure('invalid', 'invalid_input');
|
|
114
|
+
if (!isWorkflowJson(request.state) || !isObject(request.state))
|
|
115
|
+
return failure('invalid', 'invalid_state');
|
|
116
|
+
const state: JsonObject = structuredClone(request.state);
|
|
117
|
+
const input = request.input;
|
|
118
|
+
let output: WorkflowJson;
|
|
119
|
+
try {
|
|
120
|
+
switch (tool.id) {
|
|
121
|
+
case 'search': {
|
|
122
|
+
const documents = objectAt(state, 'documents');
|
|
123
|
+
if (Object.values(documents).some((value) => typeof value !== 'string'))
|
|
124
|
+
throw new SimulatedFailure('invalid_state');
|
|
125
|
+
output = {
|
|
126
|
+
matches: Object.entries(documents)
|
|
127
|
+
.filter(([, content]) =>
|
|
128
|
+
(content as string).toLowerCase().includes((input.query as string).toLowerCase())
|
|
129
|
+
)
|
|
130
|
+
.sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0))
|
|
131
|
+
.slice(0, 100)
|
|
132
|
+
.map(([id]) => ({ id })),
|
|
133
|
+
};
|
|
134
|
+
break;
|
|
135
|
+
}
|
|
136
|
+
case 'read_document': {
|
|
137
|
+
const content = objectAt(state, 'documents')[input.id as string];
|
|
138
|
+
if (content === undefined) throw new SimulatedFailure('not_found');
|
|
139
|
+
if (typeof content !== 'string') throw new SimulatedFailure('invalid_state');
|
|
140
|
+
output = { id: input.id, content };
|
|
141
|
+
break;
|
|
142
|
+
}
|
|
143
|
+
case 'query_records': {
|
|
144
|
+
const records = objectAt(state, 'records')[input.collection as string];
|
|
145
|
+
if (records === undefined) throw new SimulatedFailure('not_found');
|
|
146
|
+
if (!Array.isArray(records) || records.some((record) => !isObject(record)))
|
|
147
|
+
throw new SimulatedFailure('invalid_state');
|
|
148
|
+
output = { records: records.slice(0, 100) };
|
|
149
|
+
break;
|
|
150
|
+
}
|
|
151
|
+
case 'read_file':
|
|
152
|
+
case 'write_file': {
|
|
153
|
+
const path = input.path as string;
|
|
154
|
+
if (!isWorkflowRelativePath(path)) return failure('invalid', 'invalid_input');
|
|
155
|
+
const files = objectAt(state, 'files', tool.id === 'write_file');
|
|
156
|
+
if (tool.id === 'write_file') {
|
|
157
|
+
files[path] = input.content;
|
|
158
|
+
output = { path, written: true };
|
|
159
|
+
} else {
|
|
160
|
+
const content = files[path];
|
|
161
|
+
if (content === undefined) throw new SimulatedFailure('not_found');
|
|
162
|
+
if (typeof content !== 'string') throw new SimulatedFailure('invalid_state');
|
|
163
|
+
output = { content };
|
|
164
|
+
}
|
|
165
|
+
break;
|
|
166
|
+
}
|
|
167
|
+
case 'calculator': {
|
|
168
|
+
const a = input.a as number;
|
|
169
|
+
const b = input.b as number;
|
|
170
|
+
const value =
|
|
171
|
+
input.operation === 'add'
|
|
172
|
+
? a + b
|
|
173
|
+
: input.operation === 'subtract'
|
|
174
|
+
? a - b
|
|
175
|
+
: input.operation === 'multiply'
|
|
176
|
+
? a * b
|
|
177
|
+
: a / b;
|
|
178
|
+
if (!Number.isFinite(value)) throw new SimulatedFailure('tool_error');
|
|
179
|
+
output = { value };
|
|
180
|
+
break;
|
|
181
|
+
}
|
|
182
|
+
case 'get_workflow_state':
|
|
183
|
+
output = { state: objectAt(state, 'workflow_state') };
|
|
184
|
+
break;
|
|
185
|
+
case 'request_approval': {
|
|
186
|
+
const workflow = objectAt(state, 'workflow_state', true);
|
|
187
|
+
workflow.approvals = { requested: true, status: 'pending', reason: input.reason };
|
|
188
|
+
output = { requested: true, status: 'pending' };
|
|
189
|
+
break;
|
|
190
|
+
}
|
|
191
|
+
case 'record_decision': {
|
|
192
|
+
objectAt(state, 'workflow_state', true).decision = input.decision;
|
|
193
|
+
output = { recorded: true };
|
|
194
|
+
break;
|
|
195
|
+
}
|
|
196
|
+
case 'draft_message': {
|
|
197
|
+
const drafts = objectAt(state, 'drafts', true);
|
|
198
|
+
const id = nextId(drafts, 'draft');
|
|
199
|
+
drafts[id] = { recipient: input.recipient, body: input.body, status: 'draft' };
|
|
200
|
+
output = { id, status: 'draft' };
|
|
201
|
+
break;
|
|
202
|
+
}
|
|
203
|
+
case 'delegate_task': {
|
|
204
|
+
const tasks = objectAt(state, 'tasks', true);
|
|
205
|
+
const id = nextId(tasks, 'task');
|
|
206
|
+
tasks[id] = { task: input.task, status: 'pending' };
|
|
207
|
+
output = { id, status: 'pending' };
|
|
208
|
+
break;
|
|
209
|
+
}
|
|
210
|
+
case 'get_task_status': {
|
|
211
|
+
const task = objectAt(state, 'tasks')[input.id as string];
|
|
212
|
+
if (task === undefined) throw new SimulatedFailure('not_found');
|
|
213
|
+
if (!isObject(task)) throw new SimulatedFailure('invalid_state');
|
|
214
|
+
output = { id: input.id, status: task.status };
|
|
215
|
+
break;
|
|
216
|
+
}
|
|
217
|
+
}
|
|
218
|
+
if (!isWorkflowJson(output) || !isWorkflowJson(state) || !validator.output(output))
|
|
219
|
+
return failure('failed', 'output_limit');
|
|
220
|
+
// Do not let consumers mutate returned state through a shared output reference.
|
|
221
|
+
return {
|
|
222
|
+
status: 'succeeded',
|
|
223
|
+
output: structuredClone(output),
|
|
224
|
+
state,
|
|
225
|
+
evidence: { tool: tool.id, version: '1', status: 'succeeded' },
|
|
226
|
+
};
|
|
227
|
+
} catch (error) {
|
|
228
|
+
return failure('failed', error instanceof SimulatedFailure ? error.code : 'tool_error');
|
|
229
|
+
}
|
|
230
|
+
}
|
|
@@ -0,0 +1,280 @@
|
|
|
1
|
+
import { describe, expect, test } from 'bun:test';
|
|
2
|
+
import type { GenerateOptions, GenerateResult, ModelClient } from '../adapters/types';
|
|
3
|
+
import { type AgentTurnRequest, createModelClientTarget } from './target';
|
|
4
|
+
|
|
5
|
+
const request: AgentTurnRequest = {
|
|
6
|
+
messages: [{ role: 'user', content: 'Read the declared document.' }],
|
|
7
|
+
tools: [
|
|
8
|
+
{
|
|
9
|
+
type: 'function',
|
|
10
|
+
function: {
|
|
11
|
+
name: 'read_document',
|
|
12
|
+
parameters: {
|
|
13
|
+
type: 'object',
|
|
14
|
+
properties: { id: { type: 'string' } },
|
|
15
|
+
required: ['id'],
|
|
16
|
+
additionalProperties: false,
|
|
17
|
+
},
|
|
18
|
+
},
|
|
19
|
+
},
|
|
20
|
+
],
|
|
21
|
+
model: 'requested-model',
|
|
22
|
+
generation: { maxTokens: 200, temperature: 0, topP: 1, seed: 1, stop: ['END'] },
|
|
23
|
+
budgets: { timeoutMs: 1000, maxToolCalls: 2 },
|
|
24
|
+
};
|
|
25
|
+
const call = {
|
|
26
|
+
id: 'call-1',
|
|
27
|
+
type: 'function' as const,
|
|
28
|
+
function: { name: 'read_document', arguments: '{"id":"doc-1"}' },
|
|
29
|
+
};
|
|
30
|
+
const response: GenerateResult = {
|
|
31
|
+
id: 'response-1',
|
|
32
|
+
model: 'observed-model',
|
|
33
|
+
text: '',
|
|
34
|
+
tokens: { prompt: 10, completion: 2, total: 12 },
|
|
35
|
+
latencyMs: 1,
|
|
36
|
+
finishReason: 'tool_calls',
|
|
37
|
+
toolCalls: [call],
|
|
38
|
+
raw: { secret: 'not-retained' },
|
|
39
|
+
};
|
|
40
|
+
function client(overrides: Partial<ModelClient> = {}): ModelClient {
|
|
41
|
+
return {
|
|
42
|
+
provider: 'custom-provider',
|
|
43
|
+
capabilities: async () => ({
|
|
44
|
+
streaming: false,
|
|
45
|
+
functionCalling: true,
|
|
46
|
+
toolUse: true,
|
|
47
|
+
maxContext: 1000,
|
|
48
|
+
}),
|
|
49
|
+
generate: async () => structuredClone(response),
|
|
50
|
+
...overrides,
|
|
51
|
+
};
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
describe('ModelClient agent target', () => {
|
|
55
|
+
test('reports advertised capability and honest cancellation support', async () => {
|
|
56
|
+
expect(await createModelClientTarget(client()).capabilities({ timeoutMs: 100 })).toEqual({
|
|
57
|
+
status: 'available',
|
|
58
|
+
toolUse: true,
|
|
59
|
+
transportCancellation: false,
|
|
60
|
+
});
|
|
61
|
+
});
|
|
62
|
+
test('passes declared settings and returns normalized evidence without raw payloads', async () => {
|
|
63
|
+
let received: GenerateOptions | undefined;
|
|
64
|
+
const target = createModelClientTarget(
|
|
65
|
+
client({
|
|
66
|
+
generate: async (options) => {
|
|
67
|
+
received = options;
|
|
68
|
+
return response;
|
|
69
|
+
},
|
|
70
|
+
})
|
|
71
|
+
);
|
|
72
|
+
const result = await target.turn(request);
|
|
73
|
+
expect(received).toEqual({
|
|
74
|
+
maxRetries: 0,
|
|
75
|
+
prompt: request.messages,
|
|
76
|
+
tools: request.tools,
|
|
77
|
+
model: request.model,
|
|
78
|
+
...request.generation,
|
|
79
|
+
});
|
|
80
|
+
expect(result).toEqual({
|
|
81
|
+
status: 'completed',
|
|
82
|
+
id: response.id,
|
|
83
|
+
model: response.model,
|
|
84
|
+
message: { role: 'assistant', content: '', tool_calls: [call] },
|
|
85
|
+
tokens: response.tokens,
|
|
86
|
+
latencyMs: 1,
|
|
87
|
+
finishReason: 'tool_calls',
|
|
88
|
+
});
|
|
89
|
+
expect(JSON.stringify(result)).not.toContain('not-retained');
|
|
90
|
+
});
|
|
91
|
+
test('accepts correlated tool results and preserves IDs into continuation', async () => {
|
|
92
|
+
let received: GenerateOptions | undefined;
|
|
93
|
+
const target = createModelClientTarget(
|
|
94
|
+
client({
|
|
95
|
+
generate: async (options) => {
|
|
96
|
+
received = options;
|
|
97
|
+
return { ...response, text: 'Done', toolCalls: undefined, finishReason: 'stop' };
|
|
98
|
+
},
|
|
99
|
+
})
|
|
100
|
+
);
|
|
101
|
+
const messages: AgentTurnRequest['messages'] = [
|
|
102
|
+
...request.messages,
|
|
103
|
+
{ role: 'assistant', content: '', tool_calls: [call] },
|
|
104
|
+
{ role: 'tool', content: 'Document text', toolCallId: call.id },
|
|
105
|
+
];
|
|
106
|
+
expect((await target.turn({ ...request, messages })).status).toBe('completed');
|
|
107
|
+
expect(received?.prompt).toEqual(messages);
|
|
108
|
+
});
|
|
109
|
+
test('unsupported capabilities prevent generate calls', async () => {
|
|
110
|
+
let generated = false;
|
|
111
|
+
const target = createModelClientTarget(
|
|
112
|
+
client({
|
|
113
|
+
capabilities: async () => ({
|
|
114
|
+
streaming: false,
|
|
115
|
+
functionCalling: false,
|
|
116
|
+
toolUse: false,
|
|
117
|
+
maxContext: 1000,
|
|
118
|
+
}),
|
|
119
|
+
generate: async () => {
|
|
120
|
+
generated = true;
|
|
121
|
+
return response;
|
|
122
|
+
},
|
|
123
|
+
})
|
|
124
|
+
);
|
|
125
|
+
expect(await target.turn(request)).toEqual({
|
|
126
|
+
status: 'unsupported',
|
|
127
|
+
code: 'tool_use_unsupported',
|
|
128
|
+
});
|
|
129
|
+
expect(generated).toBe(false);
|
|
130
|
+
});
|
|
131
|
+
test('capability errors and provider errors do not expose raw exception content', async () => {
|
|
132
|
+
for (const method of ['capabilities', 'generate'] as const) {
|
|
133
|
+
const target = createModelClientTarget(
|
|
134
|
+
client({
|
|
135
|
+
[method]: async () => {
|
|
136
|
+
throw new Error('secret provider credential');
|
|
137
|
+
},
|
|
138
|
+
})
|
|
139
|
+
);
|
|
140
|
+
expect(await target.turn(request)).toEqual({ status: 'error', code: 'target_error' });
|
|
141
|
+
}
|
|
142
|
+
});
|
|
143
|
+
test.each([
|
|
144
|
+
[
|
|
145
|
+
'unknown tool',
|
|
146
|
+
{ toolCalls: [{ ...call, function: { ...call.function, name: 'send_money' } }] },
|
|
147
|
+
],
|
|
148
|
+
['duplicate ID', { toolCalls: [call, call] }],
|
|
149
|
+
[
|
|
150
|
+
'invalid JSON',
|
|
151
|
+
{ toolCalls: [{ ...call, function: { ...call.function, arguments: 'broken' } }] },
|
|
152
|
+
],
|
|
153
|
+
[
|
|
154
|
+
'nonobject arguments',
|
|
155
|
+
{ toolCalls: [{ ...call, function: { ...call.function, arguments: '[]' } }] },
|
|
156
|
+
],
|
|
157
|
+
[
|
|
158
|
+
'invalid schema arguments',
|
|
159
|
+
{ toolCalls: [{ ...call, function: { ...call.function, arguments: '{"id":3}' } }] },
|
|
160
|
+
],
|
|
161
|
+
['negative usage', { tokens: { prompt: -1, completion: 2, total: 1 } }],
|
|
162
|
+
['inconsistent usage', { tokens: { prompt: 1, completion: 2, total: 5 } }],
|
|
163
|
+
['missing calls', { toolCalls: undefined }],
|
|
164
|
+
['legacy function call', { functionCall: { name: 'read_document', arguments: '{}' } }],
|
|
165
|
+
['invalid latency', { latencyMs: Number.NaN }],
|
|
166
|
+
['missing observed model', { model: '' }],
|
|
167
|
+
])('rejects malformed response: %s', async (_name, override) => {
|
|
168
|
+
const target = createModelClientTarget(
|
|
169
|
+
client({ generate: async () => ({ ...response, ...override }) })
|
|
170
|
+
);
|
|
171
|
+
expect(await target.turn(request)).toMatchObject({
|
|
172
|
+
status: 'invalid',
|
|
173
|
+
code: 'invalid_response',
|
|
174
|
+
});
|
|
175
|
+
});
|
|
176
|
+
test('rejects tool calls beyond the declared per-turn budget', async () => {
|
|
177
|
+
expect(
|
|
178
|
+
await createModelClientTarget(client()).turn({
|
|
179
|
+
...request,
|
|
180
|
+
budgets: { ...request.budgets, maxToolCalls: 0 },
|
|
181
|
+
})
|
|
182
|
+
).toEqual({ status: 'invalid', code: 'invalid_response' });
|
|
183
|
+
});
|
|
184
|
+
test('rejects async tool argument schemas before provider invocation', async () => {
|
|
185
|
+
let reached = false;
|
|
186
|
+
const target = createModelClientTarget(
|
|
187
|
+
client({
|
|
188
|
+
generate: async () => {
|
|
189
|
+
reached = true;
|
|
190
|
+
return response;
|
|
191
|
+
},
|
|
192
|
+
})
|
|
193
|
+
);
|
|
194
|
+
const tool = request.tools[0];
|
|
195
|
+
expect(
|
|
196
|
+
await target.turn({
|
|
197
|
+
...request,
|
|
198
|
+
tools: [
|
|
199
|
+
{
|
|
200
|
+
...tool,
|
|
201
|
+
function: {
|
|
202
|
+
...tool.function,
|
|
203
|
+
parameters: { ...tool.function.parameters, $async: true },
|
|
204
|
+
},
|
|
205
|
+
},
|
|
206
|
+
],
|
|
207
|
+
})
|
|
208
|
+
).toEqual({ status: 'invalid', code: 'invalid_request' });
|
|
209
|
+
expect(reached).toBe(false);
|
|
210
|
+
});
|
|
211
|
+
test.each([
|
|
212
|
+
{ budgets: { timeoutMs: 0, maxToolCalls: 2 } },
|
|
213
|
+
{ generation: { maxTokens: -1 } },
|
|
214
|
+
{ messages: [{ role: 'tool', content: 'Orphan', toolCallId: 'missing' }] },
|
|
215
|
+
{ messages: [{ role: 'assistant', content: '', tool_calls: [call] }] },
|
|
216
|
+
{ tools: [request.tools[0], request.tools[0]] },
|
|
217
|
+
])('invalid request never reaches provider', async (override) => {
|
|
218
|
+
let reached = false;
|
|
219
|
+
const target = createModelClientTarget(
|
|
220
|
+
client({
|
|
221
|
+
generate: async () => {
|
|
222
|
+
reached = true;
|
|
223
|
+
return response;
|
|
224
|
+
},
|
|
225
|
+
})
|
|
226
|
+
);
|
|
227
|
+
expect(await target.turn({ ...request, ...override } as AgentTurnRequest)).toEqual({
|
|
228
|
+
status: 'invalid',
|
|
229
|
+
code: 'invalid_request',
|
|
230
|
+
});
|
|
231
|
+
expect(reached).toBe(false);
|
|
232
|
+
});
|
|
233
|
+
test('aborting before a turn prevents a provider call', async () => {
|
|
234
|
+
const controller = new AbortController();
|
|
235
|
+
controller.abort();
|
|
236
|
+
expect(await createModelClientTarget(client()).turn(request, controller.signal)).toEqual({
|
|
237
|
+
status: 'error',
|
|
238
|
+
code: 'aborted',
|
|
239
|
+
});
|
|
240
|
+
});
|
|
241
|
+
test('timeout bounds waiting even when generation does not settle', async () => {
|
|
242
|
+
const target = createModelClientTarget(client({ generate: () => new Promise(() => {}) }));
|
|
243
|
+
expect(await target.turn({ ...request, budgets: { timeoutMs: 10, maxToolCalls: 2 } })).toEqual({
|
|
244
|
+
status: 'error',
|
|
245
|
+
code: 'timeout',
|
|
246
|
+
});
|
|
247
|
+
});
|
|
248
|
+
test('capabilities settling after timeout cannot start generation', async () => {
|
|
249
|
+
let release: (() => void) | undefined;
|
|
250
|
+
let generated = false;
|
|
251
|
+
const target = createModelClientTarget(
|
|
252
|
+
client({
|
|
253
|
+
capabilities: async () => {
|
|
254
|
+
await new Promise<void>((resolve) => {
|
|
255
|
+
release = resolve;
|
|
256
|
+
});
|
|
257
|
+
return { streaming: false, functionCalling: true, toolUse: true, maxContext: 1 };
|
|
258
|
+
},
|
|
259
|
+
generate: async () => {
|
|
260
|
+
generated = true;
|
|
261
|
+
return response;
|
|
262
|
+
},
|
|
263
|
+
})
|
|
264
|
+
);
|
|
265
|
+
expect(await target.turn({ ...request, budgets: { timeoutMs: 10, maxToolCalls: 2 } })).toEqual({
|
|
266
|
+
status: 'error',
|
|
267
|
+
code: 'timeout',
|
|
268
|
+
});
|
|
269
|
+
release?.();
|
|
270
|
+
await new Promise((resolve) => setTimeout(resolve, 1));
|
|
271
|
+
expect(generated).toBe(false);
|
|
272
|
+
});
|
|
273
|
+
test('abort while waiting returns an explicit aborted result', async () => {
|
|
274
|
+
const controller = new AbortController();
|
|
275
|
+
const target = createModelClientTarget(client({ generate: () => new Promise(() => {}) }));
|
|
276
|
+
const pending = target.turn(request, controller.signal);
|
|
277
|
+
controller.abort();
|
|
278
|
+
expect(await pending).toEqual({ status: 'error', code: 'aborted' });
|
|
279
|
+
});
|
|
280
|
+
});
|