@artemiskit/core 0.5.3 → 0.6.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +28 -0
- package/README.md +12 -0
- package/dist/adapters/factory.d.ts +1 -1
- package/dist/adapters/index.d.ts +3 -3
- package/dist/adapters/registry.d.ts +1 -1
- package/dist/adapters/types.d.ts +7 -0
- package/dist/adapters/types.d.ts.map +1 -1
- package/dist/agent-evaluation/index.d.ts +2 -2
- package/dist/agent-evaluation/scorer.d.ts +1 -1
- package/dist/agent-workflow/catalog.d.ts +28 -0
- package/dist/agent-workflow/catalog.d.ts.map +1 -0
- package/dist/agent-workflow/environment.d.ts +47 -0
- package/dist/agent-workflow/environment.d.ts.map +1 -0
- package/dist/agent-workflow/index.d.ts +10 -0
- package/dist/agent-workflow/index.d.ts.map +1 -0
- package/dist/agent-workflow/parser.d.ts +7 -0
- package/dist/agent-workflow/parser.d.ts.map +1 -0
- package/dist/agent-workflow/sandbox-fixtures/qualify.d.ts +2 -0
- package/dist/agent-workflow/sandbox-fixtures/qualify.d.ts.map +1 -0
- package/dist/agent-workflow/sandbox.d.ts +12 -0
- package/dist/agent-workflow/sandbox.d.ts.map +1 -0
- package/dist/agent-workflow/schema.d.ts +774 -0
- package/dist/agent-workflow/schema.d.ts.map +1 -0
- package/dist/agent-workflow/session.d.ts +113 -0
- package/dist/agent-workflow/session.d.ts.map +1 -0
- package/dist/agent-workflow/simulated-tools.d.ts +35 -0
- package/dist/agent-workflow/simulated-tools.d.ts.map +1 -0
- package/dist/agent-workflow/target.d.ts +243 -0
- package/dist/agent-workflow/target.d.ts.map +1 -0
- package/dist/artifacts/index.d.ts +2 -2
- package/dist/artifacts/manifest.d.ts +1 -1
- package/dist/artifacts/types.d.ts +2 -2
- package/dist/comparison/eligibility.d.ts +1 -1
- package/dist/comparison/index.d.ts +1 -1
- package/dist/evaluators/combined.d.ts +2 -2
- package/dist/evaluators/contains.d.ts +2 -2
- package/dist/evaluators/exact.d.ts +2 -2
- package/dist/evaluators/fuzzy.d.ts +2 -2
- package/dist/evaluators/index.d.ts +13 -13
- package/dist/evaluators/inline.d.ts +2 -2
- package/dist/evaluators/json-schema.d.ts +2 -2
- package/dist/evaluators/llm-grader.d.ts +2 -2
- package/dist/evaluators/not-contains.d.ts +2 -2
- package/dist/evaluators/regex.d.ts +2 -2
- package/dist/evaluators/similarity.d.ts +2 -2
- package/dist/evaluators/tool-trace.d.ts +2 -2
- package/dist/evaluators/types.d.ts +3 -3
- package/dist/index.d.ts +15 -14
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +17827 -15174
- package/dist/provenance/execution-provenance.d.ts +1 -1
- package/dist/provenance/git.d.ts +1 -1
- package/dist/provenance/index.d.ts +4 -4
- package/dist/provenance/workload-identity.d.ts +2 -2
- package/dist/redaction/index.d.ts +2 -2
- package/dist/redaction/redactor.d.ts +1 -1
- package/dist/runner/executor.d.ts +3 -3
- package/dist/runner/index.d.ts +3 -3
- package/dist/runner/runner.d.ts +1 -1
- package/dist/runner/types.d.ts +5 -5
- package/dist/scenario/index.d.ts +4 -4
- package/dist/scenario/parser.d.ts +1 -1
- package/dist/scenario/variables.d.ts +1 -1
- package/dist/storage/factory.d.ts +1 -1
- package/dist/storage/index.d.ts +4 -4
- package/dist/storage/local.d.ts +2 -2
- package/dist/storage/supabase.d.ts +2 -2
- package/dist/storage/types.d.ts +2 -2
- package/dist/tools/fixture-executor.d.ts +2 -2
- package/dist/tools/index.d.ts +3 -3
- package/dist/tools/types.d.ts +1 -1
- package/dist/utils/index.d.ts +2 -2
- package/dist/validator/index.d.ts +2 -2
- package/dist/validator/validator.d.ts +1 -1
- package/package.json +4 -4
- package/src/adapters/types.ts +7 -0
- package/src/agent-workflow/catalog.ts +220 -0
- package/src/agent-workflow/environment.ts +207 -0
- package/src/agent-workflow/index.ts +10 -0
- package/src/agent-workflow/parser.ts +43 -0
- package/src/agent-workflow/sandbox-fixtures/qualify.ts +305 -0
- package/src/agent-workflow/sandbox.test.ts +117 -0
- package/src/agent-workflow/sandbox.ts +438 -0
- package/src/agent-workflow/schema.test.ts +218 -0
- package/src/agent-workflow/schema.ts +270 -0
- package/src/agent-workflow/session.test.ts +629 -0
- package/src/agent-workflow/session.ts +1119 -0
- package/src/agent-workflow/simulated-tools.test.ts +177 -0
- package/src/agent-workflow/simulated-tools.ts +230 -0
- package/src/agent-workflow/target.test.ts +280 -0
- package/src/agent-workflow/target.ts +357 -0
- package/src/index.ts +3 -0
|
@@ -0,0 +1,438 @@
|
|
|
1
|
+
import { spawn } from 'node:child_process';
|
|
2
|
+
import { randomUUID } from 'node:crypto';
|
|
3
|
+
import { getWorkflowTool } from './catalog';
|
|
4
|
+
import {
|
|
5
|
+
type WorkflowEnvironmentFactory,
|
|
6
|
+
WorkflowEnvironmentInitializationError,
|
|
7
|
+
isWorkflowState,
|
|
8
|
+
workflowPathAllowed,
|
|
9
|
+
} from './environment';
|
|
10
|
+
import { AgentWorkflowSchema, isWorkflowJson, isWorkflowRelativePath } from './schema';
|
|
11
|
+
import { type SimulatedToolResult, executeSimulatedTool } from './simulated-tools';
|
|
12
|
+
|
|
13
|
+
export const WORKFLOW_SANDBOX_IMAGE = 'oven/bun:1.3.10-alpine';
|
|
14
|
+
export interface DockerWorkflowEnvironmentOptions {
|
|
15
|
+
/** Per Docker operation. Run/session deadlines can abort sooner. Defaults to 5000. */
|
|
16
|
+
operationTimeoutMs?: number;
|
|
17
|
+
/** Exact owned-container cleanup deadline. Defaults to 3000; failures remain unresolved. */
|
|
18
|
+
cleanupTimeoutMs?: number;
|
|
19
|
+
}
|
|
20
|
+
const LIMIT = 1_048_576;
|
|
21
|
+
const OWNER_LABEL = 'artemiskit.workflow.owner';
|
|
22
|
+
|
|
23
|
+
// Fixed code, never assembled from an agent command, path, file content, or environment variable.
|
|
24
|
+
const FILE_PROGRAM = String.raw`
|
|
25
|
+
const fs = require('node:fs');
|
|
26
|
+
const root = '/workspace';
|
|
27
|
+
const limit = 1048576;
|
|
28
|
+
const safe = p => typeof p === 'string' && p.length <= 512 && /^[A-Za-z0-9_-][A-Za-z0-9_./-]*$/.test(p) && p.split('/').every(x => x && !['.','..','__proto__','prototype','constructor'].includes(x));
|
|
29
|
+
function checked(p, create=false) {
|
|
30
|
+
if (!safe(p)) throw 'invalid_input';
|
|
31
|
+
const parts = p.split('/'); let current = root;
|
|
32
|
+
for (let i=0;i<parts.length;i++) {
|
|
33
|
+
current += '/' + parts[i];
|
|
34
|
+
let stat;
|
|
35
|
+
try { stat = fs.lstatSync(current); } catch(e) { if(e.code !== 'ENOENT') throw 'tool_error'; }
|
|
36
|
+
if (stat && (stat.isSymbolicLink() || (i<parts.length-1 ? !stat.isDirectory() : !stat.isFile()))) throw 'invalid_input';
|
|
37
|
+
if (!stat && i<parts.length-1) { if(!create) throw 'not_found'; fs.mkdirSync(current, {mode: 0o700}); }
|
|
38
|
+
}
|
|
39
|
+
return current;
|
|
40
|
+
}
|
|
41
|
+
function write(p, content) {
|
|
42
|
+
if(typeof content !== 'string' || content.length>16384) throw 'invalid_input';
|
|
43
|
+
const path=checked(p,true); const fd=fs.openSync(path, fs.constants.O_WRONLY|fs.constants.O_CREAT|fs.constants.O_TRUNC|fs.constants.O_NOFOLLOW,0o600);
|
|
44
|
+
try { fs.writeFileSync(fd,content,'utf8'); } finally { fs.closeSync(fd); }
|
|
45
|
+
}
|
|
46
|
+
function read(p) {
|
|
47
|
+
const path=checked(p); let fd;
|
|
48
|
+
try { fd=fs.openSync(path,fs.constants.O_RDONLY|fs.constants.O_NOFOLLOW); } catch(e) { if(e.code==='ENOENT') throw 'not_found'; throw 'tool_error'; }
|
|
49
|
+
try { const stat=fs.fstatSync(fd); if(!stat.isFile() || stat.size>65536) throw 'output_limit'; const text=fs.readFileSync(fd,'utf8'); if(text.length>16384) throw 'output_limit'; return text; } finally { fs.closeSync(fd); }
|
|
50
|
+
}
|
|
51
|
+
function snapshot() {
|
|
52
|
+
const files={}; let count=0, bytes=0;
|
|
53
|
+
function walk(dir, prefix, depth) {
|
|
54
|
+
if(depth>16) throw 'output_limit';
|
|
55
|
+
for(const name of fs.readdirSync(dir).sort()) {
|
|
56
|
+
const p=prefix ? prefix+'/'+name : name;
|
|
57
|
+
if(!safe(p) || ++count>1000) throw 'output_limit';
|
|
58
|
+
const stat=fs.lstatSync(dir+'/'+name);
|
|
59
|
+
if(stat.isSymbolicLink()) throw 'invalid_input';
|
|
60
|
+
if(stat.isDirectory()) walk(dir+'/'+name,p,depth+1);
|
|
61
|
+
else if(stat.isFile()) { const value=read(p); bytes+=Buffer.byteLength(value)+Buffer.byteLength(p); if(bytes>limit) throw 'output_limit'; files[p]=value; }
|
|
62
|
+
else throw 'invalid_input';
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
walk(root,'',0); return files;
|
|
66
|
+
}
|
|
67
|
+
try {
|
|
68
|
+
const raw=await Bun.stdin.text(); if(Buffer.byteLength(raw)>limit) throw 'invalid_input';
|
|
69
|
+
const request=JSON.parse(raw); let output;
|
|
70
|
+
if(request.action==='init') { for(const [path,content] of Object.entries(request.files)) write(path,content); }
|
|
71
|
+
else if(request.action==='write') { write(request.path,request.content); output={path:request.path,written:true}; }
|
|
72
|
+
else if(request.action==='read') output={content:read(request.path)};
|
|
73
|
+
else if(request.action!=='snapshot') throw 'invalid_input';
|
|
74
|
+
const result=JSON.stringify({ok:true,files:snapshot(),...(output?{output}:{})}); if(Buffer.byteLength(result)>limit) throw 'output_limit'; process.stdout.write(result);
|
|
75
|
+
} catch(error) { process.stdout.write(JSON.stringify({ok:false,code:['invalid_input','not_found','output_limit'].includes(error)?error:'tool_error'})); process.exitCode=1; }
|
|
76
|
+
`;
|
|
77
|
+
|
|
78
|
+
class DockerFailure extends Error {
|
|
79
|
+
constructor(
|
|
80
|
+
readonly code: 'aborted' | 'timeout' | 'unavailable' | 'output_limit',
|
|
81
|
+
readonly uncertain = false
|
|
82
|
+
) {
|
|
83
|
+
super(`sandbox_${code}`);
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
interface CommandResult {
|
|
87
|
+
code: number;
|
|
88
|
+
stdout: string;
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
/** Fixed local Docker image and host-owned tools. No command, environment, mount, or image override. */
|
|
92
|
+
export function createDockerWorkflowEnvironmentFactory(
|
|
93
|
+
options: DockerWorkflowEnvironmentOptions = {}
|
|
94
|
+
): WorkflowEnvironmentFactory {
|
|
95
|
+
const operationTimeoutMs = options.operationTimeoutMs ?? 5000;
|
|
96
|
+
const cleanupTimeoutMs = options.cleanupTimeoutMs ?? 3000;
|
|
97
|
+
if (
|
|
98
|
+
Object.keys(options).some((key) => !['operationTimeoutMs', 'cleanupTimeoutMs'].includes(key)) ||
|
|
99
|
+
!Number.isInteger(operationTimeoutMs) ||
|
|
100
|
+
operationTimeoutMs < 1 ||
|
|
101
|
+
operationTimeoutMs > 30_000 ||
|
|
102
|
+
!Number.isInteger(cleanupTimeoutMs) ||
|
|
103
|
+
cleanupTimeoutMs < 1 ||
|
|
104
|
+
cleanupTimeoutMs > 10_000
|
|
105
|
+
)
|
|
106
|
+
throw new TypeError('Invalid Docker workflow environment options');
|
|
107
|
+
return async ({ workflow, initialState, signal }) => {
|
|
108
|
+
const parsed = AgentWorkflowSchema.safeParse(workflow);
|
|
109
|
+
if (
|
|
110
|
+
!parsed.success ||
|
|
111
|
+
parsed.data.environment.type !== 'sandbox' ||
|
|
112
|
+
!isWorkflowState(initialState) ||
|
|
113
|
+
signal.aborted
|
|
114
|
+
)
|
|
115
|
+
throw new WorkflowEnvironmentInitializationError({
|
|
116
|
+
status: 'completed',
|
|
117
|
+
artifacts: 'discarded',
|
|
118
|
+
pendingOperations: 0,
|
|
119
|
+
});
|
|
120
|
+
const configuration = parsed.data;
|
|
121
|
+
let state = structuredClone(initialState);
|
|
122
|
+
const files = state.files ?? {};
|
|
123
|
+
if (
|
|
124
|
+
!isWorkflowState(files) ||
|
|
125
|
+
Object.entries(files).some(
|
|
126
|
+
([path, content]) =>
|
|
127
|
+
!isWorkflowRelativePath(path) || typeof content !== 'string' || content.length > 16_384
|
|
128
|
+
) ||
|
|
129
|
+
Object.keys(files).length > 1000
|
|
130
|
+
)
|
|
131
|
+
throw new WorkflowEnvironmentInitializationError({
|
|
132
|
+
status: 'completed',
|
|
133
|
+
artifacts: 'discarded',
|
|
134
|
+
pendingOperations: 0,
|
|
135
|
+
});
|
|
136
|
+
const owner = randomUUID();
|
|
137
|
+
const name = `artemiskit-workflow-${owner}`;
|
|
138
|
+
const children = new Set<ReturnType<typeof spawn>>();
|
|
139
|
+
let closed = false;
|
|
140
|
+
let closing = false;
|
|
141
|
+
let creationAttempted = false;
|
|
142
|
+
let creationUncertain = false;
|
|
143
|
+
let busy = false;
|
|
144
|
+
const command = (
|
|
145
|
+
args: string[],
|
|
146
|
+
input: string,
|
|
147
|
+
commandSignal: AbortSignal,
|
|
148
|
+
timeoutMs = operationTimeoutMs
|
|
149
|
+
): Promise<CommandResult> => {
|
|
150
|
+
if (commandSignal.aborted) return Promise.reject(new DockerFailure('aborted'));
|
|
151
|
+
if (Buffer.byteLength(input) > LIMIT)
|
|
152
|
+
return Promise.reject(new DockerFailure('output_limit'));
|
|
153
|
+
return new Promise((resolve, reject) => {
|
|
154
|
+
let child: ReturnType<typeof spawn>;
|
|
155
|
+
try {
|
|
156
|
+
child = spawn('docker', args, { stdio: ['pipe', 'pipe', 'pipe'] });
|
|
157
|
+
} catch {
|
|
158
|
+
reject(new DockerFailure('unavailable'));
|
|
159
|
+
return;
|
|
160
|
+
}
|
|
161
|
+
children.add(child);
|
|
162
|
+
const output: Buffer[] = [];
|
|
163
|
+
let size = 0;
|
|
164
|
+
let failure: DockerFailure | undefined;
|
|
165
|
+
let settled = false;
|
|
166
|
+
const stop = (error: DockerFailure) => {
|
|
167
|
+
failure ??= error;
|
|
168
|
+
child.kill('SIGKILL');
|
|
169
|
+
};
|
|
170
|
+
const abort = () => stop(new DockerFailure('aborted', true));
|
|
171
|
+
const timer = setTimeout(() => stop(new DockerFailure('timeout', true)), timeoutMs);
|
|
172
|
+
commandSignal.addEventListener('abort', abort, { once: true });
|
|
173
|
+
child.stdout?.on('data', (chunk: Buffer) => {
|
|
174
|
+
size += chunk.length;
|
|
175
|
+
if (size > LIMIT) stop(new DockerFailure('output_limit', true));
|
|
176
|
+
else output.push(Buffer.from(chunk));
|
|
177
|
+
});
|
|
178
|
+
child.stderr?.on('data', (chunk: Buffer) => {
|
|
179
|
+
size += chunk.length;
|
|
180
|
+
if (size > LIMIT) stop(new DockerFailure('output_limit', true));
|
|
181
|
+
});
|
|
182
|
+
child.stdin?.on('error', () => {
|
|
183
|
+
/* EPIPE is handled by process exit; no raw diagnostics. */
|
|
184
|
+
});
|
|
185
|
+
const finish = (code: number | null, error?: DockerFailure) => {
|
|
186
|
+
if (settled) return;
|
|
187
|
+
settled = true;
|
|
188
|
+
clearTimeout(timer);
|
|
189
|
+
commandSignal.removeEventListener('abort', abort);
|
|
190
|
+
children.delete(child);
|
|
191
|
+
if (error || failure) reject(error ?? failure);
|
|
192
|
+
else resolve({ code: code ?? 1, stdout: Buffer.concat(output).toString('utf8') });
|
|
193
|
+
};
|
|
194
|
+
child.once('error', () => finish(null, new DockerFailure('unavailable')));
|
|
195
|
+
child.once('close', (code) => finish(code));
|
|
196
|
+
child.stdin?.end(input);
|
|
197
|
+
if (commandSignal.aborted) abort();
|
|
198
|
+
});
|
|
199
|
+
};
|
|
200
|
+
async function close(closeSignal: AbortSignal) {
|
|
201
|
+
if (closed) return { status: 'completed' as const, artifacts: 'discarded' as const };
|
|
202
|
+
closing = true;
|
|
203
|
+
for (const child of children) child.kill('SIGKILL');
|
|
204
|
+
const cleanupController = new AbortController();
|
|
205
|
+
const onAbort = () => cleanupController.abort();
|
|
206
|
+
closeSignal.addEventListener('abort', onAbort, { once: true });
|
|
207
|
+
if (closeSignal.aborted) cleanupController.abort();
|
|
208
|
+
const timer = setTimeout(onAbort, cleanupTimeoutMs);
|
|
209
|
+
try {
|
|
210
|
+
if (!creationAttempted) {
|
|
211
|
+
closed = true;
|
|
212
|
+
state = {};
|
|
213
|
+
return { status: 'completed' as const, artifacts: 'discarded' as const };
|
|
214
|
+
}
|
|
215
|
+
const inspected = await command(
|
|
216
|
+
['inspect', '--format', `{{index .Config.Labels "${OWNER_LABEL}"}}`, name],
|
|
217
|
+
'',
|
|
218
|
+
cleanupController.signal,
|
|
219
|
+
cleanupTimeoutMs
|
|
220
|
+
);
|
|
221
|
+
if (inspected.code === 0) {
|
|
222
|
+
if (inspected.stdout.trim() !== owner)
|
|
223
|
+
return { status: 'unresolved' as const, artifacts: 'unknown' as const };
|
|
224
|
+
const removed = await command(
|
|
225
|
+
['rm', '--force', name],
|
|
226
|
+
'',
|
|
227
|
+
cleanupController.signal,
|
|
228
|
+
cleanupTimeoutMs
|
|
229
|
+
);
|
|
230
|
+
if (removed.code !== 0)
|
|
231
|
+
return { status: 'unresolved' as const, artifacts: 'unknown' as const };
|
|
232
|
+
creationUncertain = false;
|
|
233
|
+
}
|
|
234
|
+
const remaining = await command(
|
|
235
|
+
['ps', '--all', '--filter', `name=^/${name}$`, '--format', '{{.Names}}'],
|
|
236
|
+
'',
|
|
237
|
+
cleanupController.signal,
|
|
238
|
+
cleanupTimeoutMs
|
|
239
|
+
);
|
|
240
|
+
if (
|
|
241
|
+
remaining.code !== 0 ||
|
|
242
|
+
remaining.stdout.trim() ||
|
|
243
|
+
creationUncertain ||
|
|
244
|
+
children.size > 0
|
|
245
|
+
)
|
|
246
|
+
return { status: 'unresolved' as const, artifacts: 'unknown' as const };
|
|
247
|
+
closed = true;
|
|
248
|
+
state = {};
|
|
249
|
+
return { status: 'completed' as const, artifacts: 'discarded' as const };
|
|
250
|
+
} catch {
|
|
251
|
+
return { status: 'unresolved' as const, artifacts: 'unknown' as const };
|
|
252
|
+
} finally {
|
|
253
|
+
clearTimeout(timer);
|
|
254
|
+
closeSignal.removeEventListener('abort', onAbort);
|
|
255
|
+
}
|
|
256
|
+
}
|
|
257
|
+
async function fileOperation(request: Record<string, unknown>, operationSignal: AbortSignal) {
|
|
258
|
+
const result = await command(
|
|
259
|
+
['exec', '--interactive', name, 'bun', '--eval', FILE_PROGRAM],
|
|
260
|
+
JSON.stringify(request),
|
|
261
|
+
operationSignal
|
|
262
|
+
);
|
|
263
|
+
let response: unknown;
|
|
264
|
+
try {
|
|
265
|
+
response = JSON.parse(result.stdout);
|
|
266
|
+
} catch {
|
|
267
|
+
throw new DockerFailure('unavailable');
|
|
268
|
+
}
|
|
269
|
+
if (!isWorkflowState(response)) throw new DockerFailure('unavailable');
|
|
270
|
+
if (
|
|
271
|
+
response.ok === false &&
|
|
272
|
+
['invalid_input', 'not_found', 'output_limit', 'tool_error'].includes(String(response.code))
|
|
273
|
+
)
|
|
274
|
+
return {
|
|
275
|
+
ok: false as const,
|
|
276
|
+
code: response.code as 'invalid_input' | 'not_found' | 'output_limit' | 'tool_error',
|
|
277
|
+
};
|
|
278
|
+
if (
|
|
279
|
+
result.code !== 0 ||
|
|
280
|
+
response.ok !== true ||
|
|
281
|
+
!isWorkflowState(response.files) ||
|
|
282
|
+
Object.entries(response.files).some(
|
|
283
|
+
([path, content]) =>
|
|
284
|
+
!isWorkflowRelativePath(path) || typeof content !== 'string' || content.length > 16_384
|
|
285
|
+
)
|
|
286
|
+
)
|
|
287
|
+
throw new DockerFailure('unavailable');
|
|
288
|
+
const next = { ...state, files: response.files };
|
|
289
|
+
if (!isWorkflowState(next)) throw new DockerFailure('output_limit');
|
|
290
|
+
state = structuredClone(next);
|
|
291
|
+
return { ok: true as const, output: response.output };
|
|
292
|
+
}
|
|
293
|
+
try {
|
|
294
|
+
const image = await command(
|
|
295
|
+
['image', 'inspect', WORKFLOW_SANDBOX_IMAGE, '--format', '{{.Id}}'],
|
|
296
|
+
'',
|
|
297
|
+
signal
|
|
298
|
+
);
|
|
299
|
+
if (image.code !== 0 || !/^sha256:[a-f0-9]{64}\s*$/.test(image.stdout))
|
|
300
|
+
throw new DockerFailure('unavailable');
|
|
301
|
+
// Pin the inspected local content ID so a concurrent tag update cannot change the selected image.
|
|
302
|
+
creationAttempted = true;
|
|
303
|
+
let created: CommandResult;
|
|
304
|
+
try {
|
|
305
|
+
created = await command(
|
|
306
|
+
[
|
|
307
|
+
'create',
|
|
308
|
+
'--pull=never',
|
|
309
|
+
'--name',
|
|
310
|
+
name,
|
|
311
|
+
'--label',
|
|
312
|
+
'artemiskit.workflow=true',
|
|
313
|
+
'--label',
|
|
314
|
+
`${OWNER_LABEL}=${owner}`,
|
|
315
|
+
'--network',
|
|
316
|
+
'none',
|
|
317
|
+
'--read-only',
|
|
318
|
+
'--cap-drop',
|
|
319
|
+
'ALL',
|
|
320
|
+
'--security-opt',
|
|
321
|
+
'no-new-privileges',
|
|
322
|
+
'--memory',
|
|
323
|
+
'128m',
|
|
324
|
+
'--memory-swap',
|
|
325
|
+
'128m',
|
|
326
|
+
'--cpus',
|
|
327
|
+
'0.5',
|
|
328
|
+
'--pids-limit',
|
|
329
|
+
'64',
|
|
330
|
+
'--user',
|
|
331
|
+
'1000:1000',
|
|
332
|
+
'--tmpfs',
|
|
333
|
+
'/workspace:rw,noexec,nosuid,nodev,size=16777216,uid=1000,gid=1000,mode=0700',
|
|
334
|
+
'--tmpfs',
|
|
335
|
+
'/tmp:rw,noexec,nosuid,nodev,size=8388608,uid=1000,gid=1000,mode=0700',
|
|
336
|
+
'--workdir',
|
|
337
|
+
'/workspace',
|
|
338
|
+
'--entrypoint',
|
|
339
|
+
'bun',
|
|
340
|
+
image.stdout.trim(),
|
|
341
|
+
'--eval',
|
|
342
|
+
'setInterval(() => {}, 1000)',
|
|
343
|
+
],
|
|
344
|
+
'',
|
|
345
|
+
signal
|
|
346
|
+
);
|
|
347
|
+
} catch (error) {
|
|
348
|
+
creationUncertain = error instanceof DockerFailure && error.uncertain;
|
|
349
|
+
throw error;
|
|
350
|
+
}
|
|
351
|
+
if (created.code !== 0) throw new DockerFailure('unavailable');
|
|
352
|
+
const started = await command(['start', name], '', signal);
|
|
353
|
+
if (started.code !== 0) throw new DockerFailure('unavailable');
|
|
354
|
+
const initialized = await fileOperation({ action: 'init', files }, signal);
|
|
355
|
+
if (!initialized.ok) throw new DockerFailure('unavailable');
|
|
356
|
+
} catch {
|
|
357
|
+
const cleanup = await close(new AbortController().signal);
|
|
358
|
+
throw new WorkflowEnvironmentInitializationError({
|
|
359
|
+
...cleanup,
|
|
360
|
+
pendingOperations: children.size + (creationUncertain ? 1 : 0),
|
|
361
|
+
});
|
|
362
|
+
}
|
|
363
|
+
return {
|
|
364
|
+
type: 'sandbox',
|
|
365
|
+
capabilities: {
|
|
366
|
+
network: 'denied',
|
|
367
|
+
commands: 'denied',
|
|
368
|
+
externalSideEffects: 'denied',
|
|
369
|
+
isolation: 'container',
|
|
370
|
+
},
|
|
371
|
+
async execute(request, operationSignal): Promise<SimulatedToolResult> {
|
|
372
|
+
const tool = getWorkflowTool(request.tool)?.id ?? 'unknown';
|
|
373
|
+
const failure = (
|
|
374
|
+
status: 'denied' | 'invalid' | 'failed',
|
|
375
|
+
code: 'permission_denied' | 'invalid_input' | 'tool_error' | 'not_found' | 'output_limit'
|
|
376
|
+
): SimulatedToolResult => ({
|
|
377
|
+
status,
|
|
378
|
+
code,
|
|
379
|
+
evidence: { tool, version: '1', status, code },
|
|
380
|
+
});
|
|
381
|
+
if (closed || closing || busy || operationSignal.aborted || signal.aborted)
|
|
382
|
+
return failure('failed', 'tool_error');
|
|
383
|
+
const checked = executeSimulatedTool({
|
|
384
|
+
tool: request.tool,
|
|
385
|
+
input: request.input,
|
|
386
|
+
state,
|
|
387
|
+
policy: configuration.environment.policy,
|
|
388
|
+
declaredTools: configuration.tools,
|
|
389
|
+
});
|
|
390
|
+
// Reject malformed original arguments, then apply authority before revealing lookup results.
|
|
391
|
+
if (checked.status === 'invalid' && checked.code === 'invalid_input') return checked;
|
|
392
|
+
if (!workflowPathAllowed(configuration, request.tool, request.input))
|
|
393
|
+
return failure('denied', 'permission_denied');
|
|
394
|
+
if (checked.status !== 'succeeded') return checked;
|
|
395
|
+
busy = true;
|
|
396
|
+
try {
|
|
397
|
+
if (request.tool === 'read_file' || request.tool === 'write_file') {
|
|
398
|
+
if (!isWorkflowState(request.input)) return failure('invalid', 'invalid_input');
|
|
399
|
+
const result = await fileOperation(
|
|
400
|
+
{ action: request.tool === 'read_file' ? 'read' : 'write', ...request.input },
|
|
401
|
+
operationSignal
|
|
402
|
+
);
|
|
403
|
+
if (!result.ok)
|
|
404
|
+
return failure(result.code === 'invalid_input' ? 'invalid' : 'failed', result.code);
|
|
405
|
+
if (!isWorkflowJson(result.output)) return failure('failed', 'tool_error');
|
|
406
|
+
return {
|
|
407
|
+
status: 'succeeded',
|
|
408
|
+
output: structuredClone(result.output),
|
|
409
|
+
state: structuredClone(state),
|
|
410
|
+
evidence: { tool, version: '1', status: 'succeeded' },
|
|
411
|
+
};
|
|
412
|
+
}
|
|
413
|
+
state = structuredClone(checked.state);
|
|
414
|
+
return checked;
|
|
415
|
+
} catch {
|
|
416
|
+
return failure('failed', 'tool_error');
|
|
417
|
+
} finally {
|
|
418
|
+
busy = false;
|
|
419
|
+
}
|
|
420
|
+
},
|
|
421
|
+
async snapshot(snapshotSignal) {
|
|
422
|
+
if (closed || closing || busy || snapshotSignal.aborted)
|
|
423
|
+
throw new DockerFailure('unavailable');
|
|
424
|
+
busy = true;
|
|
425
|
+
try {
|
|
426
|
+
const result = await fileOperation({ action: 'snapshot' }, snapshotSignal);
|
|
427
|
+
if (!result.ok) throw new DockerFailure('unavailable');
|
|
428
|
+
return structuredClone(state);
|
|
429
|
+
} finally {
|
|
430
|
+
busy = false;
|
|
431
|
+
}
|
|
432
|
+
},
|
|
433
|
+
close,
|
|
434
|
+
};
|
|
435
|
+
};
|
|
436
|
+
}
|
|
437
|
+
export const createDockerWorkflowEnvironment: WorkflowEnvironmentFactory =
|
|
438
|
+
createDockerWorkflowEnvironmentFactory();
|
|
@@ -0,0 +1,218 @@
|
|
|
1
|
+
import { describe, expect, test } from 'bun:test';
|
|
2
|
+
import { mkdtemp, rm, writeFile } from 'node:fs/promises';
|
|
3
|
+
import { tmpdir } from 'node:os';
|
|
4
|
+
import { join } from 'node:path';
|
|
5
|
+
import { stringify } from 'yaml';
|
|
6
|
+
import { loadAgentWorkflow, parseAgentWorkflow, validateAgentWorkflow } from './parser';
|
|
7
|
+
import { AgentWorkflowSchema, isWorkflowJson } from './schema';
|
|
8
|
+
|
|
9
|
+
function fixture() {
|
|
10
|
+
return {
|
|
11
|
+
version: '1',
|
|
12
|
+
kind: 'agent_workflow',
|
|
13
|
+
name: 'document-review',
|
|
14
|
+
target: { provider: 'openai', model: 'configured-by-runner' },
|
|
15
|
+
environment: {
|
|
16
|
+
type: 'simulated',
|
|
17
|
+
policy: {
|
|
18
|
+
network: 'denied',
|
|
19
|
+
side_effects: 'approval_required',
|
|
20
|
+
permissions: { documents: 'read', records: 'read', workflow_state: 'write' },
|
|
21
|
+
budgets: { max_actions: 10, max_tool_calls: 12, timeout_ms: 60_000, max_tokens: 4000 },
|
|
22
|
+
},
|
|
23
|
+
},
|
|
24
|
+
tools: ['search', 'read_document', 'query_records', 'request_approval', 'record_decision'],
|
|
25
|
+
workflow: {
|
|
26
|
+
system_instructions: 'Request review when uncertain.',
|
|
27
|
+
initial_state: 'fixtures/review-state.yaml',
|
|
28
|
+
turns: [
|
|
29
|
+
{ role: 'user', content: 'Review material.' },
|
|
30
|
+
{ role: 'user', content: 'Explain the handoff.' },
|
|
31
|
+
],
|
|
32
|
+
},
|
|
33
|
+
outcomes: {
|
|
34
|
+
deterministic: [
|
|
35
|
+
{ type: 'workflow_state', path: 'approvals.requested', equals: true },
|
|
36
|
+
{ type: 'tool_trace', tool: 'request_approval', minimum_calls: 1 },
|
|
37
|
+
{ type: 'policy', rule: 'no_undeclared_tool', expected: 'passed' },
|
|
38
|
+
],
|
|
39
|
+
semantic: [{ type: 'llm_judge', rubric: 'Explains uncertainty.', mode: 'strict_assurance' }],
|
|
40
|
+
},
|
|
41
|
+
evidence: { trace: 'summary', artifacts: 'checksums', redact: true },
|
|
42
|
+
};
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
describe('agent workflow v1 contract', () => {
|
|
46
|
+
test('loads only the selected scenario without reading referenced fixtures', async () => {
|
|
47
|
+
const directory = await mkdtemp(join(tmpdir(), 'artemiskit-workflow-'));
|
|
48
|
+
try {
|
|
49
|
+
const path = join(directory, 'scenario.yaml');
|
|
50
|
+
await writeFile(path, stringify(fixture()));
|
|
51
|
+
expect((await loadAgentWorkflow(path)).workflow.initial_state).toBe(
|
|
52
|
+
'fixtures/review-state.yaml'
|
|
53
|
+
);
|
|
54
|
+
await expect(loadAgentWorkflow(join(directory, 'missing.yaml'))).rejects.toThrow(
|
|
55
|
+
'Failed to read'
|
|
56
|
+
);
|
|
57
|
+
} finally {
|
|
58
|
+
await rm(directory, { recursive: true, force: true });
|
|
59
|
+
}
|
|
60
|
+
});
|
|
61
|
+
test('parses reviewable multi-turn YAML and keeps fixture references unresolved', () => {
|
|
62
|
+
expect(parseAgentWorkflow(stringify(fixture()))).toEqual(fixture());
|
|
63
|
+
});
|
|
64
|
+
test('preserves readable Unicode and punctuation in names without accepting blank names', () => {
|
|
65
|
+
for (const name of ['review: [draft]', 'Àyẹ̀wò ìwé', 'مراجعة المستند']) {
|
|
66
|
+
expect(parseAgentWorkflow(stringify({ ...fixture(), name })).name).toBe(name);
|
|
67
|
+
}
|
|
68
|
+
expect(AgentWorkflowSchema.safeParse({ ...fixture(), name: ' ' }).success).toBe(false);
|
|
69
|
+
});
|
|
70
|
+
test('accepts inline controlled JSON state', () => {
|
|
71
|
+
const value = {
|
|
72
|
+
...fixture(),
|
|
73
|
+
workflow: {
|
|
74
|
+
...fixture().workflow,
|
|
75
|
+
initial_state: { documents: { policy: 'Human review required' }, workflow_state: {} },
|
|
76
|
+
},
|
|
77
|
+
};
|
|
78
|
+
expect(validateAgentWorkflow(value).workflow.initial_state).toEqual(
|
|
79
|
+
value.workflow.initial_state
|
|
80
|
+
);
|
|
81
|
+
});
|
|
82
|
+
test('does not expand secrets from environment variables', () => {
|
|
83
|
+
const value = fixture();
|
|
84
|
+
value.workflow.system_instructions = '${SECRET_TOKEN}';
|
|
85
|
+
expect(parseAgentWorkflow(stringify(value)).workflow.system_instructions).toBe(
|
|
86
|
+
'${SECRET_TOKEN}'
|
|
87
|
+
);
|
|
88
|
+
});
|
|
89
|
+
test.each([
|
|
90
|
+
'../private.yaml',
|
|
91
|
+
'/tmp/private.yaml',
|
|
92
|
+
'https://example.com/file',
|
|
93
|
+
'C:\\file',
|
|
94
|
+
'a/../../x',
|
|
95
|
+
'a//b',
|
|
96
|
+
'a/constructor/b',
|
|
97
|
+
'a/./b',
|
|
98
|
+
])('rejects unsafe fixture path %s', (path) => {
|
|
99
|
+
const value = fixture();
|
|
100
|
+
value.workflow.initial_state = path;
|
|
101
|
+
expect(() => validateAgentWorkflow(value)).toThrow();
|
|
102
|
+
});
|
|
103
|
+
test.each(['constructor.x', '__proto__.x', 'x.prototype', 'a..b', '/absolute', 'x[0]'])(
|
|
104
|
+
'rejects unsafe outcome path %s',
|
|
105
|
+
(path) => {
|
|
106
|
+
const value = fixture();
|
|
107
|
+
value.outcomes.deterministic[0].path = path;
|
|
108
|
+
expect(() => validateAgentWorkflow(value)).toThrow();
|
|
109
|
+
}
|
|
110
|
+
);
|
|
111
|
+
test('rejects unknown fields, versions, tools, duplicate tools and ungranted authority', () => {
|
|
112
|
+
expect(AgentWorkflowSchema.safeParse({ ...fixture(), hidden: true }).success).toBe(false);
|
|
113
|
+
expect(AgentWorkflowSchema.safeParse({ ...fixture(), version: '2' }).success).toBe(false);
|
|
114
|
+
expect(AgentWorkflowSchema.safeParse({ ...fixture(), tools: ['send_message'] }).success).toBe(
|
|
115
|
+
false
|
|
116
|
+
);
|
|
117
|
+
expect(
|
|
118
|
+
AgentWorkflowSchema.safeParse({ ...fixture(), tools: ['search', 'search'] }).success
|
|
119
|
+
).toBe(false);
|
|
120
|
+
expect(AgentWorkflowSchema.safeParse({ ...fixture(), tools: ['write_file'] }).success).toBe(
|
|
121
|
+
false
|
|
122
|
+
);
|
|
123
|
+
const value = fixture();
|
|
124
|
+
value.target = { ...value.target, secret: 'not-allowed' } as typeof value.target;
|
|
125
|
+
expect(AgentWorkflowSchema.safeParse(value).success).toBe(false);
|
|
126
|
+
});
|
|
127
|
+
test('requires independent deterministic evidence and strict-only semantic judging', () => {
|
|
128
|
+
const value = fixture();
|
|
129
|
+
value.outcomes.deterministic = [];
|
|
130
|
+
expect(AgentWorkflowSchema.safeParse(value).success).toBe(false);
|
|
131
|
+
const other = fixture();
|
|
132
|
+
other.outcomes.semantic[0].mode = 'legacy';
|
|
133
|
+
expect(AgentWorkflowSchema.safeParse(other).success).toBe(false);
|
|
134
|
+
});
|
|
135
|
+
test('rejects inconsistent outcome references and call bounds', () => {
|
|
136
|
+
const value = fixture();
|
|
137
|
+
value.tools = ['search'];
|
|
138
|
+
expect(AgentWorkflowSchema.safeParse(value).success).toBe(false);
|
|
139
|
+
const other = fixture();
|
|
140
|
+
other.outcomes.deterministic = [
|
|
141
|
+
{
|
|
142
|
+
type: 'tool_trace',
|
|
143
|
+
tool: 'search',
|
|
144
|
+
minimum_calls: 2,
|
|
145
|
+
maximum_calls: 1,
|
|
146
|
+
} as (typeof other.outcomes.deterministic)[number],
|
|
147
|
+
];
|
|
148
|
+
expect(AgentWorkflowSchema.safeParse(other).success).toBe(false);
|
|
149
|
+
expect(
|
|
150
|
+
AgentWorkflowSchema.safeParse({
|
|
151
|
+
...fixture(),
|
|
152
|
+
outcomes: {
|
|
153
|
+
deterministic: [{ type: 'file', path: 'note.txt', exists: false, equals: 'impossible' }],
|
|
154
|
+
},
|
|
155
|
+
}).success
|
|
156
|
+
).toBe(false);
|
|
157
|
+
});
|
|
158
|
+
test.each([0, -1, 1001, Number.POSITIVE_INFINITY])(
|
|
159
|
+
'rejects invalid action budget %s',
|
|
160
|
+
(count) => {
|
|
161
|
+
const value = fixture();
|
|
162
|
+
value.environment.policy.budgets.max_actions = count;
|
|
163
|
+
expect(AgentWorkflowSchema.safeParse(value).success).toBe(false);
|
|
164
|
+
}
|
|
165
|
+
);
|
|
166
|
+
test('rejects unsupported environments, side effects, network grants and unredacted evidence', () => {
|
|
167
|
+
for (const change of [
|
|
168
|
+
{ network: 'allowed' },
|
|
169
|
+
{ side_effects: 'allowed' },
|
|
170
|
+
{ commands: ['sh'] },
|
|
171
|
+
]) {
|
|
172
|
+
const value = fixture();
|
|
173
|
+
Object.assign(value.environment.policy, change);
|
|
174
|
+
expect(AgentWorkflowSchema.safeParse(value).success).toBe(false);
|
|
175
|
+
}
|
|
176
|
+
const value = fixture();
|
|
177
|
+
value.environment.type = 'external';
|
|
178
|
+
expect(AgentWorkflowSchema.safeParse(value).success).toBe(false);
|
|
179
|
+
const other = fixture();
|
|
180
|
+
other.evidence.redact = false;
|
|
181
|
+
expect(AgentWorkflowSchema.safeParse(other).success).toBe(false);
|
|
182
|
+
});
|
|
183
|
+
test('rejects prototype keys, accessors, cycles, sparse arrays and non-JSON values', () => {
|
|
184
|
+
expect(isWorkflowJson(JSON.parse('{"__proto__":{"polluted":true}}'))).toBe(false);
|
|
185
|
+
expect(isWorkflowJson({ nested: { constructor: 'unsafe' } })).toBe(false);
|
|
186
|
+
expect(
|
|
187
|
+
isWorkflowJson({
|
|
188
|
+
get secret() {
|
|
189
|
+
throw new Error('must not run');
|
|
190
|
+
},
|
|
191
|
+
})
|
|
192
|
+
).toBe(false);
|
|
193
|
+
const cyclic: Record<string, unknown> = {};
|
|
194
|
+
cyclic.self = cyclic;
|
|
195
|
+
expect(isWorkflowJson(cyclic)).toBe(false);
|
|
196
|
+
expect(isWorkflowJson(new Array(1_000_000))).toBe(false);
|
|
197
|
+
expect(isWorkflowJson({ date: new Date() })).toBe(false);
|
|
198
|
+
expect(isWorkflowJson({ missing: undefined })).toBe(false);
|
|
199
|
+
expect(isWorkflowJson({ enormous: 'x'.repeat(1_048_577) })).toBe(false);
|
|
200
|
+
});
|
|
201
|
+
test('rejects malformed/duplicate YAML, custom tags, aliases and oversized input', () => {
|
|
202
|
+
for (const yaml of [
|
|
203
|
+
'name: [',
|
|
204
|
+
'name: a\nname: b',
|
|
205
|
+
'value: !secret abc',
|
|
206
|
+
'value: &x [1]\nother: *x',
|
|
207
|
+
'x'.repeat(1_048_577),
|
|
208
|
+
])
|
|
209
|
+
expect(() => parseAgentWorkflow(yaml)).toThrow();
|
|
210
|
+
});
|
|
211
|
+
test('diagnostics do not echo malicious YAML or raw secret values', () => {
|
|
212
|
+
try {
|
|
213
|
+
parseAgentWorkflow('secret: [DO-NOT-LEAK');
|
|
214
|
+
} catch (error) {
|
|
215
|
+
expect(String(error)).not.toContain('DO-NOT-LEAK');
|
|
216
|
+
}
|
|
217
|
+
});
|
|
218
|
+
});
|