@artemiskit/core 0.5.3 → 0.6.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (92) hide show
  1. package/CHANGELOG.md +28 -0
  2. package/README.md +12 -0
  3. package/dist/adapters/factory.d.ts +1 -1
  4. package/dist/adapters/index.d.ts +3 -3
  5. package/dist/adapters/registry.d.ts +1 -1
  6. package/dist/adapters/types.d.ts +7 -0
  7. package/dist/adapters/types.d.ts.map +1 -1
  8. package/dist/agent-evaluation/index.d.ts +2 -2
  9. package/dist/agent-evaluation/scorer.d.ts +1 -1
  10. package/dist/agent-workflow/catalog.d.ts +28 -0
  11. package/dist/agent-workflow/catalog.d.ts.map +1 -0
  12. package/dist/agent-workflow/environment.d.ts +47 -0
  13. package/dist/agent-workflow/environment.d.ts.map +1 -0
  14. package/dist/agent-workflow/index.d.ts +10 -0
  15. package/dist/agent-workflow/index.d.ts.map +1 -0
  16. package/dist/agent-workflow/parser.d.ts +7 -0
  17. package/dist/agent-workflow/parser.d.ts.map +1 -0
  18. package/dist/agent-workflow/sandbox-fixtures/qualify.d.ts +2 -0
  19. package/dist/agent-workflow/sandbox-fixtures/qualify.d.ts.map +1 -0
  20. package/dist/agent-workflow/sandbox.d.ts +12 -0
  21. package/dist/agent-workflow/sandbox.d.ts.map +1 -0
  22. package/dist/agent-workflow/schema.d.ts +774 -0
  23. package/dist/agent-workflow/schema.d.ts.map +1 -0
  24. package/dist/agent-workflow/session.d.ts +113 -0
  25. package/dist/agent-workflow/session.d.ts.map +1 -0
  26. package/dist/agent-workflow/simulated-tools.d.ts +35 -0
  27. package/dist/agent-workflow/simulated-tools.d.ts.map +1 -0
  28. package/dist/agent-workflow/target.d.ts +243 -0
  29. package/dist/agent-workflow/target.d.ts.map +1 -0
  30. package/dist/artifacts/index.d.ts +2 -2
  31. package/dist/artifacts/manifest.d.ts +1 -1
  32. package/dist/artifacts/types.d.ts +2 -2
  33. package/dist/comparison/eligibility.d.ts +1 -1
  34. package/dist/comparison/index.d.ts +1 -1
  35. package/dist/evaluators/combined.d.ts +2 -2
  36. package/dist/evaluators/contains.d.ts +2 -2
  37. package/dist/evaluators/exact.d.ts +2 -2
  38. package/dist/evaluators/fuzzy.d.ts +2 -2
  39. package/dist/evaluators/index.d.ts +13 -13
  40. package/dist/evaluators/inline.d.ts +2 -2
  41. package/dist/evaluators/json-schema.d.ts +2 -2
  42. package/dist/evaluators/llm-grader.d.ts +2 -2
  43. package/dist/evaluators/not-contains.d.ts +2 -2
  44. package/dist/evaluators/regex.d.ts +2 -2
  45. package/dist/evaluators/similarity.d.ts +2 -2
  46. package/dist/evaluators/tool-trace.d.ts +2 -2
  47. package/dist/evaluators/types.d.ts +3 -3
  48. package/dist/index.d.ts +15 -14
  49. package/dist/index.d.ts.map +1 -1
  50. package/dist/index.js +17827 -15174
  51. package/dist/provenance/execution-provenance.d.ts +1 -1
  52. package/dist/provenance/git.d.ts +1 -1
  53. package/dist/provenance/index.d.ts +4 -4
  54. package/dist/provenance/workload-identity.d.ts +2 -2
  55. package/dist/redaction/index.d.ts +2 -2
  56. package/dist/redaction/redactor.d.ts +1 -1
  57. package/dist/runner/executor.d.ts +3 -3
  58. package/dist/runner/index.d.ts +3 -3
  59. package/dist/runner/runner.d.ts +1 -1
  60. package/dist/runner/types.d.ts +5 -5
  61. package/dist/scenario/index.d.ts +4 -4
  62. package/dist/scenario/parser.d.ts +1 -1
  63. package/dist/scenario/variables.d.ts +1 -1
  64. package/dist/storage/factory.d.ts +1 -1
  65. package/dist/storage/index.d.ts +4 -4
  66. package/dist/storage/local.d.ts +2 -2
  67. package/dist/storage/supabase.d.ts +2 -2
  68. package/dist/storage/types.d.ts +2 -2
  69. package/dist/tools/fixture-executor.d.ts +2 -2
  70. package/dist/tools/index.d.ts +3 -3
  71. package/dist/tools/types.d.ts +1 -1
  72. package/dist/utils/index.d.ts +2 -2
  73. package/dist/validator/index.d.ts +2 -2
  74. package/dist/validator/validator.d.ts +1 -1
  75. package/package.json +4 -4
  76. package/src/adapters/types.ts +7 -0
  77. package/src/agent-workflow/catalog.ts +220 -0
  78. package/src/agent-workflow/environment.ts +207 -0
  79. package/src/agent-workflow/index.ts +10 -0
  80. package/src/agent-workflow/parser.ts +43 -0
  81. package/src/agent-workflow/sandbox-fixtures/qualify.ts +305 -0
  82. package/src/agent-workflow/sandbox.test.ts +117 -0
  83. package/src/agent-workflow/sandbox.ts +438 -0
  84. package/src/agent-workflow/schema.test.ts +218 -0
  85. package/src/agent-workflow/schema.ts +270 -0
  86. package/src/agent-workflow/session.test.ts +629 -0
  87. package/src/agent-workflow/session.ts +1119 -0
  88. package/src/agent-workflow/simulated-tools.test.ts +177 -0
  89. package/src/agent-workflow/simulated-tools.ts +230 -0
  90. package/src/agent-workflow/target.test.ts +280 -0
  91. package/src/agent-workflow/target.ts +357 -0
  92. package/src/index.ts +3 -0
@@ -0,0 +1,438 @@
1
+ import { spawn } from 'node:child_process';
2
+ import { randomUUID } from 'node:crypto';
3
+ import { getWorkflowTool } from './catalog';
4
+ import {
5
+ type WorkflowEnvironmentFactory,
6
+ WorkflowEnvironmentInitializationError,
7
+ isWorkflowState,
8
+ workflowPathAllowed,
9
+ } from './environment';
10
+ import { AgentWorkflowSchema, isWorkflowJson, isWorkflowRelativePath } from './schema';
11
+ import { type SimulatedToolResult, executeSimulatedTool } from './simulated-tools';
12
+
13
+ export const WORKFLOW_SANDBOX_IMAGE = 'oven/bun:1.3.10-alpine';
14
+ export interface DockerWorkflowEnvironmentOptions {
15
+ /** Per Docker operation. Run/session deadlines can abort sooner. Defaults to 5000. */
16
+ operationTimeoutMs?: number;
17
+ /** Exact owned-container cleanup deadline. Defaults to 3000; failures remain unresolved. */
18
+ cleanupTimeoutMs?: number;
19
+ }
20
+ const LIMIT = 1_048_576;
21
+ const OWNER_LABEL = 'artemiskit.workflow.owner';
22
+
23
+ // Fixed code, never assembled from an agent command, path, file content, or environment variable.
24
+ const FILE_PROGRAM = String.raw`
25
+ const fs = require('node:fs');
26
+ const root = '/workspace';
27
+ const limit = 1048576;
28
+ const safe = p => typeof p === 'string' && p.length <= 512 && /^[A-Za-z0-9_-][A-Za-z0-9_./-]*$/.test(p) && p.split('/').every(x => x && !['.','..','__proto__','prototype','constructor'].includes(x));
29
+ function checked(p, create=false) {
30
+ if (!safe(p)) throw 'invalid_input';
31
+ const parts = p.split('/'); let current = root;
32
+ for (let i=0;i<parts.length;i++) {
33
+ current += '/' + parts[i];
34
+ let stat;
35
+ try { stat = fs.lstatSync(current); } catch(e) { if(e.code !== 'ENOENT') throw 'tool_error'; }
36
+ if (stat && (stat.isSymbolicLink() || (i<parts.length-1 ? !stat.isDirectory() : !stat.isFile()))) throw 'invalid_input';
37
+ if (!stat && i<parts.length-1) { if(!create) throw 'not_found'; fs.mkdirSync(current, {mode: 0o700}); }
38
+ }
39
+ return current;
40
+ }
41
+ function write(p, content) {
42
+ if(typeof content !== 'string' || content.length>16384) throw 'invalid_input';
43
+ const path=checked(p,true); const fd=fs.openSync(path, fs.constants.O_WRONLY|fs.constants.O_CREAT|fs.constants.O_TRUNC|fs.constants.O_NOFOLLOW,0o600);
44
+ try { fs.writeFileSync(fd,content,'utf8'); } finally { fs.closeSync(fd); }
45
+ }
46
+ function read(p) {
47
+ const path=checked(p); let fd;
48
+ try { fd=fs.openSync(path,fs.constants.O_RDONLY|fs.constants.O_NOFOLLOW); } catch(e) { if(e.code==='ENOENT') throw 'not_found'; throw 'tool_error'; }
49
+ try { const stat=fs.fstatSync(fd); if(!stat.isFile() || stat.size>65536) throw 'output_limit'; const text=fs.readFileSync(fd,'utf8'); if(text.length>16384) throw 'output_limit'; return text; } finally { fs.closeSync(fd); }
50
+ }
51
+ function snapshot() {
52
+ const files={}; let count=0, bytes=0;
53
+ function walk(dir, prefix, depth) {
54
+ if(depth>16) throw 'output_limit';
55
+ for(const name of fs.readdirSync(dir).sort()) {
56
+ const p=prefix ? prefix+'/'+name : name;
57
+ if(!safe(p) || ++count>1000) throw 'output_limit';
58
+ const stat=fs.lstatSync(dir+'/'+name);
59
+ if(stat.isSymbolicLink()) throw 'invalid_input';
60
+ if(stat.isDirectory()) walk(dir+'/'+name,p,depth+1);
61
+ else if(stat.isFile()) { const value=read(p); bytes+=Buffer.byteLength(value)+Buffer.byteLength(p); if(bytes>limit) throw 'output_limit'; files[p]=value; }
62
+ else throw 'invalid_input';
63
+ }
64
+ }
65
+ walk(root,'',0); return files;
66
+ }
67
+ try {
68
+ const raw=await Bun.stdin.text(); if(Buffer.byteLength(raw)>limit) throw 'invalid_input';
69
+ const request=JSON.parse(raw); let output;
70
+ if(request.action==='init') { for(const [path,content] of Object.entries(request.files)) write(path,content); }
71
+ else if(request.action==='write') { write(request.path,request.content); output={path:request.path,written:true}; }
72
+ else if(request.action==='read') output={content:read(request.path)};
73
+ else if(request.action!=='snapshot') throw 'invalid_input';
74
+ const result=JSON.stringify({ok:true,files:snapshot(),...(output?{output}:{})}); if(Buffer.byteLength(result)>limit) throw 'output_limit'; process.stdout.write(result);
75
+ } catch(error) { process.stdout.write(JSON.stringify({ok:false,code:['invalid_input','not_found','output_limit'].includes(error)?error:'tool_error'})); process.exitCode=1; }
76
+ `;
77
+
78
+ class DockerFailure extends Error {
79
+ constructor(
80
+ readonly code: 'aborted' | 'timeout' | 'unavailable' | 'output_limit',
81
+ readonly uncertain = false
82
+ ) {
83
+ super(`sandbox_${code}`);
84
+ }
85
+ }
86
+ interface CommandResult {
87
+ code: number;
88
+ stdout: string;
89
+ }
90
+
91
+ /** Fixed local Docker image and host-owned tools. No command, environment, mount, or image override. */
92
+ export function createDockerWorkflowEnvironmentFactory(
93
+ options: DockerWorkflowEnvironmentOptions = {}
94
+ ): WorkflowEnvironmentFactory {
95
+ const operationTimeoutMs = options.operationTimeoutMs ?? 5000;
96
+ const cleanupTimeoutMs = options.cleanupTimeoutMs ?? 3000;
97
+ if (
98
+ Object.keys(options).some((key) => !['operationTimeoutMs', 'cleanupTimeoutMs'].includes(key)) ||
99
+ !Number.isInteger(operationTimeoutMs) ||
100
+ operationTimeoutMs < 1 ||
101
+ operationTimeoutMs > 30_000 ||
102
+ !Number.isInteger(cleanupTimeoutMs) ||
103
+ cleanupTimeoutMs < 1 ||
104
+ cleanupTimeoutMs > 10_000
105
+ )
106
+ throw new TypeError('Invalid Docker workflow environment options');
107
+ return async ({ workflow, initialState, signal }) => {
108
+ const parsed = AgentWorkflowSchema.safeParse(workflow);
109
+ if (
110
+ !parsed.success ||
111
+ parsed.data.environment.type !== 'sandbox' ||
112
+ !isWorkflowState(initialState) ||
113
+ signal.aborted
114
+ )
115
+ throw new WorkflowEnvironmentInitializationError({
116
+ status: 'completed',
117
+ artifacts: 'discarded',
118
+ pendingOperations: 0,
119
+ });
120
+ const configuration = parsed.data;
121
+ let state = structuredClone(initialState);
122
+ const files = state.files ?? {};
123
+ if (
124
+ !isWorkflowState(files) ||
125
+ Object.entries(files).some(
126
+ ([path, content]) =>
127
+ !isWorkflowRelativePath(path) || typeof content !== 'string' || content.length > 16_384
128
+ ) ||
129
+ Object.keys(files).length > 1000
130
+ )
131
+ throw new WorkflowEnvironmentInitializationError({
132
+ status: 'completed',
133
+ artifacts: 'discarded',
134
+ pendingOperations: 0,
135
+ });
136
+ const owner = randomUUID();
137
+ const name = `artemiskit-workflow-${owner}`;
138
+ const children = new Set<ReturnType<typeof spawn>>();
139
+ let closed = false;
140
+ let closing = false;
141
+ let creationAttempted = false;
142
+ let creationUncertain = false;
143
+ let busy = false;
144
+ const command = (
145
+ args: string[],
146
+ input: string,
147
+ commandSignal: AbortSignal,
148
+ timeoutMs = operationTimeoutMs
149
+ ): Promise<CommandResult> => {
150
+ if (commandSignal.aborted) return Promise.reject(new DockerFailure('aborted'));
151
+ if (Buffer.byteLength(input) > LIMIT)
152
+ return Promise.reject(new DockerFailure('output_limit'));
153
+ return new Promise((resolve, reject) => {
154
+ let child: ReturnType<typeof spawn>;
155
+ try {
156
+ child = spawn('docker', args, { stdio: ['pipe', 'pipe', 'pipe'] });
157
+ } catch {
158
+ reject(new DockerFailure('unavailable'));
159
+ return;
160
+ }
161
+ children.add(child);
162
+ const output: Buffer[] = [];
163
+ let size = 0;
164
+ let failure: DockerFailure | undefined;
165
+ let settled = false;
166
+ const stop = (error: DockerFailure) => {
167
+ failure ??= error;
168
+ child.kill('SIGKILL');
169
+ };
170
+ const abort = () => stop(new DockerFailure('aborted', true));
171
+ const timer = setTimeout(() => stop(new DockerFailure('timeout', true)), timeoutMs);
172
+ commandSignal.addEventListener('abort', abort, { once: true });
173
+ child.stdout?.on('data', (chunk: Buffer) => {
174
+ size += chunk.length;
175
+ if (size > LIMIT) stop(new DockerFailure('output_limit', true));
176
+ else output.push(Buffer.from(chunk));
177
+ });
178
+ child.stderr?.on('data', (chunk: Buffer) => {
179
+ size += chunk.length;
180
+ if (size > LIMIT) stop(new DockerFailure('output_limit', true));
181
+ });
182
+ child.stdin?.on('error', () => {
183
+ /* EPIPE is handled by process exit; no raw diagnostics. */
184
+ });
185
+ const finish = (code: number | null, error?: DockerFailure) => {
186
+ if (settled) return;
187
+ settled = true;
188
+ clearTimeout(timer);
189
+ commandSignal.removeEventListener('abort', abort);
190
+ children.delete(child);
191
+ if (error || failure) reject(error ?? failure);
192
+ else resolve({ code: code ?? 1, stdout: Buffer.concat(output).toString('utf8') });
193
+ };
194
+ child.once('error', () => finish(null, new DockerFailure('unavailable')));
195
+ child.once('close', (code) => finish(code));
196
+ child.stdin?.end(input);
197
+ if (commandSignal.aborted) abort();
198
+ });
199
+ };
200
+ async function close(closeSignal: AbortSignal) {
201
+ if (closed) return { status: 'completed' as const, artifacts: 'discarded' as const };
202
+ closing = true;
203
+ for (const child of children) child.kill('SIGKILL');
204
+ const cleanupController = new AbortController();
205
+ const onAbort = () => cleanupController.abort();
206
+ closeSignal.addEventListener('abort', onAbort, { once: true });
207
+ if (closeSignal.aborted) cleanupController.abort();
208
+ const timer = setTimeout(onAbort, cleanupTimeoutMs);
209
+ try {
210
+ if (!creationAttempted) {
211
+ closed = true;
212
+ state = {};
213
+ return { status: 'completed' as const, artifacts: 'discarded' as const };
214
+ }
215
+ const inspected = await command(
216
+ ['inspect', '--format', `{{index .Config.Labels "${OWNER_LABEL}"}}`, name],
217
+ '',
218
+ cleanupController.signal,
219
+ cleanupTimeoutMs
220
+ );
221
+ if (inspected.code === 0) {
222
+ if (inspected.stdout.trim() !== owner)
223
+ return { status: 'unresolved' as const, artifacts: 'unknown' as const };
224
+ const removed = await command(
225
+ ['rm', '--force', name],
226
+ '',
227
+ cleanupController.signal,
228
+ cleanupTimeoutMs
229
+ );
230
+ if (removed.code !== 0)
231
+ return { status: 'unresolved' as const, artifacts: 'unknown' as const };
232
+ creationUncertain = false;
233
+ }
234
+ const remaining = await command(
235
+ ['ps', '--all', '--filter', `name=^/${name}$`, '--format', '{{.Names}}'],
236
+ '',
237
+ cleanupController.signal,
238
+ cleanupTimeoutMs
239
+ );
240
+ if (
241
+ remaining.code !== 0 ||
242
+ remaining.stdout.trim() ||
243
+ creationUncertain ||
244
+ children.size > 0
245
+ )
246
+ return { status: 'unresolved' as const, artifacts: 'unknown' as const };
247
+ closed = true;
248
+ state = {};
249
+ return { status: 'completed' as const, artifacts: 'discarded' as const };
250
+ } catch {
251
+ return { status: 'unresolved' as const, artifacts: 'unknown' as const };
252
+ } finally {
253
+ clearTimeout(timer);
254
+ closeSignal.removeEventListener('abort', onAbort);
255
+ }
256
+ }
257
+ async function fileOperation(request: Record<string, unknown>, operationSignal: AbortSignal) {
258
+ const result = await command(
259
+ ['exec', '--interactive', name, 'bun', '--eval', FILE_PROGRAM],
260
+ JSON.stringify(request),
261
+ operationSignal
262
+ );
263
+ let response: unknown;
264
+ try {
265
+ response = JSON.parse(result.stdout);
266
+ } catch {
267
+ throw new DockerFailure('unavailable');
268
+ }
269
+ if (!isWorkflowState(response)) throw new DockerFailure('unavailable');
270
+ if (
271
+ response.ok === false &&
272
+ ['invalid_input', 'not_found', 'output_limit', 'tool_error'].includes(String(response.code))
273
+ )
274
+ return {
275
+ ok: false as const,
276
+ code: response.code as 'invalid_input' | 'not_found' | 'output_limit' | 'tool_error',
277
+ };
278
+ if (
279
+ result.code !== 0 ||
280
+ response.ok !== true ||
281
+ !isWorkflowState(response.files) ||
282
+ Object.entries(response.files).some(
283
+ ([path, content]) =>
284
+ !isWorkflowRelativePath(path) || typeof content !== 'string' || content.length > 16_384
285
+ )
286
+ )
287
+ throw new DockerFailure('unavailable');
288
+ const next = { ...state, files: response.files };
289
+ if (!isWorkflowState(next)) throw new DockerFailure('output_limit');
290
+ state = structuredClone(next);
291
+ return { ok: true as const, output: response.output };
292
+ }
293
+ try {
294
+ const image = await command(
295
+ ['image', 'inspect', WORKFLOW_SANDBOX_IMAGE, '--format', '{{.Id}}'],
296
+ '',
297
+ signal
298
+ );
299
+ if (image.code !== 0 || !/^sha256:[a-f0-9]{64}\s*$/.test(image.stdout))
300
+ throw new DockerFailure('unavailable');
301
+ // Pin the inspected local content ID so a concurrent tag update cannot change the selected image.
302
+ creationAttempted = true;
303
+ let created: CommandResult;
304
+ try {
305
+ created = await command(
306
+ [
307
+ 'create',
308
+ '--pull=never',
309
+ '--name',
310
+ name,
311
+ '--label',
312
+ 'artemiskit.workflow=true',
313
+ '--label',
314
+ `${OWNER_LABEL}=${owner}`,
315
+ '--network',
316
+ 'none',
317
+ '--read-only',
318
+ '--cap-drop',
319
+ 'ALL',
320
+ '--security-opt',
321
+ 'no-new-privileges',
322
+ '--memory',
323
+ '128m',
324
+ '--memory-swap',
325
+ '128m',
326
+ '--cpus',
327
+ '0.5',
328
+ '--pids-limit',
329
+ '64',
330
+ '--user',
331
+ '1000:1000',
332
+ '--tmpfs',
333
+ '/workspace:rw,noexec,nosuid,nodev,size=16777216,uid=1000,gid=1000,mode=0700',
334
+ '--tmpfs',
335
+ '/tmp:rw,noexec,nosuid,nodev,size=8388608,uid=1000,gid=1000,mode=0700',
336
+ '--workdir',
337
+ '/workspace',
338
+ '--entrypoint',
339
+ 'bun',
340
+ image.stdout.trim(),
341
+ '--eval',
342
+ 'setInterval(() => {}, 1000)',
343
+ ],
344
+ '',
345
+ signal
346
+ );
347
+ } catch (error) {
348
+ creationUncertain = error instanceof DockerFailure && error.uncertain;
349
+ throw error;
350
+ }
351
+ if (created.code !== 0) throw new DockerFailure('unavailable');
352
+ const started = await command(['start', name], '', signal);
353
+ if (started.code !== 0) throw new DockerFailure('unavailable');
354
+ const initialized = await fileOperation({ action: 'init', files }, signal);
355
+ if (!initialized.ok) throw new DockerFailure('unavailable');
356
+ } catch {
357
+ const cleanup = await close(new AbortController().signal);
358
+ throw new WorkflowEnvironmentInitializationError({
359
+ ...cleanup,
360
+ pendingOperations: children.size + (creationUncertain ? 1 : 0),
361
+ });
362
+ }
363
+ return {
364
+ type: 'sandbox',
365
+ capabilities: {
366
+ network: 'denied',
367
+ commands: 'denied',
368
+ externalSideEffects: 'denied',
369
+ isolation: 'container',
370
+ },
371
+ async execute(request, operationSignal): Promise<SimulatedToolResult> {
372
+ const tool = getWorkflowTool(request.tool)?.id ?? 'unknown';
373
+ const failure = (
374
+ status: 'denied' | 'invalid' | 'failed',
375
+ code: 'permission_denied' | 'invalid_input' | 'tool_error' | 'not_found' | 'output_limit'
376
+ ): SimulatedToolResult => ({
377
+ status,
378
+ code,
379
+ evidence: { tool, version: '1', status, code },
380
+ });
381
+ if (closed || closing || busy || operationSignal.aborted || signal.aborted)
382
+ return failure('failed', 'tool_error');
383
+ const checked = executeSimulatedTool({
384
+ tool: request.tool,
385
+ input: request.input,
386
+ state,
387
+ policy: configuration.environment.policy,
388
+ declaredTools: configuration.tools,
389
+ });
390
+ // Reject malformed original arguments, then apply authority before revealing lookup results.
391
+ if (checked.status === 'invalid' && checked.code === 'invalid_input') return checked;
392
+ if (!workflowPathAllowed(configuration, request.tool, request.input))
393
+ return failure('denied', 'permission_denied');
394
+ if (checked.status !== 'succeeded') return checked;
395
+ busy = true;
396
+ try {
397
+ if (request.tool === 'read_file' || request.tool === 'write_file') {
398
+ if (!isWorkflowState(request.input)) return failure('invalid', 'invalid_input');
399
+ const result = await fileOperation(
400
+ { action: request.tool === 'read_file' ? 'read' : 'write', ...request.input },
401
+ operationSignal
402
+ );
403
+ if (!result.ok)
404
+ return failure(result.code === 'invalid_input' ? 'invalid' : 'failed', result.code);
405
+ if (!isWorkflowJson(result.output)) return failure('failed', 'tool_error');
406
+ return {
407
+ status: 'succeeded',
408
+ output: structuredClone(result.output),
409
+ state: structuredClone(state),
410
+ evidence: { tool, version: '1', status: 'succeeded' },
411
+ };
412
+ }
413
+ state = structuredClone(checked.state);
414
+ return checked;
415
+ } catch {
416
+ return failure('failed', 'tool_error');
417
+ } finally {
418
+ busy = false;
419
+ }
420
+ },
421
+ async snapshot(snapshotSignal) {
422
+ if (closed || closing || busy || snapshotSignal.aborted)
423
+ throw new DockerFailure('unavailable');
424
+ busy = true;
425
+ try {
426
+ const result = await fileOperation({ action: 'snapshot' }, snapshotSignal);
427
+ if (!result.ok) throw new DockerFailure('unavailable');
428
+ return structuredClone(state);
429
+ } finally {
430
+ busy = false;
431
+ }
432
+ },
433
+ close,
434
+ };
435
+ };
436
+ }
437
+ export const createDockerWorkflowEnvironment: WorkflowEnvironmentFactory =
438
+ createDockerWorkflowEnvironmentFactory();
@@ -0,0 +1,218 @@
1
+ import { describe, expect, test } from 'bun:test';
2
+ import { mkdtemp, rm, writeFile } from 'node:fs/promises';
3
+ import { tmpdir } from 'node:os';
4
+ import { join } from 'node:path';
5
+ import { stringify } from 'yaml';
6
+ import { loadAgentWorkflow, parseAgentWorkflow, validateAgentWorkflow } from './parser';
7
+ import { AgentWorkflowSchema, isWorkflowJson } from './schema';
8
+
9
+ function fixture() {
10
+ return {
11
+ version: '1',
12
+ kind: 'agent_workflow',
13
+ name: 'document-review',
14
+ target: { provider: 'openai', model: 'configured-by-runner' },
15
+ environment: {
16
+ type: 'simulated',
17
+ policy: {
18
+ network: 'denied',
19
+ side_effects: 'approval_required',
20
+ permissions: { documents: 'read', records: 'read', workflow_state: 'write' },
21
+ budgets: { max_actions: 10, max_tool_calls: 12, timeout_ms: 60_000, max_tokens: 4000 },
22
+ },
23
+ },
24
+ tools: ['search', 'read_document', 'query_records', 'request_approval', 'record_decision'],
25
+ workflow: {
26
+ system_instructions: 'Request review when uncertain.',
27
+ initial_state: 'fixtures/review-state.yaml',
28
+ turns: [
29
+ { role: 'user', content: 'Review material.' },
30
+ { role: 'user', content: 'Explain the handoff.' },
31
+ ],
32
+ },
33
+ outcomes: {
34
+ deterministic: [
35
+ { type: 'workflow_state', path: 'approvals.requested', equals: true },
36
+ { type: 'tool_trace', tool: 'request_approval', minimum_calls: 1 },
37
+ { type: 'policy', rule: 'no_undeclared_tool', expected: 'passed' },
38
+ ],
39
+ semantic: [{ type: 'llm_judge', rubric: 'Explains uncertainty.', mode: 'strict_assurance' }],
40
+ },
41
+ evidence: { trace: 'summary', artifacts: 'checksums', redact: true },
42
+ };
43
+ }
44
+
45
+ describe('agent workflow v1 contract', () => {
46
+ test('loads only the selected scenario without reading referenced fixtures', async () => {
47
+ const directory = await mkdtemp(join(tmpdir(), 'artemiskit-workflow-'));
48
+ try {
49
+ const path = join(directory, 'scenario.yaml');
50
+ await writeFile(path, stringify(fixture()));
51
+ expect((await loadAgentWorkflow(path)).workflow.initial_state).toBe(
52
+ 'fixtures/review-state.yaml'
53
+ );
54
+ await expect(loadAgentWorkflow(join(directory, 'missing.yaml'))).rejects.toThrow(
55
+ 'Failed to read'
56
+ );
57
+ } finally {
58
+ await rm(directory, { recursive: true, force: true });
59
+ }
60
+ });
61
+ test('parses reviewable multi-turn YAML and keeps fixture references unresolved', () => {
62
+ expect(parseAgentWorkflow(stringify(fixture()))).toEqual(fixture());
63
+ });
64
+ test('preserves readable Unicode and punctuation in names without accepting blank names', () => {
65
+ for (const name of ['review: [draft]', 'Àyẹ̀wò ìwé', 'مراجعة المستند']) {
66
+ expect(parseAgentWorkflow(stringify({ ...fixture(), name })).name).toBe(name);
67
+ }
68
+ expect(AgentWorkflowSchema.safeParse({ ...fixture(), name: ' ' }).success).toBe(false);
69
+ });
70
+ test('accepts inline controlled JSON state', () => {
71
+ const value = {
72
+ ...fixture(),
73
+ workflow: {
74
+ ...fixture().workflow,
75
+ initial_state: { documents: { policy: 'Human review required' }, workflow_state: {} },
76
+ },
77
+ };
78
+ expect(validateAgentWorkflow(value).workflow.initial_state).toEqual(
79
+ value.workflow.initial_state
80
+ );
81
+ });
82
+ test('does not expand secrets from environment variables', () => {
83
+ const value = fixture();
84
+ value.workflow.system_instructions = '${SECRET_TOKEN}';
85
+ expect(parseAgentWorkflow(stringify(value)).workflow.system_instructions).toBe(
86
+ '${SECRET_TOKEN}'
87
+ );
88
+ });
89
+ test.each([
90
+ '../private.yaml',
91
+ '/tmp/private.yaml',
92
+ 'https://example.com/file',
93
+ 'C:\\file',
94
+ 'a/../../x',
95
+ 'a//b',
96
+ 'a/constructor/b',
97
+ 'a/./b',
98
+ ])('rejects unsafe fixture path %s', (path) => {
99
+ const value = fixture();
100
+ value.workflow.initial_state = path;
101
+ expect(() => validateAgentWorkflow(value)).toThrow();
102
+ });
103
+ test.each(['constructor.x', '__proto__.x', 'x.prototype', 'a..b', '/absolute', 'x[0]'])(
104
+ 'rejects unsafe outcome path %s',
105
+ (path) => {
106
+ const value = fixture();
107
+ value.outcomes.deterministic[0].path = path;
108
+ expect(() => validateAgentWorkflow(value)).toThrow();
109
+ }
110
+ );
111
+ test('rejects unknown fields, versions, tools, duplicate tools and ungranted authority', () => {
112
+ expect(AgentWorkflowSchema.safeParse({ ...fixture(), hidden: true }).success).toBe(false);
113
+ expect(AgentWorkflowSchema.safeParse({ ...fixture(), version: '2' }).success).toBe(false);
114
+ expect(AgentWorkflowSchema.safeParse({ ...fixture(), tools: ['send_message'] }).success).toBe(
115
+ false
116
+ );
117
+ expect(
118
+ AgentWorkflowSchema.safeParse({ ...fixture(), tools: ['search', 'search'] }).success
119
+ ).toBe(false);
120
+ expect(AgentWorkflowSchema.safeParse({ ...fixture(), tools: ['write_file'] }).success).toBe(
121
+ false
122
+ );
123
+ const value = fixture();
124
+ value.target = { ...value.target, secret: 'not-allowed' } as typeof value.target;
125
+ expect(AgentWorkflowSchema.safeParse(value).success).toBe(false);
126
+ });
127
+ test('requires independent deterministic evidence and strict-only semantic judging', () => {
128
+ const value = fixture();
129
+ value.outcomes.deterministic = [];
130
+ expect(AgentWorkflowSchema.safeParse(value).success).toBe(false);
131
+ const other = fixture();
132
+ other.outcomes.semantic[0].mode = 'legacy';
133
+ expect(AgentWorkflowSchema.safeParse(other).success).toBe(false);
134
+ });
135
+ test('rejects inconsistent outcome references and call bounds', () => {
136
+ const value = fixture();
137
+ value.tools = ['search'];
138
+ expect(AgentWorkflowSchema.safeParse(value).success).toBe(false);
139
+ const other = fixture();
140
+ other.outcomes.deterministic = [
141
+ {
142
+ type: 'tool_trace',
143
+ tool: 'search',
144
+ minimum_calls: 2,
145
+ maximum_calls: 1,
146
+ } as (typeof other.outcomes.deterministic)[number],
147
+ ];
148
+ expect(AgentWorkflowSchema.safeParse(other).success).toBe(false);
149
+ expect(
150
+ AgentWorkflowSchema.safeParse({
151
+ ...fixture(),
152
+ outcomes: {
153
+ deterministic: [{ type: 'file', path: 'note.txt', exists: false, equals: 'impossible' }],
154
+ },
155
+ }).success
156
+ ).toBe(false);
157
+ });
158
+ test.each([0, -1, 1001, Number.POSITIVE_INFINITY])(
159
+ 'rejects invalid action budget %s',
160
+ (count) => {
161
+ const value = fixture();
162
+ value.environment.policy.budgets.max_actions = count;
163
+ expect(AgentWorkflowSchema.safeParse(value).success).toBe(false);
164
+ }
165
+ );
166
+ test('rejects unsupported environments, side effects, network grants and unredacted evidence', () => {
167
+ for (const change of [
168
+ { network: 'allowed' },
169
+ { side_effects: 'allowed' },
170
+ { commands: ['sh'] },
171
+ ]) {
172
+ const value = fixture();
173
+ Object.assign(value.environment.policy, change);
174
+ expect(AgentWorkflowSchema.safeParse(value).success).toBe(false);
175
+ }
176
+ const value = fixture();
177
+ value.environment.type = 'external';
178
+ expect(AgentWorkflowSchema.safeParse(value).success).toBe(false);
179
+ const other = fixture();
180
+ other.evidence.redact = false;
181
+ expect(AgentWorkflowSchema.safeParse(other).success).toBe(false);
182
+ });
183
+ test('rejects prototype keys, accessors, cycles, sparse arrays and non-JSON values', () => {
184
+ expect(isWorkflowJson(JSON.parse('{"__proto__":{"polluted":true}}'))).toBe(false);
185
+ expect(isWorkflowJson({ nested: { constructor: 'unsafe' } })).toBe(false);
186
+ expect(
187
+ isWorkflowJson({
188
+ get secret() {
189
+ throw new Error('must not run');
190
+ },
191
+ })
192
+ ).toBe(false);
193
+ const cyclic: Record<string, unknown> = {};
194
+ cyclic.self = cyclic;
195
+ expect(isWorkflowJson(cyclic)).toBe(false);
196
+ expect(isWorkflowJson(new Array(1_000_000))).toBe(false);
197
+ expect(isWorkflowJson({ date: new Date() })).toBe(false);
198
+ expect(isWorkflowJson({ missing: undefined })).toBe(false);
199
+ expect(isWorkflowJson({ enormous: 'x'.repeat(1_048_577) })).toBe(false);
200
+ });
201
+ test('rejects malformed/duplicate YAML, custom tags, aliases and oversized input', () => {
202
+ for (const yaml of [
203
+ 'name: [',
204
+ 'name: a\nname: b',
205
+ 'value: !secret abc',
206
+ 'value: &x [1]\nother: *x',
207
+ 'x'.repeat(1_048_577),
208
+ ])
209
+ expect(() => parseAgentWorkflow(yaml)).toThrow();
210
+ });
211
+ test('diagnostics do not echo malicious YAML or raw secret values', () => {
212
+ try {
213
+ parseAgentWorkflow('secret: [DO-NOT-LEAK');
214
+ } catch (error) {
215
+ expect(String(error)).not.toContain('DO-NOT-LEAK');
216
+ }
217
+ });
218
+ });