@artemiskit/core 0.5.3 → 0.6.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (92) hide show
  1. package/CHANGELOG.md +28 -0
  2. package/README.md +12 -0
  3. package/dist/adapters/factory.d.ts +1 -1
  4. package/dist/adapters/index.d.ts +3 -3
  5. package/dist/adapters/registry.d.ts +1 -1
  6. package/dist/adapters/types.d.ts +7 -0
  7. package/dist/adapters/types.d.ts.map +1 -1
  8. package/dist/agent-evaluation/index.d.ts +2 -2
  9. package/dist/agent-evaluation/scorer.d.ts +1 -1
  10. package/dist/agent-workflow/catalog.d.ts +28 -0
  11. package/dist/agent-workflow/catalog.d.ts.map +1 -0
  12. package/dist/agent-workflow/environment.d.ts +47 -0
  13. package/dist/agent-workflow/environment.d.ts.map +1 -0
  14. package/dist/agent-workflow/index.d.ts +10 -0
  15. package/dist/agent-workflow/index.d.ts.map +1 -0
  16. package/dist/agent-workflow/parser.d.ts +7 -0
  17. package/dist/agent-workflow/parser.d.ts.map +1 -0
  18. package/dist/agent-workflow/sandbox-fixtures/qualify.d.ts +2 -0
  19. package/dist/agent-workflow/sandbox-fixtures/qualify.d.ts.map +1 -0
  20. package/dist/agent-workflow/sandbox.d.ts +12 -0
  21. package/dist/agent-workflow/sandbox.d.ts.map +1 -0
  22. package/dist/agent-workflow/schema.d.ts +774 -0
  23. package/dist/agent-workflow/schema.d.ts.map +1 -0
  24. package/dist/agent-workflow/session.d.ts +113 -0
  25. package/dist/agent-workflow/session.d.ts.map +1 -0
  26. package/dist/agent-workflow/simulated-tools.d.ts +35 -0
  27. package/dist/agent-workflow/simulated-tools.d.ts.map +1 -0
  28. package/dist/agent-workflow/target.d.ts +243 -0
  29. package/dist/agent-workflow/target.d.ts.map +1 -0
  30. package/dist/artifacts/index.d.ts +2 -2
  31. package/dist/artifacts/manifest.d.ts +1 -1
  32. package/dist/artifacts/types.d.ts +2 -2
  33. package/dist/comparison/eligibility.d.ts +1 -1
  34. package/dist/comparison/index.d.ts +1 -1
  35. package/dist/evaluators/combined.d.ts +2 -2
  36. package/dist/evaluators/contains.d.ts +2 -2
  37. package/dist/evaluators/exact.d.ts +2 -2
  38. package/dist/evaluators/fuzzy.d.ts +2 -2
  39. package/dist/evaluators/index.d.ts +13 -13
  40. package/dist/evaluators/inline.d.ts +2 -2
  41. package/dist/evaluators/json-schema.d.ts +2 -2
  42. package/dist/evaluators/llm-grader.d.ts +2 -2
  43. package/dist/evaluators/not-contains.d.ts +2 -2
  44. package/dist/evaluators/regex.d.ts +2 -2
  45. package/dist/evaluators/similarity.d.ts +2 -2
  46. package/dist/evaluators/tool-trace.d.ts +2 -2
  47. package/dist/evaluators/types.d.ts +3 -3
  48. package/dist/index.d.ts +15 -14
  49. package/dist/index.d.ts.map +1 -1
  50. package/dist/index.js +17827 -15174
  51. package/dist/provenance/execution-provenance.d.ts +1 -1
  52. package/dist/provenance/git.d.ts +1 -1
  53. package/dist/provenance/index.d.ts +4 -4
  54. package/dist/provenance/workload-identity.d.ts +2 -2
  55. package/dist/redaction/index.d.ts +2 -2
  56. package/dist/redaction/redactor.d.ts +1 -1
  57. package/dist/runner/executor.d.ts +3 -3
  58. package/dist/runner/index.d.ts +3 -3
  59. package/dist/runner/runner.d.ts +1 -1
  60. package/dist/runner/types.d.ts +5 -5
  61. package/dist/scenario/index.d.ts +4 -4
  62. package/dist/scenario/parser.d.ts +1 -1
  63. package/dist/scenario/variables.d.ts +1 -1
  64. package/dist/storage/factory.d.ts +1 -1
  65. package/dist/storage/index.d.ts +4 -4
  66. package/dist/storage/local.d.ts +2 -2
  67. package/dist/storage/supabase.d.ts +2 -2
  68. package/dist/storage/types.d.ts +2 -2
  69. package/dist/tools/fixture-executor.d.ts +2 -2
  70. package/dist/tools/index.d.ts +3 -3
  71. package/dist/tools/types.d.ts +1 -1
  72. package/dist/utils/index.d.ts +2 -2
  73. package/dist/validator/index.d.ts +2 -2
  74. package/dist/validator/validator.d.ts +1 -1
  75. package/package.json +4 -4
  76. package/src/adapters/types.ts +7 -0
  77. package/src/agent-workflow/catalog.ts +220 -0
  78. package/src/agent-workflow/environment.ts +207 -0
  79. package/src/agent-workflow/index.ts +10 -0
  80. package/src/agent-workflow/parser.ts +43 -0
  81. package/src/agent-workflow/sandbox-fixtures/qualify.ts +305 -0
  82. package/src/agent-workflow/sandbox.test.ts +117 -0
  83. package/src/agent-workflow/sandbox.ts +438 -0
  84. package/src/agent-workflow/schema.test.ts +218 -0
  85. package/src/agent-workflow/schema.ts +270 -0
  86. package/src/agent-workflow/session.test.ts +629 -0
  87. package/src/agent-workflow/session.ts +1119 -0
  88. package/src/agent-workflow/simulated-tools.test.ts +177 -0
  89. package/src/agent-workflow/simulated-tools.ts +230 -0
  90. package/src/agent-workflow/target.test.ts +280 -0
  91. package/src/agent-workflow/target.ts +357 -0
  92. package/src/index.ts +3 -0
@@ -0,0 +1,207 @@
1
+ import { constants } from 'node:fs';
2
+ import { lstat, open, realpath } from 'node:fs/promises';
3
+ import { join, resolve } from 'node:path';
4
+ import { parseDocument } from 'yaml';
5
+ import { getWorkflowTool } from './catalog';
6
+ import {
7
+ type AgentWorkflow,
8
+ type WorkflowJson,
9
+ isWorkflowJson,
10
+ isWorkflowRelativePath,
11
+ } from './schema';
12
+ import { type SimulatedToolResult, executeSimulatedTool } from './simulated-tools';
13
+
14
+ export type WorkflowState = { [key: string]: WorkflowJson };
15
+ export interface WorkflowEnvironmentCleanup {
16
+ status: 'completed' | 'unresolved';
17
+ artifacts: 'discarded' | 'retained' | 'unknown';
18
+ }
19
+ /** Factories report unresolved partially-created resources without disclosing engine errors. */
20
+ export class WorkflowEnvironmentInitializationError extends Error {
21
+ constructor(readonly cleanup: WorkflowEnvironmentCleanup & { pendingOperations: number }) {
22
+ super('environment_initialization_failed');
23
+ }
24
+ }
25
+ /** Trusted host extension, never constructed from model output. Implementations must honor abort. */
26
+ export interface WorkflowEnvironment {
27
+ readonly type: 'simulated' | 'sandbox';
28
+ readonly capabilities: {
29
+ network: 'denied';
30
+ commands: 'denied';
31
+ externalSideEffects: 'denied';
32
+ isolation: 'memory' | 'container';
33
+ };
34
+ execute(
35
+ request: { tool: string; input: WorkflowJson },
36
+ signal: AbortSignal
37
+ ): Promise<SimulatedToolResult>;
38
+ snapshot(signal: AbortSignal): Promise<WorkflowState>;
39
+ close(signal: AbortSignal): Promise<WorkflowEnvironmentCleanup>;
40
+ }
41
+ export type WorkflowEnvironmentFactory = (options: {
42
+ workflow: AgentWorkflow;
43
+ initialState: WorkflowState;
44
+ signal: AbortSignal;
45
+ }) => Promise<WorkflowEnvironment>;
46
+
47
+ export function isWorkflowState(value: unknown): value is WorkflowState {
48
+ return (
49
+ isWorkflowJson(value) && value !== null && typeof value === 'object' && !Array.isArray(value)
50
+ );
51
+ }
52
+
53
+ /** Exact relative path grants, no globs or path normalization that could hide traversal. */
54
+ export function workflowPathAllowed(
55
+ workflow: AgentWorkflow,
56
+ tool: string,
57
+ input: WorkflowJson
58
+ ): boolean {
59
+ if (tool !== 'read_file' && tool !== 'write_file') return true;
60
+ if (
61
+ !isWorkflowState(input) ||
62
+ typeof input.path !== 'string' ||
63
+ !isWorkflowRelativePath(input.path)
64
+ )
65
+ return false;
66
+ const paths = workflow.environment.policy.paths;
67
+ return !paths || (tool === 'read_file' ? paths.read : paths.write).includes(input.path);
68
+ }
69
+
70
+ export function workflowToolPermitted(workflow: AgentWorkflow, tool: string): boolean {
71
+ const descriptor = getWorkflowTool(tool);
72
+ if (!descriptor || !workflow.tools.includes(descriptor.id)) return false;
73
+ if (descriptor.authority.access === 'none') return true;
74
+ const grant =
75
+ workflow.environment.policy.permissions[
76
+ descriptor.authority.resource as keyof AgentWorkflow['environment']['policy']['permissions']
77
+ ];
78
+ return grant === 'write' || (grant === 'read' && descriptor.authority.access === 'read');
79
+ }
80
+
81
+ /** Fixture files are explicit, bounded JSON/YAML data. Hidden/credential paths never load. */
82
+ export async function resolveWorkflowInitialState(
83
+ workflow: AgentWorkflow,
84
+ fixtureRoot?: string
85
+ ): Promise<WorkflowState> {
86
+ const initial = workflow.workflow.initial_state;
87
+ if (typeof initial !== 'string') {
88
+ if (!isWorkflowState(initial)) throw new Error('invalid_fixture');
89
+ return structuredClone(initial);
90
+ }
91
+ if (
92
+ !fixtureRoot ||
93
+ !isWorkflowRelativePath(initial) ||
94
+ !/\.(json|ya?ml)$/i.test(initial) ||
95
+ initial
96
+ .split('/')
97
+ .some(
98
+ (part) =>
99
+ part.startsWith('.') ||
100
+ /(?:^|[._-])(?:secrets?|credentials?|tokens?|private|id_rsa|id_ed25519)(?:[._-]|$)/i.test(
101
+ part
102
+ )
103
+ )
104
+ )
105
+ throw new Error('invalid_fixture');
106
+ const root = resolve(fixtureRoot);
107
+ // The explicitly selected root may be reached via an OS alias (/tmp on macOS). Descendants may not.
108
+ const canonicalRoot = await realpath(root);
109
+ let current = canonicalRoot;
110
+ const ancestry: { path: string; ino: number; dev: number }[] = [];
111
+ const parts = initial.split('/');
112
+ for (const [index, part] of parts.entries()) {
113
+ current = join(current, part);
114
+ const stat = await lstat(current);
115
+ if (stat.isSymbolicLink() || (index < parts.length - 1 ? !stat.isDirectory() : !stat.isFile()))
116
+ throw new Error('invalid_fixture');
117
+ ancestry.push({ path: current, ino: stat.ino, dev: stat.dev });
118
+ }
119
+ const file = await open(current, constants.O_RDONLY | constants.O_NOFOLLOW);
120
+ try {
121
+ const stat = await file.stat();
122
+ const expected = ancestry[ancestry.length - 1];
123
+ if (
124
+ !stat.isFile() ||
125
+ stat.size > 1_048_576 ||
126
+ stat.ino !== expected.ino ||
127
+ stat.dev !== expected.dev
128
+ )
129
+ throw new Error('invalid_fixture');
130
+ // A bounded descriptor read prevents a concurrently growing fixture from consuming unbounded memory.
131
+ const buffer = Buffer.alloc(1_048_577);
132
+ let length = 0;
133
+ while (length < buffer.length) {
134
+ const { bytesRead } = await file.read(buffer, length, buffer.length - length, null);
135
+ if (!bytesRead) break;
136
+ length += bytesRead;
137
+ }
138
+ if (length > 1_048_576) throw new Error('invalid_fixture');
139
+ for (const entry of ancestry) {
140
+ const after = await lstat(entry.path);
141
+ if (after.isSymbolicLink() || after.ino !== entry.ino || after.dev !== entry.dev)
142
+ throw new Error('invalid_fixture');
143
+ }
144
+ const document = parseDocument(buffer.subarray(0, length).toString('utf8'), {
145
+ uniqueKeys: true,
146
+ customTags: [],
147
+ });
148
+ if (document.errors.length || document.warnings.length) throw new Error('invalid_fixture');
149
+ const value: unknown = document.toJS({ maxAliasCount: 0 });
150
+ if (!isWorkflowState(value)) throw new Error('invalid_fixture');
151
+ return value;
152
+ } finally {
153
+ await file.close();
154
+ }
155
+ }
156
+
157
+ export const createSimulatedWorkflowEnvironment: WorkflowEnvironmentFactory = async ({
158
+ workflow,
159
+ initialState,
160
+ signal,
161
+ }) => {
162
+ if (workflow.environment.type !== 'simulated' || signal.aborted || !isWorkflowState(initialState))
163
+ throw new Error('environment_unavailable');
164
+ const configuration = structuredClone(workflow);
165
+ let state = structuredClone(initialState);
166
+ let closed = false;
167
+ return {
168
+ type: 'simulated',
169
+ capabilities: {
170
+ network: 'denied',
171
+ commands: 'denied',
172
+ externalSideEffects: 'denied',
173
+ isolation: 'memory',
174
+ },
175
+ async execute(request, executionSignal) {
176
+ if (closed || executionSignal.aborted) throw new Error('environment_unavailable');
177
+ if (!workflowPathAllowed(configuration, request.tool, request.input))
178
+ return {
179
+ status: 'denied',
180
+ code: 'permission_denied',
181
+ evidence: {
182
+ tool: getWorkflowTool(request.tool)?.id ?? 'unknown',
183
+ version: '1',
184
+ status: 'denied',
185
+ code: 'permission_denied',
186
+ },
187
+ };
188
+ const result = executeSimulatedTool({
189
+ ...request,
190
+ state,
191
+ policy: configuration.environment.policy,
192
+ declaredTools: configuration.tools,
193
+ });
194
+ if (result.status === 'succeeded') state = structuredClone(result.state);
195
+ return result;
196
+ },
197
+ async snapshot(snapshotSignal) {
198
+ if (closed || snapshotSignal.aborted) throw new Error('environment_unavailable');
199
+ return structuredClone(state);
200
+ },
201
+ async close() {
202
+ closed = true;
203
+ state = {};
204
+ return { status: 'completed', artifacts: 'discarded' };
205
+ },
206
+ };
207
+ };
@@ -0,0 +1,10 @@
1
+ /** Versioned workflow contracts, primitives, and native controlled execution. */
2
+ export * from './schema';
3
+ export * from './parser';
4
+ export * from './catalog';
5
+ export * from './simulated-tools';
6
+ export * from './target';
7
+
8
+ export * from './environment';
9
+ export * from './sandbox';
10
+ export * from './session';
@@ -0,0 +1,43 @@
1
+ import { readFile } from 'node:fs/promises';
2
+ import { parseDocument } from 'yaml';
3
+ import { ArtemisError } from '../utils/errors';
4
+ import { type AgentWorkflow, AgentWorkflowSchema } from './schema';
5
+
6
+ export function validateAgentWorkflow(value: unknown): AgentWorkflow {
7
+ const result = AgentWorkflowSchema.safeParse(value);
8
+ if (!result.success) {
9
+ // Never include input values or untrusted YAML content in diagnostic messages.
10
+ throw new ArtemisError(
11
+ 'Invalid agent workflow: check version, fields, tools, policy, and outcomes',
12
+ 'SCENARIO_VALIDATION_ERROR',
13
+ { issues: result.error.issues.map((issue) => ({ path: issue.path, code: issue.code })) }
14
+ );
15
+ }
16
+ return result.data;
17
+ }
18
+
19
+ /** Parse without environment substitution or fixture dereferencing. */
20
+ export function parseAgentWorkflow(yamlText: string): AgentWorkflow {
21
+ if (Buffer.byteLength(yamlText) > 1_048_576)
22
+ throw new ArtemisError('Agent workflow exceeds 1 MiB', 'SCENARIO_PARSE_ERROR');
23
+ let value: unknown;
24
+ try {
25
+ const document = parseDocument(yamlText, { uniqueKeys: true, customTags: [] });
26
+ if (document.errors.length || document.warnings.length) throw new Error('Invalid YAML');
27
+ value = document.toJS({ maxAliasCount: 0 });
28
+ } catch {
29
+ throw new ArtemisError('Failed to parse agent workflow YAML', 'SCENARIO_PARSE_ERROR');
30
+ }
31
+ return validateAgentWorkflow(value);
32
+ }
33
+
34
+ /** Reads only the explicitly supplied scenario; the session resolves fixtures when execution begins. */
35
+ export async function loadAgentWorkflow(filePath: string): Promise<AgentWorkflow> {
36
+ let content: string;
37
+ try {
38
+ content = await readFile(filePath, 'utf8');
39
+ } catch {
40
+ throw new ArtemisError('Failed to read agent workflow file', 'SCENARIO_READ_ERROR');
41
+ }
42
+ return parseAgentWorkflow(content);
43
+ }
@@ -0,0 +1,305 @@
1
+ /** Explicit local Docker qualification. Run with ARTEMISKIT_DOCKER_TESTS=1 under Bun or bundled Node. */
2
+ import assert from 'node:assert/strict';
3
+ import { execFile } from 'node:child_process';
4
+ import { mkdtemp, writeFile } from 'node:fs/promises';
5
+ import { tmpdir } from 'node:os';
6
+ import { join } from 'node:path';
7
+ import { promisify } from 'node:util';
8
+ import { WorkflowEnvironmentInitializationError } from '../environment';
9
+ import { validateAgentWorkflow } from '../parser';
10
+ import { createDockerWorkflowEnvironmentFactory } from '../sandbox';
11
+ const exec = promisify(execFile);
12
+ const enabled = process.env.ARTEMISKIT_DOCKER_TESTS === '1';
13
+ if (!enabled) {
14
+ process.stdout.write('Docker qualification skipped; set ARTEMISKIT_DOCKER_TESTS=1.\n');
15
+ process.exit(0);
16
+ }
17
+ function workflow() {
18
+ return validateAgentWorkflow({
19
+ version: '1',
20
+ kind: 'agent_workflow',
21
+ name: 'sandbox-qualification',
22
+ target: { provider: 'custom', model: 'fixture' },
23
+ environment: {
24
+ type: 'sandbox',
25
+ policy: {
26
+ network: 'denied',
27
+ side_effects: 'approval_required',
28
+ permissions: { files: 'write', workflow_state: 'write' },
29
+ paths: { read: ['nested/note.txt', 'new.txt'], write: ['new.txt'] },
30
+ budgets: { max_actions: 10, timeout_ms: 10000 },
31
+ },
32
+ },
33
+ tools: ['read_file', 'write_file', 'request_approval'],
34
+ workflow: {
35
+ system_instructions: 'Use declared tools.',
36
+ initial_state: {},
37
+ turns: [{ role: 'user', content: 'Read.' }],
38
+ },
39
+ outcomes: {
40
+ deterministic: [{ type: 'policy', rule: 'permissions_respected', expected: 'passed' }],
41
+ },
42
+ evidence: { trace: 'summary', artifacts: 'checksums', redact: true },
43
+ });
44
+ }
45
+ const signal = () => new AbortController().signal;
46
+ const artifact = 'Àyẹ̀wò 🧪 العربية 日本語'.repeat(300);
47
+ const factory = createDockerWorkflowEnvironmentFactory();
48
+ const initialState = {
49
+ files: { 'nested/note.txt': 'original', 'forbidden.txt': 'private' },
50
+ workflow_state: {},
51
+ };
52
+ async function names() {
53
+ return (
54
+ await exec(
55
+ 'docker',
56
+ ['ps', '--all', '--filter', 'label=artemiskit.workflow=true', '--format', '{{.Names}}'],
57
+ { timeout: 5000 }
58
+ )
59
+ ).stdout
60
+ .trim()
61
+ .split('\n')
62
+ .filter(Boolean);
63
+ }
64
+ if (process.argv.includes('--unavailable-daemon') || process.argv.includes('--unavailable-image')) {
65
+ try {
66
+ await factory({ workflow: workflow(), initialState, signal: signal() });
67
+ assert.fail('expected unavailable daemon');
68
+ } catch (error) {
69
+ assert(error instanceof WorkflowEnvironmentInitializationError);
70
+ assert.equal(error.cleanup.status, 'completed');
71
+ assert.equal(error.cleanup.artifacts, 'discarded');
72
+ }
73
+ process.stdout.write('Unavailable daemon rejected before creating resources.\n');
74
+ process.exit(0);
75
+ }
76
+ const before = new Set(await names());
77
+ const environment = await factory({ workflow: workflow(), initialState, signal: signal() });
78
+ const owned = (await names()).filter((name) => !before.has(name));
79
+ assert.equal(owned.length, 1);
80
+ const name = owned[0];
81
+ try {
82
+ const config = JSON.parse((await exec('docker', ['inspect', name], { timeout: 5000 })).stdout)[0];
83
+ assert.equal(config.HostConfig.NetworkMode, 'none');
84
+ assert.equal(config.HostConfig.ReadonlyRootfs, true);
85
+ assert.deepEqual(config.HostConfig.CapDrop, ['ALL']);
86
+ assert(config.HostConfig.SecurityOpt.includes('no-new-privileges'));
87
+ assert.equal(config.Config.User, '1000:1000');
88
+ assert.equal(config.HostConfig.Binds, null);
89
+ assert.equal(config.HostConfig.Memory, 134217728);
90
+ assert.equal(config.HostConfig.PidsLimit, 64);
91
+ assert.equal(config.HostConfig.NanoCpus, 500000000);
92
+ assert(!config.Config.Env.some((entry: string) => /^(NPM_|OPENAI_|ANTHROPIC_|AWS_)/.test(entry)));
93
+ const read = await environment.execute(
94
+ { tool: 'read_file', input: { path: 'nested/note.txt' } },
95
+ signal()
96
+ );
97
+ assert.equal(read.status, 'succeeded');
98
+ if (read.status === 'succeeded') assert.deepEqual(read.output, { content: 'original' });
99
+ const written = await environment.execute(
100
+ { tool: 'write_file', input: { path: 'new.txt', content: artifact } },
101
+ signal()
102
+ );
103
+ assert.equal(written.status, 'succeeded');
104
+ assert.equal(
105
+ (
106
+ await exec(
107
+ 'docker',
108
+ [
109
+ 'exec',
110
+ name,
111
+ 'bun',
112
+ '--eval',
113
+ "process.stdout.write(require('node:fs').readFileSync('/workspace/new.txt','utf8'))",
114
+ ],
115
+ { timeout: 5000 }
116
+ )
117
+ ).stdout,
118
+ artifact
119
+ );
120
+ assert.equal(
121
+ (await environment.execute({ tool: 'read_file', input: { path: 'forbidden.txt' } }, signal()))
122
+ .status,
123
+ 'denied'
124
+ );
125
+ assert.equal(
126
+ (
127
+ await environment.execute(
128
+ { tool: 'write_file', input: { path: '../escape', content: 'no' } },
129
+ signal()
130
+ )
131
+ ).status,
132
+ 'invalid'
133
+ );
134
+ assert.equal(
135
+ (await environment.execute({ tool: 'run_command', input: { command: 'whoami' } }, signal()))
136
+ .status,
137
+ 'denied'
138
+ );
139
+ const approval = await environment.execute(
140
+ { tool: 'request_approval', input: { reason: 'Review' } },
141
+ signal()
142
+ );
143
+ assert.equal(approval.status, 'succeeded');
144
+ if (approval.status === 'succeeded')
145
+ assert.deepEqual(approval.output, { requested: true, status: 'pending' });
146
+ const snapshot = await environment.snapshot(signal());
147
+ assert.equal((snapshot.files as Record<string, string>)['new.txt'], artifact);
148
+ assert.deepEqual(initialState.files, {
149
+ 'nested/note.txt': 'original',
150
+ 'forbidden.txt': 'private',
151
+ });
152
+ const second = await factory({ workflow: workflow(), initialState, signal: signal() });
153
+ try {
154
+ assert.equal(
155
+ (await second.snapshot(signal())).files &&
156
+ ((await second.snapshot(signal())).files as Record<string, string>)['new.txt'],
157
+ undefined
158
+ );
159
+ } finally {
160
+ assert.equal((await second.close(signal())).status, 'completed');
161
+ }
162
+ const rootWrite = await exec(
163
+ 'docker',
164
+ [
165
+ 'exec',
166
+ name,
167
+ 'bun',
168
+ '--eval',
169
+ "try { require('node:fs').writeFileSync('/unauthorized','x'); process.exit(1); } catch { process.stdout.write('denied'); }",
170
+ ],
171
+ { timeout: 5000 }
172
+ );
173
+ assert.equal(rootWrite.stdout, 'denied');
174
+ const network = await exec(
175
+ 'docker',
176
+ [
177
+ 'exec',
178
+ name,
179
+ 'bun',
180
+ '--eval',
181
+ "try { await fetch('http://192.0.2.1', {signal: AbortSignal.timeout(200)}); process.exit(1); } catch { process.stdout.write('denied'); }",
182
+ ],
183
+ { timeout: 5000 }
184
+ );
185
+ assert.equal(network.stdout, 'denied');
186
+ // An out-of-band local test fixture introduces a symlink; the production fixed file program must reject it.
187
+ await exec(
188
+ 'docker',
189
+ [
190
+ 'exec',
191
+ name,
192
+ 'bun',
193
+ '--eval',
194
+ "require('node:fs').unlinkSync('/workspace/nested/note.txt'); require('node:fs').symlinkSync('/etc/passwd','/workspace/nested/note.txt')",
195
+ ],
196
+ { timeout: 5000 }
197
+ );
198
+ assert.equal(
199
+ (await environment.execute({ tool: 'read_file', input: { path: 'nested/note.txt' } }, signal()))
200
+ .status,
201
+ 'invalid'
202
+ );
203
+ } finally {
204
+ assert.equal((await environment.close(signal())).status, 'completed');
205
+ }
206
+ assert(!(await names()).includes(name));
207
+ // Cancel while the fixed exec program is in flight; admission remains closed after the run signal aborts.
208
+ const cancelled = new AbortController();
209
+ const pending = factory({ workflow: workflow(), initialState, signal: cancelled.signal });
210
+ setTimeout(() => cancelled.abort(), 20);
211
+ try {
212
+ const env = await pending;
213
+ await env.close(signal());
214
+ } catch (error) {
215
+ assert(error instanceof WorkflowEnvironmentInitializationError);
216
+ assert(['completed', 'unresolved'].includes(error.cleanup.status));
217
+ }
218
+ // A separately isolated Docker client endpoint proves unavailable-daemon behavior without changing user configuration.
219
+ const child = await exec(process.execPath, [process.argv[1], '--unavailable-daemon'], {
220
+ timeout: 10000,
221
+ env: {
222
+ PATH: process.env.PATH,
223
+ ARTEMISKIT_DOCKER_TESTS: '1',
224
+ DOCKER_HOST: 'unix:///tmp/artemiskit-deliberately-missing-docker.sock',
225
+ },
226
+ });
227
+ assert(child.stdout.includes('Unavailable daemon'));
228
+ const fakeBin = await mkdtemp(join(tmpdir(), 'artemis-docker-missing-image-'));
229
+ await writeFile(join(fakeBin, 'docker'), '#!/bin/sh\nexit 1\n', { mode: 0o700 });
230
+ const missingImage = await exec(process.execPath, [process.argv[1], '--unavailable-image'], {
231
+ timeout: 10000,
232
+ env: { PATH: fakeBin, ARTEMISKIT_DOCKER_TESTS: '1' },
233
+ });
234
+ assert(missingImage.stdout.includes('Unavailable daemon'));
235
+ try {
236
+ await factory({
237
+ workflow: workflow(),
238
+ initialState: { files: { a: 'not-a-directory', 'a/child.txt': 'invalid topology' } },
239
+ signal: signal(),
240
+ });
241
+ assert.fail('expected failed initialization');
242
+ } catch (error) {
243
+ assert(error instanceof WorkflowEnvironmentInitializationError);
244
+ assert.equal(error.cleanup.status, 'completed');
245
+ assert.equal(error.cleanup.artifacts, 'discarded');
246
+ }
247
+ const deniedWorkflow = workflow();
248
+ deniedWorkflow.environment.policy.paths = { read: [], write: [] };
249
+ const deniedEnvironment = await factory({
250
+ workflow: deniedWorkflow,
251
+ initialState,
252
+ signal: signal(),
253
+ });
254
+ try {
255
+ const present = await deniedEnvironment.execute(
256
+ { tool: 'read_file', input: { path: 'nested/note.txt' } },
257
+ signal()
258
+ );
259
+ const absent = await deniedEnvironment.execute(
260
+ { tool: 'read_file', input: { path: 'absent.txt' } },
261
+ signal()
262
+ );
263
+ assert.deepEqual(present, {
264
+ status: 'denied',
265
+ code: 'permission_denied',
266
+ evidence: { tool: 'read_file', version: '1', status: 'denied', code: 'permission_denied' },
267
+ });
268
+ assert.deepEqual(absent, present);
269
+ const malformed = await deniedEnvironment.execute(
270
+ { tool: 'read_file', input: { path: 42 } },
271
+ signal()
272
+ );
273
+ assert.equal(malformed.status, 'invalid');
274
+ assert.equal(malformed.code, 'invalid_input');
275
+ } finally {
276
+ assert.equal((await deniedEnvironment.close(signal())).status, 'completed');
277
+ }
278
+ const lifetime = new AbortController();
279
+ const active = await factory({ workflow: workflow(), initialState, signal: lifetime.signal });
280
+ const work = active.execute(
281
+ { tool: 'read_file', input: { path: 'nested/note.txt' } },
282
+ lifetime.signal
283
+ );
284
+ lifetime.abort();
285
+ assert.equal((await work).status, 'failed');
286
+ assert.equal(
287
+ (await active.execute({ tool: 'read_file', input: { path: 'nested/note.txt' } }, signal()))
288
+ .status,
289
+ 'failed'
290
+ );
291
+ assert.equal((await active.close(signal())).status, 'completed');
292
+ assert.deepEqual(new Set(await names()), before);
293
+ process.stdout.write(
294
+ `${JSON.stringify({
295
+ status: 'passed',
296
+ runtime: process.versions.bun ? 'bun' : 'node',
297
+ isolation: true,
298
+ files: true,
299
+ policy: true,
300
+ symlinks: true,
301
+ cleanup: true,
302
+ cancellation: true,
303
+ unavailableDaemon: true,
304
+ })}\n`
305
+ );
@@ -0,0 +1,117 @@
1
+ import { describe, expect, test } from 'bun:test';
2
+ import { WorkflowEnvironmentInitializationError, workflowPathAllowed } from './environment';
3
+ import { validateAgentWorkflow } from './parser';
4
+ import { createDockerWorkflowEnvironment, createDockerWorkflowEnvironmentFactory } from './sandbox';
5
+
6
+ function workflow() {
7
+ return validateAgentWorkflow({
8
+ version: '1',
9
+ kind: 'agent_workflow',
10
+ name: 'sandbox',
11
+ target: { provider: 'custom', model: 'fixture' },
12
+ environment: {
13
+ type: 'sandbox',
14
+ policy: {
15
+ network: 'denied',
16
+ side_effects: 'denied',
17
+ permissions: { files: 'write' },
18
+ budgets: { max_actions: 5, timeout_ms: 1000 },
19
+ },
20
+ },
21
+ tools: ['read_file', 'write_file'],
22
+ workflow: {
23
+ system_instructions: 'Use declared tools.',
24
+ initial_state: {},
25
+ turns: [{ role: 'user', content: 'Read.' }],
26
+ },
27
+ outcomes: {
28
+ deterministic: [{ type: 'policy', rule: 'permissions_respected', expected: 'passed' }],
29
+ },
30
+ evidence: { trace: 'summary', artifacts: 'checksums', redact: true },
31
+ });
32
+ }
33
+ describe('Docker workflow environment offline admission', () => {
34
+ test('empty path grants deny present and absent fixture names equally', () => {
35
+ const value = workflow();
36
+ value.environment.policy.paths = { read: [], write: [] };
37
+ value.workflow.initial_state = { files: { 'present.txt': 'fixture' } };
38
+ expect(workflowPathAllowed(value, 'read_file', { path: 'present.txt' })).toBe(false);
39
+ expect(workflowPathAllowed(value, 'read_file', { path: 'absent.txt' })).toBe(false);
40
+ });
41
+
42
+ test('accepts qualified default cleanup and explicit bounded timeouts', () => {
43
+ expect(typeof createDockerWorkflowEnvironmentFactory()).toBe('function');
44
+ expect(
45
+ typeof createDockerWorkflowEnvironmentFactory({
46
+ operationTimeoutMs: 5000,
47
+ cleanupTimeoutMs: 3000,
48
+ })
49
+ ).toBe('function');
50
+ });
51
+ test.each([0, -1, 30_001, Number.NaN, 1.5])(
52
+ 'rejects invalid operation timeout %s',
53
+ (operationTimeoutMs) => {
54
+ expect(() => createDockerWorkflowEnvironmentFactory({ operationTimeoutMs })).toThrow(
55
+ 'Invalid Docker'
56
+ );
57
+ }
58
+ );
59
+ test.each([0, -1, 10_001, Number.NaN, 1.5])(
60
+ 'rejects invalid cleanup timeout %s',
61
+ (cleanupTimeoutMs) => {
62
+ expect(() => createDockerWorkflowEnvironmentFactory({ cleanupTimeoutMs })).toThrow(
63
+ 'Invalid Docker'
64
+ );
65
+ }
66
+ );
67
+ test('rejects executable, image, environment and command overrides', () => {
68
+ for (const key of ['image', 'dockerCommand', 'env', 'command', 'mount'])
69
+ expect(() =>
70
+ createDockerWorkflowEnvironmentFactory({ [key]: 'untrusted' } as never)
71
+ ).toThrow();
72
+ });
73
+ test('pre-cancelled startup never needs Docker or claims remaining resources', async () => {
74
+ const controller = new AbortController();
75
+ controller.abort();
76
+ try {
77
+ await createDockerWorkflowEnvironment({
78
+ workflow: workflow(),
79
+ initialState: {},
80
+ signal: controller.signal,
81
+ });
82
+ throw new Error('expected rejection');
83
+ } catch (error) {
84
+ expect(error).toBeInstanceOf(WorkflowEnvironmentInitializationError);
85
+ expect((error as WorkflowEnvironmentInitializationError).cleanup).toEqual({
86
+ status: 'completed',
87
+ artifacts: 'discarded',
88
+ pendingOperations: 0,
89
+ });
90
+ }
91
+ });
92
+ test.each([
93
+ { files: { '../escape.txt': 'no' } },
94
+ { files: { 'x.txt': 3 } },
95
+ { files: { 'x.txt': 'x'.repeat(16385) } },
96
+ { files: [] },
97
+ ])('rejects invalid fixture before Docker access', async (initialState) => {
98
+ await expect(
99
+ createDockerWorkflowEnvironment({
100
+ workflow: workflow(),
101
+ initialState,
102
+ signal: new AbortController().signal,
103
+ })
104
+ ).rejects.toBeInstanceOf(WorkflowEnvironmentInitializationError);
105
+ });
106
+ test('rejects wrong environment and ungranted policy before Docker access', async () => {
107
+ const value = workflow();
108
+ value.environment.type = 'simulated';
109
+ await expect(
110
+ createDockerWorkflowEnvironment({
111
+ workflow: value,
112
+ initialState: {},
113
+ signal: new AbortController().signal,
114
+ })
115
+ ).rejects.toBeInstanceOf(WorkflowEnvironmentInitializationError);
116
+ });
117
+ });