@artemiskit/core 0.6.0 → 0.6.1

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@artemiskit/core",
-  "version": "0.6.0",
+  "version": "0.6.1",
   "description": "Core runner, evaluators, and storage for ArtemisKit LLM evaluation toolkit",
   "type": "module",
   "license": "Apache-2.0",

package/src/adapters/types.ts

@@ -31,6 +31,10 @@ export interface ToolCall {
  * Options for generating a completion
  */
 export interface GenerateOptions {
+  /** Optional transport cancellation; consult capabilities before relying on it. */
+  signal?: AbortSignal;
+  /** Override retries per request when a host owns cumulative admission budgets. */
+  maxRetries?: number;
   prompt: string | ChatMessage[];
   model?: string;
   maxTokens?: number;
@@ -78,6 +82,8 @@ export interface TokenUsage {
  * Result from a generation request
  */
 export interface GenerateResult {
+  /** False when zero counts are placeholders rather than provider-reported usage. */
+  usageAvailable?: boolean;
   id: string;
   model: string;
   text: string;
@@ -96,6 +102,7 @@ export interface GenerateResult {
  * Model capabilities
  */
 export interface ModelCapabilities {
+  transportCancellation?: boolean;
   streaming: boolean;
   functionCalling: boolean;
   toolUse: boolean;

package/src/agent-workflow/catalog.ts

@@ -1,4 +1,4 @@
-/** Versioned built-in capabilities. All implementations in this release are simulated. */
+/** Versioned run-local capabilities shared by simulated and disposable sandbox environments. */
 export const WORKFLOW_TOOL_IDS = [
   'search',
   'read_document',
@@ -35,6 +35,7 @@ export interface WorkflowToolDescriptor {
     resource: WorkflowResource;
     access: 'read' | 'write' | 'none';
     network: 'denied';
+    /** V1's simulated label means run-local effects; file storage follows the selected environment. */
     sideEffects: 'none' | 'simulated';
   };
   evidence: { mode: 'metadata_only'; maxBytes: 1024 };
@@ -124,7 +125,7 @@ const tools: WorkflowToolDescriptor[] = [
   descriptor(
     'read_file',
     'files',
-    'Read one relative file from in-memory fixture state.',
+    'Read one relative file from the isolated workflow environment.',
     'files',
     'read',
     object({ path: relativePath }),
@@ -133,7 +134,7 @@ const tools: WorkflowToolDescriptor[] = [
   descriptor(
     'write_file',
     'files',
-    'Write one relative file in isolated in-memory state.',
+    'Write one relative file in the isolated workflow environment.',
     'files',
     'write',
     object({ path: relativePath, content: { type: 'string', maxLength: 16_384 } }),

package/src/agent-workflow/environment.ts

@@ -0,0 +1,207 @@
+import { constants } from 'node:fs';
+import { lstat, open, realpath } from 'node:fs/promises';
+import { join, resolve } from 'node:path';
+import { parseDocument } from 'yaml';
+import { getWorkflowTool } from './catalog';
+import {
+  type AgentWorkflow,
+  type WorkflowJson,
+  isWorkflowJson,
+  isWorkflowRelativePath,
+} from './schema';
+import { type SimulatedToolResult, executeSimulatedTool } from './simulated-tools';
+
+export type WorkflowState = { [key: string]: WorkflowJson };
+export interface WorkflowEnvironmentCleanup {
+  status: 'completed' | 'unresolved';
+  artifacts: 'discarded' | 'retained' | 'unknown';
+}
+/** Factories report unresolved partially-created resources without disclosing engine errors. */
+export class WorkflowEnvironmentInitializationError extends Error {
+  constructor(readonly cleanup: WorkflowEnvironmentCleanup & { pendingOperations: number }) {
+    super('environment_initialization_failed');
+  }
+}
+/** Trusted host extension, never constructed from model output. Implementations must honor abort. */
+export interface WorkflowEnvironment {
+  readonly type: 'simulated' | 'sandbox';
+  readonly capabilities: {
+    network: 'denied';
+    commands: 'denied';
+    externalSideEffects: 'denied';
+    isolation: 'memory' | 'container';
+  };
+  execute(
+    request: { tool: string; input: WorkflowJson },
+    signal: AbortSignal
+  ): Promise<SimulatedToolResult>;
+  snapshot(signal: AbortSignal): Promise<WorkflowState>;
+  close(signal: AbortSignal): Promise<WorkflowEnvironmentCleanup>;
+}
+export type WorkflowEnvironmentFactory = (options: {
+  workflow: AgentWorkflow;
+  initialState: WorkflowState;
+  signal: AbortSignal;
+}) => Promise<WorkflowEnvironment>;
+
+export function isWorkflowState(value: unknown): value is WorkflowState {
+  return (
+    isWorkflowJson(value) && value !== null && typeof value === 'object' && !Array.isArray(value)
+  );
+}
+
+/** Exact relative path grants, no globs or path normalization that could hide traversal. */
+export function workflowPathAllowed(
+  workflow: AgentWorkflow,
+  tool: string,
+  input: WorkflowJson
+): boolean {
+  if (tool !== 'read_file' && tool !== 'write_file') return true;
+  if (
+    !isWorkflowState(input) ||
+    typeof input.path !== 'string' ||
+    !isWorkflowRelativePath(input.path)
+  )
+    return false;
+  const paths = workflow.environment.policy.paths;
+  return !paths || (tool === 'read_file' ? paths.read : paths.write).includes(input.path);
+}
+
+export function workflowToolPermitted(workflow: AgentWorkflow, tool: string): boolean {
+  const descriptor = getWorkflowTool(tool);
+  if (!descriptor || !workflow.tools.includes(descriptor.id)) return false;
+  if (descriptor.authority.access === 'none') return true;
+  const grant =
+    workflow.environment.policy.permissions[
+      descriptor.authority.resource as keyof AgentWorkflow['environment']['policy']['permissions']
+    ];
+  return grant === 'write' || (grant === 'read' && descriptor.authority.access === 'read');
+}
+
+/** Fixture files are explicit, bounded JSON/YAML data. Hidden/credential paths never load. */
+export async function resolveWorkflowInitialState(
+  workflow: AgentWorkflow,
+  fixtureRoot?: string
+): Promise<WorkflowState> {
+  const initial = workflow.workflow.initial_state;
+  if (typeof initial !== 'string') {
+    if (!isWorkflowState(initial)) throw new Error('invalid_fixture');
+    return structuredClone(initial);
+  }
+  if (
+    !fixtureRoot ||
+    !isWorkflowRelativePath(initial) ||
+    !/\.(json|ya?ml)$/i.test(initial) ||
+    initial
+      .split('/')
+      .some(
+        (part) =>
+          part.startsWith('.') ||
+          /(?:^|[._-])(?:secrets?|credentials?|tokens?|private|id_rsa|id_ed25519)(?:[._-]|$)/i.test(
+            part
+          )
+      )
+  )
+    throw new Error('invalid_fixture');
+  const root = resolve(fixtureRoot);
+  // The explicitly selected root may be reached via an OS alias (/tmp on macOS). Descendants may not.
+  const canonicalRoot = await realpath(root);
+  let current = canonicalRoot;
+  const ancestry: { path: string; ino: number; dev: number }[] = [];
+  const parts = initial.split('/');
+  for (const [index, part] of parts.entries()) {
+    current = join(current, part);
+    const stat = await lstat(current);
+    if (stat.isSymbolicLink() || (index < parts.length - 1 ? !stat.isDirectory() : !stat.isFile()))
+      throw new Error('invalid_fixture');
+    ancestry.push({ path: current, ino: stat.ino, dev: stat.dev });
+  }
+  const file = await open(current, constants.O_RDONLY | constants.O_NOFOLLOW);
+  try {
+    const stat = await file.stat();
+    const expected = ancestry[ancestry.length - 1];
+    if (
+      !stat.isFile() ||
+      stat.size > 1_048_576 ||
+      stat.ino !== expected.ino ||
+      stat.dev !== expected.dev
+    )
+      throw new Error('invalid_fixture');
+    // A bounded descriptor read prevents a concurrently growing fixture from consuming unbounded memory.
+    const buffer = Buffer.alloc(1_048_577);
+    let length = 0;
+    while (length < buffer.length) {
+      const { bytesRead } = await file.read(buffer, length, buffer.length - length, null);
+      if (!bytesRead) break;
+      length += bytesRead;
+    }
+    if (length > 1_048_576) throw new Error('invalid_fixture');
+    for (const entry of ancestry) {
+      const after = await lstat(entry.path);
+      if (after.isSymbolicLink() || after.ino !== entry.ino || after.dev !== entry.dev)
+        throw new Error('invalid_fixture');
+    }
+    const document = parseDocument(buffer.subarray(0, length).toString('utf8'), {
+      uniqueKeys: true,
+      customTags: [],
+    });
+    if (document.errors.length || document.warnings.length) throw new Error('invalid_fixture');
+    const value: unknown = document.toJS({ maxAliasCount: 0 });
+    if (!isWorkflowState(value)) throw new Error('invalid_fixture');
+    return value;
+  } finally {
+    await file.close();
+  }
+}
+
+export const createSimulatedWorkflowEnvironment: WorkflowEnvironmentFactory = async ({
+  workflow,
+  initialState,
+  signal,
+}) => {
+  if (workflow.environment.type !== 'simulated' || signal.aborted || !isWorkflowState(initialState))
+    throw new Error('environment_unavailable');
+  const configuration = structuredClone(workflow);
+  let state = structuredClone(initialState);
+  let closed = false;
+  return {
+    type: 'simulated',
+    capabilities: {
+      network: 'denied',
+      commands: 'denied',
+      externalSideEffects: 'denied',
+      isolation: 'memory',
+    },
+    async execute(request, executionSignal) {
+      if (closed || executionSignal.aborted) throw new Error('environment_unavailable');
+      if (!workflowPathAllowed(configuration, request.tool, request.input))
+        return {
+          status: 'denied',
+          code: 'permission_denied',
+          evidence: {
+            tool: getWorkflowTool(request.tool)?.id ?? 'unknown',
+            version: '1',
+            status: 'denied',
+            code: 'permission_denied',
+          },
+        };
+      const result = executeSimulatedTool({
+        ...request,
+        state,
+        policy: configuration.environment.policy,
+        declaredTools: configuration.tools,
+      });
+      if (result.status === 'succeeded') state = structuredClone(result.state);
+      return result;
+    },
+    async snapshot(snapshotSignal) {
+      if (closed || snapshotSignal.aborted) throw new Error('environment_unavailable');
+      return structuredClone(state);
+    },
+    async close() {
+      closed = true;
+      state = {};
+      return { status: 'completed', artifacts: 'discarded' };
+    },
+  };
+};

package/src/agent-workflow/index.ts

@@ -1,6 +1,10 @@
-/** Versioned agent workflow authoring and single-step primitives. */
+/** Versioned workflow contracts, primitives, and native controlled execution. */
 export * from './schema';
 export * from './parser';
 export * from './catalog';
 export * from './simulated-tools';
 export * from './target';
+
+export * from './environment';
+export * from './sandbox';
+export * from './session';

package/src/agent-workflow/parser.ts

@@ -31,7 +31,7 @@ export function parseAgentWorkflow(yamlText: string): AgentWorkflow {
   return validateAgentWorkflow(value);
 }
 
-/** Reads only the explicitly supplied scenario; fixture resolution belongs to a future runner. */
+/** Reads only the explicitly supplied scenario; the session resolves fixtures when execution begins. */
 export async function loadAgentWorkflow(filePath: string): Promise<AgentWorkflow> {
   let content: string;
   try {

package/src/agent-workflow/sandbox-fixtures/qualify.ts

@@ -0,0 +1,305 @@
+/** Explicit local Docker qualification. Run with ARTEMISKIT_DOCKER_TESTS=1 under Bun or bundled Node. */
+import assert from 'node:assert/strict';
+import { execFile } from 'node:child_process';
+import { mkdtemp, writeFile } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { promisify } from 'node:util';
+import { WorkflowEnvironmentInitializationError } from '../environment';
+import { validateAgentWorkflow } from '../parser';
+import { createDockerWorkflowEnvironmentFactory } from '../sandbox';
+const exec = promisify(execFile);
+const enabled = process.env.ARTEMISKIT_DOCKER_TESTS === '1';
+if (!enabled) {
+  process.stdout.write('Docker qualification skipped; set ARTEMISKIT_DOCKER_TESTS=1.\n');
+  process.exit(0);
+}
+function workflow() {
+  return validateAgentWorkflow({
+    version: '1',
+    kind: 'agent_workflow',
+    name: 'sandbox-qualification',
+    target: { provider: 'custom', model: 'fixture' },
+    environment: {
+      type: 'sandbox',
+      policy: {
+        network: 'denied',
+        side_effects: 'approval_required',
+        permissions: { files: 'write', workflow_state: 'write' },
+        paths: { read: ['nested/note.txt', 'new.txt'], write: ['new.txt'] },
+        budgets: { max_actions: 10, timeout_ms: 10000 },
+      },
+    },
+    tools: ['read_file', 'write_file', 'request_approval'],
+    workflow: {
+      system_instructions: 'Use declared tools.',
+      initial_state: {},
+      turns: [{ role: 'user', content: 'Read.' }],
+    },
+    outcomes: {
+      deterministic: [{ type: 'policy', rule: 'permissions_respected', expected: 'passed' }],
+    },
+    evidence: { trace: 'summary', artifacts: 'checksums', redact: true },
+  });
+}
+const signal = () => new AbortController().signal;
+const artifact = 'Àyẹ̀wò 🧪 العربية 日本語'.repeat(300);
+const factory = createDockerWorkflowEnvironmentFactory();
+const initialState = {
+  files: { 'nested/note.txt': 'original', 'forbidden.txt': 'private' },
+  workflow_state: {},
+};
+async function names() {
+  return (
+    await exec(
+      'docker',
+      ['ps', '--all', '--filter', 'label=artemiskit.workflow=true', '--format', '{{.Names}}'],
+      { timeout: 5000 }
+    )
+  ).stdout
+    .trim()
+    .split('\n')
+    .filter(Boolean);
+}
+if (process.argv.includes('--unavailable-daemon') || process.argv.includes('--unavailable-image')) {
+  try {
+    await factory({ workflow: workflow(), initialState, signal: signal() });
+    assert.fail('expected unavailable daemon');
+  } catch (error) {
+    assert(error instanceof WorkflowEnvironmentInitializationError);
+    assert.equal(error.cleanup.status, 'completed');
+    assert.equal(error.cleanup.artifacts, 'discarded');
+  }
+  process.stdout.write('Unavailable daemon rejected before creating resources.\n');
+  process.exit(0);
+}
+const before = new Set(await names());
+const environment = await factory({ workflow: workflow(), initialState, signal: signal() });
+const owned = (await names()).filter((name) => !before.has(name));
+assert.equal(owned.length, 1);
+const name = owned[0];
+try {
+  const config = JSON.parse((await exec('docker', ['inspect', name], { timeout: 5000 })).stdout)[0];
+  assert.equal(config.HostConfig.NetworkMode, 'none');
+  assert.equal(config.HostConfig.ReadonlyRootfs, true);
+  assert.deepEqual(config.HostConfig.CapDrop, ['ALL']);
+  assert(config.HostConfig.SecurityOpt.includes('no-new-privileges'));
+  assert.equal(config.Config.User, '1000:1000');
+  assert.equal(config.HostConfig.Binds, null);
+  assert.equal(config.HostConfig.Memory, 134217728);
+  assert.equal(config.HostConfig.PidsLimit, 64);
+  assert.equal(config.HostConfig.NanoCpus, 500000000);
+  assert(!config.Config.Env.some((entry: string) => /^(NPM_|OPENAI_|ANTHROPIC_|AWS_)/.test(entry)));
+  const read = await environment.execute(
+    { tool: 'read_file', input: { path: 'nested/note.txt' } },
+    signal()
+  );
+  assert.equal(read.status, 'succeeded');
+  if (read.status === 'succeeded') assert.deepEqual(read.output, { content: 'original' });
+  const written = await environment.execute(
+    { tool: 'write_file', input: { path: 'new.txt', content: artifact } },
+    signal()
+  );
+  assert.equal(written.status, 'succeeded');
+  assert.equal(
+    (
+      await exec(
+        'docker',
+        [
+          'exec',
+          name,
+          'bun',
+          '--eval',
+          "process.stdout.write(require('node:fs').readFileSync('/workspace/new.txt','utf8'))",
+        ],
+        { timeout: 5000 }
+      )
+    ).stdout,
+    artifact
+  );
+  assert.equal(
+    (await environment.execute({ tool: 'read_file', input: { path: 'forbidden.txt' } }, signal()))
+      .status,
+    'denied'
+  );
+  assert.equal(
+    (
+      await environment.execute(
+        { tool: 'write_file', input: { path: '../escape', content: 'no' } },
+        signal()
+      )
+    ).status,
+    'invalid'
+  );
+  assert.equal(
+    (await environment.execute({ tool: 'run_command', input: { command: 'whoami' } }, signal()))
+      .status,
+    'denied'
+  );
+  const approval = await environment.execute(
+    { tool: 'request_approval', input: { reason: 'Review' } },
+    signal()
+  );
+  assert.equal(approval.status, 'succeeded');
+  if (approval.status === 'succeeded')
+    assert.deepEqual(approval.output, { requested: true, status: 'pending' });
+  const snapshot = await environment.snapshot(signal());
+  assert.equal((snapshot.files as Record<string, string>)['new.txt'], artifact);
+  assert.deepEqual(initialState.files, {
+    'nested/note.txt': 'original',
+    'forbidden.txt': 'private',
+  });
+  const second = await factory({ workflow: workflow(), initialState, signal: signal() });
+  try {
+    assert.equal(
+      (await second.snapshot(signal())).files &&
+        ((await second.snapshot(signal())).files as Record<string, string>)['new.txt'],
+      undefined
+    );
+  } finally {
+    assert.equal((await second.close(signal())).status, 'completed');
+  }
+  const rootWrite = await exec(
+    'docker',
+    [
+      'exec',
+      name,
+      'bun',
+      '--eval',
+      "try { require('node:fs').writeFileSync('/unauthorized','x'); process.exit(1); } catch { process.stdout.write('denied'); }",
+    ],
+    { timeout: 5000 }
+  );
+  assert.equal(rootWrite.stdout, 'denied');
+  const network = await exec(
+    'docker',
+    [
+      'exec',
+      name,
+      'bun',
+      '--eval',
+      "try { await fetch('http://192.0.2.1', {signal: AbortSignal.timeout(200)}); process.exit(1); } catch { process.stdout.write('denied'); }",
+    ],
+    { timeout: 5000 }
+  );
+  assert.equal(network.stdout, 'denied');
+  // An out-of-band local test fixture introduces a symlink; the production fixed file program must reject it.
+  await exec(
+    'docker',
+    [
+      'exec',
+      name,
+      'bun',
+      '--eval',
+      "require('node:fs').unlinkSync('/workspace/nested/note.txt'); require('node:fs').symlinkSync('/etc/passwd','/workspace/nested/note.txt')",
+    ],
+    { timeout: 5000 }
+  );
+  assert.equal(
+    (await environment.execute({ tool: 'read_file', input: { path: 'nested/note.txt' } }, signal()))
+      .status,
+    'invalid'
+  );
+} finally {
+  assert.equal((await environment.close(signal())).status, 'completed');
+}
+assert(!(await names()).includes(name));
+// Cancel while the fixed exec program is in flight; admission remains closed after the run signal aborts.
+const cancelled = new AbortController();
+const pending = factory({ workflow: workflow(), initialState, signal: cancelled.signal });
+setTimeout(() => cancelled.abort(), 20);
+try {
+  const env = await pending;
+  await env.close(signal());
+} catch (error) {
+  assert(error instanceof WorkflowEnvironmentInitializationError);
+  assert(['completed', 'unresolved'].includes(error.cleanup.status));
+}
+// A separately isolated Docker client endpoint proves unavailable-daemon behavior without changing user configuration.
+const child = await exec(process.execPath, [process.argv[1], '--unavailable-daemon'], {
+  timeout: 10000,
+  env: {
+    PATH: process.env.PATH,
+    ARTEMISKIT_DOCKER_TESTS: '1',
+    DOCKER_HOST: 'unix:///tmp/artemiskit-deliberately-missing-docker.sock',
+  },
+});
+assert(child.stdout.includes('Unavailable daemon'));
+const fakeBin = await mkdtemp(join(tmpdir(), 'artemis-docker-missing-image-'));
+await writeFile(join(fakeBin, 'docker'), '#!/bin/sh\nexit 1\n', { mode: 0o700 });
+const missingImage = await exec(process.execPath, [process.argv[1], '--unavailable-image'], {
+  timeout: 10000,
+  env: { PATH: fakeBin, ARTEMISKIT_DOCKER_TESTS: '1' },
+});
+assert(missingImage.stdout.includes('Unavailable daemon'));
+try {
+  await factory({
+    workflow: workflow(),
+    initialState: { files: { a: 'not-a-directory', 'a/child.txt': 'invalid topology' } },
+    signal: signal(),
+  });
+  assert.fail('expected failed initialization');
+} catch (error) {
+  assert(error instanceof WorkflowEnvironmentInitializationError);
+  assert.equal(error.cleanup.status, 'completed');
+  assert.equal(error.cleanup.artifacts, 'discarded');
+}
+const deniedWorkflow = workflow();
+deniedWorkflow.environment.policy.paths = { read: [], write: [] };
+const deniedEnvironment = await factory({
+  workflow: deniedWorkflow,
+  initialState,
+  signal: signal(),
+});
+try {
+  const present = await deniedEnvironment.execute(
+    { tool: 'read_file', input: { path: 'nested/note.txt' } },
+    signal()
+  );
+  const absent = await deniedEnvironment.execute(
+    { tool: 'read_file', input: { path: 'absent.txt' } },
+    signal()
+  );
+  assert.deepEqual(present, {
+    status: 'denied',
+    code: 'permission_denied',
+    evidence: { tool: 'read_file', version: '1', status: 'denied', code: 'permission_denied' },
+  });
+  assert.deepEqual(absent, present);
+  const malformed = await deniedEnvironment.execute(
+    { tool: 'read_file', input: { path: 42 } },
+    signal()
+  );
+  assert.equal(malformed.status, 'invalid');
+  assert.equal(malformed.code, 'invalid_input');
+} finally {
+  assert.equal((await deniedEnvironment.close(signal())).status, 'completed');
+}
+const lifetime = new AbortController();
+const active = await factory({ workflow: workflow(), initialState, signal: lifetime.signal });
+const work = active.execute(
+  { tool: 'read_file', input: { path: 'nested/note.txt' } },
+  lifetime.signal
+);
+lifetime.abort();
+assert.equal((await work).status, 'failed');
+assert.equal(
+  (await active.execute({ tool: 'read_file', input: { path: 'nested/note.txt' } }, signal()))
+    .status,
+  'failed'
+);
+assert.equal((await active.close(signal())).status, 'completed');
+assert.deepEqual(new Set(await names()), before);
+process.stdout.write(
+  `${JSON.stringify({
+    status: 'passed',
+    runtime: process.versions.bun ? 'bun' : 'node',
+    isolation: true,
+    files: true,
+    policy: true,
+    symlinks: true,
+    cleanup: true,
+    cancellation: true,
+    unavailableDaemon: true,
+  })}\n`
+);