@alexkroman1/aai 0.12.3 → 1.0.3

package/host/_run-code.ts

@@ -0,0 +1,217 @@
+// Copyright 2025 the AAI authors. MIT license.
+/**
+ * run_code built-in tool — executes user JavaScript in a fresh `node:vm`
+ * context with no network, filesystem, or process access.
+ */
+
+import vm from "node:vm";
+import { z } from "zod";
+import { RUN_CODE_TIMEOUT_MS } from "../sdk/constants.ts";
+import type { ToolDef } from "../sdk/types.ts";
+import { errorMessage } from "../sdk/utils.ts";
+
+const SKIPPED_CLASS_KEYS = new Set(["constructor", "prototype", "length", "name"]);
+
+/**
+ * Copy static members from a class constructor to a wrapper function,
+ * skipping built-in keys that must not be forwarded.
+ */
+// biome-ignore lint/complexity/noBannedTypes: copying descriptors from arbitrary class constructors
+function copyStaticMembers(src: Function, dst: Function): void {
+  for (const key of Object.getOwnPropertyNames(src)) {
+    if (SKIPPED_CLASS_KEYS.has(key)) continue;
+    try {
+      const desc = Object.getOwnPropertyDescriptor(src, key);
+      if (desc) Object.defineProperty(dst, key, desc);
+    } catch {
+      // Skip non-configurable properties
+    }
+  }
+}
+
+/**
+ * Neuter the `.constructor` chain on a host function or class constructor.
+ *
+ * For plain functions: wraps the function so calling `.constructor` or
+ * `.constructor.constructor` no longer exposes the host `Function`.
+ *
+ * For class constructors: additionally copies static methods and neutralizes
+ * `prototype.constructor` so instances created via `new` also cannot escape.
+ *
+ * This prevents sandbox code from reaching the host `Function` constructor
+ * via patterns like `fn.constructor.constructor('return process')()`.
+ */
+// biome-ignore lint/complexity/noBannedTypes: wrapping arbitrary functions and class constructors
+function neutralizeConstructor<T extends Function>(fn: T): T {
+  const hasPrototype = typeof fn.prototype === "object" && fn.prototype !== null;
+
+  function Wrapper(this: unknown, ...args: unknown[]) {
+    if (hasPrototype) {
+      return new (fn as unknown as new (...a: unknown[]) => unknown)(...args);
+    }
+    return (fn as unknown as (...a: unknown[]) => unknown)(...args);
+  }
+
+  if (hasPrototype) {
+    copyStaticMembers(fn, Wrapper);
+    // Neuter prototype.constructor so instances can't escape either.
+    if (Wrapper.prototype) {
+      Object.defineProperty(Wrapper.prototype, "constructor", {
+        value: undefined,
+        writable: false,
+        configurable: false,
+      });
+    }
+  }
+
+  Object.defineProperty(Wrapper, "constructor", {
+    value: undefined,
+    writable: false,
+    configurable: false,
+  });
+
+  return Wrapper as unknown as T;
+}
+
+const runCodeParams = z.object({
+  code: z.string().describe("JavaScript code to execute. Use console.log() for output."),
+});
+
+/**
+ * Execute JavaScript code inside a fresh `node:vm` context.
+ *
+ * Each invocation creates a disposable VM context with:
+ * - No filesystem access (`node:fs` and other built-ins unavailable)
+ * - No network access (`fetch`, `http` unavailable)
+ * - No child process spawning
+ * - No environment variable access (`process` unavailable)
+ * - Execution timeout (default 5 s)
+ *
+ * The context is discarded after execution, so no state leaks between
+ * invocations or across sessions.
+ */
+export function createRunCode(): ToolDef<typeof runCodeParams> & { guidance: string } {
+  return {
+    guidance:
+      "You MUST use the run_code tool for ANY question involving math, counting, calculations, " +
+      "data processing, or code. NEVER do mental math or recite code verbally. " +
+      "run_code executes JavaScript (not Python). Always write JavaScript.",
+    description:
+      "Execute JavaScript code in a sandbox and return the output. Use this for calculations, data transformations, string manipulation, or any task that benefits from running code. Output is captured from console.log(). No network or filesystem access.",
+    parameters: runCodeParams,
+    async execute(args) {
+      return executeInIsolate(args.code);
+    },
+  };
+}
+
+/**
+ * Execute user code in a fresh `node:vm` context.
+ *
+ * @remarks
+ * The VM context only exposes standard ECMAScript globals and a console
+ * object that captures output. Node.js APIs (`process`, `require`,
+ * `import()`) are not available inside the sandbox.
+ */
+export async function executeInIsolate(code: string): Promise<string | { error: string }> {
+  const output: string[] = [];
+  const capture = (...args: unknown[]) => output.push(args.map(String).join(" "));
+
+  // Prevent timer callbacks from leaking into host event loop after execution.
+  const activeTimers = new Set<ReturnType<typeof setTimeout>>();
+
+  const sandboxSetTimeout = (
+    fn: (...args: unknown[]) => void,
+    delay?: number,
+    ...args: unknown[]
+  ): ReturnType<typeof setTimeout> => {
+    const id = setTimeout(
+      (...a: unknown[]) => {
+        activeTimers.delete(id);
+        fn(...a);
+      },
+      delay,
+      ...args,
+    );
+    activeTimers.add(id);
+    return id;
+  };
+
+  const sandboxClearTimeout = (id?: ReturnType<typeof setTimeout>): void => {
+    if (id !== undefined) {
+      activeTimers.delete(id);
+      clearTimeout(id);
+    }
+  };
+
+  const sandboxSetInterval = (
+    fn: (...args: unknown[]) => void,
+    delay?: number,
+    ...args: unknown[]
+  ): ReturnType<typeof setInterval> => {
+    const id = setInterval(fn, delay, ...args);
+    activeTimers.add(id);
+    return id;
+  };
+
+  const sandboxClearInterval = (id?: ReturnType<typeof setInterval>): void => {
+    if (id !== undefined) {
+      activeTimers.delete(id);
+      clearInterval(id);
+    }
+  };
+
+  const context = vm.createContext(
+    {
+      // Console methods wrapped to prevent .constructor escape to host Function.
+      console: Object.freeze({
+        log: neutralizeConstructor(capture),
+        info: neutralizeConstructor(capture),
+        warn: neutralizeConstructor(capture),
+        error: neutralizeConstructor(capture),
+        debug: neutralizeConstructor(capture),
+      }),
+      // Wrapped timers — neutralized to prevent .constructor escape.
+      setTimeout: neutralizeConstructor(sandboxSetTimeout),
+      clearTimeout: neutralizeConstructor(sandboxClearTimeout),
+      setInterval: neutralizeConstructor(sandboxSetInterval),
+      clearInterval: neutralizeConstructor(sandboxClearInterval),
+      // Standard web-compat globals — constructor chain neutered.
+      URL: neutralizeConstructor(URL),
+      URLSearchParams: neutralizeConstructor(URLSearchParams),
+      TextEncoder: neutralizeConstructor(TextEncoder),
+      TextDecoder: neutralizeConstructor(TextDecoder),
+      atob: neutralizeConstructor(atob),
+      btoa: neutralizeConstructor(btoa),
+      structuredClone: neutralizeConstructor(structuredClone),
+    },
+    {
+      // Block string-based code generation within the sandbox realm.
+      codeGeneration: { strings: false, wasm: false },
+    },
+  );
+
+  try {
+    // Wrap user code in an async IIFE so top-level `await` works.
+    const wrapped = `(async () => {\n${code}\n})()`;
+    const script = new vm.Script(wrapped, { filename: "run_code.js" });
+
+    // runInContext's `timeout` enforces the execution limit.
+    const result = await script.runInContext(context, { timeout: RUN_CODE_TIMEOUT_MS });
+    void result;
+
+    const text = output.join("\n").trim();
+    return text || "Code ran successfully (no output)";
+  } catch (err: unknown) {
+    return { error: errorMessage(err) };
+  } finally {
+    // Cancel all sandbox timers that are still pending. This prevents
+    // setInterval/setTimeout callbacks from running in the host event loop
+    // after the sandbox execution has completed or timed out.
+    for (const id of activeTimers) {
+      clearTimeout(id);
+      clearInterval(id);
+    }
+    activeTimers.clear();
+  }
+}

package/host/_runtime-conformance.ts

@@ -0,0 +1,143 @@
+// Copyright 2025 the AAI authors. MIT license.
+/**
+ * Shared runtime conformance tests.
+ *
+ * Both the self-hosted direct executor and the platform sandbox must satisfy
+ * the same behavioral contract. This module defines that contract as a
+ * reusable test suite that can be wired to either runtime.
+ *
+ * Inspired by Nitro's `testNitro()` pattern: one test fixture, many runtimes.
+ *
+ * @example Direct executor (unit test)
+ * ```ts
+ * import { testRuntime } from "./_runtime-conformance.ts";
+ *
+ * testRuntime("direct", () => {
+ *   const exec = createRuntime({ agent: CONFORMANCE_AGENT, env: { MY_VAR: "test-value" } });
+ *   return { executeTool: exec.executeTool, hooks: exec.hooks };
+ * });
+ * ```
+ *
+ * @example Sandbox (integration test in aai-server)
+ * ```ts
+ * // Internal module — import the .ts source directly from this package.
+ * import { testRuntime } from "../../aai/host/_runtime-conformance.ts";
+ *
+ * testRuntime("sandbox", async () => {
+ *   // ... start isolate with a bundled agent
+ *   return { executeTool: buildExecuteTool(...), hooks: buildHookInvoker(...) };
+ * });
+ * ```
+ */
+
+import { describe, expect, test } from "vitest";
+import { z } from "zod";
+import type { ExecuteTool } from "../sdk/_internal-types.ts";
+import type { AgentDef } from "../sdk/types.ts";
+
+// ── Shared context type ────────────────────────────────────────────────────
+
+/**
+ * Minimal runtime surface needed for conformance tests.
+ *
+ * Both `Runtime` and `buildExecuteTool`/`buildHookInvoker` from the
+ * sandbox produce objects that satisfy this interface.
+ */
+export type RuntimeTestContext = {
+  executeTool: ExecuteTool;
+};
+
+// ── Conformance agent ──────────────────────────────────────────────────────
+
+/** Agent definition used by the conformance suite (direct executor path). */
+export const CONFORMANCE_AGENT: AgentDef = {
+  name: "conformance-test",
+  systemPrompt: "Conformance test agent.",
+  greeting: "Hello!",
+  maxSteps: 5,
+  state: () => ({ count: 0, lastTurn: "" }),
+  tools: {
+    echo: {
+      description: "Echo input",
+      parameters: z.object({ text: z.string() }),
+      execute: ({ text }: { text: string }) => `echo:${text}`,
+    },
+    get_env: {
+      description: "Get MY_VAR from env",
+      execute: (_args: unknown, ctx) => ctx.env.MY_VAR ?? "missing",
+    },
+    get_state: {
+      description: "Get session state",
+      execute: (_args: unknown, ctx) => JSON.stringify(ctx.state),
+    },
+    echo_messages: {
+      description: "Return messages as JSON",
+      execute: (_args: unknown, ctx) => JSON.stringify(ctx.messages),
+    },
+    kv_roundtrip: {
+      description: "KV set then get",
+      parameters: z.object({ value: z.string() }),
+      execute: async ({ value }: { value: string }, ctx) => {
+        await ctx.kv.set("test-key", value);
+        const result = await ctx.kv.get<string>("test-key");
+        return `stored:${JSON.stringify(result)}`;
+      },
+    },
+  },
+};
+
+// ── Shared conformance suite ───────────────────────────────────────────────
+
+/**
+ * Run the runtime conformance test suite against a given runtime context.
+ *
+ * The `getContext` callback is invoked once per test to retrieve the
+ * current {@link RuntimeTestContext}. This allows the caller to set up
+ * the runtime in a `beforeAll` and return it lazily.
+ *
+ * All tests assume the runtime was created with {@link CONFORMANCE_AGENT}
+ * (or its bundle equivalent) and `env: { MY_VAR: "test-value" }`.
+ */
+export function testRuntime(label: string, getContext: () => RuntimeTestContext): void {
+  describe(`runtime conformance: ${label}`, () => {
+    // ── Tool execution ───────────────────────────────────────────────
+
+    test("executes tool and returns result", async () => {
+      const { executeTool } = getContext();
+      const result = await executeTool("echo", { text: "hello" }, "s1", []);
+      expect(result).toBe("echo:hello");
+    });
+
+    test("tool receives env variables", async () => {
+      const { executeTool } = getContext();
+      const result = await executeTool("get_env", {}, "s1", []);
+      expect(result).toBe("test-value");
+    });
+
+    test("tool receives conversation messages", async () => {
+      const { executeTool } = getContext();
+      const msgs = [
+        { role: "user" as const, content: "hi" },
+        { role: "assistant" as const, content: "hello" },
+      ];
+      const result = await executeTool("echo_messages", {}, "s1", msgs);
+      expect(JSON.parse(result)).toEqual(msgs);
+    });
+
+    test("KV round-trip through tool context", async () => {
+      const { executeTool } = getContext();
+      const result = await executeTool("kv_roundtrip", { value: "abc" }, "s1", []);
+      expect(result).toBe('stored:"abc"');
+    });
+
+    // ── Session state ────────────────────────────────────────────────
+
+    test("session state is initialized from factory", async () => {
+      const { executeTool } = getContext();
+      const result = await executeTool("get_state", {}, "state-init", []);
+      const state = JSON.parse(result);
+      expect(state).toHaveProperty("count", 0);
+      expect(state).toHaveProperty("lastTurn", "");
+    });
+  });
+}

package/host/_test-utils.ts

@@ -0,0 +1,276 @@
+// Copyright 2025 the AAI authors. MIT license.
+
+import { readFileSync } from "node:fs";
+import { resolve } from "node:path";
+import { createNanoEvents } from "nanoevents";
+import { vi } from "vitest";
+import type { AgentConfig } from "../sdk/_internal-types.ts";
+import type { ClientSink } from "../sdk/protocol.ts";
+import type { AgentDef, ToolContext, ToolDef } from "../sdk/types.ts";
+import { DEFAULT_SYSTEM_PROMPT } from "../sdk/types.ts";
+import { createRuntime } from "./runtime.ts";
+import type { S2sEvents, S2sHandle } from "./s2s.ts";
+import type { Session } from "./session.ts";
+import { _internals, type S2sSessionOptions } from "./session.ts";
+
+/** Yield to the microtask queue so pending promises settle. */
+export function flush(): Promise<void> {
+  return new Promise<void>((r) => queueMicrotask(r));
+}
+
+export function createMockToolContext(overrides?: Partial<ToolContext>): ToolContext {
+  return {
+    env: {},
+    state: {},
+    kv: {} as never,
+    messages: [],
+    sessionId: "test-session",
+    ...overrides,
+  };
+}
+
+export function makeTool(overrides?: Partial<ToolDef>): ToolDef {
+  return { description: "test tool", execute: () => "ok", ...overrides };
+}
+
+export function makeAgent(overrides?: Partial<AgentDef>): AgentDef {
+  return {
+    name: "test-agent",
+    systemPrompt: "Be helpful.",
+    greeting: "Hello!",
+    maxSteps: 5,
+    tools: {},
+    ...overrides,
+  };
+}
+
+export function makeConfig(overrides: Partial<AgentConfig> = {}): AgentConfig {
+  return {
+    name: "test-agent",
+    systemPrompt: DEFAULT_SYSTEM_PROMPT,
+    greeting: "Hello",
+    ...overrides,
+  };
+}
+
+/** Create a stub Session with all methods as vi.fn() spies. */
+export function makeStubSession(overrides?: Partial<Session>): Session {
+  return {
+    start: vi.fn(() => Promise.resolve()),
+    stop: vi.fn(() => Promise.resolve()),
+    onAudio: vi.fn(),
+    onAudioReady: vi.fn(),
+    onCancel: vi.fn(),
+    onReset: vi.fn(),
+    onHistory: vi.fn(),
+    waitForTurn: vi.fn(() => Promise.resolve()),
+    ...overrides,
+  };
+}
+
+// ─── Session test helpers ───────────────────────────────────────────────────
+
+export type MockS2sHandle = S2sHandle & {
+  _fire: <K extends keyof S2sEvents>(type: K, ...args: Parameters<S2sEvents[K]>) => void;
+};
+
+/** Create a mock S2sHandle backed by nanoevents. */
+export function makeMockHandle(): MockS2sHandle {
+  const emitter = createNanoEvents<S2sEvents>();
+  return {
+    on: emitter.on.bind(emitter),
+    sendAudio: vi.fn(),
+    sendToolResult: vi.fn(),
+    updateSession: vi.fn(),
+    resumeSession: vi.fn(),
+    close: vi.fn(),
+    _fire<K extends keyof S2sEvents>(type: K, ...args: Parameters<S2sEvents[K]>) {
+      emitter.emit(type, ...args);
+    },
+  };
+}
+
+/** Minimal client that tracks events and audio. All methods are vi.fn() spies. */
+export function makeClient(): ClientSink & {
+  events: unknown[];
+  audioChunks: Uint8Array[];
+  audioDoneCount: number;
+} {
+  const events: unknown[] = [];
+  const audioChunks: Uint8Array[] = [];
+  let audioDoneCount = 0;
+  return {
+    open: true,
+    events,
+    audioChunks,
+    get audioDoneCount() {
+      return audioDoneCount;
+    },
+    event: vi.fn((e: unknown) => {
+      events.push(e);
+    }),
+    playAudioChunk: vi.fn((chunk: Uint8Array) => {
+      audioChunks.push(chunk);
+    }),
+    playAudioDone: vi.fn(() => {
+      audioDoneCount++;
+    }),
+  };
+}
+
+export const silentLogger: {
+  info: (...args: unknown[]) => void;
+  warn: (...args: unknown[]) => void;
+  error: (...args: unknown[]) => void;
+  debug: (...args: unknown[]) => void;
+} = {
+  info: vi.fn(),
+  warn: vi.fn(),
+  error: vi.fn(),
+  debug: vi.fn(),
+};
+
+export function makeSessionOpts(overrides?: Partial<S2sSessionOptions>): S2sSessionOptions {
+  return {
+    id: "session-1",
+    agent: "test-agent",
+    client: makeClient(),
+    agentConfig: {
+      name: "test-agent",
+      systemPrompt: DEFAULT_SYSTEM_PROMPT,
+      greeting: "Hello!",
+    },
+    toolSchemas: [],
+    apiKey: "test-key",
+    s2sConfig: {
+      wssUrl: "wss://fake",
+      inputSampleRate: 16_000,
+      outputSampleRate: 24_000,
+    },
+    executeTool: vi.fn(async () => "tool-result"),
+    createWebSocket: vi.fn(),
+    logger: silentLogger,
+    ...overrides,
+  };
+}
+
+// ─── Fixture replay helpers ──────────────────────────────────────────────────
+
+const FIXTURE_DIR = resolve(import.meta.dirname, "fixtures");
+
+/** Load a JSON fixture from fixtures/. */
+export function loadFixture<T = Record<string, unknown>[]>(name: string): T {
+  return JSON.parse(readFileSync(resolve(FIXTURE_DIR, name), "utf-8"));
+}
+
+/**
+ * Wire-format → event translator: maps a single raw S2S API message to
+ * a `_fire()` call on the mock handle.  Returns false if the message
+ * type is not dispatchable (audio, content_part, unknown).
+ */
+type FireFn = (handle: MockS2sHandle, msg: Record<string, unknown>) => void;
+
+const FIXTURE_DISPATCH: Record<string, FireFn> = {
+  "session.ready": (h, m) => h._fire("ready", { sessionId: m.session_id as string }),
+  "session.updated": () => {
+    /* dropped — no longer dispatched */
+  },
+  "session.error": (h, m) => {
+    const code = m.code as string;
+    if (code === "session_not_found" || code === "session_forbidden") h._fire("sessionExpired");
+    else h._fire("error", new Error(m.message as string));
+  },
+  error: (h, m) => h._fire("error", new Error(m.message as string)),
+  "input.speech.started": (h) => h._fire("event", { type: "speech_started" }),
+  "input.speech.stopped": (h) => h._fire("event", { type: "speech_stopped" }),
+  "transcript.user": (h, m) =>
+    h._fire("event", { type: "user_transcript", text: m.text as string }),
+  "reply.started": (h, m) => h._fire("replyStarted", { replyId: (m.reply_id as string) ?? "" }),
+  "transcript.agent": (h, m) =>
+    h._fire("event", {
+      type: "agent_transcript",
+      text: (m.text as string) ?? "",
+      _interrupted: m.interrupted === true,
+    }),
+  "tool.call": (h, m) =>
+    h._fire("event", {
+      type: "tool_call",
+      toolCallId: m.call_id as string,
+      toolName: m.name as string,
+      args: (m.args as Record<string, unknown>) ?? {},
+    }),
+  "reply.done": (h, m) => {
+    if (m.status === "interrupted") h._fire("event", { type: "cancelled" });
+    else h._fire("event", { type: "reply_done" });
+  },
+};
+
+/**
+ * Replay recorded S2S API messages through a MockS2sHandle.
+ *
+ * Converts raw wire-format JSON (from fixtures/) into typed `_fire()` calls.
+ * This is the inverse of `dispatchS2sMessage` in s2s.ts — it translates
+ * snake_case API fields to camelCase event payloads.
+ *
+ * Messages that don't map to an event (audio, `reply.content_part.*`) are skipped.
+ */
+export function replayFixtureMessages(
+  handle: MockS2sHandle,
+  messages: Record<string, unknown>[],
+): void {
+  for (const msg of messages) {
+    FIXTURE_DISPATCH[msg.type as string]?.(handle, msg);
+  }
+}
+
+// ─── Real-executor fixture replay ────────────────────────────────────────────
+
+/**
+ * Create a real Runtime-backed session for fixture replay testing.
+ *
+ * Uses a real `Runtime` (real tool execution, real hooks) but replaces the
+ * S2S WebSocket with a mock handle so fixture messages can be replayed
+ * through the full orchestration layer.
+ *
+ * Exercises: AgentDef → toAgentConfig → tool schemas → Zod arg validation
+ * → executeToolCall → session orchestration (reply guards, tool buffering,
+ * turnPromise chaining).
+ *
+ * Call `cleanup()` when done to restore the connectS2s spy.
+ */
+export function createFixtureSession(
+  // biome-ignore lint/suspicious/noExplicitAny: test helper accepts any agent state type
+  agent: AgentDef<any>,
+  opts?: { env?: Record<string, string> },
+) {
+  const mockHandle = makeMockHandle();
+  const connectSpy = vi.spyOn(_internals, "connectS2s").mockResolvedValue(mockHandle);
+  const client = makeClient();
+
+  const executor = createRuntime({
+    agent,
+    env: opts?.env ?? {},
+    logger: silentLogger,
+  });
+
+  const session = executor.createSession({
+    id: "fixture-session",
+    agent: agent.name,
+    client,
+  });
+
+  return {
+    session,
+    client,
+    mockHandle,
+    executor,
+    /** Replay a fixture file through the session's S2S handle. */
+    replay(fixtureName: string) {
+      replayFixtureMessages(mockHandle, loadFixture(fixtureName));
+    },
+    /** Restore the connectS2s spy. Call in afterEach. */
+    cleanup() {
+      connectSpy.mockRestore();
+    },
+  };
+}