@poncho-ai/harness 0.35.0 → 0.36.1

package/src/isolate/run-code-tool.ts

@@ -0,0 +1,220 @@
+// ---------------------------------------------------------------------------
+// run_code tool – sandboxed JavaScript/TypeScript execution in V8 isolates.
+// ---------------------------------------------------------------------------
+
+import { defineTool, type ToolDefinition } from "@poncho-ai/sdk";
+import type { IsolateConfig, IsolateBinding, NetworkConfig } from "../config.js";
+import type { BashEnvironmentManager } from "../vfs/bash-manager.js";
+import { createIsolateRuntime, type IsolateRuntime } from "./runtime.js";
+import { createVfsBindings, createFetchBinding, mergeBuilderBindings } from "./bindings.js";
+import { buildPolyfillPreamble } from "./polyfills.js";
+
+// ---------------------------------------------------------------------------
+// TS stripping via esbuild (dynamic import)
+// ---------------------------------------------------------------------------
+
+let esbuildTransform: typeof import("esbuild").transform | undefined;
+
+async function loadEsbuild(): Promise<typeof import("esbuild").transform> {
+  if (esbuildTransform) return esbuildTransform;
+  try {
+    const mod = await import("esbuild");
+    esbuildTransform = mod.transform;
+    return esbuildTransform;
+  } catch {
+    throw new Error(
+      "Code execution requires esbuild for TypeScript stripping. Run: pnpm add esbuild",
+    );
+  }
+}
+
+async function stripTypeScript(code: string): Promise<string> {
+  const transform = await loadEsbuild();
+  // Wrap in an async function before transforming so that top-level
+  // `await` + `return` don't trigger esbuild's ESM detection
+  // (ESM forbids top-level return).
+  const wrapped = "async function __poncho_wrapper__() {\n" + code + "\n}";
+  const result = await transform(wrapped, { loader: "ts" });
+  // Unwrap: remove the function declaration and closing brace
+  const stripped = result.code
+    .replace(/^async function __poncho_wrapper__\(\)\s*\{\n?/, "")
+    .replace(/\n?\}\s*$/, "");
+  return stripped;
+}
+
+// ---------------------------------------------------------------------------
+// Allowed file extensions for VFS file execution
+// ---------------------------------------------------------------------------
+
+const ALLOWED_EXTENSIONS = new Set([".js", ".ts", ".mjs", ".mts"]);
+
+function hasAllowedExtension(path: string): boolean {
+  const dot = path.lastIndexOf(".");
+  if (dot === -1) return false;
+  return ALLOWED_EXTENSIONS.has(path.slice(dot).toLowerCase());
+}
+
+// ---------------------------------------------------------------------------
+// Factory
+// ---------------------------------------------------------------------------
+
+export interface CreateRunCodeToolOptions {
+  config: IsolateConfig;
+  bashManager: BashEnvironmentManager;
+  /** Pre-built library preamble (from bundler). null if no libraries configured. */
+  libraryPreamble: string | null;
+  /** Dynamic tool description built from config. */
+  description: string;
+  /** Top-level network config — auto-registers fetch binding when set. */
+  network?: NetworkConfig;
+}
+
+export function createRunCodeTool(opts: CreateRunCodeToolOptions): ToolDefinition {
+  const { config, bashManager, libraryPreamble, description } = opts;
+
+  const memoryLimit = config.memoryLimit ?? 128;
+  const timeout = config.timeLimit ?? 10_000;
+  const outputLimit = config.outputLimit ?? 65_536;
+  const codeLimit = config.codeLimit ?? 102_400;
+
+  const runtime: IsolateRuntime = createIsolateRuntime({
+    memoryLimit,
+    timeout,
+    outputLimit,
+  });
+
+  // Static bindings (created once, reused across calls)
+  const staticBindings: Record<string, IsolateBinding> = {};
+
+  // Explicit isolate.apis.fetch takes precedence, then top-level network config
+  if (config.apis?.fetch) {
+    staticBindings.__poncho_fetch = createFetchBinding(config.apis.fetch.allowedDomains);
+  } else if (opts.network) {
+    const net = opts.network;
+    if (net.dangerouslyAllowAll) {
+      staticBindings.__poncho_fetch = createFetchBinding([], net);
+    } else if (net.allowedUrls?.length) {
+      const domains: string[] = [];
+      for (const entry of net.allowedUrls) {
+        const urlStr = typeof entry === "string" ? entry : entry.url;
+        try { domains.push(new URL(urlStr).hostname); } catch { /* skip invalid */ }
+      }
+      if (domains.length > 0) {
+        staticBindings.__poncho_fetch = createFetchBinding(domains, net);
+      }
+    }
+  }
+
+  if (config.bindings) {
+    Object.assign(staticBindings, mergeBuilderBindings(config.bindings));
+  }
+
+  const hasNetwork = "__poncho_fetch" in staticBindings;
+  const polyfillPreamble = buildPolyfillPreamble(hasNetwork);
+
+  return defineTool({
+    name: "run_code",
+    description,
+    inputSchema: {
+      type: "object",
+      properties: {
+        code: {
+          type: "string",
+          description: "JavaScript or TypeScript code to execute",
+        },
+        file: {
+          type: "string",
+          description: "Path to a .js/.ts file in the VFS to execute instead of inline code",
+        },
+      },
+      additionalProperties: false,
+    },
+    handler: async (input, context) => {
+      const code = input.code as string | undefined;
+      const file = input.file as string | undefined;
+
+      // Validate exactly one of code/file
+      if (code && file) {
+        return { error: "Provide either `code` or `file`, not both." };
+      }
+      if (!code && !file) {
+        return { error: "Provide either `code` (inline) or `file` (VFS path)." };
+      }
+
+      // Resolve source code
+      let source: string;
+      if (file) {
+        if (!hasAllowedExtension(file)) {
+          return {
+            error: `File must have a .js, .ts, .mjs, or .mts extension. Got: "${file}"`,
+          };
+        }
+        const tenantId = context.tenantId ?? "__default__";
+        const adapter = bashManager.getAdapter(tenantId);
+        try {
+          source = await adapter.readFile(file);
+        } catch (err) {
+          return {
+            error: `Failed to read file "${file}": ${err instanceof Error ? err.message : String(err)}`,
+          };
+        }
+      } else {
+        source = code!;
+      }
+
+      // Code size limit
+      if (source.length > codeLimit) {
+        return {
+          error: `Code exceeds size limit: ${source.length} bytes > ${codeLimit} byte max.`,
+        };
+      }
+
+      // Strip TypeScript
+      let jsCode: string;
+      try {
+        jsCode = await stripTypeScript(source);
+      } catch (err: unknown) {
+        const msg = err instanceof Error ? err.message : String(err);
+        return { error: `TypeScript parse error: ${msg}` };
+      }
+
+      // Build per-call VFS bindings + merge with static bindings
+      const tenantId = context.tenantId ?? "__default__";
+      const adapter = bashManager.getAdapter(tenantId);
+      const vfsBindings = createVfsBindings(adapter);
+      const allBindings: Record<string, IsolateBinding> = {
+        ...vfsBindings,
+        ...staticBindings,
+      };
+
+      // Execute
+      const result = await runtime.execute(
+        jsCode,
+        allBindings,
+        libraryPreamble,
+        context.abortSignal,
+        polyfillPreamble,
+      );
+
+      // Format output
+      if (result.error) {
+        return {
+          error: result.error.message,
+          errorName: result.error.name,
+          line: result.error.line,
+          column: result.error.column,
+          stdout: result.stdout || undefined,
+          stderr: result.stderr || undefined,
+          executionTimeMs: result.executionTimeMs,
+        };
+      }
+
+      return {
+        result: result.result,
+        stdout: result.stdout || undefined,
+        stderr: result.stderr || undefined,
+        executionTimeMs: result.executionTimeMs,
+      };
+    },
+  });
+}

package/src/isolate/runtime.ts

@@ -0,0 +1,286 @@
+// ---------------------------------------------------------------------------
+// Isolate Runtime – thin wrapper around isolated-vm with async bridging,
+// timeout, memory limits, and console capture.
+// ---------------------------------------------------------------------------
+
+import type { IsolateBinding } from "../config.js";
+
+// ---------------------------------------------------------------------------
+// Types
+// ---------------------------------------------------------------------------
+
+export interface ExecutionResult {
+  result?: unknown;
+  stdout: string;
+  stderr: string;
+  error?: { message: string; name?: string; line?: number; column?: number };
+  executionTimeMs: number;
+}
+
+export interface IsolateRuntime {
+  execute(
+    code: string,
+    bindings: Record<string, IsolateBinding>,
+    preamble: string | null,
+    signal?: AbortSignal,
+    polyfillPreamble?: string | null,
+  ): Promise<ExecutionResult>;
+}
+
+// ---------------------------------------------------------------------------
+// Dynamic import helper
+// ---------------------------------------------------------------------------
+
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+let ivmModule: any | undefined;
+
+async function loadIvm(): Promise<typeof import("isolated-vm")> {
+  if (ivmModule) return ivmModule;
+  try {
+    const mod = await import("isolated-vm");
+    // CJS native module: handle both ESM interop shapes
+    ivmModule = mod.default ?? mod;
+    return ivmModule;
+  } catch {
+    throw new Error(
+      "Code execution requires isolated-vm. Run: pnpm add isolated-vm",
+    );
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Runtime preamble – injected into every isolate context.
+//
+// Provides:
+//   - console.log / console.error / console.warn  (captured to buffers)
+//   - Ergonomic wrappers for __binding_* callbacks (JSON marshal/unmarshal)
+// ---------------------------------------------------------------------------
+
+function buildRuntimePreamble(): string {
+  return `
+// --- console capture ---
+const __stdout = [];
+const __stderr = [];
+let __outputBytes = 0;
+const __outputLimit = typeof __OUTPUT_LIMIT === "number" ? __OUTPUT_LIMIT : 65536;
+
+function __serialize(v) {
+  if (typeof v === "string") return v;
+  try {
+    const seen = new WeakSet();
+    return JSON.stringify(v, function(_k, val) {
+      if (typeof val === "object" && val !== null) {
+        if (seen.has(val)) return "[Circular]";
+        seen.add(val);
+      }
+      return val;
+    }, 2);
+  } catch { return String(v); }
+}
+
+function __capture(arr, args) {
+  const line = Array.from(args).map(__serialize).join(" ");
+  const bytes = line.length;
+  if (__outputBytes + bytes > __outputLimit) {
+    arr.push("[output truncated at " + __outputLimit + " bytes]");
+    __outputBytes = __outputLimit;
+    return;
+  }
+  __outputBytes += bytes;
+  arr.push(line);
+}
+
+const console = {
+  log:   function() { __capture(__stdout, arguments); },
+  info:  function() { __capture(__stdout, arguments); },
+  warn:  function() { __capture(__stderr, arguments); },
+  error: function() { __capture(__stderr, arguments); },
+  debug: function() { __capture(__stdout, arguments); },
+};
+`;
+}
+
+// ---------------------------------------------------------------------------
+// Factory
+// ---------------------------------------------------------------------------
+
+export function createIsolateRuntime(config: {
+  memoryLimit: number;
+  timeout: number;
+  outputLimit: number;
+}): IsolateRuntime {
+  return {
+    async execute(code, bindings, preamble, signal, polyfillPreamble) {
+      const ivm = await loadIvm();
+
+      const isolate = new ivm.Isolate({
+        memoryLimit: config.memoryLimit,
+      });
+
+      // Wire abort → dispose
+      let abortHandler: (() => void) | undefined;
+      let aborted = false;
+      if (signal) {
+        if (signal.aborted) {
+          isolate.dispose();
+          return {
+            stdout: "",
+            stderr: "",
+            error: { message: "Execution cancelled", name: "AbortError" },
+            executionTimeMs: 0,
+          };
+        }
+        abortHandler = () => {
+          aborted = true;
+          isolate.dispose();
+        };
+        signal.addEventListener("abort", abortHandler, { once: true });
+      }
+
+      const t0 = performance.now();
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any
+      let context: any;
+      try {
+        context = await isolate.createContext();
+        const jail = context.global;
+
+        // Inject output limit constant
+        jail.setSync("__OUTPUT_LIMIT", config.outputLimit);
+
+        // Inject binding References and build wrapper declarations
+        const bindingNames = Object.keys(bindings);
+        const wrapperDecls: string[] = [];
+        for (const name of bindingNames) {
+          const binding = bindings[name]!;
+          const ref = new ivm.Reference(async (inputJson: string) => {
+            const input = JSON.parse(inputJson) as Record<string, unknown>;
+            const result = await binding.handler(input);
+            return JSON.stringify(result ?? null);
+          });
+          jail.setSync(`__binding_${name}`, ref);
+          wrapperDecls.push(
+            `async function ${name}(input) {\n` +
+            `  const raw = await __binding_${name}.apply(undefined, [JSON.stringify(input)], { result: { promise: true, copy: true } });\n` +
+            `  return JSON.parse(raw);\n` +
+            `}`,
+          );
+        }
+
+        // Evaluate runtime preamble (console capture + binding wrappers)
+        const runtimePreamble = buildRuntimePreamble() + "\n" + wrapperDecls.join("\n");
+        await context.eval(runtimePreamble, { filename: "<runtime>" });
+
+        // Evaluate polyfill preamble (standard APIs wrapping internal bindings)
+        if (polyfillPreamble) {
+          await context.eval(polyfillPreamble, { filename: "<polyfills>" });
+        }
+
+        // Evaluate library preamble (bundled libs + require shim)
+        if (preamble) {
+          await context.eval(preamble, { filename: "<libraries>" });
+        }
+
+        // Wrap user code in async IIFE and execute via context.eval
+        // (context.eval + promise option handles Reference.apply resolution
+        // correctly, unlike compileScript().run())
+        const wrapped = `(async () => {\n${code}\n})()`;
+        const rawResult = await context.eval(wrapped, {
+          filename: "<user-code>",
+          promise: true,
+          copy: true,
+          timeout: config.timeout,
+        });
+
+        // Read captured stdout/stderr from isolate
+        const stdout = (await context.eval("__stdout.join('\\n')", { copy: true })) as string;
+        const stderr = (await context.eval("__stderr.join('\\n')", { copy: true })) as string;
+
+        // Serialize result
+        let result: unknown;
+        try {
+          result =
+            rawResult === undefined || rawResult === null
+              ? rawResult
+              : JSON.parse(JSON.stringify(rawResult));
+        } catch {
+          result = undefined;
+        }
+
+        return {
+          result,
+          stdout,
+          stderr,
+          executionTimeMs: performance.now() - t0,
+        };
+      } catch (err: unknown) {
+        const elapsed = performance.now() - t0;
+
+        if (aborted) {
+          return {
+            stdout: "",
+            stderr: "",
+            error: { message: "Execution cancelled", name: "AbortError" },
+            executionTimeMs: elapsed,
+          };
+        }
+
+        // Try to recover stdout/stderr captured before the error
+        let stdout = "";
+        let stderr = "";
+        if (context) {
+          try {
+            stdout = (await context.eval("__stdout.join('\\n')", { copy: true })) as string;
+            stderr = (await context.eval("__stderr.join('\\n')", { copy: true })) as string;
+          } catch {
+            // Context may be disposed or unavailable
+          }
+        }
+
+        const error = err instanceof Error ? err : new Error(String(err));
+        const parsed = parseV8Error(error);
+        return {
+          stdout,
+          stderr,
+          error: parsed,
+          executionTimeMs: elapsed,
+        };
+      } finally {
+        if (abortHandler && signal) {
+          signal.removeEventListener("abort", abortHandler);
+        }
+        try {
+          isolate.dispose();
+        } catch {
+          // Already disposed (e.g. via abort)
+        }
+      }
+    },
+  };
+}
+
+// ---------------------------------------------------------------------------
+// Error parsing – extract line/column from V8 stack traces
+// ---------------------------------------------------------------------------
+
+function parseV8Error(error: Error): {
+  message: string;
+  name?: string;
+  line?: number;
+  column?: number;
+} {
+  const result: { message: string; name?: string; line?: number; column?: number } = {
+    message: error.message,
+    name: error.name,
+  };
+
+  // Match "<user-code>:N:N" in stack trace
+  const match = error.stack?.match(/<user-code>:(\d+):(\d+)/);
+  if (match) {
+    // Subtract 1 for the async IIFE wrapper line
+    const rawLine = parseInt(match[1]!, 10);
+    result.line = Math.max(1, rawLine - 1);
+    result.column = parseInt(match[2]!, 10);
+  }
+
+  return result;
+}

package/src/isolate/type-stubs.ts

@@ -0,0 +1,196 @@
+// ---------------------------------------------------------------------------
+// Type stubs – generate TypeScript declarations for the isolate system prompt.
+// ---------------------------------------------------------------------------
+
+import type { IsolateConfig, IsolateBinding } from "../config.js";
+
+/**
+ * Build a TypeScript declaration block listing all APIs available inside the
+ * isolate. Included in the agent system prompt when `run_code` is registered.
+ */
+export function generateIsolateTypeStubs(config: IsolateConfig): string {
+  const lines: string[] = [];
+
+  // Standard APIs
+  lines.push(
+    "// Standard Web/Node.js APIs available in the sandbox",
+    "",
+    "// --- fetch (standard Web API) ---",
+    "declare function fetch(url: string, init?: { method?: string; headers?: Record<string, string>; body?: string }): Promise<Response>;",
+    "declare class Response {",
+    "  readonly ok: boolean;",
+    "  readonly status: number;",
+    "  readonly statusText: string;",
+    "  readonly headers: Headers;",
+    "  text(): Promise<string>;",
+    "  json(): Promise<any>;",
+    "  arrayBuffer(): Promise<ArrayBuffer>;",
+    "  blob(): Promise<Blob>;",
+    "}",
+    "",
+    "// --- fs (Node.js-compatible) ---",
+    "declare const fs: {",
+    "  readFile(path: string, encoding?: string): Promise<string | Buffer>;",
+    "  writeFile(path: string, data: string | Buffer | Uint8Array): Promise<void>;",
+    "  readdir(path: string): Promise<string[]>;",
+    "  mkdir(path: string): Promise<void>;",
+    "  stat(path: string): Promise<{ isFile(): boolean; isDirectory(): boolean; size: number; mtime: Date }>;",
+    "  exists(path: string): Promise<boolean>;",
+    "  unlink(path: string): Promise<void>;",
+    "  rm(path: string): Promise<void>;",
+    "};",
+    "",
+    "// --- path ---",
+    "declare const path: {",
+    "  join(...parts: string[]): string;",
+    "  resolve(...parts: string[]): string;",
+    "  basename(p: string, ext?: string): string;",
+    "  dirname(p: string): string;",
+    "  extname(p: string): string;",
+    "};",
+    "",
+    "// --- Buffer, encoding, crypto ---",
+    "declare class Buffer extends Uint8Array {",
+    "  static from(input: string | ArrayBuffer | Uint8Array | number[], encoding?: string): Buffer;",
+    "  static alloc(size: number, fill?: number): Buffer;",
+    "  static concat(list: Uint8Array[]): Buffer;",
+    "  toString(encoding?: 'utf-8' | 'base64' | 'hex'): string;",
+    "}",
+    "declare function atob(data: string): string;",
+    "declare function btoa(data: string): string;",
+    "declare function setTimeout(fn: () => void, ms?: number): number;",
+    "declare function clearTimeout(id: number): void;",
+    "declare const crypto: { randomUUID(): string; getRandomValues(arr: Uint8Array): Uint8Array };",
+    "declare function structuredClone<T>(value: T): T;",
+  );
+
+  // Console
+  lines.push(
+    "",
+    "// Console (output captured and returned in tool result)",
+    "declare const console: {",
+    "  log(...args: unknown[]): void; error(...args: unknown[]): void;",
+    "  warn(...args: unknown[]): void; info(...args: unknown[]): void;",
+    "  table(data: unknown): void; time(label?: string): void; timeEnd(label?: string): void;",
+    "};",
+  );
+
+  // Builder custom bindings
+  if (config.bindings) {
+    const entries = Object.entries(config.bindings);
+    if (entries.length > 0) {
+      lines.push("", "// Custom bindings");
+      for (const [name, binding] of entries) {
+        lines.push(formatBindingStub(name, binding));
+      }
+    }
+  }
+
+  // Libraries
+  if (config.libraries?.length) {
+    lines.push(
+      "",
+      `// Pre-bundled libraries (use require())`,
+      `declare function require(name: ${config.libraries.map((l) => `"${l}"`).join(" | ")}): any;`,
+    );
+  }
+
+  return lines.join("\n");
+}
+
+/**
+ * Build the dynamic tool description for `run_code` based on the isolate config.
+ */
+export function buildRunCodeDescription(
+  config: IsolateConfig,
+  hasNetwork?: boolean,
+): string {
+  const parts: string[] = [
+    "Execute JavaScript/TypeScript code in a sandboxed V8 isolate with standard Node.js/Web APIs.",
+    "",
+    "Input: provide either `code` (inline string) or `file` (path to a .js/.ts file in the VFS).",
+    "",
+    "Available standard APIs:",
+    "- fs.readFile(path, encoding?) / fs.writeFile(path, data) / fs.readdir(path) / fs.mkdir(path)",
+    "- fs.stat(path) / fs.exists(path) / fs.unlink(path)",
+    "- path.join() / path.resolve() / path.basename() / path.dirname() / path.extname()",
+    "- Buffer.from() / Buffer.alloc() / Buffer.concat() / buf.toString(encoding)",
+    "- atob() / btoa() / setTimeout() / crypto.randomUUID() / structuredClone()",
+    "- console.log() / console.error() / console.table()",
+  ];
+
+  if (hasNetwork || config.apis?.fetch) {
+    parts.push(
+      "- fetch(url, init?) — standard Web fetch API with Response.text(), .json(), .arrayBuffer()",
+    );
+  } else {
+    parts.push(
+      "- fetch() — not available (enable `network` in poncho.config.js)",
+    );
+  }
+
+  if (config.bindings) {
+    for (const [name, binding] of Object.entries(config.bindings)) {
+      parts.push(`- ${name}({...}) — ${binding.description}`);
+    }
+  }
+
+  if (config.libraries?.length) {
+    parts.push(
+      "",
+      `Pre-bundled libraries (use require()):`,
+      `- ${config.libraries.join(", ")}`,
+    );
+  }
+
+  const memoryLimit = config.memoryLimit ?? 128;
+  const timeLimit = config.timeLimit ?? 10_000;
+  const codeLimit = config.codeLimit ?? 102_400;
+  parts.push(
+    "",
+    "Notes:",
+    "- Code is wrapped in an async IIFE. Use `return` to return a value.",
+    "- Files written during execution persist even if the code throws an error.",
+    "- TypeScript is supported (type annotations are stripped before execution).",
+    `- Execution timeout: ${timeLimit / 1000}s. Memory limit: ${memoryLimit}MB. Max code size: ${Math.round(codeLimit / 1024)}KB.`,
+  );
+
+  return parts.join("\n");
+}
+
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+
+function formatBindingStub(name: string, binding: IsolateBinding): string {
+  const schema = binding.inputSchema;
+  const props = schema.properties ?? {};
+  const required = new Set(schema.required ?? []);
+  const params = Object.entries(props)
+    .map(([k, v]) => {
+      const opt = required.has(k) ? "" : "?";
+      const tsType = jsonSchemaToTsType(v);
+      return `${k}${opt}: ${tsType}`;
+    })
+    .join("; ");
+
+  return `declare function ${name}(input: { ${params} }): Promise<unknown>; // ${binding.description}`;
+}
+
+function jsonSchemaToTsType(schema: { type?: string; [key: string]: unknown }): string {
+  switch (schema.type) {
+    case "string":
+      return "string";
+    case "number":
+    case "integer":
+      return "number";
+    case "boolean":
+      return "boolean";
+    case "array":
+      return "unknown[]";
+    case "object":
+      return "Record<string, unknown>";
+    default:
+      return "unknown";
+  }
+}