@mammothb/pi-eval 1.0.0

package/LICENSE

@@ -0,0 +1,18 @@
+MIT License
+
+Copyright (c) 2026 mammothb
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
+associated documentation files (the "Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the
+following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial
+portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT
+LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO
+EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
+USE OR OTHER DEALINGS IN THE SOFTWARE.

package/README.md

@@ -0,0 +1,29 @@
+# pi-eval
+
+A [pi](https://pi.dev) extension that adds an `eval` tool for executing
+JavaScript and Python code in isolated subprocesses.
+
+## Usage
+
+Once installed, the LLM can call the `eval` tool to run code snippets without
+managing temp files. Each call spawns a fresh subprocess — no state persists
+between calls.
+
+### Tool parameters
+
+| Parameter          | Type   | Default    | Description |
+| ------------------ | ------ | ---------- | ----------- |
+| `language`         | string | _(required)_ | Programming language: `"javascript"` or `"python"` |
+| `code`             | string | _(required)_ | Code to execute |
+| `nodeModulesPath`  | string | —          | Path to node_modules for `require()` resolution |
+| `pythonPath`       | string | —          | Path to python3 binary (e.g., `.venv/bin/python3`) |
+
+### Features
+
+- **JavaScript**: Writes code to a temp file, spawns `node` as a subprocess.
+  Console output is captured as labeled `STDOUT:` / `STDERR:` sections.
+- **Python**: Spawns `python3` with `-c`, capturing stdout/stderr identically.
+  Supports virtual environments via the `pythonPath` parameter.
+- **Safety**: 30-second timeout, 1 MB output cap, abort-on-Escape support.
+- **Dependency isolation**: Use `nodeModulesPath` to resolve packages from a
+  project's `node_modules/`. Use `pythonPath` to target a venv.

package/index.ts

@@ -0,0 +1,6 @@
+import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
+import { createEvalTool } from "./src/eval.js";
+
+export default function (pi: ExtensionAPI) {
+  pi.registerTool(createEvalTool());
+}

package/package.json

@@ -0,0 +1,27 @@
+{
+  "name": "@mammothb/pi-eval",
+  "version": "1.0.0",
+  "description": "A pi extension that adds an eval tool for executing JavaScript and Python code in isolated subprocesses",
+  "keywords": [
+    "pi-package"
+  ],
+  "license": "MIT",
+  "files": [
+    "index.ts",
+    "src"
+  ],
+  "pi": {
+    "extensions": [
+      "./index.ts"
+    ]
+  },
+  "devDependencies": {
+    "typebox": "^1.2.0"
+  },
+  "peerDependencies": {
+    "@earendil-works/pi-ai": "*",
+    "@earendil-works/pi-coding-agent": "*",
+    "@earendil-works/pi-tui": "*",
+    "typebox": "*"
+  }
+}

package/src/eval.ts

@@ -0,0 +1,85 @@
+import type { ToolDefinition } from "@earendil-works/pi-coding-agent";
+import { Type } from "typebox";
+import { executeJavaScript } from "./javascript.js";
+import { executePython } from "./python.js";
+import {
+  EvalCancelledError,
+  type EvalDetails,
+  EvalUnsupportedLanguageError,
+} from "./types.js";
+
+export const TIMEOUT_MS = 30_000;
+
+const Parameters = Type.Object({
+  language: Type.Union([Type.Literal("javascript"), Type.Literal("python")]),
+  code: Type.String({ description: "Code to execute" }),
+  pythonPath: Type.Optional(
+    Type.String({
+      description:
+        "Path to python3 binary (e.g., '.venv/bin/python3' for venvs). " +
+        "Defaults to 'python3'.",
+    }),
+  ),
+  nodeModulesPath: Type.Optional(
+    Type.String({
+      description:
+        "Path to a node_modules directory. When set, NODE_PATH is passed " +
+        "to the subprocess so require() resolves from this directory. " +
+        "Use './node_modules' for project-local packages.",
+    }),
+  ),
+});
+
+export function createEvalTool(): ToolDefinition<
+  typeof Parameters,
+  EvalDetails
+> {
+  return {
+    name: "eval",
+    label: "Eval",
+    description: `Execute JavaScript or Python code in an isolated subprocess.
+
+- Each call is a fresh subprocess — no state persists between calls
+- 30-second timeout; press Escape to cancel a running evaluation
+- Working directory is the agent's current working directory (like bash)
+- Use nodeModulesPath to resolve require() from a project directory
+- Use pythonPath to target a virtual environment`,
+    promptSnippet:
+      "Execute JavaScript or Python code in an isolated subprocess",
+    parameters: Parameters,
+    async execute(_toolCallId, params, signal, _onUpdate, ctx) {
+      const { language, code, pythonPath, nodeModulesPath } = params;
+
+      // Validate language (belt-and-suspenders: TypeBox schema already constrains it,
+      // but a raw API call could bypass validation)
+      if (language !== "javascript" && language !== "python") {
+        throw new EvalUnsupportedLanguageError(language);
+      }
+
+      if (signal?.aborted) {
+        throw new EvalCancelledError();
+      }
+
+      // Build timeout signal — passed separately from the user signal so
+      // subprocess.ts can discriminate timeout vs user cancel
+      const timeoutSignal = AbortSignal.timeout(TIMEOUT_MS);
+
+      if (timeoutSignal.aborted) {
+        throw new EvalCancelledError();
+      }
+
+      if (language === "python") {
+        return executePython(code, pythonPath, signal, timeoutSignal, ctx.cwd);
+      }
+
+      // ── JavaScript execution via temp file + node subprocess ──
+      return executeJavaScript(
+        code,
+        nodeModulesPath,
+        signal,
+        timeoutSignal,
+        ctx.cwd,
+      );
+    },
+  };
+}

package/src/format.ts

@@ -0,0 +1,59 @@
+import type { AgentToolResult } from "@earendil-works/pi-coding-agent";
+import {
+  type EvalDetails,
+  EvalToolError,
+  type SubprocessResult,
+} from "./types.js";
+
+export const MAX_OUTPUT = 1024 * 1024; // 1 MB
+
+export interface FormatOutputOptions {
+  stdout: string;
+  stderr: string;
+  truncated?: boolean;
+  exitSignal?: string | null;
+}
+
+export function formatOutput(opts: FormatOutputOptions): string {
+  const parts: string[] = [];
+  parts.push(`STDOUT:\n${opts.stdout || "(no output)"}`);
+  if (opts.stderr) {
+    parts.push(`STDERR:\n${opts.stderr}`);
+  }
+  if (opts.truncated) {
+    parts.push("[Output truncated at 1 MB]");
+  }
+  if (opts.exitSignal) {
+    parts.push(`[Process killed by signal: ${opts.exitSignal}]`);
+  }
+  return parts.join("\n\n");
+}
+
+/**
+ * Convert a subprocess result into an AgentToolResult, throwing on failure.
+ *
+ * @throws {EvalToolError} if exitCode is non-zero or the process was killed by a signal.
+ */
+export function assertSuccessOrThrow(
+  language: string,
+  result: SubprocessResult,
+): AgentToolResult<EvalDetails> {
+  const output = formatOutput({
+    stdout: result.stdout,
+    stderr: result.stderr,
+    truncated: result.truncated,
+    exitSignal: result.exitSignal,
+  });
+  // Treat non-zero exit code or signal kill as failure
+  if (result.exitCode !== 0 || result.exitSignal != null) {
+    throw new EvalToolError(output, "NON_ZERO_EXIT");
+  }
+  return {
+    content: [{ type: "text" as const, text: output }],
+    details: {
+      language,
+      exitCode: result.exitCode,
+      exitSignal: result.exitSignal,
+    },
+  };
+}

package/src/javascript.ts

@@ -0,0 +1,42 @@
+import { randomUUID } from "node:crypto";
+import { rm, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { isAbsolute, join, resolve } from "node:path";
+import type { AgentToolResult } from "@earendil-works/pi-coding-agent";
+import { assertSuccessOrThrow } from "./format.js";
+import { run } from "./subprocess.js";
+import type { EvalDetails } from "./types.js";
+
+export async function executeJavaScript(
+  code: string,
+  nodeModulesPath: string | undefined,
+  userSignal: AbortSignal | undefined,
+  timeoutSignal: AbortSignal,
+  cwd: string,
+): Promise<AgentToolResult<EvalDetails>> {
+  const tmpFile = join(tmpdir(), `pi-eval-${randomUUID()}.js`);
+
+  try {
+    await writeFile(tmpFile, code, "utf-8");
+
+    const env: NodeJS.ProcessEnv = { ...process.env };
+    if (nodeModulesPath) {
+      env.NODE_PATH = isAbsolute(nodeModulesPath)
+        ? nodeModulesPath
+        : resolve(cwd, nodeModulesPath);
+    }
+
+    const result = await run(
+      "node",
+      [tmpFile],
+      cwd,
+      env,
+      userSignal,
+      timeoutSignal,
+    );
+    return assertSuccessOrThrow("javascript", result);
+  } finally {
+    // await ensures cleanup completes before result/error propagates
+    await rm(tmpFile, { force: true });
+  }
+}

package/src/python.ts

@@ -0,0 +1,63 @@
+import { constants } from "node:fs";
+import { access } from "node:fs/promises";
+import { delimiter, isAbsolute, join, resolve } from "node:path";
+import type { AgentToolResult } from "@earendil-works/pi-coding-agent";
+import { assertSuccessOrThrow } from "./format.js";
+import { run } from "./subprocess.js";
+import { EvalBinaryNotFoundError, type EvalDetails } from "./types.js";
+
+export async function resolvePythonBinary(
+  pythonPath: string | undefined,
+  cwd: string,
+): Promise<string> {
+  if (pythonPath) {
+    // Explicit path: resolve relative to cwd, then validate
+    const resolved = isAbsolute(pythonPath)
+      ? pythonPath
+      : resolve(cwd, pythonPath);
+    try {
+      await access(resolved, constants.X_OK);
+    } catch {
+      throw new EvalBinaryNotFoundError(resolved);
+    }
+    return resolved;
+  }
+
+  // Default: search PATH for python3
+  const pathDirs = (process.env.PATH || "").split(delimiter);
+  const candidates = [
+    ...pathDirs.map((dir) => join(dir, "python3")),
+    "/usr/bin/python3",
+    "/usr/local/bin/python3",
+  ];
+
+  for (const candidate of candidates) {
+    try {
+      await access(candidate, constants.X_OK);
+      return candidate;
+    } catch {
+      // try next
+    }
+  }
+
+  throw new EvalBinaryNotFoundError("python3");
+}
+
+export async function executePython(
+  code: string,
+  pythonPath: string | undefined,
+  userSignal: AbortSignal | undefined,
+  timeoutSignal: AbortSignal,
+  cwd: string,
+): Promise<AgentToolResult<EvalDetails>> {
+  const bin = await resolvePythonBinary(pythonPath, cwd);
+  const result = await run(
+    bin,
+    ["-c", code],
+    cwd,
+    {},
+    userSignal,
+    timeoutSignal,
+  );
+  return assertSuccessOrThrow("python", result);
+}

package/src/subprocess.ts

@@ -0,0 +1,130 @@
+import { spawn } from "node:child_process";
+import { MAX_OUTPUT } from "./format.js";
+import {
+  EvalCancelledError,
+  EvalSpawnError,
+  EvalTimeoutError,
+  type SubprocessResult,
+} from "./types.js";
+
+/**
+ * Spawn a subprocess and capture stdout/stderr.
+ *
+ * @throws {EvalTimeoutError} when {@link timeoutSignal} fires
+ * @throws {EvalCancelledError} when {@link userSignal} fires
+ * @throws {EvalSpawnError} when the process fails to start
+ */
+export function run(
+  file: string,
+  args: string[],
+  cwd: string,
+  env: NodeJS.ProcessEnv,
+  userSignal: AbortSignal | undefined,
+  timeoutSignal: AbortSignal,
+): Promise<SubprocessResult> {
+  return new Promise((resolvePromise, reject) => {
+    // Combine signals for spawn (either will kill the child)
+    const combinedSignal = userSignal
+      ? AbortSignal.any([userSignal, timeoutSignal])
+      : timeoutSignal;
+
+    const child = spawn(file, args, {
+      cwd,
+      env,
+      signal: combinedSignal,
+      stdio: ["ignore", "pipe", "pipe"],
+    });
+
+    let stdout = "";
+    let stderr = "";
+    let truncated = false;
+
+    const onData = (target: "stdout" | "stderr") => (chunk: Buffer) => {
+      const used = stdout.length + stderr.length;
+      const remaining = MAX_OUTPUT - used;
+      if (remaining <= 0) {
+        truncated = true;
+        child.kill();
+        return;
+      }
+      // If the chunk is larger than remaining capacity, only keep what fits
+      // and mark as truncated immediately (don't wait for the next chunk).
+      const fits = Math.min(chunk.length, remaining);
+      const text = chunk.toString("utf-8", 0, fits);
+      if (target === "stdout") {
+        stdout += text;
+      } else {
+        stderr += text;
+      }
+      if (fits < chunk.length) {
+        truncated = true;
+        child.kill();
+      }
+    };
+
+    child.stdout.on("data", onData("stdout"));
+    child.stderr.on("data", onData("stderr"));
+
+    let settled = false;
+
+    child.on("close", (exitCode, exitSignal) => {
+      if (settled) {
+        return;
+      }
+      // Truncation kill (we killed it) — resolve with partial output.
+      // Treat as a normal termination: the truncated flag communicates the
+      // condition, not the exit signal.
+      if (truncated) {
+        settled = true;
+        resolvePromise({
+          stdout,
+          stderr,
+          exitCode: 0,
+          exitSignal: null,
+          truncated: true,
+        });
+        return;
+      }
+      // Check our abort signals first (they kill the child via spawn's signal option)
+      if (timeoutSignal.aborted) {
+        settled = true;
+        reject(new EvalTimeoutError());
+        return;
+      }
+      if (userSignal?.aborted) {
+        settled = true;
+        reject(new EvalCancelledError());
+        return;
+      }
+      // Normal exit (exitCode may be null if killed by an external signal)
+      settled = true;
+      resolvePromise({
+        stdout,
+        stderr,
+        exitCode,
+        exitSignal,
+        truncated,
+      });
+    });
+
+    child.on("error", (err) => {
+      if (settled) {
+        return;
+      }
+      // Abort during spawn: Node throws an error before the process starts.
+      // Discriminate which signal caused it.
+      if (timeoutSignal.aborted) {
+        settled = true;
+        reject(new EvalTimeoutError());
+        return;
+      }
+      if (userSignal?.aborted) {
+        settled = true;
+        reject(new EvalCancelledError());
+        return;
+      }
+      settled = true;
+      reject(new EvalSpawnError(file, err.message));
+    });
+  });
+}

package/src/types.ts

@@ -0,0 +1,60 @@
+export class EvalToolError extends Error {
+  constructor(
+    message: string,
+    public readonly code: string,
+  ) {
+    super(message);
+    this.name = "EvalToolError";
+  }
+}
+
+export class EvalTimeoutError extends EvalToolError {
+  constructor() {
+    super("Evaluation cancelled or timed out after 30 seconds", "TIMEOUT");
+  }
+}
+
+export class EvalCancelledError extends EvalToolError {
+  constructor() {
+    super("Evaluation cancelled", "CANCELLED");
+  }
+}
+
+export class EvalBinaryNotFoundError extends EvalToolError {
+  constructor(binary: string) {
+    super(
+      `Python binary not found or not executable: ${binary}`,
+      "BINARY_NOT_FOUND",
+    );
+  }
+}
+
+export class EvalSpawnError extends EvalToolError {
+  constructor(file: string, cause: string) {
+    super(`Failed to spawn ${file}: ${cause}`, "SPAWN_FAILED");
+  }
+}
+
+export class EvalUnsupportedLanguageError extends EvalToolError {
+  constructor(language: string) {
+    super(
+      `Unsupported language: "${language}". Use "javascript" or "python".`,
+      "UNSUPPORTED_LANGUAGE",
+    );
+  }
+}
+
+export interface SubprocessResult {
+  stdout: string;
+  stderr: string;
+  /** null when the process was killed by a signal rather than exiting normally */
+  exitCode: number | null;
+  exitSignal: string | null;
+  truncated: boolean;
+}
+
+export interface EvalDetails {
+  language: string;
+  exitCode: number | null;
+  exitSignal: string | null;
+}