@theaiteam/promptdiff 1.0.0-rc.1

package/src/runner/index.ts

@@ -0,0 +1,31 @@
+import type { Runner } from "../types";
+import { ClaudePrintRunner } from "./claude-p";
+import { OpenAiCompatRunner, type ModelPricing } from "./openai-compat";
+
+export const RUNNER_NAMES = ["claude-p", "openai"] as const;
+
+export type RunnerName = (typeof RUNNER_NAMES)[number];
+
+export interface CreateRunnerOptions {
+  /** OpenAI-compatible endpoint base URL; ignored by claude-p. */
+  baseUrl?: string;
+  /** Transient-failure retries for the openai runner; ignored by claude-p. */
+  retries?: number;
+  /** USD-per-M-token pricing for the openai runner; claude-p prices itself. */
+  pricing?: ModelPricing;
+}
+
+export function createRunner(name: RunnerName, options: CreateRunnerOptions = {}): Runner {
+  switch (name) {
+    case "claude-p":
+      return new ClaudePrintRunner();
+    case "openai":
+      return new OpenAiCompatRunner({ baseUrl: options.baseUrl, retries: options.retries, pricing: options.pricing });
+  }
+}
+
+export function runnerNameValue(value: unknown, fallback: RunnerName): RunnerName {
+  if (value === undefined) return fallback;
+  if (value === "claude-p" || value === "openai") return value;
+  throw new Error(`runner must be one of: ${RUNNER_NAMES.join(", ")}`);
+}

package/src/runner/openai-compat.ts

@@ -0,0 +1,228 @@
+import { readFileSync } from "node:fs";
+import { extname } from "node:path";
+import type { RunResult, Runner, RunnerRunOptions } from "../types";
+
+/** USD per million input/output tokens for one model. */
+export interface ModelPricing {
+  input: number;
+  output: number;
+}
+
+export interface OpenAiCompatRunnerConfig {
+  /** Defaults to $OPENAI_BASE_URL, then https://api.openai.com/v1. */
+  baseUrl?: string;
+  /** Defaults to $OPENAI_API_KEY. Optional — local servers often need none. */
+  apiKey?: string;
+  /** Extra attempts after a transient failure (timeout, connect error, 429/5xx). Default 2. */
+  retries?: number;
+  /** Backoff before retry attempt N (1-based); injectable for tests. */
+  backoffMs?: (attempt: number) => number;
+  /**
+   * Prices responses from usage tokens and makes maxBudgetUsd enforce.
+   * Unset, cost reports 0 — true for local servers, decorative for paid ones.
+   */
+  pricing?: ModelPricing;
+  fetchFn?: typeof fetch;
+}
+
+/** Failure modes worth retrying: the request produced nothing durable and the cause is time-bound. */
+class TransientError extends Error {}
+
+export type ChatContentPart =
+  | { type: "text"; text: string }
+  | { type: "image_url"; image_url: { url: string } };
+
+export interface ChatRequest {
+  model: string;
+  messages: Array<{ role: "system" | "user"; content: string | ChatContentPart[] }>;
+  [param: string]: unknown;
+}
+
+const IMAGE_MIME: Record<string, string> = {
+  ".jpg": "image/jpeg",
+  ".jpeg": "image/jpeg",
+  ".png": "image/png",
+  ".webp": "image/webp",
+  ".gif": "image/gif",
+};
+
+/** File path → data URI. Throws on unsupported extensions before any paid run. */
+export function imageDataUri(path: string): string {
+  const mime = IMAGE_MIME[extname(path).toLowerCase()];
+  if (!mime) {
+    throw new Error(`unsupported image type "${extname(path)}" (${path}); supported: ${Object.keys(IMAGE_MIME).join(", ")}`);
+  }
+  return `data:${mime};base64,${readFileSync(path).toString("base64")}`;
+}
+
+interface ChatCompletion {
+  model?: unknown;
+  choices?: unknown;
+  usage?: unknown;
+}
+
+export function buildChatRequest(options: RunnerRunOptions): ChatRequest {
+  if (options.systemPromptMode === "append") {
+    throw new Error(
+      'runner "openai" has no default harness prompt to append to (install delivery is claude-p only)',
+    );
+  }
+  const messages: ChatRequest["messages"] = [];
+  if (options.systemPrompt.trim().length > 0) {
+    messages.push({ role: "system", content: options.systemPrompt });
+  }
+  const images = options.images ?? [];
+  if (images.length > 0) {
+    messages.push({
+      role: "user",
+      content: [
+        ...images.map((path): ChatContentPart => ({ type: "image_url", image_url: { url: imageDataUri(path) } })),
+        { type: "text", text: options.userPrompt },
+      ],
+    });
+  } else {
+    messages.push({ role: "user", content: options.userPrompt });
+  }
+  // Spread first so extra params can never clobber model or messages.
+  return { ...(options.requestParams ?? {}), model: options.model, messages };
+}
+
+/**
+ * Single-shot chat-completion runner for any OpenAI-compatible endpoint
+ * (OpenAI, ollama, vLLM, llama.cpp, OpenRouter, ...). Text mode only: no
+ * tools, no sandbox execution, no skill registry.
+ */
+export class OpenAiCompatRunner implements Runner {
+  readonly name = "openai";
+  readonly capabilities = { sandboxTools: false, skillRegistry: false, images: true };
+
+  private readonly baseUrl: string;
+  private readonly apiKey: string | undefined;
+  private readonly retries: number;
+  private readonly backoffMs: (attempt: number) => number;
+  private readonly pricing: ModelPricing | undefined;
+  private readonly fetchFn: typeof fetch;
+
+  constructor(config: OpenAiCompatRunnerConfig = {}) {
+    this.baseUrl = (config.baseUrl ?? process.env.OPENAI_BASE_URL ?? "https://api.openai.com/v1").replace(/\/+$/, "");
+    this.apiKey = config.apiKey ?? process.env.OPENAI_API_KEY;
+    this.retries = config.retries ?? 2;
+    this.backoffMs = config.backoffMs ?? ((attempt) => Math.min(4_000, 500 * 2 ** (attempt - 1)));
+    this.pricing = config.pricing;
+    this.fetchFn = config.fetchFn ?? fetch;
+  }
+
+  async run(options: RunnerRunOptions): Promise<RunResult> {
+    // Transient failures (timeout, connect error, 429/5xx) retry with backoff —
+    // one 503 at run 4-of-5 must not throw away a whole paid compare. Anything
+    // deterministic (4xx, bad JSON, malformed completion) still throws at once.
+    let lastError: Error | undefined;
+    for (let attempt = 0; attempt <= this.retries; attempt += 1) {
+      if (attempt > 0) {
+        await Bun.sleep(this.backoffMs(attempt));
+      }
+      try {
+        return await this.runOnce(options);
+      } catch (error) {
+        if (!(error instanceof TransientError)) throw error;
+        lastError = error;
+      }
+    }
+    throw new Error(`${lastError?.message} (after ${this.retries + 1} attempts)`);
+  }
+
+  private async runOnce(options: RunnerRunOptions): Promise<RunResult> {
+    const url = `${this.baseUrl}/chat/completions`;
+    const headers: Record<string, string> = { "content-type": "application/json" };
+    if (this.apiKey) {
+      headers.authorization = `Bearer ${this.apiKey}`;
+    }
+
+    const started = Date.now();
+    let response: Response;
+    try {
+      response = await this.fetchFn(url, {
+        method: "POST",
+        headers,
+        body: JSON.stringify(buildChatRequest(options)),
+        signal: AbortSignal.timeout(options.timeoutMs),
+      });
+    } catch (error) {
+      if (error instanceof DOMException && error.name === "TimeoutError") {
+        throw new TransientError(`${url} timed out after ${options.timeoutMs}ms`);
+      }
+      const reason = error instanceof Error ? error.message : String(error);
+      throw new TransientError(`request to ${url} failed: ${reason}`);
+    }
+
+    const body = await response.text();
+    if (!response.ok) {
+      const message = `${url} returned ${response.status}: ${body.slice(0, 1_500)}`;
+      // 429/5xx are load conditions; other statuses are wrong requests.
+      if (response.status === 429 || response.status >= 500) {
+        throw new TransientError(message);
+      }
+      throw new Error(message);
+    }
+
+    let parsed: ChatCompletion;
+    try {
+      parsed = JSON.parse(body) as ChatCompletion;
+    } catch {
+      throw new Error(`${url} returned invalid JSON: ${body.slice(0, 1_500)}`);
+    }
+
+    const run = normalizeChatCompletion(parsed, options.model, Date.now() - started, this.pricing);
+    // A single completion can't be aborted mid-request, so the cap enforces
+    // post-hoc, like claude-p's between-turns check. Not a TransientError:
+    // retrying an over-budget run would spend the overage again.
+    if (this.pricing && run.costUsd > options.maxBudgetUsd) {
+      throw new Error(
+        `run cost $${run.costUsd.toFixed(4)} exceeded the $${options.maxBudgetUsd} max budget — raise maxBudgetUsd or cap max_tokens via requestParams`,
+      );
+    }
+    return run;
+  }
+}
+
+function normalizeChatCompletion(
+  result: ChatCompletion,
+  requestedModel: string,
+  durationMs: number,
+  pricing: ModelPricing | undefined,
+): RunResult {
+  const choice = Array.isArray(result.choices) ? result.choices[0] : undefined;
+  const message = isRecord(choice) && isRecord(choice.message) ? choice.message : undefined;
+  const content = message?.content;
+  if (typeof content !== "string") {
+    throw new Error(`chat completion has no choices[0].message.content string: ${JSON.stringify(result).slice(0, 1_500)}`);
+  }
+
+  return {
+    output: content,
+    costUsd: computeCostUsd(result.usage, pricing),
+    turns: 1,
+    durationMs,
+    models: [typeof result.model === "string" ? result.model : requestedModel],
+    raw: result,
+  };
+}
+
+function computeCostUsd(usage: unknown, pricing: ModelPricing | undefined): number {
+  // Unpriced endpoints report 0 — true for local servers; usage stays in raw.
+  if (!pricing) return 0;
+  const promptTokens = isRecord(usage) ? usage.prompt_tokens : undefined;
+  const completionTokens = isRecord(usage) ? usage.completion_tokens : undefined;
+  if (typeof promptTokens !== "number" || typeof completionTokens !== "number") {
+    // Silent $0 with pricing configured is exactly the decorative-budget bug;
+    // an endpoint that won't report usage cannot be budget-enforced.
+    throw new Error(
+      `pricing is configured but the endpoint returned no usage.prompt_tokens/completion_tokens: ${JSON.stringify(usage).slice(0, 300)}`,
+    );
+  }
+  return (promptTokens * pricing.input + completionTokens * pricing.output) / 1_000_000;
+}
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return typeof value === "object" && value !== null;
+}

package/src/types.ts

@@ -0,0 +1,65 @@
+export type RunMode = "text" | "artifact";
+
+export interface RunResult {
+  output: string;
+  /** USD cost when the runner reports one; 0 when the provider only reports tokens. */
+  costUsd: number;
+  /** Agentic turns; plain completion runners always report 1. */
+  turns: number;
+  durationMs: number;
+  models: string[];
+  raw: unknown;
+}
+
+/**
+ * What a runner can do beyond "system prompt + user prompt in, text out".
+ * The engine validates a scenario's demands against these before any paid run.
+ */
+export interface RunnerCapabilities {
+  /**
+   * The runner executes tools inside the sandbox cwd — required for artifact
+   * mode, command graders, and any non-empty `tools` value.
+   */
+  sandboxTools: boolean;
+  /**
+   * The runner has a harness-managed skill registry and a default system
+   * prompt that can be appended to — required for install delivery.
+   */
+  skillRegistry: boolean;
+  /**
+   * The runner can attach image files to the user message (vision models) —
+   * required for scenarios with `images`.
+   */
+  images: boolean;
+}
+
+export interface RunnerRunOptions {
+  systemPrompt: string;
+  /**
+   * "replace" (default) swaps the whole system prompt — right for controlled
+   * inline evals. "append" layers the text on top of the runner's default
+   * harness prompt, preserving harness machinery like the skill registry;
+   * required for install-delivery evals (skillRegistry runners only).
+   */
+  systemPromptMode?: "replace" | "append";
+  userPrompt: string;
+  /** Image file paths attached to the user message (vision evals). */
+  images?: string[];
+  /** Extra request-body fields for completion runners (e.g. max_tokens, temperature). */
+  requestParams?: Record<string, unknown>;
+  model: string;
+  /** Sandbox directory; graders run here. Agentic runners also execute in it. */
+  cwd: string;
+  timeoutMs: number;
+  // The remaining fields only apply to sandboxTools runners; the engine
+  // guarantees they hold their defaults ("", [], any budget) otherwise.
+  tools: string;
+  addDirs: string[];
+  maxBudgetUsd: number;
+}
+
+export interface Runner {
+  readonly name: string;
+  readonly capabilities: RunnerCapabilities;
+  run(options: RunnerRunOptions): Promise<RunResult>;
+}