@aliou/pi-evals 0.0.1

package/dist/index.js

@@ -0,0 +1,455 @@
+import {
+  __export,
+  defineConfig,
+  getCurrentFile,
+  registerEval
+} from "./chunk-342JG3E3.js";
+
+// src/scorers/index.ts
+var scorers_exports = {};
+__export(scorers_exports, {
+  bash: () => bash,
+  files: () => files,
+  llmJudge: () => llmJudge,
+  outputContains: () => outputContains,
+  outputMatches: () => outputMatches,
+  toolCalled: () => toolCalled,
+  toolCalledWith: () => toolCalledWith
+});
+
+// src/scorers/bash.ts
+import { exec } from "child_process";
+import { promisify } from "util";
+var execAsync = promisify(exec);
+function bash(command, options = {}) {
+  const { exitCode: expectedCode = 0, timeout = 3e4 } = options;
+  return {
+    name: "bash",
+    score: async (ctx) => {
+      try {
+        const _result = await execAsync(command, {
+          cwd: ctx.cwd,
+          timeout,
+          env: { ...process.env, PATH: process.env.PATH }
+        });
+        if (expectedCode === 0) {
+          return {
+            name: "bash",
+            score: 1,
+            reason: `Command succeeded: ${command}`
+          };
+        } else {
+          return {
+            name: "bash",
+            score: 0,
+            reason: `Expected exit code ${expectedCode}, got 0`
+          };
+        }
+      } catch (err) {
+        const error = err;
+        if (error.killed) {
+          return {
+            name: "bash",
+            score: 0,
+            reason: `Command timed out after ${timeout}ms: ${command}`
+          };
+        }
+        const actualCode = error.code ?? 1;
+        if (actualCode === expectedCode) {
+          return {
+            name: "bash",
+            score: 1,
+            reason: `Command exited with expected code ${expectedCode}: ${command}`
+          };
+        }
+        const stderr = error.stderr ? `
+${truncate(error.stderr, 200)}` : "";
+        return {
+          name: "bash",
+          score: 0,
+          reason: `Command failed with code ${actualCode}: ${command}${stderr}`
+        };
+      }
+    }
+  };
+}
+function truncate(str, maxLen) {
+  if (str.length <= maxLen) return str;
+  return `${str.slice(0, maxLen - 3)}...`;
+}
+
+// src/scorers/contains.ts
+function outputContains() {
+  return {
+    name: "outputContains",
+    score: async (ctx) => {
+      const expectedOutput = ctx.expected?.output;
+      if (!expectedOutput) {
+        return {
+          name: "outputContains",
+          score: 1,
+          reason: "No output expected"
+        };
+      }
+      const contains = ctx.output.includes(expectedOutput);
+      return {
+        name: "outputContains",
+        score: contains ? 1 : 0,
+        reason: contains ? `Output contains "${truncate2(expectedOutput, 50)}"` : `Output missing "${truncate2(expectedOutput, 50)}"`
+      };
+    }
+  };
+}
+function truncate2(str, maxLen) {
+  if (str.length <= maxLen) return str;
+  return `${str.slice(0, maxLen - 3)}...`;
+}
+
+// src/scorers/files.ts
+import * as fs from "fs/promises";
+import * as path from "path";
+function files() {
+  return {
+    name: "files",
+    score: async (ctx) => {
+      const expectedFiles = ctx.expected?.files;
+      if (!expectedFiles || Object.keys(expectedFiles).length === 0) {
+        return {
+          name: "files",
+          score: 1,
+          reason: "No files expected"
+        };
+      }
+      const results = [];
+      for (const [filePath, expectedContent] of Object.entries(expectedFiles)) {
+        const fullPath = path.join(ctx.cwd, filePath);
+        try {
+          const content = await fs.readFile(fullPath, "utf-8");
+          if (content.includes(expectedContent)) {
+            results.push({
+              file: filePath,
+              ok: true,
+              reason: "exists with expected content"
+            });
+          } else {
+            results.push({
+              file: filePath,
+              ok: false,
+              reason: `exists but missing expected content: "${truncate3(expectedContent, 50)}"`
+            });
+          }
+        } catch (err) {
+          if (err.code === "ENOENT") {
+            results.push({
+              file: filePath,
+              ok: false,
+              reason: "file not found"
+            });
+          } else {
+            results.push({
+              file: filePath,
+              ok: false,
+              reason: `error reading: ${err.message}`
+            });
+          }
+        }
+      }
+      const passed = results.filter((r) => r.ok).length;
+      const total = results.length;
+      const score = total > 0 ? passed / total : 1;
+      const reasons = results.map(
+        (r) => `${r.ok ? "+" : "-"} ${r.file}: ${r.reason}`
+      );
+      return {
+        name: "files",
+        score,
+        reason: reasons.join("\n")
+      };
+    }
+  };
+}
+function truncate3(str, maxLen) {
+  if (str.length <= maxLen) return str;
+  return `${str.slice(0, maxLen - 3)}...`;
+}
+
+// src/scorers/llm-judge.ts
+function llmJudge(options) {
+  const { criteria, model = "gpt-4o-mini", provider = "openai" } = options;
+  return {
+    name: "llmJudge",
+    score: async (ctx) => {
+      const prompt = buildJudgePrompt(criteria, ctx.input, ctx.output);
+      try {
+        const result = await callLlm(prompt, model, provider);
+        return parseJudgeResponse(result);
+      } catch (err) {
+        return {
+          name: "llmJudge",
+          score: 0,
+          reason: `LLM judge error: ${err.message}`
+        };
+      }
+    }
+  };
+}
+function buildJudgePrompt(criteria, input, output) {
+  return `You are evaluating an AI coding assistant's response.
+
+## Task given to the assistant
+${input}
+
+## Assistant's response
+${output}
+
+## Evaluation criteria
+${criteria}
+
+## Instructions
+Evaluate the response against the criteria. Respond with a JSON object:
+{
+  "score": <number from 0 to 1>,
+  "reason": "<brief explanation>"
+}
+
+Score meanings:
+- 1.0: Fully meets criteria
+- 0.7-0.9: Mostly meets criteria with minor issues
+- 0.4-0.6: Partially meets criteria
+- 0.1-0.3: Barely meets criteria
+- 0.0: Does not meet criteria
+
+Respond ONLY with the JSON object, no other text.`;
+}
+async function callLlm(prompt, model, provider) {
+  if (provider === "openai") {
+    return callOpenAI(prompt, model);
+  } else if (provider === "anthropic") {
+    return callAnthropic(prompt, model);
+  } else {
+    throw new Error(`Unsupported LLM judge provider: ${provider}`);
+  }
+}
+async function callOpenAI(prompt, model) {
+  const apiKey = process.env.OPENAI_API_KEY;
+  if (!apiKey) {
+    throw new Error("OPENAI_API_KEY not set");
+  }
+  const response = await fetch("https://api.openai.com/v1/chat/completions", {
+    method: "POST",
+    headers: {
+      "Content-Type": "application/json",
+      Authorization: `Bearer ${apiKey}`
+    },
+    body: JSON.stringify({
+      model,
+      messages: [{ role: "user", content: prompt }],
+      temperature: 0
+    })
+  });
+  if (!response.ok) {
+    const text = await response.text();
+    throw new Error(`OpenAI API error: ${response.status} ${text}`);
+  }
+  const data = await response.json();
+  return data.choices[0]?.message?.content ?? "";
+}
+async function callAnthropic(prompt, model) {
+  const apiKey = process.env.ANTHROPIC_API_KEY;
+  if (!apiKey) {
+    throw new Error("ANTHROPIC_API_KEY not set");
+  }
+  const response = await fetch("https://api.anthropic.com/v1/messages", {
+    method: "POST",
+    headers: {
+      "Content-Type": "application/json",
+      "x-api-key": apiKey,
+      "anthropic-version": "2023-06-01"
+    },
+    body: JSON.stringify({
+      model,
+      max_tokens: 256,
+      messages: [{ role: "user", content: prompt }]
+    })
+  });
+  if (!response.ok) {
+    const text = await response.text();
+    throw new Error(`Anthropic API error: ${response.status} ${text}`);
+  }
+  const data = await response.json();
+  const textBlock = data.content.find((c) => c.type === "text");
+  return textBlock?.text ?? "";
+}
+function parseJudgeResponse(response) {
+  try {
+    const jsonMatch = response.match(/\{[\s\S]*\}/);
+    if (!jsonMatch) {
+      throw new Error("No JSON found in response");
+    }
+    const parsed = JSON.parse(jsonMatch[0]);
+    if (typeof parsed.score !== "number" || parsed.score < 0 || parsed.score > 1) {
+      throw new Error("Invalid score in response");
+    }
+    return {
+      name: "llmJudge",
+      score: parsed.score,
+      reason: parsed.reason
+    };
+  } catch (err) {
+    return {
+      name: "llmJudge",
+      score: 0,
+      reason: `Failed to parse judge response: ${err.message}`
+    };
+  }
+}
+
+// src/scorers/regex.ts
+function outputMatches(pattern) {
+  return {
+    name: "outputMatches",
+    score: async (ctx) => {
+      const matches = pattern.test(ctx.output);
+      return {
+        name: "outputMatches",
+        score: matches ? 1 : 0,
+        reason: matches ? `Output matches ${pattern}` : `Output does not match ${pattern}`
+      };
+    }
+  };
+}
+
+// src/scorers/tool-called.ts
+function toolCalled(name) {
+  return {
+    name: `toolCalled(${name})`,
+    score: async (ctx) => {
+      const called = ctx.toolCalls.some((tc) => tc.name === name);
+      return {
+        name: `toolCalled(${name})`,
+        score: called ? 1 : 0,
+        reason: called ? `Tool "${name}" was called` : `Tool "${name}" was not called. Called: ${formatToolNames(ctx)}`
+      };
+    }
+  };
+}
+function formatToolNames(ctx) {
+  if (ctx.toolCalls.length === 0) return "(none)";
+  const unique = [...new Set(ctx.toolCalls.map((tc) => tc.name))];
+  return unique.join(", ");
+}
+
+// src/scorers/tool-called-with.ts
+import * as path2 from "path";
+function toolCalledWith(name, expectedArgs) {
+  const label = `toolCalledWith(${name})`;
+  return {
+    name: label,
+    score: async (ctx) => {
+      const matching = ctx.toolCalls.filter((tc) => tc.name === name);
+      if (matching.length === 0) {
+        return {
+          name: label,
+          score: 0,
+          reason: `Tool "${name}" was not called. Called: ${formatToolNames2(ctx)}`
+        };
+      }
+      for (const tc of matching) {
+        if (argsMatch(tc.args, expectedArgs, ctx.cwd)) {
+          return {
+            name: label,
+            score: 1,
+            reason: `Tool "${name}" called with matching args`
+          };
+        }
+      }
+      const firstCall = matching[0];
+      const mismatches = getArgMismatches(
+        firstCall.args,
+        expectedArgs,
+        ctx.cwd
+      );
+      return {
+        name: label,
+        score: 0,
+        reason: `Tool "${name}" called ${matching.length} time(s) but args did not match. ${mismatches}`
+      };
+    }
+  };
+}
+function argsMatch(actual, expected, cwd) {
+  for (const [key, expectedValue] of Object.entries(expected)) {
+    const actualValue = actual[key];
+    if (isPathArg(key)) {
+      const resolvedActual = resolvePath(actualValue, cwd);
+      const resolvedExpected = resolvePath(expectedValue, cwd);
+      if (resolvedActual !== resolvedExpected) return false;
+    } else {
+      if (!deepEqual(actualValue, expectedValue)) return false;
+    }
+  }
+  return true;
+}
+function getArgMismatches(actual, expected, cwd) {
+  const parts = [];
+  for (const [key, expectedValue] of Object.entries(expected)) {
+    const actualValue = actual[key];
+    if (isPathArg(key)) {
+      const resolvedActual = resolvePath(actualValue, cwd);
+      const resolvedExpected = resolvePath(expectedValue, cwd);
+      if (resolvedActual !== resolvedExpected) {
+        parts.push(
+          `${key}: expected "${resolvedExpected}", got "${resolvedActual}"`
+        );
+      }
+    } else if (!deepEqual(actualValue, expectedValue)) {
+      parts.push(
+        `${key}: expected ${JSON.stringify(expectedValue)}, got ${JSON.stringify(actualValue)}`
+      );
+    }
+  }
+  return parts.join("; ");
+}
+function isPathArg(key) {
+  return key === "path" || key === "file" || key.endsWith("Path");
+}
+function resolvePath(value, cwd) {
+  if (typeof value !== "string") return String(value);
+  return path2.resolve(cwd, value);
+}
+function deepEqual(a, b) {
+  if (a === b) return true;
+  if (a === null || b === null) return false;
+  if (typeof a !== typeof b) return false;
+  if (Array.isArray(a) && Array.isArray(b)) {
+    if (a.length !== b.length) return false;
+    return a.every((val, i) => deepEqual(val, b[i]));
+  }
+  if (typeof a === "object" && typeof b === "object") {
+    const aObj = a;
+    const bObj = b;
+    const aKeys = Object.keys(aObj);
+    const bKeys = Object.keys(bObj);
+    if (aKeys.length !== bKeys.length) return false;
+    return aKeys.every((key) => deepEqual(aObj[key], bObj[key]));
+  }
+  return false;
+}
+function formatToolNames2(ctx) {
+  if (ctx.toolCalls.length === 0) return "(none)";
+  const unique = [...new Set(ctx.toolCalls.map((tc) => tc.name))];
+  return unique.join(", ");
+}
+
+// src/index.ts
+var Scorers = scorers_exports;
+function evaluate(name, options) {
+  const file = getCurrentFile() || "unknown";
+  registerEval(name, options, file);
+}
+export {
+  Scorers,
+  defineConfig,
+  evaluate
+};
+//# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/scorers/index.ts","../src/scorers/bash.ts","../src/scorers/contains.ts","../src/scorers/files.ts","../src/scorers/llm-judge.ts","../src/scorers/regex.ts","../src/scorers/tool-called.ts","../src/scorers/tool-called-with.ts","../src/index.ts"],"sourcesContent":["/**\n * Built-in scorers for pi-eval\n */\n\nexport type { BashOptions } from \"./bash\";\nexport { bash } from \"./bash\";\nexport { outputContains } from \"./contains\";\nexport { files } from \"./files\";\nexport type { LlmJudgeOptions } from \"./llm-judge\";\nexport { llmJudge } from \"./llm-judge\";\nexport { outputMatches } from \"./regex\";\nexport { toolCalled } from \"./tool-called\";\nexport { toolCalledWith } from \"./tool-called-with\";\n","/**\n * Bash scorer - runs a command and checks exit code\n */\nimport { exec } from \"node:child_process\";\nimport { promisify } from \"node:util\";\nimport type { Expected, Scorer } from \"../types\";\n\nconst execAsync = promisify(exec);\n\nexport interface BashOptions {\n  /** Expected exit code (default: 0) */\n  exitCode?: number;\n  /** Timeout in ms (default: 30000) */\n  timeout?: number;\n}\n\n/**\n * Creates a scorer that runs a bash command and checks the exit code.\n * Useful for running tests, linters, or other validation commands.\n */\nexport function bash(\n  command: string,\n  options: BashOptions = {},\n): Scorer<Expected> {\n  const { exitCode: expectedCode = 0, timeout = 30000 } = options;\n\n  return {\n    name: \"bash\",\n    score: async (ctx) => {\n      try {\n        const _result = await execAsync(command, {\n          cwd: ctx.cwd,\n          timeout,\n          env: { ...process.env, PATH: process.env.PATH },\n        });\n\n        // Command succeeded (exit code 0)\n        if (expectedCode === 0) {\n          return {\n            name: \"bash\",\n            score: 1,\n            reason: `Command succeeded: ${command}`,\n          };\n        } else {\n          return {\n            name: \"bash\",\n            score: 0,\n            reason: `Expected exit code ${expectedCode}, got 0`,\n          };\n        }\n      } catch (err) {\n        const error = err as Error & {\n          code?: number;\n          killed?: boolean;\n          stdout?: string;\n          stderr?: string;\n        };\n\n        if (error.killed) {\n          return {\n            name: \"bash\",\n            score: 0,\n            reason: `Command timed out after ${timeout}ms: ${command}`,\n          };\n        }\n\n        const actualCode = error.code ?? 1;\n\n        if (actualCode === expectedCode) {\n          return {\n            name: \"bash\",\n            score: 1,\n            reason: `Command exited with expected code ${expectedCode}: ${command}`,\n          };\n        }\n\n        const stderr = error.stderr ? `\\n${truncate(error.stderr, 200)}` : \"\";\n\n        return {\n          name: \"bash\",\n          score: 0,\n          reason: `Command failed with code ${actualCode}: ${command}${stderr}`,\n        };\n      }\n    },\n  };\n}\n\nfunction truncate(str: string, maxLen: number): string {\n  if (str.length <= maxLen) return str;\n  return `${str.slice(0, maxLen - 3)}...`;\n}\n","/**\n * Output contains scorer - checks that output contains expected substring\n */\nimport type { Expected, ScoreContext, Scorer } from \"../types\";\n\n/**\n * Creates a scorer that checks if the agent's output contains expected.output\n */\nexport function outputContains(): Scorer<Expected> {\n  return {\n    name: \"outputContains\",\n    score: async (ctx: ScoreContext<Expected>) => {\n      const expectedOutput = ctx.expected?.output;\n\n      if (!expectedOutput) {\n        return {\n          name: \"outputContains\",\n          score: 1,\n          reason: \"No output expected\",\n        };\n      }\n\n      const contains = ctx.output.includes(expectedOutput);\n\n      return {\n        name: \"outputContains\",\n        score: contains ? 1 : 0,\n        reason: contains\n          ? `Output contains \"${truncate(expectedOutput, 50)}\"`\n          : `Output missing \"${truncate(expectedOutput, 50)}\"`,\n      };\n    },\n  };\n}\n\nfunction truncate(str: string, maxLen: number): string {\n  if (str.length <= maxLen) return str;\n  return `${str.slice(0, maxLen - 3)}...`;\n}\n","/**\n * Files scorer - checks that expected files exist with expected content\n */\nimport * as fs from \"node:fs/promises\";\nimport * as path from \"node:path\";\nimport type { Expected, ScoreContext, Scorer } from \"../types\";\n\n/**\n * Creates a scorer that checks if expected files exist and contain expected content.\n * Uses substring matching for content comparison.\n */\nexport function files(): Scorer<Expected> {\n  return {\n    name: \"files\",\n    score: async (ctx: ScoreContext<Expected>) => {\n      const expectedFiles = ctx.expected?.files;\n\n      if (!expectedFiles || Object.keys(expectedFiles).length === 0) {\n        return {\n          name: \"files\",\n          score: 1,\n          reason: \"No files expected\",\n        };\n      }\n\n      const results: { file: string; ok: boolean; reason: string }[] = [];\n\n      for (const [filePath, expectedContent] of Object.entries(expectedFiles)) {\n        const fullPath = path.join(ctx.cwd, filePath);\n\n        try {\n          const content = await fs.readFile(fullPath, \"utf-8\");\n\n          if (content.includes(expectedContent)) {\n            results.push({\n              file: filePath,\n              ok: true,\n              reason: \"exists with expected content\",\n            });\n          } else {\n            results.push({\n              file: filePath,\n              ok: false,\n              reason: `exists but missing expected content: \"${truncate(expectedContent, 50)}\"`,\n            });\n          }\n        } catch (err) {\n          if ((err as NodeJS.ErrnoException).code === \"ENOENT\") {\n            results.push({\n              file: filePath,\n              ok: false,\n              reason: \"file not found\",\n            });\n          } else {\n            results.push({\n              file: filePath,\n              ok: false,\n              reason: `error reading: ${(err as Error).message}`,\n            });\n          }\n        }\n      }\n\n      const passed = results.filter((r) => r.ok).length;\n      const total = results.length;\n      const score = total > 0 ? passed / total : 1;\n\n      const reasons = results.map(\n        (r) => `${r.ok ? \"+\" : \"-\"} ${r.file}: ${r.reason}`,\n      );\n\n      return {\n        name: \"files\",\n        score,\n        reason: reasons.join(\"\\n\"),\n      };\n    },\n  };\n}\n\nfunction truncate(str: string, maxLen: number): string {\n  if (str.length <= maxLen) return str;\n  return `${str.slice(0, maxLen - 3)}...`;\n}\n","/**\n * LLM Judge scorer - uses an LLM to evaluate the output\n */\nimport type { Expected, Scorer } from \"../types\";\n\nexport interface LlmJudgeOptions {\n  /** Criteria for the LLM to evaluate against */\n  criteria: string;\n  /** Model to use (default: gpt-4o-mini) */\n  model?: string;\n  /** Provider (default: openai) */\n  provider?: string;\n}\n\n/**\n * Creates a scorer that uses an LLM to evaluate the output against criteria.\n * Uses a cheap, fast model by default.\n *\n * Note: Requires OPENAI_API_KEY or appropriate provider API key.\n */\nexport function llmJudge(options: LlmJudgeOptions): Scorer<Expected> {\n  const { criteria, model = \"gpt-4o-mini\", provider = \"openai\" } = options;\n\n  return {\n    name: \"llmJudge\",\n    score: async (ctx) => {\n      const prompt = buildJudgePrompt(criteria, ctx.input, ctx.output);\n\n      try {\n        const result = await callLlm(prompt, model, provider);\n        return parseJudgeResponse(result);\n      } catch (err) {\n        return {\n          name: \"llmJudge\",\n          score: 0,\n          reason: `LLM judge error: ${(err as Error).message}`,\n        };\n      }\n    },\n  };\n}\n\nfunction buildJudgePrompt(\n  criteria: string,\n  input: string,\n  output: string,\n): string {\n  return `You are evaluating an AI coding assistant's response.\n\n## Task given to the assistant\n${input}\n\n## Assistant's response\n${output}\n\n## Evaluation criteria\n${criteria}\n\n## Instructions\nEvaluate the response against the criteria. Respond with a JSON object:\n{\n  \"score\": <number from 0 to 1>,\n  \"reason\": \"<brief explanation>\"\n}\n\nScore meanings:\n- 1.0: Fully meets criteria\n- 0.7-0.9: Mostly meets criteria with minor issues\n- 0.4-0.6: Partially meets criteria\n- 0.1-0.3: Barely meets criteria\n- 0.0: Does not meet criteria\n\nRespond ONLY with the JSON object, no other text.`;\n}\n\nasync function callLlm(\n  prompt: string,\n  model: string,\n  provider: string,\n): Promise<string> {\n  if (provider === \"openai\") {\n    return callOpenAI(prompt, model);\n  } else if (provider === \"anthropic\") {\n    return callAnthropic(prompt, model);\n  } else {\n    throw new Error(`Unsupported LLM judge provider: ${provider}`);\n  }\n}\n\nasync function callOpenAI(prompt: string, model: string): Promise<string> {\n  const apiKey = process.env.OPENAI_API_KEY;\n  if (!apiKey) {\n    throw new Error(\"OPENAI_API_KEY not set\");\n  }\n\n  const response = await fetch(\"https://api.openai.com/v1/chat/completions\", {\n    method: \"POST\",\n    headers: {\n      \"Content-Type\": \"application/json\",\n      Authorization: `Bearer ${apiKey}`,\n    },\n    body: JSON.stringify({\n      model,\n      messages: [{ role: \"user\", content: prompt }],\n      temperature: 0,\n    }),\n  });\n\n  if (!response.ok) {\n    const text = await response.text();\n    throw new Error(`OpenAI API error: ${response.status} ${text}`);\n  }\n\n  const data = (await response.json()) as {\n    choices: { message: { content: string } }[];\n  };\n  return data.choices[0]?.message?.content ?? \"\";\n}\n\nasync function callAnthropic(prompt: string, model: string): Promise<string> {\n  const apiKey = process.env.ANTHROPIC_API_KEY;\n  if (!apiKey) {\n    throw new Error(\"ANTHROPIC_API_KEY not set\");\n  }\n\n  const response = await fetch(\"https://api.anthropic.com/v1/messages\", {\n    method: \"POST\",\n    headers: {\n      \"Content-Type\": \"application/json\",\n      \"x-api-key\": apiKey,\n      \"anthropic-version\": \"2023-06-01\",\n    },\n    body: JSON.stringify({\n      model,\n      max_tokens: 256,\n      messages: [{ role: \"user\", content: prompt }],\n    }),\n  });\n\n  if (!response.ok) {\n    const text = await response.text();\n    throw new Error(`Anthropic API error: ${response.status} ${text}`);\n  }\n\n  const data = (await response.json()) as {\n    content: { type: string; text: string }[];\n  };\n  const textBlock = data.content.find((c) => c.type === \"text\");\n  return textBlock?.text ?? \"\";\n}\n\nfunction parseJudgeResponse(response: string): {\n  name: string;\n  score: number;\n  reason?: string;\n} {\n  try {\n    // Try to extract JSON from the response\n    const jsonMatch = response.match(/\\{[\\s\\S]*\\}/);\n    if (!jsonMatch) {\n      throw new Error(\"No JSON found in response\");\n    }\n\n    const parsed = JSON.parse(jsonMatch[0]) as {\n      score: number;\n      reason?: string;\n    };\n\n    if (\n      typeof parsed.score !== \"number\" ||\n      parsed.score < 0 ||\n      parsed.score > 1\n    ) {\n      throw new Error(\"Invalid score in response\");\n    }\n\n    return {\n      name: \"llmJudge\",\n      score: parsed.score,\n      reason: parsed.reason,\n    };\n  } catch (err) {\n    return {\n      name: \"llmJudge\",\n      score: 0,\n      reason: `Failed to parse judge response: ${(err as Error).message}`,\n    };\n  }\n}\n","/**\n * Regex scorer - checks that output matches a pattern\n */\nimport type { Expected, Scorer } from \"../types\";\n\n/**\n * Creates a scorer that checks if the agent's output matches a regex pattern\n */\nexport function outputMatches(pattern: RegExp): Scorer<Expected> {\n  return {\n    name: \"outputMatches\",\n    score: async (ctx) => {\n      const matches = pattern.test(ctx.output);\n\n      return {\n        name: \"outputMatches\",\n        score: matches ? 1 : 0,\n        reason: matches\n          ? `Output matches ${pattern}`\n          : `Output does not match ${pattern}`,\n      };\n    },\n  };\n}\n","/**\n * Tool called scorer - checks that a specific tool was called during the session\n */\nimport type { Expected, ScoreContext, Scorer } from \"../types\";\n\n/**\n * Creates a scorer that checks if a specific tool was called.\n *\n * @param name - The tool name to check for (e.g., \"read\", \"bash\", \"linkup_web_search\")\n */\nexport function toolCalled(name: string): Scorer<Expected> {\n  return {\n    name: `toolCalled(${name})`,\n    score: async (ctx: ScoreContext<Expected>) => {\n      const called = ctx.toolCalls.some((tc) => tc.name === name);\n\n      return {\n        name: `toolCalled(${name})`,\n        score: called ? 1 : 0,\n        reason: called\n          ? `Tool \"${name}\" was called`\n          : `Tool \"${name}\" was not called. Called: ${formatToolNames(ctx)}`,\n      };\n    },\n  };\n}\n\nfunction formatToolNames(ctx: ScoreContext<Expected>): string {\n  if (ctx.toolCalls.length === 0) return \"(none)\";\n  const unique = [...new Set(ctx.toolCalls.map((tc) => tc.name))];\n  return unique.join(\", \");\n}\n","/**\n * Tool called with args scorer - checks that a tool was called with specific arguments\n */\nimport * as path from \"node:path\";\nimport type { Expected, ScoreContext, Scorer } from \"../types\";\n\n/**\n * Creates a scorer that checks if a tool was called with specific arguments.\n *\n * For `path` arguments, both expected and actual values are resolved to\n * absolute paths before comparison. All other arguments use direct equality.\n *\n * @param name - The tool name to check for\n * @param expectedArgs - Key-value pairs the tool call args must contain\n */\nexport function toolCalledWith(\n  name: string,\n  expectedArgs: Record<string, unknown>,\n): Scorer<Expected> {\n  const label = `toolCalledWith(${name})`;\n\n  return {\n    name: label,\n    score: async (ctx: ScoreContext<Expected>) => {\n      const matching = ctx.toolCalls.filter((tc) => tc.name === name);\n\n      if (matching.length === 0) {\n        return {\n          name: label,\n          score: 0,\n          reason: `Tool \"${name}\" was not called. Called: ${formatToolNames(ctx)}`,\n        };\n      }\n\n      // Check if any call matches all expected args\n      for (const tc of matching) {\n        if (argsMatch(tc.args, expectedArgs, ctx.cwd)) {\n          return {\n            name: label,\n            score: 1,\n            reason: `Tool \"${name}\" called with matching args`,\n          };\n        }\n      }\n\n      // Show the closest call for debugging\n      const firstCall = matching[0];\n      const mismatches = getArgMismatches(\n        firstCall.args,\n        expectedArgs,\n        ctx.cwd,\n      );\n\n      return {\n        name: label,\n        score: 0,\n        reason: `Tool \"${name}\" called ${matching.length} time(s) but args did not match. ${mismatches}`,\n      };\n    },\n  };\n}\n\n/**\n * Check if actual args contain all expected key-value pairs.\n * Path args are resolved to absolute paths before comparison.\n */\nfunction argsMatch(\n  actual: Record<string, unknown>,\n  expected: Record<string, unknown>,\n  cwd: string,\n): boolean {\n  for (const [key, expectedValue] of Object.entries(expected)) {\n    const actualValue = actual[key];\n\n    if (isPathArg(key)) {\n      const resolvedActual = resolvePath(actualValue, cwd);\n      const resolvedExpected = resolvePath(expectedValue, cwd);\n      if (resolvedActual !== resolvedExpected) return false;\n    } else {\n      if (!deepEqual(actualValue, expectedValue)) return false;\n    }\n  }\n\n  return true;\n}\n\n/**\n * Get human-readable mismatch descriptions for debugging.\n */\nfunction getArgMismatches(\n  actual: Record<string, unknown>,\n  expected: Record<string, unknown>,\n  cwd: string,\n): string {\n  const parts: string[] = [];\n\n  for (const [key, expectedValue] of Object.entries(expected)) {\n    const actualValue = actual[key];\n\n    if (isPathArg(key)) {\n      const resolvedActual = resolvePath(actualValue, cwd);\n      const resolvedExpected = resolvePath(expectedValue, cwd);\n      if (resolvedActual !== resolvedExpected) {\n        parts.push(\n          `${key}: expected \"${resolvedExpected}\", got \"${resolvedActual}\"`,\n        );\n      }\n    } else if (!deepEqual(actualValue, expectedValue)) {\n      parts.push(\n        `${key}: expected ${JSON.stringify(expectedValue)}, got ${JSON.stringify(actualValue)}`,\n      );\n    }\n  }\n\n  return parts.join(\"; \");\n}\n\n/**\n * Check if an argument key represents a file path.\n */\nfunction isPathArg(key: string): boolean {\n  return key === \"path\" || key === \"file\" || key.endsWith(\"Path\");\n}\n\n/**\n * Resolve a value as an absolute path relative to cwd.\n */\nfunction resolvePath(value: unknown, cwd: string): string {\n  if (typeof value !== \"string\") return String(value);\n  return path.resolve(cwd, value);\n}\n\n/**\n * Simple deep equality for JSON-compatible values.\n */\nfunction deepEqual(a: unknown, b: unknown): boolean {\n  if (a === b) return true;\n  if (a === null || b === null) return false;\n  if (typeof a !== typeof b) return false;\n\n  if (Array.isArray(a) && Array.isArray(b)) {\n    if (a.length !== b.length) return false;\n    return a.every((val, i) => deepEqual(val, b[i]));\n  }\n\n  if (typeof a === \"object\" && typeof b === \"object\") {\n    const aObj = a as Record<string, unknown>;\n    const bObj = b as Record<string, unknown>;\n    const aKeys = Object.keys(aObj);\n    const bKeys = Object.keys(bObj);\n    if (aKeys.length !== bKeys.length) return false;\n    return aKeys.every((key) => deepEqual(aObj[key], bObj[key]));\n  }\n\n  return false;\n}\n\nfunction formatToolNames(ctx: ScoreContext<Expected>): string {\n  if (ctx.toolCalls.length === 0) return \"(none)\";\n  const unique = [...new Set(ctx.toolCalls.map((tc) => tc.name))];\n  return unique.join(\", \");\n}\n","/**\n * pi-eval - Eval framework for pi coding agent\n */\n\n// Main API\nexport { defineConfig } from \"./config\";\n\n// Scorers namespace\nimport * as ScorersModule from \"./scorers/index\";\nexport const Scorers = ScorersModule;\n\n// Re-export scorer option types\nexport type { BashOptions, LlmJudgeOptions } from \"./scorers/index\";\n// Types\nexport type {\n  CliOptions,\n  EvalDefinition,\n  EvalOptions,\n  EvalRunSummary,\n  Expected,\n  GlobalConfig,\n  PiConfig,\n  ScoreContext,\n  ScoreResult,\n  Scorer,\n  SessionStats,\n  TestCase,\n  TestResult,\n  TestSetup,\n  TokenStats,\n  ToolCall,\n} from \"./types\";\n\nimport { getCurrentFile, registerEval } from \"./discovery\";\nimport type { EvalOptions, Expected } from \"./types\";\n\n/**\n * Define and register an eval.\n * This is the main API for creating evals.\n *\n * @example\n * ```typescript\n * import { evaluate, Scorers } from \"@aliou/pi-evals\";\n *\n * evaluate(\"Create hello file\", {\n *   config: {\n *     model: \"claude-sonnet-4-20250514\",\n *     provider: \"anthropic\",\n *   },\n *   data: [\n *     {\n *       input: 'Create a file called hello.txt containing \"Hello World\"',\n *       expected: { files: { \"hello.txt\": \"Hello World\" } },\n *     },\n *   ],\n *   scorers: [Scorers.files()],\n * });\n * ```\n */\nexport function evaluate<TExpected = Expected>(\n  name: string,\n  options: EvalOptions<TExpected>,\n): void {\n  const file = getCurrentFile() || \"unknown\";\n  registerEval(name, options, file);\n}\n"],"mappings":";;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;;;ACGA,SAAS,YAAY;AACrB,SAAS,iBAAiB;AAG1B,IAAM,YAAY,UAAU,IAAI;AAazB,SAAS,KACd,SACA,UAAuB,CAAC,GACN;AAClB,QAAM,EAAE,UAAU,eAAe,GAAG,UAAU,IAAM,IAAI;AAExD,SAAO;AAAA,IACL,MAAM;AAAA,IACN,OAAO,OAAO,QAAQ;AACpB,UAAI;AACF,cAAM,UAAU,MAAM,UAAU,SAAS;AAAA,UACvC,KAAK,IAAI;AAAA,UACT;AAAA,UACA,KAAK,EAAE,GAAG,QAAQ,KAAK,MAAM,QAAQ,IAAI,KAAK;AAAA,QAChD,CAAC;AAGD,YAAI,iBAAiB,GAAG;AACtB,iBAAO;AAAA,YACL,MAAM;AAAA,YACN,OAAO;AAAA,YACP,QAAQ,sBAAsB,OAAO;AAAA,UACvC;AAAA,QACF,OAAO;AACL,iBAAO;AAAA,YACL,MAAM;AAAA,YACN,OAAO;AAAA,YACP,QAAQ,sBAAsB,YAAY;AAAA,UAC5C;AAAA,QACF;AAAA,MACF,SAAS,KAAK;AACZ,cAAM,QAAQ;AAOd,YAAI,MAAM,QAAQ;AAChB,iBAAO;AAAA,YACL,MAAM;AAAA,YACN,OAAO;AAAA,YACP,QAAQ,2BAA2B,OAAO,OAAO,OAAO;AAAA,UAC1D;AAAA,QACF;AAEA,cAAM,aAAa,MAAM,QAAQ;AAEjC,YAAI,eAAe,cAAc;AAC/B,iBAAO;AAAA,YACL,MAAM;AAAA,YACN,OAAO;AAAA,YACP,QAAQ,qCAAqC,YAAY,KAAK,OAAO;AAAA,UACvE;AAAA,QACF;AAEA,cAAM,SAAS,MAAM,SAAS;AAAA,EAAK,SAAS,MAAM,QAAQ,GAAG,CAAC,KAAK;AAEnE,eAAO;AAAA,UACL,MAAM;AAAA,UACN,OAAO;AAAA,UACP,QAAQ,4BAA4B,UAAU,KAAK,OAAO,GAAG,MAAM;AAAA,QACrE;AAAA,MACF;AAAA,IACF;AAAA,EACF;AACF;AAEA,SAAS,SAAS,KAAa,QAAwB;AACrD,MAAI,IAAI,UAAU,OAAQ,QAAO;AACjC,SAAO,GAAG,IAAI,MAAM,GAAG,SAAS,CAAC,CAAC;AACpC;;;ACnFO,SAAS,iBAAmC;AACjD,SAAO;AAAA,IACL,MAAM;AAAA,IACN,OAAO,OAAO,QAAgC;AAC5C,YAAM,iBAAiB,IAAI,UAAU;AAErC,UAAI,CAAC,gBAAgB;AACnB,eAAO;AAAA,UACL,MAAM;AAAA,UACN,OAAO;AAAA,UACP,QAAQ;AAAA,QACV;AAAA,MACF;AAEA,YAAM,WAAW,IAAI,OAAO,SAAS,cAAc;AAEnD,aAAO;AAAA,QACL,MAAM;AAAA,QACN,OAAO,WAAW,IAAI;AAAA,QACtB,QAAQ,WACJ,oBAAoBA,UAAS,gBAAgB,EAAE,CAAC,MAChD,mBAAmBA,UAAS,gBAAgB,EAAE,CAAC;AAAA,MACrD;AAAA,IACF;AAAA,EACF;AACF;AAEA,SAASA,UAAS,KAAa,QAAwB;AACrD,MAAI,IAAI,UAAU,OAAQ,QAAO;AACjC,SAAO,GAAG,IAAI,MAAM,GAAG,SAAS,CAAC,CAAC;AACpC;;;ACnCA,YAAY,QAAQ;AACpB,YAAY,UAAU;AAOf,SAAS,QAA0B;AACxC,SAAO;AAAA,IACL,MAAM;AAAA,IACN,OAAO,OAAO,QAAgC;AAC5C,YAAM,gBAAgB,IAAI,UAAU;AAEpC,UAAI,CAAC,iBAAiB,OAAO,KAAK,aAAa,EAAE,WAAW,GAAG;AAC7D,eAAO;AAAA,UACL,MAAM;AAAA,UACN,OAAO;AAAA,UACP,QAAQ;AAAA,QACV;AAAA,MACF;AAEA,YAAM,UAA2D,CAAC;AAElE,iBAAW,CAAC,UAAU,eAAe,KAAK,OAAO,QAAQ,aAAa,GAAG;AACvE,cAAM,WAAgB,UAAK,IAAI,KAAK,QAAQ;AAE5C,YAAI;AACF,gBAAM,UAAU,MAAS,YAAS,UAAU,OAAO;AAEnD,cAAI,QAAQ,SAAS,eAAe,GAAG;AACrC,oBAAQ,KAAK;AAAA,cACX,MAAM;AAAA,cACN,IAAI;AAAA,cACJ,QAAQ;AAAA,YACV,CAAC;AAAA,UACH,OAAO;AACL,oBAAQ,KAAK;AAAA,cACX,MAAM;AAAA,cACN,IAAI;AAAA,cACJ,QAAQ,yCAAyCC,UAAS,iBAAiB,EAAE,CAAC;AAAA,YAChF,CAAC;AAAA,UACH;AAAA,QACF,SAAS,KAAK;AACZ,cAAK,IAA8B,SAAS,UAAU;AACpD,oBAAQ,KAAK;AAAA,cACX,MAAM;AAAA,cACN,IAAI;AAAA,cACJ,QAAQ;AAAA,YACV,CAAC;AAAA,UACH,OAAO;AACL,oBAAQ,KAAK;AAAA,cACX,MAAM;AAAA,cACN,IAAI;AAAA,cACJ,QAAQ,kBAAmB,IAAc,OAAO;AAAA,YAClD,CAAC;AAAA,UACH;AAAA,QACF;AAAA,MACF;AAEA,YAAM,SAAS,QAAQ,OAAO,CAAC,MAAM,EAAE,EAAE,EAAE;AAC3C,YAAM,QAAQ,QAAQ;AACtB,YAAM,QAAQ,QAAQ,IAAI,SAAS,QAAQ;AAE3C,YAAM,UAAU,QAAQ;AAAA,QACtB,CAAC,MAAM,GAAG,EAAE,KAAK,MAAM,GAAG,IAAI,EAAE,IAAI,KAAK,EAAE,MAAM;AAAA,MACnD;AAEA,aAAO;AAAA,QACL,MAAM;AAAA,QACN;AAAA,QACA,QAAQ,QAAQ,KAAK,IAAI;AAAA,MAC3B;AAAA,IACF;AAAA,EACF;AACF;AAEA,SAASA,UAAS,KAAa,QAAwB;AACrD,MAAI,IAAI,UAAU,OAAQ,QAAO;AACjC,SAAO,GAAG,IAAI,MAAM,GAAG,SAAS,CAAC,CAAC;AACpC;;;AC/DO,SAAS,SAAS,SAA4C;AACnE,QAAM,EAAE,UAAU,QAAQ,eAAe,WAAW,SAAS,IAAI;AAEjE,SAAO;AAAA,IACL,MAAM;AAAA,IACN,OAAO,OAAO,QAAQ;AACpB,YAAM,SAAS,iBAAiB,UAAU,IAAI,OAAO,IAAI,MAAM;AAE/D,UAAI;AACF,cAAM,SAAS,MAAM,QAAQ,QAAQ,OAAO,QAAQ;AACpD,eAAO,mBAAmB,MAAM;AAAA,MAClC,SAAS,KAAK;AACZ,eAAO;AAAA,UACL,MAAM;AAAA,UACN,OAAO;AAAA,UACP,QAAQ,oBAAqB,IAAc,OAAO;AAAA,QACpD;AAAA,MACF;AAAA,IACF;AAAA,EACF;AACF;AAEA,SAAS,iBACP,UACA,OACA,QACQ;AACR,SAAO;AAAA;AAAA;AAAA,EAGP,KAAK;AAAA;AAAA;AAAA,EAGL,MAAM;AAAA;AAAA;AAAA,EAGN,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAiBV;AAEA,eAAe,QACb,QACA,OACA,UACiB;AACjB,MAAI,aAAa,UAAU;AACzB,WAAO,WAAW,QAAQ,KAAK;AAAA,EACjC,WAAW,aAAa,aAAa;AACnC,WAAO,cAAc,QAAQ,KAAK;AAAA,EACpC,OAAO;AACL,UAAM,IAAI,MAAM,mCAAmC,QAAQ,EAAE;AAAA,EAC/D;AACF;AAEA,eAAe,WAAW,QAAgB,OAAgC;AACxE,QAAM,SAAS,QAAQ,IAAI;AAC3B,MAAI,CAAC,QAAQ;AACX,UAAM,IAAI,MAAM,wBAAwB;AAAA,EAC1C;AAEA,QAAM,WAAW,MAAM,MAAM,8CAA8C;AAAA,IACzE,QAAQ;AAAA,IACR,SAAS;AAAA,MACP,gBAAgB;AAAA,MAChB,eAAe,UAAU,MAAM;AAAA,IACjC;AAAA,IACA,MAAM,KAAK,UAAU;AAAA,MACnB;AAAA,MACA,UAAU,CAAC,EAAE,MAAM,QAAQ,SAAS,OAAO,CAAC;AAAA,MAC5C,aAAa;AAAA,IACf,CAAC;AAAA,EACH,CAAC;AAED,MAAI,CAAC,SAAS,IAAI;AAChB,UAAM,OAAO,MAAM,SAAS,KAAK;AACjC,UAAM,IAAI,MAAM,qBAAqB,SAAS,MAAM,IAAI,IAAI,EAAE;AAAA,EAChE;AAEA,QAAM,OAAQ,MAAM,SAAS,KAAK;AAGlC,SAAO,KAAK,QAAQ,CAAC,GAAG,SAAS,WAAW;AAC9C;AAEA,eAAe,cAAc,QAAgB,OAAgC;AAC3E,QAAM,SAAS,QAAQ,IAAI;AAC3B,MAAI,CAAC,QAAQ;AACX,UAAM,IAAI,MAAM,2BAA2B;AAAA,EAC7C;AAEA,QAAM,WAAW,MAAM,MAAM,yCAAyC;AAAA,IACpE,QAAQ;AAAA,IACR,SAAS;AAAA,MACP,gBAAgB;AAAA,MAChB,aAAa;AAAA,MACb,qBAAqB;AAAA,IACvB;AAAA,IACA,MAAM,KAAK,UAAU;AAAA,MACnB;AAAA,MACA,YAAY;AAAA,MACZ,UAAU,CAAC,EAAE,MAAM,QAAQ,SAAS,OAAO,CAAC;AAAA,IAC9C,CAAC;AAAA,EACH,CAAC;AAED,MAAI,CAAC,SAAS,IAAI;AAChB,UAAM,OAAO,MAAM,SAAS,KAAK;AACjC,UAAM,IAAI,MAAM,wBAAwB,SAAS,MAAM,IAAI,IAAI,EAAE;AAAA,EACnE;AAEA,QAAM,OAAQ,MAAM,SAAS,KAAK;AAGlC,QAAM,YAAY,KAAK,QAAQ,KAAK,CAAC,MAAM,EAAE,SAAS,MAAM;AAC5D,SAAO,WAAW,QAAQ;AAC5B;AAEA,SAAS,mBAAmB,UAI1B;AACA,MAAI;AAEF,UAAM,YAAY,SAAS,MAAM,aAAa;AAC9C,QAAI,CAAC,WAAW;AACd,YAAM,IAAI,MAAM,2BAA2B;AAAA,IAC7C;AAEA,UAAM,SAAS,KAAK,MAAM,UAAU,CAAC,CAAC;AAKtC,QACE,OAAO,OAAO,UAAU,YACxB,OAAO,QAAQ,KACf,OAAO,QAAQ,GACf;AACA,YAAM,IAAI,MAAM,2BAA2B;AAAA,IAC7C;AAEA,WAAO;AAAA,MACL,MAAM;AAAA,MACN,OAAO,OAAO;AAAA,MACd,QAAQ,OAAO;AAAA,IACjB;AAAA,EACF,SAAS,KAAK;AACZ,WAAO;AAAA,MACL,MAAM;AAAA,MACN,OAAO;AAAA,MACP,QAAQ,mCAAoC,IAAc,OAAO;AAAA,IACnE;AAAA,EACF;AACF;;;ACpLO,SAAS,cAAc,SAAmC;AAC/D,SAAO;AAAA,IACL,MAAM;AAAA,IACN,OAAO,OAAO,QAAQ;AACpB,YAAM,UAAU,QAAQ,KAAK,IAAI,MAAM;AAEvC,aAAO;AAAA,QACL,MAAM;AAAA,QACN,OAAO,UAAU,IAAI;AAAA,QACrB,QAAQ,UACJ,kBAAkB,OAAO,KACzB,yBAAyB,OAAO;AAAA,MACtC;AAAA,IACF;AAAA,EACF;AACF;;;ACbO,SAAS,WAAW,MAAgC;AACzD,SAAO;AAAA,IACL,MAAM,cAAc,IAAI;AAAA,IACxB,OAAO,OAAO,QAAgC;AAC5C,YAAM,SAAS,IAAI,UAAU,KAAK,CAAC,OAAO,GAAG,SAAS,IAAI;AAE1D,aAAO;AAAA,QACL,MAAM,cAAc,IAAI;AAAA,QACxB,OAAO,SAAS,IAAI;AAAA,QACpB,QAAQ,SACJ,SAAS,IAAI,iBACb,SAAS,IAAI,6BAA6B,gBAAgB,GAAG,CAAC;AAAA,MACpE;AAAA,IACF;AAAA,EACF;AACF;AAEA,SAAS,gBAAgB,KAAqC;AAC5D,MAAI,IAAI,UAAU,WAAW,EAAG,QAAO;AACvC,QAAM,SAAS,CAAC,GAAG,IAAI,IAAI,IAAI,UAAU,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,CAAC;AAC9D,SAAO,OAAO,KAAK,IAAI;AACzB;;;AC5BA,YAAYC,WAAU;AAYf,SAAS,eACd,MACA,cACkB;AAClB,QAAM,QAAQ,kBAAkB,IAAI;AAEpC,SAAO;AAAA,IACL,MAAM;AAAA,IACN,OAAO,OAAO,QAAgC;AAC5C,YAAM,WAAW,IAAI,UAAU,OAAO,CAAC,OAAO,GAAG,SAAS,IAAI;AAE9D,UAAI,SAAS,WAAW,GAAG;AACzB,eAAO;AAAA,UACL,MAAM;AAAA,UACN,OAAO;AAAA,UACP,QAAQ,SAAS,IAAI,6BAA6BC,iBAAgB,GAAG,CAAC;AAAA,QACxE;AAAA,MACF;AAGA,iBAAW,MAAM,UAAU;AACzB,YAAI,UAAU,GAAG,MAAM,cAAc,IAAI,GAAG,GAAG;AAC7C,iBAAO;AAAA,YACL,MAAM;AAAA,YACN,OAAO;AAAA,YACP,QAAQ,SAAS,IAAI;AAAA,UACvB;AAAA,QACF;AAAA,MACF;AAGA,YAAM,YAAY,SAAS,CAAC;AAC5B,YAAM,aAAa;AAAA,QACjB,UAAU;AAAA,QACV;AAAA,QACA,IAAI;AAAA,MACN;AAEA,aAAO;AAAA,QACL,MAAM;AAAA,QACN,OAAO;AAAA,QACP,QAAQ,SAAS,IAAI,YAAY,SAAS,MAAM,oCAAoC,UAAU;AAAA,MAChG;AAAA,IACF;AAAA,EACF;AACF;AAMA,SAAS,UACP,QACA,UACA,KACS;AACT,aAAW,CAAC,KAAK,aAAa,KAAK,OAAO,QAAQ,QAAQ,GAAG;AAC3D,UAAM,cAAc,OAAO,GAAG;AAE9B,QAAI,UAAU,GAAG,GAAG;AAClB,YAAM,iBAAiB,YAAY,aAAa,GAAG;AACnD,YAAM,mBAAmB,YAAY,eAAe,GAAG;AACvD,UAAI,mBAAmB,iBAAkB,QAAO;AAAA,IAClD,OAAO;AACL,UAAI,CAAC,UAAU,aAAa,aAAa,EAAG,QAAO;AAAA,IACrD;AAAA,EACF;AAEA,SAAO;AACT;AAKA,SAAS,iBACP,QACA,UACA,KACQ;AACR,QAAM,QAAkB,CAAC;AAEzB,aAAW,CAAC,KAAK,aAAa,KAAK,OAAO,QAAQ,QAAQ,GAAG;AAC3D,UAAM,cAAc,OAAO,GAAG;AAE9B,QAAI,UAAU,GAAG,GAAG;AAClB,YAAM,iBAAiB,YAAY,aAAa,GAAG;AACnD,YAAM,mBAAmB,YAAY,eAAe,GAAG;AACvD,UAAI,mBAAmB,kBAAkB;AACvC,cAAM;AAAA,UACJ,GAAG,GAAG,eAAe,gBAAgB,WAAW,cAAc;AAAA,QAChE;AAAA,MACF;AAAA,IACF,WAAW,CAAC,UAAU,aAAa,aAAa,GAAG;AACjD,YAAM;AAAA,QACJ,GAAG,GAAG,cAAc,KAAK,UAAU,aAAa,CAAC,SAAS,KAAK,UAAU,WAAW,CAAC;AAAA,MACvF;AAAA,IACF;AAAA,EACF;AAEA,SAAO,MAAM,KAAK,IAAI;AACxB;AAKA,SAAS,UAAU,KAAsB;AACvC,SAAO,QAAQ,UAAU,QAAQ,UAAU,IAAI,SAAS,MAAM;AAChE;AAKA,SAAS,YAAY,OAAgB,KAAqB;AACxD,MAAI,OAAO,UAAU,SAAU,QAAO,OAAO,KAAK;AAClD,SAAY,cAAQ,KAAK,KAAK;AAChC;AAKA,SAAS,UAAU,GAAY,GAAqB;AAClD,MAAI,MAAM,EAAG,QAAO;AACpB,MAAI,MAAM,QAAQ,MAAM,KAAM,QAAO;AACrC,MAAI,OAAO,MAAM,OAAO,EAAG,QAAO;AAElC,MAAI,MAAM,QAAQ,CAAC,KAAK,MAAM,QAAQ,CAAC,GAAG;AACxC,QAAI,EAAE,WAAW,EAAE,OAAQ,QAAO;AAClC,WAAO,EAAE,MAAM,CAAC,KAAK,MAAM,UAAU,KAAK,EAAE,CAAC,CAAC,CAAC;AAAA,EACjD;AAEA,MAAI,OAAO,MAAM,YAAY,OAAO,MAAM,UAAU;AAClD,UAAM,OAAO;AACb,UAAM,OAAO;AACb,UAAM,QAAQ,OAAO,KAAK,IAAI;AAC9B,UAAM,QAAQ,OAAO,KAAK,IAAI;AAC9B,QAAI,MAAM,WAAW,MAAM,OAAQ,QAAO;AAC1C,WAAO,MAAM,MAAM,CAAC,QAAQ,UAAU,KAAK,GAAG,GAAG,KAAK,GAAG,CAAC,CAAC;AAAA,EAC7D;AAEA,SAAO;AACT;AAEA,SAASA,iBAAgB,KAAqC;AAC5D,MAAI,IAAI,UAAU,WAAW,EAAG,QAAO;AACvC,QAAM,SAAS,CAAC,GAAG,IAAI,IAAI,IAAI,UAAU,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,CAAC;AAC9D,SAAO,OAAO,KAAK,IAAI;AACzB;;;ACxJO,IAAM,UAAU;AAkDhB,SAAS,SACd,MACA,SACM;AACN,QAAM,OAAO,eAAe,KAAK;AACjC,eAAa,MAAM,SAAS,IAAI;AAClC;","names":["truncate","truncate","path","formatToolNames"]}

package/package.json

@@ -0,0 +1,61 @@
+{
+  "name": "@aliou/pi-evals",
+  "version": "0.0.1",
+  "description": "Eval framework for pi coding agent",
+  "type": "module",
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "bin": {
+    "pi-evals": "dist/cli.js"
+  },
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js"
+    }
+  },
+  "dependencies": {
+    "@mariozechner/pi-ai": "^0.51.6",
+    "@mariozechner/pi-coding-agent": "^0.51.6",
+    "glob": "^11.0.0"
+  },
+  "devDependencies": {
+    "@biomejs/biome": "^2.3.14",
+    "@changesets/cli": "^2.29.8",
+    "@types/node": "^22.0.0",
+    "husky": "^9.1.7",
+    "tsup": "^8.5.1",
+    "tsx": "^4.21.0",
+    "typescript": "^5.7.0"
+  },
+  "engines": {
+    "node": ">=20"
+  },
+  "files": [
+    "dist"
+  ],
+  "keywords": [
+    "pi",
+    "eval",
+    "llm",
+    "testing"
+  ],
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/aliou/pi-evals"
+  },
+  "publishConfig": {
+    "access": "public"
+  },
+  "scripts": {
+    "build": "tsup",
+    "dev": "tsup --watch",
+    "typecheck": "tsc --noEmit",
+    "lint": "biome check",
+    "format": "biome check --write",
+    "eval": "tsx src/cli.ts",
+    "eval:build": "node dist/cli.js",
+    "changeset": "changeset"
+  }
+}