@aliou/pi-evals 0.0.1

package/README.md

@@ -0,0 +1,205 @@
+# @aliou/pi-evals
+
+Eval framework for the pi coding agent.
+
+## Installation
+
+```bash
+pnpm add @aliou/pi-evals
+```
+
+## Quick Start
+
+Create an eval file in `evals/`:
+
+```typescript
+// evals/hello.eval.ts
+import { evaluate, Scorers } from "@aliou/pi-evals";
+
+evaluate("Create hello file", {
+  config: {
+    model: "claude-sonnet-4-20250514",
+    provider: "anthropic",
+  },
+  data: [
+    {
+      input: 'Create a file called hello.txt containing "Hello World"',
+      expected: { files: { "hello.txt": "Hello World" } },
+    },
+  ],
+  scorers: [Scorers.files()],
+});
+```
+
+Run evals:
+
+```bash
+npx pi-evals
+```
+
+## Configuration
+
+Create `pi-evals.config.ts`:
+
+```typescript
+import { defineConfig } from "@aliou/pi-evals";
+
+export default defineConfig({
+  defaults: {
+    model: "claude-sonnet-4-20250514",
+    provider: "anthropic",
+  },
+  evalsDir: "./evals",
+  delayBetweenTests: 500,
+  timeout: 60_000,
+  warnTestCount: 30,
+});
+```
+
+## CLI Options
+
+```
+pi-evals [options]
+
+Options:
+  -h, --help              Show help
+  -f, --filter <pattern>  Filter evals by name
+  -t, --threshold <pct>   Minimum pass percentage to exit 0
+  -c, --config <path>     Config file path
+  -m, --model <model>     Override model
+  -p, --provider <name>   Override provider
+  -v, --verbose           Verbose output
+  --json                  Output results as JSON
+
+Environment Variables:
+  PI_EVAL_MODEL           Override model (lower priority than -m)
+  PI_EVAL_PROVIDER        Override provider (lower priority than -p)
+```
+
+Examples:
+```bash
+pi-evals                                # Run all evals
+pi-evals -p github -m gpt-4o            # Use GitHub Models
+PI_EVAL_PROVIDER=github pi-evals        # Via env var
+```
+
+## Built-in Scorers
+
+### `Scorers.files()`
+
+Checks that expected files exist with expected content.
+
+```typescript
+{
+  expected: { files: { "hello.txt": "Hello World" } },
+  scorers: [Scorers.files()],
+}
+```
+
+### `Scorers.outputContains()`
+
+Checks that the agent's output contains expected substring.
+
+```typescript
+{
+  expected: { output: "created file" },
+  scorers: [Scorers.outputContains()],
+}
+```
+
+### `Scorers.outputMatches(pattern)`
+
+Checks that the agent's output matches a regex.
+
+```typescript
+{
+  scorers: [Scorers.outputMatches(/function \w+\(/)],
+}
+```
+
+### `Scorers.bash(command, options?)`
+
+Runs a command and checks the exit code.
+
+```typescript
+{
+  scorers: [Scorers.bash("npm test")],
+}
+```
+
+Options:
+- `exitCode`: Expected exit code (default: 0)
+- `timeout`: Command timeout in ms (default: 30000)
+
+### `Scorers.llmJudge(options)`
+
+Uses an LLM to evaluate the output.
+
+```typescript
+{
+  scorers: [
+    Scorers.llmJudge({
+      criteria: "The response correctly explains the solution",
+      model: "gpt-4o-mini", // optional
+      provider: "openai", // optional
+    }),
+  ],
+}
+```
+
+## Test Case Options
+
+```typescript
+{
+  input: "Create a file",
+  expected: { files: { "file.txt": "content" } },
+  setup: {
+    files: { "existing.txt": "existing content" },
+    commands: ["npm init -y"],
+  },
+  timeout: 30_000,
+  only: false, // Run only this test
+  skip: false, // Skip this test
+}
+```
+
+## Custom Scorers
+
+```typescript
+const customScorer: Scorer = {
+  name: "custom",
+  score: async (ctx) => {
+    const fileExists = await fs.access(path.join(ctx.cwd, "output.txt"))
+      .then(() => true)
+      .catch(() => false);
+
+    return {
+      name: "custom",
+      score: fileExists ? 1 : 0,
+      reason: fileExists ? "File exists" : "File not found",
+    };
+  },
+};
+```
+
+## CI Integration
+
+```yaml
+- name: Run evals
+  env:
+    PI_EVAL_PROVIDER: github
+    PI_EVAL_MODEL: gpt-4o
+    GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+  run: npx pi-evals --json > results.json
+
+- name: Check results
+  run: |
+    PASSED=$(jq '.passed' results.json)
+    TOTAL=$(jq '.total' results.json)
+    echo "Passed: $PASSED/$TOTAL"
+    if [ "$PASSED" -lt "$TOTAL" ]; then exit 1; fi
+```
+
+## License
+
+MIT

package/dist/chunk-342JG3E3.js

@@ -0,0 +1,117 @@
+var __defProp = Object.defineProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+
+// src/config.ts
+import * as fs from "fs/promises";
+import * as path from "path";
+import { pathToFileURL } from "url";
+var DEFAULT_CONFIG_FILES = [
+  "pi-evals.config.ts",
+  "pi-evals.config.js",
+  "pi-evals.config.mjs"
+];
+var DEFAULT_CONFIG = {
+  defaults: {},
+  evalsDir: "./evals",
+  delayBetweenTests: 500,
+  timeout: 6e4,
+  warnTestCount: 30
+};
+function defineConfig(config) {
+  return config;
+}
+async function loadConfig(configPath) {
+  const cwd = process.cwd();
+  if (configPath) {
+    const fullPath = path.resolve(cwd, configPath);
+    return loadConfigFile(fullPath);
+  }
+  for (const filename of DEFAULT_CONFIG_FILES) {
+    const fullPath = path.join(cwd, filename);
+    try {
+      await fs.access(fullPath);
+      return loadConfigFile(fullPath);
+    } catch {
+    }
+  }
+  return DEFAULT_CONFIG;
+}
+async function loadConfigFile(filePath) {
+  try {
+    const fileUrl = pathToFileURL(filePath).href;
+    const module = await import(fileUrl);
+    const userConfig = module.default ?? {};
+    return {
+      ...DEFAULT_CONFIG,
+      ...userConfig,
+      defaults: {
+        ...DEFAULT_CONFIG.defaults,
+        ...userConfig.defaults
+      }
+    };
+  } catch (err) {
+    console.error(`Failed to load config from ${filePath}:`, err);
+    return DEFAULT_CONFIG;
+  }
+}
+
+// src/discovery.ts
+import * as path2 from "path";
+import { pathToFileURL as pathToFileURL2 } from "url";
+import { glob } from "glob";
+var evalRegistry = [];
+function registerEval(name, options, file) {
+  evalRegistry.push({
+    name,
+    // Cast to unknown first to avoid type overlap issues
+    options,
+    file
+  });
+}
+function clearRegistry() {
+  evalRegistry.length = 0;
+}
+function getRegisteredEvals() {
+  return [...evalRegistry];
+}
+async function discoverEvals(evalsDir) {
+  const cwd = process.cwd();
+  const fullDir = path2.resolve(cwd, evalsDir);
+  const pattern = path2.join(fullDir, "**/*.eval.{ts,js,mjs}");
+  const files = await glob(pattern, { absolute: true });
+  if (files.length === 0) {
+    console.warn(`No eval files found in ${evalsDir}`);
+    return [];
+  }
+  clearRegistry();
+  for (const file of files) {
+    try {
+      setCurrentFile(file);
+      const fileUrl = pathToFileURL2(file).href;
+      await import(fileUrl);
+    } catch (err) {
+      console.error(`Failed to load eval file ${file}:`, err);
+    }
+  }
+  return getRegisteredEvals();
+}
+var currentFile = "";
+function setCurrentFile(file) {
+  currentFile = file;
+}
+function getCurrentFile() {
+  return currentFile;
+}
+
+export {
+  __export,
+  defineConfig,
+  loadConfig,
+  registerEval,
+  discoverEvals,
+  getCurrentFile
+};
+//# sourceMappingURL=chunk-342JG3E3.js.map

package/dist/chunk-342JG3E3.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/config.ts","../src/discovery.ts"],"sourcesContent":["/**\n * Config loading for pi-eval\n */\nimport * as fs from \"node:fs/promises\";\nimport * as path from \"node:path\";\nimport { pathToFileURL } from \"node:url\";\nimport type { GlobalConfig } from \"./types\";\n\nconst DEFAULT_CONFIG_FILES = [\n  \"pi-evals.config.ts\",\n  \"pi-evals.config.js\",\n  \"pi-evals.config.mjs\",\n];\n\nconst DEFAULT_CONFIG: Required<GlobalConfig> = {\n  defaults: {},\n  evalsDir: \"./evals\",\n  delayBetweenTests: 500,\n  timeout: 60_000,\n  warnTestCount: 30,\n};\n\n/**\n * Helper for defining config with type inference\n */\nexport function defineConfig(config: GlobalConfig): GlobalConfig {\n  return config;\n}\n\n/**\n * Load config from file or return defaults\n */\nexport async function loadConfig(\n  configPath?: string,\n): Promise<Required<GlobalConfig>> {\n  const cwd = process.cwd();\n\n  // If explicit path provided, use it\n  if (configPath) {\n    const fullPath = path.resolve(cwd, configPath);\n    return loadConfigFile(fullPath);\n  }\n\n  // Try default config file names\n  for (const filename of DEFAULT_CONFIG_FILES) {\n    const fullPath = path.join(cwd, filename);\n    try {\n      await fs.access(fullPath);\n      return loadConfigFile(fullPath);\n    } catch {\n      // File doesn't exist, try next\n    }\n  }\n\n  // No config file found, return defaults\n  return DEFAULT_CONFIG;\n}\n\nasync function loadConfigFile(\n  filePath: string,\n): Promise<Required<GlobalConfig>> {\n  try {\n    // For TypeScript files, we need to compile or use a loader\n    // For now, assume the file is pre-compiled or use tsx/ts-node\n    const fileUrl = pathToFileURL(filePath).href;\n    const module = (await import(fileUrl)) as { default?: GlobalConfig };\n\n    const userConfig = module.default ?? {};\n\n    return {\n      ...DEFAULT_CONFIG,\n      ...userConfig,\n      defaults: {\n        ...DEFAULT_CONFIG.defaults,\n        ...userConfig.defaults,\n      },\n    };\n  } catch (err) {\n    console.error(`Failed to load config from ${filePath}:`, err);\n    return DEFAULT_CONFIG;\n  }\n}\n","/**\n * Eval file discovery\n */\n\nimport * as path from \"node:path\";\nimport { pathToFileURL } from \"node:url\";\nimport { glob } from \"glob\";\nimport type { EvalDefinition, EvalOptions } from \"./types\";\n\n// Global registry for evals (populated when eval files are imported)\nconst evalRegistry: EvalDefinition[] = [];\n\n/**\n * Register an eval definition (called by evaluate())\n */\nexport function registerEval<TExpected>(\n  name: string,\n  options: EvalOptions<TExpected>,\n  file: string,\n): void {\n  evalRegistry.push({\n    name,\n    // Cast to unknown first to avoid type overlap issues\n    options: options as unknown as EvalOptions,\n    file,\n  });\n}\n\n/**\n * Clear the eval registry (for testing)\n */\nexport function clearRegistry(): void {\n  evalRegistry.length = 0;\n}\n\n/**\n * Get all registered evals\n */\nexport function getRegisteredEvals(): EvalDefinition[] {\n  return [...evalRegistry];\n}\n\n/**\n * Discover and load all eval files from a directory\n */\nexport async function discoverEvals(\n  evalsDir: string,\n): Promise<EvalDefinition[]> {\n  const cwd = process.cwd();\n  const fullDir = path.resolve(cwd, evalsDir);\n\n  // Find all *.eval.ts and *.eval.js files\n  const pattern = path.join(fullDir, \"**/*.eval.{ts,js,mjs}\");\n  const files = await glob(pattern, { absolute: true });\n\n  if (files.length === 0) {\n    console.warn(`No eval files found in ${evalsDir}`);\n    return [];\n  }\n\n  // Clear registry before loading\n  clearRegistry();\n\n  // Import each file (this triggers evaluate() calls which register evals)\n  for (const file of files) {\n    try {\n      // Set current file context for registration\n      setCurrentFile(file);\n      const fileUrl = pathToFileURL(file).href;\n      await import(fileUrl);\n    } catch (err) {\n      console.error(`Failed to load eval file ${file}:`, err);\n    }\n  }\n\n  return getRegisteredEvals();\n}\n\n// Track current file being loaded (for registration)\nlet currentFile = \"\";\n\nexport function setCurrentFile(file: string): void {\n  currentFile = file;\n}\n\nexport function getCurrentFile(): string {\n  return currentFile;\n}\n"],"mappings":";;;;;;;AAGA,YAAY,QAAQ;AACpB,YAAY,UAAU;AACtB,SAAS,qBAAqB;AAG9B,IAAM,uBAAuB;AAAA,EAC3B;AAAA,EACA;AAAA,EACA;AACF;AAEA,IAAM,iBAAyC;AAAA,EAC7C,UAAU,CAAC;AAAA,EACX,UAAU;AAAA,EACV,mBAAmB;AAAA,EACnB,SAAS;AAAA,EACT,eAAe;AACjB;AAKO,SAAS,aAAa,QAAoC;AAC/D,SAAO;AACT;AAKA,eAAsB,WACpB,YACiC;AACjC,QAAM,MAAM,QAAQ,IAAI;AAGxB,MAAI,YAAY;AACd,UAAM,WAAgB,aAAQ,KAAK,UAAU;AAC7C,WAAO,eAAe,QAAQ;AAAA,EAChC;AAGA,aAAW,YAAY,sBAAsB;AAC3C,UAAM,WAAgB,UAAK,KAAK,QAAQ;AACxC,QAAI;AACF,YAAS,UAAO,QAAQ;AACxB,aAAO,eAAe,QAAQ;AAAA,IAChC,QAAQ;AAAA,IAER;AAAA,EACF;AAGA,SAAO;AACT;AAEA,eAAe,eACb,UACiC;AACjC,MAAI;AAGF,UAAM,UAAU,cAAc,QAAQ,EAAE;AACxC,UAAM,SAAU,MAAM,OAAO;AAE7B,UAAM,aAAa,OAAO,WAAW,CAAC;AAEtC,WAAO;AAAA,MACL,GAAG;AAAA,MACH,GAAG;AAAA,MACH,UAAU;AAAA,QACR,GAAG,eAAe;AAAA,QAClB,GAAG,WAAW;AAAA,MAChB;AAAA,IACF;AAAA,EACF,SAAS,KAAK;AACZ,YAAQ,MAAM,8BAA8B,QAAQ,KAAK,GAAG;AAC5D,WAAO;AAAA,EACT;AACF;;;AC7EA,YAAYA,WAAU;AACtB,SAAS,iBAAAC,sBAAqB;AAC9B,SAAS,YAAY;AAIrB,IAAM,eAAiC,CAAC;AAKjC,SAAS,aACd,MACA,SACA,MACM;AACN,eAAa,KAAK;AAAA,IAChB;AAAA;AAAA,IAEA;AAAA,IACA;AAAA,EACF,CAAC;AACH;AAKO,SAAS,gBAAsB;AACpC,eAAa,SAAS;AACxB;AAKO,SAAS,qBAAuC;AACrD,SAAO,CAAC,GAAG,YAAY;AACzB;AAKA,eAAsB,cACpB,UAC2B;AAC3B,QAAM,MAAM,QAAQ,IAAI;AACxB,QAAM,UAAe,cAAQ,KAAK,QAAQ;AAG1C,QAAM,UAAe,WAAK,SAAS,uBAAuB;AAC1D,QAAM,QAAQ,MAAM,KAAK,SAAS,EAAE,UAAU,KAAK,CAAC;AAEpD,MAAI,MAAM,WAAW,GAAG;AACtB,YAAQ,KAAK,0BAA0B,QAAQ,EAAE;AACjD,WAAO,CAAC;AAAA,EACV;AAGA,gBAAc;AAGd,aAAW,QAAQ,OAAO;AACxB,QAAI;AAEF,qBAAe,IAAI;AACnB,YAAM,UAAUA,eAAc,IAAI,EAAE;AACpC,YAAM,OAAO;AAAA,IACf,SAAS,KAAK;AACZ,cAAQ,MAAM,4BAA4B,IAAI,KAAK,GAAG;AAAA,IACxD;AAAA,EACF;AAEA,SAAO,mBAAmB;AAC5B;AAGA,IAAI,cAAc;AAEX,SAAS,eAAe,MAAoB;AACjD,gBAAc;AAChB;AAEO,SAAS,iBAAyB;AACvC,SAAO;AACT;","names":["path","pathToFileURL"]}

package/dist/cli.d.ts

@@ -0,0 +1 @@
+#!/usr/bin/env node