@bradtaylorsf/alpha-loop 1.2.0 → 1.4.0

package/dist/lib/config.js

@@ -1,12 +1,23 @@
 import { readFileSync, existsSync } from 'node:fs';
 import { execSync } from 'node:child_process';
 import { parse as parseYaml } from 'yaml';
+/**
+ * Estimate cost in USD from token counts and a pricing table.
+ * Returns 0 if the model is not in the pricing table.
+ */
+export function estimateCost(model, inputTokens, outputTokens, pricing) {
+    const p = pricing[model];
+    if (!p)
+        return 0;
+    return (inputTokens * p.input + outputTokens * p.output) / 1_000_000;
+}
 const DEFAULTS = {
     repo: '',
     repoOwner: '',
     project: 2,
-    model: 'opus',
-    reviewModel: 'opus',
+    agent: 'claude',
+    model: '',
+    reviewModel: '',
     pollInterval: 60,
     dryRun: false,
     baseBranch: 'master',
@@ -15,7 +26,6 @@ const DEFAULTS = {
     maxTestRetries: 3,
     testCommand: 'pnpm test',
     devCommand: 'pnpm dev',
-    port: 3000,
     skipTests: false,
     skipReview: false,
     skipInstall: false,
@@ -32,12 +42,27 @@ const DEFAULTS = {
     runFull: false,
     verbose: false,
     harnesses: [],
+    setupCommand: '',
+    evalDir: '.alpha-loop/evals',
+    evalModel: '',
+    skipEval: false,
+    evalTimeout: 300,
+    autoCapture: true,
+    skipPostSessionReview: false,
+    skipPostSessionSecurity: false,
+    pricing: {
+        'claude-opus-4-6': { input: 15.0, output: 75.0 },
+        'claude-sonnet-4-6': { input: 3.0, output: 15.0 },
+        'claude-haiku-4-5': { input: 0.80, output: 4.0 },
+        'codex-mini': { input: 1.50, output: 6.0 },
+    },
 };
 /** Map from YAML key (snake_case) to Config key (camelCase). */
 const YAML_KEY_MAP = {
     harnesses: 'harnesses',
     repo: 'repo',
     project: 'project',
+    agent: 'agent',
     model: 'model',
     review_model: 'reviewModel',
     poll_interval: 'pollInterval',
@@ -48,7 +73,6 @@ const YAML_KEY_MAP = {
     max_test_retries: 'maxTestRetries',
     test_command: 'testCommand',
     dev_command: 'devCommand',
-    port: 'port',
     skip_tests: 'skipTests',
     skip_review: 'skipReview',
     skip_install: 'skipInstall',
@@ -64,11 +88,18 @@ const YAML_KEY_MAP = {
     auto_cleanup: 'autoCleanup',
     run_full: 'runFull',
     verbose: 'verbose',
+    setup_command: 'setupCommand',
+    eval_dir: 'evalDir',
+    eval_model: 'evalModel',
+    skip_eval: 'skipEval',
+    eval_timeout: 'evalTimeout',
+    auto_capture: 'autoCapture',
 };
 /** Map from env var name to Config key. */
 const ENV_KEY_MAP = {
     REPO: 'repo',
-    PROJECT_NUM: 'project',
+    PROJECT: 'project',
+    AGENT: 'agent',
     MODEL: 'model',
     REVIEW_MODEL: 'reviewModel',
     POLL_INTERVAL: 'pollInterval',
@@ -79,7 +110,6 @@ const ENV_KEY_MAP = {
     MAX_TEST_RETRIES: 'maxTestRetries',
     TEST_COMMAND: 'testCommand',
     DEV_COMMAND: 'devCommand',
-    PORT: 'port',
     SKIP_TESTS: 'skipTests',
     SKIP_REVIEW: 'skipReview',
     SKIP_INSTALL: 'skipInstall',
@@ -95,6 +125,14 @@ const ENV_KEY_MAP = {
     AUTO_CLEANUP: 'autoCleanup',
     RUN_FULL: 'runFull',
     VERBOSE: 'verbose',
+    SETUP_COMMAND: 'setupCommand',
+    EVAL_DIR: 'evalDir',
+    EVAL_MODEL: 'evalModel',
+    SKIP_EVAL: 'skipEval',
+    EVAL_TIMEOUT: 'evalTimeout',
+    AUTO_CAPTURE: 'autoCapture',
+    SKIP_POST_SESSION_REVIEW: 'skipPostSessionReview',
+    SKIP_POST_SESSION_SECURITY: 'skipPostSessionSecurity',
 };
 function coerce(value, current) {
     if (typeof current === 'number')
@@ -103,8 +141,10 @@ function coerce(value, current) {
         return value === 'true' || value === '1';
     return value;
 }
-/** Validate a string contains only safe shell characters. */
+/** Validate a string contains only safe shell characters. Empty strings are allowed (model is optional). */
 export function assertSafeShellArg(value, name) {
+    if (value === '')
+        return value;
     if (!/^[a-zA-Z0-9._\-/]+$/.test(value)) {
         throw new Error(`Invalid ${name}: contains unsafe characters: ${value}`);
     }
@@ -143,6 +183,27 @@ function loadYamlConfig(configPath) {
             result[configKey] = parsed[yamlKey];
         }
     }
+    // Handle post_session nested config
+    if (parsed.post_session && typeof parsed.post_session === 'object') {
+        const ps = parsed.post_session;
+        if (ps.review === false)
+            result.skipPostSessionReview = true;
+        if (ps.security_scan === false)
+            result.skipPostSessionSecurity = true;
+    }
+    // Handle pricing table (nested object, not in YAML_KEY_MAP)
+    if (parsed.pricing && typeof parsed.pricing === 'object') {
+        const pricing = {};
+        for (const [model, value] of Object.entries(parsed.pricing)) {
+            const v = value;
+            if (typeof v?.input === 'number' && typeof v?.output === 'number') {
+                pricing[model] = { input: v.input, output: v.output };
+            }
+        }
+        if (Object.keys(pricing).length > 0) {
+            result.pricing = pricing;
+        }
+    }
     return result;
 }
 function loadEnvConfig() {
@@ -165,13 +226,25 @@ export function loadConfig(overrides) {
         autoDetect.repo = detectedRepo;
     }
     // Precedence: overrides (CLI flags) > env vars > config file > auto-detect > defaults
+    // Pricing is merged specially: YAML/overrides extend defaults rather than replacing
+    const mergedPricing = {
+        ...DEFAULTS.pricing,
+        ...yamlConfig.pricing,
+        ...overrides?.pricing,
+    };
     const merged = {
         ...DEFAULTS,
         ...autoDetect,
         ...yamlConfig,
         ...envConfig,
         ...overrides,
+        pricing: mergedPricing,
     };
+    // Validate agent is a known value
+    const VALID_AGENTS = ['claude', 'codex', 'opencode'];
+    if (!VALID_AGENTS.includes(merged.agent)) {
+        throw new Error(`Invalid agent: "${merged.agent}". Supported agents: ${VALID_AGENTS.join(', ')}`);
+    }
     // Derive repoOwner from repo
     if (merged.repo) {
         merged.repoOwner = merged.repo.split('/')[0] ?? '';

package/dist/lib/config.js.map

@@ -1 +1 @@
-{"version":3,"file":"config.js","sourceRoot":"","sources":["../../src/lib/config.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AACnD,OAAO,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAC9C,OAAO,EAAE,KAAK,IAAI,SAAS,EAAE,MAAM,MAAM,CAAC;AAmC1C,MAAM,QAAQ,GAAW;IACvB,IAAI,EAAE,EAAE;IACR,SAAS,EAAE,EAAE;IACb,OAAO,EAAE,CAAC;IACV,KAAK,EAAE,MAAM;IACb,WAAW,EAAE,MAAM;IACnB,YAAY,EAAE,EAAE;IAChB,MAAM,EAAE,KAAK;IACb,UAAU,EAAE,QAAQ;IACpB,MAAM,EAAE,MAAM;IACd,UAAU,EAAE,OAAO;IACnB,cAAc,EAAE,CAAC;IACjB,WAAW,EAAE,WAAW;IACxB,UAAU,EAAE,UAAU;IACtB,IAAI,EAAE,IAAI;IACV,SAAS,EAAE,KAAK;IAChB,UAAU,EAAE,KAAK;IACjB,WAAW,EAAE,KAAK;IAClB,aAAa,EAAE,KAAK;IACpB,UAAU,EAAE,KAAK;IACjB,SAAS,EAAE,KAAK;IAChB,OAAO,EAAE,KAAK;IACd,SAAS,EAAE,CAAC;IACZ,kBAAkB,EAAE,CAAC;IACrB,SAAS,EAAE,EAAE;IACb,SAAS,EAAE,IAAI;IACf,OAAO,EAAE,EAAE;IACX,WAAW,EAAE,IAAI;IACjB,OAAO,EAAE,KAAK;IACd,OAAO,EAAE,KAAK;IACd,SAAS,EAAE,EAAE;CACd,CAAC;AAEF,gEAAgE;AAChE,MAAM,YAAY,GAAiC;IACjD,SAAS,EAAE,WAAW;IACtB,IAAI,EAAE,MAAM;IACZ,OAAO,EAAE,SAAS;IAClB,KAAK,EAAE,OAAO;IACd,YAAY,EAAE,aAAa;IAC3B,aAAa,EAAE,cAAc;IAC7B,OAAO,EAAE,QAAQ;IACjB,WAAW,EAAE,YAAY;IACzB,OAAO,EAAE,QAAQ;IACjB,KAAK,EAAE,YAAY;IACnB,gBAAgB,EAAE,gBAAgB;IAClC,YAAY,EAAE,aAAa;IAC3B,WAAW,EAAE,YAAY;IACzB,IAAI,EAAE,MAAM;IACZ,UAAU,EAAE,WAAW;IACvB,WAAW,EAAE,YAAY;IACzB,YAAY,EAAE,aAAa;IAC3B,cAAc,EAAE,eAAe;IAC/B,WAAW,EAAE,YAAY;IACzB,UAAU,EAAE,WAAW;IACvB,QAAQ,EAAE,SAAS;IACnB,UAAU,EAAE,WAAW;IACvB,oBAAoB,EAAE,oBAAoB;IAC1C,SAAS,EAAE,WAAW;IACtB,UAAU,EAAE,WAAW;IACvB,QAAQ,EAAE,SAAS;IACnB,YAAY,EAAE,aAAa;IAC3B,QAAQ,EAAE,SAAS;IACnB,OAAO,EAAE,SAAS;CACnB,CAAC;AAEF,2CAA2C;AAC3C,MAAM,WAAW,GAAiC;IAChD,IAAI,EAAE,MAAM;IACZ,WAAW,EAAE,SAAS;IACtB,KAAK,EAAE,OAAO;IACd,YAAY,EAAE,aAAa;IAC3B,aAAa,EAAE,cAAc;IAC7B,OAAO,EAAE,QAAQ;IACjB,WAAW,EAAE,YAAY;IACzB,OAAO,EAAE,QAAQ;IACjB,WAAW,EAAE,YAAY;IACzB,gBAAgB,EAAE,gBAAgB;IAClC,YAAY,EAAE,aAAa;IAC3B,WAAW,EAAE,YAAY;IACzB,IAAI,EAAE,MAAM;IACZ,UAAU,EAAE,WAAW;IACvB,WAAW,EAAE,YAAY;IACzB,YAAY,EAAE,aAAa;IAC3B,cAAc,EAAE,eAAe;IAC/B,WAAW,EAAE,YAAY;IACzB,UAAU,EAAE,WAAW;IACvB,QAAQ,EAAE,SAAS;IACnB,UAAU,EAAE,WAAW;IACvB,oBAAoB,EAAE,oBAAoB;IAC1C,SAAS,EAAE,WAAW;IACtB,UAAU,EAAE,WAAW;IACvB,QAAQ,EAAE,SAAS;IACnB,YAAY,EAAE,aAAa;IAC3B,QAAQ,EAAE,SAAS;IACnB,OAAO,EAAE,SAAS;CACnB,CAAC;AAEF,SAAS,MAAM,CAAC,KAAa,EAAE,OAAgB;IAC7C,IAAI,OAAO,OAAO,KAAK,QAAQ;QAAE,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC;IACtD,IAAI,OAAO,OAAO,KAAK,SAAS;QAAE,OAAO,KAAK,KAAK,MAAM,IAAI,KAAK,KAAK,GAAG,CAAC;IAC3E,OAAO,KAAK,CAAC;AACf,CAAC;AAED,6DAA6D;AAC7D,MAAM,UAAU,kBAAkB,CAAC,KAAa,EAAE,IAAY;IAC5D,IAAI,CAAC,qBAAqB,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;QACvC,MAAM,IAAI,KAAK,CAAC,WAAW,IAAI,iCAAiC,KAAK,EAAE,CAAC,CAAC;IAC3E,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,MAAM,UAAU,UAAU;IACxB,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,QAAQ,CAAC,2BAA2B,EAAE;YAChD,QAAQ,EAAE,OAAO;YACjB,KAAK,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC;SAChC,CAAC,CAAC,IAAI,EAAE,CAAC;QAEV,2CAA2C;QAC3C,MAAM,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC,gCAAgC,CAAC,CAAC;QAC1D,IAAI,KAAK;YAAE,OAAO,GAAG,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC;QAE5C,qCAAqC;QACrC,MAAM,GAAG,GAAG,GAAG,CAAC,KAAK,CAAC,+BAA+B,CAAC,CAAC;QACvD,IAAI,GAAG;YAAE,OAAO,GAAG,GAAG,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;IACxC,CAAC;IAAC,MAAM,CAAC;QACP,8BAA8B;IAChC,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,cAAc,CAAC,UAAkB;IACxC,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC;QAAE,OAAO,EAAE,CAAC;IAEvC,MAAM,GAAG,GAAG,YAAY,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;IAC9C,MAAM,MAAM,GAAG,SAAS,CAAC,GAAG,CAAmC,CAAC;IAChE,IAAI,CAAC,MAAM,IAAI,OAAO,MAAM,KAAK,QAAQ;QAAE,OAAO,EAAE,CAAC;IAErD,MAAM,MAAM,GAAoB,EAAE,CAAC;IACnC,KAAK,MAAM,CAAC,OAAO,EAAE,SAAS,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,YAAY,CAAC,EAAE,CAAC;QAChE,IAAI,OAAO,IAAI,MAAM,EAAE,CAAC;YACrB,MAAkC,CAAC,SAAS,CAAC,GAAG,MAAM,CAAC,OAAO,CAAC,CAAC;QACnE,CAAC;IACH,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,aAAa;IACpB,MAAM,MAAM,GAAoB,EAAE,CAAC;IACnC,KAAK,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,WAAW,CAAC,EAAE,CAAC;QAC9D,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QAChC,IAAI,GAAG,KAAK,SAAS,EAAE,CAAC;YACrB,MAAkC,CAAC,SAAS,CAAC,GAAG,MAAM,CAAC,GAAG,EAAE,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAC;QACpF,CAAC;IACH,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,SAA2B;IACpD,MAAM,UAAU,GAAG,cAAc,CAAC,kBAAkB,CAAC,CAAC;IACtD,MAAM,SAAS,GAAG,aAAa,EAAE,CAAC;IAElC,4CAA4C;IAC5C,MAAM,YAAY,GAAG,UAAU,EAAE,CAAC;IAClC,MAAM,UAAU,GAAoB,EAAE,CAAC;IACvC,IAAI,YAAY,EAAE,CAAC;QACjB,UAAU,CAAC,IAAI,GAAG,YAAY,CAAC;IACjC,CAAC;IAED,sFAAsF;IACtF,MAAM,MAAM,GAAW;QACrB,GAAG,QAAQ;QACX,GAAG,UAAU;QACb,GAAG,UAAU;QACb,GAAG,SAAS;QACZ,GAAG,SAAS;KACb,CAAC;IAEF,6BAA6B;IAC7B,IAAI,MAAM,CAAC,IAAI,EAAE,CAAC;QAChB,MAAM,CAAC,SAAS,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IACrD,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC"}
+{"version":3,"file":"config.js","sourceRoot":"","sources":["../../src/lib/config.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AACnD,OAAO,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAC9C,OAAO,EAAE,KAAK,IAAI,SAAS,EAAE,MAAM,MAAM,CAAC;AAQ1C;;;GAGG;AACH,MAAM,UAAU,YAAY,CAC1B,KAAa,EACb,WAAmB,EACnB,YAAoB,EACpB,OAAqC;IAErC,MAAM,CAAC,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC;IACzB,IAAI,CAAC,CAAC;QAAE,OAAO,CAAC,CAAC;IACjB,OAAO,CAAC,WAAW,GAAG,CAAC,CAAC,KAAK,GAAG,YAAY,GAAG,CAAC,CAAC,MAAM,CAAC,GAAG,SAAS,CAAC;AACvE,CAAC;AAgDD,MAAM,QAAQ,GAAW;IACvB,IAAI,EAAE,EAAE;IACR,SAAS,EAAE,EAAE;IACb,OAAO,EAAE,CAAC;IACV,KAAK,EAAE,QAAQ;IACf,KAAK,EAAE,EAAE;IACT,WAAW,EAAE,EAAE;IACf,YAAY,EAAE,EAAE;IAChB,MAAM,EAAE,KAAK;IACb,UAAU,EAAE,QAAQ;IACpB,MAAM,EAAE,MAAM;IACd,UAAU,EAAE,OAAO;IACnB,cAAc,EAAE,CAAC;IACjB,WAAW,EAAE,WAAW;IACxB,UAAU,EAAE,UAAU;IACtB,SAAS,EAAE,KAAK;IAChB,UAAU,EAAE,KAAK;IACjB,WAAW,EAAE,KAAK;IAClB,aAAa,EAAE,KAAK;IACpB,UAAU,EAAE,KAAK;IACjB,SAAS,EAAE,KAAK;IAChB,OAAO,EAAE,KAAK;IACd,SAAS,EAAE,CAAC;IACZ,kBAAkB,EAAE,CAAC;IACrB,SAAS,EAAE,EAAE;IACb,SAAS,EAAE,IAAI;IACf,OAAO,EAAE,EAAE;IACX,WAAW,EAAE,IAAI;IACjB,OAAO,EAAE,KAAK;IACd,OAAO,EAAE,KAAK;IACd,SAAS,EAAE,EAAE;IACb,YAAY,EAAE,EAAE;IAChB,OAAO,EAAE,mBAAmB;IAC5B,SAAS,EAAE,EAAE;IACb,QAAQ,EAAE,KAAK;IACf,WAAW,EAAE,GAAG;IAChB,WAAW,EAAE,IAAI;IACjB,qBAAqB,EAAE,KAAK;IAC5B,uBAAuB,EAAE,KAAK;IAC9B,OAAO,EAAE;QACP,iBAAiB,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE;QAChD,mBAAmB,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE;QACjD,kBAAkB,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE;QAChD,YAAY,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE;KAC3C;CACF,CAAC;AAEF,gEAAgE;AAChE,MAAM,YAAY,GAAiC;IACjD,SAAS,EAAE,WAAW;IACtB,IAAI,EAAE,MAAM;IACZ,OAAO,EAAE,SAAS;IAClB,KAAK,EAAE,OAAO;IACd,KAAK,EAAE,OAAO;IACd,YAAY,EAAE,aAAa;IAC3B,aAAa,EAAE,cAAc;IAC7B,OAAO,EAAE,QAAQ;IACjB,WAAW,EAAE,YAAY;IACzB,OAAO,EAAE,QAAQ;IACjB,KAAK,EAAE,YAAY;IACnB,gBAAgB,EAAE,gBAAgB;IAClC,YAAY,EAAE,aAAa;IAC3B,WAAW,EAAE,YAAY;IACzB,UAAU,EAAE,WAAW;IACvB,WAAW,EAAE,YAAY;IACzB,YAAY,EAAE,aAAa;IAC3B,cAAc,EAAE,eAAe;IAC/B,WAAW,EAAE,YAAY;IACzB,UAAU,EAAE,WAAW;IACvB,QAAQ,EAAE,SAAS;IACnB,UAAU,EAAE,WAAW;IACvB,oBAAoB,EAAE,oBAAoB;IAC1C,SAAS,EAAE,WAAW;IACtB,UAAU,EAAE,WAAW;IACvB,QAAQ,EAAE,SAAS;IACnB,YAAY,EAAE,aAAa;IAC3B,QAAQ,EAAE,SAAS;IACnB,OAAO,EAAE,SAAS;IAClB,aAAa,EAAE,cAAc;IAC7B,QAAQ,EAAE,SAAS;IACnB,UAAU,EAAE,WAAW;IACvB,SAAS,EAAE,UAAU;IACrB,YAAY,EAAE,aAAa;IAC3B,YAAY,EAAE,aAAa;CAC5B,CAAC;AAEF,2CAA2C;AAC3C,MAAM,WAAW,GAAiC;IAChD,IAAI,EAAE,MAAM;IACZ,OAAO,EAAE,SAAS;IAClB,KAAK,EAAE,OAAO;IACd,KAAK,EAAE,OAAO;IACd,YAAY,EAAE,aAAa;IAC3B,aAAa,EAAE,cAAc;IAC7B,OAAO,EAAE,QAAQ;IACjB,WAAW,EAAE,YAAY;IACzB,OAAO,EAAE,QAAQ;IACjB,WAAW,EAAE,YAAY;IACzB,gBAAgB,EAAE,gBAAgB;IAClC,YAAY,EAAE,aAAa;IAC3B,WAAW,EAAE,YAAY;IACzB,UAAU,EAAE,WAAW;IACvB,WAAW,EAAE,YAAY;IACzB,YAAY,EAAE,aAAa;IAC3B,cAAc,EAAE,eAAe;IAC/B,WAAW,EAAE,YAAY;IACzB,UAAU,EAAE,WAAW;IACvB,QAAQ,EAAE,SAAS;IACnB,UAAU,EAAE,WAAW;IACvB,oBAAoB,EAAE,oBAAoB;IAC1C,SAAS,EAAE,WAAW;IACtB,UAAU,EAAE,WAAW;IACvB,QAAQ,EAAE,SAAS;IACnB,YAAY,EAAE,aAAa;IAC3B,QAAQ,EAAE,SAAS;IACnB,OAAO,EAAE,SAAS;IAClB,aAAa,EAAE,cAAc;IAC7B,QAAQ,EAAE,SAAS;IACnB,UAAU,EAAE,WAAW;IACvB,SAAS,EAAE,UAAU;IACrB,YAAY,EAAE,aAAa;IAC3B,YAAY,EAAE,aAAa;IAC3B,wBAAwB,EAAE,uBAAuB;IACjD,0BAA0B,EAAE,yBAAyB;CACtD,CAAC;AAEF,SAAS,MAAM,CAAC,KAAa,EAAE,OAAgB;IAC7C,IAAI,OAAO,OAAO,KAAK,QAAQ;QAAE,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC;IACtD,IAAI,OAAO,OAAO,KAAK,SAAS;QAAE,OAAO,KAAK,KAAK,MAAM,IAAI,KAAK,KAAK,GAAG,CAAC;IAC3E,OAAO,KAAK,CAAC;AACf,CAAC;AAED,4GAA4G;AAC5G,MAAM,UAAU,kBAAkB,CAAC,KAAa,EAAE,IAAY;IAC5D,IAAI,KAAK,KAAK,EAAE;QAAE,OAAO,KAAK,CAAC;IAC/B,IAAI,CAAC,qBAAqB,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;QACvC,MAAM,IAAI,KAAK,CAAC,WAAW,IAAI,iCAAiC,KAAK,EAAE,CAAC,CAAC;IAC3E,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,MAAM,UAAU,UAAU;IACxB,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,QAAQ,CAAC,2BAA2B,EAAE;YAChD,QAAQ,EAAE,OAAO;YACjB,KAAK,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC;SAChC,CAAC,CAAC,IAAI,EAAE,CAAC;QAEV,2CAA2C;QAC3C,MAAM,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC,gCAAgC,CAAC,CAAC;QAC1D,IAAI,KAAK;YAAE,OAAO,GAAG,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC;QAE5C,qCAAqC;QACrC,MAAM,GAAG,GAAG,GAAG,CAAC,KAAK,CAAC,+BAA+B,CAAC,CAAC;QACvD,IAAI,GAAG;YAAE,OAAO,GAAG,GAAG,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;IACxC,CAAC;IAAC,MAAM,CAAC;QACP,8BAA8B;IAChC,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,cAAc,CAAC,UAAkB;IACxC,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC;QAAE,OAAO,EAAE,CAAC;IAEvC,MAAM,GAAG,GAAG,YAAY,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;IAC9C,MAAM,MAAM,GAAG,SAAS,CAAC,GAAG,CAAmC,CAAC;IAChE,IAAI,CAAC,MAAM,IAAI,OAAO,MAAM,KAAK,QAAQ;QAAE,OAAO,EAAE,CAAC;IAErD,MAAM,MAAM,GAAoB,EAAE,CAAC;IACnC,KAAK,MAAM,CAAC,OAAO,EAAE,SAAS,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,YAAY,CAAC,EAAE,CAAC;QAChE,IAAI,OAAO,IAAI,MAAM,EAAE,CAAC;YACrB,MAAkC,CAAC,SAAS,CAAC,GAAG,MAAM,CAAC,OAAO,CAAC,CAAC;QACnE,CAAC;IACH,CAAC;IAED,oCAAoC;IACpC,IAAI,MAAM,CAAC,YAAY,IAAI,OAAO,MAAM,CAAC,YAAY,KAAK,QAAQ,EAAE,CAAC;QACnE,MAAM,EAAE,GAAG,MAAM,CAAC,YAAuC,CAAC;QAC1D,IAAI,EAAE,CAAC,MAAM,KAAK,KAAK;YAAE,MAAM,CAAC,qBAAqB,GAAG,IAAI,CAAC;QAC7D,IAAI,EAAE,CAAC,aAAa,KAAK,KAAK;YAAE,MAAM,CAAC,uBAAuB,GAAG,IAAI,CAAC;IACxE,CAAC;IAED,4DAA4D;IAC5D,IAAI,MAAM,CAAC,OAAO,IAAI,OAAO,MAAM,CAAC,OAAO,KAAK,QAAQ,EAAE,CAAC;QACzD,MAAM,OAAO,GAAsD,EAAE,CAAC;QACtE,KAAK,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,OAAkC,CAAC,EAAE,CAAC;YACvF,MAAM,CAAC,GAAG,KAAgC,CAAC;YAC3C,IAAI,OAAO,CAAC,EAAE,KAAK,KAAK,QAAQ,IAAI,OAAO,CAAC,EAAE,MAAM,KAAK,QAAQ,EAAE,CAAC;gBAClE,OAAO,CAAC,KAAK,CAAC,GAAG,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC;YACxD,CAAC;QACH,CAAC;QACD,IAAI,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACpC,MAAM,CAAC,OAAO,GAAG,OAAO,CAAC;QAC3B,CAAC;IACH,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,aAAa;IACpB,MAAM,MAAM,GAAoB,EAAE,CAAC;IACnC,KAAK,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,WAAW,CAAC,EAAE,CAAC;QAC9D,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QAChC,IAAI,GAAG,KAAK,SAAS,EAAE,CAAC;YACrB,MAAkC,CAAC,SAAS,CAAC,GAAG,MAAM,CAAC,GAAG,EAAE,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAC;QACpF,CAAC;IACH,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,SAA2B;IACpD,MAAM,UAAU,GAAG,cAAc,CAAC,kBAAkB,CAAC,CAAC;IACtD,MAAM,SAAS,GAAG,aAAa,EAAE,CAAC;IAElC,4CAA4C;IAC5C,MAAM,YAAY,GAAG,UAAU,EAAE,CAAC;IAClC,MAAM,UAAU,GAAoB,EAAE,CAAC;IACvC,IAAI,YAAY,EAAE,CAAC;QACjB,UAAU,CAAC,IAAI,GAAG,YAAY,CAAC;IACjC,CAAC;IAED,sFAAsF;IACtF,oFAAoF;IACpF,MAAM,aAAa,GAAG;QACpB,GAAG,QAAQ,CAAC,OAAO;QACnB,GAAG,UAAU,CAAC,OAAO;QACrB,GAAG,SAAS,EAAE,OAAO;KACtB,CAAC;IAEF,MAAM,MAAM,GAAW;QACrB,GAAG,QAAQ;QACX,GAAG,UAAU;QACb,GAAG,UAAU;QACb,GAAG,SAAS;QACZ,GAAG,SAAS;QACZ,OAAO,EAAE,aAAa;KACvB,CAAC;IAEF,kCAAkC;IAClC,MAAM,YAAY,GAAG,CAAC,QAAQ,EAAE,OAAO,EAAE,UAAU,CAAU,CAAC;IAC9D,IAAI,CAAC,YAAY,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAoC,CAAC,EAAE,CAAC;QACxE,MAAM,IAAI,KAAK,CAAC,mBAAmB,MAAM,CAAC,KAAK,wBAAwB,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACpG,CAAC;IAED,6BAA6B;IAC7B,IAAI,MAAM,CAAC,IAAI,EAAE,CAAC;QAChB,MAAM,CAAC,SAAS,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IACrD,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC"}

package/dist/lib/eval-checks.d.ts

@@ -0,0 +1,91 @@
+/** Result of running a single check. */
+export type CheckResult = {
+    passed: boolean;
+    score: number;
+    detail: string;
+};
+/** Base check with a type discriminator. */
+type BaseCheck = {
+    type: string;
+};
+/** All tests must pass. */
+export type TestPassCheck = BaseCheck & {
+    type: 'test_pass';
+};
+/** A specific file must exist. */
+export type FileExistsCheck = BaseCheck & {
+    type: 'file_exists';
+    path: string;
+};
+/** A pattern must exist in a file. */
+export type GrepCheck = BaseCheck & {
+    type: 'grep';
+    file: string;
+    pattern: string;
+};
+/** HTTP endpoint check. */
+export type HttpCheck = BaseCheck & {
+    type: 'http';
+    method: string;
+    path: string;
+    port?: number;
+    expect_status: number;
+    expect_body_contains?: string;
+};
+/** Diff size limit check. */
+export type DiffSizeCheck = BaseCheck & {
+    type: 'diff_size';
+    max_files?: number;
+    max_lines?: number;
+};
+/** Keywords that must be present in output. */
+export type KeywordPresentCheck = BaseCheck & {
+    type: 'keyword_present';
+    keywords: string[];
+};
+/** Keywords that must be absent from output. */
+export type KeywordAbsentCheck = BaseCheck & {
+    type: 'keyword_absent';
+    keywords: string[];
+};
+/** LLM-judge evaluation. */
+export type LlmJudgeCheck = BaseCheck & {
+    type: 'llm_judge';
+    model: string;
+    rubric: string;
+    min_score: number;
+};
+/** Union of all check types. */
+export type CheckDefinition = TestPassCheck | FileExistsCheck | GrepCheck | HttpCheck | DiffSizeCheck | KeywordPresentCheck | KeywordAbsentCheck | LlmJudgeCheck;
+/** Context passed to check runners. */
+export type CheckContext = {
+    /** Working directory (worktree root). */
+    cwd: string;
+    /** Test command to run. */
+    testCommand?: string;
+    /** Agent output (for step evals). */
+    output?: string;
+    /** Git diff of changes. */
+    diff?: string;
+    /** List of changed files. */
+    filesChanged?: string[];
+    /** Model to use for LLM judge (fallback). */
+    judgeModel?: string;
+};
+/**
+ * Run a single check against the given context.
+ */
+export declare function runCheck(check: CheckDefinition, ctx: CheckContext): Promise<CheckResult>;
+/**
+ * Run all checks and return aggregate results.
+ */
+export declare function runChecks(checks: CheckDefinition[], ctx: CheckContext): Promise<{
+    results: CheckResult[];
+    allPassed: boolean;
+    avgScore: number;
+}>;
+/**
+ * Parse check definitions from a checks.yaml object.
+ */
+export declare function parseChecks(raw: unknown): CheckDefinition[];
+export {};

package/dist/lib/eval-checks.js

@@ -0,0 +1,254 @@
+/**
+ * Eval Check Executor — runs machine-checkable acceptance criteria.
+ *
+ * Check types:
+ *   E2E: test_pass, file_exists, grep, http, diff_size
+ *   Step: keyword_present, keyword_absent, llm_judge
+ *
+ * Each check returns { passed, score, detail }.
+ */
+import { existsSync, readFileSync } from 'node:fs';
+import { join } from 'node:path';
+import { exec } from './shell.js';
+import { spawnAgent } from './agent.js';
+import { log } from './logger.js';
+/**
+ * Run a single check against the given context.
+ */
+export async function runCheck(check, ctx) {
+    switch (check.type) {
+        case 'test_pass':
+            return runTestPassCheck(check, ctx);
+        case 'file_exists':
+            return runFileExistsCheck(check, ctx);
+        case 'grep':
+            return runGrepCheck(check, ctx);
+        case 'http':
+            return runHttpCheck(check, ctx);
+        case 'diff_size':
+            return runDiffSizeCheck(check, ctx);
+        case 'keyword_present':
+            return runKeywordPresentCheck(check, ctx);
+        case 'keyword_absent':
+            return runKeywordAbsentCheck(check, ctx);
+        case 'llm_judge':
+            return runLlmJudgeCheck(check, ctx);
+        default:
+            return { passed: false, score: 0, detail: `Unknown check type: ${check.type}` };
+    }
+}
+/**
+ * Run all checks and return aggregate results.
+ */
+export async function runChecks(checks, ctx) {
+    const results = [];
+    for (const check of checks) {
+        try {
+            const result = await runCheck(check, ctx);
+            results.push(result);
+        }
+        catch (err) {
+            results.push({
+                passed: false,
+                score: 0,
+                detail: `Check ${check.type} threw: ${err instanceof Error ? err.message : String(err)}`,
+            });
+        }
+    }
+    const allPassed = results.every((r) => r.passed);
+    const avgScore = results.length > 0
+        ? results.reduce((sum, r) => sum + r.score, 0) / results.length
+        : 0;
+    return { results, allPassed, avgScore };
+}
+// --- Individual check runners ---
+async function runTestPassCheck(_check, ctx) {
+    const cmd = ctx.testCommand ?? 'npm test';
+    const result = exec(cmd, { cwd: ctx.cwd, timeout: 120_000 });
+    const passed = result.exitCode === 0;
+    return {
+        passed,
+        score: passed ? 1 : 0,
+        detail: passed ? 'All tests passed' : `Tests failed (exit ${result.exitCode}): ${result.stderr.slice(0, 500)}`,
+    };
+}
+async function runFileExistsCheck(check, ctx) {
+    const fullPath = join(ctx.cwd, check.path);
+    const passed = existsSync(fullPath);
+    return {
+        passed,
+        score: passed ? 1 : 0,
+        detail: passed ? `File exists: ${check.path}` : `File not found: ${check.path}`,
+    };
+}
+async function runGrepCheck(check, ctx) {
+    const fullPath = join(ctx.cwd, check.file);
+    if (!existsSync(fullPath)) {
+        return { passed: false, score: 0, detail: `File not found: ${check.file}` };
+    }
+    const content = readFileSync(fullPath, 'utf-8');
+    const regex = new RegExp(check.pattern);
+    const passed = regex.test(content);
+    return {
+        passed,
+        score: passed ? 1 : 0,
+        detail: passed ? `Pattern "${check.pattern}" found in ${check.file}` : `Pattern "${check.pattern}" not found in ${check.file}`,
+    };
+}
+async function runHttpCheck(check, ctx) {
+    try {
+        const url = `http://localhost:${check.port ?? 3000}${check.path}`;
+        const response = await fetch(url, { method: check.method, signal: AbortSignal.timeout(10_000) });
+        const body = await response.text();
+        const statusMatch = response.status === check.expect_status;
+        const bodyMatch = check.expect_body_contains ? body.includes(check.expect_body_contains) : true;
+        const passed = statusMatch && bodyMatch;
+        return {
+            passed,
+            score: passed ? 1 : 0,
+            detail: passed
+                ? `HTTP ${check.method} ${check.path} returned ${response.status}`
+                : `HTTP check failed: status=${response.status} (expected ${check.expect_status}), body match=${bodyMatch}`,
+        };
+    }
+    catch (err) {
+        return {
+            passed: false,
+            score: 0,
+            detail: `HTTP check failed: ${err instanceof Error ? err.message : String(err)}`,
+        };
+    }
+}
+async function runDiffSizeCheck(check, ctx) {
+    const filesCount = ctx.filesChanged?.length ?? 0;
+    const diff = ctx.diff ?? '';
+    const lineCount = diff.split('\n').filter((l) => l.startsWith('+') || l.startsWith('-')).length;
+    const fileOk = check.max_files == null || filesCount <= check.max_files;
+    const lineOk = check.max_lines == null || lineCount <= check.max_lines;
+    const passed = fileOk && lineOk;
+    return {
+        passed,
+        score: passed ? 1 : 0,
+        detail: `Diff: ${filesCount} files, ${lineCount} lines` +
+            (!fileOk ? ` (max ${check.max_files} files exceeded)` : '') +
+            (!lineOk ? ` (max ${check.max_lines} lines exceeded)` : ''),
+    };
+}
+async function runKeywordPresentCheck(check, ctx) {
+    const output = ctx.output ?? '';
+    const found = check.keywords.filter((kw) => output.includes(kw));
+    const missing = check.keywords.filter((kw) => !output.includes(kw));
+    const passed = missing.length === 0;
+    return {
+        passed,
+        score: check.keywords.length > 0 ? found.length / check.keywords.length : 1,
+        detail: passed
+            ? `All keywords found: ${check.keywords.join(', ')}`
+            : `Missing keywords: ${missing.join(', ')}`,
+    };
+}
+async function runKeywordAbsentCheck(check, ctx) {
+    const output = ctx.output ?? '';
+    const present = check.keywords.filter((kw) => output.includes(kw));
+    const passed = present.length === 0;
+    return {
+        passed,
+        score: passed ? 1 : 0,
+        detail: passed
+            ? `No forbidden keywords found`
+            : `Forbidden keywords present: ${present.join(', ')}`,
+    };
+}
+async function runLlmJudgeCheck(check, ctx) {
+    const model = check.model || ctx.judgeModel || 'claude-haiku-4-5';
+    const prompt = `You are an evaluation judge. Score the following output on a scale of 1-5 based on this rubric.
+
+## Rubric
+${check.rubric}
+
+## Output to evaluate
+${(ctx.output ?? '').slice(0, 8000)}
+
+Respond with ONLY a single number (1-5) on the first line, followed by a brief explanation.`;
+    try {
+        const result = await spawnAgent({
+            agent: 'claude',
+            model,
+            prompt,
+            cwd: ctx.cwd,
+            timeout: 60_000,
+            maxTurns: 1,
+        });
+        const scoreMatch = result.output.match(/^(\d)/m);
+        const score = scoreMatch ? parseInt(scoreMatch[1], 10) : 0;
+        const normalizedScore = Math.max(0, Math.min(5, score));
+        const passed = normalizedScore >= check.min_score;
+        return {
+            passed,
+            score: normalizedScore / 5,
+            detail: `LLM judge score: ${normalizedScore}/5 (min: ${check.min_score}). ${result.output.slice(0, 200)}`,
+        };
+    }
+    catch (err) {
+        log.warn(`LLM judge check failed: ${err instanceof Error ? err.message : String(err)}`);
+        return {
+            passed: false,
+            score: 0,
+            detail: `LLM judge failed: ${err instanceof Error ? err.message : String(err)}`,
+        };
+    }
+}
+/**
+ * Parse check definitions from a checks.yaml object.
+ */
+export function parseChecks(raw) {
+    if (!raw || typeof raw !== 'object')
+        return [];
+    const obj = raw;
+    const checks = Array.isArray(obj.checks) ? obj.checks : [];
+    return checks.map((c) => {
+        const type = String(c.type ?? '');
+        switch (type) {
+            case 'test_pass':
+                return { type: 'test_pass' };
+            case 'file_exists':
+                return { type: 'file_exists', path: String(c.path ?? '') };
+            case 'grep':
+                return { type: 'grep', file: String(c.file ?? ''), pattern: String(c.pattern ?? '') };
+            case 'http':
+                return {
+                    type: 'http',
+                    method: String(c.method ?? 'GET'),
+                    path: String(c.path ?? '/'),
+                    expect_status: Number(c.expect_status ?? 200),
+                    expect_body_contains: c.expect_body_contains ? String(c.expect_body_contains) : undefined,
+                };
+            case 'diff_size':
+                return {
+                    type: 'diff_size',
+                    max_files: c.max_files != null ? Number(c.max_files) : undefined,
+                    max_lines: c.max_lines != null ? Number(c.max_lines) : undefined,
+                };
+            case 'keyword_present':
+                return {
+                    type: 'keyword_present',
+                    keywords: Array.isArray(c.keywords) ? c.keywords.map(String) : [],
+                };
+            case 'keyword_absent':
+                return {
+                    type: 'keyword_absent',
+                    keywords: Array.isArray(c.keywords) ? c.keywords.map(String) : [],
+                };
+            case 'llm_judge':
+                return {
+                    type: 'llm_judge',
+                    model: String(c.model ?? ''),
+                    rubric: String(c.rubric ?? ''),
+                    min_score: Number(c.min_score ?? 3),
+                };
+            default:
+                return { type };
+        }
+    });
+}
+//# sourceMappingURL=eval-checks.js.map

package/dist/lib/eval-checks.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval-checks.js","sourceRoot":"","sources":["../../src/lib/eval-checks.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AACH,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AACnD,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAClC,OAAO,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AACxC,OAAO,EAAE,GAAG,EAAE,MAAM,aAAa,CAAC;AA8FlC;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,QAAQ,CAAC,KAAsB,EAAE,GAAiB;IACtE,QAAQ,KAAK,CAAC,IAAI,EAAE,CAAC;QACnB,KAAK,WAAW;YACd,OAAO,gBAAgB,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;QACtC,KAAK,aAAa;YAChB,OAAO,kBAAkB,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;QACxC,KAAK,MAAM;YACT,OAAO,YAAY,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;QAClC,KAAK,MAAM;YACT,OAAO,YAAY,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;QAClC,KAAK,WAAW;YACd,OAAO,gBAAgB,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;QACtC,KAAK,iBAAiB;YACpB,OAAO,sBAAsB,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;QAC5C,KAAK,gBAAgB;YACnB,OAAO,qBAAqB,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;QAC3C,KAAK,WAAW;YACd,OAAO,gBAAgB,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;QACtC;YACE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,uBAAwB,KAAmB,CAAC,IAAI,EAAE,EAAE,CAAC;IACnG,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,MAAyB,EAAE,GAAiB;IAK1E,MAAM,OAAO,GAAkB,EAAE,CAAC;IAClC,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;YAC1C,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACvB,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,OAAO,CAAC,IAAI,CAAC;gBACX,MAAM,EAAE,KAAK;gBACb,KAAK,EAAE,CAAC;gBACR,MAAM,EAAE,SAAS,KAAK,CAAC,IAAI,WAAW,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE;aACzF,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,MAAM,SAAS,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;IACjD,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,GAAG,CAAC;QACjC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,GAAG,OAAO,CAAC,MAAM;QAC/D,CAAC,CAAC,CAAC,CAAC;IAEN,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE,QAAQ,EAAE,CAAC;AAC1C,CAAC;AAED,mCAAmC;AAEnC,KAAK,UAAU,gBAAgB,CAAC,MAAqB,EAAE,GAAiB;IACtE,MAAM,GAAG,GAAG,GAAG,CAAC,WAAW,IAAI,UAAU,CAAC;IAC1C,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,EAAE,EAAE,GAAG,EAAE,GAAG,CAAC,GAAG,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,CAAC;IAC7D,MAAM,MAAM,GAAG,MAAM,CAAC,QAAQ,KAAK,CAAC,CAAC;IACrC,OAAO;QACL,MAAM;QACN,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACrB,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC,kBAAkB,CAAC,CAAC,CAAC,sBAAsB,MAAM,CAAC,QAAQ,MAAM,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE;KAC/G,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,kBAAkB,CAAC,KAAsB,EAAE,GAAiB;IACzE,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;IAC3C,MAAM,MAAM,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC;IACpC,OAAO;QACL,MAAM;QACN,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACrB,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC,gBAAgB,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,mBAAmB,KAAK,CAAC,IAAI,EAAE;KAChF,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,YAAY,CAAC,KAAgB,EAAE,GAAiB;IAC7D,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;IAC3C,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC1B,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,mBAAmB,KAAK,CAAC,IAAI,EAAE,EAAE,CAAC;IAC9E,CAAC;IACD,MAAM,OAAO,GAAG,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IAChD,MAAM,KAAK,GAAG,IAAI,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;IACxC,MAAM,MAAM,GAAG,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACnC,OAAO;QACL,MAAM;QACN,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACrB,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC,YAAY,KAAK,CAAC,OAAO,cAAc,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,YAAY,KAAK,CAAC,OAAO,kBAAkB,KAAK,CAAC,IAAI,EAAE;KAC/H,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,YAAY,CAAC,KAAgB,EAAE,GAAiB;IAC7D,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,oBAAoB,KAAK,CAAC,IAAI,IAAI,IAAI,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC;QAClE,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;QACjG,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;QACnC,MAAM,WAAW,GAAG,QAAQ,CAAC,MAAM,KAAK,KAAK,CAAC,aAAa,CAAC;QAC5D,MAAM,SAAS,GAAG,KAAK,CAAC,oBAAoB,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,oBAAoB,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QAChG,MAAM,MAAM,GAAG,WAAW,IAAI,SAAS,CAAC;QACxC,OAAO;YACL,MAAM;YACN,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACrB,MAAM,EAAE,MAAM;gBACZ,CAAC,CAAC,QAAQ,KAAK,CAAC,MAAM,IAAI,KAAK,CAAC,IAAI,aAAa,QAAQ,CAAC,MAAM,EAAE;gBAClE,CAAC,CAAC,6BAA6B,QAAQ,CAAC,MAAM,cAAc,KAAK,CAAC,aAAa,iBAAiB,SAAS,EAAE;SAC9G,CAAC;IACJ,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,OAAO;YACL,MAAM,EAAE,KAAK;YACb,KAAK,EAAE,CAAC;YACR,MAAM,EAAE,sBAAsB,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE;SACjF,CAAC;IACJ,CAAC;AACH,CAAC;AAED,KAAK,UAAU,gBAAgB,CAAC,KAAoB,EAAE,GAAiB;IACrE,MAAM,UAAU,GAAG,GAAG,CAAC,YAAY,EAAE,MAAM,IAAI,CAAC,CAAC;IACjD,MAAM,IAAI,GAAG,GAAG,CAAC,IAAI,IAAI,EAAE,CAAC;IAC5B,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC;IAEhG,MAAM,MAAM,GAAG,KAAK,CAAC,SAAS,IAAI,IAAI,IAAI,UAAU,IAAI,KAAK,CAAC,SAAS,CAAC;IACxE,MAAM,MAAM,GAAG,KAAK,CAAC,SAAS,IAAI,IAAI,IAAI,SAAS,IAAI,KAAK,CAAC,SAAS,CAAC;IACvE,MAAM,MAAM,GAAG,MAAM,IAAI,MAAM,CAAC;IAEhC,OAAO;QACL,MAAM;QACN,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACrB,MAAM,EAAE,SAAS,UAAU,WAAW,SAAS,QAAQ;YACrD,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,KAAK,CAAC,SAAS,kBAAkB,CAAC,CAAC,CAAC,EAAE,CAAC;YAC3D,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,KAAK,CAAC,SAAS,kBAAkB,CAAC,CAAC,CAAC,EAAE,CAAC;KAC9D,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,sBAAsB,CAAC,KAA0B,EAAE,GAAiB;IACjF,MAAM,MAAM,GAAG,GAAG,CAAC,MAAM,IAAI,EAAE,CAAC;IAChC,MAAM,KAAK,GAAG,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC;IACjE,MAAM,OAAO,GAAG,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC;IACpE,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,KAAK,CAAC,CAAC;IACpC,OAAO;QACL,MAAM;QACN,KAAK,EAAE,KAAK,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,GAAG,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;QAC3E,MAAM,EAAE,MAAM;YACZ,CAAC,CAAC,uBAAuB,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;YACpD,CAAC,CAAC,qBAAqB,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;KAC9C,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,qBAAqB,CAAC,KAAyB,EAAE,GAAiB;IAC/E,MAAM,MAAM,GAAG,GAAG,CAAC,MAAM,IAAI,EAAE,CAAC;IAChC,MAAM,OAAO,GAAG,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC;IACnE,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,KAAK,CAAC,CAAC;IACpC,OAAO;QACL,MAAM;QACN,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACrB,MAAM,EAAE,MAAM;YACZ,CAAC,CAAC,6BAA6B;YAC/B,CAAC,CAAC,+BAA+B,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;KACxD,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,gBAAgB,CAAC,KAAoB,EAAE,GAAiB;IACrE,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,IAAI,GAAG,CAAC,UAAU,IAAI,kBAAkB,CAAC;IAClE,MAAM,MAAM,GAAG;;;EAGf,KAAK,CAAC,MAAM;;;EAGZ,CAAC,GAAG,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC;;4FAEyD,CAAC;IAE3F,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC;YAC9B,KAAK,EAAE,QAAQ;YACf,KAAK;YACL,MAAM;YACN,GAAG,EAAE,GAAG,CAAC,GAAG;YACZ,OAAO,EAAE,MAAM;YACf,QAAQ,EAAE,CAAC;SACZ,CAAC,CAAC;QAEH,MAAM,UAAU,GAAG,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;QACjD,MAAM,KAAK,GAAG,UAAU,CAAC,CAAC,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAC3D,MAAM,eAAe,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC;QACxD,MAAM,MAAM,GAAG,eAAe,IAAI,KAAK,CAAC,SAAS,CAAC;QAElD,OAAO;YACL,MAAM;YACN,KAAK,EAAE,eAAe,GAAG,CAAC;YAC1B,MAAM,EAAE,oBAAoB,eAAe,YAAY,KAAK,CAAC,SAAS,MAAM,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE;SAC1G,CAAC;IACJ,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,GAAG,CAAC,IAAI,CAAC,2BAA2B,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QACxF,OAAO;YACL,MAAM,EAAE,KAAK;YACb,KAAK,EAAE,CAAC;YACR,MAAM,EAAE,qBAAqB,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE;SAChF,CAAC;IACJ,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,WAAW,CAAC,GAAY;IACtC,IAAI,CAAC,GAAG,IAAI,OAAO,GAAG,KAAK,QAAQ;QAAE,OAAO,EAAE,CAAC;IAC/C,MAAM,GAAG,GAAG,GAA8B,CAAC;IAC3C,MAAM,MAAM,GAAG,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC;IAE3D,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,CAA0B,EAAE,EAAE;QAC/C,MAAM,IAAI,GAAG,MAAM,CAAC,CAAC,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC;QAClC,QAAQ,IAAI,EAAE,CAAC;YACb,KAAK,WAAW;gBACd,OAAO,EAAE,IAAI,EAAE,WAAW,EAAmB,CAAC;YAChD,KAAK,aAAa;gBAChB,OAAO,EAAE,IAAI,EAAE,aAAa,EAAE,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC,IAAI,IAAI,EAAE,CAAC,EAAqB,CAAC;YAChF,KAAK,MAAM;gBACT,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC,IAAI,IAAI,EAAE,CAAC,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC,OAAO,IAAI,EAAE,CAAC,EAAe,CAAC;YACrG,KAAK,MAAM;gBACT,OAAO;oBACL,IAAI,EAAE,MAAM;oBACZ,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC,MAAM,IAAI,KAAK,CAAC;oBACjC,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC,IAAI,IAAI,GAAG,CAAC;oBAC3B,aAAa,EAAE,MAAM,CAAC,CAAC,CAAC,aAAa,IAAI,GAAG,CAAC;oBAC7C,oBAAoB,EAAE,CAAC,CAAC,oBAAoB,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,oBAAoB,CAAC,CAAC,CAAC,CAAC,SAAS;iBAC7E,CAAC;YACjB,KAAK,WAAW;gBACd,OAAO;oBACL,IAAI,EAAE,WAAW;oBACjB,SAAS,EAAE,CAAC,CAAC,SAAS,IAAI,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,SAAS;oBAChE,SAAS,EAAE,CAAC,CAAC,SAAS,IAAI,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,SAAS;iBAChD,CAAC;YACrB,KAAK,iBAAiB;gBACpB,OAAO;oBACL,IAAI,EAAE,iBAAiB;oBACvB,QAAQ,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE;iBAC3C,CAAC;YAC3B,KAAK,gBAAgB;gBACnB,OAAO;oBACL,IAAI,EAAE,gBAAgB;oBACtB,QAAQ,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE;iBAC5C,CAAC;YAC1B,KAAK,WAAW;gBACd,OAAO;oBACL,IAAI,EAAE,WAAW;oBACjB,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;oBAC5B,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC,MAAM,IAAI,EAAE,CAAC;oBAC9B,SAAS,EAAE,MAAM,CAAC,CAAC,CAAC,SAAS,IAAI,CAAC,CAAC;iBACnB,CAAC;YACrB;gBACE,OAAO,EAAE,IAAI,EAAkC,CAAC;QACpD,CAAC;IACH,CAAC,CAAC,CAAC;AACL,CAAC"}

package/dist/lib/eval-runner.d.ts

@@ -0,0 +1,29 @@
+import type { Config } from './config.js';
+import type { CheckDefinition } from './eval-checks.js';
+import type { EvalCase, EvalSuiteResult } from './eval.js';
+/** Options for running the eval suite. */
+export type EvalRunOptions = {
+    /** Only run this specific case ID (prefix match). */
+    caseId?: string;
+    /** Verbose output. */
+    verbose?: boolean;
+};
+/** Extended eval case with parsed check definitions. */
+export type EvalCaseWithChecks = EvalCase & {
+    checks?: CheckDefinition[];
+    /** For step cases: raw input text. */
+    inputText?: string;
+};
+/**
+ * Run a full eval suite: iterate over cases, execute, collect results.
+ */
+export declare function runEvalSuite(cases: EvalCaseWithChecks[], config: Config, options?: EvalRunOptions): Promise<EvalSuiteResult>;
+/**
+ * Prepare a fixture directory for an e2e eval case.
+ * Clones the repo at the specified ref into a worktree.
+ */
+export declare function prepareFixture(evalCase: EvalCase, projectDir: string): string;
+/**
+ * Snapshot the current harness state (prompts + skills + config) as a hash.
+ */
+export declare function snapshotHarness(config: Config): string;