@honeybee-ai/incubator 1.1.5 → 1.1.6

package/dist/bin.d.ts

@@ -1,2 +1 @@
-#!/usr/bin/env node
 export {};

package/dist/bin.js

@@ -1,4 +1,3 @@
-#!/usr/bin/env node
 import { main } from './index.js';
 main().catch((err) => {
     console.error('[incubator] Fatal error:', err.message);

package/dist/bin.js.map

@@ -1 +1 @@
-{"version":3,"file":"bin.js","sourceRoot":"","sources":["../src/bin.ts"],"names":[],"mappings":";AACA,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAElC,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;IACnB,OAAO,CAAC,KAAK,CAAC,0BAA0B,EAAG,GAAa,CAAC,OAAO,CAAC,CAAC;IAClE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC,CAAC,CAAC"}
+{"version":3,"file":"bin.js","sourceRoot":"","sources":["../src/bin.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAElC,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;IACnB,OAAO,CAAC,KAAK,CAAC,0BAA0B,EAAG,GAAa,CAAC,OAAO,CAAC,CAAC;IAClE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC,CAAC,CAAC"}

package/dist/eval/checks/security.d.ts

@@ -0,0 +1,2 @@
+import type { EvalCheck } from '../types.js';
+export declare const securityCheck: EvalCheck;

package/dist/eval/checks/security.js

@@ -0,0 +1,106 @@
+/**
+ * Security check — Carapace scan on generated source files.
+ */
+import { readdirSync, readFileSync } from 'node:fs';
+import { join } from 'node:path';
+/** Default weight if not configured in task. */
+const DEFAULT_WEIGHT = 2;
+/**
+ * Recursively collect .ts/.js files from a directory.
+ */
+function collectSourceFiles(dir, base) {
+    const result = [];
+    const entries = readdirSync(dir, { withFileTypes: true });
+    for (const entry of entries) {
+        if (entry.name === 'node_modules' || entry.name === '.git' || entry.name === 'dist')
+            continue;
+        const rel = base ? `${base}/${entry.name}` : entry.name;
+        if (entry.isDirectory()) {
+            result.push(...collectSourceFiles(join(dir, entry.name), rel));
+        }
+        else if (/\.(ts|js|tsx|jsx)$/.test(entry.name) && !entry.name.endsWith('.d.ts')) {
+            result.push(rel);
+        }
+    }
+    return result;
+}
+export const securityCheck = {
+    name: 'security',
+    async run(workDir, task) {
+        const start = Date.now();
+        const weight = task.checks?.security?.weight ?? DEFAULT_WEIGHT;
+        const files = collectSourceFiles(workDir);
+        if (files.length === 0) {
+            return {
+                name: 'security',
+                passed: true,
+                score: 1.0,
+                weight,
+                details: 'No source files to scan',
+                duration_ms: Date.now() - start,
+            };
+        }
+        // Use createEdgeScanner() — pre-loads patterns via static require()
+        // which esbuild inlines. The default scan() uses __dirname-relative
+        // fs.readFileSync for patterns, which breaks when bundled.
+        let scanner;
+        try {
+            const carapace = await import('@honeybee-ai/carapace');
+            const create = carapace.createEdgeScanner;
+            if (create) {
+                scanner = create();
+            }
+            else {
+                // Fallback: wrap scan() directly
+                const scanFn = carapace.scan;
+                scanner = { scan: scanFn };
+            }
+        }
+        catch {
+            return {
+                name: 'security',
+                passed: true,
+                score: 1.0,
+                weight,
+                details: 'Carapace not available — skipped',
+                duration_ms: Date.now() - start,
+            };
+        }
+        let warnCount = 0;
+        let blockCount = 0;
+        const findings = [];
+        for (const file of files) {
+            const content = readFileSync(join(workDir, file), 'utf-8');
+            if (!content.trim())
+                continue;
+            const result = scanner.scan(content);
+            if (result.action === 'BLOCK') {
+                blockCount++;
+                findings.push(`BLOCK: ${file} (score: ${result.score})`);
+            }
+            else if (result.action === 'WARN') {
+                warnCount++;
+                findings.push(`WARN: ${file} (score: ${result.score})`);
+            }
+        }
+        // Score: 1.0 if all PASS, -0.1 per WARN, 0 on any BLOCK
+        let score;
+        if (blockCount > 0) {
+            score = 0;
+        }
+        else {
+            score = Math.max(0, 1 - warnCount * 0.1);
+        }
+        const passed = blockCount === 0 && warnCount === 0;
+        return {
+            name: 'security',
+            passed,
+            score,
+            weight,
+            details: `Scanned ${files.length} files: ${blockCount} blocked, ${warnCount} warnings${findings.length > 0 ? '\n' + findings.join('\n') : ''}`,
+            errors: findings.length > 0 ? findings : undefined,
+            duration_ms: Date.now() - start,
+        };
+    },
+};
+//# sourceMappingURL=security.js.map

package/dist/eval/checks/security.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"security.js","sourceRoot":"","sources":["../../../src/eval/checks/security.ts"],"names":[],"mappings":"AAAA;;GAEG;AACH,OAAO,EAAE,WAAW,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AACpD,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAGjC,gDAAgD;AAChD,MAAM,cAAc,GAAG,CAAC,CAAC;AAEzB;;GAEG;AACH,SAAS,kBAAkB,CAAC,GAAW,EAAE,IAAa;IACpD,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,MAAM,OAAO,GAAG,WAAW,CAAC,GAAG,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC;IAE1D,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;QAC5B,IAAI,KAAK,CAAC,IAAI,KAAK,cAAc,IAAI,KAAK,CAAC,IAAI,KAAK,MAAM,IAAI,KAAK,CAAC,IAAI,KAAK,MAAM;YAAE,SAAS;QAC9F,MAAM,GAAG,GAAG,IAAI,CAAC,CAAC,CAAC,GAAG,IAAI,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC;QACxD,IAAI,KAAK,CAAC,WAAW,EAAE,EAAE,CAAC;YACxB,MAAM,CAAC,IAAI,CAAC,GAAG,kBAAkB,CAAC,IAAI,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC;QACjE,CAAC;aAAM,IAAI,oBAAoB,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC;YAClF,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACnB,CAAC;IACH,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,CAAC,MAAM,aAAa,GAAc;IACtC,IAAI,EAAE,UAAU;IAEhB,KAAK,CAAC,GAAG,CAAC,OAAe,EAAE,IAAc;QACvC,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACzB,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,EAAE,QAAQ,EAAE,MAAM,IAAI,cAAc,CAAC;QAE/D,MAAM,KAAK,GAAG,kBAAkB,CAAC,OAAO,CAAC,CAAC;QAC1C,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACvB,OAAO;gBACL,IAAI,EAAE,UAAU;gBAChB,MAAM,EAAE,IAAI;gBACZ,KAAK,EAAE,GAAG;gBACV,MAAM;gBACN,OAAO,EAAE,yBAAyB;gBAClC,WAAW,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK;aAChC,CAAC;QACJ,CAAC;QAED,oEAAoE;QACpE,oEAAoE;QACpE,2DAA2D;QAC3D,IAAI,OAAsE,CAAC;QAC3E,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,uBAAuB,CAAC,CAAC;YACvD,MAAM,MAAM,GAAI,QAAoC,CAAC,iBAC4D,CAAC;YAClH,IAAI,MAAM,EAAE,CAAC;gBACX,OAAO,GAAG,MAAM,EAAE,CAAC;YACrB,CAAC;iBAAM,CAAC;gBACN,iCAAiC;gBACjC,MAAM,MAAM,GAAI,QAAoC,CAAC,IAA2D,CAAC;gBACjH,OAAO,GAAG,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC;YAC7B,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YACP,OAAO;gBACL,IAAI,EAAE,UAAU;gBAChB,MAAM,EAAE,IAAI;gBACZ,KAAK,EAAE,GAAG;gBACV,MAAM;gBACN,OAAO,EAAE,kCAAkC;gBAC3C,WAAW,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK;aAChC,CAAC;QACJ,CAAC;QAED,IAAI,SAAS,GAAG,CAAC,CAAC;QAClB,IAAI,UAAU,GAAG,CAAC,CAAC;QACnB,MAAM,QAAQ,GAAa,EAAE,CAAC;QAE9B,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,MAAM,OAAO,GAAG,YAAY,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,EAAE,OAAO,CAAC,CAAC;YAC3D,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE;gBAAE,SAAS;YAE9B,MAAM,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YACrC,IAAI,MAAM,CAAC,MAAM,KAAK,OAAO,EAAE,CAAC;gBAC9B,UAAU,EAAE,CAAC;gBACb,QAAQ,CAAC,IAAI,CAAC,UAAU,IAAI,YAAY,MAAM,CAAC,KAAK,GAAG,CAAC,CAAC;YAC3D,CAAC;iBAAM,IAAI,MAAM,CAAC,MAAM,KAAK,MAAM,EAAE,CAAC;gBACpC,SAAS,EAAE,CAAC;gBACZ,QAAQ,CAAC,IAAI,CAAC,SAAS,IAAI,YAAY,MAAM,CAAC,KAAK,GAAG,CAAC,CAAC;YAC1D,CAAC;QACH,CAAC;QAED,wDAAwD;QACxD,IAAI,KAAa,CAAC;QAClB,IAAI,UAAU,GAAG,CAAC,EAAE,CAAC;YACnB,KAAK,GAAG,CAAC,CAAC;QACZ,CAAC;aAAM,CAAC;YACN,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,GAAG,SAAS,GAAG,GAAG,CAAC,CAAC;QAC3C,CAAC;QAED,MAAM,MAAM,GAAG,UAAU,KAAK,CAAC,IAAI,SAAS,KAAK,CAAC,CAAC;QAEnD,OAAO;YACL,IAAI,EAAE,UAAU;YAChB,MAAM;YACN,KAAK;YACL,MAAM;YACN,OAAO,EAAE,WAAW,KAAK,CAAC,MAAM,WAAW,UAAU,aAAa,SAAS,YAAY,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,GAAG,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE;YAC9I,MAAM,EAAE,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,SAAS;YAClD,WAAW,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK;SAChC,CAAC;IACJ,CAAC;CACF,CAAC"}

package/dist/eval/checks/structure.d.ts

@@ -0,0 +1,2 @@
+import type { EvalCheck } from '../types.js';
+export declare const structureCheck: EvalCheck;

package/dist/eval/checks/structure.js

@@ -0,0 +1,100 @@
+/**
+ * File structure check — validates expected files exist and are non-empty.
+ */
+import { existsSync, statSync, readdirSync } from 'node:fs';
+import { join } from 'node:path';
+/** Default weight if not configured in task. */
+const DEFAULT_WEIGHT = 1;
+/**
+ * Simple glob match (supports `*` and `**`).
+ * Not a full glob engine — handles the common patterns.
+ */
+function matchGlob(pattern, filePath) {
+    // Exact match
+    if (pattern === filePath)
+        return true;
+    // Convert glob to regex (safe — patterns come from task YAML, not user input)
+    let regexStr = pattern
+        .replace(/[.+^${}()|[\]\\]/g, '\\$&') // escape regex chars (not * or ?)
+        .replace(/\*\*\//g, '{{GLOBSTAR_SEP}}') // **/ = zero or more dirs
+        .replace(/\*\*/g, '{{GLOBSTAR}}') // ** at end
+        .replace(/\*/g, '[^/]*') // * = one segment
+        .replace(/\{\{GLOBSTAR_SEP\}\}/g, '(?:.+/)?') // **/ matches "" or "a/" or "a/b/"
+        .replace(/\{\{GLOBSTAR\}\}/g, '.*');
+    return new RegExp(`^${regexStr}$`).test(filePath);
+}
+/**
+ * Recursively list all files in a directory (relative paths).
+ */
+function listFiles(dir, base) {
+    const result = [];
+    const entries = readdirSync(dir, { withFileTypes: true });
+    for (const entry of entries) {
+        if (entry.name === 'node_modules' || entry.name === '.git')
+            continue;
+        const rel = base ? `${base}/${entry.name}` : entry.name;
+        if (entry.isDirectory()) {
+            result.push(...listFiles(join(dir, entry.name), rel));
+        }
+        else {
+            result.push(rel);
+        }
+    }
+    return result;
+}
+export const structureCheck = {
+    name: 'structure',
+    async run(workDir, task) {
+        const start = Date.now();
+        const weight = task.checks?.structure?.weight ?? DEFAULT_WEIGHT;
+        const patterns = task.expected_files;
+        if (!patterns || patterns.length === 0) {
+            return {
+                name: 'structure',
+                passed: true,
+                score: 1.0,
+                weight,
+                details: 'No expected files configured — skipped',
+                duration_ms: Date.now() - start,
+            };
+        }
+        const allFiles = listFiles(workDir);
+        const results = [];
+        for (const pattern of patterns) {
+            // Check if any file matches this pattern
+            const matches = allFiles.filter(f => matchGlob(pattern, f));
+            // Verify matched files are non-empty
+            const nonEmpty = matches.filter(f => {
+                const fullPath = join(workDir, f);
+                return existsSync(fullPath) && statSync(fullPath).size > 0;
+            });
+            results.push({
+                pattern,
+                matched: nonEmpty.length > 0,
+                files: nonEmpty,
+            });
+        }
+        const matchedCount = results.filter(r => r.matched).length;
+        const score = patterns.length > 0 ? matchedCount / patterns.length : 1.0;
+        const passed = matchedCount === patterns.length;
+        const matchedPatterns = results.filter(r => r.matched).map(r => r.pattern);
+        const missingPatterns = results.filter(r => !r.matched).map(r => r.pattern);
+        const parts = [];
+        if (matchedPatterns.length > 0) {
+            parts.push(`Matched: ${matchedPatterns.join(', ')}`);
+        }
+        if (missingPatterns.length > 0) {
+            parts.push(`Missing: ${missingPatterns.join(', ')}`);
+        }
+        return {
+            name: 'structure',
+            passed,
+            score,
+            weight,
+            details: `${matchedCount}/${patterns.length} expected file patterns found. ${parts.join('. ')}`,
+            errors: missingPatterns.length > 0 ? missingPatterns.map(p => `Missing: ${p}`) : undefined,
+            duration_ms: Date.now() - start,
+        };
+    },
+};
+//# sourceMappingURL=structure.js.map

package/dist/eval/checks/structure.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"structure.js","sourceRoot":"","sources":["../../../src/eval/checks/structure.ts"],"names":[],"mappings":"AAAA;;GAEG;AACH,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,WAAW,EAAE,MAAM,SAAS,CAAC;AAC5D,OAAO,EAAE,IAAI,EAAY,MAAM,WAAW,CAAC;AAG3C,gDAAgD;AAChD,MAAM,cAAc,GAAG,CAAC,CAAC;AAEzB;;;GAGG;AACH,SAAS,SAAS,CAAC,OAAe,EAAE,QAAgB;IAClD,cAAc;IACd,IAAI,OAAO,KAAK,QAAQ;QAAE,OAAO,IAAI,CAAC;IAEtC,8EAA8E;IAC9E,IAAI,QAAQ,GAAG,OAAO;SACnB,OAAO,CAAC,mBAAmB,EAAE,MAAM,CAAC,CAAE,kCAAkC;SACxE,OAAO,CAAC,SAAS,EAAE,kBAAkB,CAAC,CAAC,0BAA0B;SACjE,OAAO,CAAC,OAAO,EAAE,cAAc,CAAC,CAAO,YAAY;SACnD,OAAO,CAAC,KAAK,EAAE,OAAO,CAAC,CAAgB,kBAAkB;SACzD,OAAO,CAAC,uBAAuB,EAAE,UAAU,CAAC,CAAE,mCAAmC;SACjF,OAAO,CAAC,mBAAmB,EAAE,IAAI,CAAC,CAAC;IAEtC,OAAO,IAAI,MAAM,CAAC,IAAI,QAAQ,GAAG,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;AACpD,CAAC;AAED;;GAEG;AACH,SAAS,SAAS,CAAC,GAAW,EAAE,IAAa;IAC3C,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,MAAM,OAAO,GAAG,WAAW,CAAC,GAAG,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC;IAE1D,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;QAC5B,IAAI,KAAK,CAAC,IAAI,KAAK,cAAc,IAAI,KAAK,CAAC,IAAI,KAAK,MAAM;YAAE,SAAS;QACrE,MAAM,GAAG,GAAG,IAAI,CAAC,CAAC,CAAC,GAAG,IAAI,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC;QACxD,IAAI,KAAK,CAAC,WAAW,EAAE,EAAE,CAAC;YACxB,MAAM,CAAC,IAAI,CAAC,GAAG,SAAS,CAAC,IAAI,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC;QACxD,CAAC;aAAM,CAAC;YACN,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACnB,CAAC;IACH,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,CAAC,MAAM,cAAc,GAAc;IACvC,IAAI,EAAE,WAAW;IAEjB,KAAK,CAAC,GAAG,CAAC,OAAe,EAAE,IAAc;QACvC,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACzB,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,EAAE,SAAS,EAAE,MAAM,IAAI,cAAc,CAAC;QAChE,MAAM,QAAQ,GAAG,IAAI,CAAC,cAAc,CAAC;QAErC,IAAI,CAAC,QAAQ,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACvC,OAAO;gBACL,IAAI,EAAE,WAAW;gBACjB,MAAM,EAAE,IAAI;gBACZ,KAAK,EAAE,GAAG;gBACV,MAAM;gBACN,OAAO,EAAE,wCAAwC;gBACjD,WAAW,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK;aAChC,CAAC;QACJ,CAAC;QAED,MAAM,QAAQ,GAAG,SAAS,CAAC,OAAO,CAAC,CAAC;QACpC,MAAM,OAAO,GAAkE,EAAE,CAAC;QAElF,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;YAC/B,yCAAyC;YACzC,MAAM,OAAO,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,SAAS,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC;YAE5D,qCAAqC;YACrC,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE;gBAClC,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;gBAClC,OAAO,UAAU,CAAC,QAAQ,CAAC,IAAI,QAAQ,CAAC,QAAQ,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC;YAC7D,CAAC,CAAC,CAAC;YAEH,OAAO,CAAC,IAAI,CAAC;gBACX,OAAO;gBACP,OAAO,EAAE,QAAQ,CAAC,MAAM,GAAG,CAAC;gBAC5B,KAAK,EAAE,QAAQ;aAChB,CAAC,CAAC;QACL,CAAC;QAED,MAAM,YAAY,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC;QAC3D,MAAM,KAAK,GAAG,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,YAAY,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC;QACzE,MAAM,MAAM,GAAG,YAAY,KAAK,QAAQ,CAAC,MAAM,CAAC;QAEhD,MAAM,eAAe,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;QAC3E,MAAM,eAAe,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;QAE5E,MAAM,KAAK,GAAa,EAAE,CAAC;QAC3B,IAAI,eAAe,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC/B,KAAK,CAAC,IAAI,CAAC,YAAY,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACvD,CAAC;QACD,IAAI,eAAe,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC/B,KAAK,CAAC,IAAI,CAAC,YAAY,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACvD,CAAC;QAED,OAAO;YACL,IAAI,EAAE,WAAW;YACjB,MAAM;YACN,KAAK;YACL,MAAM;YACN,OAAO,EAAE,GAAG,YAAY,IAAI,QAAQ,CAAC,MAAM,kCAAkC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;YAC/F,MAAM,EAAE,eAAe,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,YAAY,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,SAAS;YAC1F,WAAW,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK;SAChC,CAAC;IACJ,CAAC;CACF,CAAC"}

package/dist/eval/checks/test.d.ts

@@ -0,0 +1,2 @@
+import type { EvalCheck } from '../types.js';
+export declare const testCheck: EvalCheck;

package/dist/eval/checks/test.js

@@ -0,0 +1,80 @@
+/**
+ * Test runner check — install deps and run the project's test command.
+ */
+import { execFileSync } from 'node:child_process';
+import { existsSync } from 'node:fs';
+import { join } from 'node:path';
+/** Default weight if not configured in task. */
+const DEFAULT_WEIGHT = 4;
+export const testCheck = {
+    name: 'test',
+    async run(workDir, task) {
+        const start = Date.now();
+        const weight = task.checks?.test?.weight ?? DEFAULT_WEIGHT;
+        // Check if package.json exists
+        if (!existsSync(join(workDir, 'package.json'))) {
+            return {
+                name: 'test',
+                passed: false,
+                score: 0,
+                weight,
+                details: 'No package.json found — cannot run tests',
+                duration_ms: Date.now() - start,
+            };
+        }
+        // Install dependencies
+        try {
+            execFileSync('npm', ['install', '--ignore-scripts'], {
+                cwd: workDir,
+                timeout: 120_000,
+                stdio: ['ignore', 'pipe', 'pipe'],
+            });
+        }
+        catch (err) {
+            const stderr = err?.stderr?.toString()?.slice(0, 500) ?? '';
+            return {
+                name: 'test',
+                passed: false,
+                score: 0,
+                weight,
+                details: `npm install failed: ${stderr}`.slice(0, 2000),
+                errors: ['npm install failed'],
+                duration_ms: Date.now() - start,
+            };
+        }
+        // Run tests
+        const testCommand = task.checks?.test?.command ?? 'npm test';
+        const [cmd, ...cmdArgs] = testCommand.split(' ');
+        try {
+            const result = execFileSync(cmd, cmdArgs, {
+                cwd: workDir,
+                timeout: 120_000,
+                stdio: ['ignore', 'pipe', 'pipe'],
+            });
+            const output = result.toString().trim();
+            return {
+                name: 'test',
+                passed: true,
+                score: 1.0,
+                weight,
+                details: `Tests passed.\n${output}`.slice(0, 2000),
+                duration_ms: Date.now() - start,
+            };
+        }
+        catch (err) {
+            const stderr = err?.stderr?.toString() ?? '';
+            const stdout = err?.stdout?.toString() ?? '';
+            const output = (stderr + '\n' + stdout).trim();
+            return {
+                name: 'test',
+                passed: false,
+                score: 0,
+                weight,
+                details: `Tests failed.\n${output}`.slice(0, 2000),
+                errors: ['Test suite failed'],
+                duration_ms: Date.now() - start,
+            };
+        }
+    },
+};
+//# sourceMappingURL=test.js.map

package/dist/eval/checks/test.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"test.js","sourceRoot":"","sources":["../../../src/eval/checks/test.ts"],"names":[],"mappings":"AAAA;;GAEG;AACH,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AACrC,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAGjC,gDAAgD;AAChD,MAAM,cAAc,GAAG,CAAC,CAAC;AAEzB,MAAM,CAAC,MAAM,SAAS,GAAc;IAClC,IAAI,EAAE,MAAM;IAEZ,KAAK,CAAC,GAAG,CAAC,OAAe,EAAE,IAAc;QACvC,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACzB,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,EAAE,IAAI,EAAE,MAAM,IAAI,cAAc,CAAC;QAE3D,+BAA+B;QAC/B,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,OAAO,EAAE,cAAc,CAAC,CAAC,EAAE,CAAC;YAC/C,OAAO;gBACL,IAAI,EAAE,MAAM;gBACZ,MAAM,EAAE,KAAK;gBACb,KAAK,EAAE,CAAC;gBACR,MAAM;gBACN,OAAO,EAAE,0CAA0C;gBACnD,WAAW,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK;aAChC,CAAC;QACJ,CAAC;QAED,uBAAuB;QACvB,IAAI,CAAC;YACH,YAAY,CAAC,KAAK,EAAE,CAAC,SAAS,EAAE,kBAAkB,CAAC,EAAE;gBACnD,GAAG,EAAE,OAAO;gBACZ,OAAO,EAAE,OAAO;gBAChB,KAAK,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,CAAC;aAClC,CAAC,CAAC;QACL,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,MAAM,GAAI,GAA2B,EAAE,MAAM,EAAE,QAAQ,EAAE,EAAE,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,IAAI,EAAE,CAAC;YACrF,OAAO;gBACL,IAAI,EAAE,MAAM;gBACZ,MAAM,EAAE,KAAK;gBACb,KAAK,EAAE,CAAC;gBACR,MAAM;gBACN,OAAO,EAAE,uBAAuB,MAAM,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC;gBACvD,MAAM,EAAE,CAAC,oBAAoB,CAAC;gBAC9B,WAAW,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK;aAChC,CAAC;QACJ,CAAC;QAED,YAAY;QACZ,MAAM,WAAW,GAAG,IAAI,CAAC,MAAM,EAAE,IAAI,EAAE,OAAO,IAAI,UAAU,CAAC;QAC7D,MAAM,CAAC,GAAG,EAAE,GAAG,OAAO,CAAC,GAAG,WAAW,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAEjD,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,YAAY,CAAC,GAAG,EAAE,OAAO,EAAE;gBACxC,GAAG,EAAE,OAAO;gBACZ,OAAO,EAAE,OAAO;gBAChB,KAAK,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,CAAC;aAClC,CAAC,CAAC;YAEH,MAAM,MAAM,GAAG,MAAM,CAAC,QAAQ,EAAE,CAAC,IAAI,EAAE,CAAC;YAExC,OAAO;gBACL,IAAI,EAAE,MAAM;gBACZ,MAAM,EAAE,IAAI;gBACZ,KAAK,EAAE,GAAG;gBACV,MAAM;gBACN,OAAO,EAAE,kBAAkB,MAAM,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC;gBAClD,WAAW,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK;aAChC,CAAC;QACJ,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,MAAM,GAAI,GAA2B,EAAE,MAAM,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;YACtE,MAAM,MAAM,GAAI,GAA2B,EAAE,MAAM,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;YACtE,MAAM,MAAM,GAAG,CAAC,MAAM,GAAG,IAAI,GAAG,MAAM,CAAC,CAAC,IAAI,EAAE,CAAC;YAE/C,OAAO;gBACL,IAAI,EAAE,MAAM;gBACZ,MAAM,EAAE,KAAK;gBACb,KAAK,EAAE,CAAC;gBACR,MAAM;gBACN,OAAO,EAAE,kBAAkB,MAAM,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC;gBAClD,MAAM,EAAE,CAAC,mBAAmB,CAAC;gBAC7B,WAAW,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK;aAChC,CAAC;QACJ,CAAC;IACH,CAAC;CACF,CAAC"}

package/dist/eval/checks/typecheck.d.ts

@@ -0,0 +1,2 @@
+import type { EvalCheck } from '../types.js';
+export declare const typecheckCheck: EvalCheck;

package/dist/eval/checks/typecheck.js

@@ -0,0 +1,78 @@
+/**
+ * TypeScript compilation check — runs tsc --noEmit.
+ */
+import { execFileSync } from 'node:child_process';
+import { existsSync } from 'node:fs';
+import { join } from 'node:path';
+import { createRequire } from 'node:module';
+const require = createRequire(import.meta.url);
+/** Resolve tsc binary from our own typescript installation. */
+function findTsc() {
+    try {
+        return require.resolve('typescript/bin/tsc');
+    }
+    catch {
+        return 'tsc'; // fallback to PATH
+    }
+}
+/** Default weight if not configured in task. */
+const DEFAULT_WEIGHT = 3;
+export const typecheckCheck = {
+    name: 'typecheck',
+    async run(workDir, task) {
+        const start = Date.now();
+        const weight = task.checks?.typecheck?.weight ?? DEFAULT_WEIGHT;
+        // Skip if no tsconfig.json
+        if (!existsSync(join(workDir, 'tsconfig.json'))) {
+            return {
+                name: 'typecheck',
+                passed: true,
+                score: 1.0,
+                weight,
+                details: 'Skipped: no tsconfig.json found',
+                duration_ms: Date.now() - start,
+            };
+        }
+        try {
+            execFileSync('node', [findTsc(), '--noEmit', '--pretty'], {
+                cwd: workDir,
+                timeout: 120_000,
+                stdio: ['ignore', 'pipe', 'pipe'],
+            });
+            return {
+                name: 'typecheck',
+                passed: true,
+                score: 1.0,
+                weight,
+                details: 'TypeScript compilation passed with no errors',
+                duration_ms: Date.now() - start,
+            };
+        }
+        catch (err) {
+            const stderr = err?.stderr?.toString() ?? '';
+            const stdout = err?.stdout?.toString() ?? '';
+            const output = (stderr + stdout).trim();
+            // Count errors from tsc output
+            const errorMatch = output.match(/Found (\d+) errors?/);
+            const errorCount = errorMatch ? parseInt(errorMatch[1], 10) : 1;
+            // Score degrades with errors: 0 errors = 1.0, each error costs 0.1
+            const score = Math.max(0, 1 - errorCount * 0.1);
+            // Extract error lines (truncate to 2000 chars)
+            const lines = output.split('\n');
+            const errorLines = lines.filter(l => l.includes('error TS')).slice(0, 20);
+            const details = errorLines.length > 0
+                ? `${errorCount} TypeScript error(s):\n${errorLines.join('\n')}`
+                : `TypeScript compilation failed (${errorCount} error(s))`;
+            return {
+                name: 'typecheck',
+                passed: false,
+                score,
+                weight,
+                details: details.slice(0, 2000),
+                errors: errorLines,
+                duration_ms: Date.now() - start,
+            };
+        }
+    },
+};
+//# sourceMappingURL=typecheck.js.map

package/dist/eval/checks/typecheck.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"typecheck.js","sourceRoot":"","sources":["../../../src/eval/checks/typecheck.ts"],"names":[],"mappings":"AAAA;;GAEG;AACH,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AACrC,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAG5C,MAAM,OAAO,GAAG,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AAE/C,+DAA+D;AAC/D,SAAS,OAAO;IACd,IAAI,CAAC;QACH,OAAO,OAAO,CAAC,OAAO,CAAC,oBAAoB,CAAC,CAAC;IAC/C,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC,CAAC,mBAAmB;IACnC,CAAC;AACH,CAAC;AAED,gDAAgD;AAChD,MAAM,cAAc,GAAG,CAAC,CAAC;AAEzB,MAAM,CAAC,MAAM,cAAc,GAAc;IACvC,IAAI,EAAE,WAAW;IAEjB,KAAK,CAAC,GAAG,CAAC,OAAe,EAAE,IAAc;QACvC,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACzB,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,EAAE,SAAS,EAAE,MAAM,IAAI,cAAc,CAAC;QAEhE,2BAA2B;QAC3B,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,OAAO,EAAE,eAAe,CAAC,CAAC,EAAE,CAAC;YAChD,OAAO;gBACL,IAAI,EAAE,WAAW;gBACjB,MAAM,EAAE,IAAI;gBACZ,KAAK,EAAE,GAAG;gBACV,MAAM;gBACN,OAAO,EAAE,iCAAiC;gBAC1C,WAAW,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK;aAChC,CAAC;QACJ,CAAC;QAED,IAAI,CAAC;YACH,YAAY,CAAC,MAAM,EAAE,CAAC,OAAO,EAAE,EAAE,UAAU,EAAE,UAAU,CAAC,EAAE;gBACxD,GAAG,EAAE,OAAO;gBACZ,OAAO,EAAE,OAAO;gBAChB,KAAK,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,CAAC;aAClC,CAAC,CAAC;YAEH,OAAO;gBACL,IAAI,EAAE,WAAW;gBACjB,MAAM,EAAE,IAAI;gBACZ,KAAK,EAAE,GAAG;gBACV,MAAM;gBACN,OAAO,EAAE,8CAA8C;gBACvD,WAAW,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK;aAChC,CAAC;QACJ,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,MAAM,GAAI,GAA2B,EAAE,MAAM,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;YACtE,MAAM,MAAM,GAAI,GAA2B,EAAE,MAAM,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;YACtE,MAAM,MAAM,GAAG,CAAC,MAAM,GAAG,MAAM,CAAC,CAAC,IAAI,EAAE,CAAC;YAExC,+BAA+B;YAC/B,MAAM,UAAU,GAAG,MAAM,CAAC,KAAK,CAAC,qBAAqB,CAAC,CAAC;YACvD,MAAM,UAAU,GAAG,UAAU,CAAC,CAAC,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YAEhE,mEAAmE;YACnE,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,GAAG,UAAU,GAAG,GAAG,CAAC,CAAC;YAEhD,+CAA+C;YAC/C,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;YACjC,MAAM,UAAU,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YAC1E,MAAM,OAAO,GAAG,UAAU,CAAC,MAAM,GAAG,CAAC;gBACnC,CAAC,CAAC,GAAG,UAAU,0BAA0B,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;gBAChE,CAAC,CAAC,kCAAkC,UAAU,YAAY,CAAC;YAE7D,OAAO;gBACL,IAAI,EAAE,WAAW;gBACjB,MAAM,EAAE,KAAK;gBACb,KAAK;gBACL,MAAM;gBACN,OAAO,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC;gBAC/B,MAAM,EAAE,UAAU;gBAClB,WAAW,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK;aAChC,CAAC;QACJ,CAAC;IACH,CAAC;CACF,CAAC"}

package/dist/eval/index.d.ts

@@ -0,0 +1,4 @@
+export { runEval } from './runner.js';
+export { parseEvalTask } from './task.js';
+export { computeScore } from './scorer.js';
+export type { EvalTask, EvalResult, EvalCheckResult, EvalProcessMetrics, EvalRunOptions, } from './types.js';

package/dist/eval/index.js

@@ -0,0 +1,4 @@
+export { runEval } from './runner.js';
+export { parseEvalTask } from './task.js';
+export { computeScore } from './scorer.js';
+//# sourceMappingURL=index.js.map

package/dist/eval/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/eval/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,aAAa,CAAC;AACtC,OAAO,EAAE,aAAa,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC"}

package/dist/eval/runner.d.ts

@@ -0,0 +1,5 @@
+import type { EvalTask, EvalResult, EvalRunOptions } from './types.js';
+/**
+ * Run a full evaluation: spawn agents, wait for completion, run checks, score.
+ */
+export declare function runEval(task: EvalTask, opts?: EvalRunOptions): Promise<EvalResult>;