@pauly4010/evalai-sdk 1.8.0 → 1.9.1

package/dist/cli/discover.d.ts

@@ -0,0 +1,84 @@
+/**
+ * TICKET 1 — evalai discover
+ *
+ * Your first "holy shit" moment feature
+ *
+ * Goal:
+ * npm install
+ * evalai discover
+ *
+ * Output:
+ * Found 42 behavioral specifications
+ * Safety: 12
+ * Accuracy: 18
+ * Agents: 7
+ * Tools: 5
+ *
+ * Why this matters:
+ * - makes EvalAI feel alive
+ * - proves DSL works
+ * - enables intelligence layer
+ *
+ * This becomes your entry point command.
+ */
+/**
+ * Discovered specification statistics
+ */
+export interface DiscoveryStats {
+    /** Total number of specifications found */
+    totalSpecs: number;
+    /** Specifications by category/tag */
+    categories: Record<string, number>;
+    /** Specifications by file */
+    files: Record<string, number>;
+    /** Execution mode information */
+    executionMode: {
+        mode: string;
+        hasSpecRuntime: boolean;
+        hasLegacyRuntime: boolean;
+        specFiles: string[];
+        legacyConfig?: string;
+    };
+    /** Project metadata */
+    project: {
+        root: string;
+        name: string;
+        hasPackageJson: boolean;
+        hasGit: boolean;
+    };
+}
+/**
+ * Specification analysis result
+ */
+export interface SpecAnalysis {
+    /** Specification ID */
+    id: string;
+    /** Specification name */
+    name: string;
+    /** File path */
+    file: string;
+    /** Tags/categories */
+    tags: string[];
+    /** Has assertions */
+    hasAssertions: boolean;
+    /** Uses external models */
+    usesModels: boolean;
+    /** Uses tools */
+    usesTools: boolean;
+    /** Estimated complexity */
+    complexity: "simple" | "medium" | "complex";
+}
+/**
+ * Discover and analyze behavioral specifications in the current project
+ */
+export declare function discoverSpecs(options?: {
+    manifest?: boolean;
+}): Promise<DiscoveryStats>;
+/**
+ * Print discovery results in a beautiful format
+ */
+export declare function printDiscoveryResults(stats: DiscoveryStats): void;
+/**
+ * Run discovery command
+ */
+export declare function runDiscover(): Promise<void>;

package/dist/cli/discover.js

@@ -0,0 +1,419 @@
+"use strict";
+/**
+ * TICKET 1 — evalai discover
+ *
+ * Your first "holy shit" moment feature
+ *
+ * Goal:
+ * npm install
+ * evalai discover
+ *
+ * Output:
+ * Found 42 behavioral specifications
+ * Safety: 12
+ * Accuracy: 18
+ * Agents: 7
+ * Tools: 5
+ *
+ * Why this matters:
+ * - makes EvalAI feel alive
+ * - proves DSL works
+ * - enables intelligence layer
+ *
+ * This becomes your entry point command.
+ */
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.discoverSpecs = discoverSpecs;
+exports.printDiscoveryResults = printDiscoveryResults;
+exports.runDiscover = runDiscover;
+const fs = __importStar(require("node:fs/promises"));
+const path = __importStar(require("node:path"));
+const execution_mode_1 = require("../runtime/execution-mode");
+const manifest_1 = require("./manifest");
+/**
+ * Discover and analyze behavioral specifications in the current project
+ */
+async function discoverSpecs(options = {}) {
+    try {
+        const projectRoot = process.cwd();
+        const executionMode = await (0, execution_mode_1.getExecutionMode)(projectRoot);
+        // Get project metadata
+        const project = await getProjectMetadata(projectRoot);
+        if (executionMode.specFiles.length === 0) {
+            console.log("\n✨ No behavioral specifications found.");
+            console.log("💡 Create files with defineEval() calls to get started.");
+            return {
+                totalSpecs: 0,
+                categories: {},
+                files: {},
+                executionMode: {
+                    mode: executionMode.mode,
+                    hasSpecRuntime: executionMode.hasSpecRuntime,
+                    hasLegacyRuntime: executionMode.hasLegacyRuntime,
+                    specFiles: executionMode.specFiles,
+                    legacyConfig: executionMode.legacyConfig,
+                },
+                project,
+            };
+        }
+        // Analyze specifications
+        const specs = await analyzeSpecifications(executionMode.specFiles);
+        // Generate manifest if requested
+        if (options.manifest) {
+            console.log("🔧 Generating evaluation manifest...");
+            const manifest = await (0, manifest_1.generateManifest)(specs, projectRoot, project.name, executionMode);
+            await (0, manifest_1.writeManifest)(manifest, projectRoot);
+            console.log(`✅ Manifest written to .evalai/manifest.json`);
+            console.log(`✅ Lock file written to .evalai/manifest.lock.json`);
+        }
+        // Calculate statistics
+        const stats = calculateStats(specs, executionMode, project);
+        printDiscoveryResults(stats);
+        return stats;
+    }
+    catch (error) {
+        console.error("❌ Discovery failed:", error instanceof Error ? error.message : String(error));
+        throw error;
+    }
+}
+/**
+ * Get project metadata
+ */
+async function getProjectMetadata(projectRoot) {
+    const packageJsonPath = path.join(projectRoot, "package.json");
+    const gitPath = path.join(projectRoot, ".git");
+    let hasPackageJson = false;
+    let projectName = "unknown";
+    try {
+        const packageJson = await fs.readFile(packageJsonPath, "utf-8");
+        const parsed = JSON.parse(packageJson);
+        hasPackageJson = true;
+        projectName = parsed.name || "unknown";
+    }
+    catch (_error) {
+        // No package.json
+    }
+    const hasGit = await fs
+        .access(gitPath)
+        .then(() => true)
+        .catch(() => false);
+    return {
+        root: projectRoot,
+        name: projectName,
+        hasPackageJson,
+        hasGit,
+    };
+}
+/**
+ * Analyze specification files
+ */
+async function analyzeSpecifications(specFiles) {
+    const specs = [];
+    for (const filePath of specFiles) {
+        try {
+            const content = await fs.readFile(filePath, "utf-8");
+            const analysis = analyzeSpecFile(filePath, content);
+            specs.push(analysis);
+        }
+        catch (error) {
+            console.warn(`Warning: Could not analyze ${filePath}: ${error instanceof Error ? error.message : String(error)}`);
+        }
+    }
+    return specs;
+}
+/**
+ * Analyze a single specification file
+ */
+function analyzeSpecFile(filePath, content) {
+    // Extract defineEval calls
+    const defineEvalMatches = content.match(/defineEval\s*\([^)]+\)/g) || [];
+    const specNames = defineEvalMatches.map((match) => {
+        const nameMatch = match.match(/["'`](.+?)["'`](?:\s*,|\s*\))/);
+        return nameMatch ? nameMatch[1] : "unnamed";
+    });
+    // Extract tags
+    const tags = extractTags(content);
+    // Analyze complexity
+    const complexity = analyzeComplexity(content);
+    // Check for models and tools
+    const usesModels = content.includes("model:") ||
+        content.includes("model=") ||
+        content.includes("openai") ||
+        content.includes("anthropic");
+    const usesTools = content.includes("tool:") ||
+        content.includes("function.") ||
+        content.includes("call(");
+    // Check for assertions
+    const hasAssertions = content.includes("assert") ||
+        content.includes("expect") ||
+        content.includes("should");
+    // Generate ID from file path
+    const id = generateSpecId(filePath);
+    return {
+        id,
+        name: specNames[0] || path.basename(filePath, ".ts"),
+        file: path.relative(process.cwd(), filePath),
+        tags,
+        hasAssertions,
+        usesModels,
+        usesTools,
+        complexity,
+    };
+}
+/**
+ * Extract tags from specification content
+ */
+function extractTags(content) {
+    const tags = [];
+    // Extract tags parameter
+    const tagsMatch = content.match(/tags\s*:\s*\[([^\]]+)\]/);
+    if (tagsMatch) {
+        const tagContent = tagsMatch[1];
+        const tagStrings = tagContent.match(/["'`](.+?)["'`](?:\s*,|\s*)/g) || [];
+        tags.push(...tagStrings.map((tag) => tag.replace(/["'`](.+?)["'`](?:\s*,|\s*)/, "$1")));
+    }
+    // Extract from description and metadata
+    const descriptionMatch = content.match(/description\s*:\s*["'`](.+?)["'`](?:\s*,|\s*)/);
+    if (descriptionMatch) {
+        const description = descriptionMatch[1].toLowerCase();
+        // Auto-categorize based on description
+        if (description.includes("safety") || description.includes("security"))
+            tags.push("safety");
+        if (description.includes("accuracy") || description.includes("precision"))
+            tags.push("accuracy");
+        if (description.includes("agent") || description.includes("autonomous"))
+            tags.push("agents");
+        if (description.includes("tool") || description.includes("function"))
+            tags.push("tools");
+        if (description.includes("latency") || description.includes("speed"))
+            tags.push("performance");
+        if (description.includes("hallucination") || description.includes("fact"))
+            tags.push("factual");
+        if (description.includes("bias") || description.includes("fairness"))
+            tags.push("bias");
+        if (description.includes("privacy") || description.includes("pii"))
+            tags.push("privacy");
+    }
+    return [...new Set(tags)]; // Remove duplicates
+}
+/**
+ * Analyze specification complexity
+ */
+function analyzeComplexity(content) {
+    const lines = content.split("\n").length;
+    const hasAsync = content.includes("async") || content.includes("await");
+    const hasLoops = content.includes("for") || content.includes("while");
+    const hasConditionals = content.includes("if") || content.includes("switch");
+    const hasTryCatch = content.includes("try") || content.includes("catch");
+    const hasExternalCalls = content.includes("fetch") ||
+        content.includes("http") ||
+        content.includes("api");
+    let complexityScore = 0;
+    if (lines > 50)
+        complexityScore += 2;
+    if (lines > 100)
+        complexityScore += 3;
+    if (hasAsync)
+        complexityScore += 2;
+    if (hasLoops)
+        complexityScore += 1;
+    if (hasConditionals)
+        complexityScore += 1;
+    if (hasTryCatch)
+        complexityScore += 1;
+    if (hasExternalCalls)
+        complexityScore += 2;
+    if (complexityScore <= 2)
+        return "simple";
+    if (complexityScore <= 5)
+        return "medium";
+    return "complex";
+}
+/**
+ * Generate specification ID from file path
+ */
+function generateSpecId(filePath) {
+    const relativePath = path.relative(process.cwd(), filePath);
+    const hash = Buffer.from(relativePath)
+        .toString("base64")
+        .replace(/[+/=]/g, "")
+        .slice(0, 8);
+    return hash;
+}
+/**
+ * Calculate discovery statistics
+ */
+function calculateStats(specs, executionMode, project) {
+    const categories = {};
+    const files = {};
+    // Count by categories
+    for (const spec of specs) {
+        for (const tag of spec.tags) {
+            categories[tag] = (categories[tag] || 0) + 1;
+        }
+        // Count by files
+        files[spec.file] = (files[spec.file] || 0) + 1;
+    }
+    // Add default categories if none found
+    if (Object.keys(categories).length === 0) {
+        categories.general = specs.length;
+    }
+    return {
+        totalSpecs: specs.length,
+        categories,
+        files,
+        executionMode: {
+            mode: executionMode.mode,
+            hasSpecRuntime: executionMode.hasSpecRuntime,
+            hasLegacyRuntime: executionMode.hasLegacyRuntime,
+            specFiles: executionMode.specFiles,
+            legacyConfig: executionMode.legacyConfig,
+        },
+        project,
+    };
+}
+/**
+ * Print discovery results in a beautiful format
+ */
+function printDiscoveryResults(stats) {
+    console.log(`🔍 EvalAI Discovery Results`);
+    console.log(``);
+    console.log(`📊 Found ${stats.totalSpecs} behavioral specifications`);
+    console.log(``);
+    // Print categories
+    if (Object.keys(stats.categories).length > 0) {
+        console.log(`📋 Categories:`);
+        const sortedCategories = Object.entries(stats.categories)
+            .sort(([, a], [, b]) => b - a)
+            .slice(0, 10); // Top 10 categories
+        for (const [category, count] of sortedCategories) {
+            const icon = getCategoryIcon(category);
+            console.log(`   ${icon} ${category}: ${count}`);
+        }
+        console.log(``);
+    }
+    // Print execution mode
+    console.log(`⚙️  Execution Mode: ${stats.executionMode.mode.toUpperCase()}`);
+    if (stats.executionMode.hasSpecRuntime) {
+        console.log(`   ✅ Spec runtime: ${stats.executionMode.specFiles.length} files`);
+    }
+    if (stats.executionMode.hasLegacyRuntime) {
+        console.log(`   ✅ Legacy runtime: ${stats.executionMode.legacyConfig ? path.basename(stats.executionMode.legacyConfig) : "config"}`);
+    }
+    console.log(``);
+    // Print project info
+    console.log(`📁 Project: ${stats.project.name}`);
+    console.log(`   📍 Root: ${stats.project.root}`);
+    console.log(`   📦 Package.json: ${stats.project.hasPackageJson ? "✅" : "❌"}`);
+    console.log(`   🔄 Git: ${stats.project.hasGit ? "✅" : "❌"}`);
+    console.log(``);
+    // Print recommendations
+    printRecommendations(stats);
+}
+/**
+ * Get icon for category
+ */
+function getCategoryIcon(category) {
+    const icons = {
+        safety: "🛡️",
+        security: "🔒",
+        accuracy: "🎯",
+        precision: "🎯",
+        agents: "🤖",
+        autonomous: "🤖",
+        tools: "🔧",
+        functions: "🔧",
+        performance: "⚡",
+        latency: "⚡",
+        speed: "⚡",
+        factual: "📊",
+        hallucination: "📊",
+        bias: "⚖️",
+        fairness: "⚖️",
+        privacy: "🔐",
+        pii: "🔐",
+        general: "📝",
+    };
+    return icons[category.toLowerCase()] || "📝";
+}
+/**
+ * Print recommendations based on discovery results
+ */
+function printRecommendations(stats) {
+    console.log(`💡 Recommendations:`);
+    if (stats.totalSpecs === 0) {
+        console.log(`   🚀 No specifications found. Create your first eval with:
+   echo 'import { defineEval } from "@pauly4010/evalai-sdk";
+   defineEval("hello-world", async (context) => {
+     return { pass: true, score: 100 };
+   });' > eval/hello.spec.ts`);
+    }
+    else if (stats.totalSpecs < 5) {
+        console.log(`   📈 Add more specifications to improve coverage`);
+    }
+    else if (stats.totalSpecs < 20) {
+        console.log(`   🎯 Good start! Consider organizing by categories`);
+    }
+    else {
+        console.log(`   🏆 Excellent coverage! Consider running evalai run`);
+    }
+    if (!stats.executionMode.hasSpecRuntime &&
+        !stats.executionMode.hasLegacyRuntime) {
+        console.log(`   🆕 New project? Try 'evalai init' to get started`);
+    }
+    if (stats.executionMode.hasLegacyRuntime &&
+        !stats.executionMode.hasSpecRuntime) {
+        console.log(`   🔄 Legacy project detected. Try 'evalai migrate config' to upgrade`);
+    }
+    if (stats.executionMode.hasSpecRuntime) {
+        console.log(`   🚀 Ready to run! Use 'evalai run' to execute specifications`);
+    }
+    console.log(``);
+}
+/**
+ * Run discovery command
+ */
+async function runDiscover() {
+    try {
+        const stats = await discoverSpecs();
+        printDiscoveryResults(stats);
+        process.exit(0);
+    }
+    catch (error) {
+        console.error(`❌ Discovery failed: ${error instanceof Error ? error.message : String(error)}`);
+        process.exit(1);
+    }
+}

package/dist/cli/doctor.js

@@ -96,8 +96,10 @@ function parseFlags(argv) {
     const baseUrl = raw.baseUrl || process.env.EVALAI_BASE_URL || "http://localhost:3000";
     const apiKey = raw.apiKey || process.env.EVALAI_API_KEY || "";
     let evaluationId = raw.evaluationId || "";
-    const baseline = (raw.baseline === "previous" ? "previous"
-        : raw.baseline === "production" ? "production"
+    const baseline = (raw.baseline === "previous"
+        ? "previous"
+        : raw.baseline === "production"
+            ? "production"
             : "published");
     // Try to fill evaluationId from config
     if (!evaluationId) {
@@ -111,7 +113,15 @@ function parseFlags(argv) {
             evaluationId = String(merged.evaluationId);
     }
     const strict = raw.strict === "true" || raw.strict === "1";
-    return { report, format: report ? "json" : fmt, strict, baseUrl, apiKey, evaluationId, baseline };
+    return {
+        report,
+        format: report ? "json" : fmt,
+        strict,
+        baseUrl,
+        apiKey,
+        evaluationId,
+        baseline,
+    };
 }
 // ── Individual checks ──
 function checkProject(cwd) {
@@ -222,7 +232,10 @@ function checkBaseline(cwd) {
         };
     }
     const schemaVersion = typeof data.schemaVersion === "number" ? data.schemaVersion : undefined;
-    const hash = (0, node_crypto_1.createHash)("sha256").update(JSON.stringify(data)).digest("hex").slice(0, 12);
+    const hash = (0, node_crypto_1.createHash)("sha256")
+        .update(JSON.stringify(data))
+        .digest("hex")
+        .slice(0, 12);
     const updatedAt = typeof data.updatedAt === "string" ? data.updatedAt : undefined;
     // Staleness: warn if baseline older than 30 days
     let stale = false;
@@ -237,7 +250,12 @@ function checkBaseline(cwd) {
             status: "fail",
             message: `Unsupported baseline schemaVersion: ${schemaVersion ?? "missing"}`,
             remediation: "Run: npx evalai baseline init  (creates schemaVersion 1)",
-            baselineInfo: { path: "evals/baseline.json", exists: true, hash, schemaVersion },
+            baselineInfo: {
+                path: "evals/baseline.json",
+                exists: true,
+                hash,
+                schemaVersion,
+            },
         };
     }
     if (stale) {
@@ -247,7 +265,13 @@ function checkBaseline(cwd) {
             status: "warn",
             message: `Baseline is stale (last updated ${updatedAt})`,
             remediation: "Run: npx evalai baseline update",
-            baselineInfo: { path: "evals/baseline.json", exists: true, hash, schemaVersion, stale },
+            baselineInfo: {
+                path: "evals/baseline.json",
+                exists: true,
+                hash,
+                schemaVersion,
+                stale,
+            },
         };
     }
     return {
@@ -255,7 +279,13 @@ function checkBaseline(cwd) {
         label: "Baseline file",
         status: "pass",
         message: `schemaVersion ${schemaVersion}, hash ${hash}`,
-        baselineInfo: { path: "evals/baseline.json", exists: true, hash, schemaVersion, stale },
+        baselineInfo: {
+            path: "evals/baseline.json",
+            exists: true,
+            hash,
+            schemaVersion,
+            stale,
+        },
     };
 }
 function checkAuth(apiKey) {
@@ -269,9 +299,7 @@ function checkAuth(apiKey) {
         };
     }
     // Redact key for display
-    const redacted = apiKey.length > 8
-        ? `${apiKey.slice(0, 4)}...${apiKey.slice(-4)}`
-        : "****";
+    const redacted = apiKey.length > 8 ? `${apiKey.slice(0, 4)}...${apiKey.slice(-4)}` : "****";
     return {
         id: "auth",
         label: "Authentication",
@@ -437,7 +465,8 @@ function checkCiWiring(cwd) {
             ciInfo: { workflowPath, exists: true },
         };
     }
-    if (!content.includes("evalai") && !content.includes("@pauly4010/evalai-sdk")) {
+    if (!content.includes("evalai") &&
+        !content.includes("@pauly4010/evalai-sdk")) {
         return {
             id: "ci_wiring",
             label: "CI wiring",
@@ -480,10 +509,14 @@ function checkProviderEnv() {
 // ── Output formatting ──
 function icon(status) {
     switch (status) {
-        case "pass": return "\u2705"; // ✅
-        case "fail": return "\u274C"; // ❌
-        case "warn": return "\u26A0\uFE0F"; // ⚠️
-        case "skip": return "\u23ED\uFE0F"; // ⏭️
+        case "pass":
+            return "\u2705"; // ✅
+        case "fail":
+            return "\u274C"; // ❌
+        case "warn":
+            return "\u26A0\uFE0F"; // ⚠️
+        case "skip":
+            return "\u23ED\uFE0F"; // ⏭️
     }
 }
 function printHuman(checks, overall) {
@@ -539,10 +572,17 @@ async function runDoctor(argv) {
             message: "Infrastructure error during connectivity check",
         });
         infraError = true;
-        connectivityResult = { id: "connectivity", label: "API connectivity", status: "fail", message: "" };
+        connectivityResult = {
+            id: "connectivity",
+            label: "API connectivity",
+            status: "fail",
+            message: "",
+        };
     }
     // 7. Eval access (async, depends on auth + connectivity)
-    if (flags.apiKey && flags.evaluationId && connectivityResult.status !== "fail") {
+    if (flags.apiKey &&
+        flags.evaluationId &&
+        connectivityResult.status !== "fail") {
         try {
             const accessResult = await checkEvalAccess(flags.baseUrl, flags.apiKey, flags.evaluationId, flags.baseline);
             checks.push(accessResult);
@@ -583,7 +623,9 @@ async function runDoctor(argv) {
     if (flags.report || flags.format === "json") {
         const redactedConfig = {
             ...(configResult.config ?? {}),
-            path: configResult.configPath ? path.relative(cwd, configResult.configPath) : null,
+            path: configResult.configPath
+                ? path.relative(cwd, configResult.configPath)
+                : null,
         };
         const bundle = {
             timestamp: new Date().toISOString(),
@@ -595,7 +637,8 @@ async function runDoctor(argv) {
             config: redactedConfig,
             baseline: baselineResult.baselineInfo,
             api: {
-                reachable: connectivityResult.status === "pass" || connectivityResult.status === "warn",
+                reachable: connectivityResult.status === "pass" ||
+                    connectivityResult.status === "warn",
                 latencyMs: connectivityResult.latencyMs,
             },
             ci: ciResult.ciInfo,

package/dist/cli/env.d.ts

@@ -0,0 +1,21 @@
+/**
+ * CORE-401: Centralized environment detection
+ *
+ * Provides unified environment detection for all EvalAI CLI commands
+ */
+/**
+ * Check if running in CI environment
+ */
+export declare function isCI(): boolean;
+/**
+ * Check if running in GitHub Actions
+ */
+export declare function isGitHubActions(): boolean;
+/**
+ * Get GitHub Step Summary path if available
+ */
+export declare function getGitHubStepSummaryPath(): string | undefined;
+/**
+ * Check if string looks like a git reference
+ */
+export declare function isGitRef(ref: string): boolean;