@pauly4010/evalai-sdk 1.8.0 → 1.9.1

package/dist/cli/env.js

@@ -0,0 +1,42 @@
+"use strict";
+/**
+ * CORE-401: Centralized environment detection
+ *
+ * Provides unified environment detection for all EvalAI CLI commands
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.isCI = isCI;
+exports.isGitHubActions = isGitHubActions;
+exports.getGitHubStepSummaryPath = getGitHubStepSummaryPath;
+exports.isGitRef = isGitRef;
+/**
+ * Check if running in CI environment
+ */
+function isCI() {
+    return !!(process.env.GITHUB_ACTIONS ||
+        process.env.CI ||
+        process.env.CONTINUOUS_INTEGRATION ||
+        process.env.BUILDKITE ||
+        process.env.CIRCLECI ||
+        process.env.TRAVIS ||
+        process.env.JENKINS_URL);
+}
+/**
+ * Check if running in GitHub Actions
+ */
+function isGitHubActions() {
+    return !!process.env.GITHUB_ACTIONS;
+}
+/**
+ * Get GitHub Step Summary path if available
+ */
+function getGitHubStepSummaryPath() {
+    return process.env.GITHUB_STEP_SUMMARY;
+}
+/**
+ * Check if string looks like a git reference
+ */
+function isGitRef(ref) {
+    // Common git ref patterns
+    return /^(main|master|develop|dev|origin\/|remotes\/|feature\/|hotfix\/|release\/|v\d+\.\d+\.\d+|.*\.\.\..*|nonexistent-branch|test-branch|ci-branch)/.test(ref);
+}

package/dist/cli/explain.js

@@ -87,7 +87,9 @@ const REPORT_SEARCH_PATHS = [
 ];
 function findReport(cwd, explicitPath) {
     if (explicitPath) {
-        const abs = path.isAbsolute(explicitPath) ? explicitPath : path.join(cwd, explicitPath);
+        const abs = path.isAbsolute(explicitPath)
+            ? explicitPath
+            : path.join(cwd, explicitPath);
         return fs.existsSync(abs) ? abs : null;
     }
     for (const rel of REPORT_SEARCH_PATHS) {
@@ -115,11 +117,13 @@ function classifyRootCauses(report) {
         causes.push("cost_regression");
     }
     // Latency regression
-    if (reasonCode === "LATENCY_BUDGET_EXCEEDED" || reasonCode === "LATENCY_RISK") {
+    if (reasonCode === "LATENCY_BUDGET_EXCEEDED" ||
+        reasonCode === "LATENCY_RISK") {
         causes.push("latency_regression");
     }
     // Coverage drop (test count decreased)
-    if (reasonCode === "LOW_SAMPLE_SIZE" || reasonCode === "INSUFFICIENT_EVIDENCE") {
+    if (reasonCode === "LOW_SAMPLE_SIZE" ||
+        reasonCode === "INSUFFICIENT_EVIDENCE") {
         causes.push("coverage_drop");
     }
     // Analyze failed cases for drift patterns
@@ -173,52 +177,164 @@ function classifyRootCauses(report) {
 // ── Suggested fixes ──
 const ROOT_CAUSE_FIXES = {
     prompt_drift: [
-        { action: "Review prompt changes", detail: "Compare current prompt with the version used in baseline run. Diff system/user messages.", priority: "high" },
-        { action: "Pin model version", detail: "Use a specific model snapshot (e.g. gpt-4-0613) instead of a rolling alias.", priority: "medium" },
-        { action: "Update baseline", detail: "If changes are intentional, run: npx evalai baseline update", priority: "low" },
+        {
+            action: "Review prompt changes",
+            detail: "Compare current prompt with the version used in baseline run. Diff system/user messages.",
+            priority: "high",
+        },
+        {
+            action: "Pin model version",
+            detail: "Use a specific model snapshot (e.g. gpt-4-0613) instead of a rolling alias.",
+            priority: "medium",
+        },
+        {
+            action: "Update baseline",
+            detail: "If changes are intentional, run: npx evalai baseline update",
+            priority: "low",
+        },
     ],
     retrieval_drift: [
-        { action: "Check retrieval pipeline", detail: "Verify embeddings, index, and chunk strategy haven't changed.", priority: "high" },
-        { action: "Update test case context", detail: "If knowledge base changed, update expected outputs in test cases.", priority: "medium" },
-        { action: "Add retrieval-specific tests", detail: "Add test cases that verify document retrieval before generation.", priority: "low" },
+        {
+            action: "Check retrieval pipeline",
+            detail: "Verify embeddings, index, and chunk strategy haven't changed.",
+            priority: "high",
+        },
+        {
+            action: "Update test case context",
+            detail: "If knowledge base changed, update expected outputs in test cases.",
+            priority: "medium",
+        },
+        {
+            action: "Add retrieval-specific tests",
+            detail: "Add test cases that verify document retrieval before generation.",
+            priority: "low",
+        },
     ],
     formatting_drift: [
-        { action: "Update output format instructions", detail: "Check if system prompt format instructions match expected output structure.", priority: "high" },
-        { action: "Add format validators", detail: "Use schema assertions to validate output structure (JSON schema, regex).", priority: "medium" },
-        { action: "Refresh baseline", detail: "If new format is intentional, run: npx evalai baseline update", priority: "low" },
+        {
+            action: "Update output format instructions",
+            detail: "Check if system prompt format instructions match expected output structure.",
+            priority: "high",
+        },
+        {
+            action: "Add format validators",
+            detail: "Use schema assertions to validate output structure (JSON schema, regex).",
+            priority: "medium",
+        },
+        {
+            action: "Refresh baseline",
+            detail: "If new format is intentional, run: npx evalai baseline update",
+            priority: "low",
+        },
     ],
     tool_use_drift: [
-        { action: "Verify tool definitions", detail: "Check that tool/function schemas match what the model expects.", priority: "high" },
-        { action: "Review tool call patterns", detail: "Compare tool call sequences in failing vs passing cases.", priority: "medium" },
-        { action: "Add tool-use assertions", detail: "Assert specific tool calls are made (or not made) per test case.", priority: "low" },
+        {
+            action: "Verify tool definitions",
+            detail: "Check that tool/function schemas match what the model expects.",
+            priority: "high",
+        },
+        {
+            action: "Review tool call patterns",
+            detail: "Compare tool call sequences in failing vs passing cases.",
+            priority: "medium",
+        },
+        {
+            action: "Add tool-use assertions",
+            detail: "Assert specific tool calls are made (or not made) per test case.",
+            priority: "low",
+        },
     ],
     safety_regression: [
-        { action: "Review safety assertions", detail: "Check which safety test cases are failing and why.", priority: "high" },
-        { action: "Strengthen guardrails", detail: "Add or update content filters, system prompt safety instructions.", priority: "high" },
-        { action: "Update rubric", detail: "If safety criteria changed, update the LLM judge rubric.", priority: "medium" },
+        {
+            action: "Review safety assertions",
+            detail: "Check which safety test cases are failing and why.",
+            priority: "high",
+        },
+        {
+            action: "Strengthen guardrails",
+            detail: "Add or update content filters, system prompt safety instructions.",
+            priority: "high",
+        },
+        {
+            action: "Update rubric",
+            detail: "If safety criteria changed, update the LLM judge rubric.",
+            priority: "medium",
+        },
     ],
     cost_regression: [
-        { action: "Check token usage", detail: "Compare input/output token counts between baseline and current run.", priority: "high" },
-        { action: "Optimize prompts", detail: "Reduce prompt length or use a smaller model for non-critical paths.", priority: "medium" },
-        { action: "Update cost budget", detail: "If higher cost is expected, adjust --max-cost-usd threshold.", priority: "low" },
+        {
+            action: "Check token usage",
+            detail: "Compare input/output token counts between baseline and current run.",
+            priority: "high",
+        },
+        {
+            action: "Optimize prompts",
+            detail: "Reduce prompt length or use a smaller model for non-critical paths.",
+            priority: "medium",
+        },
+        {
+            action: "Update cost budget",
+            detail: "If higher cost is expected, adjust --max-cost-usd threshold.",
+            priority: "low",
+        },
     ],
     latency_regression: [
-        { action: "Check response times", detail: "Compare per-test-case latency between baseline and current run.", priority: "high" },
-        { action: "Reduce prompt complexity", detail: "Simplify prompts or use streaming to reduce perceived latency.", priority: "medium" },
-        { action: "Update latency budget", detail: "If higher latency is expected, adjust --max-latency-ms threshold.", priority: "low" },
+        {
+            action: "Check response times",
+            detail: "Compare per-test-case latency between baseline and current run.",
+            priority: "high",
+        },
+        {
+            action: "Reduce prompt complexity",
+            detail: "Simplify prompts or use streaming to reduce perceived latency.",
+            priority: "medium",
+        },
+        {
+            action: "Update latency budget",
+            detail: "If higher latency is expected, adjust --max-latency-ms threshold.",
+            priority: "low",
+        },
     ],
     coverage_drop: [
-        { action: "Add test cases", detail: "Current test count is below minimum. Add more test cases to the evaluation.", priority: "high" },
-        { action: "Check test case filtering", detail: "Verify no test cases were accidentally deleted or filtered out.", priority: "medium" },
+        {
+            action: "Add test cases",
+            detail: "Current test count is below minimum. Add more test cases to the evaluation.",
+            priority: "high",
+        },
+        {
+            action: "Check test case filtering",
+            detail: "Verify no test cases were accidentally deleted or filtered out.",
+            priority: "medium",
+        },
     ],
     baseline_stale: [
-        { action: "Create baseline", detail: "Run: npx evalai baseline init  (or publish a run from the dashboard)", priority: "high" },
-        { action: "Use --baseline previous", detail: "Compare against the previous run instead of a published baseline.", priority: "medium" },
+        {
+            action: "Create baseline",
+            detail: "Run: npx evalai baseline init  (or publish a run from the dashboard)",
+            priority: "high",
+        },
+        {
+            action: "Use --baseline previous",
+            detail: "Compare against the previous run instead of a published baseline.",
+            priority: "medium",
+        },
     ],
     unknown: [
-        { action: "Run evalai doctor", detail: "Run: npx evalai doctor  to check your full CI/CD setup.", priority: "high" },
-        { action: "Check logs", detail: "Review CI logs for errors or unexpected behavior.", priority: "medium" },
-        { action: "Update baseline", detail: "If changes are intentional, run: npx evalai baseline update", priority: "low" },
+        {
+            action: "Run evalai doctor",
+            detail: "Run: npx evalai doctor  to check your full CI/CD setup.",
+            priority: "high",
+        },
+        {
+            action: "Check logs",
+            detail: "Review CI logs for errors or unexpected behavior.",
+            priority: "medium",
+        },
+        {
+            action: "Update baseline",
+            detail: "If changes are intentional, run: npx evalai baseline update",
+            priority: "low",
+        },
     ],
 };
 function suggestFixes(causes) {
@@ -248,7 +364,9 @@ function buildExplainOutput(report, reportPath) {
 function buildFromCheckReport(report, reportPath) {
     const failedCases = report.failedCases ?? [];
     // Top failures (up to 3)
-    const topFailures = failedCases.slice(0, 3).map((fc, i) => ({
+    const topFailures = failedCases
+        .slice(0, 3)
+        .map((fc, i) => ({
         rank: i + 1,
         name: fc.name,
         input: fc.inputSnippet || fc.input,
@@ -336,7 +454,11 @@ function buildFromBuiltinReport(report, reportPath) {
 }
 // ── Output formatting ──
 function printHuman(output) {
-    const verdictIcon = output.verdict === "pass" ? "\u2705" : output.verdict === "warn" ? "\u26A0\uFE0F" : "\u274C";
+    const verdictIcon = output.verdict === "pass"
+        ? "\u2705"
+        : output.verdict === "warn"
+            ? "\u26A0\uFE0F"
+            : "\u274C";
     console.log(`\n  evalai explain\n`);
     console.log(`  ${verdictIcon} Verdict: ${output.verdict.toUpperCase()}`);
     if (output.score != null) {
@@ -352,7 +474,11 @@ function printHuman(output) {
     if (output.changes.length > 0) {
         console.log("\n  What changed:");
         for (const c of output.changes) {
-            const arrow = c.direction === "worse" ? "\u2193" : c.direction === "better" ? "\u2191" : "\u2192";
+            const arrow = c.direction === "worse"
+                ? "\u2193"
+                : c.direction === "better"
+                    ? "\u2191"
+                    : "\u2192";
             console.log(`    ${arrow} ${c.metric}: ${c.baseline} \u2192 ${c.current}`);
         }
     }
@@ -382,7 +508,11 @@ function printHuman(output) {
     if (output.suggestedFixes.length > 0) {
         console.log("\n  Suggested fixes:");
         for (const fix of output.suggestedFixes) {
-            const pIcon = fix.priority === "high" ? "\u203C\uFE0F" : fix.priority === "medium" ? "\u2757" : "\u2022";
+            const pIcon = fix.priority === "high"
+                ? "\u203C\uFE0F"
+                : fix.priority === "medium"
+                    ? "\u2757"
+                    : "\u2022";
             console.log(`    ${pIcon} ${fix.action}`);
             console.log(`      ${fix.detail}`);
         }
@@ -413,7 +543,9 @@ async function runExplain(argv) {
         return 1;
     }
     // Schema version compatibility check
-    const reportSchema = typeof reportData.schemaVersion === "number" ? reportData.schemaVersion : undefined;
+    const reportSchema = typeof reportData.schemaVersion === "number"
+        ? reportData.schemaVersion
+        : undefined;
     if (reportSchema != null && reportSchema > types_1.CHECK_REPORT_SCHEMA_VERSION) {
         console.error(`\n  \u26A0\uFE0F  Report schema version ${reportSchema} is newer than this CLI supports (v${types_1.CHECK_REPORT_SCHEMA_VERSION}).`);
         console.error("  Update your SDK: npm install @pauly4010/evalai-sdk@latest\n");

package/dist/cli/formatters/human.js

@@ -43,7 +43,10 @@ function formatHuman(report) {
         lines.push("Next: View full report above, fix failing cases, or adjust gate with --minScore / --maxDrop / --warnDrop");
     }
     if (report.explain &&
-        (report.breakdown01 || report.contribPts || report.flags?.length || report.policyEvidence)) {
+        (report.breakdown01 ||
+            report.contribPts ||
+            report.flags?.length ||
+            report.policyEvidence)) {
         lines.push("");
         lines.push("--- Explain ---");
         if (report.contribPts) {

package/dist/cli/formatters/pr-comment.js

@@ -34,7 +34,9 @@ function buildPrComment(report) {
         }
     }
     else {
-        lines.push(passed ? "## ✅ EvalAI Regression Gate — PASSED" : "## 🚨 EvalAI Regression Gate — FAILED");
+        lines.push(passed
+            ? "## ✅ EvalAI Regression Gate — PASSED"
+            : "## 🚨 EvalAI Regression Gate — FAILED");
     }
     lines.push("");
     // Score + Delta (skip when gate not applied)

package/dist/cli/gate.js

@@ -55,7 +55,9 @@ function evaluateGate(args, quality) {
             reasonMessage: `cost $${costUsd.toFixed(4)} exceeds maxCostUsd $${args.maxCostUsd.toFixed(4)}`,
         };
     }
-    if (args.maxLatencyMs != null && avgLatencyMs != null && avgLatencyMs > args.maxLatencyMs) {
+    if (args.maxLatencyMs != null &&
+        avgLatencyMs != null &&
+        avgLatencyMs > args.maxLatencyMs) {
         return {
             exitCode: constants_1.EXIT.SCORE_BELOW,
             passed: false,
@@ -102,7 +104,9 @@ function evaluateGate(args, quality) {
         };
     }
     // warnDrop: soft warning band; maxDrop: hard fail
-    if (args.maxDrop !== undefined && regressionDelta !== null && regressionDelta < -args.maxDrop) {
+    if (args.maxDrop !== undefined &&
+        regressionDelta !== null &&
+        regressionDelta < -args.maxDrop) {
         return {
             exitCode: constants_1.EXIT.REGRESSION,
             passed: false,

package/dist/cli/impact-analysis.d.ts

@@ -0,0 +1,63 @@
+/**
+ * TICKET 3 — Impact Analysis CLI Command (v0)
+ *
+ * Goal: Modal-like perceived speed via incremental intelligence
+ *
+ * Algorithm v0 (practical, shippable):
+ * - Inputs: manifest.json + git diff --name-only base...HEAD
+ * - Rules: Direct file mapping, dependency tracking, safe fallback
+ * - Output: Human-readable counts + JSON for automation
+ */
+import type { EvaluationManifest } from "./manifest";
+/**
+ * Impact analysis result
+ */
+export interface ImpactAnalysisResult {
+    /** Impacted specification IDs */
+    impactedSpecIds: string[];
+    /** Reason for each impacted spec */
+    reasonBySpecId: Record<string, string>;
+    /** Changed files that triggered the analysis */
+    changedFiles: string[];
+    /** Analysis metadata */
+    metadata: {
+        baseBranch: string;
+        totalSpecs: number;
+        impactedCount: number;
+        analysisTime: number;
+    };
+}
+/**
+ * Impact analysis options
+ */
+export interface ImpactAnalysisOptions {
+    /** Base branch to compare against */
+    baseBranch: string;
+    /** Optional explicit list of changed files (for CI) */
+    changedFiles?: string[];
+    /** Output format */
+    format?: "human" | "json";
+}
+/**
+ * Run impact analysis
+ */
+export declare function runImpactAnalysis(options: ImpactAnalysisOptions, projectRoot?: string): Promise<ImpactAnalysisResult>;
+/**
+ * Analyze impact of changed files
+ */
+export declare function analyzeImpact(changedFiles: string[], manifest: EvaluationManifest): {
+    impactedSpecIds: string[];
+    reasonBySpecId: Record<string, string>;
+};
+/**
+ * Print human-readable results
+ */
+export declare function printHumanResults(result: ImpactAnalysisResult): void;
+/**
+ * Print JSON results
+ */
+export declare function printJsonResults(result: ImpactAnalysisResult): void;
+/**
+ * CLI entry point
+ */
+export declare function runImpactAnalysisCLI(options: ImpactAnalysisOptions): Promise<void>;