@pauly4010/evalai-sdk 1.4.0 → 1.5.0

package/dist/cli/check.js

@@ -15,7 +15,7 @@
  *   --minN <n>           Fail if total test cases < n (low sample size)
  *   --allowWeakEvidence  If false (default), fail when evidenceLevel is 'weak'
  *   --policy <name>      Enforce a compliance policy (e.g. HIPAA, SOC2, GDPR)
- *   --baseline <mode>    Baseline comparison mode: "published" (default) or "previous"
+ *   --baseline <mode>   Baseline comparison mode: "published" (default), "previous", or "production"
  *   --evaluationId <id>  Required. The evaluation to gate on.
  *   --baseUrl <url>      API base URL (default: EVALAI_BASE_URL or http://localhost:3000)
  *   --apiKey <key>       API key (default: EVALAI_API_KEY env var)
@@ -38,17 +38,17 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.EXIT = void 0;
 exports.parseArgs = parseArgs;
 exports.runCheck = runCheck;
-// Standardized exit codes
-exports.EXIT = {
-    PASS: 0,
-    SCORE_BELOW: 1,
-    REGRESSION: 2,
-    POLICY_VIOLATION: 3,
-    API_ERROR: 4,
-    BAD_ARGS: 5,
-    LOW_N: 6,
-    WEAK_EVIDENCE: 7,
-};
+const config_1 = require("./config");
+const api_1 = require("./api");
+const ci_context_1 = require("./ci-context");
+const gate_1 = require("./gate");
+const build_check_report_1 = require("./report/build-check-report");
+const human_1 = require("./formatters/human");
+const json_1 = require("./formatters/json");
+const github_1 = require("./formatters/github");
+const constants_1 = require("./constants");
+var constants_2 = require("./constants");
+Object.defineProperty(exports, "EXIT", { enumerable: true, get: function () { return constants_2.EXIT; } });
 function parseArgs(argv) {
     const args = {};
     for (let i = 0; i < argv.length; i++) {
@@ -61,144 +61,134 @@ function parseArgs(argv) {
                 i++;
             }
             else {
-                args[key] = 'true'; // bare flag
+                args[key] = 'true';
             }
         }
     }
-    const baseUrl = args.baseUrl || process.env.EVALAI_BASE_URL || 'http://localhost:3000';
+    let baseUrl = args.baseUrl || process.env.EVALAI_BASE_URL || 'http://localhost:3000';
     const apiKey = args.apiKey || process.env.EVALAI_API_KEY || '';
-    const minScore = parseInt(args.minScore || '0');
+    let minScore = parseInt(args.minScore || '0');
     const maxDrop = args.maxDrop ? parseInt(args.maxDrop) : undefined;
-    const minN = args.minN ? parseInt(args.minN) : undefined;
-    const allowWeakEvidence = args.allowWeakEvidence === 'true' || args.allowWeakEvidence === '1';
-    const evaluationId = args.evaluationId || '';
+    let minN = args.minN ? parseInt(args.minN) : undefined;
+    let allowWeakEvidence = args.allowWeakEvidence === 'true' || args.allowWeakEvidence === '1';
+    let evaluationId = args.evaluationId || '';
     const policy = args.policy || undefined;
-    const baseline = (args.baseline === 'previous' ? 'previous' : 'published');
+    const formatRaw = args.format || 'human';
+    const format = formatRaw === 'json' ? 'json' : formatRaw === 'github' ? 'github' : 'human';
+    const explain = args.explain === 'true' || args.explain === '1';
+    const onFail = args.onFail === 'import' ? 'import' : undefined;
+    let baseline = (args.baseline === 'previous'
+        ? 'previous'
+        : args.baseline === 'production'
+            ? 'production'
+            : 'published');
+    if (!evaluationId) {
+        const config = (0, config_1.loadConfig)(process.cwd());
+        const merged = (0, config_1.mergeConfigWithArgs)(config, {
+            evaluationId: args.evaluationId,
+            baseUrl: args.baseUrl || process.env.EVALAI_BASE_URL,
+            minScore: args.minScore,
+            minN: args.minN,
+            allowWeakEvidence: args.allowWeakEvidence,
+            baseline: args.baseline,
+        });
+        if (merged.evaluationId)
+            evaluationId = merged.evaluationId;
+        if (merged.baseUrl)
+            baseUrl = merged.baseUrl;
+        if (merged.minScore != null && !args.minScore)
+            minScore = merged.minScore ?? 0;
+        if (merged.minN != null && !args.minN)
+            minN = merged.minN;
+        if (merged.allowWeakEvidence != null && !args.allowWeakEvidence)
+            allowWeakEvidence = merged.allowWeakEvidence ?? false;
+        if (merged.baseline && !args.baseline)
+            baseline = merged.baseline;
+    }
     if (!apiKey) {
-        console.error('Error: --apiKey or EVALAI_API_KEY is required');
-        process.exit(exports.EXIT.BAD_ARGS);
+        return { ok: false, exitCode: constants_1.EXIT.BAD_ARGS, message: 'Error: --apiKey or EVALAI_API_KEY is required' };
     }
     if (!evaluationId) {
-        console.error('Error: --evaluationId is required');
-        process.exit(exports.EXIT.BAD_ARGS);
+        return { ok: false, exitCode: constants_1.EXIT.BAD_ARGS, message: 'Run npx evalai init and paste your evaluationId, or pass --evaluationId.' };
     }
     if (isNaN(minScore) || minScore < 0 || minScore > 100) {
-        console.error('Error: --minScore must be 0-100');
-        process.exit(exports.EXIT.BAD_ARGS);
+        return { ok: false, exitCode: constants_1.EXIT.BAD_ARGS, message: 'Error: --minScore must be 0-100' };
     }
     if (minN !== undefined && (isNaN(minN) || minN < 1)) {
-        console.error('Error: --minN must be a positive number');
-        process.exit(exports.EXIT.BAD_ARGS);
+        return { ok: false, exitCode: constants_1.EXIT.BAD_ARGS, message: 'Error: --minN must be a positive number' };
     }
-    return { baseUrl, apiKey, minScore, maxDrop, minN, allowWeakEvidence, evaluationId, policy, baseline };
+    return {
+        ok: true,
+        args: { baseUrl, apiKey, minScore, maxDrop, minN, allowWeakEvidence, evaluationId, policy, baseline, format, explain, onFail },
+    };
 }
 async function runCheck(args) {
-    const headers = { Authorization: `Bearer ${args.apiKey}` };
-    // ── 1. Fetch latest quality score ──
-    const scoreUrl = `${args.baseUrl}/api/quality?evaluationId=${args.evaluationId}&action=latest`;
-    let scoreRes;
-    try {
-        scoreRes = await fetch(scoreUrl, { headers });
-    }
-    catch (err) {
-        console.error(`EvalAI gate ERROR: Network failure — ${err.message}`);
-        return exports.EXIT.API_ERROR;
-    }
-    if (!scoreRes.ok) {
-        const body = await scoreRes.text();
-        console.error(`EvalAI gate ERROR: API returned ${scoreRes.status} — ${body}`);
-        return exports.EXIT.API_ERROR;
-    }
-    const data = (await scoreRes.json());
-    const score = data?.score ?? 0;
-    const total = data?.total ?? null;
-    const evidenceLevel = data?.evidenceLevel ?? null;
-    const baselineScore = data?.baselineScore ?? null;
-    const regressionDelta = data?.regressionDelta ?? null;
-    const breakdown = data?.breakdown ?? {};
-    // ── Gate: minN (low sample size) ──
-    if (args.minN !== undefined && total !== null && total < args.minN) {
-        console.error(`\n✗ FAILED: total test cases (${total}) < minN (${args.minN})`);
-        return exports.EXIT.LOW_N;
-    }
-    // ── Gate: allowWeakEvidence ──
-    if (!args.allowWeakEvidence && evidenceLevel === 'weak') {
-        console.error(`\n✗ FAILED: evidence level is 'weak' (use --allowWeakEvidence to permit)`);
-        return exports.EXIT.WEAK_EVIDENCE;
-    }
-    // ── Print summary ──
-    console.log('┌─────────────────────────────────────────┐');
-    console.log(`│  EvalAI Quality Score: ${String(score).padStart(3)}/100            │`);
-    console.log('├─────────────────────────────────────────┤');
-    if (baselineScore !== null) {
-        const delta = regressionDelta ?? 0;
-        const arrow = delta >= 0 ? '▲' : '▼';
-        console.log(`│  Baseline: ${baselineScore}  ${arrow} ${Math.abs(delta)} pts          │`);
-    }
-    if (breakdown) {
-        const pct = (v) => `${Math.round((v ?? 0) * 100)}%`;
-        console.log(`│  Pass: ${pct(breakdown.passRate)}  Safety: ${pct(breakdown.safety)}  Judge: ${pct(breakdown.judge)} │`);
-    }
-    if (data?.flags && data.flags.length > 0) {
-        console.log(`│  Flags: ${data.flags.join(', ').padEnd(30)} │`);
-    }
-    console.log('└─────────────────────────────────────────┘');
-    // ── 2. Gate: minimum score ──
-    if (args.minScore > 0 && score < args.minScore) {
-        console.error(`\n✗ FAILED: score=${score} < minScore=${args.minScore}`);
-        return exports.EXIT.SCORE_BELOW;
-    }
-    // ── 3. Gate: maximum drop from baseline ──
-    if (args.maxDrop !== undefined && regressionDelta !== null && regressionDelta < -(args.maxDrop)) {
-        console.error(`\n✗ FAILED: score dropped ${Math.abs(regressionDelta)} pts from baseline ` +
-            `(max allowed: ${args.maxDrop})`);
-        return exports.EXIT.REGRESSION;
-    }
-    // ── 4. Gate: policy compliance ──
-    if (args.policy) {
-        const policyUrl = `${args.baseUrl}/api/quality?evaluationId=${args.evaluationId}&action=latest`;
-        // Check policy-specific flags
-        const policyFlags = (data?.flags ?? []);
-        // Policy mapping: each policy has a set of required conditions
-        const policyChecks = {
-            HIPAA: { requiredSafetyRate: 0.99, maxFlags: ['SAFETY_RISK'] },
-            SOC2: { requiredSafetyRate: 0.95, maxFlags: ['SAFETY_RISK', 'LOW_PASS_RATE'] },
-            GDPR: { requiredSafetyRate: 0.95, maxFlags: ['SAFETY_RISK'] },
-            PCI_DSS: { requiredSafetyRate: 0.99, maxFlags: ['SAFETY_RISK', 'LOW_PASS_RATE'] },
-            FINRA_4511: { requiredSafetyRate: 0.95, maxFlags: ['SAFETY_RISK'] },
-        };
-        const policyName = args.policy.toUpperCase();
-        const check = policyChecks[policyName];
-        if (!check) {
-            console.error(`\n✗ Unknown policy: ${args.policy}. Available: ${Object.keys(policyChecks).join(', ')}`);
-            return exports.EXIT.BAD_ARGS;
+    const qualityResult = await (0, api_1.fetchQualityLatest)(args.baseUrl, args.apiKey, args.evaluationId, args.baseline);
+    if (!qualityResult.ok) {
+        if (qualityResult.status === 0) {
+            console.error(`EvalAI gate ERROR: Network failure — ${qualityResult.body}`);
         }
-        // Check safety rate
-        const safetyRate = breakdown?.safety ?? 0;
-        if (safetyRate < check.requiredSafetyRate) {
-            console.error(`\n✗ POLICY VIOLATION (${policyName}): safety rate ${Math.round(safetyRate * 100)}% < ` +
-                `required ${Math.round(check.requiredSafetyRate * 100)}%`);
-            return exports.EXIT.POLICY_VIOLATION;
+        else {
+            console.error(`EvalAI gate ERROR: API returned ${qualityResult.status} — ${qualityResult.body}`);
         }
-        // Check for disqualifying flags
-        const violations = policyFlags.filter(f => check.maxFlags.includes(f));
-        if (violations.length > 0) {
-            console.error(`\n✗ POLICY VIOLATION (${policyName}): ${violations.join(', ')}`);
-            return exports.EXIT.POLICY_VIOLATION;
+        return constants_1.EXIT.API_ERROR;
+    }
+    const { data: quality, requestId } = qualityResult;
+    const evaluationRunId = quality?.evaluationRunId;
+    let runDetails = null;
+    if (evaluationRunId != null) {
+        const runRes = await (0, api_1.fetchRunDetails)(args.baseUrl, args.apiKey, args.evaluationId, evaluationRunId);
+        if (runRes.ok)
+            runDetails = runRes.data;
+    }
+    const gateResult = (0, gate_1.evaluateGate)(args, quality);
+    const report = (0, build_check_report_1.buildCheckReport)({
+        args,
+        quality,
+        runDetails,
+        gateResult,
+        requestId,
+    });
+    const formatted = args.format === 'json'
+        ? (0, json_1.formatJson)(report)
+        : args.format === 'github'
+            ? (0, github_1.formatGitHub)(report)
+            : (0, human_1.formatHuman)(report);
+    console.log(formatted);
+    // --onFail import: when gate fails, import run with CI context
+    if (!gateResult.passed && args.onFail === 'import' && runDetails?.results && quality?.evaluationRunId) {
+        const importResults = runDetails.results
+            .filter((r) => r.testCaseId != null && (r.status === 'passed' || r.status === 'failed'))
+            .map((r) => ({
+            testCaseId: r.testCaseId,
+            status: r.status,
+            output: r.output ?? '',
+            latencyMs: r.durationMs,
+            assertionsJson: r.assertionsJson,
+        }));
+        if (importResults.length > 0) {
+            const ci = (0, ci_context_1.captureCiContext)();
+            const idempotencyKey = ci ? (0, ci_context_1.computeIdempotencyKey)(args.evaluationId, ci) : undefined;
+            const importRes = await (0, api_1.importRunOnFail)(args.baseUrl, args.apiKey, args.evaluationId, importResults, { idempotencyKey, ci, importClientVersion: 'evalai-cli' });
+            if (!importRes.ok) {
+                console.error(`EvalAI import (onFail): ${importRes.status} — ${importRes.body}`);
+            }
         }
-        console.log(`\n✓ Policy ${policyName}: COMPLIANT`);
     }
-    console.log('\n✓ EvalAI gate PASSED');
-    return exports.EXIT.PASS;
+    return gateResult.exitCode;
 }
 // Main entry point
 const isDirectRun = typeof require !== 'undefined' && require.main === module;
 if (isDirectRun) {
-    const args = parseArgs(process.argv.slice(2));
-    runCheck(args).then((code) => {
-        process.exit(code);
-    }).catch((err) => {
-        console.error(`EvalAI gate ERROR: ${err.message}`);
-        process.exit(exports.EXIT.API_ERROR);
+    const parsed = parseArgs(process.argv.slice(2));
+    if (!parsed.ok) {
+        console.error(parsed.message);
+        process.exit(parsed.exitCode);
+    }
+    runCheck(parsed.args)
+        .then((code) => process.exit(code))
+        .catch((err) => {
+        console.error(`EvalAI gate ERROR: ${err instanceof Error ? err.message : String(err)}`);
+        process.exit(constants_1.EXIT.API_ERROR);
     });
 }

package/dist/cli/ci-context.d.ts

@@ -0,0 +1,6 @@
+/**
+ * CI context capture and idempotency key for --onFail import.
+ */
+import type { CiContext } from './api';
+export declare function captureCiContext(): CiContext | undefined;
+export declare function computeIdempotencyKey(evaluationId: string, ci: CiContext): string | undefined;

package/dist/cli/ci-context.js

@@ -0,0 +1,51 @@
+"use strict";
+/**
+ * CI context capture and idempotency key for --onFail import.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.captureCiContext = captureCiContext;
+exports.computeIdempotencyKey = computeIdempotencyKey;
+const crypto_1 = require("crypto");
+function captureCiContext() {
+    const repo = process.env.GITHUB_REPOSITORY;
+    const sha = process.env.GITHUB_SHA;
+    const ref = process.env.GITHUB_REF;
+    const runId = process.env.GITHUB_RUN_ID;
+    const workflow = process.env.GITHUB_WORKFLOW;
+    const job = process.env.GITHUB_JOB;
+    const actor = process.env.GITHUB_ACTOR;
+    if (!repo && !sha)
+        return undefined;
+    let provider = 'unknown';
+    if (process.env.GITHUB_ACTIONS)
+        provider = 'github';
+    else if (process.env.GITLAB_CI)
+        provider = 'gitlab';
+    else if (process.env.CIRCLECI)
+        provider = 'circle';
+    let runUrl;
+    if (repo && runId) {
+        runUrl = `https://github.com/${repo}/actions/runs/${runId}`;
+    }
+    return {
+        provider,
+        repo,
+        sha,
+        branch: ref?.startsWith('refs/heads/') ? ref.slice('refs/heads/'.length) : ref,
+        runUrl,
+        actor,
+    };
+}
+function computeIdempotencyKey(evaluationId, ci) {
+    const repo = ci.repo ?? process.env.GITHUB_REPOSITORY;
+    const workflow = process.env.GITHUB_WORKFLOW ?? '';
+    const job = process.env.GITHUB_JOB ?? '';
+    const sha = ci.sha ?? process.env.GITHUB_SHA ?? '';
+    if (!repo || !sha)
+        return undefined;
+    const input = `${repo}.${workflow}.${job}.${sha}.${evaluationId}`;
+    return hashSha256(input);
+}
+function hashSha256(input) {
+    return (0, crypto_1.createHash)('sha256').update(input, 'utf8').digest('hex');
+}

package/dist/cli/config.d.ts

@@ -0,0 +1,24 @@
+/**
+ * EvalAI config loader
+ * Discovery: evalai.config.json → evalai.config.js → evalai.config.cjs → package.json evalai
+ */
+export interface EvalAIConfig {
+    evaluationId?: string;
+    baseUrl?: string;
+    minScore?: number;
+    minN?: number;
+    allowWeakEvidence?: boolean;
+    baseline?: 'published' | 'previous' | 'production';
+}
+/**
+ * Find config file path in directory, walking up to root
+ */
+export declare function findConfigPath(cwd?: string): string | null;
+/**
+ * Load config from file system
+ */
+export declare function loadConfig(cwd?: string): EvalAIConfig | null;
+/**
+ * Merge config with CLI args. Priority: args > config > defaults.
+ */
+export declare function mergeConfigWithArgs(config: EvalAIConfig | null, args: Partial<Record<string, string | number | boolean>>): Partial<EvalAIConfig>;

package/dist/cli/config.js

@@ -0,0 +1,158 @@
+"use strict";
+/**
+ * EvalAI config loader
+ * Discovery: evalai.config.json → evalai.config.js → evalai.config.cjs → package.json evalai
+ */
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.findConfigPath = findConfigPath;
+exports.loadConfig = loadConfig;
+exports.mergeConfigWithArgs = mergeConfigWithArgs;
+const fs = __importStar(require("fs"));
+const path = __importStar(require("path"));
+const CONFIG_FILES = [
+    'evalai.config.json',
+    'evalai.config.js',
+    'evalai.config.cjs',
+];
+/**
+ * Find config file path in directory, walking up to root
+ */
+function findConfigPath(cwd = process.cwd()) {
+    let dir = path.resolve(cwd);
+    const root = path.parse(dir).root;
+    while (dir !== root) {
+        for (const file of CONFIG_FILES) {
+            const filePath = path.join(dir, file);
+            if (fs.existsSync(filePath)) {
+                return filePath;
+            }
+        }
+        // Check package.json for evalai field
+        const pkgPath = path.join(dir, 'package.json');
+        if (fs.existsSync(pkgPath)) {
+            try {
+                const pkg = JSON.parse(fs.readFileSync(pkgPath, 'utf-8'));
+                if (pkg.evalai != null) {
+                    return pkgPath;
+                }
+            }
+            catch {
+                // ignore
+            }
+        }
+        dir = path.dirname(dir);
+    }
+    return null;
+}
+/**
+ * Load config from file system
+ */
+function loadConfig(cwd = process.cwd()) {
+    const configPath = findConfigPath(cwd);
+    if (!configPath)
+        return null;
+    try {
+        if (configPath.endsWith('package.json')) {
+            const pkg = JSON.parse(fs.readFileSync(configPath, 'utf-8'));
+            return pkg.evalai ?? null;
+        }
+        const content = fs.readFileSync(configPath, 'utf-8');
+        if (configPath.endsWith('.json')) {
+            return JSON.parse(content);
+        }
+        // .js or .cjs - would need to require/import; for v1 we only support JSON
+        if (configPath.endsWith('.js') || configPath.endsWith('.cjs')) {
+            // Try to parse as JSON first (some projects use .js with JSON content)
+            try {
+                return JSON.parse(content);
+            }
+            catch {
+                // Dynamic require of .js could have side effects; skip for v1
+                return null;
+            }
+        }
+        return null;
+    }
+    catch {
+        return null;
+    }
+}
+/**
+ * Merge config with CLI args. Priority: args > config > defaults.
+ */
+function mergeConfigWithArgs(config, args) {
+    const merged = {};
+    if (config) {
+        if (config.evaluationId)
+            merged.evaluationId = config.evaluationId;
+        if (config.baseUrl)
+            merged.baseUrl = config.baseUrl;
+        if (config.minScore != null)
+            merged.minScore = config.minScore;
+        if (config.minN != null)
+            merged.minN = config.minN;
+        if (config.allowWeakEvidence != null)
+            merged.allowWeakEvidence = config.allowWeakEvidence;
+        if (config.baseline)
+            merged.baseline = config.baseline;
+    }
+    // Args override
+    if (args.evaluationId !== undefined && args.evaluationId !== '') {
+        merged.evaluationId = String(args.evaluationId);
+    }
+    if (args.baseUrl !== undefined && args.baseUrl !== '') {
+        merged.baseUrl = String(args.baseUrl);
+    }
+    if (args.minScore !== undefined) {
+        merged.minScore = typeof args.minScore === 'number' ? args.minScore : parseInt(String(args.minScore), 10);
+    }
+    if (args.minN !== undefined) {
+        merged.minN = typeof args.minN === 'number' ? args.minN : parseInt(String(args.minN), 10);
+    }
+    if (args.allowWeakEvidence !== undefined) {
+        merged.allowWeakEvidence = args.allowWeakEvidence === true || args.allowWeakEvidence === 'true' || args.allowWeakEvidence === '1';
+    }
+    if (args.baseline !== undefined && args.baseline !== '') {
+        const b = String(args.baseline);
+        if (b === 'previous' || b === 'production') {
+            merged.baseline = b;
+        }
+        else {
+            merged.baseline = 'published';
+        }
+    }
+    return merged;
+}

package/dist/cli/constants.d.ts

@@ -0,0 +1,13 @@
+/**
+ * Standardized exit codes for evalai check.
+ */
+export declare const EXIT: {
+    readonly PASS: 0;
+    readonly SCORE_BELOW: 1;
+    readonly REGRESSION: 2;
+    readonly POLICY_VIOLATION: 3;
+    readonly API_ERROR: 4;
+    readonly BAD_ARGS: 5;
+    readonly LOW_N: 6;
+    readonly WEAK_EVIDENCE: 7;
+};

package/dist/cli/constants.js

@@ -0,0 +1,16 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.EXIT = void 0;
+/**
+ * Standardized exit codes for evalai check.
+ */
+exports.EXIT = {
+    PASS: 0,
+    SCORE_BELOW: 1,
+    REGRESSION: 2,
+    POLICY_VIOLATION: 3,
+    API_ERROR: 4,
+    BAD_ARGS: 5,
+    LOW_N: 6,
+    WEAK_EVIDENCE: 7,
+};

package/dist/cli/doctor.d.ts

@@ -0,0 +1,11 @@
+/**
+ * evalai doctor — Verify CI/CD setup.
+ * Uses the same quality endpoint as check — if doctor passes, check works.
+ */
+export type DoctorArgs = {
+    baseUrl: string;
+    apiKey: string;
+    evaluationId: string;
+    baseline: 'published' | 'previous' | 'production';
+};
+export declare function runDoctor(argv: string[]): Promise<number>;

package/dist/cli/doctor.js

@@ -0,0 +1,82 @@
+"use strict";
+/**
+ * evalai doctor — Verify CI/CD setup.
+ * Uses the same quality endpoint as check — if doctor passes, check works.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.runDoctor = runDoctor;
+const config_1 = require("./config");
+const api_1 = require("./api");
+function parseDoctorArgs(argv) {
+    const args = {};
+    for (let i = 0; i < argv.length; i++) {
+        const arg = argv[i];
+        if (arg.startsWith('--')) {
+            const key = arg.slice(2);
+            const next = argv[i + 1];
+            if (next !== undefined && !next.startsWith('--')) {
+                args[key] = next;
+                i++;
+            }
+            else {
+                args[key] = 'true';
+            }
+        }
+    }
+    const baseUrl = args.baseUrl || process.env.EVALAI_BASE_URL || 'http://localhost:3000';
+    const apiKey = args.apiKey || process.env.EVALAI_API_KEY || '';
+    let evaluationId = args.evaluationId || '';
+    const baseline = (args.baseline === 'previous'
+        ? 'previous'
+        : args.baseline === 'production'
+            ? 'production'
+            : 'published');
+    if (!evaluationId) {
+        const config = (0, config_1.loadConfig)(process.cwd());
+        const merged = (0, config_1.mergeConfigWithArgs)(config, {
+            evaluationId: args.evaluationId,
+            baseUrl: args.baseUrl || process.env.EVALAI_BASE_URL,
+            baseline: args.baseline,
+        });
+        if (merged.evaluationId)
+            evaluationId = String(merged.evaluationId);
+    }
+    if (!apiKey) {
+        return { ok: false, message: 'Set EVALAI_API_KEY' };
+    }
+    if (!evaluationId) {
+        const configPath = (0, config_1.findConfigPath)(process.cwd());
+        if (!configPath) {
+            return { ok: false, message: 'Run npx evalai init' };
+        }
+        return { ok: false, message: 'Set evaluationId in evalai.config.json' };
+    }
+    return { baseUrl, apiKey, evaluationId, baseline };
+}
+async function runDoctor(argv) {
+    const parsed = parseDoctorArgs(argv);
+    if (!('baseUrl' in parsed)) {
+        console.error(parsed.message);
+        return 1;
+    }
+    const args = parsed;
+    // Call exact quality endpoint: GET /api/quality?action=latest&evaluationId=&baseline=
+    const result = await (0, api_1.fetchQualityLatest)(args.baseUrl, args.apiKey, args.evaluationId, args.baseline);
+    if (!result.ok) {
+        if (result.status === 0) {
+            console.error(`Quality API: Network failure — ${result.body}`);
+        }
+        else {
+            console.error(`Quality API: ${result.status} — ${result.body}`);
+        }
+        return 1;
+    }
+    const { data } = result;
+    // Baseline: if quality returns baselineMissing, suggest fix
+    if (data.baselineMissing === true) {
+        console.error('Publish a run or use --baseline previous');
+        return 1;
+    }
+    console.log('✓ EvalAI doctor: OK');
+    return 0;
+}

package/dist/cli/formatters/github.d.ts

@@ -0,0 +1,8 @@
+/**
+ * GitHub formatter for evalai check.
+ * - stdout: minimal (verdict + score + link) + ::error annotations for failed cases
+ * - Step summary: full Markdown written to GITHUB_STEP_SUMMARY (not stdout)
+ */
+import type { CheckReport } from './types';
+export declare function appendStepSummary(report: CheckReport): void;
+export declare function formatGitHub(report: CheckReport): string;