@pauly4010/evalai-sdk 1.4.1 → 1.5.5

package/dist/cli/check.js

@@ -15,10 +15,14 @@
  *   --minN <n>           Fail if total test cases < n (low sample size)
  *   --allowWeakEvidence  If false (default), fail when evidenceLevel is 'weak'
  *   --policy <name>      Enforce a compliance policy (e.g. HIPAA, SOC2, GDPR)
- *   --baseline <mode>    Baseline comparison mode: "published" (default), "previous", or "production"
+ *   --baseline <mode>   Baseline comparison mode: "published" (default), "previous", or "production"
  *   --evaluationId <id>  Required. The evaluation to gate on.
  *   --baseUrl <url>      API base URL (default: EVALAI_BASE_URL or http://localhost:3000)
  *   --apiKey <key>       API key (default: EVALAI_API_KEY env var)
+ *   --share <mode>       Share link: "always" | "fail" | "never" (default: never)
+ *                        fail = create public share link only when gate fails (CI-friendly)
+ *   --pr-comment-out <file>  Write PR comment markdown to file (for GitHub Action to post)
+ *   --profile <name>         Preset: strict (95/0/30), balanced (90/2/10), fast (85/5/5). Explicit flags override.
  *
  * Exit codes:
  *   0  — Gate passed
@@ -29,187 +33,290 @@
  *   5  — Invalid arguments
  *   6  — Gate failed: total test cases < minN
  *   7  — Gate failed: weak evidence (evidenceLevel === 'weak')
+ *   8  — Gate warned: near-regression (warnDrop ≤ drop < maxDrop)
  *
  * Environment:
  *   EVALAI_BASE_URL  — API base URL (default: http://localhost:3000)
  *   EVALAI_API_KEY   — API key for authentication
  */
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.EXIT = void 0;
 exports.parseArgs = parseArgs;
 exports.runCheck = runCheck;
-// Standardized exit codes
-exports.EXIT = {
-    PASS: 0,
-    SCORE_BELOW: 1,
-    REGRESSION: 2,
-    POLICY_VIOLATION: 3,
-    API_ERROR: 4,
-    BAD_ARGS: 5,
-    LOW_N: 6,
-    WEAK_EVIDENCE: 7,
-};
+const fs = __importStar(require("node:fs"));
+const api_1 = require("./api");
+const ci_context_1 = require("./ci-context");
+const config_1 = require("./config");
+const constants_1 = require("./constants");
+const github_1 = require("./formatters/github");
+const human_1 = require("./formatters/human");
+const json_1 = require("./formatters/json");
+const pr_comment_1 = require("./formatters/pr-comment");
+const gate_1 = require("./gate");
+const build_check_report_1 = require("./report/build-check-report");
+var constants_2 = require("./constants");
+Object.defineProperty(exports, "EXIT", { enumerable: true, get: function () { return constants_2.EXIT; } });
 function parseArgs(argv) {
     const args = {};
     for (let i = 0; i < argv.length; i++) {
         const arg = argv[i];
-        if (arg.startsWith('--')) {
+        if (arg.startsWith("--")) {
             const key = arg.slice(2);
             const next = argv[i + 1];
-            if (next !== undefined && !next.startsWith('--')) {
+            if (next !== undefined && !next.startsWith("--")) {
                 args[key] = next;
                 i++;
             }
             else {
-                args[key] = 'true'; // bare flag
+                args[key] = "true";
             }
         }
     }
-    const baseUrl = args.baseUrl || process.env.EVALAI_BASE_URL || 'http://localhost:3000';
-    const apiKey = args.apiKey || process.env.EVALAI_API_KEY || '';
-    const minScore = parseInt(args.minScore || '0');
-    const maxDrop = args.maxDrop ? parseInt(args.maxDrop) : undefined;
-    const minN = args.minN ? parseInt(args.minN) : undefined;
-    const allowWeakEvidence = args.allowWeakEvidence === 'true' || args.allowWeakEvidence === '1';
-    const evaluationId = args.evaluationId || '';
+    let baseUrl = args.baseUrl || process.env.EVALAI_BASE_URL || "http://localhost:3000";
+    const apiKey = args.apiKey || process.env.EVALAI_API_KEY || "";
+    let minScore = parseInt(args.minScore || "0", 10);
+    let maxDrop = args.maxDrop ? parseInt(args.maxDrop, 10) : undefined;
+    let warnDrop = args.warnDrop ? parseInt(args.warnDrop, 10) : undefined;
+    let minN = args.minN ? parseInt(args.minN, 10) : undefined;
+    let allowWeakEvidence = args.allowWeakEvidence === "true" || args.allowWeakEvidence === "1";
+    let evaluationId = args.evaluationId || "";
     const policy = args.policy || undefined;
-    const baseline = (args.baseline === 'previous'
-        ? 'previous'
-        : args.baseline === 'production'
-            ? 'production'
-            : 'published');
+    const formatRaw = args.format || "human";
+    const format = formatRaw === "json" ? "json" : formatRaw === "github" ? "github" : "human";
+    const explain = args.explain === "true" || args.explain === "1";
+    const onFail = args.onFail === "import" ? "import" : undefined;
+    const shareRaw = args.share || "never";
+    const share = shareRaw === "always" ? "always" : shareRaw === "fail" ? "fail" : "never";
+    const prCommentOut = args["pr-comment-out"] || args.prCommentOut || undefined;
+    const maxCostUsd = args["max-cost-usd"] || args.maxCostUsd
+        ? parseFloat(args["max-cost-usd"] || args.maxCostUsd || "0")
+        : undefined;
+    const maxLatencyMs = args["max-latency-ms"] || args.maxLatencyMs
+        ? parseInt(args["max-latency-ms"] || args.maxLatencyMs || "0", 10)
+        : undefined;
+    const maxCostDeltaUsd = args["max-cost-delta-usd"] || args.maxCostDeltaUsd
+        ? parseFloat(args["max-cost-delta-usd"] || args.maxCostDeltaUsd || "0")
+        : undefined;
+    const profile = (args.profile || args.profile);
+    let baseline = (args.baseline === "auto"
+        ? "auto"
+        : args.baseline === "previous"
+            ? "previous"
+            : args.baseline === "production"
+                ? "production"
+                : "published");
+    const config = (0, config_1.loadConfig)(process.cwd());
+    const merged = (0, config_1.mergeConfigWithArgs)(config, {
+        evaluationId: args.evaluationId,
+        baseUrl: args.baseUrl || process.env.EVALAI_BASE_URL,
+        minScore: args.minScore,
+        maxDrop: args.maxDrop,
+        warnDrop: args.warnDrop,
+        minN: args.minN,
+        allowWeakEvidence: args.allowWeakEvidence,
+        baseline: args.baseline,
+        profile: profile,
+        prCommentOut: args["pr-comment-out"] ?? args.prCommentOut,
+    });
+    if (!evaluationId && merged.evaluationId)
+        evaluationId = merged.evaluationId;
+    if (merged.baseUrl)
+        baseUrl = merged.baseUrl;
+    if (merged.minScore != null && args.minScore === undefined)
+        minScore = merged.minScore ?? 0;
+    if (merged.maxDrop != null && args.maxDrop === undefined)
+        maxDrop = merged.maxDrop;
+    if (merged.warnDrop != null && args.warnDrop === undefined)
+        warnDrop = merged.warnDrop;
+    if (merged.minN != null && args.minN === undefined)
+        minN = merged.minN;
+    if (merged.allowWeakEvidence != null && args.allowWeakEvidence === undefined)
+        allowWeakEvidence = merged.allowWeakEvidence ?? false;
+    if (merged.baseline && !args.baseline)
+        baseline = merged.baseline;
     if (!apiKey) {
-        console.error('Error: --apiKey or EVALAI_API_KEY is required');
-        process.exit(exports.EXIT.BAD_ARGS);
+        return {
+            ok: false,
+            exitCode: constants_1.EXIT.BAD_ARGS,
+            message: "Error: --apiKey or EVALAI_API_KEY is required",
+        };
     }
     if (!evaluationId) {
-        console.error('Error: --evaluationId is required');
-        process.exit(exports.EXIT.BAD_ARGS);
+        return {
+            ok: false,
+            exitCode: constants_1.EXIT.BAD_ARGS,
+            message: "Run npx evalai init and paste your evaluationId, or pass --evaluationId.",
+        };
     }
-    if (isNaN(minScore) || minScore < 0 || minScore > 100) {
-        console.error('Error: --minScore must be 0-100');
-        process.exit(exports.EXIT.BAD_ARGS);
+    if (Number.isNaN(minScore) || minScore < 0 || minScore > 100) {
+        return { ok: false, exitCode: constants_1.EXIT.BAD_ARGS, message: "Error: --minScore must be 0-100" };
     }
-    if (minN !== undefined && (isNaN(minN) || minN < 1)) {
-        console.error('Error: --minN must be a positive number');
-        process.exit(exports.EXIT.BAD_ARGS);
+    if (minN !== undefined && (Number.isNaN(minN) || minN < 1)) {
+        return {
+            ok: false,
+            exitCode: constants_1.EXIT.BAD_ARGS,
+            message: "Error: --minN must be a positive number",
+        };
     }
-    return { baseUrl, apiKey, minScore, maxDrop, minN, allowWeakEvidence, evaluationId, policy, baseline };
+    return {
+        ok: true,
+        args: {
+            baseUrl,
+            apiKey,
+            minScore,
+            maxDrop,
+            warnDrop,
+            minN,
+            allowWeakEvidence,
+            evaluationId,
+            policy,
+            baseline,
+            format,
+            explain,
+            onFail,
+            share,
+            prCommentOut,
+            maxCostUsd: maxCostUsd != null && !Number.isNaN(maxCostUsd) ? maxCostUsd : undefined,
+            maxLatencyMs: maxLatencyMs != null && !Number.isNaN(maxLatencyMs) ? maxLatencyMs : undefined,
+            maxCostDeltaUsd: maxCostDeltaUsd != null && !Number.isNaN(maxCostDeltaUsd) ? maxCostDeltaUsd : undefined,
+        },
+    };
 }
 async function runCheck(args) {
-    const headers = { Authorization: `Bearer ${args.apiKey}` };
-    // ── 1. Fetch latest quality score ──
-    const scoreUrl = `${args.baseUrl}/api/quality?evaluationId=${args.evaluationId}&action=latest&baseline=${args.baseline}`;
-    let scoreRes;
-    try {
-        scoreRes = await fetch(scoreUrl, { headers });
-    }
-    catch (err) {
-        console.error(`EvalAI gate ERROR: Network failure — ${err.message}`);
-        return exports.EXIT.API_ERROR;
-    }
-    if (!scoreRes.ok) {
-        const body = await scoreRes.text();
-        console.error(`EvalAI gate ERROR: API returned ${scoreRes.status} — ${body}`);
-        return exports.EXIT.API_ERROR;
-    }
-    const data = (await scoreRes.json());
-    const score = data?.score ?? 0;
-    const total = data?.total ?? null;
-    const evidenceLevel = data?.evidenceLevel ?? null;
-    const baselineScore = data?.baselineScore ?? null;
-    const regressionDelta = data?.regressionDelta ?? null;
-    const baselineMissing = data?.baselineMissing === true;
-    const breakdown = data?.breakdown ?? {};
-    // ── Gate: baseline missing (when baseline comparison requested) ──
-    if (baselineMissing && (args.baseline !== 'published' || args.maxDrop !== undefined)) {
-        console.error(`\n✗ FAILED: baseline (${args.baseline}) not found. ` +
-            `Ensure a baseline run exists (e.g. published run, previous run, or prod-tagged run).`);
-        return exports.EXIT.API_ERROR;
-    }
-    // ── Gate: minN (low sample size) ──
-    if (args.minN !== undefined && total !== null && total < args.minN) {
-        console.error(`\n✗ FAILED: total test cases (${total}) < minN (${args.minN})`);
-        return exports.EXIT.LOW_N;
-    }
-    // ── Gate: allowWeakEvidence ──
-    if (!args.allowWeakEvidence && evidenceLevel === 'weak') {
-        console.error(`\n✗ FAILED: evidence level is 'weak' (use --allowWeakEvidence to permit)`);
-        return exports.EXIT.WEAK_EVIDENCE;
-    }
-    // ── Print summary ──
-    console.log('┌─────────────────────────────────────────┐');
-    console.log(`│  EvalAI Quality Score: ${String(score).padStart(3)}/100            │`);
-    console.log('├─────────────────────────────────────────┤');
-    if (baselineScore !== null) {
-        const delta = regressionDelta ?? 0;
-        const arrow = delta >= 0 ? '▲' : '▼';
-        console.log(`│  Baseline: ${baselineScore}  ${arrow} ${Math.abs(delta)} pts          │`);
-    }
-    if (breakdown) {
-        const pct = (v) => `${Math.round((v ?? 0) * 100)}%`;
-        console.log(`│  Pass: ${pct(breakdown.passRate)}  Safety: ${pct(breakdown.safety)}  Judge: ${pct(breakdown.judge)} │`);
-    }
-    if (data?.flags && data.flags.length > 0) {
-        console.log(`│  Flags: ${data.flags.join(', ').padEnd(30)} │`);
+    const qualityResult = await (0, api_1.fetchQualityLatest)(args.baseUrl, args.apiKey, args.evaluationId, args.baseline);
+    if (!qualityResult.ok) {
+        if (qualityResult.status === 0) {
+            console.error(`EvalAI gate ERROR: Network failure — ${qualityResult.body}`);
+        }
+        else {
+            console.error(`EvalAI gate ERROR: API returned ${qualityResult.status} — ${qualityResult.body}`);
+        }
+        return constants_1.EXIT.API_ERROR;
     }
-    console.log('└─────────────────────────────────────────┘');
-    // ── 2. Gate: minimum score ──
-    if (args.minScore > 0 && score < args.minScore) {
-        console.error(`\n✗ FAILED: score=${score} < minScore=${args.minScore}`);
-        return exports.EXIT.SCORE_BELOW;
+    const { data: quality, requestId } = qualityResult;
+    const evaluationRunId = quality?.evaluationRunId;
+    let runDetails = null;
+    if (evaluationRunId != null) {
+        const runRes = await (0, api_1.fetchRunDetails)(args.baseUrl, args.apiKey, args.evaluationId, evaluationRunId);
+        if (runRes.ok)
+            runDetails = runRes.data;
     }
-    // ── 3. Gate: maximum drop from baseline ──
-    if (args.maxDrop !== undefined && regressionDelta !== null && regressionDelta < -(args.maxDrop)) {
-        console.error(`\n✗ FAILED: score dropped ${Math.abs(regressionDelta)} pts from baseline ` +
-            `(max allowed: ${args.maxDrop})`);
-        return exports.EXIT.REGRESSION;
+    const gateResult = (0, gate_1.evaluateGate)(args, quality);
+    // Create share before report when PR comment needs shareUrl (--pr-comment-out + --share fail + gate failed)
+    let shareUrl;
+    const shouldCreateShare = quality?.evaluationRunId != null &&
+        (args.share === "always" || (args.share === "fail" && !gateResult.passed));
+    if (shouldCreateShare) {
+        const exportRes = await (0, api_1.fetchRunExport)(args.baseUrl, args.apiKey, args.evaluationId, quality.evaluationRunId);
+        if (exportRes.ok) {
+            const publishRes = await (0, api_1.publishShare)(args.baseUrl, args.apiKey, args.evaluationId, exportRes.exportData, quality.evaluationRunId);
+            if (publishRes.ok) {
+                shareUrl = publishRes.data.shareUrl;
+                console.error(`\nPublic share link created: ${shareUrl}`);
+            }
+        }
     }
-    // ── 4. Gate: policy compliance ──
-    if (args.policy) {
-        const policyUrl = `${args.baseUrl}/api/quality?evaluationId=${args.evaluationId}&action=latest`;
-        // Check policy-specific flags
-        const policyFlags = (data?.flags ?? []);
-        // Policy mapping: each policy has a set of required conditions
-        const policyChecks = {
-            HIPAA: { requiredSafetyRate: 0.99, maxFlags: ['SAFETY_RISK'] },
-            SOC2: { requiredSafetyRate: 0.95, maxFlags: ['SAFETY_RISK', 'LOW_PASS_RATE'] },
-            GDPR: { requiredSafetyRate: 0.95, maxFlags: ['SAFETY_RISK'] },
-            PCI_DSS: { requiredSafetyRate: 0.99, maxFlags: ['SAFETY_RISK', 'LOW_PASS_RATE'] },
-            FINRA_4511: { requiredSafetyRate: 0.95, maxFlags: ['SAFETY_RISK'] },
-        };
-        const policyName = args.policy.toUpperCase();
-        const check = policyChecks[policyName];
-        if (!check) {
-            console.error(`\n✗ Unknown policy: ${args.policy}. Available: ${Object.keys(policyChecks).join(', ')}`);
-            return exports.EXIT.BAD_ARGS;
+    const ci = (0, ci_context_1.captureCiContext)();
+    const report = (0, build_check_report_1.buildCheckReport)({
+        args,
+        quality,
+        runDetails,
+        gateResult,
+        requestId,
+        shareUrl,
+        baselineRunId: quality?.baselineRunId ?? undefined,
+        ciRunUrl: ci?.runUrl ?? undefined,
+    });
+    const formatted = args.format === "json"
+        ? (0, json_1.formatJson)(report)
+        : args.format === "github"
+            ? (0, github_1.formatGitHub)(report)
+            : (0, human_1.formatHuman)(report);
+    console.log(formatted);
+    // --pr-comment-out: write markdown to file for GitHub Action to post
+    if (args.prCommentOut) {
+        try {
+            const markdown = (0, pr_comment_1.buildPrComment)(report);
+            fs.writeFileSync(args.prCommentOut, markdown, "utf8");
         }
-        // Check safety rate
-        const safetyRate = breakdown?.safety ?? 0;
-        if (safetyRate < check.requiredSafetyRate) {
-            console.error(`\n✗ POLICY VIOLATION (${policyName}): safety rate ${Math.round(safetyRate * 100)}% < ` +
-                `required ${Math.round(check.requiredSafetyRate * 100)}%`);
-            return exports.EXIT.POLICY_VIOLATION;
+        catch (err) {
+            console.error(`EvalAI: failed to write PR comment to ${args.prCommentOut}: ${err instanceof Error ? err.message : String(err)}`);
         }
-        // Check for disqualifying flags
-        const violations = policyFlags.filter(f => check.maxFlags.includes(f));
-        if (violations.length > 0) {
-            console.error(`\n✗ POLICY VIOLATION (${policyName}): ${violations.join(', ')}`);
-            return exports.EXIT.POLICY_VIOLATION;
+    }
+    // --onFail import: when gate fails, import run with CI context
+    if (!gateResult.passed &&
+        args.onFail === "import" &&
+        runDetails?.results &&
+        quality?.evaluationRunId) {
+        const importResults = runDetails.results
+            .filter((r) => r.testCaseId != null && (r.status === "passed" || r.status === "failed"))
+            .map((r) => ({
+            testCaseId: r.testCaseId,
+            status: r.status,
+            output: r.output ?? "",
+            latencyMs: r.durationMs,
+            assertionsJson: r.assertionsJson,
+        }));
+        if (importResults.length > 0) {
+            const idempotencyKey = ci ? (0, ci_context_1.computeIdempotencyKey)(args.evaluationId, ci) : undefined;
+            const importRes = await (0, api_1.importRunOnFail)(args.baseUrl, args.apiKey, args.evaluationId, importResults, {
+                idempotencyKey,
+                ci,
+                importClientVersion: "evalai-cli",
+                checkReport: report,
+            });
+            if (!importRes.ok) {
+                console.error(`EvalAI import (onFail): ${importRes.status} — ${importRes.body}`);
+            }
         }
-        console.log(`\n✓ Policy ${policyName}: COMPLIANT`);
     }
-    console.log('\n✓ EvalAI gate PASSED');
-    return exports.EXIT.PASS;
+    return gateResult.exitCode;
 }
 // Main entry point
-const isDirectRun = typeof require !== 'undefined' && require.main === module;
+const isDirectRun = typeof require !== "undefined" && require.main === module;
 if (isDirectRun) {
-    const args = parseArgs(process.argv.slice(2));
-    runCheck(args).then((code) => {
-        process.exit(code);
-    }).catch((err) => {
-        console.error(`EvalAI gate ERROR: ${err.message}`);
-        process.exit(exports.EXIT.API_ERROR);
+    const parsed = parseArgs(process.argv.slice(2));
+    if (!parsed.ok) {
+        console.error(parsed.message);
+        process.exit(parsed.exitCode);
+    }
+    runCheck(parsed.args)
+        .then((code) => process.exit(code))
+        .catch((err) => {
+        console.error(`EvalAI gate ERROR: ${err instanceof Error ? err.message : String(err)}`);
+        process.exit(constants_1.EXIT.API_ERROR);
     });
 }

package/dist/cli/ci-context.d.ts

@@ -0,0 +1,6 @@
+/**
+ * CI context capture and idempotency key for --onFail import.
+ */
+import type { CiContext } from "./api";
+export declare function captureCiContext(): CiContext | undefined;
+export declare function computeIdempotencyKey(evaluationId: string, ci: CiContext): string | undefined;

package/dist/cli/ci-context.js

@@ -0,0 +1,110 @@
+"use strict";
+/**
+ * CI context capture and idempotency key for --onFail import.
+ */
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.captureCiContext = captureCiContext;
+exports.computeIdempotencyKey = computeIdempotencyKey;
+const node_crypto_1 = require("node:crypto");
+const fs = __importStar(require("node:fs"));
+function readPrFromEventPath() {
+    const path = process.env.GITHUB_EVENT_PATH;
+    if (!path)
+        return undefined;
+    try {
+        const raw = fs.readFileSync(path, "utf8");
+        const event = JSON.parse(raw);
+        return event.pull_request?.number;
+    }
+    catch {
+        return undefined;
+    }
+}
+function readPrFromRef() {
+    const ref = process.env.GITHUB_REF;
+    if (!ref)
+        return undefined;
+    const m = ref.match(/^refs\/pull\/(\d+)\/merge$/);
+    return m ? parseInt(m[1], 10) : undefined;
+}
+function captureCiContext() {
+    const repo = process.env.GITHUB_REPOSITORY;
+    const sha = process.env.GITHUB_SHA;
+    const ref = process.env.GITHUB_REF;
+    const runId = process.env.GITHUB_RUN_ID;
+    const _workflow = process.env.GITHUB_WORKFLOW;
+    const _job = process.env.GITHUB_JOB;
+    const actor = process.env.GITHUB_ACTOR;
+    if (!repo && !sha)
+        return undefined;
+    let provider = "unknown";
+    if (process.env.GITHUB_ACTIONS)
+        provider = "github";
+    else if (process.env.GITLAB_CI)
+        provider = "gitlab";
+    else if (process.env.CIRCLECI)
+        provider = "circle";
+    let runUrl;
+    if (repo && runId) {
+        runUrl = `https://github.com/${repo}/actions/runs/${runId}`;
+    }
+    let pr;
+    if (process.env.GITHUB_EVENT_NAME === "pull_request") {
+        pr = readPrFromEventPath() ?? readPrFromRef();
+    }
+    return {
+        provider,
+        repo,
+        sha,
+        branch: ref?.startsWith("refs/heads/") ? ref.slice("refs/heads/".length) : ref,
+        runUrl,
+        actor,
+        pr,
+    };
+}
+function computeIdempotencyKey(evaluationId, ci) {
+    const repo = ci.repo ?? process.env.GITHUB_REPOSITORY;
+    const workflow = process.env.GITHUB_WORKFLOW ?? "";
+    const job = process.env.GITHUB_JOB ?? "";
+    const sha = ci.sha ?? process.env.GITHUB_SHA ?? "";
+    if (!repo || !sha)
+        return undefined;
+    const input = `${repo}.${workflow}.${job}.${sha}.${evaluationId}`;
+    return hashSha256(input);
+}
+function hashSha256(input) {
+    return (0, node_crypto_1.createHash)("sha256").update(input, "utf8").digest("hex");
+}

package/dist/cli/config.d.ts

@@ -0,0 +1,30 @@
+/**
+ * EvalAI config loader
+ * Discovery: evalai.config.json → evalai.config.js → evalai.config.cjs → package.json evalai
+ */
+import { type ProfileName } from "./profiles";
+export interface EvalAIConfig {
+    evaluationId?: string;
+    baseUrl?: string;
+    minScore?: number;
+    minN?: number;
+    maxDrop?: number;
+    warnDrop?: number;
+    allowWeakEvidence?: boolean;
+    baseline?: "published" | "previous" | "production" | "auto";
+    profile?: ProfileName;
+    /** Monorepo: package path → config. Key = path relative to config dir (e.g. "apps/web", "packages/api"). */
+    packages?: Record<string, Partial<EvalAIConfig>>;
+}
+/**
+ * Find config file path in directory, walking up to root
+ */
+export declare function findConfigPath(cwd?: string): string | null;
+/**
+ * Load config from file system
+ */
+export declare function loadConfig(cwd?: string): EvalAIConfig | null;
+/**
+ * Merge config with CLI args. Priority: args > profile > config > defaults.
+ */
+export declare function mergeConfigWithArgs(config: EvalAIConfig | null, args: Partial<Record<string, string | number | boolean>>): Partial<EvalAIConfig>;