@pauly4010/evalai-sdk 1.7.0 → 1.8.0

package/dist/cli/print-config.js

@@ -0,0 +1,251 @@
+"use strict";
+/**
+ * evalai print-config — Show resolved configuration with source-of-truth annotations.
+ *
+ * Prints every config field, where it came from (file, env, default, CLI arg),
+ * and redacts secrets. Useful for debugging "why is it using this baseUrl?"
+ *
+ * Usage:
+ *   evalai print-config
+ *   evalai print-config --format json
+ *
+ * Exit codes:
+ *   0 — Always (informational only)
+ */
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.runPrintConfig = runPrintConfig;
+const path = __importStar(require("node:path"));
+const version_1 = require("../version");
+const config_1 = require("./config");
+const profiles_1 = require("./profiles");
+function parseFlags(argv) {
+    const raw = {};
+    for (let i = 0; i < argv.length; i++) {
+        const arg = argv[i];
+        if (arg.startsWith("--")) {
+            const key = arg.slice(2);
+            const next = argv[i + 1];
+            if (next !== undefined && !next.startsWith("--")) {
+                raw[key] = next;
+                i++;
+            }
+            else {
+                raw[key] = "true";
+            }
+        }
+    }
+    return {
+        format: raw.format === "json" ? "json" : "human",
+        evaluationId: raw.evaluationId,
+        baseUrl: raw.baseUrl,
+        apiKey: raw.apiKey,
+        baseline: raw.baseline,
+        profile: raw.profile,
+        minScore: raw.minScore,
+        maxDrop: raw.maxDrop,
+        warnDrop: raw.warnDrop,
+        minN: raw.minN,
+    };
+}
+// ── Helpers ──
+function redact(value) {
+    if (!value)
+        return null;
+    if (value.length > 8)
+        return `${value.slice(0, 4)}...${value.slice(-4)}`;
+    return "****";
+}
+// ── Build resolved config ──
+function buildResolvedConfig(cwd, flags) {
+    const configPath = (0, config_1.findConfigPath)(cwd);
+    const fileConfig = (0, config_1.loadConfig)(cwd);
+    // Build CLI args object (only what was explicitly passed)
+    const cliArgs = {};
+    if (flags.evaluationId)
+        cliArgs.evaluationId = flags.evaluationId;
+    if (flags.baseUrl)
+        cliArgs.baseUrl = flags.baseUrl;
+    if (flags.baseline)
+        cliArgs.baseline = flags.baseline;
+    if (flags.profile)
+        cliArgs.profile = flags.profile;
+    if (flags.minScore)
+        cliArgs.minScore = flags.minScore;
+    if (flags.maxDrop)
+        cliArgs.maxDrop = flags.maxDrop;
+    if (flags.warnDrop)
+        cliArgs.warnDrop = flags.warnDrop;
+    if (flags.minN)
+        cliArgs.minN = flags.minN;
+    const merged = (0, config_1.mergeConfigWithArgs)(fileConfig, cliArgs);
+    // Determine source of each field
+    const fields = [];
+    // evaluationId
+    const evalIdSource = flags.evaluationId ? "arg"
+        : fileConfig?.evaluationId ? "file"
+            : "default";
+    fields.push({
+        key: "evaluationId",
+        value: merged.evaluationId ?? null,
+        source: evalIdSource,
+    });
+    // baseUrl
+    const envBaseUrl = process.env.EVALAI_BASE_URL;
+    const baseUrlSource = flags.baseUrl ? "arg"
+        : envBaseUrl ? "env"
+            : fileConfig?.baseUrl ? "file"
+                : "default";
+    fields.push({
+        key: "baseUrl",
+        value: flags.baseUrl || envBaseUrl || fileConfig?.baseUrl || "http://localhost:3000",
+        source: baseUrlSource,
+    });
+    // apiKey (always redacted)
+    const envApiKey = process.env.EVALAI_API_KEY;
+    const rawApiKey = flags.apiKey || envApiKey || "";
+    const apiKeySource = flags.apiKey ? "arg"
+        : envApiKey ? "env"
+            : "default";
+    fields.push({
+        key: "apiKey",
+        value: redact(rawApiKey) ?? "(not set)",
+        source: apiKeySource,
+        raw: rawApiKey ? "(redacted)" : undefined,
+    });
+    // profile
+    const profileName = (flags.profile || fileConfig?.profile);
+    const profileSource = flags.profile ? "arg" : fileConfig?.profile ? "file" : "default";
+    fields.push({
+        key: "profile",
+        value: profileName ?? null,
+        source: profileSource,
+    });
+    // Numeric gate fields: minScore, maxDrop, warnDrop, minN, allowWeakEvidence
+    const numericFields = [
+        { key: "minScore" },
+        { key: "maxDrop" },
+        { key: "warnDrop" },
+        { key: "minN" },
+        { key: "allowWeakEvidence" },
+    ];
+    for (const { key } of numericFields) {
+        const argVal = cliArgs[key];
+        const fileVal = fileConfig?.[key];
+        const profileVal = profileName && profileName in profiles_1.PROFILES
+            ? profiles_1.PROFILES[profileName][key]
+            : undefined;
+        const source = argVal !== undefined ? "arg"
+            : fileVal !== undefined ? "file"
+                : profileVal !== undefined ? "profile"
+                    : "default";
+        fields.push({
+            key,
+            value: merged[key] ?? null,
+            source,
+        });
+    }
+    // baseline
+    const baselineSource = flags.baseline ? "arg"
+        : fileConfig?.baseline ? "file"
+            : "default";
+    fields.push({
+        key: "baseline",
+        value: merged.baseline ?? "published",
+        source: baselineSource,
+    });
+    // Environment variables summary
+    const envVars = {
+        EVALAI_API_KEY: redact(envApiKey),
+        EVALAI_BASE_URL: envBaseUrl ?? null,
+        OPENAI_API_KEY: redact(process.env.OPENAI_API_KEY),
+        ANTHROPIC_API_KEY: redact(process.env.ANTHROPIC_API_KEY),
+        AZURE_OPENAI_API_KEY: redact(process.env.AZURE_OPENAI_API_KEY),
+        GITHUB_ACTIONS: process.env.GITHUB_ACTIONS ?? null,
+        CI: process.env.CI ?? null,
+    };
+    return {
+        cliVersion: version_1.SDK_VERSION,
+        configFile: configPath ? path.relative(cwd, configPath) : null,
+        cwd,
+        resolved: fields,
+        env: envVars,
+    };
+}
+// ── Output formatting ──
+function printHuman(output) {
+    console.log("\n  evalai print-config\n");
+    console.log(`  CLI version: ${output.cliVersion}`);
+    console.log(`  Config file: ${output.configFile ?? "(none found)"}`);
+    console.log(`  Working dir: ${output.cwd}`);
+    console.log("");
+    console.log("  Resolved configuration:");
+    console.log("");
+    const maxKeyLen = Math.max(...output.resolved.map((f) => f.key.length));
+    for (const field of output.resolved) {
+        const val = field.value === null ? "(not set)" : String(field.value);
+        const pad = " ".repeat(maxKeyLen - field.key.length);
+        const sourceTag = `[${field.source}]`;
+        console.log(`    ${field.key}${pad}  ${val}  ${sourceTag}`);
+    }
+    console.log("");
+    console.log("  Environment variables:");
+    console.log("");
+    for (const [key, val] of Object.entries(output.env)) {
+        if (val !== null) {
+            console.log(`    ${key} = ${val}`);
+        }
+    }
+    const unsetEnv = Object.entries(output.env)
+        .filter(([, v]) => v === null)
+        .map(([k]) => k);
+    if (unsetEnv.length > 0) {
+        console.log(`    (not set: ${unsetEnv.join(", ")})`);
+    }
+    console.log("");
+}
+// ── Main ──
+function runPrintConfig(argv) {
+    const flags = parseFlags(argv);
+    const cwd = process.cwd();
+    const output = buildResolvedConfig(cwd, flags);
+    if (flags.format === "json") {
+        console.log(JSON.stringify(output, null, 2));
+    }
+    else {
+        printHuman(output);
+    }
+    return 0;
+}

package/dist/cli/report/build-check-report.d.ts

@@ -4,7 +4,7 @@
  */
 import type { QualityLatestData, RunDetailsData } from "../api";
 import type { CheckArgs } from "../check";
-import type { CheckReport } from "../formatters/types";
+import { type CheckReport } from "../formatters/types";
 import type { GateResult } from "../gate";
 export type BuildReportInput = {
     args: CheckArgs;

package/dist/cli/report/build-check-report.js

@@ -5,6 +5,7 @@
  */
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.buildCheckReport = buildCheckReport;
+const types_1 = require("../formatters/types");
 const snippet_1 = require("../render/snippet");
 const sort_1 = require("../render/sort");
 const TOP_N = 3;
@@ -69,6 +70,7 @@ function buildCheckReport(input) {
         : (gateResult.reasonMessage ?? undefined);
     const verdict = gateResult.reasonCode === "WARN_REGRESSION" ? "warn" : gateResult.passed ? "pass" : "fail";
     const report = {
+        schemaVersion: types_1.CHECK_REPORT_SCHEMA_VERSION,
         evaluationId: args.evaluationId,
         runId: evaluationRunId,
         verdict,

package/dist/version.d.ts

@@ -3,5 +3,5 @@
  * X-EvalAI-SDK-Version: SDK package version
  * X-EvalAI-Spec-Version: OpenAPI spec version (docs/openapi.json info.version)
  */
-export declare const SDK_VERSION = "1.6.0";
+export declare const SDK_VERSION = "1.8.0";
 export declare const SPEC_VERSION = "1.0.0";

package/dist/version.js

@@ -6,5 +6,5 @@ exports.SPEC_VERSION = exports.SDK_VERSION = void 0;
  * X-EvalAI-SDK-Version: SDK package version
  * X-EvalAI-Spec-Version: OpenAPI spec version (docs/openapi.json info.version)
  */
-exports.SDK_VERSION = "1.6.0";
+exports.SDK_VERSION = "1.8.0";
 exports.SPEC_VERSION = "1.0.0";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pauly4010/evalai-sdk",
-  "version": "1.7.0",
+  "version": "1.8.0",
   "description": "AI Evaluation Platform SDK - Complete API Coverage with Performance Optimizations",
   "main": "dist/index.js",
   "module": "dist/index.js",