@pauly4010/evalai-sdk 1.4.1 → 1.5.5

package/dist/cli/api.d.ts

@@ -0,0 +1,108 @@
+/**
+ * API fetch helpers for evalai check.
+ * Captures x-request-id from response headers.
+ * Sends X-EvalAI-SDK-Version and X-EvalAI-Spec-Version on all requests.
+ */
+export type QualityLatestData = {
+    score?: number;
+    total?: number | null;
+    evidenceLevel?: string | null;
+    baselineScore?: number | null;
+    regressionDelta?: number | null;
+    baselineMissing?: boolean | null;
+    breakdown?: {
+        passRate?: number;
+        safety?: number;
+        judge?: number;
+    };
+    flags?: string[];
+    evaluationRunId?: number;
+    evaluationId?: number;
+    avgLatencyMs?: number | null;
+    costUsd?: number | null;
+    baselineCostUsd?: number | null;
+    baselineRunId?: number | null;
+};
+export type RunDetailsData = {
+    results?: Array<{
+        testCaseId?: number;
+        status?: string;
+        output?: string;
+        durationMs?: number;
+        assertionsJson?: Record<string, unknown>;
+        test_cases?: {
+            name?: string;
+            input?: string;
+            expectedOutput?: string;
+        };
+    }>;
+};
+export declare function fetchQualityLatest(baseUrl: string, apiKey: string, evaluationId: string, baseline: string): Promise<{
+    ok: true;
+    data: QualityLatestData;
+    requestId?: string;
+} | {
+    ok: false;
+    status: number;
+    body: string;
+    requestId?: string;
+}>;
+export declare function fetchRunDetails(baseUrl: string, apiKey: string, evaluationId: string, runId: number): Promise<{
+    ok: true;
+    data: RunDetailsData;
+} | {
+    ok: false;
+}>;
+export type CiContext = {
+    provider?: "github" | "gitlab" | "circle" | "unknown";
+    repo?: string;
+    sha?: string;
+    branch?: string;
+    pr?: number;
+    runUrl?: string;
+    actor?: string;
+};
+export type ImportResult = {
+    testCaseId: number;
+    status: "passed" | "failed";
+    output: string;
+    latencyMs?: number;
+    costUsd?: number;
+    assertionsJson?: Record<string, unknown>;
+};
+export type PublishShareResult = {
+    shareId: string;
+    shareUrl: string;
+    shareScope: string;
+};
+export declare function fetchRunExport(baseUrl: string, apiKey: string, evaluationId: string, runId: number): Promise<{
+    ok: true;
+    exportData: Record<string, unknown>;
+} | {
+    ok: false;
+    status: number;
+    body: string;
+}>;
+export declare function publishShare(baseUrl: string, apiKey: string, evaluationId: string, exportData: Record<string, unknown>, evaluationRunId: number, options?: {
+    expiresInDays?: number;
+}): Promise<{
+    ok: true;
+    data: PublishShareResult;
+} | {
+    ok: false;
+    status: number;
+    body: string;
+}>;
+export declare function importRunOnFail(baseUrl: string, apiKey: string, evaluationId: string, results: ImportResult[], options: {
+    idempotencyKey?: string;
+    ci?: CiContext;
+    importClientVersion?: string;
+    checkReport?: Record<string, unknown>;
+}): Promise<{
+    ok: true;
+    runId: number;
+} | {
+    ok: false;
+    status: number;
+    body: string;
+}>;

package/dist/cli/api.js

@@ -0,0 +1,130 @@
+"use strict";
+/**
+ * API fetch helpers for evalai check.
+ * Captures x-request-id from response headers.
+ * Sends X-EvalAI-SDK-Version and X-EvalAI-Spec-Version on all requests.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.fetchQualityLatest = fetchQualityLatest;
+exports.fetchRunDetails = fetchRunDetails;
+exports.fetchRunExport = fetchRunExport;
+exports.publishShare = publishShare;
+exports.importRunOnFail = importRunOnFail;
+const version_1 = require("../version");
+const API_HEADERS = {
+    "X-EvalAI-SDK-Version": version_1.SDK_VERSION,
+    "X-EvalAI-Spec-Version": version_1.SPEC_VERSION,
+};
+async function fetchQualityLatest(baseUrl, apiKey, evaluationId, baseline) {
+    const headers = { ...API_HEADERS, Authorization: `Bearer ${apiKey}` };
+    const url = `${baseUrl.replace(/\/$/, "")}/api/quality?evaluationId=${evaluationId}&action=latest&baseline=${baseline}`;
+    try {
+        const res = await fetch(url, { headers });
+        const requestId = res.headers.get("x-request-id") ?? undefined;
+        const body = await res.text();
+        if (!res.ok) {
+            return { ok: false, status: res.status, body, requestId };
+        }
+        const data = JSON.parse(body);
+        return { ok: true, data, requestId };
+    }
+    catch (err) {
+        const msg = err instanceof Error ? err.message : String(err);
+        return { ok: false, status: 0, body: msg, requestId: undefined };
+    }
+}
+async function fetchRunDetails(baseUrl, apiKey, evaluationId, runId) {
+    const headers = { ...API_HEADERS, Authorization: `Bearer ${apiKey}` };
+    const url = `${baseUrl.replace(/\/$/, "")}/api/evaluations/${evaluationId}/runs/${runId}`;
+    try {
+        const res = await fetch(url, { headers });
+        if (!res.ok)
+            return { ok: false };
+        const data = (await res.json());
+        return { ok: true, data };
+    }
+    catch {
+        return { ok: false };
+    }
+}
+async function fetchRunExport(baseUrl, apiKey, evaluationId, runId) {
+    const headers = { ...API_HEADERS, Authorization: `Bearer ${apiKey}` };
+    const url = `${baseUrl.replace(/\/$/, "")}/api/evaluations/${evaluationId}/runs/${runId}/export`;
+    try {
+        const res = await fetch(url, { headers });
+        const text = await res.text();
+        if (!res.ok)
+            return { ok: false, status: res.status, body: text };
+        const exportData = JSON.parse(text);
+        return { ok: true, exportData };
+    }
+    catch (err) {
+        const msg = err instanceof Error ? err.message : String(err);
+        return { ok: false, status: 0, body: msg };
+    }
+}
+async function publishShare(baseUrl, apiKey, evaluationId, exportData, evaluationRunId, options) {
+    const headers = {
+        ...API_HEADERS,
+        Authorization: `Bearer ${apiKey}`,
+        "Content-Type": "application/json",
+    };
+    const body = {
+        exportData,
+        shareScope: "run",
+        evaluationRunId,
+        ...(options?.expiresInDays != null && { expiresInDays: options.expiresInDays }),
+    };
+    const url = `${baseUrl.replace(/\/$/, "")}/api/evaluations/${evaluationId}/publish`;
+    try {
+        const res = await fetch(url, {
+            method: "POST",
+            headers,
+            body: JSON.stringify(body),
+        });
+        const text = await res.text();
+        if (!res.ok)
+            return { ok: false, status: res.status, body: text };
+        const data = JSON.parse(text);
+        return { ok: true, data };
+    }
+    catch (err) {
+        const msg = err instanceof Error ? err.message : String(err);
+        return { ok: false, status: 0, body: msg };
+    }
+}
+async function importRunOnFail(baseUrl, apiKey, evaluationId, results, options) {
+    const headers = {
+        ...API_HEADERS,
+        Authorization: `Bearer ${apiKey}`,
+        "Content-Type": "application/json",
+    };
+    if (options.idempotencyKey) {
+        headers["Idempotency-Key"] = options.idempotencyKey;
+    }
+    const body = {
+        environment: "dev",
+        results,
+        importClientVersion: options.importClientVersion ?? "evalai-cli",
+        ci: options.ci,
+        ...(options.checkReport != null && { checkReport: options.checkReport }),
+    };
+    const url = `${baseUrl.replace(/\/$/, "")}/api/evaluations/${evaluationId}/runs/import`;
+    try {
+        const res = await fetch(url, {
+            method: "POST",
+            headers,
+            body: JSON.stringify(body),
+        });
+        const text = await res.text();
+        if (!res.ok) {
+            return { ok: false, status: res.status, body: text };
+        }
+        const data = JSON.parse(text);
+        return { ok: true, runId: data.runId };
+    }
+    catch (err) {
+        const msg = err instanceof Error ? err.message : String(err);
+        return { ok: false, status: 0, body: msg };
+    }
+}

package/dist/cli/check.d.ts

@@ -14,10 +14,14 @@
  *   --minN <n>           Fail if total test cases < n (low sample size)
  *   --allowWeakEvidence  If false (default), fail when evidenceLevel is 'weak'
  *   --policy <name>      Enforce a compliance policy (e.g. HIPAA, SOC2, GDPR)
- *   --baseline <mode>    Baseline comparison mode: "published" (default), "previous", or "production"
+ *   --baseline <mode>   Baseline comparison mode: "published" (default), "previous", or "production"
  *   --evaluationId <id>  Required. The evaluation to gate on.
  *   --baseUrl <url>      API base URL (default: EVALAI_BASE_URL or http://localhost:3000)
  *   --apiKey <key>       API key (default: EVALAI_API_KEY env var)
+ *   --share <mode>       Share link: "always" | "fail" | "never" (default: never)
+ *                        fail = create public share link only when gate fails (CI-friendly)
+ *   --pr-comment-out <file>  Write PR comment markdown to file (for GitHub Action to post)
+ *   --profile <name>         Preset: strict (95/0/30), balanced (90/2/10), fast (85/5/5). Explicit flags override.
  *
  * Exit codes:
  *   0  — Gate passed
@@ -28,31 +32,42 @@
  *   5  — Invalid arguments
  *   6  — Gate failed: total test cases < minN
  *   7  — Gate failed: weak evidence (evidenceLevel === 'weak')
+ *   8  — Gate warned: near-regression (warnDrop ≤ drop < maxDrop)
  *
  * Environment:
  *   EVALAI_BASE_URL  — API base URL (default: http://localhost:3000)
  *   EVALAI_API_KEY   — API key for authentication
  */
-export declare const EXIT: {
-    readonly PASS: 0;
-    readonly SCORE_BELOW: 1;
-    readonly REGRESSION: 2;
-    readonly POLICY_VIOLATION: 3;
-    readonly API_ERROR: 4;
-    readonly BAD_ARGS: 5;
-    readonly LOW_N: 6;
-    readonly WEAK_EVIDENCE: 7;
-};
+export { EXIT } from "./constants";
+export type FormatType = "human" | "json" | "github";
+export type ShareMode = "always" | "fail" | "never";
 export interface CheckArgs {
     baseUrl: string;
     apiKey: string;
     minScore: number;
     maxDrop?: number;
+    warnDrop?: number;
     minN?: number;
     allowWeakEvidence: boolean;
     evaluationId: string;
     policy?: string;
-    baseline: 'published' | 'previous' | 'production';
+    baseline: "published" | "previous" | "production" | "auto";
+    format: FormatType;
+    explain: boolean;
+    onFail?: "import";
+    share: ShareMode;
+    prCommentOut?: string;
+    maxCostUsd?: number;
+    maxLatencyMs?: number;
+    maxCostDeltaUsd?: number;
 }
-export declare function parseArgs(argv: string[]): CheckArgs;
+export type ParseArgsResult = {
+    ok: true;
+    args: CheckArgs;
+} | {
+    ok: false;
+    exitCode: number;
+    message: string;
+};
+export declare function parseArgs(argv: string[]): ParseArgsResult;
 export declare function runCheck(args: CheckArgs): Promise<number>;