@pauly4010/evalai-sdk 1.4.1 → 1.5.5

package/dist/cli/policy-packs.js

@@ -0,0 +1,83 @@
+"use strict";
+/**
+ * Versioned policy packs for evalai check --policy.
+ * Schema: policyId, version, thresholds, rationale, checks.
+ * Usage: --policy HIPAA@1
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.POLICY_PACKS = void 0;
+exports.resolvePolicyPack = resolvePolicyPack;
+exports.getValidPolicyVersions = getValidPolicyVersions;
+exports.POLICY_PACKS = {
+    HIPAA: {
+        1: {
+            policyId: "HIPAA",
+            version: 1,
+            thresholds: { requiredSafetyRate: 0.99, maxFlags: ["SAFETY_RISK"] },
+            rationale: "HIPAA requires high safety and no safety risks for PHI handling.",
+            checks: ["safety_rate", "no_safety_flags"],
+        },
+    },
+    SOC2: {
+        1: {
+            policyId: "SOC2",
+            version: 1,
+            thresholds: { requiredSafetyRate: 0.95, maxFlags: ["SAFETY_RISK", "LOW_PASS_RATE"] },
+            rationale: "SOC2 trust criteria for security and availability.",
+            checks: ["safety_rate", "flag_restrictions"],
+        },
+    },
+    GDPR: {
+        1: {
+            policyId: "GDPR",
+            version: 1,
+            thresholds: { requiredSafetyRate: 0.95, maxFlags: ["SAFETY_RISK"] },
+            rationale: "GDPR data protection and privacy requirements.",
+            checks: ["safety_rate", "no_safety_flags"],
+        },
+    },
+    PCI_DSS: {
+        1: {
+            policyId: "PCI_DSS",
+            version: 1,
+            thresholds: { requiredSafetyRate: 0.99, maxFlags: ["SAFETY_RISK", "LOW_PASS_RATE"] },
+            rationale: "PCI DSS cardholder data security standards.",
+            checks: ["safety_rate", "flag_restrictions"],
+        },
+    },
+    FINRA_4511: {
+        1: {
+            policyId: "FINRA_4511",
+            version: 1,
+            thresholds: { requiredSafetyRate: 0.95, maxFlags: ["SAFETY_RISK"] },
+            rationale: "FINRA 4511 supervisory control requirements.",
+            checks: ["safety_rate", "no_safety_flags"],
+        },
+    },
+};
+/**
+ * Parse --policy flag (e.g. "HIPAA@1" or "HIPAA") and resolve to PolicyPack.
+ * Default version is 1 when omitted.
+ */
+function resolvePolicyPack(spec) {
+    const at = spec.indexOf("@");
+    const policyId = (at >= 0 ? spec.slice(0, at) : spec).toUpperCase();
+    const version = at >= 0 ? parseInt(spec.slice(at + 1), 10) : 1;
+    if (Number.isNaN(version) || version < 1)
+        return null;
+    const versions = exports.POLICY_PACKS[policyId];
+    if (!versions)
+        return null;
+    const pack = versions[version];
+    return pack ?? null;
+}
+/** List valid policy@version specs for error messages */
+function getValidPolicyVersions() {
+    const out = [];
+    for (const [policyId, versions] of Object.entries(exports.POLICY_PACKS)) {
+        for (const v of Object.keys(versions)) {
+            out.push(`${policyId}@${v}`);
+        }
+    }
+    return out.sort();
+}

package/dist/cli/profiles.d.ts

@@ -0,0 +1,28 @@
+/**
+ * Gate profile presets: strict, balanced, fast.
+ * Profiles override flags unless explicitly set.
+ */
+export declare const PROFILES: {
+    readonly strict: {
+        readonly minScore: 95;
+        readonly maxDrop: 0;
+        readonly warnDrop: 0;
+        readonly minN: 30;
+        readonly allowWeakEvidence: false;
+    };
+    readonly balanced: {
+        readonly minScore: 90;
+        readonly maxDrop: 2;
+        readonly warnDrop: 1;
+        readonly minN: 10;
+        readonly allowWeakEvidence: false;
+    };
+    readonly fast: {
+        readonly minScore: 85;
+        readonly maxDrop: 5;
+        readonly warnDrop: 2;
+        readonly minN: 5;
+        readonly allowWeakEvidence: true;
+    };
+};
+export type ProfileName = keyof typeof PROFILES;

package/dist/cli/profiles.js

@@ -0,0 +1,30 @@
+"use strict";
+/**
+ * Gate profile presets: strict, balanced, fast.
+ * Profiles override flags unless explicitly set.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.PROFILES = void 0;
+exports.PROFILES = {
+    strict: {
+        minScore: 95,
+        maxDrop: 0,
+        warnDrop: 0,
+        minN: 30,
+        allowWeakEvidence: false,
+    },
+    balanced: {
+        minScore: 90,
+        maxDrop: 2,
+        warnDrop: 1,
+        minN: 10,
+        allowWeakEvidence: false,
+    },
+    fast: {
+        minScore: 85,
+        maxDrop: 5,
+        warnDrop: 2,
+        minN: 5,
+        allowWeakEvidence: true,
+    },
+};

package/dist/cli/reason-codes.d.ts

@@ -0,0 +1,17 @@
+/**
+ * Canonical reason codes for gate evaluation.
+ * Used by gate.ts and formatters for consistent failure classification.
+ */
+export declare const REASON_CODES: {
+    readonly PASS: "PASS";
+    readonly WARN_REGRESSION: "WARN_REGRESSION";
+    readonly LOW_SAMPLE_SIZE: "LOW_SAMPLE_SIZE";
+    readonly BASELINE_MISSING: "BASELINE_MISSING";
+    readonly SCORE_TOO_LOW: "SCORE_TOO_LOW";
+    readonly DELTA_TOO_HIGH: "DELTA_TOO_HIGH";
+    readonly COST_BUDGET_EXCEEDED: "COST_BUDGET_EXCEEDED";
+    readonly LATENCY_BUDGET_EXCEEDED: "LATENCY_BUDGET_EXCEEDED";
+    readonly POLICY_FAILED: "POLICY_FAILED";
+    readonly UNKNOWN: "UNKNOWN";
+};
+export type ReasonCode = (typeof REASON_CODES)[keyof typeof REASON_CODES];

package/dist/cli/reason-codes.js

@@ -0,0 +1,19 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.REASON_CODES = void 0;
+/**
+ * Canonical reason codes for gate evaluation.
+ * Used by gate.ts and formatters for consistent failure classification.
+ */
+exports.REASON_CODES = {
+    PASS: "PASS",
+    WARN_REGRESSION: "WARN_REGRESSION",
+    LOW_SAMPLE_SIZE: "LOW_SAMPLE_SIZE",
+    BASELINE_MISSING: "BASELINE_MISSING",
+    SCORE_TOO_LOW: "SCORE_TOO_LOW",
+    DELTA_TOO_HIGH: "DELTA_TOO_HIGH",
+    COST_BUDGET_EXCEEDED: "COST_BUDGET_EXCEEDED",
+    LATENCY_BUDGET_EXCEEDED: "LATENCY_BUDGET_EXCEEDED",
+    POLICY_FAILED: "POLICY_FAILED",
+    UNKNOWN: "UNKNOWN",
+};

package/dist/cli/render/snippet.d.ts

@@ -0,0 +1,5 @@
+/**
+ * Truncate a string for deterministic output.
+ * Replaces newlines with space, caps length.
+ */
+export declare function truncateSnippet(s: string | undefined | null, maxLen?: number): string;

package/dist/cli/render/snippet.js

@@ -0,0 +1,15 @@
+"use strict";
+/**
+ * Truncate a string for deterministic output.
+ * Replaces newlines with space, caps length.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.truncateSnippet = truncateSnippet;
+function truncateSnippet(s, maxLen = 140) {
+    if (s == null)
+        return "";
+    const normalized = s.replace(/\s+/g, " ").trim();
+    if (normalized.length <= maxLen)
+        return normalized;
+    return `${normalized.slice(0, maxLen)}…`;
+}

package/dist/cli/render/sort.d.ts

@@ -0,0 +1,10 @@
+/**
+ * Deterministic ordering for failed cases.
+ * Sort by status severity (failed > error > skipped > passed), then by testCaseId asc.
+ */
+export interface SortableCase {
+    status?: string;
+    testCaseId?: number;
+    [key: string]: unknown;
+}
+export declare function sortFailedCases<T extends SortableCase>(cases: T[]): T[];

package/dist/cli/render/sort.js

@@ -0,0 +1,24 @@
+"use strict";
+/**
+ * Deterministic ordering for failed cases.
+ * Sort by status severity (failed > error > skipped > passed), then by testCaseId asc.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.sortFailedCases = sortFailedCases;
+const STATUS_SEVERITY = {
+    failed: 0,
+    error: 1,
+    skipped: 2,
+    passed: 3,
+};
+function sortFailedCases(cases) {
+    return [...cases].sort((a, b) => {
+        const sevA = STATUS_SEVERITY[a.status?.toLowerCase() ?? ""] ?? 4;
+        const sevB = STATUS_SEVERITY[b.status?.toLowerCase() ?? ""] ?? 4;
+        if (sevA !== sevB)
+            return sevA - sevB;
+        const idA = a.testCaseId ?? 0;
+        const idB = b.testCaseId ?? 0;
+        return idA - idB;
+    });
+}

package/dist/cli/report/build-check-report.d.ts

@@ -0,0 +1,19 @@
+/**
+ * Build CheckReport from API data and gate result.
+ * Normalizes failed cases (truncate, sort), dashboard URL, top N + more.
+ */
+import type { QualityLatestData, RunDetailsData } from "../api";
+import type { CheckArgs } from "../check";
+import type { CheckReport } from "../formatters/types";
+import type { GateResult } from "../gate";
+export type BuildReportInput = {
+    args: CheckArgs;
+    quality: QualityLatestData;
+    runDetails?: RunDetailsData | null;
+    gateResult: GateResult;
+    requestId?: string;
+    shareUrl?: string;
+    baselineRunId?: number | null;
+    ciRunUrl?: string | null;
+};
+export declare function buildCheckReport(input: BuildReportInput): CheckReport;

package/dist/cli/report/build-check-report.js

@@ -0,0 +1,124 @@
+"use strict";
+/**
+ * Build CheckReport from API data and gate result.
+ * Normalizes failed cases (truncate, sort), dashboard URL, top N + more.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.buildCheckReport = buildCheckReport;
+const snippet_1 = require("../render/snippet");
+const sort_1 = require("../render/sort");
+const TOP_N = 3;
+/** ContribPts from weights: passRate*50, safety*25, (0.6*judge+0.4*schema)*15, (0.6*latency+0.4*cost)*10 */
+function computeContribPts(b) {
+    const pr = b.passRate ?? 0;
+    const s = b.safety ?? 0;
+    const j = b.judge ?? 0;
+    const sc = b.schema ?? 0;
+    const l = b.latency ?? 0;
+    const c = b.cost ?? 0;
+    return {
+        passRatePts: Math.round(pr * 50 * 10) / 10,
+        safetyPts: Math.round(s * 25 * 10) / 10,
+        compliancePts: Math.round((0.6 * j + 0.4 * sc) * 15 * 10) / 10,
+        performancePts: Math.round((0.6 * l + 0.4 * c) * 10 * 10) / 10,
+    };
+}
+const SNIPPET_MAX = 50;
+function buildCheckReport(input) {
+    const { args, quality, runDetails, gateResult, requestId } = input;
+    const score = quality?.score ?? 0;
+    const total = quality?.total ?? null;
+    const baselineScore = quality?.baselineScore ?? null;
+    const regressionDelta = quality?.regressionDelta ?? null;
+    const evaluationRunId = quality?.evaluationRunId;
+    const breakdown = quality?.breakdown ?? {};
+    const flags = (quality?.flags ?? []);
+    const baseUrl = args.baseUrl.replace(/\/$/, "");
+    const dashboardUrl = evaluationRunId != null
+        ? `${baseUrl}/evaluations/${args.evaluationId}/runs/${evaluationRunId}`
+        : undefined;
+    // Build failed cases from run details
+    let failedCases = [];
+    if (runDetails?.results && evaluationRunId != null) {
+        const raw = runDetails.results
+            .filter((r) => r.status === "failed")
+            .map((r) => ({
+            testCaseId: r.testCaseId,
+            status: "failed",
+            name: r.test_cases?.name,
+            input: r.test_cases?.input,
+            expectedOutput: r.test_cases?.expectedOutput,
+            output: r.output,
+        }));
+        failedCases = (0, sort_1.sortFailedCases)(raw).map((fc) => ({
+            ...fc,
+            inputSnippet: (0, snippet_1.truncateSnippet)(fc.input, SNIPPET_MAX),
+            expectedSnippet: (0, snippet_1.truncateSnippet)(fc.expectedOutput, SNIPPET_MAX),
+            outputSnippet: (0, snippet_1.truncateSnippet)(fc.output, SNIPPET_MAX),
+        }));
+    }
+    const failedCasesShown = Math.min(failedCases.length, TOP_N);
+    const failedCasesMore = failedCases.length - failedCasesShown;
+    const breakdown01 = Object.keys(breakdown).length > 0 ? breakdown : undefined;
+    const contribPts = args.explain && breakdown01 ? computeContribPts(breakdown01) : undefined;
+    const gateSkipped = gateResult.gateSkipped === true;
+    const gateApplied = !gateSkipped;
+    const gateMode = gateSkipped ? "neutral" : "enforced";
+    const actionableMessage = gateSkipped
+        ? "Gate not applied: baseline missing. Publish a baseline from the dashboard, or run with --baseline previous once you have runs."
+        : (gateResult.reasonMessage ?? undefined);
+    const verdict = gateResult.reasonCode === "WARN_REGRESSION" ? "warn" : gateResult.passed ? "pass" : "fail";
+    const report = {
+        evaluationId: args.evaluationId,
+        runId: evaluationRunId,
+        verdict,
+        gateApplied,
+        gateMode,
+        actionableMessage,
+        shareUrl: input.shareUrl,
+        policy: args.policy,
+        baselineRunId: input.baselineRunId ?? quality?.baselineRunId ?? undefined,
+        ciRunUrl: input.ciRunUrl ?? undefined,
+        reasonCode: gateResult.reasonCode,
+        reasonMessage: gateResult.reasonMessage ?? undefined,
+        score,
+        baselineScore: baselineScore ?? undefined,
+        delta: regressionDelta ?? undefined,
+        n: total ?? undefined,
+        evidenceLevel: quality?.evidenceLevel ?? undefined,
+        baselineMissing: quality?.baselineMissing === true,
+        baselineStatus: quality?.baselineMissing === true
+            ? "missing"
+            : quality?.baselineScore != null
+                ? "found"
+                : undefined,
+        flags: flags.length > 0 ? [...flags].sort() : undefined,
+        breakdown01,
+        contribPts,
+        thresholds: {
+            minScore: args.minScore,
+            maxDrop: args.maxDrop,
+            warnDrop: args.warnDrop,
+            minN: args.minN,
+            allowWeakEvidence: args.allowWeakEvidence,
+            baseline: args.baseline,
+            maxCostUsd: args.maxCostUsd,
+            maxLatencyMs: args.maxLatencyMs,
+            maxCostDeltaUsd: args.maxCostDeltaUsd,
+        },
+        dashboardUrl,
+        failedCases,
+        failedCasesShown: failedCases.length > 0 ? failedCasesShown : undefined,
+        failedCasesMore: failedCasesMore > 0 ? failedCasesMore : undefined,
+        requestId,
+        explain: args.explain,
+        policyEvidence: args.explain && gateResult.policyEvidence
+            ? {
+                failedCheck: gateResult.policyEvidence.failedCheck,
+                remediation: gateResult.policyEvidence.remediation,
+                snapshot: gateResult.policyEvidence.snapshot,
+            }
+            : undefined,
+    };
+    return report;
+}

package/dist/cli/share.d.ts

@@ -0,0 +1,17 @@
+/**
+ * evalai share — Create a share link for a run.
+ * Usage: evalai share --scope run --expires 7d
+ */
+export type ShareArgs = {
+    baseUrl: string;
+    apiKey: string;
+    evaluationId: string;
+    runId: number;
+    scope: "run";
+    expires: string;
+    expiresInDays: number;
+};
+export declare function parseShareArgs(argv: string[]): ShareArgs | {
+    error: string;
+};
+export declare function runShare(args: ShareArgs): Promise<number>;

package/dist/cli/share.js

@@ -0,0 +1,83 @@
+"use strict";
+/**
+ * evalai share — Create a share link for a run.
+ * Usage: evalai share --scope run --expires 7d
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.parseShareArgs = parseShareArgs;
+exports.runShare = runShare;
+const api_1 = require("./api");
+function parseExpires(spec) {
+    const m = spec.match(/^(\d+)(d|h|m|s)$/i);
+    if (!m)
+        return null;
+    const n = parseInt(m[1], 10);
+    const unit = m[2].toLowerCase();
+    if (unit === "d")
+        return n;
+    if (unit === "h")
+        return n / 24;
+    if (unit === "m")
+        return n / (24 * 60);
+    if (unit === "s")
+        return n / (24 * 60 * 60);
+    return null;
+}
+function parseShareArgs(argv) {
+    const args = {};
+    for (let i = 0; i < argv.length; i++) {
+        const arg = argv[i];
+        if (arg.startsWith("--")) {
+            const key = arg.slice(2);
+            const next = argv[i + 1];
+            if (next !== undefined && !next.startsWith("--")) {
+                args[key] = next;
+                i++;
+            }
+            else {
+                args[key] = "true";
+            }
+        }
+    }
+    const baseUrl = args.baseUrl || process.env.EVALAI_BASE_URL || "http://localhost:3000";
+    const apiKey = args.apiKey || process.env.EVALAI_API_KEY || "";
+    const evaluationId = args.evaluationId || "";
+    const runId = args.runId ? parseInt(args.runId, 10) : NaN;
+    const scope = args.scope === "run" ? "run" : "run";
+    const expires = args.expires || "7d";
+    if (!apiKey)
+        return { error: "Error: --apiKey or EVALAI_API_KEY is required" };
+    if (!evaluationId)
+        return { error: "Error: --evaluationId is required" };
+    if (Number.isNaN(runId) || runId < 1)
+        return { error: "Error: --runId is required and must be a positive number" };
+    const expiresInDays = parseExpires(expires);
+    if (expiresInDays == null || expiresInDays <= 0)
+        return { error: "Error: --expires must be e.g. 7d, 24h, 60m, 1s" };
+    return {
+        baseUrl,
+        apiKey,
+        evaluationId,
+        runId,
+        scope,
+        expires,
+        expiresInDays,
+    };
+}
+async function runShare(args) {
+    const exportRes = await (0, api_1.fetchRunExport)(args.baseUrl, args.apiKey, args.evaluationId, args.runId);
+    if (!exportRes.ok) {
+        console.error(`EvalAI share: failed to fetch export — ${exportRes.status} ${exportRes.body}`);
+        return 1;
+    }
+    const publishRes = await (0, api_1.publishShare)(args.baseUrl, args.apiKey, args.evaluationId, exportRes.exportData, args.runId, { expiresInDays: args.expiresInDays });
+    if (!publishRes.ok) {
+        console.error(`EvalAI share: failed to publish — ${publishRes.status} ${publishRes.body}`);
+        return 1;
+    }
+    const shareUrl = publishRes.data.shareUrl ??
+        `${args.baseUrl.replace(/\/$/, "")}/share/${publishRes.data.shareId}`;
+    console.log(`Share link created (expires in ${args.expires}):`);
+    console.log(shareUrl);
+    return 0;
+}

package/dist/client.d.ts

@@ -1,5 +1,5 @@
-import { ClientConfig, Trace, CreateTraceParams, ListTracesParams, Evaluation, CreateEvaluationParams, UpdateEvaluationParams, ListEvaluationsParams, LLMJudgeResult, RunLLMJudgeParams, TestCase, CreateTestCaseParams, EvaluationRun, CreateRunParams, Span, CreateSpanParams, UpdateTraceParams, OrganizationLimits, Annotation, CreateAnnotationParams, ListAnnotationsParams, AnnotationTask, CreateAnnotationTaskParams, ListAnnotationTasksParams, AnnotationItem, CreateAnnotationItemParams, ListAnnotationItemsParams, APIKey, APIKeyWithSecret, CreateAPIKeyParams, UpdateAPIKeyParams, ListAPIKeysParams, APIKeyUsage, Webhook, CreateWebhookParams, UpdateWebhookParams, ListWebhooksParams, WebhookDelivery, ListWebhookDeliveriesParams, UsageStats, GetUsageParams, UsageSummary, LLMJudgeConfig, CreateLLMJudgeConfigParams, ListLLMJudgeConfigsParams, ListLLMJudgeResultsParams, LLMJudgeAlignment, GetLLMJudgeAlignmentParams, Organization } from './types';
-import { Logger } from './logger';
+import { type Logger } from "./logger";
+import type { Annotation, AnnotationItem, AnnotationTask, APIKey, APIKeyUsage, APIKeyWithSecret, ClientConfig, CreateAnnotationItemParams, CreateAnnotationParams, CreateAnnotationTaskParams, CreateAPIKeyParams, CreateEvaluationParams, CreateLLMJudgeConfigParams, CreateRunParams, CreateSpanParams, CreateTestCaseParams, CreateTraceParams, CreateWebhookParams, Evaluation, EvaluationRun, GetLLMJudgeAlignmentParams, GetUsageParams, ListAnnotationItemsParams, ListAnnotationsParams, ListAnnotationTasksParams, ListAPIKeysParams, ListEvaluationsParams, ListLLMJudgeConfigsParams, ListLLMJudgeResultsParams, ListTracesParams, ListWebhookDeliveriesParams, ListWebhooksParams, LLMJudgeAlignment, LLMJudgeConfig, LLMJudgeResult, Organization, OrganizationLimits, RunLLMJudgeParams, Span, TestCase, Trace, UpdateAPIKeyParams, UpdateEvaluationParams, UpdateTraceParams, UpdateWebhookParams, UsageStats, UsageSummary, Webhook, WebhookDelivery } from "./types";
 /**
  * AI Evaluation Platform SDK Client
  *