@pauly4010/evalai-sdk 1.9.0 → 1.9.1

package/dist/cli/regression-gate.js

@@ -137,7 +137,10 @@ function runBuiltinGate(cwd) {
         };
     }
     const baselineMeta = baselineData.updatedAt
-        ? { updatedAt: baselineData.updatedAt, updatedBy: baselineData.updatedBy ?? "unknown" }
+        ? {
+            updatedAt: baselineData.updatedAt,
+            updatedBy: baselineData.updatedBy ?? "unknown",
+        }
         : null;
     // Run tests
     const isWin = process.platform === "win32";
@@ -302,7 +305,10 @@ function runGate(argv) {
                 process.stdout.write(fs.readFileSync(reportPath, "utf-8"));
             }
             else {
-                console.error(JSON.stringify({ error: "regression-report.json not found", exitCode }));
+                console.error(JSON.stringify({
+                    error: "regression-report.json not found",
+                    exitCode,
+                }));
             }
         }
         else if (args.format === "github") {

package/dist/cli/report/build-check-report.js

@@ -60,7 +60,9 @@ function buildCheckReport(input) {
     }
     const failedCasesShown = Math.min(failedCases.length, TOP_N);
     const failedCasesMore = failedCases.length - failedCasesShown;
-    const breakdown01 = Object.keys(breakdown).length > 0 ? breakdown : undefined;
+    const breakdown01 = Object.keys(breakdown).length > 0
+        ? breakdown
+        : undefined;
     const contribPts = args.explain && breakdown01 ? computeContribPts(breakdown01) : undefined;
     const gateSkipped = gateResult.gateSkipped === true;
     const gateApplied = !gateSkipped;
@@ -68,7 +70,11 @@ function buildCheckReport(input) {
     const actionableMessage = gateSkipped
         ? "Gate not applied: baseline missing. Publish a baseline from the dashboard, or run with --baseline previous once you have runs."
         : (gateResult.reasonMessage ?? undefined);
-    const verdict = gateResult.reasonCode === "WARN_REGRESSION" ? "warn" : gateResult.passed ? "pass" : "fail";
+    const verdict = gateResult.reasonCode === "WARN_REGRESSION"
+        ? "warn"
+        : gateResult.passed
+            ? "pass"
+            : "fail";
     const report = {
         schemaVersion: types_1.CHECK_REPORT_SCHEMA_VERSION,
         evaluationId: args.evaluationId,

package/dist/cli/run.js

@@ -49,9 +49,9 @@ exports.runEvaluations = runEvaluations;
 exports.printHumanResults = printHumanResults;
 exports.printJsonResults = printJsonResults;
 exports.runEvaluationsCLI = runEvaluationsCLI;
+const node_child_process_1 = require("node:child_process");
 const fs = __importStar(require("node:fs/promises"));
 const path = __importStar(require("node:path"));
-const node_child_process_1 = require("node:child_process");
 const impact_analysis_1 = require("./impact-analysis");
 /**
  * Generate deterministic run ID
@@ -133,7 +133,7 @@ async function loadManifest(projectRoot = process.cwd()) {
         const content = await fs.readFile(manifestPath, "utf-8");
         return JSON.parse(content);
     }
-    catch (error) {
+    catch (_error) {
         return null;
     }
 }
@@ -279,7 +279,7 @@ async function updateRunIndex(result, projectRoot = process.cwd()) {
         const existingContent = await fs.readFile(indexPath, "utf-8");
         index = JSON.parse(existingContent);
     }
-    catch (error) {
+    catch (_error) {
         // Index doesn't exist yet, start with empty array
     }
     // Add new entry
@@ -350,8 +350,14 @@ function printHumanResults(result) {
     console.log(`   📊 Pass Rate: ${(result.summary.passRate * 100).toFixed(1)}%`);
     console.log("\n📋 Individual Results:");
     for (const spec of result.results) {
-        const status = spec.result.status === "passed" ? "✅" : spec.result.status === "failed" ? "❌" : "⏭️";
-        const score = spec.result.score ? ` (${(spec.result.score * 100).toFixed(1)}%)` : "";
+        const status = spec.result.status === "passed"
+            ? "✅"
+            : spec.result.status === "failed"
+                ? "❌"
+                : "⏭️";
+        const score = spec.result.score
+            ? ` (${(spec.result.score * 100).toFixed(1)}%)`
+            : "";
         const error = spec.result.error ? ` - ${spec.result.error}` : "";
         console.log(`   ${status} ${spec.name}${score}${error}`);
     }

package/dist/cli/share.js

@@ -50,7 +50,9 @@ function parseShareArgs(argv) {
     if (!evaluationId)
         return { error: "Error: --evaluationId is required" };
     if (Number.isNaN(runId) || runId < 1)
-        return { error: "Error: --runId is required and must be a positive number" };
+        return {
+            error: "Error: --runId is required and must be a positive number",
+        };
     const expiresInDays = parseExpires(expires);
     if (expiresInDays == null || expiresInDays <= 0)
         return { error: "Error: --expires must be e.g. 7d, 24h, 60m, 1s" };

package/dist/cli/upgrade.js

@@ -275,7 +275,8 @@ function addNpmScripts(cwd) {
         changed = true;
     }
     if (!scripts["eval:baseline-update"]) {
-        scripts["eval:baseline-update"] = "npx tsx scripts/regression-gate.ts --update-baseline";
+        scripts["eval:baseline-update"] =
+            "npx tsx scripts/regression-gate.ts --update-baseline";
         changed = true;
     }
     if (changed) {

package/dist/client.d.ts

@@ -1,5 +1,5 @@
 import { type Logger } from "./logger";
-import type { Annotation, AnnotationItem, AnnotationTask, APIKey, APIKeyUsage, APIKeyWithSecret, ClientConfig, CreateAnnotationItemParams, CreateAnnotationParams, CreateAnnotationTaskParams, CreateAPIKeyParams, CreateEvaluationParams, CreateLLMJudgeConfigParams, CreateRunParams, CreateSpanParams, CreateTestCaseParams, CreateTraceParams, CreateWebhookParams, Evaluation, EvaluationRun, GetLLMJudgeAlignmentParams, GetUsageParams, ListAnnotationItemsParams, ListAnnotationsParams, ListAnnotationTasksParams, ListAPIKeysParams, ListEvaluationsParams, ListLLMJudgeConfigsParams, ListLLMJudgeResultsParams, ListTracesParams, ListWebhookDeliveriesParams, ListWebhooksParams, LLMJudgeAlignment, LLMJudgeConfig, LLMJudgeResult, Organization, OrganizationLimits, RunLLMJudgeParams, Span, TestCase, Trace, UpdateAPIKeyParams, UpdateEvaluationParams, UpdateTraceParams, UpdateWebhookParams, UsageStats, UsageSummary, Webhook, WebhookDelivery } from "./types";
+import type { Annotation, AnnotationItem, AnnotationTask, APIKey, APIKeyUsage, APIKeyWithSecret, ClientConfig, CreateAnnotationItemParams, CreateAnnotationParams, CreateAnnotationTaskParams, CreateAPIKeyParams, CreateEvaluationParams, CreateLLMJudgeConfigParams, CreateRunParams, CreateSpanParams, CreateTestCaseParams, CreateTraceParams, CreateWebhookParams, Evaluation, EvaluationRun, EvaluationRunDetail, GetLLMJudgeAlignmentParams, GetUsageParams, ListAnnotationItemsParams, ListAnnotationsParams, ListAnnotationTasksParams, ListAPIKeysParams, ListEvaluationsParams, ListLLMJudgeConfigsParams, ListLLMJudgeResultsParams, ListTracesParams, ListWebhookDeliveriesParams, ListWebhooksParams, LLMJudgeAlignment, LLMJudgeConfig, LLMJudgeEvaluateResult, LLMJudgeResult, Organization, OrganizationLimits, RunLLMJudgeParams, Span, TestCase, Trace, TraceDetail, UpdateAPIKeyParams, UpdateEvaluationParams, UpdateTraceParams, UpdateWebhookParams, UsageStats, UsageSummary, Webhook, WebhookDelivery } from "./types";
 /**
  * AI Evaluation Platform SDK Client
  *
@@ -81,15 +81,8 @@ export declare class AIEvalClient {
      */
     getLogger(): Logger;
     /**
-     * Get organization resource limits and usage
-     * Returns feature usage data for per-organization quotas
-     *
-     * @example
-     * ```typescript
-     * const limits = await client.getOrganizationLimits();
-     * console.log('Traces:', limits.traces_per_organization);
-     * console.log('Evaluations:', limits.evals_per_organization);
-     * ```
+     * @deprecated The /api/organizations/:id/limits endpoint does not exist.
+     * Use `organizations.getCurrent()` to get org info instead.
      */
     getOrganizationLimits(): Promise<OrganizationLimits>;
 }
@@ -123,9 +116,9 @@ declare class TraceAPI {
         message: string;
     }>;
     /**
-     * Get a single trace by ID
+     * Get a single trace by ID, including its spans
      */
-    get(id: number): Promise<Trace>;
+    get(id: number): Promise<TraceDetail>;
     /**
      * Update an existing trace (e.g. set status, duration, metadata on completion)
      *
@@ -193,9 +186,9 @@ declare class EvaluationAPI {
      */
     listRuns(evaluationId: number): Promise<EvaluationRun[]>;
     /**
-     * Get a specific run
+     * Get a specific run with its results
      */
-    getRun(evaluationId: number, runId: number): Promise<EvaluationRun>;
+    getRun(evaluationId: number, runId: number): Promise<EvaluationRunDetail>;
 }
 /**
  * LLM Judge API methods
@@ -207,8 +200,7 @@ declare class LLMJudgeAPI {
      * Run an LLM judge evaluation
      */
     evaluate(params: RunLLMJudgeParams): Promise<{
-        result: LLMJudgeResult;
-        config: unknown;
+        result: LLMJudgeEvaluateResult;
     }>;
     /**
      * Create an LLM judge configuration
@@ -289,11 +281,13 @@ declare class DeveloperAPI {
     /**
      * Get usage statistics
      */
-    getUsage(params: GetUsageParams): Promise<UsageStats>;
+    getUsage(params?: GetUsageParams): Promise<UsageStats>;
     /**
      * Get usage summary
      */
-    getUsageSummary(organizationId: number): Promise<UsageSummary>;
+    getUsageSummary(params?: {
+        period?: "7d" | "30d" | "90d" | "all";
+    }): Promise<UsageSummary>;
 }
 /**
  * API Keys API methods
@@ -355,7 +349,10 @@ declare class WebhooksAPI {
     /**
      * Get webhook deliveries
      */
-    getDeliveries(webhookId: number, params?: ListWebhookDeliveriesParams): Promise<WebhookDelivery[]>;
+    getDeliveries(webhookId: number, params?: ListWebhookDeliveriesParams): Promise<{
+        deliveries: WebhookDelivery[];
+        total: number;
+    }>;
 }
 /**
  * Organizations API methods

package/dist/client.js

@@ -44,17 +44,24 @@ class AIEvalClient {
     constructor(config = {}) {
         // Tier 1.1: Zero-config with env variable detection (works in Node.js and browsers)
         this.apiKey =
-            config.apiKey || getEnvVar("EVALAI_API_KEY") || getEnvVar("AI_EVAL_API_KEY") || "";
+            config.apiKey ||
+                getEnvVar("EVALAI_API_KEY") ||
+                getEnvVar("AI_EVAL_API_KEY") ||
+                "";
         if (!this.apiKey) {
             throw new errors_1.EvalAIError("API key is required. Provide via config.apiKey or EVALAI_API_KEY environment variable.", "MISSING_API_KEY", 0);
         }
         // Auto-detect organization ID from env
-        const orgIdFromEnv = getEnvVar("EVALAI_ORGANIZATION_ID") || getEnvVar("AI_EVAL_ORGANIZATION_ID");
+        const orgIdFromEnv = getEnvVar("EVALAI_ORGANIZATION_ID") ||
+            getEnvVar("AI_EVAL_ORGANIZATION_ID");
         this.organizationId =
-            config.organizationId || (orgIdFromEnv ? parseInt(orgIdFromEnv, 10) : undefined);
-        // Default to relative URLs for browser, or allow custom baseUrl
+            config.organizationId ||
+                (orgIdFromEnv ? parseInt(orgIdFromEnv, 10) : undefined);
         const isBrowser = typeof globalThis.window !== "undefined";
-        this.baseUrl = config.baseUrl || (isBrowser ? "" : "http://localhost:3000");
+        this.baseUrl =
+            config.baseUrl ||
+                getEnvVar("EVALAI_BASE_URL") ||
+                (isBrowser ? "" : "http://localhost:3000");
         this.timeout = config.timeout || 30000;
         // Tier 4.17: Debug mode with request logging
         const logLevel = config.logLevel || (config.debug ? "debug" : "info");
@@ -292,22 +299,11 @@ class AIEvalClient {
         return this.logger;
     }
     /**
-     * Get organization resource limits and usage
-     * Returns feature usage data for per-organization quotas
-     *
-     * @example
-     * ```typescript
-     * const limits = await client.getOrganizationLimits();
-     * console.log('Traces:', limits.traces_per_organization);
-     * console.log('Evaluations:', limits.evals_per_organization);
-     * ```
+     * @deprecated The /api/organizations/:id/limits endpoint does not exist.
+     * Use `organizations.getCurrent()` to get org info instead.
      */
     async getOrganizationLimits() {
-        const orgId = this.getOrganizationId();
-        if (!orgId) {
-            throw new errors_1.EvalAIError("Organization ID is required", "MISSING_ORGANIZATION_ID", 0);
-        }
-        return this.request(`/api/organizations/${orgId}/limits`);
+        return {};
     }
 }
 exports.AIEvalClient = AIEvalClient;
@@ -370,7 +366,7 @@ class TraceAPI {
         });
     }
     /**
-     * Get a single trace by ID
+     * Get a single trace by ID, including its spans
      */
     async get(id) {
         return this.client.request(`/api/traces/${id}`);
@@ -504,7 +500,7 @@ class EvaluationAPI {
         return this.client.request(`/api/evaluations/${evaluationId}/runs`);
     }
     /**
-     * Get a specific run
+     * Get a specific run with its results
      */
     async getRun(evaluationId, runId) {
         return this.client.request(`/api/evaluations/${evaluationId}/runs/${runId}`);
@@ -547,7 +543,9 @@ class LLMJudgeAPI {
         if (params.offset)
             searchParams.set("offset", params.offset.toString());
         const query = searchParams.toString();
-        const endpoint = query ? `/api/llm-judge/configs?${query}` : "/api/llm-judge/configs";
+        const endpoint = query
+            ? `/api/llm-judge/configs?${query}`
+            : "/api/llm-judge/configs";
         return this.client.request(endpoint);
     }
     /**
@@ -564,7 +562,9 @@ class LLMJudgeAPI {
         if (params.offset)
             searchParams.set("offset", params.offset.toString());
         const query = searchParams.toString();
-        const endpoint = query ? `/api/llm-judge/results?${query}` : "/api/llm-judge/results";
+        const endpoint = query
+            ? `/api/llm-judge/results?${query}`
+            : "/api/llm-judge/results";
         return this.client.request(endpoint);
     }
     /**
@@ -572,11 +572,7 @@ class LLMJudgeAPI {
      */
     async getAlignment(params) {
         const searchParams = new URLSearchParams();
-        searchParams.set("configId", params.configId.toString());
-        if (params.startDate)
-            searchParams.set("startDate", params.startDate);
-        if (params.endDate)
-            searchParams.set("endDate", params.endDate);
+        searchParams.set("evaluationRunId", params.evaluationRunId.toString());
         const query = searchParams.toString();
         return this.client.request(`/api/llm-judge/alignment?${query}`);
     }
@@ -651,14 +647,18 @@ class AnnotationTasksAPI {
         if (params.offset)
             searchParams.set("offset", params.offset.toString());
         const query = searchParams.toString();
-        const endpoint = query ? `/api/annotations/tasks?${query}` : "/api/annotations/tasks";
+        const endpoint = query
+            ? `/api/annotations/tasks?${query}`
+            : "/api/annotations/tasks";
         return this.client.request(endpoint);
     }
     /**
      * Get an annotation task
      */
     async get(taskId) {
-        return this.client.request(`/api/annotations/tasks/${taskId}`);
+        return this.client
+            .request(`/api/annotations/tasks/${taskId}`)
+            .then((res) => res.task);
     }
 }
 /**
@@ -705,21 +705,34 @@ class DeveloperAPI {
     /**
      * Get usage statistics
      */
-    async getUsage(params) {
+    async getUsage(params = {}) {
         const searchParams = new URLSearchParams();
-        searchParams.set("organizationId", params.organizationId.toString());
-        if (params.startDate)
-            searchParams.set("startDate", params.startDate);
-        if (params.endDate)
-            searchParams.set("endDate", params.endDate);
+        if (params.period)
+            searchParams.set("period", params.period);
+        if (params.groupBy)
+            searchParams.set("groupBy", params.groupBy);
+        if (params.limit)
+            searchParams.set("limit", params.limit.toString());
+        if (params.offset)
+            searchParams.set("offset", params.offset.toString());
         const query = searchParams.toString();
-        return this.client.request(`/api/developer/usage?${query}`);
+        const endpoint = query
+            ? `/api/developer/usage?${query}`
+            : "/api/developer/usage";
+        return this.client.request(endpoint);
     }
     /**
      * Get usage summary
      */
-    async getUsageSummary(organizationId) {
-        return this.client.request(`/api/developer/usage/summary?organizationId=${organizationId}`);
+    async getUsageSummary(params = {}) {
+        const searchParams = new URLSearchParams();
+        if (params.period)
+            searchParams.set("period", params.period);
+        const query = searchParams.toString();
+        const endpoint = query
+            ? `/api/developer/usage/summary?${query}`
+            : "/api/developer/usage/summary";
+        return this.client.request(endpoint);
     }
 }
 /**
@@ -750,7 +763,9 @@ class APIKeysAPI {
         if (params.offset)
             searchParams.set("offset", params.offset.toString());
         const query = searchParams.toString();
-        const endpoint = query ? `/api/developer/api-keys?${query}` : "/api/developer/api-keys";
+        const endpoint = query
+            ? `/api/developer/api-keys?${query}`
+            : "/api/developer/api-keys";
         return this.client.request(endpoint);
     }
     /**
@@ -840,8 +855,8 @@ class WebhooksAPI {
             searchParams.set("limit", params.limit.toString());
         if (params.offset)
             searchParams.set("offset", params.offset.toString());
-        if (params.success !== undefined)
-            searchParams.set("success", params.success.toString());
+        if (params.status)
+            searchParams.set("status", params.status);
         const query = searchParams.toString();
         const endpoint = query
             ? `/api/developer/webhooks/${webhookId}/deliveries?${query}`
@@ -860,6 +875,8 @@ class OrganizationsAPI {
      * Get current organization
      */
     async getCurrent() {
-        return this.client.request("/api/organizations/current");
+        return this.client
+            .request("/api/organizations/current")
+            .then((res) => res.organization);
     }
 }

package/dist/client.request.test.d.ts

@@ -1 +1 @@
-export {};
+export {};