@pauly4010/evalai-sdk 1.8.0 → 1.9.1

package/dist/cli/workspace.js

@@ -0,0 +1,58 @@
+"use strict";
+/**
+ * CORE-402: Centralized .evalai workspace resolution
+ *
+ * Provides unified workspace path resolution for all EvalAI CLI commands
+ */
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.resolveEvalWorkspace = resolveEvalWorkspace;
+const path = __importStar(require("node:path"));
+/**
+ * Resolve EvalAI workspace paths
+ */
+function resolveEvalWorkspace(projectRoot = process.cwd()) {
+    const evalaiDir = path.join(projectRoot, ".evalai");
+    const runsDir = path.join(evalaiDir, "runs");
+    return {
+        root: projectRoot,
+        evalaiDir,
+        runsDir,
+        manifestPath: path.join(evalaiDir, "manifest.json"),
+        lastRunPath: path.join(evalaiDir, "last-run.json"),
+        indexPath: path.join(runsDir, "index.json"),
+        baselinePath: path.join(evalaiDir, "baseline-run.json"),
+    };
+}

package/dist/client.d.ts

@@ -1,5 +1,5 @@
 import { type Logger } from "./logger";
-import type { Annotation, AnnotationItem, AnnotationTask, APIKey, APIKeyUsage, APIKeyWithSecret, ClientConfig, CreateAnnotationItemParams, CreateAnnotationParams, CreateAnnotationTaskParams, CreateAPIKeyParams, CreateEvaluationParams, CreateLLMJudgeConfigParams, CreateRunParams, CreateSpanParams, CreateTestCaseParams, CreateTraceParams, CreateWebhookParams, Evaluation, EvaluationRun, GetLLMJudgeAlignmentParams, GetUsageParams, ListAnnotationItemsParams, ListAnnotationsParams, ListAnnotationTasksParams, ListAPIKeysParams, ListEvaluationsParams, ListLLMJudgeConfigsParams, ListLLMJudgeResultsParams, ListTracesParams, ListWebhookDeliveriesParams, ListWebhooksParams, LLMJudgeAlignment, LLMJudgeConfig, LLMJudgeResult, Organization, OrganizationLimits, RunLLMJudgeParams, Span, TestCase, Trace, UpdateAPIKeyParams, UpdateEvaluationParams, UpdateTraceParams, UpdateWebhookParams, UsageStats, UsageSummary, Webhook, WebhookDelivery } from "./types";
+import type { Annotation, AnnotationItem, AnnotationTask, APIKey, APIKeyUsage, APIKeyWithSecret, ClientConfig, CreateAnnotationItemParams, CreateAnnotationParams, CreateAnnotationTaskParams, CreateAPIKeyParams, CreateEvaluationParams, CreateLLMJudgeConfigParams, CreateRunParams, CreateSpanParams, CreateTestCaseParams, CreateTraceParams, CreateWebhookParams, Evaluation, EvaluationRun, EvaluationRunDetail, GetLLMJudgeAlignmentParams, GetUsageParams, ListAnnotationItemsParams, ListAnnotationsParams, ListAnnotationTasksParams, ListAPIKeysParams, ListEvaluationsParams, ListLLMJudgeConfigsParams, ListLLMJudgeResultsParams, ListTracesParams, ListWebhookDeliveriesParams, ListWebhooksParams, LLMJudgeAlignment, LLMJudgeConfig, LLMJudgeEvaluateResult, LLMJudgeResult, Organization, OrganizationLimits, RunLLMJudgeParams, Span, TestCase, Trace, TraceDetail, UpdateAPIKeyParams, UpdateEvaluationParams, UpdateTraceParams, UpdateWebhookParams, UsageStats, UsageSummary, Webhook, WebhookDelivery } from "./types";
 /**
  * AI Evaluation Platform SDK Client
  *
@@ -81,15 +81,8 @@ export declare class AIEvalClient {
      */
     getLogger(): Logger;
     /**
-     * Get organization resource limits and usage
-     * Returns feature usage data for per-organization quotas
-     *
-     * @example
-     * ```typescript
-     * const limits = await client.getOrganizationLimits();
-     * console.log('Traces:', limits.traces_per_organization);
-     * console.log('Evaluations:', limits.evals_per_organization);
-     * ```
+     * @deprecated The /api/organizations/:id/limits endpoint does not exist.
+     * Use `organizations.getCurrent()` to get org info instead.
      */
     getOrganizationLimits(): Promise<OrganizationLimits>;
 }
@@ -123,9 +116,9 @@ declare class TraceAPI {
         message: string;
     }>;
     /**
-     * Get a single trace by ID
+     * Get a single trace by ID, including its spans
      */
-    get(id: number): Promise<Trace>;
+    get(id: number): Promise<TraceDetail>;
     /**
      * Update an existing trace (e.g. set status, duration, metadata on completion)
      *
@@ -193,9 +186,9 @@ declare class EvaluationAPI {
      */
     listRuns(evaluationId: number): Promise<EvaluationRun[]>;
     /**
-     * Get a specific run
+     * Get a specific run with its results
      */
-    getRun(evaluationId: number, runId: number): Promise<EvaluationRun>;
+    getRun(evaluationId: number, runId: number): Promise<EvaluationRunDetail>;
 }
 /**
  * LLM Judge API methods
@@ -207,8 +200,7 @@ declare class LLMJudgeAPI {
      * Run an LLM judge evaluation
      */
     evaluate(params: RunLLMJudgeParams): Promise<{
-        result: LLMJudgeResult;
-        config: unknown;
+        result: LLMJudgeEvaluateResult;
     }>;
     /**
      * Create an LLM judge configuration
@@ -289,11 +281,13 @@ declare class DeveloperAPI {
     /**
      * Get usage statistics
      */
-    getUsage(params: GetUsageParams): Promise<UsageStats>;
+    getUsage(params?: GetUsageParams): Promise<UsageStats>;
     /**
      * Get usage summary
      */
-    getUsageSummary(organizationId: number): Promise<UsageSummary>;
+    getUsageSummary(params?: {
+        period?: "7d" | "30d" | "90d" | "all";
+    }): Promise<UsageSummary>;
 }
 /**
  * API Keys API methods
@@ -355,7 +349,10 @@ declare class WebhooksAPI {
     /**
      * Get webhook deliveries
      */
-    getDeliveries(webhookId: number, params?: ListWebhookDeliveriesParams): Promise<WebhookDelivery[]>;
+    getDeliveries(webhookId: number, params?: ListWebhookDeliveriesParams): Promise<{
+        deliveries: WebhookDelivery[];
+        total: number;
+    }>;
 }
 /**
  * Organizations API methods

package/dist/client.js

@@ -44,17 +44,24 @@ class AIEvalClient {
     constructor(config = {}) {
         // Tier 1.1: Zero-config with env variable detection (works in Node.js and browsers)
         this.apiKey =
-            config.apiKey || getEnvVar("EVALAI_API_KEY") || getEnvVar("AI_EVAL_API_KEY") || "";
+            config.apiKey ||
+                getEnvVar("EVALAI_API_KEY") ||
+                getEnvVar("AI_EVAL_API_KEY") ||
+                "";
         if (!this.apiKey) {
             throw new errors_1.EvalAIError("API key is required. Provide via config.apiKey or EVALAI_API_KEY environment variable.", "MISSING_API_KEY", 0);
         }
         // Auto-detect organization ID from env
-        const orgIdFromEnv = getEnvVar("EVALAI_ORGANIZATION_ID") || getEnvVar("AI_EVAL_ORGANIZATION_ID");
+        const orgIdFromEnv = getEnvVar("EVALAI_ORGANIZATION_ID") ||
+            getEnvVar("AI_EVAL_ORGANIZATION_ID");
         this.organizationId =
-            config.organizationId || (orgIdFromEnv ? parseInt(orgIdFromEnv, 10) : undefined);
-        // Default to relative URLs for browser, or allow custom baseUrl
+            config.organizationId ||
+                (orgIdFromEnv ? parseInt(orgIdFromEnv, 10) : undefined);
         const isBrowser = typeof globalThis.window !== "undefined";
-        this.baseUrl = config.baseUrl || (isBrowser ? "" : "http://localhost:3000");
+        this.baseUrl =
+            config.baseUrl ||
+                getEnvVar("EVALAI_BASE_URL") ||
+                (isBrowser ? "" : "http://localhost:3000");
         this.timeout = config.timeout || 30000;
         // Tier 4.17: Debug mode with request logging
         const logLevel = config.logLevel || (config.debug ? "debug" : "info");
@@ -292,22 +299,11 @@ class AIEvalClient {
         return this.logger;
     }
     /**
-     * Get organization resource limits and usage
-     * Returns feature usage data for per-organization quotas
-     *
-     * @example
-     * ```typescript
-     * const limits = await client.getOrganizationLimits();
-     * console.log('Traces:', limits.traces_per_organization);
-     * console.log('Evaluations:', limits.evals_per_organization);
-     * ```
+     * @deprecated The /api/organizations/:id/limits endpoint does not exist.
+     * Use `organizations.getCurrent()` to get org info instead.
      */
     async getOrganizationLimits() {
-        const orgId = this.getOrganizationId();
-        if (!orgId) {
-            throw new errors_1.EvalAIError("Organization ID is required", "MISSING_ORGANIZATION_ID", 0);
-        }
-        return this.request(`/api/organizations/${orgId}/limits`);
+        return {};
     }
 }
 exports.AIEvalClient = AIEvalClient;
@@ -370,7 +366,7 @@ class TraceAPI {
         });
     }
     /**
-     * Get a single trace by ID
+     * Get a single trace by ID, including its spans
      */
     async get(id) {
         return this.client.request(`/api/traces/${id}`);
@@ -504,7 +500,7 @@ class EvaluationAPI {
         return this.client.request(`/api/evaluations/${evaluationId}/runs`);
     }
     /**
-     * Get a specific run
+     * Get a specific run with its results
      */
     async getRun(evaluationId, runId) {
         return this.client.request(`/api/evaluations/${evaluationId}/runs/${runId}`);
@@ -547,7 +543,9 @@ class LLMJudgeAPI {
         if (params.offset)
             searchParams.set("offset", params.offset.toString());
         const query = searchParams.toString();
-        const endpoint = query ? `/api/llm-judge/configs?${query}` : "/api/llm-judge/configs";
+        const endpoint = query
+            ? `/api/llm-judge/configs?${query}`
+            : "/api/llm-judge/configs";
         return this.client.request(endpoint);
     }
     /**
@@ -564,7 +562,9 @@ class LLMJudgeAPI {
         if (params.offset)
             searchParams.set("offset", params.offset.toString());
         const query = searchParams.toString();
-        const endpoint = query ? `/api/llm-judge/results?${query}` : "/api/llm-judge/results";
+        const endpoint = query
+            ? `/api/llm-judge/results?${query}`
+            : "/api/llm-judge/results";
         return this.client.request(endpoint);
     }
     /**
@@ -572,11 +572,7 @@ class LLMJudgeAPI {
      */
     async getAlignment(params) {
         const searchParams = new URLSearchParams();
-        searchParams.set("configId", params.configId.toString());
-        if (params.startDate)
-            searchParams.set("startDate", params.startDate);
-        if (params.endDate)
-            searchParams.set("endDate", params.endDate);
+        searchParams.set("evaluationRunId", params.evaluationRunId.toString());
         const query = searchParams.toString();
         return this.client.request(`/api/llm-judge/alignment?${query}`);
     }
@@ -651,14 +647,18 @@ class AnnotationTasksAPI {
         if (params.offset)
             searchParams.set("offset", params.offset.toString());
         const query = searchParams.toString();
-        const endpoint = query ? `/api/annotations/tasks?${query}` : "/api/annotations/tasks";
+        const endpoint = query
+            ? `/api/annotations/tasks?${query}`
+            : "/api/annotations/tasks";
         return this.client.request(endpoint);
     }
     /**
      * Get an annotation task
      */
     async get(taskId) {
-        return this.client.request(`/api/annotations/tasks/${taskId}`);
+        return this.client
+            .request(`/api/annotations/tasks/${taskId}`)
+            .then((res) => res.task);
     }
 }
 /**
@@ -705,21 +705,34 @@ class DeveloperAPI {
     /**
      * Get usage statistics
      */
-    async getUsage(params) {
+    async getUsage(params = {}) {
         const searchParams = new URLSearchParams();
-        searchParams.set("organizationId", params.organizationId.toString());
-        if (params.startDate)
-            searchParams.set("startDate", params.startDate);
-        if (params.endDate)
-            searchParams.set("endDate", params.endDate);
+        if (params.period)
+            searchParams.set("period", params.period);
+        if (params.groupBy)
+            searchParams.set("groupBy", params.groupBy);
+        if (params.limit)
+            searchParams.set("limit", params.limit.toString());
+        if (params.offset)
+            searchParams.set("offset", params.offset.toString());
         const query = searchParams.toString();
-        return this.client.request(`/api/developer/usage?${query}`);
+        const endpoint = query
+            ? `/api/developer/usage?${query}`
+            : "/api/developer/usage";
+        return this.client.request(endpoint);
     }
     /**
      * Get usage summary
      */
-    async getUsageSummary(organizationId) {
-        return this.client.request(`/api/developer/usage/summary?organizationId=${organizationId}`);
+    async getUsageSummary(params = {}) {
+        const searchParams = new URLSearchParams();
+        if (params.period)
+            searchParams.set("period", params.period);
+        const query = searchParams.toString();
+        const endpoint = query
+            ? `/api/developer/usage/summary?${query}`
+            : "/api/developer/usage/summary";
+        return this.client.request(endpoint);
     }
 }
 /**
@@ -750,7 +763,9 @@ class APIKeysAPI {
         if (params.offset)
             searchParams.set("offset", params.offset.toString());
         const query = searchParams.toString();
-        const endpoint = query ? `/api/developer/api-keys?${query}` : "/api/developer/api-keys";
+        const endpoint = query
+            ? `/api/developer/api-keys?${query}`
+            : "/api/developer/api-keys";
         return this.client.request(endpoint);
     }
     /**
@@ -840,8 +855,8 @@ class WebhooksAPI {
             searchParams.set("limit", params.limit.toString());
         if (params.offset)
             searchParams.set("offset", params.offset.toString());
-        if (params.success !== undefined)
-            searchParams.set("success", params.success.toString());
+        if (params.status)
+            searchParams.set("status", params.status);
         const query = searchParams.toString();
         const endpoint = query
             ? `/api/developer/webhooks/${webhookId}/deliveries?${query}`
@@ -860,6 +875,8 @@ class OrganizationsAPI {
      * Get current organization
      */
     async getCurrent() {
-        return this.client.request("/api/organizations/current");
+        return this.client
+            .request("/api/organizations/current")
+            .then((res) => res.organization);
     }
 }

package/dist/client.request.test.d.ts

@@ -1 +1 @@
-export {};
+export {};