@pauly4010/evalai-sdk 1.8.0 → 1.9.1

package/dist/testing.d.ts

@@ -90,6 +90,47 @@ export interface TestSuiteResult {
     /** Case IDs that were retried (flaky recovery) */
     retriedCases?: string[];
 }
+/**
+ * Test definition for introspection
+ * COMPAT-201: Public TestSuite introspection (minimal getters)
+ */
+export interface TestDefinition {
+    /** Test case ID */
+    id: string;
+    /** Test input */
+    input: string;
+    /** Expected output */
+    expected?: string;
+    /** Test metadata */
+    metadata?: Record<string, unknown>;
+    /** Whether test has assertions */
+    hasAssertions: boolean;
+    /** Number of assertions */
+    assertionCount: number;
+}
+/**
+ * Portable suite representation
+ * COMPAT-201: Public TestSuite introspection (minimal getters)
+ */
+export interface PortableSuite {
+    /** Suite name */
+    name: string;
+    /** Suite configuration */
+    config: TestSuiteConfig;
+    /** Test definitions */
+    tests: TestDefinition[];
+    /** Suite metadata */
+    metadata: {
+        suiteName?: string;
+        tags?: string[];
+        defaults?: {
+            timeout?: number;
+            parallel?: boolean;
+            stopOnFailure?: boolean;
+            retries?: number;
+        };
+    };
+}
 /**
  * Test Suite for declarative evaluation testing
  */
@@ -115,6 +156,30 @@ export declare class TestSuite {
      * Get suite configuration
      */
     getConfig(): TestSuiteConfig;
+    /**
+     * Get test definitions for introspection
+     * COMPAT-201: Public TestSuite introspection (minimal getters)
+     */
+    getTests(): TestDefinition[];
+    /**
+     * Get suite metadata for introspection
+     * COMPAT-201: Public TestSuite introspection (minimal getters)
+     */
+    getMetadata(): {
+        suiteName?: string;
+        tags?: string[];
+        defaults?: {
+            timeout?: number;
+            parallel?: boolean;
+            stopOnFailure?: boolean;
+            retries?: number;
+        };
+    };
+    /**
+     * Convert to portable suite representation
+     * COMPAT-201: Public TestSuite introspection (minimal getters)
+     */
+    toJSON(): PortableSuite;
 }
 /**
  * Create a test suite

package/dist/testing.js

@@ -59,7 +59,10 @@ class TestSuite {
                 if (this.config.executor) {
                     const timeout = this.config.timeout || 30000;
                     const timeoutPromise = new Promise((_, reject) => setTimeout(() => reject(new Error(`Test timeout after ${timeout}ms`)), timeout));
-                    actual = await Promise.race([this.config.executor(testCase.input), timeoutPromise]);
+                    actual = await Promise.race([
+                        this.config.executor(testCase.input),
+                        timeoutPromise,
+                    ]);
                 }
                 else if (testCase.expected) {
                     actual = testCase.expected; // Use expected as actual if no executor
@@ -127,7 +130,9 @@ class TestSuite {
         const retriedCases = [];
         const retries = this.config.retries ?? 0;
         if (retries > 0 && results.length > 0) {
-            const failingIndices = results.map((r, i) => (r.passed ? -1 : i)).filter((i) => i >= 0);
+            const failingIndices = results
+                .map((r, i) => (r.passed ? -1 : i))
+                .filter((i) => i >= 0);
             for (let attempt = 0; attempt < retries && failingIndices.length > 0; attempt++) {
                 const toRetry = [...failingIndices];
                 failingIndices.length = 0;
@@ -169,6 +174,48 @@ class TestSuite {
     getConfig() {
         return { ...this.config };
     }
+    /**
+     * Get test definitions for introspection
+     * COMPAT-201: Public TestSuite introspection (minimal getters)
+     */
+    getTests() {
+        return this.config.cases.map((testCase, index) => ({
+            id: testCase.id || `case-${index}`,
+            input: testCase.input,
+            expected: testCase.expected,
+            metadata: testCase.metadata,
+            hasAssertions: !!testCase.assertions && testCase.assertions.length > 0,
+            assertionCount: testCase.assertions?.length || 0,
+        }));
+    }
+    /**
+     * Get suite metadata for introspection
+     * COMPAT-201: Public TestSuite introspection (minimal getters)
+     */
+    getMetadata() {
+        return {
+            suiteName: this.name,
+            tags: [], // TestSuite doesn't have tags, but include for future compatibility
+            defaults: {
+                timeout: this.config.timeout,
+                parallel: this.config.parallel,
+                stopOnFailure: this.config.stopOnFailure,
+                retries: this.config.retries,
+            },
+        };
+    }
+    /**
+     * Convert to portable suite representation
+     * COMPAT-201: Public TestSuite introspection (minimal getters)
+     */
+    toJSON() {
+        return {
+            name: this.name,
+            config: this.getConfig(),
+            tests: this.getTests(),
+            metadata: this.getMetadata(),
+        };
+    }
 }
 exports.TestSuite = TestSuite;
 /**

package/dist/types.d.ts

@@ -150,11 +150,15 @@ export interface Span<TMetadata = Record<string, unknown>> {
 export interface CreateSpanParams<TMetadata = Record<string, unknown>> {
     name: string;
     spanId: string;
+    type: string;
     parentSpanId?: string;
     startTime: string;
     endTime?: string;
     durationMs?: number;
+    input?: unknown;
+    output?: unknown;
     metadata?: TMetadata;
+    evaluationRunId?: number | null;
 }
 /**
  * Evaluation object representing a test evaluation
@@ -227,17 +231,30 @@ export interface CreateTestCaseParams {
 export interface EvaluationRun {
     id: number;
     evaluationId: number;
-    status: "pending" | "running" | "completed" | "failed";
-    results: Record<string, unknown> | null;
-    createdAt: string;
+    organizationId: number;
+    status: string;
+    totalCases: number | null;
+    passedCases: number | null;
+    failedCases: number | null;
+    environment: string | null;
+    startedAt: string | null;
     completedAt: string | null;
+    createdAt: string;
+}
+/**
+ * Result of getRun — includes the run and its test results
+ */
+export interface EvaluationRunDetail {
+    run: EvaluationRun;
+    results: Array<Record<string, unknown>>;
+    baselineResults?: Array<Record<string, unknown>>;
+    compareRunId?: number;
 }
 /**
  * Parameters for creating an evaluation run
  */
 export interface CreateRunParams {
-    status?: "pending" | "running" | "completed" | "failed";
-    results?: Record<string, unknown>;
+    environment?: string;
 }
 /**
  * LLM Judge evaluation result
@@ -252,6 +269,15 @@ export interface LLMJudgeResult {
     metadata: Record<string, unknown> | null;
     createdAt: string;
 }
+/**
+ * Result of a single LLM judge evaluation call
+ */
+export interface LLMJudgeEvaluateResult {
+    score: number;
+    reasoning: string;
+    passed: boolean;
+    details: unknown;
+}
 /**
  * Parameters for running an LLM judge evaluation
  */
@@ -295,6 +321,10 @@ export declare class SDKError extends Error {
 export type AIEvalConfig = ClientConfig;
 export type TraceData<TMetadata = unknown> = Trace<TMetadata>;
 export type SpanData<TMetadata = unknown> = Span<TMetadata>;
+export interface TraceDetail<TMetadata = Record<string, unknown>> {
+    trace: Trace<TMetadata>;
+    spans: Span<TMetadata>[];
+}
 export type EvaluationData<TMetadata = unknown> = Evaluation<TMetadata>;
 export type LLMJudgeData = LLMJudgeResult;
 export type AnnotationData = unknown;
@@ -537,7 +567,7 @@ export interface Webhook {
     events: string[];
     secret: string;
     status: "active" | "inactive";
-    lastTriggeredAt: string | null;
+    lastDeliveredAt: string | null;
     createdAt: string;
     updatedAt: string;
 }
@@ -572,12 +602,12 @@ export interface ListWebhooksParams {
 export interface WebhookDelivery {
     id: number;
     webhookId: number;
-    event: string;
+    eventType: string;
     payload: Record<string, unknown>;
-    response: string | null;
-    statusCode: number | null;
-    success: boolean;
-    attempt: number;
+    responseBody: string | null;
+    responseStatus: number | null;
+    status: string;
+    attemptCount: number;
     createdAt: string;
 }
 /**
@@ -586,51 +616,59 @@ export interface WebhookDelivery {
 export interface ListWebhookDeliveriesParams {
     limit?: number;
     offset?: number;
-    success?: boolean;
+    status?: "success" | "failed" | "pending";
 }
 /**
  * Usage statistics
  */
 export interface UsageStats {
-    organizationId: number;
+    analytics: {
+        totalRequests: number;
+        avgResponseTime: number;
+        errorRate: number;
+        successRate: number;
+        groupedData: Array<{
+            key: string;
+            count: number;
+            avgResponseTime: number;
+        }>;
+    };
     period: {
         start: string;
         end: string;
     };
-    traces: {
-        total: number;
-        byStatus: Record<string, number>;
-    };
-    evaluations: {
-        total: number;
-        byType: Record<string, number>;
-    };
-    apiCalls: {
-        total: number;
-        byEndpoint: Record<string, number>;
-    };
 }
 /**
  * Parameters for getting usage stats
  */
 export interface GetUsageParams {
-    organizationId: number;
-    startDate?: string;
-    endDate?: string;
+    period?: "7d" | "30d" | "90d";
+    groupBy?: "endpoint" | "method" | "day";
+    limit?: number;
+    offset?: number;
 }
 /**
  * Usage summary
  */
 export interface UsageSummary {
-    organizationId: number;
-    currentPeriod: {
-        traces: number;
-        evaluations: number;
-        annotations: number;
-        apiCalls: number;
+    summary: {
+        totalRequests: number;
+        avgResponseTime: number;
+        minResponseTime: number;
+        maxResponseTime: number;
+        errorRate: number;
+        successRate: number;
+        requestsByStatusCode: Record<string, number>;
+        topEndpoints: Array<{
+            endpoint: string;
+            count: number;
+        }>;
+        requestsOverTime: Array<{
+            date: string;
+            count: number;
+        }>;
     };
-    limits: OrganizationLimits;
-    billingPeriod: {
+    period: {
         start: string;
         end: string;
     };
@@ -641,13 +679,12 @@ export interface UsageSummary {
 export interface LLMJudgeConfig {
     id: number;
     name: string;
-    description: string | null;
     model: string;
-    rubric: string;
-    temperature: number;
-    maxTokens: number;
+    promptTemplate: string;
+    criteria: unknown;
+    settings: unknown;
     organizationId: number;
-    createdBy: number;
+    createdBy: string;
     createdAt: string;
     updatedAt: string;
 }
@@ -656,13 +693,10 @@ export interface LLMJudgeConfig {
  */
 export interface CreateLLMJudgeConfigParams {
     name: string;
-    description?: string;
     model: string;
-    rubric: string;
-    temperature?: number;
-    maxTokens?: number;
-    organizationId: number;
-    createdBy: number;
+    promptTemplate: string;
+    criteria?: Record<string, unknown>;
+    settings?: Record<string, unknown>;
 }
 /**
  * Parameters for listing LLM judge configs
@@ -685,28 +719,25 @@ export interface ListLLMJudgeResultsParams {
  * LLM Judge alignment analysis
  */
 export interface LLMJudgeAlignment {
-    configId: number;
-    totalEvaluations: number;
-    averageScore: number;
-    alignmentMetrics: {
-        accuracy: number;
-        precision: number;
-        recall: number;
-        f1Score: number;
-    };
-    scoreDistribution: Record<string, number>;
-    comparisonWithHuman?: {
-        agreement: number;
-        correlation: number;
+    alignmentData: Array<{
+        testCaseId: number;
+        humanScore: number;
+        judgeScore: number;
+        alignment: number;
+    }>;
+    metrics: {
+        averageAlignment: number;
+        totalComparisons: number;
+        highAlignment: number;
+        lowAlignment: number;
+        alignmentRate: number;
     };
 }
 /**
  * Parameters for getting alignment analysis
  */
 export interface GetLLMJudgeAlignmentParams {
-    configId: number;
-    startDate?: string;
-    endDate?: string;
+    evaluationRunId: number;
 }
 /**
  * Organization object
@@ -714,10 +745,10 @@ export interface GetLLMJudgeAlignmentParams {
 export interface Organization {
     id: number;
     name: string;
-    slug: string;
-    plan: string;
-    status: "active" | "suspended" | "cancelled";
-    createdAt: string;
-    updatedAt: string;
-    metadata?: Record<string, unknown>;
+    role?: string;
+    slug?: string;
+    plan?: string;
+    status?: string;
+    createdAt?: string;
+    updatedAt?: string;
 }

package/dist/utils/input-hash.js

@@ -34,5 +34,8 @@ function normalizeInput(input) {
 }
 /** SHA-256 hash of normalized input. */
 function sha256Input(s) {
-    return node_crypto_1.default.createHash("sha256").update(normalizeInput(s), "utf8").digest("hex");
+    return node_crypto_1.default
+        .createHash("sha256")
+        .update(normalizeInput(s), "utf8")
+        .digest("hex");
 }

package/dist/version.d.ts

@@ -3,5 +3,5 @@
  * X-EvalAI-SDK-Version: SDK package version
  * X-EvalAI-Spec-Version: OpenAPI spec version (docs/openapi.json info.version)
  */
-export declare const SDK_VERSION = "1.8.0";
+export declare const SDK_VERSION = "1.9.1";
 export declare const SPEC_VERSION = "1.0.0";

package/dist/version.js

@@ -6,5 +6,5 @@ exports.SPEC_VERSION = exports.SDK_VERSION = void 0;
  * X-EvalAI-SDK-Version: SDK package version
  * X-EvalAI-Spec-Version: OpenAPI spec version (docs/openapi.json info.version)
  */
-exports.SDK_VERSION = "1.8.0";
+exports.SDK_VERSION = "1.9.1";
 exports.SPEC_VERSION = "1.0.0";

package/dist/workflows.js

@@ -192,6 +192,7 @@ class WorkflowTracer {
         await this.client.traces.createSpan(this.currentWorkflow.traceId, {
             name: `Agent: ${agentName}`,
             spanId,
+            type: "agent",
             parentSpanId,
             startTime,
             metadata: (0, context_1.mergeWithContext)({
@@ -215,6 +216,7 @@ class WorkflowTracer {
         await this.client.traces.createSpan(this.currentWorkflow.traceId, {
             name: `Agent: ${span.agentName}`,
             spanId: `${span.spanId}-end`,
+            type: "agent",
             parentSpanId: span.spanId,
             startTime: span.startTime,
             endTime,
@@ -226,7 +228,11 @@ class WorkflowTracer {
             }),
         });
         this.activeSpans.delete(span.spanId);
-        this.log("Ended agent span", { agentName: span.agentName, spanId: span.spanId, durationMs });
+        this.log("Ended agent span", {
+            agentName: span.agentName,
+            spanId: span.spanId,
+            durationMs,
+        });
     }
     // ==========================================================================
     // HANDOFFS
@@ -261,6 +267,7 @@ class WorkflowTracer {
         await this.client.traces.createSpan(this.currentWorkflow.traceId, {
             name: `Handoff: ${fromAgent || "start"} → ${toAgent}`,
             spanId,
+            type: "handoff",
             startTime: handoff.timestamp,
             endTime: handoff.timestamp,
             durationMs: 0,
@@ -306,6 +313,7 @@ class WorkflowTracer {
         await this.client.traces.createSpan(this.currentWorkflow.traceId, {
             name: `Decision: ${params.agent} chose ${params.chosen}`,
             spanId,
+            type: "decision",
             startTime: timestamp,
             endTime: timestamp,
             durationMs: 0,
@@ -369,6 +377,7 @@ class WorkflowTracer {
             await this.client.traces.createSpan(this.currentWorkflow.traceId, {
                 name: `Cost: ${params.provider}/${params.model}`,
                 spanId,
+                type: "cost",
                 startTime: timestamp,
                 endTime: timestamp,
                 durationMs: 0,
@@ -417,23 +426,62 @@ class WorkflowTracer {
         // Default pricing (can be extended with API lookup)
         const knownPricing = {
             // OpenAI
-            "openai/gpt-4": { inputPricePerMillion: 30.0, outputPricePerMillion: 60.0 },
-            "openai/gpt-4-turbo": { inputPricePerMillion: 10.0, outputPricePerMillion: 30.0 },
-            "openai/gpt-4o": { inputPricePerMillion: 5.0, outputPricePerMillion: 15.0 },
-            "openai/gpt-4o-mini": { inputPricePerMillion: 0.15, outputPricePerMillion: 0.6 },
-            "openai/gpt-3.5-turbo": { inputPricePerMillion: 0.5, outputPricePerMillion: 1.5 },
+            "openai/gpt-4": {
+                inputPricePerMillion: 30.0,
+                outputPricePerMillion: 60.0,
+            },
+            "openai/gpt-4-turbo": {
+                inputPricePerMillion: 10.0,
+                outputPricePerMillion: 30.0,
+            },
+            "openai/gpt-4o": {
+                inputPricePerMillion: 5.0,
+                outputPricePerMillion: 15.0,
+            },
+            "openai/gpt-4o-mini": {
+                inputPricePerMillion: 0.15,
+                outputPricePerMillion: 0.6,
+            },
+            "openai/gpt-3.5-turbo": {
+                inputPricePerMillion: 0.5,
+                outputPricePerMillion: 1.5,
+            },
             // Anthropic
-            "anthropic/claude-3-opus": { inputPricePerMillion: 15.0, outputPricePerMillion: 75.0 },
-            "anthropic/claude-3-sonnet": { inputPricePerMillion: 3.0, outputPricePerMillion: 15.0 },
-            "anthropic/claude-3-haiku": { inputPricePerMillion: 0.25, outputPricePerMillion: 1.25 },
-            "anthropic/claude-3.5-sonnet": { inputPricePerMillion: 3.0, outputPricePerMillion: 15.0 },
+            "anthropic/claude-3-opus": {
+                inputPricePerMillion: 15.0,
+                outputPricePerMillion: 75.0,
+            },
+            "anthropic/claude-3-sonnet": {
+                inputPricePerMillion: 3.0,
+                outputPricePerMillion: 15.0,
+            },
+            "anthropic/claude-3-haiku": {
+                inputPricePerMillion: 0.25,
+                outputPricePerMillion: 1.25,
+            },
+            "anthropic/claude-3.5-sonnet": {
+                inputPricePerMillion: 3.0,
+                outputPricePerMillion: 15.0,
+            },
             // Google
-            "google/gemini-pro": { inputPricePerMillion: 0.5, outputPricePerMillion: 1.5 },
-            "google/gemini-1.5-pro": { inputPricePerMillion: 3.5, outputPricePerMillion: 10.5 },
-            "google/gemini-1.5-flash": { inputPricePerMillion: 0.075, outputPricePerMillion: 0.3 },
+            "google/gemini-pro": {
+                inputPricePerMillion: 0.5,
+                outputPricePerMillion: 1.5,
+            },
+            "google/gemini-1.5-pro": {
+                inputPricePerMillion: 3.5,
+                outputPricePerMillion: 10.5,
+            },
+            "google/gemini-1.5-flash": {
+                inputPricePerMillion: 0.075,
+                outputPricePerMillion: 0.3,
+            },
         };
         const key = `${provider}/${model}`;
-        return knownPricing[key] || { inputPricePerMillion: 1.0, outputPricePerMillion: 3.0 };
+        return (knownPricing[key] || {
+            inputPricePerMillion: 1.0,
+            outputPricePerMillion: 3.0,
+        });
     }
     /**
      * Generate a unique ID