@pauly4010/evalai-sdk 1.4.1 → 1.5.0

package/dist/cli/report/build-check-report.d.ts

@@ -0,0 +1,16 @@
+/**
+ * Build CheckReport from API data and gate result.
+ * Normalizes failed cases (truncate, sort), dashboard URL, top N + more.
+ */
+import type { CheckArgs } from '../check';
+import type { QualityLatestData, RunDetailsData } from '../api';
+import type { GateResult } from '../gate';
+import type { CheckReport } from '../formatters/types';
+export type BuildReportInput = {
+    args: CheckArgs;
+    quality: QualityLatestData;
+    runDetails?: RunDetailsData | null;
+    gateResult: GateResult;
+    requestId?: string;
+};
+export declare function buildCheckReport(input: BuildReportInput): CheckReport;

package/dist/cli/report/build-check-report.js

@@ -0,0 +1,94 @@
+"use strict";
+/**
+ * Build CheckReport from API data and gate result.
+ * Normalizes failed cases (truncate, sort), dashboard URL, top N + more.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.buildCheckReport = buildCheckReport;
+const snippet_1 = require("../render/snippet");
+const sort_1 = require("../render/sort");
+const TOP_N = 3;
+/** ContribPts from weights: passRate*50, safety*25, (0.6*judge+0.4*schema)*15, (0.6*latency+0.4*cost)*10 */
+function computeContribPts(b) {
+    const pr = b.passRate ?? 0;
+    const s = b.safety ?? 0;
+    const j = b.judge ?? 0;
+    const sc = b.schema ?? 0;
+    const l = b.latency ?? 0;
+    const c = b.cost ?? 0;
+    return {
+        passRatePts: Math.round(pr * 50 * 10) / 10,
+        safetyPts: Math.round(s * 25 * 10) / 10,
+        compliancePts: Math.round((0.6 * j + 0.4 * sc) * 15 * 10) / 10,
+        performancePts: Math.round((0.6 * l + 0.4 * c) * 10 * 10) / 10,
+    };
+}
+const SNIPPET_MAX = 50;
+function buildCheckReport(input) {
+    const { args, quality, runDetails, gateResult, requestId } = input;
+    const score = quality?.score ?? 0;
+    const total = quality?.total ?? null;
+    const baselineScore = quality?.baselineScore ?? null;
+    const regressionDelta = quality?.regressionDelta ?? null;
+    const evaluationRunId = quality?.evaluationRunId;
+    const breakdown = quality?.breakdown ?? {};
+    const flags = (quality?.flags ?? []);
+    const baseUrl = args.baseUrl.replace(/\/$/, '');
+    const dashboardUrl = evaluationRunId != null
+        ? `${baseUrl}/evaluations/${args.evaluationId}/runs/${evaluationRunId}`
+        : undefined;
+    // Build failed cases from run details
+    let failedCases = [];
+    if (runDetails?.results && evaluationRunId != null) {
+        const raw = runDetails.results
+            .filter((r) => r.status === 'failed')
+            .map((r) => ({
+            testCaseId: r.testCaseId,
+            status: 'failed',
+            name: r.test_cases?.name,
+            input: r.test_cases?.input,
+            expectedOutput: r.test_cases?.expectedOutput,
+            output: r.output,
+        }));
+        failedCases = (0, sort_1.sortFailedCases)(raw).map((fc) => ({
+            ...fc,
+            inputSnippet: (0, snippet_1.truncateSnippet)(fc.input, SNIPPET_MAX),
+            expectedSnippet: (0, snippet_1.truncateSnippet)(fc.expectedOutput, SNIPPET_MAX),
+            outputSnippet: (0, snippet_1.truncateSnippet)(fc.output, SNIPPET_MAX),
+        }));
+    }
+    const failedCasesShown = Math.min(failedCases.length, TOP_N);
+    const failedCasesMore = failedCases.length - failedCasesShown;
+    const breakdown01 = Object.keys(breakdown).length > 0 ? breakdown : undefined;
+    const contribPts = args.explain && breakdown01 ? computeContribPts(breakdown01) : undefined;
+    const report = {
+        evaluationId: args.evaluationId,
+        runId: evaluationRunId,
+        verdict: gateResult.passed ? 'pass' : 'fail',
+        reasonCode: gateResult.reasonCode,
+        reasonMessage: gateResult.reasonMessage ?? undefined,
+        score,
+        baselineScore: baselineScore ?? undefined,
+        delta: regressionDelta ?? undefined,
+        n: total ?? undefined,
+        evidenceLevel: quality?.evidenceLevel ?? undefined,
+        baselineMissing: quality?.baselineMissing === true,
+        flags: flags.length > 0 ? [...flags].sort() : undefined,
+        breakdown01,
+        contribPts,
+        thresholds: {
+            minScore: args.minScore,
+            maxDrop: args.maxDrop,
+            minN: args.minN,
+            allowWeakEvidence: args.allowWeakEvidence,
+            baseline: args.baseline,
+        },
+        dashboardUrl,
+        failedCases,
+        failedCasesShown: failedCases.length > 0 ? failedCasesShown : undefined,
+        failedCasesMore: failedCasesMore > 0 ? failedCasesMore : undefined,
+        requestId,
+        explain: args.explain,
+    };
+    return report;
+}

package/dist/index.d.ts

@@ -27,6 +27,7 @@ export { RequestBatcher } from './batch';
 export { Logger } from './logger';
 export { traceOpenAI } from './integrations/openai';
 export { traceAnthropic } from './integrations/anthropic';
+export { openAIChatEval, type OpenAIChatEvalOptions, type OpenAIChatEvalResult, type OpenAIChatEvalCase, } from './integrations/openai-eval';
 export { WorkflowTracer, createWorkflowTracer, traceWorkflowStep, traceLangChainAgent, traceCrewAI, traceAutoGen, type WorkflowNode, type WorkflowEdge, type WorkflowDefinition, type WorkflowContext, type WorkflowStatus, type HandoffType, type AgentHandoff, type DecisionAlternative, type DecisionType, type RecordDecisionParams, type LLMProvider, type CostCategory, type RecordCostParams, type CostRecord, type WorkflowTracerOptions, type AgentSpanContext, } from './workflows';
 export type { ClientConfig as AIEvalConfig, Trace as TraceData, Span as SpanData, Evaluation as EvaluationData, LLMJudgeResult as LLMJudgeData, RetryConfig, GenericMetadata as AnnotationData, TracedResponse, TestCase, TestResult, SnapshotData, ExportOptions, ImportOptions, StreamOptions, BatchOptions } from './types';
 export { EvaluationTemplates, type EvaluationTemplateType, type FeatureUsage, type OrganizationLimits } from './types';

package/dist/index.js

@@ -9,7 +9,7 @@
  */
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.decodeCursor = exports.encodeCursor = exports.autoPaginate = exports.createPaginatedIterator = exports.PaginatedIterator = exports.CacheTTL = exports.RequestCache = exports.RateLimiter = exports.batchRead = exports.streamEvaluation = exports.batchProcess = exports.importData = exports.exportData = exports.compareSnapshots = exports.saveSnapshot = exports.compareWithSnapshot = exports.snapshot = exports.TestSuite = exports.createTestSuite = exports.ContextManager = exports.withContext = exports.getContext = exports.createContext = exports.hasValidCodeSyntax = exports.containsAllRequiredFields = exports.followsInstructions = exports.hasNoToxicity = exports.respondedWithinTime = exports.hasFactualAccuracy = exports.containsLanguage = exports.hasReadabilityScore = exports.matchesSchema = exports.hasNoHallucinations = exports.isValidURL = exports.isValidEmail = exports.withinRange = exports.similarTo = exports.hasSentiment = exports.notContainsPII = exports.containsJSON = exports.hasLength = exports.matchesPattern = exports.containsKeywords = exports.expect = exports.NetworkError = exports.ValidationError = exports.AuthenticationError = exports.RateLimitError = exports.EvalAIError = exports.AIEvalClient = void 0;
-exports.EXIT = exports.runCheck = exports.parseArgs = exports.EvaluationTemplates = exports.traceAutoGen = exports.traceCrewAI = exports.traceLangChainAgent = exports.traceWorkflowStep = exports.createWorkflowTracer = exports.WorkflowTracer = exports.traceAnthropic = exports.traceOpenAI = exports.Logger = exports.RequestBatcher = void 0;
+exports.EXIT = exports.runCheck = exports.parseArgs = exports.EvaluationTemplates = exports.traceAutoGen = exports.traceCrewAI = exports.traceLangChainAgent = exports.traceWorkflowStep = exports.createWorkflowTracer = exports.WorkflowTracer = exports.openAIChatEval = exports.traceAnthropic = exports.traceOpenAI = exports.Logger = exports.RequestBatcher = void 0;
 // Main SDK exports
 var client_1 = require("./client");
 Object.defineProperty(exports, "AIEvalClient", { enumerable: true, get: function () { return client_1.AIEvalClient; } });
@@ -94,6 +94,9 @@ var openai_1 = require("./integrations/openai");
 Object.defineProperty(exports, "traceOpenAI", { enumerable: true, get: function () { return openai_1.traceOpenAI; } });
 var anthropic_1 = require("./integrations/anthropic");
 Object.defineProperty(exports, "traceAnthropic", { enumerable: true, get: function () { return anthropic_1.traceAnthropic; } });
+// OpenAI regression eval (local-first, no account required)
+var openai_eval_1 = require("./integrations/openai-eval");
+Object.defineProperty(exports, "openAIChatEval", { enumerable: true, get: function () { return openai_eval_1.openAIChatEval; } });
 // Workflow tracing (Orchestration Layer)
 var workflows_1 = require("./workflows");
 Object.defineProperty(exports, "WorkflowTracer", { enumerable: true, get: function () { return workflows_1.WorkflowTracer; } });

package/dist/integrations/openai-eval.d.ts

@@ -0,0 +1,53 @@
+/**
+ * openAIChatEval — One-function OpenAI chat regression testing
+ *
+ * Run local regression tests with OpenAI. No EvalAI account required.
+ * CI-friendly output. Optional reportToEvalAI in v1.5.
+ *
+ * @example
+ * ```typescript
+ * import { openAIChatEval } from '@pauly4010/evalai-sdk';
+ *
+ * await openAIChatEval({
+ *   name: 'chat-regression',
+ *   cases: [
+ *     { input: 'Hello', expectedOutput: 'greeting' },
+ *     { input: '2 + 2 = ?', expectedOutput: '4' }
+ *   ]
+ * });
+ * ```
+ */
+import type { TestSuiteCaseResult } from '../testing';
+export interface OpenAIChatEvalCase {
+    input: string;
+    expectedOutput?: string;
+    /** Platform test case ID. When provided, used directly for reportToEvalAI (no input matching). */
+    testCaseId?: number;
+    assertions?: ((output: string) => import('../assertions').AssertionResult)[];
+}
+export interface OpenAIChatEvalOptions {
+    name: string;
+    model?: string;
+    apiKey?: string;
+    cases: OpenAIChatEvalCase[];
+    /** v1.5: Upload results to EvalAI platform for an existing evaluation. Requires evaluationId and EVALAI_API_KEY. */
+    reportToEvalAI?: boolean;
+    /** Evaluation ID (from config or arg). Required when reportToEvalAI is true. */
+    evaluationId?: string;
+    /** EvalAI API base URL. Default: EVALAI_BASE_URL or http://localhost:3000 */
+    baseUrl?: string;
+    /** Idempotency key for import (e.g. CI run ID). Prevents duplicate runs on retry. */
+    idempotencyKey?: string;
+}
+export interface OpenAIChatEvalResult {
+    passed: number;
+    total: number;
+    score: number;
+    results: TestSuiteCaseResult[];
+    durationMs: number;
+}
+/**
+ * Run OpenAI chat regression tests locally.
+ * No EvalAI account required. Returns score and prints CI-friendly summary.
+ */
+export declare function openAIChatEval(options: OpenAIChatEvalOptions): Promise<OpenAIChatEvalResult>;

package/dist/integrations/openai-eval.js

@@ -0,0 +1,226 @@
+"use strict";
+/**
+ * openAIChatEval — One-function OpenAI chat regression testing
+ *
+ * Run local regression tests with OpenAI. No EvalAI account required.
+ * CI-friendly output. Optional reportToEvalAI in v1.5.
+ *
+ * @example
+ * ```typescript
+ * import { openAIChatEval } from '@pauly4010/evalai-sdk';
+ *
+ * await openAIChatEval({
+ *   name: 'chat-regression',
+ *   cases: [
+ *     { input: 'Hello', expectedOutput: 'greeting' },
+ *     { input: '2 + 2 = ?', expectedOutput: '4' }
+ *   ]
+ * });
+ * ```
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.openAIChatEval = openAIChatEval;
+const testing_1 = require("../testing");
+const assertions_1 = require("../assertions");
+const config_1 = require("../cli/config");
+const input_hash_1 = require("../utils/input-hash");
+const MAX_FAILED_CASES_TO_SHOW = 5;
+function getOpenAI() {
+    try {
+        // eslint-disable-next-line @typescript-eslint/no-var-requires
+        const OpenAI = require('openai');
+        return OpenAI;
+    }
+    catch {
+        throw new Error('openai package is required for openAIChatEval. Install with: npm install openai');
+    }
+}
+function createExecutor(model, apiKey) {
+    const OpenAI = getOpenAI();
+    const openai = new OpenAI({ apiKey });
+    return async (input) => {
+        const response = await openai.chat.completions.create({
+            model,
+            messages: [{ role: 'user', content: input }],
+            temperature: 0.1,
+        });
+        return response.choices[0]?.message?.content ?? '';
+    };
+}
+function printSummary(result) {
+    const { passed, total, results } = result;
+    const score = total > 0 ? Math.round((passed / total) * 100) : 0;
+    const failed = results.filter((r) => !r.passed);
+    const status = failed.length === 0 ? 'PASS' : 'FAIL';
+    console.log(`\n${status} ${passed}/${total}  (score: ${score})\n`);
+    if (failed.length > 0) {
+        const toShow = failed.slice(0, MAX_FAILED_CASES_TO_SHOW);
+        const more = failed.length - toShow.length;
+        console.log(`${failed.length} failing case${failed.length === 1 ? '' : 's'}:`);
+        for (const r of toShow) {
+            const expected = r.expected ?? '(no expected)';
+            console.log(`- "${r.input}" → expected: ${expected}`);
+        }
+        if (more > 0) {
+            console.log(`+ ${more} more`);
+        }
+        console.log('\nGate this in CI:');
+        console.log('  npx -y @pauly4010/evalai-sdk@^1 init');
+    }
+    else {
+        console.log('Tip: Want dashboards and history?');
+        console.log('Set EVALAI_API_KEY and connect this to the platform.');
+    }
+}
+/**
+ * Run OpenAI chat regression tests locally.
+ * No EvalAI account required. Returns score and prints CI-friendly summary.
+ */
+async function openAIChatEval(options) {
+    const { name, model = 'gpt-4o-mini', apiKey, cases } = options;
+    const resolvedApiKey = apiKey ?? (typeof process !== 'undefined' && process.env?.OPENAI_API_KEY);
+    if (!resolvedApiKey) {
+        throw new Error('OPENAI_API_KEY is required. Set it in the environment or pass apiKey to openAIChatEval.');
+    }
+    const executor = createExecutor(model, resolvedApiKey);
+    const suiteCases = cases.map((c) => {
+        const assertions = c.assertions
+            ? [...c.assertions]
+            : c.expectedOutput
+                ? [(output) => (0, assertions_1.expect)(output).toContainKeywords(c.expectedOutput.split(/\s+/).filter(Boolean))]
+                : undefined;
+        return {
+            input: c.input,
+            expected: c.expectedOutput,
+            assertions,
+        };
+    });
+    const suite = (0, testing_1.createTestSuite)(name, {
+        cases: suiteCases,
+        executor,
+        parallel: true,
+    });
+    const result = await suite.run();
+    const score = result.total > 0 ? Math.round((result.passed / result.total) * 100) : 0;
+    const evalResult = {
+        passed: result.passed,
+        total: result.total,
+        score,
+        results: result.results,
+        durationMs: result.durationMs,
+    };
+    printSummary(evalResult);
+    // v1.5: Optional report to EvalAI platform
+    if (options.reportToEvalAI) {
+        const config = typeof process !== 'undefined' && process.cwd ? (0, config_1.loadConfig)(process.cwd()) : null;
+        const evalId = options.evaluationId || config?.evaluationId;
+        if (!evalId || String(evalId).trim() === '') {
+            console.log('Run evalai init and set evaluationId to upload results.');
+            return evalResult;
+        }
+        const evalaiKey = (typeof process !== 'undefined' && process.env?.EVALAI_API_KEY) || '';
+        if (!evalaiKey) {
+            console.log('Set EVALAI_API_KEY to upload results.');
+            return evalResult;
+        }
+        const baseUrl = options.baseUrl ||
+            config?.baseUrl ||
+            (typeof process !== 'undefined' && process.env?.EVALAI_BASE_URL) ||
+            'http://localhost:3000';
+        const url = String(baseUrl).replace(/\/$/, '');
+        try {
+            // Resolve testCaseId for each result: explicit testCaseId in cases, or match by inputHash
+            const importResults = [];
+            const hasExplicitIds = cases.some((c) => c.testCaseId != null);
+            if (hasExplicitIds) {
+                // Use testCaseId from cases (same order as results)
+                for (let i = 0; i < result.results.length; i++) {
+                    const tcId = cases[i]?.testCaseId;
+                    if (tcId == null) {
+                        console.log('reportToEvalAI: All cases must have testCaseId when any has it.');
+                        return evalResult;
+                    }
+                    importResults.push({
+                        testCaseId: tcId,
+                        status: result.results[i].passed ? 'passed' : 'failed',
+                        output: result.results[i].actual ?? '',
+                        latencyMs: result.results[i].durationMs,
+                    });
+                }
+            }
+            else {
+                // Match by inputHash (same canonicalization as platform)
+                const tcRes = await fetch(`${url}/api/evaluations/${evalId}/test-cases?limit=500`, {
+                    headers: { Authorization: `Bearer ${evalaiKey}` },
+                });
+                if (!tcRes.ok) {
+                    console.log('Could not fetch test cases. Check evaluationId and EVALAI_API_KEY.');
+                    return evalResult;
+                }
+                const platformCases = (await tcRes.json());
+                const hashToIds = new Map();
+                for (const tc of platformCases) {
+                    const input = tc.input ?? '';
+                    if (!input.trim())
+                        continue;
+                    const hash = (0, input_hash_1.sha256Input)(input);
+                    const existing = hashToIds.get(hash) ?? [];
+                    existing.push(tc.id);
+                    hashToIds.set(hash, existing);
+                }
+                for (const r of result.results) {
+                    const hash = (0, input_hash_1.sha256Input)(r.input ?? '');
+                    const ids = hashToIds.get(hash);
+                    if (ids == null || ids.length === 0) {
+                        console.log(`No platform test case matches input: "${(r.input ?? '').slice(0, 50)}…"`);
+                        return evalResult;
+                    }
+                    if (ids.length > 1) {
+                        console.log(`Multiple platform test cases share the same input (hash collision). Use testCaseId in cases.`);
+                        return evalResult;
+                    }
+                    importResults.push({
+                        testCaseId: ids[0],
+                        status: r.passed ? 'passed' : 'failed',
+                        output: r.actual ?? '',
+                        latencyMs: r.durationMs,
+                    });
+                }
+            }
+            if (importResults.length !== result.results.length) {
+                console.log('Could not match all results to platform test cases.');
+                return evalResult;
+            }
+            const sdkVersion = '1.4.1';
+            const headers = {
+                'Content-Type': 'application/json',
+                Authorization: `Bearer ${evalaiKey}`,
+            };
+            if (options.idempotencyKey) {
+                headers['Idempotency-Key'] = options.idempotencyKey;
+            }
+            const importRes = await fetch(`${url}/api/evaluations/${evalId}/runs/import`, {
+                method: 'POST',
+                headers,
+                body: JSON.stringify({
+                    environment: 'dev',
+                    results: importResults,
+                    importClientVersion: sdkVersion,
+                }),
+            });
+            if (!importRes.ok) {
+                const body = await importRes.text();
+                console.log(`Upload failed: ${importRes.status} — ${body}`);
+                return evalResult;
+            }
+            const importData = (await importRes.json());
+            if (importData.dashboardUrl) {
+                console.log(`Dashboard: ${importData.dashboardUrl}`);
+            }
+        }
+        catch (err) {
+            console.log('Upload failed:', err instanceof Error ? err.message : String(err));
+        }
+    }
+    return evalResult;
+}

package/dist/utils/input-hash.d.ts

@@ -0,0 +1,8 @@
+/**
+ * Input normalization and hashing for deterministic matching.
+ * Must match platform's @/lib/utils/input-hash.ts for reportToEvalAI.
+ */
+/** Normalize input for stable matching (whitespace, JSON key order). */
+export declare function normalizeInput(input: string): string;
+/** SHA-256 hash of normalized input. */
+export declare function sha256Input(s: string): string;

package/dist/utils/input-hash.js

@@ -0,0 +1,38 @@
+"use strict";
+/**
+ * Input normalization and hashing for deterministic matching.
+ * Must match platform's @/lib/utils/input-hash.ts for reportToEvalAI.
+ */
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.normalizeInput = normalizeInput;
+exports.sha256Input = sha256Input;
+const crypto_1 = __importDefault(require("crypto"));
+function sortKeys(obj) {
+    const sorted = {};
+    for (const k of Object.keys(obj).sort()) {
+        const v = obj[k];
+        sorted[k] =
+            v != null && typeof v === 'object' && !Array.isArray(v)
+                ? sortKeys(v)
+                : v;
+    }
+    return sorted;
+}
+/** Normalize input for stable matching (whitespace, JSON key order). */
+function normalizeInput(input) {
+    const s = input.trim();
+    try {
+        const obj = JSON.parse(s);
+        return JSON.stringify(sortKeys(obj));
+    }
+    catch {
+        return s.replace(/\s+/g, ' ');
+    }
+}
+/** SHA-256 hash of normalized input. */
+function sha256Input(s) {
+    return crypto_1.default.createHash('sha256').update(normalizeInput(s), 'utf8').digest('hex');
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pauly4010/evalai-sdk",
-  "version": "1.4.1",
+  "version": "1.5.0",
   "description": "AI Evaluation Platform SDK - Complete API Coverage with Performance Optimizations",
   "main": "dist/index.js",
   "module": "dist/index.js",
@@ -85,6 +85,10 @@
     "./integrations/anthropic": {
       "import": "./dist/integrations/anthropic.js",
       "types": "./dist/integrations/anthropic.d.ts"
+    },
+    "./integrations/openai-eval": {
+      "import": "./dist/integrations/openai-eval.js",
+      "types": "./dist/integrations/openai-eval.d.ts"
     }
   }
 }

package/dist/__tests__/assertions.test.d.ts

@@ -1 +0,0 @@
-export {};