@evalgate/sdk 2.0.0

package/dist/matchers/index.js

@@ -0,0 +1,6 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.toPassGate = exports.extendExpectWithToPassGate = void 0;
+var to_pass_gate_1 = require("./to-pass-gate");
+Object.defineProperty(exports, "extendExpectWithToPassGate", { enumerable: true, get: function () { return to_pass_gate_1.extendExpectWithToPassGate; } });
+Object.defineProperty(exports, "toPassGate", { enumerable: true, get: function () { return to_pass_gate_1.toPassGate; } });

package/dist/matchers/to-pass-gate.d.ts

@@ -0,0 +1,29 @@
+/**
+ * Vitest/Jest matcher: expect(result).toPassGate()
+ * Use with openAIChatEval: expect(await openAIChatEval(...)).toPassGate()
+ *
+ * @example
+ * ```ts
+ * import { openAIChatEval } from '@evalgate/sdk';
+ * import { expect } from 'vitest';
+ * import { extendExpectWithToPassGate } from '@evalgate/sdk/matchers';
+ *
+ * extendExpectWithToPassGate(expect);
+ *
+ * it('passes gate', async () => {
+ *   const result = await openAIChatEval({ name: 'test', cases: [...] });
+ *   expect(result).toPassGate();
+ * });
+ * ```
+ */
+import type { OpenAIChatEvalResult } from "../integrations/openai-eval";
+export declare function toPassGate(this: {
+    isNot?: boolean;
+}, received: OpenAIChatEvalResult): {
+    pass: boolean;
+    message: () => string;
+};
+/** Register toPassGate matcher with expect. Call in test setup. */
+export declare function extendExpectWithToPassGate(expect: {
+    extend: (matchers: object) => void;
+}): void;

package/dist/matchers/to-pass-gate.js

@@ -0,0 +1,35 @@
+"use strict";
+/**
+ * Vitest/Jest matcher: expect(result).toPassGate()
+ * Use with openAIChatEval: expect(await openAIChatEval(...)).toPassGate()
+ *
+ * @example
+ * ```ts
+ * import { openAIChatEval } from '@evalgate/sdk';
+ * import { expect } from 'vitest';
+ * import { extendExpectWithToPassGate } from '@evalgate/sdk/matchers';
+ *
+ * extendExpectWithToPassGate(expect);
+ *
+ * it('passes gate', async () => {
+ *   const result = await openAIChatEval({ name: 'test', cases: [...] });
+ *   expect(result).toPassGate();
+ * });
+ * ```
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.toPassGate = toPassGate;
+exports.extendExpectWithToPassGate = extendExpectWithToPassGate;
+function toPassGate(received) {
+    const passed = received.passed === received.total && received.total > 0;
+    const isNot = this.isNot ?? false;
+    const success = isNot ? !passed : passed;
+    const message = () => isNot
+        ? `Expected result not to pass gate (${received.passed}/${received.total} passed, score ${received.score})`
+        : `Expected result to pass gate but ${received.passed}/${received.total} passed (score ${received.score})`;
+    return { pass: success, message };
+}
+/** Register toPassGate matcher with expect. Call in test setup. */
+function extendExpectWithToPassGate(expect) {
+    expect.extend({ toPassGate });
+}

package/dist/pagination.d.ts

@@ -0,0 +1,74 @@
+/**
+ * Cursor-based pagination utilities for efficient data fetching
+ */
+export interface PaginationParams {
+    limit?: number;
+    cursor?: string;
+    offset?: number;
+}
+export interface PaginatedResponse<T> {
+    data: T[];
+    pagination: {
+        hasMore: boolean;
+        nextCursor?: string;
+        prevCursor?: string;
+        total?: number;
+        limit: number;
+        offset?: number;
+    };
+}
+export interface PaginationOptions {
+    limit?: number;
+    offset?: number;
+}
+/**
+ * Iterator for paginated results
+ * Allows for easy iteration over all pages
+ */
+export declare class PaginatedIterator<T> implements AsyncIterableIterator<T[]> {
+    private fetchFn;
+    private limit;
+    private currentOffset;
+    private hasMore;
+    constructor(fetchFn: (offset: number, limit: number) => Promise<{
+        data: T[];
+        hasMore: boolean;
+    }>, limit?: number);
+    next(): Promise<IteratorResult<T[]>>;
+    [Symbol.asyncIterator](): AsyncIterableIterator<T[]>;
+    /**
+     * Collect all pages into a single array
+     * Warning: Use with caution on large datasets
+     */
+    toArray(): Promise<T[]>;
+}
+/**
+ * Helper to create paginated iterator
+ */
+export declare function createPaginatedIterator<T>(fetchFn: (offset: number, limit: number) => Promise<{
+    data: T[];
+    hasMore: boolean;
+}>, limit?: number): PaginatedIterator<T>;
+/**
+ * Auto-paginate helper that fetches all pages automatically
+ */
+export declare function autoPaginate<T>(fetchFn: (offset: number, limit: number) => Promise<T[]>, limit?: number): AsyncGenerator<T, void, unknown>;
+/**
+ * Encode cursor for pagination (base64)
+ */
+export declare function encodeCursor(data: unknown): string;
+/**
+ * Decode cursor from base64
+ */
+export declare function decodeCursor(cursor: string): unknown;
+/**
+ * Create pagination metadata from response
+ */
+export declare function createPaginationMeta<T>(items: T[], limit: number, offset: number, total?: number): PaginatedResponse<T>["pagination"];
+/**
+ * Parse pagination params from cursor or offset
+ */
+export declare function parsePaginationParams(params: PaginationParams): {
+    limit: number;
+    offset: number;
+};

package/dist/pagination.js

@@ -0,0 +1,139 @@
+"use strict";
+/**
+ * Cursor-based pagination utilities for efficient data fetching
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.PaginatedIterator = void 0;
+exports.createPaginatedIterator = createPaginatedIterator;
+exports.autoPaginate = autoPaginate;
+exports.encodeCursor = encodeCursor;
+exports.decodeCursor = decodeCursor;
+exports.createPaginationMeta = createPaginationMeta;
+exports.parsePaginationParams = parsePaginationParams;
+/**
+ * Iterator for paginated results
+ * Allows for easy iteration over all pages
+ */
+class PaginatedIterator {
+    constructor(fetchFn, limit = 50) {
+        this.fetchFn = fetchFn;
+        this.limit = limit;
+        this.currentOffset = 0;
+        this.hasMore = true;
+    }
+    async next() {
+        if (!this.hasMore) {
+            return { done: true, value: undefined };
+        }
+        const result = await this.fetchFn(this.currentOffset, this.limit);
+        this.hasMore = result.hasMore;
+        this.currentOffset += this.limit;
+        return {
+            done: false,
+            value: result.data,
+        };
+    }
+    [Symbol.asyncIterator]() {
+        return this;
+    }
+    /**
+     * Collect all pages into a single array
+     * Warning: Use with caution on large datasets
+     */
+    async toArray() {
+        const allItems = [];
+        for await (const page of this) {
+            allItems.push(...page);
+        }
+        return allItems;
+    }
+}
+exports.PaginatedIterator = PaginatedIterator;
+/**
+ * Helper to create paginated iterator
+ */
+function createPaginatedIterator(fetchFn, limit = 50) {
+    return new PaginatedIterator(fetchFn, limit);
+}
+/**
+ * Auto-paginate helper that fetches all pages automatically
+ */
+async function* autoPaginate(fetchFn, limit = 50) {
+    let offset = 0;
+    let hasMore = true;
+    while (hasMore) {
+        const items = await fetchFn(offset, limit);
+        if (items.length === 0) {
+            break;
+        }
+        for (const item of items) {
+            yield item;
+        }
+        hasMore = items.length === limit;
+        offset += limit;
+    }
+}
+/**
+ * Encode cursor for pagination (base64)
+ */
+function encodeCursor(data) {
+    const json = JSON.stringify(data);
+    if (typeof globalThis !== "undefined" && "btoa" in globalThis) {
+        return globalThis.btoa(json);
+    }
+    else {
+        return Buffer.from(json).toString("base64");
+    }
+}
+/**
+ * Decode cursor from base64
+ */
+function decodeCursor(cursor) {
+    try {
+        let json;
+        if (typeof globalThis !== "undefined" && "atob" in globalThis) {
+            json = globalThis.atob(cursor);
+        }
+        else {
+            json = Buffer.from(cursor, "base64").toString("utf-8");
+        }
+        return JSON.parse(json);
+    }
+    catch (_error) {
+        throw new Error("Invalid cursor format");
+    }
+}
+/**
+ * Create pagination metadata from response
+ */
+function createPaginationMeta(items, limit, offset, total) {
+    const hasMore = items.length === limit;
+    return {
+        hasMore,
+        limit,
+        offset,
+        total,
+        nextCursor: hasMore
+            ? encodeCursor({ offset: offset + limit, limit })
+            : undefined,
+        prevCursor: offset > 0
+            ? encodeCursor({ offset: Math.max(0, offset - limit), limit })
+            : undefined,
+    };
+}
+/**
+ * Parse pagination params from cursor or offset
+ */
+function parsePaginationParams(params) {
+    if (params.cursor) {
+        const decoded = decodeCursor(params.cursor);
+        return {
+            limit: decoded.limit || 50,
+            offset: decoded.offset || 0,
+        };
+    }
+    return {
+        limit: params.limit || 50,
+        offset: params.offset || 0,
+    };
+}

package/dist/regression.d.ts

@@ -0,0 +1,100 @@
+/**
+ * Regression gate constants and types.
+ *
+ * These mirror the contracts defined in scripts/regression-gate.ts
+ * and evals/schemas/regression-report.schema.json so that SDK consumers
+ * can programmatically inspect gate results without parsing strings.
+ *
+ * @packageDocumentation
+ */
+/** Exit codes emitted by `evalgate gate` / `scripts/regression-gate.ts`. */
+export declare const GATE_EXIT: {
+    /** Gate passed — no regressions detected */
+    readonly PASS: 0;
+    /** One or more regression thresholds exceeded */
+    readonly REGRESSION: 1;
+    /** Infrastructure error (baseline missing, summary missing, etc.) */
+    readonly INFRA_ERROR: 2;
+    /** Confidence tests failed (test suite red) */
+    readonly CONFIDENCE_FAILED: 3;
+    /** Confidence summary file missing (test infra crashed) */
+    readonly CONFIDENCE_MISSING: 4;
+};
+export type GateExitCode = (typeof GATE_EXIT)[keyof typeof GATE_EXIT];
+/** Categories written to regression-report.json `category` field. */
+export declare const GATE_CATEGORY: {
+    readonly PASS: "pass";
+    readonly REGRESSION: "regression";
+    readonly INFRA_ERROR: "infra_error";
+};
+export type GateCategory = (typeof GATE_CATEGORY)[keyof typeof GATE_CATEGORY];
+/** Current schema version for regression-report.json. */
+export declare const REPORT_SCHEMA_VERSION = 1;
+export interface RegressionDelta {
+    metric: string;
+    baseline: number | string;
+    current: number | string;
+    delta: string;
+    status: "pass" | "fail";
+}
+export interface RegressionReport {
+    schemaVersion: number;
+    timestamp: string;
+    exitCode: GateExitCode;
+    category: GateCategory;
+    passed: boolean;
+    failures: string[];
+    deltas: RegressionDelta[];
+}
+export interface BaselineTolerance {
+    scoreDrop: number;
+    passRateDrop: number;
+    maxLatencyIncreaseMs: number;
+    maxCostIncreaseUsd: number;
+}
+export interface Baseline {
+    schemaVersion: number;
+    description: string;
+    generatedAt: string;
+    generatedBy: string;
+    commitSha: string;
+    updatedAt: string;
+    updatedBy: string;
+    tolerance: BaselineTolerance;
+    goldenEval: {
+        score: number;
+        passRate: number;
+        totalCases: number;
+        passedCases: number;
+    };
+    qualityScore: {
+        overall: number;
+        grade: string;
+        accuracy: number;
+        safety: number;
+        latency: number;
+        cost: number;
+        consistency: number;
+    };
+    confidenceTests: {
+        unitPassed: boolean;
+        unitTotal: number;
+        dbPassed: boolean;
+        dbTotal: number;
+    };
+    productMetrics: {
+        p95ApiLatencyMs?: number;
+        goldenCostUsd?: number;
+    };
+    qualityMetrics?: {
+        unitLaneDurationMs?: number;
+        dbLaneDurationMs?: number;
+    };
+}
+/** Well-known artifact paths relative to project root. */
+export declare const ARTIFACTS: {
+    readonly BASELINE: "evals/baseline.json";
+    readonly REGRESSION_REPORT: "evals/regression-report.json";
+    readonly CONFIDENCE_SUMMARY: "evals/confidence-summary.json";
+    readonly LATENCY_BENCHMARK: "evals/latency-benchmark.json";
+};

package/dist/regression.js

@@ -0,0 +1,44 @@
+"use strict";
+/**
+ * Regression gate constants and types.
+ *
+ * These mirror the contracts defined in scripts/regression-gate.ts
+ * and evals/schemas/regression-report.schema.json so that SDK consumers
+ * can programmatically inspect gate results without parsing strings.
+ *
+ * @packageDocumentation
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ARTIFACTS = exports.REPORT_SCHEMA_VERSION = exports.GATE_CATEGORY = exports.GATE_EXIT = void 0;
+// ── Exit codes ──
+/** Exit codes emitted by `evalgate gate` / `scripts/regression-gate.ts`. */
+exports.GATE_EXIT = {
+    /** Gate passed — no regressions detected */
+    PASS: 0,
+    /** One or more regression thresholds exceeded */
+    REGRESSION: 1,
+    /** Infrastructure error (baseline missing, summary missing, etc.) */
+    INFRA_ERROR: 2,
+    /** Confidence tests failed (test suite red) */
+    CONFIDENCE_FAILED: 3,
+    /** Confidence summary file missing (test infra crashed) */
+    CONFIDENCE_MISSING: 4,
+};
+// ── Report categories ──
+/** Categories written to regression-report.json `category` field. */
+exports.GATE_CATEGORY = {
+    PASS: "pass",
+    REGRESSION: "regression",
+    INFRA_ERROR: "infra_error",
+};
+// ── Schema version ──
+/** Current schema version for regression-report.json. */
+exports.REPORT_SCHEMA_VERSION = 1;
+// ── Artifact paths ──
+/** Well-known artifact paths relative to project root. */
+exports.ARTIFACTS = {
+    BASELINE: "evals/baseline.json",
+    REGRESSION_REPORT: "evals/regression-report.json",
+    CONFIDENCE_SUMMARY: "evals/confidence-summary.json",
+    LATENCY_BENCHMARK: "evals/latency-benchmark.json",
+};

package/dist/runtime/adapters/config-to-dsl.d.ts

@@ -0,0 +1,33 @@
+/**
+ * Config → DSL Adapter - LAYER 2 Compatibility Bridge
+ *
+ * Migrates existing evalgate.config.json and TestSuite configurations
+ * to the new defineEval() DSL without breaking user workflows.
+ */
+import type { TestSuite } from "../../testing";
+/**
+ * Migration result information
+ */
+interface MigrationResult {
+    success: boolean;
+    specsGenerated: number;
+    errors: string[];
+    warnings: string[];
+    outputPath: string;
+}
+/**
+ * Convert TestSuite to defineEval() specifications
+ */
+export declare function migrateTestSuiteToDSL(testSuite: TestSuite, outputPath: string): MigrationResult;
+/**
+ * Convert evalgate.config.json to DSL specifications
+ */
+export declare function migrateConfigToDSL(configPath: string, outputPath: string): MigrationResult;
+/**
+ * Discover and migrate all TestSuite configurations in a project
+ */
+export declare function migrateProjectToDSL(projectRoot: string, options?: {
+    outputDir?: string;
+    dryRun?: boolean;
+}): MigrationResult;
+export {};