@evalgate/sdk 2.0.0

package/dist/runtime/run-report.js

@@ -0,0 +1,222 @@
+"use strict";
+/**
+ * RUNTIME-104: Deterministic Report Serialization (RunReport v1)
+ *
+ * Stable report format for downstream processing (explain, diff, history).
+ * Mirrors CheckReport conventions for consistency.
+ */
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.RunReportBuilder = exports.RUN_REPORT_SCHEMA_VERSION = void 0;
+exports.createRunReport = createRunReport;
+exports.parseRunReport = parseRunReport;
+/**
+ * RunReport schema version - increment when breaking changes occur
+ */
+exports.RUN_REPORT_SCHEMA_VERSION = "1";
+/**
+ * RunReport builder for creating deterministic reports
+ */
+class RunReportBuilder {
+    /**
+     * Initialize report with basic metadata
+     */
+    constructor(runId, runtimeInfo) {
+        this.report = {
+            schemaVersion: exports.RUN_REPORT_SCHEMA_VERSION,
+            results: [],
+            failures: [],
+            summary: {
+                total: 0,
+                passed: 0,
+                failed: 0,
+                errors: 0,
+                timeouts: 0,
+                passRate: 0,
+                averageScore: 0,
+                totalDurationMs: 0,
+                success: true,
+            },
+        };
+        this.report.startedAt = new Date().toISOString();
+        this.report.runId = runId;
+        this.report.runtime = runtimeInfo;
+    }
+    /**
+     * Add a test result to the report
+     */
+    addResult(testId, testName, filePath, position, input, result) {
+        const runResult = {
+            testId,
+            testName,
+            filePath,
+            position,
+            input,
+            pass: result.pass,
+            score: result.score,
+            durationMs: result.durationMs || 0,
+            metadata: result.metadata,
+            tags: [], // TODO: Extract from spec
+            assertions: result.assertions?.map((assertion, index) => ({
+                name: assertion.name || `assertion-${index}`,
+                passed: assertion.passed,
+                message: assertion.message,
+            })),
+        };
+        this.report.results?.push(runResult);
+        // Update summary
+        this.updateSummary(result);
+        // Add to failures if needed
+        if (!result.pass ||
+            result.classification === "error" ||
+            result.classification === "timeout") {
+            this.addFailure(testId, testName, filePath, position, result);
+        }
+    }
+    /**
+     * Update summary statistics
+     */
+    updateSummary(result) {
+        const summary = this.report.summary;
+        summary.total++;
+        summary.totalDurationMs += result.durationMs || 0;
+        if (result.pass) {
+            summary.passed++;
+        }
+        else if (result.classification === "error") {
+            summary.errors++;
+            summary.success = false;
+        }
+        else if (result.classification === "timeout") {
+            summary.timeouts++;
+            summary.success = false;
+        }
+        else {
+            summary.failed++;
+        }
+        // Calculate rates and averages
+        summary.passRate =
+            summary.total > 0 ? (summary.passed / summary.total) * 100 : 0;
+        // Average score calculation (excluding errors/timeouts)
+        const scoredResults = this.report.results?.filter((r) => r.score > 0) || [];
+        summary.averageScore =
+            scoredResults.length > 0
+                ? scoredResults.reduce((sum, r) => sum + r.score, 0) /
+                    scoredResults.length
+                : 0;
+    }
+    /**
+     * Add a failure to the report
+     */
+    addFailure(testId, testName, filePath, position, result) {
+        const failure = {
+            testId,
+            testName,
+            filePath,
+            position,
+            classification: result.classification === "error"
+                ? "error"
+                : result.classification === "timeout"
+                    ? "timeout"
+                    : "failed",
+            errorEnvelope: result.errorEnvelope,
+            message: result.error || "Test failed",
+            timestamp: new Date().toISOString(),
+        };
+        this.report.failures?.push(failure);
+    }
+    /**
+     * Set execution configuration
+     */
+    setConfig(config) {
+        this.report.config = {
+            executorType: "local",
+            defaultTimeout: 30000,
+            environment: {
+                nodeVersion: process.version,
+                platform: process.platform,
+                arch: process.arch,
+            },
+            ...config,
+        };
+    }
+    /**
+     * Finalize and return the complete report
+     */
+    build() {
+        // Sort results and failures by testId for determinism
+        this.report.results?.sort((a, b) => a.testId.localeCompare(b.testId));
+        this.report.failures?.sort((a, b) => a.testId.localeCompare(b.testId));
+        // Set completion timestamp
+        this.report.finishedAt = new Date().toISOString();
+        const finalReport = this.report;
+        // Add toJSON method
+        finalReport.toJSON = () => JSON.stringify(finalReport, null, 2);
+        return finalReport;
+    }
+    /**
+     * Serialize report to JSON string
+     * Ensures deterministic output
+     */
+    toJSON() {
+        const report = this.build();
+        return JSON.stringify(report, null, 2);
+    }
+    /**
+     * Write report to file
+     */
+    async writeToFile(filePath) {
+        const fs = await Promise.resolve().then(() => __importStar(require("node:fs/promises")));
+        await fs.writeFile(filePath, this.toJSON(), "utf-8");
+    }
+}
+exports.RunReportBuilder = RunReportBuilder;
+/**
+ * Create a new RunReport builder
+ */
+function createRunReport(runId, runtimeInfo) {
+    return new RunReportBuilder(runId, runtimeInfo);
+}
+/**
+ * Parse a RunReport from JSON string
+ */
+function parseRunReport(json) {
+    const report = JSON.parse(json);
+    // Validate schema version
+    if (report.schemaVersion !== exports.RUN_REPORT_SCHEMA_VERSION) {
+        throw new Error(`Unsupported RunReport schema version: ${report.schemaVersion}`);
+    }
+    return report;
+}

package/dist/runtime/types.d.ts

@@ -0,0 +1,356 @@
+/**
+ * EvalGate Runtime Types - Layer 1 Foundation
+ *
+ * Core types for the evaluation specification programming model.
+ * Everything revolves around the Evaluation Specification primitive.
+ */
+import type { AssertionResult } from "../assertions";
+/**
+ * Core evaluation specification - the single primitive
+ * Represents a behavioral specification test
+ */
+export interface EvalSpec {
+    /** Content-addressable unique identifier */
+    id: string;
+    /** Human-readable name */
+    name: string;
+    /** File path where this spec is defined */
+    filePath: string;
+    /** AST position for identity stability */
+    position: {
+        line: number;
+        column: number;
+    };
+    /** Specification description */
+    description?: string;
+    /** Categorization tags */
+    tags?: string[];
+    /** Specification execution function */
+    executor: EvalExecutor;
+    /** Specification metadata */
+    metadata?: Record<string, unknown>;
+    /** Execution configuration */
+    config?: {
+        timeout?: number;
+        retries?: number;
+        budget?: string;
+        model?: string | "auto";
+    };
+}
+/**
+ * Specification execution context
+ */
+export interface EvalContext {
+    /** Test input data */
+    input: string;
+    /** Context metadata */
+    metadata?: Record<string, unknown>;
+    /** Execution options */
+    options?: EvalOptions;
+}
+/**
+ * Execution options
+ */
+export interface EvalOptions {
+    /** Model selection */
+    model?: string | "auto";
+    /** Budget constraint */
+    budget?: string;
+    /** Timeout override */
+    timeout?: number;
+    /** Custom execution parameters */
+    [key: string]: unknown;
+}
+/**
+ * Specification executor function
+ */
+export type EvalExecutor = (context: EvalContext) => Promise<EvalResult>;
+/**
+ * Specification execution result
+ */
+export interface EvalResult {
+    /** Pass/fail determination */
+    pass: boolean;
+    /** Numeric score (0-100) */
+    score: number;
+    /** Assertion results */
+    assertions?: AssertionResult[];
+    /** Result metadata */
+    metadata?: Record<string, unknown>;
+    /** Execution duration */
+    durationMs?: number;
+    /** Execution error if failed */
+    error?: string;
+}
+/**
+ * Scoped runtime context - prevents cross-run contamination
+ */
+export interface EvalRuntime {
+    /** Runtime unique identifier */
+    id: string;
+    /** Project namespace */
+    namespace: string;
+    /** Runtime creation timestamp */
+    createdAt: Date;
+    /** Registered specifications */
+    specs: Map<string, EvalSpec>;
+    /** Runtime statistics */
+    stats: RuntimeStats;
+    /** Register a new specification */
+    register(spec: EvalSpec): void;
+    /** Get specification by ID */
+    get(id: string): EvalSpec | undefined;
+    /** List all specifications */
+    list(): EvalSpec[];
+    /** Find specifications by criteria */
+    find(criteria: SpecSearchCriteria): EvalSpec[];
+    /** Clear all specifications (dispose) */
+    clear(): void;
+    /** Get runtime health metrics */
+    getHealth(): RuntimeHealth;
+}
+/**
+ * Specification search criteria
+ */
+export interface SpecSearchCriteria {
+    tags?: string[];
+    files?: string[];
+    names?: string[];
+    metadata?: Record<string, unknown>;
+}
+/**
+ * Runtime statistics
+ */
+export interface RuntimeStats {
+    /** Total registered specs */
+    totalSpecs: number;
+    /** Specs by tag */
+    specsByTag: Record<string, number>;
+    /** Specs by file */
+    specsByFile: Record<string, number>;
+    /** Memory usage estimate */
+    memoryUsage: number;
+    /** Last updated timestamp */
+    lastUpdated: Date;
+}
+/**
+ * Runtime health information
+ */
+export interface RuntimeHealth {
+    /** Runtime status */
+    status: "healthy" | "warning" | "error";
+    /** Memory usage in bytes */
+    memoryUsage: number;
+    /** Number of registered specs */
+    specCount: number;
+    /** Health issues if any */
+    issues: string[];
+}
+/**
+ * Content-addressable identity components
+ */
+export interface SpecIdentity {
+    /** Project namespace hash */
+    namespace: string;
+    /** Relative file path */
+    filePath: string;
+    /** Specification name */
+    name: string;
+    /** Suite path if nested */
+    suitePath?: string;
+    /** AST position */
+    position: {
+        line: number;
+        column: number;
+    };
+}
+/**
+ * Specification definition function signature
+ * This is the defineEval() DSL function signature
+ */
+export interface DefineEvalFunction {
+    /**
+     * Define a behavioral specification
+     * @param name - Human-readable specification name
+     * @param executor - Specification execution function
+     * @param options - Optional configuration
+     */
+    <TInput = string>(name: string, executor: (context: EvalContext & {
+        input: TInput;
+    }) => Promise<EvalResult>, options?: SpecOptions): void;
+    /**
+     * Define a specification with full configuration
+     * @param config - Complete specification configuration
+     */
+    (config: SpecConfig): void;
+}
+/**
+ * Specification definition options
+ */
+export interface SpecOptions {
+    /** Specification description */
+    description?: string;
+    /** Categorization tags */
+    tags?: string[];
+    /** Execution timeout */
+    timeout?: number;
+    /** Retry attempts */
+    retries?: number;
+    /** Budget constraint */
+    budget?: string;
+    /** Model selection */
+    model?: string | "auto";
+    /** Specification metadata */
+    metadata?: Record<string, unknown>;
+    /** Dependencies for impact analysis */
+    dependsOn?: {
+        prompts?: string[];
+        datasets?: string[];
+        tools?: string[];
+        code?: string[];
+    };
+}
+/**
+ * Complete specification configuration
+ */
+export interface SpecConfig extends SpecOptions {
+    /** Specification name (required) */
+    name: string;
+    /** Specification executor (required) */
+    executor: EvalExecutor;
+}
+/**
+ * Execution interface - abstraction layer
+ * Prevents worker coupling and enables multiple backends
+ */
+export interface EvalExecutorInterface {
+    /** Execute a single specification */
+    executeSpec(spec: EvalSpec, input: string): Promise<EvalResult>;
+    /** Execute multiple specifications */
+    executeSpecs(specs: EvalSpec[], inputs: string[]): Promise<EvalResult[]>;
+    /** Get executor capabilities */
+    getCapabilities(): ExecutorCapabilities;
+}
+/**
+ * Executor capabilities
+ */
+export interface ExecutorCapabilities {
+    /** Executor type */
+    type: "local" | "cloud" | "worker";
+    /** Parallel execution support */
+    parallel: boolean;
+    /** Maximum parallel workers */
+    maxParallel?: number;
+    /** Supported models */
+    supportedModels: string[];
+    /** Cost tracking support */
+    costTracking: boolean;
+    /** Streaming support */
+    streaming: boolean;
+}
+/**
+ * Local executor implementation
+ */
+export interface LocalExecutor extends EvalExecutorInterface {
+    type: "local";
+}
+/**
+ * Cloud executor implementation
+ */
+export interface CloudExecutor extends EvalExecutorInterface {
+    type: "cloud";
+    /** Cloud provider */
+    provider: string;
+    /** Region */
+    region?: string;
+}
+/**
+ * Worker executor implementation
+ */
+export interface WorkerExecutor extends EvalExecutorInterface {
+    type: "worker";
+    /** Worker pool configuration */
+    workerPool: {
+        size: number;
+        maxMemory: number;
+        timeout: number;
+    };
+}
+/**
+ * Runtime errors
+ */
+export declare class EvalRuntimeError extends Error {
+    code: string;
+    details?: Record<string, unknown> | undefined;
+    constructor(message: string, code: string, details?: Record<string, unknown> | undefined);
+}
+export declare class SpecRegistrationError extends EvalRuntimeError {
+    constructor(message: string, details?: Record<string, unknown>);
+}
+export declare class SpecExecutionError extends EvalRuntimeError {
+    constructor(message: string, details?: Record<string, unknown>);
+}
+export declare class RuntimeError extends EvalRuntimeError {
+    constructor(message: string, details?: Record<string, unknown>);
+}
+/**
+ * EvalExecutionError wrapper for safe error boundaries
+ */
+export declare class EvalExecutionError extends Error {
+    readonly name = "EvalExecutionError";
+    readonly code: string;
+    readonly testId: string;
+    readonly filePath: string;
+    readonly position: {
+        line: number;
+        column: number;
+    };
+    readonly originalError?: Error;
+    constructor(message: string, context: {
+        testId: string;
+        filePath: string;
+        position: {
+            line: number;
+            column: number;
+        };
+        code?: string;
+        originalError?: Error;
+    });
+    /**
+     * Convert to normalized error envelope for reporting
+     */
+    toEnvelope(): ExecutionErrorEnvelope;
+}
+/**
+ * Normalized error envelope for reporting
+ */
+export interface ExecutionErrorEnvelope {
+    /** Error classification */
+    classification: "execution_error" | "timeout_error" | "assertion_error" | "system_error";
+    /** Error code for programmatic handling */
+    code: string;
+    /** Human-readable error message */
+    message: string;
+    /** Stack trace if available */
+    stack?: string;
+    /** Test specification ID */
+    testId: string;
+    /** File where error occurred */
+    filePath: string;
+    /** Position in file where error occurred */
+    position: {
+        line: number;
+        column: number;
+    };
+    /** Error timestamp */
+    timestamp: string;
+}
+/**
+ * Enhanced execution result with error classification
+ */
+export interface EnhancedEvalResult extends EvalResult {
+    /** Error envelope if execution failed */
+    errorEnvelope?: ExecutionErrorEnvelope;
+    /** Execution classification */
+    classification: "passed" | "failed" | "error" | "timeout";
+}

package/dist/runtime/types.js

@@ -0,0 +1,76 @@
+"use strict";
+/**
+ * EvalGate Runtime Types - Layer 1 Foundation
+ *
+ * Core types for the evaluation specification programming model.
+ * Everything revolves around the Evaluation Specification primitive.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.EvalExecutionError = exports.RuntimeError = exports.SpecExecutionError = exports.SpecRegistrationError = exports.EvalRuntimeError = void 0;
+/**
+ * Runtime errors
+ */
+class EvalRuntimeError extends Error {
+    constructor(message, code, details) {
+        super(message);
+        this.code = code;
+        this.details = details;
+        this.name = "EvalRuntimeError";
+    }
+}
+exports.EvalRuntimeError = EvalRuntimeError;
+class SpecRegistrationError extends EvalRuntimeError {
+    constructor(message, details) {
+        super(message, "SPEC_REGISTRATION_ERROR", details);
+        this.name = "SpecRegistrationError";
+    }
+}
+exports.SpecRegistrationError = SpecRegistrationError;
+class SpecExecutionError extends EvalRuntimeError {
+    constructor(message, details) {
+        super(message, "SPEC_EXECUTION_ERROR", details);
+        this.name = "SpecExecutionError";
+    }
+}
+exports.SpecExecutionError = SpecExecutionError;
+class RuntimeError extends EvalRuntimeError {
+    constructor(message, details) {
+        super(message, "RUNTIME_ERROR", details);
+        this.name = "RuntimeError";
+    }
+}
+exports.RuntimeError = RuntimeError;
+/**
+ * EvalExecutionError wrapper for safe error boundaries
+ */
+class EvalExecutionError extends Error {
+    constructor(message, context) {
+        super(message);
+        this.name = "EvalExecutionError";
+        this.code = context.code || "EXECUTION_ERROR";
+        this.testId = context.testId;
+        this.filePath = context.filePath;
+        this.position = context.position;
+        this.originalError = context.originalError;
+        // Maintain proper stack trace
+        if (Error.captureStackTrace) {
+            Error.captureStackTrace(this, EvalExecutionError);
+        }
+    }
+    /**
+     * Convert to normalized error envelope for reporting
+     */
+    toEnvelope() {
+        return {
+            classification: "execution_error",
+            code: this.code,
+            message: this.message,
+            stack: this.stack,
+            testId: this.testId,
+            filePath: this.filePath,
+            position: this.position,
+            timestamp: new Date().toISOString(),
+        };
+    }
+}
+exports.EvalExecutionError = EvalExecutionError;