@runtypelabs/sdk 1.7.2 → 1.8.0

package/dist/eval-builder.js

@@ -1,225 +0,0 @@
-"use strict";
-/**
- * EvalBuilder - Fluent builder for evaluation runs
- *
- * Provides a chainable API for building evaluation configurations
- * that test flows against records with model overrides and comparisons.
- *
- * @example
- * ```typescript
- * import { EvalBuilder } from '@runtypelabs/sdk'
- *
- * // Single model eval with overrides
- * const eval1 = await new EvalBuilder()
- *   .useFlow('flow_abc123')
- *   .forRecordType('test_data')
- *   .withModelOverrides([{ stepName: 'Analyze', model: 'gpt-4o' }])
- *   .run(apiClient)
- *
- * // Multi-model comparison
- * const eval2 = await new EvalBuilder()
- *   .useFlow('flow_abc123')
- *   .forRecordType('test_data')
- *   .compareModels([
- *     { stepName: 'Analyze', model: 'gpt-4o' },
- *     { stepName: 'Analyze', model: 'claude-3-opus' },
- *   ])
- *   .run(apiClient)
- *
- * // Virtual flow eval
- * const eval3 = await new EvalBuilder()
- *   .useVirtualFlow(flowBuilder)
- *   .forRecordType('test_data')
- *   .run(apiClient)
- * ```
- */
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.ClientEvalBuilder = exports.EvalBuilder = void 0;
-// ============================================================================
-// EvalBuilder Class
-// ============================================================================
-class EvalBuilder {
-    constructor() {
-        this.evalOptions = {};
-    }
-    /**
-     * Use an existing flow by ID for evaluation
-     */
-    useFlow(flowId) {
-        this.flowId = flowId;
-        this.virtualFlow = undefined;
-        return this;
-    }
-    /**
-     * Use a virtual flow (from FlowBuilder) for evaluation
-     *
-     * @example
-     * ```typescript
-     * const flowBuilder = new FlowBuilder()
-     *   .createFlow({ name: 'Test Flow' })
-     *   .prompt({ name: 'Analyze', model: 'gpt-4', userPrompt: '...' })
-     *
-     * const eval = await new EvalBuilder()
-     *   .useVirtualFlow(flowBuilder)
-     *   .forRecordType('test_data')
-     *   .run(apiClient)
-     * ```
-     */
-    useVirtualFlow(builder) {
-        const config = builder.build();
-        if (config.flow && 'name' in config.flow && 'steps' in config.flow) {
-            this.virtualFlow = {
-                name: config.flow.name,
-                steps: config.flow.steps,
-            };
-        }
-        this.flowId = undefined;
-        return this;
-    }
-    /**
-     * Specify the record type to evaluate against
-     */
-    forRecordType(recordType) {
-        this.recordType = recordType;
-        this.inlineRecords = undefined;
-        return this;
-    }
-    /**
-     * Use inline records for ad-hoc evaluation (no record type lookup)
-     *
-     * @example
-     * ```typescript
-     * .withRecords([
-     *   { name: 'Test 1', type: 'test', metadata: { input: 'hello' } },
-     *   { name: 'Test 2', type: 'test', metadata: { input: 'world' } },
-     * ])
-     * ```
-     */
-    withRecords(records) {
-        this.inlineRecords = records;
-        this.recordType = undefined;
-        return this;
-    }
-    /**
-     * Set model overrides for single-model evaluation
-     *
-     * @example
-     * ```typescript
-     * .withModelOverrides([
-     *   { stepName: 'Analyze', model: 'gpt-4o', temperature: 0.2 }
-     * ])
-     * ```
-     */
-    withModelOverrides(overrides) {
-        this.modelOverrides = overrides;
-        this.modelConfigs = undefined;
-        return this;
-    }
-    /**
-     * Compare multiple model configurations
-     *
-     * @example
-     * ```typescript
-     * .compareModels([
-     *   [{ stepName: 'Analyze', model: 'gpt-4o' }],
-     *   [{ stepName: 'Analyze', model: 'claude-3-opus' }],
-     *   [{ stepName: 'Analyze', model: 'gemini-pro' }],
-     * ])
-     * ```
-     */
-    compareModels(configs) {
-        this.modelConfigs = configs;
-        this.modelOverrides = undefined;
-        return this;
-    }
-    /**
-     * Set evaluation options
-     */
-    withOptions(options) {
-        this.evalOptions = { ...this.evalOptions, ...options };
-        return this;
-    }
-    /**
-     * Filter records to evaluate
-     */
-    withFilter(filter) {
-        this.filterConfig = filter;
-        return this;
-    }
-    /**
-     * Limit the number of records to evaluate
-     */
-    withLimit(limit) {
-        this.limitConfig = limit;
-        return this;
-    }
-    /**
-     * Build the eval request configuration
-     */
-    build() {
-        if (!this.flowId && !this.virtualFlow) {
-            throw new Error('EvalBuilder: flow is required. Call .useFlow(id) or .useVirtualFlow(builder) first.');
-        }
-        if (!this.recordType && !this.inlineRecords) {
-            throw new Error('EvalBuilder: records are required. Call .forRecordType(type) or .withRecords([...]) first.');
-        }
-        const request = {};
-        if (this.flowId) {
-            request.flowId = this.flowId;
-        }
-        else if (this.virtualFlow) {
-            request.flow = this.virtualFlow;
-        }
-        if (this.recordType) {
-            request.recordType = this.recordType;
-        }
-        else if (this.inlineRecords) {
-            request.records = this.inlineRecords;
-        }
-        if (this.modelOverrides) {
-            request.modelOverrides = this.modelOverrides;
-        }
-        else if (this.modelConfigs) {
-            request.modelConfigs = this.modelConfigs;
-        }
-        if (Object.keys(this.evalOptions).length > 0) {
-            request.options = this.evalOptions;
-        }
-        if (this.filterConfig) {
-            request.filter = this.filterConfig;
-        }
-        if (this.limitConfig !== undefined) {
-            request.limit = this.limitConfig;
-        }
-        return request;
-    }
-    /**
-     * Execute the evaluation
-     *
-     * @param client - Client with eval capability
-     * @returns EvalResult with evaluation status and results
-     */
-    async run(client) {
-        const config = this.build();
-        return client.eval(config);
-    }
-}
-exports.EvalBuilder = EvalBuilder;
-/**
- * EvalBuilder that is bound to a client for direct execution
- */
-class ClientEvalBuilder extends EvalBuilder {
-    constructor(client) {
-        super();
-        this.boundClient = client;
-    }
-    /**
-     * Execute the evaluation using the bound client
-     */
-    async run() {
-        const config = this.build();
-        return this.boundClient.eval(config);
-    }
-}
-exports.ClientEvalBuilder = ClientEvalBuilder;
-//# sourceMappingURL=eval-builder.js.map

package/dist/eval-builder.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"eval-builder.js","sourceRoot":"","sources":["../src/eval-builder.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAiCG;;;AAkFH,+EAA+E;AAC/E,oBAAoB;AACpB,+EAA+E;AAE/E,MAAa,WAAW;IAAxB;QAOU,gBAAW,GAAgB,EAAE,CAAA;IAwLvC,CAAC;IApLC;;OAEG;IACH,OAAO,CAAC,MAAc;QACpB,IAAI,CAAC,MAAM,GAAG,MAAM,CAAA;QACpB,IAAI,CAAC,WAAW,GAAG,SAAS,CAAA;QAC5B,OAAO,IAAI,CAAA;IACb,CAAC;IAED;;;;;;;;;;;;;;OAcG;IACH,cAAc,CAAC,OAAoB;QACjC,MAAM,MAAM,GAAG,OAAO,CAAC,KAAK,EAAE,CAAA;QAC9B,IAAI,MAAM,CAAC,IAAI,IAAI,MAAM,IAAI,MAAM,CAAC,IAAI,IAAI,OAAO,IAAI,MAAM,CAAC,IAAI,EAAE,CAAC;YACnE,IAAI,CAAC,WAAW,GAAG;gBACjB,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,IAAc;gBAChC,KAAK,EAAE,MAAM,CAAC,IAAI,CAAC,KAAc;aAClC,CAAA;QACH,CAAC;QACD,IAAI,CAAC,MAAM,GAAG,SAAS,CAAA;QACvB,OAAO,IAAI,CAAA;IACb,CAAC;IAED;;OAEG;IACH,aAAa,CAAC,UAAkB;QAC9B,IAAI,CAAC,UAAU,GAAG,UAAU,CAAA;QAC5B,IAAI,CAAC,aAAa,GAAG,SAAS,CAAA;QAC9B,OAAO,IAAI,CAAA;IACb,CAAC;IAED;;;;;;;;;;OAUG;IACH,WAAW,CAAC,OAAqB;QAC/B,IAAI,CAAC,aAAa,GAAG,OAAO,CAAA;QAC5B,IAAI,CAAC,UAAU,GAAG,SAAS,CAAA;QAC3B,OAAO,IAAI,CAAA;IACb,CAAC;IAED;;;;;;;;;OASG;IACH,kBAAkB,CAAC,SAA0B;QAC3C,IAAI,CAAC,cAAc,GAAG,SAAS,CAAA;QAC/B,IAAI,CAAC,YAAY,GAAG,SAAS,CAAA;QAC7B,OAAO,IAAI,CAAA;IACb,CAAC;IAED;;;;;;;;;;;OAWG;IACH,aAAa,CAAC,OAA0B;QACtC,IAAI,CAAC,YAAY,GAAG,OAAO,CAAA;QAC3B,IAAI,CAAC,cAAc,GAAG,SAAS,CAAA;QAC/B,OAAO,IAAI,CAAA;IACb,CAAC;IAED;;OAEG;IACH,WAAW,CAAC,OAAoB;QAC9B,IAAI,CAAC,WAAW,GAAG,EAAE,GAAG,IAAI,CAAC,WAAW,EAAE,GAAG,OAAO,EAAE,CAAA;QACtD,OAAO,IAAI,CAAA;IACb,CAAC;IAED;;OAEG;IACH,UAAU,CAAC,MAA2B;QACpC,IAAI,CAAC,YAAY,GAAG,MAAM,CAAA;QAC1B,OAAO,IAAI,CAAA;IACb,CAAC;IAED;;OAEG;IACH,SAAS,CAAC,KAAa;QACrB,IAAI,CAAC,WAAW,GAAG,KAAK,CAAA;QACxB,OAAO,IAAI,CAAA;IACb,CAAC;IAED;;OAEG;IACH,KAAK;QACH,IAAI,CAAC,IAAI,CAAC,MAAM,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;YACtC,MAAM,IAAI,KAAK,CACb,qFAAqF,CACtF,CAAA;QACH,CAAC;QACD,IAAI,CAAC,IAAI,CAAC,UAAU,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,CAAC;YAC5C,MAAM,IAAI,KAAK,CACb,4FAA4F,CAC7F,CAAA;QACH,CAAC;QAED,MAAM,OAAO,GAAgB,EAAE,CAAA;QAE/B,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YAChB,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAA;QAC9B,CAAC;aAAM,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;YAC5B,OAAO,CAAC,IAAI,GAAG,IAAI,CAAC,WAAW,CAAA;QACjC,CAAC;QAED,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YACpB,OAAO,CAAC,UAAU,GAAG,IAAI,CAAC,UAAU,CAAA;QACtC,CAAC;aAAM,IAAI,IAAI,CAAC,aAAa,EAAE,CAAC;YAC9B,OAAO,CAAC,OAAO,GAAG,IAAI,CAAC,aAAa,CAAA;QACtC,CAAC;QAED,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;YACxB,OAAO,CAAC,cAAc,GAAG,IAAI,CAAC,cAAc,CAAA;QAC9C,CAAC;aAAM,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;YAC7B,OAAO,CAAC,YAAY,GAAG,IAAI,CAAC,YAAY,CAAA;QAC1C,CAAC;QAED,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC7C,OAAO,CAAC,OAAO,GAAG,IAAI,CAAC,WAAW,CAAA;QACpC,CAAC;QAED,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;YACtB,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,YAAY,CAAA;QACpC,CAAC;QAED,IAAI,IAAI,CAAC,WAAW,KAAK,SAAS,EAAE,CAAC;YACnC,OAAO,CAAC,KAAK,GAAG,IAAI,CAAC,WAAW,CAAA;QAClC,CAAC;QAED,OAAO,OAAO,CAAA;IAChB,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,GAAG,CAAC,MAAkB;QAC1B,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,EAAE,CAAA;QAC3B,OAAO,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;IAC5B,CAAC;CACF;AA/LD,kCA+LC;AAaD;;GAEG;AACH,MAAa,iBAAkB,SAAQ,WAAW;IAGhD,YAAY,MAAkB;QAC5B,KAAK,EAAE,CAAA;QACP,IAAI,CAAC,WAAW,GAAG,MAAM,CAAA;IAC3B,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,GAAG;QACP,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,EAAE,CAAA;QAC3B,OAAO,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;IACtC,CAAC;CACF;AAfD,8CAeC"}

package/dist/evals-namespace.d.ts

@@ -1,205 +0,0 @@
-/**
- * EvalsNamespace - Static namespace for evaluation operations
- *
- * Provides methods for running evaluations and comparing model performance.
- * Evals can be streamed for real-time results or submitted as batch jobs.
- */
-import type { RuntypeClient } from './runtype';
-import { FlowResult } from './flow-result';
-import type { StreamCallbacks } from './flow-builder';
-export interface ModelOverride {
-    /** Name of the step to override */
-    stepName: string;
-    /** Model to use for this step */
-    model: string;
-    /** Optional temperature override */
-    temperature?: number;
-    /** Optional max tokens override */
-    maxTokens?: number;
-}
-export interface EvalRunConfig {
-    /** Flow ID to evaluate */
-    flowId?: string;
-    /** Virtual flow definition (alternative to flowId) */
-    flow?: {
-        name: string;
-        steps: any[];
-    };
-    /** Record type to evaluate against */
-    recordType?: string;
-    /** Inline records for ad-hoc evaluation */
-    records?: Array<{
-        name: string;
-        type: string;
-        metadata: Record<string, any>;
-    }>;
-    /** Model overrides for single-model evaluation */
-    models?: ModelOverride[];
-    /** Multiple model configurations for comparison */
-    compareModels?: ModelOverride[][];
-    /** Store evaluation results */
-    storeResults?: boolean;
-    /** Run evaluations in parallel */
-    parallel?: boolean;
-    /** Maximum concurrent evaluations */
-    concurrency?: number;
-    /** Continue on individual record failures */
-    continueOnError?: boolean;
-    /** Optional filter for records */
-    filter?: Record<string, any>;
-    /** Optional limit on number of records */
-    limit?: number;
-}
-export interface EvalStatus {
-    evalId: string;
-    status: 'queued' | 'running' | 'completed' | 'failed';
-    totalRecords: number;
-    completedRecords: number;
-    failedRecords: number;
-    results?: Array<{
-        modelConfig: ModelOverride[];
-        records: Array<{
-            recordId: string;
-            status: 'success' | 'error';
-            result?: any;
-            error?: string;
-            executionTime: number;
-        }>;
-    }>;
-}
-export interface EvalListParams {
-    /** Filter by status */
-    status?: 'queued' | 'running' | 'completed' | 'failed';
-    /** Filter by flow ID */
-    flowId?: string;
-    /** Pagination limit */
-    limit?: number;
-    /** Pagination offset */
-    offset?: number;
-}
-/**
- * EvalRunner - Builder returned by Runtype.evals.run()
- *
- * Provides terminal methods for executing evaluations:
- * - .stream() - Execute and stream results in real-time
- * - .submit() - Submit as a batch job for async processing
- */
-export declare class EvalRunner {
-    private getClient;
-    private config;
-    constructor(getClient: () => RuntypeClient, config: EvalRunConfig);
-    /**
-     * Execute the evaluation with streaming results
-     *
-     * Streams evaluation results as they complete.
-     * Good for real-time feedback during development/testing.
-     *
-     * @example
-     * ```typescript
-     * const result = await Runtype.evals.run({
-     *   flowId: 'flow_123',
-     *   recordType: 'test_data',
-     *   models: [{ stepName: 'Analyze', model: 'gpt-4o' }]
-     * }).stream()
-     *
-     * // Process with callbacks
-     * await result.stream({
-     *   onStepComplete: (result, event) => {
-     *     console.log('Step completed:', event.name, result)
-     *   },
-     * })
-     * ```
-     */
-    stream(callbacks?: StreamCallbacks): Promise<FlowResult>;
-    /**
-     * Submit the evaluation as a batch job
-     *
-     * Queues the evaluation to run asynchronously.
-     * Good for large evaluations or scheduled runs.
-     *
-     * @example
-     * ```typescript
-     * const eval = await Runtype.evals.run({
-     *   flowId: 'flow_123',
-     *   recordType: 'test_data',
-     *   compareModels: [
-     *     [{ stepName: 'Analyze', model: 'gpt-4o' }],
-     *     [{ stepName: 'Analyze', model: 'claude-3-opus' }],
-     *   ]
-     * }).submit()
-     *
-     * console.log('Eval queued:', eval.evalId)
-     *
-     * // Check status later
-     * const status = await Runtype.evals.get(eval.evalId)
-     * ```
-     */
-    submit(): Promise<EvalStatus>;
-    /**
-     * Build the evaluation payload
-     */
-    private buildPayload;
-}
-export declare class EvalsNamespace {
-    private getClient;
-    constructor(getClient: () => RuntypeClient);
-    /**
-     * Run an evaluation
-     *
-     * Returns an EvalRunner with terminal methods:
-     * - .stream() - Execute and stream results
-     * - .submit() - Submit as batch job
-     *
-     * @example
-     * ```typescript
-     * // Single model evaluation with streaming
-     * const result = await Runtype.evals.run({
-     *   flowId: 'flow_123',
-     *   recordType: 'test_data',
-     *   models: [{ stepName: 'Analyze', model: 'gpt-4o' }]
-     * }).stream()
-     *
-     * // Multi-model comparison as batch
-     * const eval = await Runtype.evals.run({
-     *   flowId: 'flow_123',
-     *   recordType: 'test_data',
-     *   compareModels: [
-     *     [{ stepName: 'Analyze', model: 'gpt-4o' }],
-     *     [{ stepName: 'Analyze', model: 'claude-3-opus' }],
-     *     [{ stepName: 'Analyze', model: 'gemini-pro' }],
-     *   ]
-     * }).submit()
-     * ```
-     */
-    run(config: EvalRunConfig): EvalRunner;
-    /**
-     * Get evaluation status by ID
-     *
-     * @example
-     * ```typescript
-     * const status = await Runtype.evals.get('eval_123')
-     * console.log(status.status, status.completedRecords, '/', status.totalRecords)
-     * ```
-     */
-    get(evalId: string): Promise<EvalStatus>;
-    /**
-     * List evaluations
-     *
-     * @example
-     * ```typescript
-     * // List all evals
-     * const evals = await Runtype.evals.list()
-     *
-     * // Filter by status
-     * const running = await Runtype.evals.list({ status: 'running' })
-     *
-     * // Filter by flow
-     * const flowEvals = await Runtype.evals.list({ flowId: 'flow_123' })
-     * ```
-     */
-    list(params?: EvalListParams): Promise<{
-        data: EvalStatus[];
-        total: number;
-    }>;
-}
-//# sourceMappingURL=evals-namespace.d.ts.map

package/dist/evals-namespace.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"evals-namespace.d.ts","sourceRoot":"","sources":["../src/evals-namespace.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,WAAW,CAAA;AAC9C,OAAO,EAAE,UAAU,EAAE,MAAM,eAAe,CAAA;AAC1C,OAAO,KAAK,EAAE,eAAe,EAAe,MAAM,gBAAgB,CAAA;AAMlE,MAAM,WAAW,aAAa;IAC5B,mCAAmC;IACnC,QAAQ,EAAE,MAAM,CAAA;IAChB,iCAAiC;IACjC,KAAK,EAAE,MAAM,CAAA;IACb,oCAAoC;IACpC,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,mCAAmC;IACnC,SAAS,CAAC,EAAE,MAAM,CAAA;CACnB;AAED,MAAM,WAAW,aAAa;IAC5B,0BAA0B;IAC1B,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,sDAAsD;IACtD,IAAI,CAAC,EAAE;QACL,IAAI,EAAE,MAAM,CAAA;QACZ,KAAK,EAAE,GAAG,EAAE,CAAA;KACb,CAAA;IACD,sCAAsC;IACtC,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB,2CAA2C;IAC3C,OAAO,CAAC,EAAE,KAAK,CAAC;QACd,IAAI,EAAE,MAAM,CAAA;QACZ,IAAI,EAAE,MAAM,CAAA;QACZ,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAA;KAC9B,CAAC,CAAA;IACF,kDAAkD;IAClD,MAAM,CAAC,EAAE,aAAa,EAAE,CAAA;IACxB,mDAAmD;IACnD,aAAa,CAAC,EAAE,aAAa,EAAE,EAAE,CAAA;IACjC,+BAA+B;IAC/B,YAAY,CAAC,EAAE,OAAO,CAAA;IACtB,kCAAkC;IAClC,QAAQ,CAAC,EAAE,OAAO,CAAA;IAClB,qCAAqC;IACrC,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,6CAA6C;IAC7C,eAAe,CAAC,EAAE,OAAO,CAAA;IACzB,kCAAkC;IAClC,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAA;IAC5B,0CAA0C;IAC1C,KAAK,CAAC,EAAE,MAAM,CAAA;CACf;AAED,MAAM,WAAW,UAAU;IACzB,MAAM,EAAE,MAAM,CAAA;IACd,MAAM,EAAE,QAAQ,GAAG,SAAS,GAAG,WAAW,GAAG,QAAQ,CAAA;IACrD,YAAY,EAAE,MAAM,CAAA;IACpB,gBAAgB,EAAE,MAAM,CAAA;IACxB,aAAa,EAAE,MAAM,CAAA;IACrB,OAAO,CAAC,EAAE,KAAK,CAAC;QACd,WAAW,EAAE,aAAa,EAAE,CAAA;QAC5B,OAAO,EAAE,KAAK,CAAC;YACb,QAAQ,EAAE,MAAM,CAAA;YAChB,MAAM,EAAE,SAAS,GAAG,OAAO,CAAA;YAC3B,MAAM,CAAC,EAAE,GAAG,CAAA;YACZ,KAAK,CAAC,EAAE,MAAM,CAAA;YACd,aAAa,EAAE,MAAM,CAAA;SACtB,CAAC,CAAA;KACH,CAAC,CAAA;CACH;AAED,MAAM,WAAW,cAAc;IAC7B,uBAAuB;IACvB,MAAM,CAAC,EAAE,QAAQ,GAAG,SAAS,GAAG,WAAW,GAAG,QAAQ,CAAA;IACtD,wBAAwB;IACxB,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,uBAAuB;IACvB,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,wBAAwB;IACxB,MAAM,CAAC,EAAE,MAAM,CAAA;CAChB;AAMD;;;;;;GAMG;AACH,qBAAa,UAAU;IAEnB,OAAO,CAAC,SAAS;IACjB,OAAO,CAAC,MAAM;gBADN,SAAS,EAAE,MAAM,aAAa,EAC9B,MAAM,EAAE,aAAa;IAG/B;;;;;;;;;;;;;;;;;;;;;OAqBG;IACG,MAAM,CAAC,SAAS,CAAC,EAAE,eAAe,GAAG,OAAO,CAAC,UAAU,CAAC;IAoB9D;;;;;;;;;;;;;;;;;;;;;;OAsBG;IACG,MAAM,IAAI,OAAO,CAAC,UAAU,CAAC;IASnC;;OAEG;IACH,OAAO,CAAC,YAAY;CA0CrB;AAMD,qBAAa,cAAc;IACb,OAAO,CAAC,SAAS;gBAAT,SAAS,EAAE,MAAM,aAAa;IAElD;;;;;;;;;;;;;;;;;;;;;;;;;;;OA2BG;IACH,GAAG,CAAC,MAAM,EAAE,aAAa,GAAG,UAAU;IAItC;;;;;;;;OAQG;IACG,GAAG,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,UAAU,CAAC;IAK9C;;;;;;;;;;;;;;OAcG;IACG,IAAI,CAAC,MAAM,CAAC,EAAE,cAAc,GAAG,OAAO,CAAC;QAAE,IAAI,EAAE,UAAU,EAAE,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC;CAIpF"}

package/dist/evals-namespace.js

@@ -1,208 +0,0 @@
-"use strict";
-/**
- * EvalsNamespace - Static namespace for evaluation operations
- *
- * Provides methods for running evaluations and comparing model performance.
- * Evals can be streamed for real-time results or submitted as batch jobs.
- */
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.EvalsNamespace = exports.EvalRunner = void 0;
-const flow_result_1 = require("./flow-result");
-// ============================================================================
-// EvalRunner
-// ============================================================================
-/**
- * EvalRunner - Builder returned by Runtype.evals.run()
- *
- * Provides terminal methods for executing evaluations:
- * - .stream() - Execute and stream results in real-time
- * - .submit() - Submit as a batch job for async processing
- */
-class EvalRunner {
-    constructor(getClient, config) {
-        this.getClient = getClient;
-        this.config = config;
-    }
-    /**
-     * Execute the evaluation with streaming results
-     *
-     * Streams evaluation results as they complete.
-     * Good for real-time feedback during development/testing.
-     *
-     * @example
-     * ```typescript
-     * const result = await Runtype.evals.run({
-     *   flowId: 'flow_123',
-     *   recordType: 'test_data',
-     *   models: [{ stepName: 'Analyze', model: 'gpt-4o' }]
-     * }).stream()
-     *
-     * // Process with callbacks
-     * await result.stream({
-     *   onStepComplete: (result, event) => {
-     *     console.log('Step completed:', event.name, result)
-     *   },
-     * })
-     * ```
-     */
-    async stream(callbacks) {
-        const client = this.getClient();
-        const payload = this.buildPayload();
-        payload.stream = true;
-        const response = await client.requestStream('/eval/stream', {
-            method: 'POST',
-            body: JSON.stringify(payload),
-        });
-        const result = new flow_result_1.FlowResult(response);
-        if (callbacks) {
-            await result.stream(callbacks);
-        }
-        return result;
-    }
-    /**
-     * Submit the evaluation as a batch job
-     *
-     * Queues the evaluation to run asynchronously.
-     * Good for large evaluations or scheduled runs.
-     *
-     * @example
-     * ```typescript
-     * const eval = await Runtype.evals.run({
-     *   flowId: 'flow_123',
-     *   recordType: 'test_data',
-     *   compareModels: [
-     *     [{ stepName: 'Analyze', model: 'gpt-4o' }],
-     *     [{ stepName: 'Analyze', model: 'claude-3-opus' }],
-     *   ]
-     * }).submit()
-     *
-     * console.log('Eval queued:', eval.evalId)
-     *
-     * // Check status later
-     * const status = await Runtype.evals.get(eval.evalId)
-     * ```
-     */
-    async submit() {
-        const client = this.getClient();
-        const payload = this.buildPayload();
-        payload.async = true;
-        return client.post('/evals', payload);
-    }
-    /**
-     * Build the evaluation payload
-     */
-    buildPayload() {
-        const payload = {};
-        if (this.config.flowId) {
-            payload.flowId = this.config.flowId;
-        }
-        else if (this.config.flow) {
-            payload.flow = this.config.flow;
-        }
-        if (this.config.recordType) {
-            payload.recordType = this.config.recordType;
-        }
-        else if (this.config.records) {
-            payload.records = this.config.records;
-        }
-        if (this.config.models) {
-            payload.modelOverrides = this.config.models;
-        }
-        else if (this.config.compareModels) {
-            payload.modelConfigs = this.config.compareModels;
-        }
-        const options = {};
-        if (this.config.storeResults !== undefined)
-            options.storeResults = this.config.storeResults;
-        if (this.config.parallel !== undefined)
-            options.parallel = this.config.parallel;
-        if (this.config.concurrency !== undefined)
-            options.concurrency = this.config.concurrency;
-        if (this.config.continueOnError !== undefined)
-            options.continueOnError = this.config.continueOnError;
-        if (Object.keys(options).length > 0) {
-            payload.options = options;
-        }
-        if (this.config.filter) {
-            payload.filter = this.config.filter;
-        }
-        if (this.config.limit !== undefined) {
-            payload.limit = this.config.limit;
-        }
-        return payload;
-    }
-}
-exports.EvalRunner = EvalRunner;
-// ============================================================================
-// EvalsNamespace
-// ============================================================================
-class EvalsNamespace {
-    constructor(getClient) {
-        this.getClient = getClient;
-    }
-    /**
-     * Run an evaluation
-     *
-     * Returns an EvalRunner with terminal methods:
-     * - .stream() - Execute and stream results
-     * - .submit() - Submit as batch job
-     *
-     * @example
-     * ```typescript
-     * // Single model evaluation with streaming
-     * const result = await Runtype.evals.run({
-     *   flowId: 'flow_123',
-     *   recordType: 'test_data',
-     *   models: [{ stepName: 'Analyze', model: 'gpt-4o' }]
-     * }).stream()
-     *
-     * // Multi-model comparison as batch
-     * const eval = await Runtype.evals.run({
-     *   flowId: 'flow_123',
-     *   recordType: 'test_data',
-     *   compareModels: [
-     *     [{ stepName: 'Analyze', model: 'gpt-4o' }],
-     *     [{ stepName: 'Analyze', model: 'claude-3-opus' }],
-     *     [{ stepName: 'Analyze', model: 'gemini-pro' }],
-     *   ]
-     * }).submit()
-     * ```
-     */
-    run(config) {
-        return new EvalRunner(this.getClient, config);
-    }
-    /**
-     * Get evaluation status by ID
-     *
-     * @example
-     * ```typescript
-     * const status = await Runtype.evals.get('eval_123')
-     * console.log(status.status, status.completedRecords, '/', status.totalRecords)
-     * ```
-     */
-    async get(evalId) {
-        const client = this.getClient();
-        return client.get(`/evals/${evalId}`);
-    }
-    /**
-     * List evaluations
-     *
-     * @example
-     * ```typescript
-     * // List all evals
-     * const evals = await Runtype.evals.list()
-     *
-     * // Filter by status
-     * const running = await Runtype.evals.list({ status: 'running' })
-     *
-     * // Filter by flow
-     * const flowEvals = await Runtype.evals.list({ flowId: 'flow_123' })
-     * ```
-     */
-    async list(params) {
-        const client = this.getClient();
-        return client.get('/evals', params);
-    }
-}
-exports.EvalsNamespace = EvalsNamespace;
-//# sourceMappingURL=evals-namespace.js.map

package/dist/evals-namespace.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"evals-namespace.js","sourceRoot":"","sources":["../src/evals-namespace.ts"],"names":[],"mappings":";AAAA;;;;;GAKG;;;AAGH,+CAA0C;AAiF1C,+EAA+E;AAC/E,aAAa;AACb,+EAA+E;AAE/E;;;;;;GAMG;AACH,MAAa,UAAU;IACrB,YACU,SAA8B,EAC9B,MAAqB;QADrB,cAAS,GAAT,SAAS,CAAqB;QAC9B,WAAM,GAAN,MAAM,CAAe;IAC5B,CAAC;IAEJ;;;;;;;;;;;;;;;;;;;;;OAqBG;IACH,KAAK,CAAC,MAAM,CAAC,SAA2B;QACtC,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,EAAE,CAAA;QAE/B,MAAM,OAAO,GAAG,IAAI,CAAC,YAAY,EAAE,CAAA;QACnC,OAAO,CAAC,MAAM,GAAG,IAAI,CAAA;QAErB,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,aAAa,CAAC,cAAc,EAAE;YAC1D,MAAM,EAAE,MAAM;YACd,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC;SAC9B,CAAC,CAAA;QAEF,MAAM,MAAM,GAAG,IAAI,wBAAU,CAAC,QAAQ,CAAC,CAAA;QAEvC,IAAI,SAAS,EAAE,CAAC;YACd,MAAM,MAAM,CAAC,MAAM,CAAC,SAAS,CAAC,CAAA;QAChC,CAAC;QAED,OAAO,MAAM,CAAA;IACf,CAAC;IAED;;;;;;;;;;;;;;;;;;;;;;OAsBG;IACH,KAAK,CAAC,MAAM;QACV,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,EAAE,CAAA;QAE/B,MAAM,OAAO,GAAG,IAAI,CAAC,YAAY,EAAE,CAAA;QACnC,OAAO,CAAC,KAAK,GAAG,IAAI,CAAA;QAEpB,OAAO,MAAM,CAAC,IAAI,CAAa,QAAQ,EAAE,OAAO,CAAC,CAAA;IACnD,CAAC;IAED;;OAEG;IACK,YAAY;QAClB,MAAM,OAAO,GAAQ,EAAE,CAAA;QAEvB,IAAI,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;YACvB,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM,CAAA;QACrC,CAAC;aAAM,IAAI,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;YAC5B,OAAO,CAAC,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAA;QACjC,CAAC;QAED,IAAI,IAAI,CAAC,MAAM,CAAC,UAAU,EAAE,CAAC;YAC3B,OAAO,CAAC,UAAU,GAAG,IAAI,CAAC,MAAM,CAAC,UAAU,CAAA;QAC7C,CAAC;aAAM,IAAI,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;YAC/B,OAAO,CAAC,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,OAAO,CAAA;QACvC,CAAC;QAED,IAAI,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;YACvB,OAAO,CAAC,cAAc,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM,CAAA;QAC7C,CAAC;aAAM,IAAI,IAAI,CAAC,MAAM,CAAC,aAAa,EAAE,CAAC;YACrC,OAAO,CAAC,YAAY,GAAG,IAAI,CAAC,MAAM,CAAC,aAAa,CAAA;QAClD,CAAC;QAED,MAAM,OAAO,GAAQ,EAAE,CAAA;QACvB,IAAI,IAAI,CAAC,MAAM,CAAC,YAAY,KAAK,SAAS;YAAE,OAAO,CAAC,YAAY,GAAG,IAAI,CAAC,MAAM,CAAC,YAAY,CAAA;QAC3F,IAAI,IAAI,CAAC,MAAM,CAAC,QAAQ,KAAK,SAAS;YAAE,OAAO,CAAC,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAA;QAC/E,IAAI,IAAI,CAAC,MAAM,CAAC,WAAW,KAAK,SAAS;YAAE,OAAO,CAAC,WAAW,GAAG,IAAI,CAAC,MAAM,CAAC,WAAW,CAAA;QACxF,IAAI,IAAI,CAAC,MAAM,CAAC,eAAe,KAAK,SAAS;YAC3C,OAAO,CAAC,eAAe,GAAG,IAAI,CAAC,MAAM,CAAC,eAAe,CAAA;QAEvD,IAAI,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACpC,OAAO,CAAC,OAAO,GAAG,OAAO,CAAA;QAC3B,CAAC;QAED,IAAI,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;YACvB,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM,CAAA;QACrC,CAAC;QAED,IAAI,IAAI,CAAC,MAAM,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;YACpC,OAAO,CAAC,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,CAAA;QACnC,CAAC;QAED,OAAO,OAAO,CAAA;IAChB,CAAC;CACF;AA7HD,gCA6HC;AAED,+EAA+E;AAC/E,iBAAiB;AACjB,+EAA+E;AAE/E,MAAa,cAAc;IACzB,YAAoB,SAA8B;QAA9B,cAAS,GAAT,SAAS,CAAqB;IAAG,CAAC;IAEtD;;;;;;;;;;;;;;;;;;;;;;;;;;;OA2BG;IACH,GAAG,CAAC,MAAqB;QACvB,OAAO,IAAI,UAAU,CAAC,IAAI,CAAC,SAAS,EAAE,MAAM,CAAC,CAAA;IAC/C,CAAC;IAED;;;;;;;;OAQG;IACH,KAAK,CAAC,GAAG,CAAC,MAAc;QACtB,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,EAAE,CAAA;QAC/B,OAAO,MAAM,CAAC,GAAG,CAAa,UAAU,MAAM,EAAE,CAAC,CAAA;IACnD,CAAC;IAED;;;;;;;;;;;;;;OAcG;IACH,KAAK,CAAC,IAAI,CAAC,MAAuB;QAChC,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,EAAE,CAAA;QAC/B,OAAO,MAAM,CAAC,GAAG,CAAwC,QAAQ,EAAE,MAAM,CAAC,CAAA;IAC5E,CAAC;CACF;AApED,wCAoEC"}