@runtypelabs/sdk 0.1.1

package/dist/evals-namespace.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"evals-namespace.d.ts","sourceRoot":"","sources":["../src/evals-namespace.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,WAAW,CAAA;AAC9C,OAAO,EAAE,UAAU,EAAE,MAAM,eAAe,CAAA;AAC1C,OAAO,KAAK,EAAE,eAAe,EAAe,MAAM,gBAAgB,CAAA;AAMlE,MAAM,WAAW,aAAa;IAC5B,mCAAmC;IACnC,QAAQ,EAAE,MAAM,CAAA;IAChB,iCAAiC;IACjC,KAAK,EAAE,MAAM,CAAA;IACb,oCAAoC;IACpC,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,mCAAmC;IACnC,SAAS,CAAC,EAAE,MAAM,CAAA;CACnB;AAED,MAAM,WAAW,aAAa;IAC5B,0BAA0B;IAC1B,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,sDAAsD;IACtD,IAAI,CAAC,EAAE;QACL,IAAI,EAAE,MAAM,CAAA;QACZ,KAAK,EAAE,GAAG,EAAE,CAAA;KACb,CAAA;IACD,sCAAsC;IACtC,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB,2CAA2C;IAC3C,OAAO,CAAC,EAAE,KAAK,CAAC;QACd,IAAI,EAAE,MAAM,CAAA;QACZ,IAAI,EAAE,MAAM,CAAA;QACZ,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAA;KAC9B,CAAC,CAAA;IACF,kDAAkD;IAClD,MAAM,CAAC,EAAE,aAAa,EAAE,CAAA;IACxB,mDAAmD;IACnD,aAAa,CAAC,EAAE,aAAa,EAAE,EAAE,CAAA;IACjC,+BAA+B;IAC/B,YAAY,CAAC,EAAE,OAAO,CAAA;IACtB,kCAAkC;IAClC,QAAQ,CAAC,EAAE,OAAO,CAAA;IAClB,qCAAqC;IACrC,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,6CAA6C;IAC7C,eAAe,CAAC,EAAE,OAAO,CAAA;IACzB,kCAAkC;IAClC,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAA;IAC5B,0CAA0C;IAC1C,KAAK,CAAC,EAAE,MAAM,CAAA;CACf;AAED,MAAM,WAAW,UAAU;IACzB,MAAM,EAAE,MAAM,CAAA;IACd,MAAM,EAAE,QAAQ,GAAG,SAAS,GAAG,WAAW,GAAG,QAAQ,CAAA;IACrD,YAAY,EAAE,MAAM,CAAA;IACpB,gBAAgB,EAAE,MAAM,CAAA;IACxB,aAAa,EAAE,MAAM,CAAA;IACrB,OAAO,CAAC,EAAE,KAAK,CAAC;QACd,WAAW,EAAE,aAAa,EAAE,CAAA;QAC5B,OAAO,EAAE,KAAK,CAAC;YACb,QAAQ,EAAE,MAAM,CAAA;YAChB,MAAM,EAAE,SAAS,GAAG,OAAO,CAAA;YAC3B,MAAM,CAAC,EAAE,GAAG,CAAA;YACZ,KAAK,CAAC,EAAE,MAAM,CAAA;YACd,aAAa,EAAE,MAAM,CAAA;SACtB,CAAC,CAAA;KACH,CAAC,CAAA;CACH;AAED,MAAM,WAAW,cAAc;IAC7B,uBAAuB;IACvB,MAAM,CAAC,EAAE,QAAQ,GAAG,SAAS,GAAG,WAAW,GAAG,QAAQ,CAAA;IACtD,wBAAwB;IACxB,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,uBAAuB;IACvB,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,wBAAwB;IACxB,MAAM,CAAC,EAAE,MAAM,CAAA;CAChB;AAMD;;;;;;GAMG;AACH,qBAAa,UAAU;IAEnB,OAAO,CAAC,SAAS;IACjB,OAAO,CAAC,MAAM;gBADN,SAAS,EAAE,MAAM,aAAa,EAC9B,MAAM,EAAE,aAAa;IAG/B;;;;;;;;;;;;;;;;;;;;;OAqBG;IACG,MAAM,CAAC,SAAS,CAAC,EAAE,eAAe,GAAG,OAAO,CAAC,UAAU,CAAC;IAoB9D;;;;;;;;;;;;;;;;;;;;;;OAsBG;IACG,MAAM,IAAI,OAAO,CAAC,UAAU,CAAC;IASnC;;OAEG;IACH,OAAO,CAAC,YAAY;CAyCrB;AAMD,qBAAa,cAAc;IACb,OAAO,CAAC,SAAS;gBAAT,SAAS,EAAE,MAAM,aAAa;IAElD;;;;;;;;;;;;;;;;;;;;;;;;;;;OA2BG;IACH,GAAG,CAAC,MAAM,EAAE,aAAa,GAAG,UAAU;IAItC;;;;;;;;OAQG;IACG,GAAG,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,UAAU,CAAC;IAK9C;;;;;;;;;;;;;;OAcG;IACG,IAAI,CAAC,MAAM,CAAC,EAAE,cAAc,GAAG,OAAO,CAAC;QAAE,IAAI,EAAE,UAAU,EAAE,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC;CAIpF"}

package/dist/evals-namespace.js

@@ -0,0 +1,208 @@
+"use strict";
+/**
+ * EvalsNamespace - Static namespace for evaluation operations
+ *
+ * Provides methods for running evaluations and comparing model performance.
+ * Evals can be streamed for real-time results or submitted as batch jobs.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.EvalsNamespace = exports.EvalRunner = void 0;
+const flow_result_1 = require("./flow-result");
+// ============================================================================
+// EvalRunner
+// ============================================================================
+/**
+ * EvalRunner - Builder returned by Runtype.evals.run()
+ *
+ * Provides terminal methods for executing evaluations:
+ * - .stream() - Execute and stream results in real-time
+ * - .submit() - Submit as a batch job for async processing
+ */
+class EvalRunner {
+    constructor(getClient, config) {
+        this.getClient = getClient;
+        this.config = config;
+    }
+    /**
+     * Execute the evaluation with streaming results
+     *
+     * Streams evaluation results as they complete.
+     * Good for real-time feedback during development/testing.
+     *
+     * @example
+     * ```typescript
+     * const result = await Runtype.evals.run({
+     *   flowId: 'flow_123',
+     *   recordType: 'test_data',
+     *   models: [{ stepName: 'Analyze', model: 'gpt-4o' }]
+     * }).stream()
+     *
+     * // Process with callbacks
+     * await result.stream({
+     *   onStepComplete: (result, event) => {
+     *     console.log('Step completed:', event.name, result)
+     *   },
+     * })
+     * ```
+     */
+    async stream(callbacks) {
+        const client = this.getClient();
+        const payload = this.buildPayload();
+        payload.stream = true;
+        const response = await client.requestStream('/eval/stream', {
+            method: 'POST',
+            body: JSON.stringify(payload)
+        });
+        const result = new flow_result_1.FlowResult(response);
+        if (callbacks) {
+            await result.stream(callbacks);
+        }
+        return result;
+    }
+    /**
+     * Submit the evaluation as a batch job
+     *
+     * Queues the evaluation to run asynchronously.
+     * Good for large evaluations or scheduled runs.
+     *
+     * @example
+     * ```typescript
+     * const eval = await Runtype.evals.run({
+     *   flowId: 'flow_123',
+     *   recordType: 'test_data',
+     *   compareModels: [
+     *     [{ stepName: 'Analyze', model: 'gpt-4o' }],
+     *     [{ stepName: 'Analyze', model: 'claude-3-opus' }],
+     *   ]
+     * }).submit()
+     *
+     * console.log('Eval queued:', eval.evalId)
+     *
+     * // Check status later
+     * const status = await Runtype.evals.get(eval.evalId)
+     * ```
+     */
+    async submit() {
+        const client = this.getClient();
+        const payload = this.buildPayload();
+        payload.async = true;
+        return client.post('/evals', payload);
+    }
+    /**
+     * Build the evaluation payload
+     */
+    buildPayload() {
+        const payload = {};
+        if (this.config.flowId) {
+            payload.flowId = this.config.flowId;
+        }
+        else if (this.config.flow) {
+            payload.flow = this.config.flow;
+        }
+        if (this.config.recordType) {
+            payload.recordType = this.config.recordType;
+        }
+        else if (this.config.records) {
+            payload.records = this.config.records;
+        }
+        if (this.config.models) {
+            payload.modelOverrides = this.config.models;
+        }
+        else if (this.config.compareModels) {
+            payload.modelConfigs = this.config.compareModels;
+        }
+        const options = {};
+        if (this.config.storeResults !== undefined)
+            options.storeResults = this.config.storeResults;
+        if (this.config.parallel !== undefined)
+            options.parallel = this.config.parallel;
+        if (this.config.concurrency !== undefined)
+            options.concurrency = this.config.concurrency;
+        if (this.config.continueOnError !== undefined)
+            options.continueOnError = this.config.continueOnError;
+        if (Object.keys(options).length > 0) {
+            payload.options = options;
+        }
+        if (this.config.filter) {
+            payload.filter = this.config.filter;
+        }
+        if (this.config.limit !== undefined) {
+            payload.limit = this.config.limit;
+        }
+        return payload;
+    }
+}
+exports.EvalRunner = EvalRunner;
+// ============================================================================
+// EvalsNamespace
+// ============================================================================
+class EvalsNamespace {
+    constructor(getClient) {
+        this.getClient = getClient;
+    }
+    /**
+     * Run an evaluation
+     *
+     * Returns an EvalRunner with terminal methods:
+     * - .stream() - Execute and stream results
+     * - .submit() - Submit as batch job
+     *
+     * @example
+     * ```typescript
+     * // Single model evaluation with streaming
+     * const result = await Runtype.evals.run({
+     *   flowId: 'flow_123',
+     *   recordType: 'test_data',
+     *   models: [{ stepName: 'Analyze', model: 'gpt-4o' }]
+     * }).stream()
+     *
+     * // Multi-model comparison as batch
+     * const eval = await Runtype.evals.run({
+     *   flowId: 'flow_123',
+     *   recordType: 'test_data',
+     *   compareModels: [
+     *     [{ stepName: 'Analyze', model: 'gpt-4o' }],
+     *     [{ stepName: 'Analyze', model: 'claude-3-opus' }],
+     *     [{ stepName: 'Analyze', model: 'gemini-pro' }],
+     *   ]
+     * }).submit()
+     * ```
+     */
+    run(config) {
+        return new EvalRunner(this.getClient, config);
+    }
+    /**
+     * Get evaluation status by ID
+     *
+     * @example
+     * ```typescript
+     * const status = await Runtype.evals.get('eval_123')
+     * console.log(status.status, status.completedRecords, '/', status.totalRecords)
+     * ```
+     */
+    async get(evalId) {
+        const client = this.getClient();
+        return client.get(`/evals/${evalId}`);
+    }
+    /**
+     * List evaluations
+     *
+     * @example
+     * ```typescript
+     * // List all evals
+     * const evals = await Runtype.evals.list()
+     *
+     * // Filter by status
+     * const running = await Runtype.evals.list({ status: 'running' })
+     *
+     * // Filter by flow
+     * const flowEvals = await Runtype.evals.list({ flowId: 'flow_123' })
+     * ```
+     */
+    async list(params) {
+        const client = this.getClient();
+        return client.get('/evals', params);
+    }
+}
+exports.EvalsNamespace = EvalsNamespace;
+//# sourceMappingURL=evals-namespace.js.map

package/dist/evals-namespace.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"evals-namespace.js","sourceRoot":"","sources":["../src/evals-namespace.ts"],"names":[],"mappings":";AAAA;;;;;GAKG;;;AAGH,+CAA0C;AAiF1C,+EAA+E;AAC/E,aAAa;AACb,+EAA+E;AAE/E;;;;;;GAMG;AACH,MAAa,UAAU;IACrB,YACU,SAA8B,EAC9B,MAAqB;QADrB,cAAS,GAAT,SAAS,CAAqB;QAC9B,WAAM,GAAN,MAAM,CAAe;IAC5B,CAAC;IAEJ;;;;;;;;;;;;;;;;;;;;;OAqBG;IACH,KAAK,CAAC,MAAM,CAAC,SAA2B;QACtC,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,EAAE,CAAA;QAE/B,MAAM,OAAO,GAAG,IAAI,CAAC,YAAY,EAAE,CAAA;QACnC,OAAO,CAAC,MAAM,GAAG,IAAI,CAAA;QAErB,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,aAAa,CAAC,cAAc,EAAE;YAC1D,MAAM,EAAE,MAAM;YACd,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC;SAC9B,CAAC,CAAA;QAEF,MAAM,MAAM,GAAG,IAAI,wBAAU,CAAC,QAAQ,CAAC,CAAA;QAEvC,IAAI,SAAS,EAAE,CAAC;YACd,MAAM,MAAM,CAAC,MAAM,CAAC,SAAS,CAAC,CAAA;QAChC,CAAC;QAED,OAAO,MAAM,CAAA;IACf,CAAC;IAED;;;;;;;;;;;;;;;;;;;;;;OAsBG;IACH,KAAK,CAAC,MAAM;QACV,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,EAAE,CAAA;QAE/B,MAAM,OAAO,GAAG,IAAI,CAAC,YAAY,EAAE,CAAA;QACnC,OAAO,CAAC,KAAK,GAAG,IAAI,CAAA;QAEpB,OAAO,MAAM,CAAC,IAAI,CAAa,QAAQ,EAAE,OAAO,CAAC,CAAA;IACnD,CAAC;IAED;;OAEG;IACK,YAAY;QAClB,MAAM,OAAO,GAAQ,EAAE,CAAA;QAEvB,IAAI,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;YACvB,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM,CAAA;QACrC,CAAC;aAAM,IAAI,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;YAC5B,OAAO,CAAC,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAA;QACjC,CAAC;QAED,IAAI,IAAI,CAAC,MAAM,CAAC,UAAU,EAAE,CAAC;YAC3B,OAAO,CAAC,UAAU,GAAG,IAAI,CAAC,MAAM,CAAC,UAAU,CAAA;QAC7C,CAAC;aAAM,IAAI,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;YAC/B,OAAO,CAAC,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,OAAO,CAAA;QACvC,CAAC;QAED,IAAI,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;YACvB,OAAO,CAAC,cAAc,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM,CAAA;QAC7C,CAAC;aAAM,IAAI,IAAI,CAAC,MAAM,CAAC,aAAa,EAAE,CAAC;YACrC,OAAO,CAAC,YAAY,GAAG,IAAI,CAAC,MAAM,CAAC,aAAa,CAAA;QAClD,CAAC;QAED,MAAM,OAAO,GAAQ,EAAE,CAAA;QACvB,IAAI,IAAI,CAAC,MAAM,CAAC,YAAY,KAAK,SAAS;YAAE,OAAO,CAAC,YAAY,GAAG,IAAI,CAAC,MAAM,CAAC,YAAY,CAAA;QAC3F,IAAI,IAAI,CAAC,MAAM,CAAC,QAAQ,KAAK,SAAS;YAAE,OAAO,CAAC,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAA;QAC/E,IAAI,IAAI,CAAC,MAAM,CAAC,WAAW,KAAK,SAAS;YAAE,OAAO,CAAC,WAAW,GAAG,IAAI,CAAC,MAAM,CAAC,WAAW,CAAA;QACxF,IAAI,IAAI,CAAC,MAAM,CAAC,eAAe,KAAK,SAAS;YAAE,OAAO,CAAC,eAAe,GAAG,IAAI,CAAC,MAAM,CAAC,eAAe,CAAA;QAEpG,IAAI,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACpC,OAAO,CAAC,OAAO,GAAG,OAAO,CAAA;QAC3B,CAAC;QAED,IAAI,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;YACvB,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM,CAAA;QACrC,CAAC;QAED,IAAI,IAAI,CAAC,MAAM,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;YACpC,OAAO,CAAC,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,CAAA;QACnC,CAAC;QAED,OAAO,OAAO,CAAA;IAChB,CAAC;CACF;AA5HD,gCA4HC;AAED,+EAA+E;AAC/E,iBAAiB;AACjB,+EAA+E;AAE/E,MAAa,cAAc;IACzB,YAAoB,SAA8B;QAA9B,cAAS,GAAT,SAAS,CAAqB;IAAG,CAAC;IAEtD;;;;;;;;;;;;;;;;;;;;;;;;;;;OA2BG;IACH,GAAG,CAAC,MAAqB;QACvB,OAAO,IAAI,UAAU,CAAC,IAAI,CAAC,SAAS,EAAE,MAAM,CAAC,CAAA;IAC/C,CAAC;IAED;;;;;;;;OAQG;IACH,KAAK,CAAC,GAAG,CAAC,MAAc;QACtB,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,EAAE,CAAA;QAC/B,OAAO,MAAM,CAAC,GAAG,CAAa,UAAU,MAAM,EAAE,CAAC,CAAA;IACnD,CAAC;IAED;;;;;;;;;;;;;;OAcG;IACH,KAAK,CAAC,IAAI,CAAC,MAAuB;QAChC,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,EAAE,CAAA;QAC/B,OAAO,MAAM,CAAC,GAAG,CAAwC,QAAQ,EAAE,MAAM,CAAC,CAAA;IAC5E,CAAC;CACF;AApED,wCAoEC"}