honeyhive 1.0.19 → 1.0.21

package/dist/sdk/tracer.js

@@ -0,0 +1,308 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __generator = (this && this.__generator) || function (thisArg, body) {
+    var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g;
+    return g = { next: verb(0), "throw": verb(1), "return": verb(2) }, typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g;
+    function verb(n) { return function (v) { return step([n, v]); }; }
+    function step(op) {
+        if (f) throw new TypeError("Generator is already executing.");
+        while (g && (g = 0, op[0] && (_ = 0)), _) try {
+            if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t;
+            if (y = 0, t) op = [op[0] & 2, t.value];
+            switch (op[0]) {
+                case 0: case 1: t = op; break;
+                case 4: _.label++; return { value: op[1], done: false };
+                case 5: _.label++; y = op[1]; op = [0]; continue;
+                case 7: op = _.ops.pop(); _.trys.pop(); continue;
+                default:
+                    if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; }
+                    if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; }
+                    if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; }
+                    if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; }
+                    if (t[2]) _.ops.pop();
+                    _.trys.pop(); continue;
+            }
+            op = body.call(thisArg, _);
+        } catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; }
+        if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true };
+    }
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.HoneyHiveTracer = void 0;
+var sdk_1 = require("./sdk");
+var telemetry_1 = require("./telemetry");
+var traceloop = __importStar(require("@traceloop/node-server-sdk"));
+var openai_1 = __importDefault(require("openai"));
+var anthropic = __importStar(require("@anthropic-ai/sdk"));
+var azureOpenAI = __importStar(require("@azure/openai"));
+var cohere = __importStar(require("cohere-ai"));
+var bedrock = __importStar(require("@aws-sdk/client-bedrock-runtime"));
+var google_aiplatform = __importStar(require("@google-cloud/aiplatform"));
+var pinecone = __importStar(require("@pinecone-database/pinecone"));
+var ChainsModule = __importStar(require("langchain/chains"));
+var AgentsModule = __importStar(require("langchain/agents"));
+var ToolsModule = __importStar(require("langchain/tools"));
+var chromadb = __importStar(require("chromadb"));
+var HoneyHiveTracer = /** @class */ (function () {
+    function HoneyHiveTracer(sdk) {
+        this.sdk = sdk;
+    }
+    HoneyHiveTracer.prototype.initSession = function (project, sessionName, source, apiKey, serverUrl) {
+        var _a;
+        return __awaiter(this, void 0, void 0, function () {
+            var requestBody, res, error_1;
+            return __generator(this, function (_b) {
+                switch (_b.label) {
+                    case 0:
+                        _b.trys.push([0, 2, , 3]);
+                        requestBody = {
+                            session: {
+                                project: project,
+                                sessionName: sessionName,
+                                source: source,
+                            },
+                        };
+                        return [4 /*yield*/, this.sdk.session.startSession(requestBody)];
+                    case 1:
+                        res = _b.sent();
+                        this.sessionId = (_a = res.object) === null || _a === void 0 ? void 0 : _a.sessionId;
+                        if (this.sessionId) {
+                            traceloop.initialize({
+                                baseUrl: "".concat(serverUrl, "/opentelemetry"),
+                                apiKey: apiKey,
+                                disableBatch: true,
+                                instrumentModules: {
+                                    openAI: openai_1.default,
+                                    anthropic: anthropic,
+                                    azureOpenAI: azureOpenAI,
+                                    cohere: cohere,
+                                    bedrock: bedrock,
+                                    google_aiplatform: google_aiplatform,
+                                    pinecone: pinecone,
+                                    langchain: {
+                                        chainsModule: ChainsModule,
+                                        agentsModule: AgentsModule,
+                                        toolsModule: ToolsModule,
+                                    },
+                                    chromadb: chromadb,
+                                },
+                            });
+                        }
+                        return [3 /*break*/, 3];
+                    case 2:
+                        error_1 = _b.sent();
+                        console.error("Failed to create session:", error_1);
+                        return [3 /*break*/, 3];
+                    case 3: return [2 /*return*/];
+                }
+            });
+        });
+    };
+    HoneyHiveTracer.prototype.initSessionFromId = function (sessionId, apiKey, serverUrl) {
+        return __awaiter(this, void 0, void 0, function () {
+            return __generator(this, function (_a) {
+                this.sessionId = sessionId;
+                traceloop.initialize({
+                    baseUrl: "".concat(serverUrl, "/opentelemetry"),
+                    apiKey: apiKey,
+                    disableBatch: true,
+                    instrumentModules: {
+                        openAI: openai_1.default,
+                        anthropic: anthropic,
+                        azureOpenAI: azureOpenAI,
+                        cohere: cohere,
+                        bedrock: bedrock,
+                        google_aiplatform: google_aiplatform,
+                        pinecone: pinecone,
+                        langchain: {
+                            chainsModule: ChainsModule,
+                            agentsModule: AgentsModule,
+                            toolsModule: ToolsModule,
+                        },
+                        chromadb: chromadb,
+                    },
+                });
+                return [2 /*return*/];
+            });
+        });
+    };
+    HoneyHiveTracer.init = function (_a) {
+        var apiKey = _a.apiKey, project = _a.project, sessionName = _a.sessionName, source = _a.source, _b = _a.serverUrl, serverUrl = _b === void 0 ? "https://api.honeyhive.ai" : _b;
+        return __awaiter(this, void 0, void 0, function () {
+            var sdk, tracer;
+            return __generator(this, function (_c) {
+                switch (_c.label) {
+                    case 0:
+                        sdk = new sdk_1.HoneyHive({
+                            bearerAuth: apiKey,
+                            serverURL: serverUrl,
+                        });
+                        tracer = new HoneyHiveTracer(sdk);
+                        return [4 /*yield*/, tracer.initSession(project, sessionName, source, apiKey, serverUrl)];
+                    case 1:
+                        _c.sent();
+                        return [4 /*yield*/, telemetry_1.Telemetry.getInstance().capture("tracer_init", { "hhai_session_id": tracer.sessionId })];
+                    case 2:
+                        _c.sent();
+                        return [2 /*return*/, tracer];
+                }
+            });
+        });
+    };
+    HoneyHiveTracer.initFromSessionId = function (_a) {
+        var apiKey = _a.apiKey, sessionId = _a.sessionId, _b = _a.serverUrl, serverUrl = _b === void 0 ? "https://api.honeyhive.ai" : _b;
+        return __awaiter(this, void 0, void 0, function () {
+            var sdk, tracer;
+            return __generator(this, function (_c) {
+                switch (_c.label) {
+                    case 0:
+                        sdk = new sdk_1.HoneyHive({
+                            bearerAuth: apiKey,
+                            serverURL: serverUrl,
+                        });
+                        tracer = new HoneyHiveTracer(sdk);
+                        return [4 /*yield*/, tracer.initSessionFromId(sessionId, apiKey, serverUrl)];
+                    case 1:
+                        _c.sent();
+                        return [2 /*return*/, tracer];
+                }
+            });
+        });
+    };
+    HoneyHiveTracer.prototype.setFeedback = function (feedback) {
+        return __awaiter(this, void 0, void 0, function () {
+            var error_2;
+            return __generator(this, function (_a) {
+                switch (_a.label) {
+                    case 0:
+                        if (!this.sessionId) return [3 /*break*/, 5];
+                        _a.label = 1;
+                    case 1:
+                        _a.trys.push([1, 3, , 4]);
+                        return [4 /*yield*/, this.sdk.events.updateEvent({
+                                eventId: this.sessionId,
+                                feedback: feedback
+                            })];
+                    case 2:
+                        _a.sent();
+                        return [3 /*break*/, 4];
+                    case 3:
+                        error_2 = _a.sent();
+                        console.error("Failed to set feedback:", error_2);
+                        return [3 /*break*/, 4];
+                    case 4: return [3 /*break*/, 6];
+                    case 5:
+                        console.error("Session ID is not initialized");
+                        _a.label = 6;
+                    case 6: return [2 /*return*/];
+                }
+            });
+        });
+    };
+    HoneyHiveTracer.prototype.setMetric = function (metrics) {
+        return __awaiter(this, void 0, void 0, function () {
+            var error_3;
+            return __generator(this, function (_a) {
+                switch (_a.label) {
+                    case 0:
+                        if (!this.sessionId) return [3 /*break*/, 5];
+                        _a.label = 1;
+                    case 1:
+                        _a.trys.push([1, 3, , 4]);
+                        return [4 /*yield*/, this.sdk.events.updateEvent({
+                                eventId: this.sessionId,
+                                metrics: metrics
+                            })];
+                    case 2:
+                        _a.sent();
+                        return [3 /*break*/, 4];
+                    case 3:
+                        error_3 = _a.sent();
+                        console.error("Failed to set metric:", error_3);
+                        return [3 /*break*/, 4];
+                    case 4: return [3 /*break*/, 6];
+                    case 5:
+                        console.error("Session ID is not initialized");
+                        _a.label = 6;
+                    case 6: return [2 /*return*/];
+                }
+            });
+        });
+    };
+    HoneyHiveTracer.prototype.setMetadata = function (metadata) {
+        return __awaiter(this, void 0, void 0, function () {
+            var error_4;
+            return __generator(this, function (_a) {
+                switch (_a.label) {
+                    case 0:
+                        if (!this.sessionId) return [3 /*break*/, 5];
+                        _a.label = 1;
+                    case 1:
+                        _a.trys.push([1, 3, , 4]);
+                        return [4 /*yield*/, this.sdk.events.updateEvent({
+                                eventId: this.sessionId,
+                                metadata: metadata
+                            })];
+                    case 2:
+                        _a.sent();
+                        return [3 /*break*/, 4];
+                    case 3:
+                        error_4 = _a.sent();
+                        console.error("Failed to set metadata:", error_4);
+                        return [3 /*break*/, 4];
+                    case 4: return [3 /*break*/, 6];
+                    case 5:
+                        console.error("Session ID is not initialized");
+                        _a.label = 6;
+                    case 6: return [2 /*return*/];
+                }
+            });
+        });
+    };
+    HoneyHiveTracer.prototype.trace = function (fn) {
+        if (this.sessionId) {
+            traceloop.withAssociationProperties({
+                session_id: this.sessionId,
+            }, fn);
+        }
+        else {
+            console.error("Session ID is not initialized");
+        }
+    };
+    return HoneyHiveTracer;
+}());
+exports.HoneyHiveTracer = HoneyHiveTracer;

package/dist/types/index.d.ts

@@ -0,0 +1 @@
+export * from "./rfcdate";

package/dist/types/index.js

@@ -0,0 +1,20 @@
+"use strict";
+/*
+ * Code generated by Speakeasy (https://speakeasyapi.dev). DO NOT EDIT.
+ */
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __exportStar = (this && this.__exportStar) || function(m, exports) {
+    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+__exportStar(require("./rfcdate"), exports);

package/dist/types/rfcdate.d.ts

@@ -0,0 +1,9 @@
+export declare class RFCDate {
+    private date;
+    constructor(date: Date | {
+        date: string;
+    } | string | undefined);
+    getDate(): Date;
+    toJSON(): string;
+    toString(): string;
+}

package/dist/types/rfcdate.js

@@ -0,0 +1,59 @@
+"use strict";
+/*
+ * Code generated by Speakeasy (https://speakeasyapi.dev). DO NOT EDIT.
+ */
+var __read = (this && this.__read) || function (o, n) {
+    var m = typeof Symbol === "function" && o[Symbol.iterator];
+    if (!m) return o;
+    var i = m.call(o), r, ar = [], e;
+    try {
+        while ((n === void 0 || n-- > 0) && !(r = i.next()).done) ar.push(r.value);
+    }
+    catch (error) { e = { error: error }; }
+    finally {
+        try {
+            if (r && !r.done && (m = i["return"])) m.call(i);
+        }
+        finally { if (e) throw e.error; }
+    }
+    return ar;
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.RFCDate = void 0;
+var RFCDate = /** @class */ (function () {
+    function RFCDate(date) {
+        if (!date) {
+            this.date = new Date();
+            return;
+        }
+        if (typeof date === "string") {
+            this.date = new Date(date);
+            return;
+        }
+        if (date instanceof Date) {
+            this.date = date;
+            return;
+        }
+        var anyDate = date;
+        if (date && !!anyDate.date) {
+            this.date = new Date(anyDate.date);
+        }
+    }
+    RFCDate.prototype.getDate = function () {
+        return this.date;
+    };
+    RFCDate.prototype.toJSON = function () {
+        return this.toString();
+    };
+    RFCDate.prototype.toString = function () {
+        var dateRegex = /^(\d{4})-(\d{2})-(\d{2})/;
+        var matches = this.date.toISOString().match(dateRegex);
+        if (matches == null) {
+            throw new Error("Date format is not valid");
+        }
+        var _a = __read(matches, 4), year = _a[1], month = _a[2], day = _a[3];
+        return "".concat(year, "-").concat(month, "-").concat(day);
+    };
+    return RFCDate;
+}());
+exports.RFCDate = RFCDate;

package/funcs/runsCreateRun.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"runsCreateRun.d.ts","sourceRoot":"","sources":["../src/funcs/runsCreateRun.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAI3C,OAAO,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAC;AAGhD,OAAO,KAAK,UAAU,MAAM,+BAA+B,CAAC;AAC5D,OAAO,EACL,eAAe,EACf,mBAAmB,EACnB,mBAAmB,EACnB,mBAAmB,EACnB,qBAAqB,EACtB,MAAM,sCAAsC,CAAC;AAC9C,OAAO,EAAE,QAAQ,EAAE,MAAM,8BAA8B,CAAC;AACxD,OAAO,EAAE,kBAAkB,EAAE,MAAM,wCAAwC,CAAC;AAC5E,OAAO,EAAE,MAAM,EAAE,MAAM,gBAAgB,CAAC;AAExC;;GAEG;AACH,wBAAsB,aAAa,CACjC,MAAM,EAAE,aAAa,EACrB,OAAO,EAAE,UAAU,CAAC,gBAAgB,EACpC,OAAO,CAAC,EAAE,cAAc,GACvB,OAAO,CACR,MAAM,CACJ,UAAU,CAAC,iBAAiB,EAC1B,QAAQ,GACR,kBAAkB,GAClB,qBAAqB,GACrB,mBAAmB,GACnB,mBAAmB,GACnB,mBAAmB,GACnB,eAAe,CAClB,CACF,CAuEA"}
+{"version":3,"file":"runsCreateRun.d.ts","sourceRoot":"","sources":["../src/funcs/runsCreateRun.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAI3C,OAAO,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAC;AAGhD,OAAO,KAAK,UAAU,MAAM,+BAA+B,CAAC;AAC5D,OAAO,EACL,eAAe,EACf,mBAAmB,EACnB,mBAAmB,EACnB,mBAAmB,EACnB,qBAAqB,EACtB,MAAM,sCAAsC,CAAC;AAC9C,OAAO,EAAE,QAAQ,EAAE,MAAM,8BAA8B,CAAC;AACxD,OAAO,EAAE,kBAAkB,EAAE,MAAM,wCAAwC,CAAC;AAC5E,OAAO,EAAE,MAAM,EAAE,MAAM,gBAAgB,CAAC;AAExC;;GAEG;AACH,wBAAsB,aAAa,CACjC,MAAM,EAAE,aAAa,EACrB,OAAO,EAAE,UAAU,CAAC,gBAAgB,EACpC,OAAO,CAAC,EAAE,cAAc,GACvB,OAAO,CACR,MAAM,CACJ,UAAU,CAAC,iBAAiB,EAC1B,QAAQ,GACR,kBAAkB,GAClB,qBAAqB,GACrB,mBAAmB,GACnB,mBAAmB,GACnB,mBAAmB,GACnB,eAAe,CAClB,CACF,CAyEA"}

package/funcs/runsCreateRun.js

@@ -37,7 +37,8 @@ const components = __importStar(require("../models/components/index.js"));
  * Create a new evaluation run
  */
 async function runsCreateRun(client, request, options) {
-    const parsed = (0, schemas_js_1.safeParse)(request, (value) => components.CreateRunRequest$outboundSchema.parse(value), "Input validation failed");
+    const input = request;
+    const parsed = (0, schemas_js_1.safeParse)(input, (value) => components.CreateRunRequest$outboundSchema.parse(value), "Input validation failed");
     if (!parsed.ok) {
         return parsed;
     }

package/funcs/runsCreateRun.js.map

@@ -1 +1 @@
-{"version":3,"file":"runsCreateRun.js","sourceRoot":"","sources":["../src/funcs/runsCreateRun.ts"],"names":[],"mappings":";AAAA;;GAEG;;;;;;;;;;;;;;;;;;;;;;;;;AAwBH,sCAsFC;AA3GD,sDAAiD;AACjD,sDAAwC;AACxC,kDAA8C;AAE9C,oDAA4E;AAC5E,0CAA2C;AAC3C,0EAA4D;AAY5D;;GAEG;AACI,KAAK,UAAU,aAAa,CACjC,MAAqB,EACrB,OAAoC,EACpC,OAAwB;IAaxB,MAAM,MAAM,GAAG,IAAA,sBAAS,EACtB,OAAO,EACP,CAAC,KAAK,EAAE,EAAE,CAAC,UAAU,CAAC,+BAA+B,CAAC,KAAK,CAAC,KAAK,CAAC,EAClE,yBAAyB,CAC1B,CAAC;IACF,IAAI,CAAC,MAAM,CAAC,EAAE,EAAE,CAAC;QACf,OAAO,MAAM,CAAC;IAChB,CAAC;IACD,MAAM,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC;IAC7B,MAAM,IAAI,GAAG,IAAA,yBAAU,EAAC,MAAM,EAAE,OAAO,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC;IAE5D,MAAM,IAAI,GAAG,IAAA,mBAAU,EAAC,OAAO,CAAC,EAAE,CAAC;IAEnC,MAAM,OAAO,GAAG,IAAI,OAAO,CAAC;QAC1B,cAAc,EAAE,kBAAkB;QAClC,MAAM,EAAE,kBAAkB;KAC3B,CAAC,CAAC;IAEH,MAAM,SAAS,GAAG,MAAM,IAAA,6BAAe,EAAC,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC;IACpE,MAAM,aAAa,GAAG,SAAS,IAAI,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,SAAS,EAAE,CAAC;IACzE,MAAM,OAAO,GAAG;QACd,WAAW,EAAE,WAAW;QACxB,YAAY,EAAE,EAAE;QAChB,cAAc,EAAE,MAAM,CAAC,QAAQ,CAAC,UAAU;KAC3C,CAAC;IACF,MAAM,eAAe,GAAG,IAAA,mCAAqB,EAAC,aAAa,CAAC,CAAC;IAE7D,MAAM,UAAU,GAAG,MAAM,CAAC,cAAc,CAAC,OAAO,EAAE;QAChD,QAAQ,EAAE,eAAe;QACzB,MAAM,EAAE,MAAM;QACd,IAAI,EAAE,IAAI;QACV,OAAO,EAAE,OAAO;QAChB,IAAI,EAAE,IAAI;QACV,SAAS,EAAE,CAAA,OAAO,aAAP,OAAO,uBAAP,OAAO,CAAE,SAAS,KAAI,MAAM,CAAC,QAAQ,CAAC,SAAS,IAAI,CAAC,CAAC;KACjE,EAAE,OAAO,CAAC,CAAC;IACZ,IAAI,CAAC,UAAU,CAAC,EAAE,EAAE,CAAC;QACnB,OAAO,UAAU,CAAC;IACpB,CAAC;IACD,MAAM,GAAG,GAAG,UAAU,CAAC,KAAK,CAAC;IAE7B,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE;QACrC,OAAO;QACP,UAAU,EAAE,CAAC,KAAK,EAAE,KAAK,EAAE,KAAK,CAAC;QACjC,WAAW,EAAE,CAAA,OAAO,aAAP,OAAO,uBAAP,OAAO,CAAE,OAAO;eACxB,MAAM,CAAC,QAAQ,CAAC,WAAW;QAChC,UAAU,EAAE,CAAA,OAAO,aAAP,OAAO,uBAAP,OAAO,CAAE,UAAU,KAAI,CAAC,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,CAAC;KACvE,CAAC,CAAC;IACH,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;QACjB,OAAO,QAAQ,CAAC;IAClB,CAAC;IACD,MAAM,QAAQ,GAAG,QAAQ,CAAC,KAAK,CAAC;IAEhC,MAAM,CAAC,MAAM,CAAC,GAAG,MAAM,CAAC,CAAC,KAAK,CAU5B,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,UAAU,CAAC,+BAA+B,CAAC,EACvD,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC,CAC5B,CAAC,QAAQ,CAAC,CAAC;IACZ,IAAI,CAAC,MAAM,CAAC,EAAE,EAAE,CAAC;QACf,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC"}
+{"version":3,"file":"runsCreateRun.js","sourceRoot":"","sources":["../src/funcs/runsCreateRun.ts"],"names":[],"mappings":";AAAA;;GAEG;;;;;;;;;;;;;;;;;;;;;;;;;AAwBH,sCAwFC;AA7GD,sDAAiD;AACjD,sDAAwC;AACxC,kDAA8C;AAE9C,oDAA4E;AAC5E,0CAA2C;AAC3C,0EAA4D;AAY5D;;GAEG;AACI,KAAK,UAAU,aAAa,CACjC,MAAqB,EACrB,OAAoC,EACpC,OAAwB;IAaxB,MAAM,KAAK,GAAG,OAAO,CAAC;IAEtB,MAAM,MAAM,GAAG,IAAA,sBAAS,EACtB,KAAK,EACL,CAAC,KAAK,EAAE,EAAE,CAAC,UAAU,CAAC,+BAA+B,CAAC,KAAK,CAAC,KAAK,CAAC,EAClE,yBAAyB,CAC1B,CAAC;IACF,IAAI,CAAC,MAAM,CAAC,EAAE,EAAE,CAAC;QACf,OAAO,MAAM,CAAC;IAChB,CAAC;IACD,MAAM,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC;IAC7B,MAAM,IAAI,GAAG,IAAA,yBAAU,EAAC,MAAM,EAAE,OAAO,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC;IAE5D,MAAM,IAAI,GAAG,IAAA,mBAAU,EAAC,OAAO,CAAC,EAAE,CAAC;IAEnC,MAAM,OAAO,GAAG,IAAI,OAAO,CAAC;QAC1B,cAAc,EAAE,kBAAkB;QAClC,MAAM,EAAE,kBAAkB;KAC3B,CAAC,CAAC;IAEH,MAAM,SAAS,GAAG,MAAM,IAAA,6BAAe,EAAC,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC;IACpE,MAAM,aAAa,GAAG,SAAS,IAAI,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,SAAS,EAAE,CAAC;IACzE,MAAM,OAAO,GAAG;QACd,WAAW,EAAE,WAAW;QACxB,YAAY,EAAE,EAAE;QAChB,cAAc,EAAE,MAAM,CAAC,QAAQ,CAAC,UAAU;KAC3C,CAAC;IACF,MAAM,eAAe,GAAG,IAAA,mCAAqB,EAAC,aAAa,CAAC,CAAC;IAE7D,MAAM,UAAU,GAAG,MAAM,CAAC,cAAc,CAAC,OAAO,EAAE;QAChD,QAAQ,EAAE,eAAe;QACzB,MAAM,EAAE,MAAM;QACd,IAAI,EAAE,IAAI;QACV,OAAO,EAAE,OAAO;QAChB,IAAI,EAAE,IAAI;QACV,SAAS,EAAE,CAAA,OAAO,aAAP,OAAO,uBAAP,OAAO,CAAE,SAAS,KAAI,MAAM,CAAC,QAAQ,CAAC,SAAS,IAAI,CAAC,CAAC;KACjE,EAAE,OAAO,CAAC,CAAC;IACZ,IAAI,CAAC,UAAU,CAAC,EAAE,EAAE,CAAC;QACnB,OAAO,UAAU,CAAC;IACpB,CAAC;IACD,MAAM,GAAG,GAAG,UAAU,CAAC,KAAK,CAAC;IAE7B,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE;QACrC,OAAO;QACP,UAAU,EAAE,CAAC,KAAK,EAAE,KAAK,EAAE,KAAK,CAAC;QACjC,WAAW,EAAE,CAAA,OAAO,aAAP,OAAO,uBAAP,OAAO,CAAE,OAAO;eACxB,MAAM,CAAC,QAAQ,CAAC,WAAW;QAChC,UAAU,EAAE,CAAA,OAAO,aAAP,OAAO,uBAAP,OAAO,CAAE,UAAU,KAAI,CAAC,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,CAAC;KACvE,CAAC,CAAC;IACH,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;QACjB,OAAO,QAAQ,CAAC;IAClB,CAAC;IACD,MAAM,QAAQ,GAAG,QAAQ,CAAC,KAAK,CAAC;IAEhC,MAAM,CAAC,MAAM,CAAC,GAAG,MAAM,CAAC,CAAC,KAAK,CAU5B,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,UAAU,CAAC,+BAA+B,CAAC,EACvD,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC,CAC5B,CAAC,QAAQ,CAAC,CAAC;IACZ,IAAI,CAAC,MAAM,CAAC,EAAE,EAAE,CAAC;QACf,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC"}

package/funcs/runsUpdateRun.d.ts

@@ -8,5 +8,5 @@ import { Result } from "../types/fp.js";
 /**
  * Update an evaluation run
  */
-export declare function runsUpdateRun(client: HoneyHiveCore, updateRunRequest: components.UpdateRunRequest, runId: string, options?: RequestOptions): Promise<Result<components.UpdateRunResponse, SDKError | SDKValidationError | UnexpectedClientError | InvalidRequestError | RequestAbortedError | RequestTimeoutError | ConnectionError>>;
+export declare function runsUpdateRun(client: HoneyHiveCore, runId: string, updateRunRequest: components.UpdateRunRequest, options?: RequestOptions): Promise<Result<components.UpdateRunResponse, SDKError | SDKValidationError | UnexpectedClientError | InvalidRequestError | RequestAbortedError | RequestTimeoutError | ConnectionError>>;
 //# sourceMappingURL=runsUpdateRun.d.ts.map

package/funcs/runsUpdateRun.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"runsUpdateRun.d.ts","sourceRoot":"","sources":["../src/funcs/runsUpdateRun.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAI3C,OAAO,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAC;AAGhD,OAAO,KAAK,UAAU,MAAM,+BAA+B,CAAC;AAC5D,OAAO,EACL,eAAe,EACf,mBAAmB,EACnB,mBAAmB,EACnB,mBAAmB,EACnB,qBAAqB,EACtB,MAAM,sCAAsC,CAAC;AAC9C,OAAO,EAAE,QAAQ,EAAE,MAAM,8BAA8B,CAAC;AACxD,OAAO,EAAE,kBAAkB,EAAE,MAAM,wCAAwC,CAAC;AAE5E,OAAO,EAAE,MAAM,EAAE,MAAM,gBAAgB,CAAC;AAExC;;GAEG;AACH,wBAAsB,aAAa,CACjC,MAAM,EAAE,aAAa,EACrB,gBAAgB,EAAE,UAAU,CAAC,gBAAgB,EAC7C,KAAK,EAAE,MAAM,EACb,OAAO,CAAC,EAAE,cAAc,GACvB,OAAO,CACR,MAAM,CACJ,UAAU,CAAC,iBAAiB,EAC1B,QAAQ,GACR,kBAAkB,GAClB,qBAAqB,GACrB,mBAAmB,GACnB,mBAAmB,GACnB,mBAAmB,GACnB,eAAe,CAClB,CACF,CAmFA"}
+{"version":3,"file":"runsUpdateRun.d.ts","sourceRoot":"","sources":["../src/funcs/runsUpdateRun.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAI3C,OAAO,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAC;AAGhD,OAAO,KAAK,UAAU,MAAM,+BAA+B,CAAC;AAC5D,OAAO,EACL,eAAe,EACf,mBAAmB,EACnB,mBAAmB,EACnB,mBAAmB,EACnB,qBAAqB,EACtB,MAAM,sCAAsC,CAAC;AAC9C,OAAO,EAAE,QAAQ,EAAE,MAAM,8BAA8B,CAAC;AACxD,OAAO,EAAE,kBAAkB,EAAE,MAAM,wCAAwC,CAAC;AAE5E,OAAO,EAAE,MAAM,EAAE,MAAM,gBAAgB,CAAC;AAExC;;GAEG;AACH,wBAAsB,aAAa,CACjC,MAAM,EAAE,aAAa,EACrB,KAAK,EAAE,MAAM,EACb,gBAAgB,EAAE,UAAU,CAAC,gBAAgB,EAC7C,OAAO,CAAC,EAAE,cAAc,GACvB,OAAO,CACR,MAAM,CACJ,UAAU,CAAC,iBAAiB,EAC1B,QAAQ,GACR,kBAAkB,GAClB,qBAAqB,GACrB,mBAAmB,GACnB,mBAAmB,GACnB,mBAAmB,GACnB,eAAe,CAClB,CACF,CAmFA"}

package/funcs/runsUpdateRun.js

@@ -37,10 +37,10 @@ const operations = __importStar(require("../models/operations/index.js"));
 /**
  * Update an evaluation run
  */
-async function runsUpdateRun(client, updateRunRequest, runId, options) {
+async function runsUpdateRun(client, runId, updateRunRequest, options) {
     const input = {
-        updateRunRequest: updateRunRequest,
         runId: runId,
+        updateRunRequest: updateRunRequest,
     };
     const parsed = (0, schemas_js_1.safeParse)(input, (value) => operations.UpdateRunRequest$outboundSchema.parse(value), "Input validation failed");
     if (!parsed.ok) {

package/funcs/runsUpdateRun.js.map

@@ -1 +1 @@
-{"version":3,"file":"runsUpdateRun.js","sourceRoot":"","sources":["../src/funcs/runsUpdateRun.ts"],"names":[],"mappings":";AAAA;;GAEG;;;;;;;;;;;;;;;;;;;;;;;;;AAyBH,sCAmGC;AAzHD,sDAA+D;AAC/D,sDAAwC;AACxC,kDAA8C;AAE9C,oDAA4E;AAC5E,0CAA2C;AAC3C,0EAA4D;AAU5D,0EAA4D;AAG5D;;GAEG;AACI,KAAK,UAAU,aAAa,CACjC,MAAqB,EACrB,gBAA6C,EAC7C,KAAa,EACb,OAAwB;IAaxB,MAAM,KAAK,GAAgC;QACzC,gBAAgB,EAAE,gBAAgB;QAClC,KAAK,EAAE,KAAK;KACb,CAAC;IAEF,MAAM,MAAM,GAAG,IAAA,sBAAS,EACtB,KAAK,EACL,CAAC,KAAK,EAAE,EAAE,CAAC,UAAU,CAAC,+BAA+B,CAAC,KAAK,CAAC,KAAK,CAAC,EAClE,yBAAyB,CAC1B,CAAC;IACF,IAAI,CAAC,MAAM,CAAC,EAAE,EAAE,CAAC;QACf,OAAO,MAAM,CAAC;IAChB,CAAC;IACD,MAAM,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC;IAC7B,MAAM,IAAI,GAAG,IAAA,yBAAU,EAAC,MAAM,EAAE,OAAO,CAAC,gBAAgB,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC;IAE7E,MAAM,UAAU,GAAG;QACjB,MAAM,EAAE,IAAA,2BAAY,EAAC,QAAQ,EAAE,OAAO,CAAC,MAAM,EAAE;YAC7C,OAAO,EAAE,KAAK;YACd,YAAY,EAAE,SAAS;SACxB,CAAC;KACH,CAAC;IAEF,MAAM,IAAI,GAAG,IAAA,mBAAU,EAAC,gBAAgB,CAAC,CAAC,UAAU,CAAC,CAAC;IAEtD,MAAM,OAAO,GAAG,IAAI,OAAO,CAAC;QAC1B,cAAc,EAAE,kBAAkB;QAClC,MAAM,EAAE,kBAAkB;KAC3B,CAAC,CAAC;IAEH,MAAM,SAAS,GAAG,MAAM,IAAA,6BAAe,EAAC,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC;IACpE,MAAM,aAAa,GAAG,SAAS,IAAI,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,SAAS,EAAE,CAAC;IACzE,MAAM,OAAO,GAAG;QACd,WAAW,EAAE,WAAW;QACxB,YAAY,EAAE,EAAE;QAChB,cAAc,EAAE,MAAM,CAAC,QAAQ,CAAC,UAAU;KAC3C,CAAC;IACF,MAAM,eAAe,GAAG,IAAA,mCAAqB,EAAC,aAAa,CAAC,CAAC;IAE7D,MAAM,UAAU,GAAG,MAAM,CAAC,cAAc,CAAC,OAAO,EAAE;QAChD,QAAQ,EAAE,eAAe;QACzB,MAAM,EAAE,KAAK;QACb,IAAI,EAAE,IAAI;QACV,OAAO,EAAE,OAAO;QAChB,IAAI,EAAE,IAAI;QACV,SAAS,EAAE,CAAA,OAAO,aAAP,OAAO,uBAAP,OAAO,CAAE,SAAS,KAAI,MAAM,CAAC,QAAQ,CAAC,SAAS,IAAI,CAAC,CAAC;KACjE,EAAE,OAAO,CAAC,CAAC;IACZ,IAAI,CAAC,UAAU,CAAC,EAAE,EAAE,CAAC;QACnB,OAAO,UAAU,CAAC;IACpB,CAAC;IACD,MAAM,GAAG,GAAG,UAAU,CAAC,KAAK,CAAC;IAE7B,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE;QACrC,OAAO;QACP,UAAU,EAAE,CAAC,KAAK,EAAE,KAAK,EAAE,KAAK,CAAC;QACjC,WAAW,EAAE,CAAA,OAAO,aAAP,OAAO,uBAAP,OAAO,CAAE,OAAO;eACxB,MAAM,CAAC,QAAQ,CAAC,WAAW;QAChC,UAAU,EAAE,CAAA,OAAO,aAAP,OAAO,uBAAP,OAAO,CAAE,UAAU,KAAI,CAAC,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,CAAC;KACvE,CAAC,CAAC;IACH,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;QACjB,OAAO,QAAQ,CAAC;IAClB,CAAC;IACD,MAAM,QAAQ,GAAG,QAAQ,CAAC,KAAK,CAAC;IAEhC,MAAM,CAAC,MAAM,CAAC,GAAG,MAAM,CAAC,CAAC,KAAK,CAU5B,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,UAAU,CAAC,+BAA+B,CAAC,EACvD,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC,CAC5B,CAAC,QAAQ,CAAC,CAAC;IACZ,IAAI,CAAC,MAAM,CAAC,EAAE,EAAE,CAAC;QACf,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC"}
+{"version":3,"file":"runsUpdateRun.js","sourceRoot":"","sources":["../src/funcs/runsUpdateRun.ts"],"names":[],"mappings":";AAAA;;GAEG;;;;;;;;;;;;;;;;;;;;;;;;;AAyBH,sCAmGC;AAzHD,sDAA+D;AAC/D,sDAAwC;AACxC,kDAA8C;AAE9C,oDAA4E;AAC5E,0CAA2C;AAC3C,0EAA4D;AAU5D,0EAA4D;AAG5D;;GAEG;AACI,KAAK,UAAU,aAAa,CACjC,MAAqB,EACrB,KAAa,EACb,gBAA6C,EAC7C,OAAwB;IAaxB,MAAM,KAAK,GAAgC;QACzC,KAAK,EAAE,KAAK;QACZ,gBAAgB,EAAE,gBAAgB;KACnC,CAAC;IAEF,MAAM,MAAM,GAAG,IAAA,sBAAS,EACtB,KAAK,EACL,CAAC,KAAK,EAAE,EAAE,CAAC,UAAU,CAAC,+BAA+B,CAAC,KAAK,CAAC,KAAK,CAAC,EAClE,yBAAyB,CAC1B,CAAC;IACF,IAAI,CAAC,MAAM,CAAC,EAAE,EAAE,CAAC;QACf,OAAO,MAAM,CAAC;IAChB,CAAC;IACD,MAAM,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC;IAC7B,MAAM,IAAI,GAAG,IAAA,yBAAU,EAAC,MAAM,EAAE,OAAO,CAAC,gBAAgB,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC;IAE7E,MAAM,UAAU,GAAG;QACjB,MAAM,EAAE,IAAA,2BAAY,EAAC,QAAQ,EAAE,OAAO,CAAC,MAAM,EAAE;YAC7C,OAAO,EAAE,KAAK;YACd,YAAY,EAAE,SAAS;SACxB,CAAC;KACH,CAAC;IAEF,MAAM,IAAI,GAAG,IAAA,mBAAU,EAAC,gBAAgB,CAAC,CAAC,UAAU,CAAC,CAAC;IAEtD,MAAM,OAAO,GAAG,IAAI,OAAO,CAAC;QAC1B,cAAc,EAAE,kBAAkB;QAClC,MAAM,EAAE,kBAAkB;KAC3B,CAAC,CAAC;IAEH,MAAM,SAAS,GAAG,MAAM,IAAA,6BAAe,EAAC,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC;IACpE,MAAM,aAAa,GAAG,SAAS,IAAI,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,SAAS,EAAE,CAAC;IACzE,MAAM,OAAO,GAAG;QACd,WAAW,EAAE,WAAW;QACxB,YAAY,EAAE,EAAE;QAChB,cAAc,EAAE,MAAM,CAAC,QAAQ,CAAC,UAAU;KAC3C,CAAC;IACF,MAAM,eAAe,GAAG,IAAA,mCAAqB,EAAC,aAAa,CAAC,CAAC;IAE7D,MAAM,UAAU,GAAG,MAAM,CAAC,cAAc,CAAC,OAAO,EAAE;QAChD,QAAQ,EAAE,eAAe;QACzB,MAAM,EAAE,KAAK;QACb,IAAI,EAAE,IAAI;QACV,OAAO,EAAE,OAAO;QAChB,IAAI,EAAE,IAAI;QACV,SAAS,EAAE,CAAA,OAAO,aAAP,OAAO,uBAAP,OAAO,CAAE,SAAS,KAAI,MAAM,CAAC,QAAQ,CAAC,SAAS,IAAI,CAAC,CAAC;KACjE,EAAE,OAAO,CAAC,CAAC;IACZ,IAAI,CAAC,UAAU,CAAC,EAAE,EAAE,CAAC;QACnB,OAAO,UAAU,CAAC;IACpB,CAAC;IACD,MAAM,GAAG,GAAG,UAAU,CAAC,KAAK,CAAC;IAE7B,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE;QACrC,OAAO;QACP,UAAU,EAAE,CAAC,KAAK,EAAE,KAAK,EAAE,KAAK,CAAC;QACjC,WAAW,EAAE,CAAA,OAAO,aAAP,OAAO,uBAAP,OAAO,CAAE,OAAO;eACxB,MAAM,CAAC,QAAQ,CAAC,WAAW;QAChC,UAAU,EAAE,CAAA,OAAO,aAAP,OAAO,uBAAP,OAAO,CAAE,UAAU,KAAI,CAAC,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,CAAC;KACvE,CAAC,CAAC;IACH,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;QACjB,OAAO,QAAQ,CAAC;IAClB,CAAC;IACD,MAAM,QAAQ,GAAG,QAAQ,CAAC,KAAK,CAAC;IAEhC,MAAM,CAAC,MAAM,CAAC,GAAG,MAAM,CAAC,CAAC,KAAK,CAU5B,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,UAAU,CAAC,+BAA+B,CAAC,EACvD,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC,CAC5B,CAAC,QAAQ,CAAC,CAAC;IACZ,IAAI,CAAC,MAAM,CAAC,EAAE,EAAE,CAAC;QACf,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "honeyhive",
-  "version": "1.0.19",
+  "version": "1.0.21",
   "author": "HoneyHive",
   "main": "./index.js",
   "sideEffects": false,
@@ -31,9 +31,10 @@
     "@aws-sdk/client-bedrock-runtime": "^3.577.0",
     "@azure/openai": "^1.0.0-beta.12",
     "@pinecone-database/pinecone": "^2.2.1",
-    "@traceloop/node-server-sdk": "^0.8.4",
-    "chromadb": "^1.8.1",
+    "@traceloop/node-server-sdk": "^0.11.5",
+    "chromadb": "1.8.1",
     "cohere-ai": "^7.10.0",
+    "js-tiktoken": "^1.0.15",
     "langchain": "^0.2.0",
     "llamaindex": "^0.3.12",
     "openai": "^4.47.1",

package/sdk/evaluation.d.ts

@@ -1,3 +1,4 @@
+import { Status } from '../models/components';
 type Dict<T> = {
     [key: string]: T;
 };
@@ -12,6 +13,11 @@ interface EvaluationConfig {
     runs?: number;
     evaluators?: Function[];
 }
-declare function evaluate(config: EvaluationConfig): Promise<void>;
+declare function evaluate(config: EvaluationConfig): Promise<{
+    run_id: string;
+    dataset_id: string | undefined;
+    session_ids: string[];
+    status: Status;
+}>;
 export { evaluate };
 //# sourceMappingURL=evaluation.d.ts.map

package/sdk/evaluation.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"evaluation.d.ts","sourceRoot":"","sources":["../src/sdk/evaluation.ts"],"names":[],"mappings":"AAIA,KAAK,IAAI,CAAC,CAAC,IAAI;IAAE,CAAC,GAAG,EAAE,MAAM,GAAG,CAAC,CAAA;CAAE,CAAC;AACpC,KAAK,GAAG,GAAG,GAAG,CAAC;AAEf,UAAU,gBAAgB;IACtB,kBAAkB,EAAE,QAAQ,CAAC;IAC7B,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,UAAU,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;IACzB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,UAAU,CAAC,EAAE,QAAQ,EAAE,CAAC;CAC3B;AAkLD,iBAAe,QAAQ,CACnB,MAAM,EAAE,gBAAgB,GACzB,OAAO,CAAC,IAAI,CAAC,CA8Bf;AAED,OAAO,EAAE,QAAQ,EAAE,CAAC"}
+{"version":3,"file":"evaluation.d.ts","sourceRoot":"","sources":["../src/sdk/evaluation.ts"],"names":[],"mappings":"AACA,OAAO,EAAqB,MAAM,EAAE,MAAM,sBAAsB,CAAC;AAIjE,KAAK,IAAI,CAAC,CAAC,IAAI;IAAE,CAAC,GAAG,EAAE,MAAM,GAAG,CAAC,CAAA;CAAE,CAAC;AACpC,KAAK,GAAG,GAAG,GAAG,CAAC;AAEf,UAAU,gBAAgB;IACtB,kBAAkB,EAAE,QAAQ,CAAC;IAC7B,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,UAAU,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;IACzB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,UAAU,CAAC,EAAE,QAAQ,EAAE,CAAC;CAC3B;AAiMD,iBAAe,QAAQ,CACnB,MAAM,EAAE,gBAAgB,GACzB,OAAO,CAAC;IACP,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,EAAE,MAAM,GAAG,SAAS,CAAC;IAC/B,WAAW,EAAE,MAAM,EAAE,CAAC;IACtB,MAAM,EAAE,MAAM,CAAC;CAClB,CAAC,CAuCD;AAED,OAAO,EAAE,QAAQ,EAAE,CAAC"}

package/sdk/evaluation.js

@@ -1,8 +1,33 @@
 "use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.evaluate = evaluate;
 const sdk_1 = require("./sdk");
+const components_1 = require("../models/components");
 const tracer_1 = require("./tracer");
+const crypto = __importStar(require("crypto"));
 function validateRequirements(config) {
     if (!config.hh_api_key) {
         throw new Error("Honeyhive API key not found. Please set 'hh_api_key' to initiate Honeyhive Tracer. Cannot run Evaluation");
@@ -17,6 +42,11 @@ function validateRequirements(config) {
         throw new Error("No valid 'dataset_id' or 'query_list' found. Please provide one to iterate the evaluation over.");
     }
 }
+function _generateHash(inputString) {
+    const hashObject = crypto.createHash('md5');
+    hashObject.update(inputString, 'utf-8');
+    return `EXT-${hashObject.digest('hex').slice(0, 24)}`;
+}
 async function loadDataset(hhai, config) {
     if (!config.dataset_id) {
         return null;
@@ -76,8 +106,7 @@ async function runEvaluation(tracer, evalconfig, inputs) {
             (async () => {
                 try {
                     const agentResponse = await evalconfig.evaluationFunction(inputs);
-                    if (agentResponse && typeof agentResponse === 'object')
-                        output = agentResponse;
+                    output = agentResponse;
                     console.log(agentResponse);
                 }
                 finally {
@@ -128,6 +157,13 @@ async function addTraceMetadata(tracer, state, config, evaluation_output, metric
                 tracing_metadata['datapoint_id'] = state.hh_dataset.datapoints[run_id];
                 tracing_metadata['dataset_id'] = config.dataset_id;
             }
+            if (state.external_dataset_id && config.query_list) {
+                tracing_metadata['datapoint_id'] = _generateHash(JSON.stringify(config.query_list[run_id]));
+                tracing_metadata['dataset_id'] = state.external_dataset_id;
+            }
+            if (typeof evaluation_output !== 'object') {
+                evaluation_output = { output: evaluation_output };
+            }
             await tracer.enrichSession({
                 metadata: tracing_metadata,
                 outputs: evaluation_output,
@@ -143,17 +179,19 @@ async function setupEvaluation(state, config) {
     const eval_run = await state.hhai.experiments.createRun({
         project: config.hh_project,
         name: config.name,
-        datasetId: config.dataset_id,
+        datasetId: config.dataset_id || state.external_dataset_id,
         eventIds: [],
+        status: state.state,
     });
     state.eval_run = eval_run;
 }
 async function windupEvaluation(state) {
     try {
+        state.state = components_1.Status.Completed;
         if (state.eval_run && state.eval_run.runId) {
             await state.hhai.experiments.updateRun({
                 eventIds: state.evaluation_session_ids,
-                status: "completed"
+                status: state.state
             }, state.eval_run.runId);
         }
     }
@@ -162,12 +200,15 @@ async function windupEvaluation(state) {
     }
 }
 async function evaluate(config) {
+    var _a;
     validateRequirements(config);
     const state = {
         hhai: new sdk_1.HoneyHive({ bearerAuth: config.hh_api_key }),
         hh_dataset: null,
+        external_dataset_id: config.query_list ? _generateHash(JSON.stringify(config.query_list)) : undefined,
         evaluation_session_ids: [],
-        eval_run: null
+        eval_run: null,
+        state: components_1.Status.Pending
     };
     state.hh_dataset = await loadDataset(state.hhai, config);
     const runs = config.runs || (state.hh_dataset ? state.hh_dataset.datapoints.length : (config.query_list ? config.query_list.length : 0));
@@ -183,5 +224,11 @@ async function evaluate(config) {
             state.evaluation_session_ids.push(tracer.sessionId);
     }
     await windupEvaluation(state);
+    return {
+        run_id: ((_a = state.eval_run) === null || _a === void 0 ? void 0 : _a.runId) || '',
+        dataset_id: config.dataset_id || state.external_dataset_id,
+        session_ids: state.evaluation_session_ids,
+        status: state.state
+    };
 }
 //# sourceMappingURL=evaluation.js.map