honeyhive 1.0.13 → 1.0.16

package/src/funcs/metricsUpdateMetric.ts

@@ -44,10 +44,8 @@ export async function metricsUpdateMetric(
     | ConnectionError
   >
 > {
-  const input = request;
-
   const parsed = safeParse(
-    input,
+    request,
     (value) => components.MetricEdit$outboundSchema.parse(value),
     "Input validation failed",
   );

package/src/funcs/projectsCreateProject.ts

@@ -40,10 +40,8 @@ export async function projectsCreateProject(
     | ConnectionError
   >
 > {
-  const input = request;
-
   const parsed = safeParse(
-    input,
+    request,
     (value) => components.CreateProjectRequest$outboundSchema.parse(value),
     "Input validation failed",
   );

package/src/funcs/projectsUpdateProject.ts

@@ -41,10 +41,8 @@ export async function projectsUpdateProject(
     | ConnectionError
   >
 > {
-  const input = request;
-
   const parsed = safeParse(
-    input,
+    request,
     (value) => components.UpdateProjectRequest$outboundSchema.parse(value),
     "Input validation failed",
   );

package/src/funcs/runsCreateRun.ts

@@ -40,10 +40,8 @@ export async function runsCreateRun(
     | ConnectionError
   >
 > {
-  const input = request;
-
   const parsed = safeParse(
-    input,
+    request,
     (value) => components.CreateRunRequest$outboundSchema.parse(value),
     "Input validation failed",
   );

package/src/funcs/runsUpdateRun.ts

@@ -27,8 +27,8 @@ import { Result } from "../types/fp.js";
  */
 export async function runsUpdateRun(
   client: HoneyHiveCore,
-  runId: string,
   updateRunRequest: components.UpdateRunRequest,
+  runId: string,
   options?: RequestOptions,
 ): Promise<
   Result<
@@ -43,8 +43,8 @@ export async function runsUpdateRun(
   >
 > {
   const input: operations.UpdateRunRequest = {
-    runId: runId,
     updateRunRequest: updateRunRequest,
+    runId: runId,
   };
 
   const parsed = safeParse(

package/src/funcs/sessionStartSession.ts

@@ -40,10 +40,8 @@ export async function sessionStartSession(
     | ConnectionError
   >
 > {
-  const input = request;
-
   const parsed = safeParse(
-    input,
+    request,
     (value) => operations.StartSessionRequestBody$outboundSchema.parse(value),
     "Input validation failed",
   );

package/src/funcs/toolsCreateTool.ts

@@ -41,10 +41,8 @@ export async function toolsCreateTool(
     | ConnectionError
   >
 > {
-  const input = request;
-
   const parsed = safeParse(
-    input,
+    request,
     (value) => components.CreateToolRequest$outboundSchema.parse(value),
     "Input validation failed",
   );

package/src/funcs/toolsUpdateTool.ts

@@ -41,10 +41,8 @@ export async function toolsUpdateTool(
     | ConnectionError
   >
 > {
-  const input = request;
-
   const parsed = safeParse(
-    input,
+    request,
     (value) => components.UpdateToolRequest$outboundSchema.parse(value),
     "Input validation failed",
   );

package/src/index.ts

@@ -7,3 +7,4 @@ export * as files from "./lib/files.js";
 export * from "./sdk/sdk.js";
 export * from "./sdk/tracer.js";
 export * from "./sdk/lc_tracer.js";
+export * from "./sdk/evaluation.js";

package/src/lib/config.ts

@@ -53,8 +53,8 @@ export function serverURLFromOptions(options: SDKOptions): URL | null {
 
 export const SDK_METADATA = {
   language: "typescript",
-  openapiDocVersion: "1.0.2",
-  sdkVersion: "1.1.1",
-  genVersion: "2.428.1",
-  userAgent: "speakeasy-sdk/typescript 1.1.1 2.428.1 1.0.2 honeyhive",
+  openapiDocVersion: "1.0.4",
+  sdkVersion: "1.2.1",
+  genVersion: "2.429.0",
+  userAgent: "speakeasy-sdk/typescript 1.2.1 2.429.0 1.0.4 honeyhive",
 } as const;

package/src/models/components/datapoint.ts

@@ -15,8 +15,8 @@ export type Datapoint = {
    * UUID for the project where the datapoint is stored
    */
   projectId?: string | undefined;
-  createdAt?: Date | undefined;
-  updatedAt?: Date | undefined;
+  createdAt?: string | undefined;
+  updatedAt?: string | undefined;
   /**
    * Arbitrary JSON object containing the inputs for the datapoint
    */
@@ -55,10 +55,8 @@ export const Datapoint$inboundSchema: z.ZodType<
   _id: z.string().optional(),
   tenant: z.string().optional(),
   project_id: z.string().optional(),
-  created_at: z.string().datetime({ offset: true }).transform(v => new Date(v))
-    .optional(),
-  updated_at: z.string().datetime({ offset: true }).transform(v => new Date(v))
-    .optional(),
+  created_at: z.string().optional(),
+  updated_at: z.string().optional(),
   inputs: z.record(z.any()).optional(),
   history: z.array(z.record(z.any())).optional(),
   ground_truth: z.record(z.any()).optional(),
@@ -108,8 +106,8 @@ export const Datapoint$outboundSchema: z.ZodType<
   id: z.string().optional(),
   tenant: z.string().optional(),
   projectId: z.string().optional(),
-  createdAt: z.date().transform(v => v.toISOString()).optional(),
-  updatedAt: z.date().transform(v => v.toISOString()).optional(),
+  createdAt: z.string().optional(),
+  updatedAt: z.string().optional(),
   inputs: z.record(z.any()).optional(),
   history: z.array(z.record(z.any())).optional(),
   groundTruth: z.record(z.any()).optional(),

package/src/models/components/dataset.ts

@@ -67,11 +67,11 @@ export type Dataset = {
   /**
    * Timestamp of when the dataset was created
    */
-  createdAt?: Date | undefined;
+  createdAt?: string | undefined;
   /**
    * Timestamp of when the dataset was last updated
    */
-  updatedAt?: Date | undefined;
+  updatedAt?: string | undefined;
 };
 
 /** @internal */
@@ -124,12 +124,8 @@ export const Dataset$inboundSchema: z.ZodType<Dataset, z.ZodTypeDef, unknown> =
     linked_evals: z.array(z.string()).optional(),
     saved: z.boolean().optional(),
     pipeline_type: PipelineType$inboundSchema.optional(),
-    created_at: z.string().datetime({ offset: true }).transform(v =>
-      new Date(v)
-    ).optional(),
-    updated_at: z.string().datetime({ offset: true }).transform(v =>
-      new Date(v)
-    ).optional(),
+    created_at: z.string().optional(),
+    updated_at: z.string().optional(),
   }).transform((v) => {
     return remap$(v, {
       "num_points": "numPoints",
@@ -170,8 +166,8 @@ export const Dataset$outboundSchema: z.ZodType<
   linkedEvals: z.array(z.string()).optional(),
   saved: z.boolean().optional(),
   pipelineType: PipelineType$outboundSchema.optional(),
-  createdAt: z.date().transform(v => v.toISOString()).optional(),
-  updatedAt: z.date().transform(v => v.toISOString()).optional(),
+  createdAt: z.string().optional(),
+  updatedAt: z.string().optional(),
 }).transform((v) => {
   return remap$(v, {
     numPoints: "num_points",

package/src/sdk/configurations.ts

@@ -48,14 +48,14 @@ export class Configurations extends ClientSDK {
    * Update an existing configuration
    */
   async updateConfiguration(
-    id: string,
     putConfigurationRequest: components.PutConfigurationRequest,
+    id: string,
     options?: RequestOptions,
   ): Promise<void> {
     return unwrapAsync(configurationsUpdateConfiguration(
       this,
-      id,
       putConfigurationRequest,
+      id,
       options,
     ));
   }

package/src/sdk/datapoints.ts

@@ -63,14 +63,14 @@ export class Datapoints extends ClientSDK {
    * Update a specific datapoint
    */
   async updateDatapoint(
-    id: string,
     updateDatapointRequest: components.UpdateDatapointRequest,
+    id: string,
     options?: RequestOptions,
   ): Promise<void> {
     return unwrapAsync(datapointsUpdateDatapoint(
       this,
-      id,
       updateDatapointRequest,
+      id,
       options,
     ));
   }

package/src/sdk/datasets.ts

@@ -77,14 +77,14 @@ export class Datasets extends ClientSDK {
    * Add datapoints to a dataset
    */
   async addDatapoints(
-    datasetId: string,
     requestBody: operations.AddDatapointsRequestBody,
+    datasetId: string,
     options?: RequestOptions,
   ): Promise<operations.AddDatapointsResponseBody> {
     return unwrapAsync(datasetsAddDatapoints(
       this,
-      datasetId,
       requestBody,
+      datasetId,
       options,
     ));
   }

package/src/sdk/evaluation.ts

@@ -0,0 +1,205 @@
+import { HoneyHive } from "./sdk";
+import { CreateRunResponse, } from '../models/components';
+import { HoneyHiveTracer } from './tracer';
+
+type Dict<T> = { [key: string]: T };
+type Any = any;
+
+interface EvaluationConfig {
+    evaluationFunction: Function,
+    hh_api_key: string;
+    hh_project: string;
+    name: string;
+    dataset_id?: string;
+    query_list?: Dict<Any>[];
+    runs?: number;
+}
+
+interface EvaluationState {
+    hhai: HoneyHive;
+    hh_dataset: Any;
+    evaluation_session_ids: string[];
+    eval_run: CreateRunResponse | null;
+}
+
+function validateRequirements(config: EvaluationConfig): void {
+    if (!config.hh_api_key) {
+        throw new Error("Honeyhive API key not found. Please set 'hh_api_key' to initiate Honeyhive Tracer. Cannot run Evaluation");
+    }
+    if (!config.hh_project) {
+        throw new Error("Honeyhive Project not found. Please set 'hh_project' to initiate Honeyhive Tracer. Cannot run Evaluation");
+    }
+    if (!config.name) {
+        throw new Error("Evaluation name not found. Please set 'name' to initiate Honeyhive Evaluation.");
+    }
+    if (!config.dataset_id && !config.query_list) {
+        throw new Error("No valid 'dataset_id' or 'query_list' found. Please provide one to iterate the evaluation over.");
+    }
+}
+
+async function loadDataset(hhai: HoneyHive, config: EvaluationConfig): Promise<any> {
+    if (!config.dataset_id) {
+        return null;
+    }
+    try {
+        const dataset = await hhai.datasets.getDatasets(
+            config.hh_project,
+            undefined,
+            config.dataset_id
+        );
+        if (dataset && dataset.testcases && dataset.testcases.length > 0) {
+            return dataset.testcases[0];
+        }
+    } catch (error) {
+        throw new Error(`No dataset found with id - ${config.dataset_id} for project - ${config.hh_project}`);
+    }
+    return null;
+}
+
+async function getInputs(state: EvaluationState, config: EvaluationConfig, run_id: number): Promise<any> {
+    if (state.hh_dataset && state.hh_dataset.datapoints && state.hh_dataset.datapoints.length > 0) {
+        try {
+            const datapoint_id = state.hh_dataset.datapoints[run_id];
+            const datapoint_response = await state.hhai.datapoints.getDatapoint(datapoint_id);
+            const datapointList = datapoint_response.datapoint;
+            if (datapointList && datapointList[0])
+                return datapointList[0].inputs;
+        } catch (error) {
+            console.error(`Error getting datapoint: ${error}`);
+        }
+    } else if (config.query_list) {
+        return config.query_list[run_id];
+    }
+    return null;
+}
+
+async function initializeTracer(config: EvaluationConfig): Promise<HoneyHiveTracer> {
+    try {
+        return await HoneyHiveTracer.init({
+            apiKey: config.hh_api_key,
+            project: config.hh_project,
+            source: 'evaluation',
+            sessionName: config.name
+        });
+    } catch (error) {
+        throw new Error("Unable to initiate Honeyhive Tracer. Cannot run Evaluation");
+    }
+}
+
+async function runEvaluation(tracer: HoneyHiveTracer, evalconfig: EvaluationConfig, inputs: any): Promise<Any> {
+    try {
+        let output = {};
+        let promiseResolve: () => void;
+        const promise = new Promise<void>((resolve) => {
+            promiseResolve = resolve;
+        });
+
+        tracer.trace(() => {
+            (async () => {
+                try {
+                    const agentResponse = await evalconfig.evaluationFunction(inputs);
+                    if (agentResponse && typeof agentResponse === 'object')
+                        output = agentResponse;
+                    console.log(agentResponse);
+                } finally {
+                    promiseResolve();
+                }
+            })();
+        });
+
+        await promise;
+        await tracer.flush();
+
+        return output;
+    } catch (error) {
+        console.error(`Error in evaluation function: ${error}`);
+        return null;
+    }
+}
+
+async function addTraceMetadata(tracer: HoneyHiveTracer, state: EvaluationState, config: EvaluationConfig, inputs: any, evaluation_output: any, run_id: number): Promise<void> {
+    try {
+        if (state.eval_run && state.eval_run?.runId) {
+            const tracing_metadata: Dict<Any> = {
+                run_id: state.eval_run.runId,
+                inputs: inputs
+            };
+            if (state.hh_dataset) {
+                tracing_metadata['datapoint_id'] = state.hh_dataset.datapoints[run_id];
+                tracing_metadata['dataset_id'] = config.dataset_id;
+            }
+            if (evaluation_output) {
+                tracing_metadata['outputs'] = evaluation_output;
+            }
+            await tracer.enrichSession({
+                metadata: tracing_metadata,
+                outputs: evaluation_output
+            });
+        }
+
+    } catch (error) {
+        console.error(`Error adding trace metadata: ${error}`);
+    }
+}
+
+async function setupEvaluation(state: EvaluationState, config: EvaluationConfig): Promise<void> {
+    const eval_run = await state.hhai.runs.createRun({
+        project: config.hh_project,
+        name: config.name,
+        datasetId: config.dataset_id,
+        eventIds: [],
+
+    });
+    state.eval_run = eval_run;
+}
+
+async function windupEvaluation(state: EvaluationState): Promise<void> {
+    try {
+        if (state.eval_run && state.eval_run.runId) {
+            await state.hhai.runs.updateRun(
+                {
+                    eventIds: state.evaluation_session_ids,
+                    status: "completed"
+                },
+                state.eval_run.runId,
+            );
+        }
+    } catch (error) {
+        console.warn("Warning: Unable to mark evaluation as `Completed`");
+    }
+}
+
+async function evaluate(
+    config: EvaluationConfig
+): Promise<void> {
+    validateRequirements(config);
+
+    const state: EvaluationState = {
+        hhai: new HoneyHive({ bearerAuth: config.hh_api_key }),
+        hh_dataset: null,
+        evaluation_session_ids: [],
+        eval_run: null
+    };
+
+    state.hh_dataset = await loadDataset(state.hhai, config);
+    const runs = config.runs || (state.hh_dataset ? state.hh_dataset.datapoints.length : (config.query_list ? config.query_list.length : 0));
+
+    await setupEvaluation(state, config);
+
+    for (let run_id = 0; run_id < runs; run_id++) {
+        console.log(`---------- RUN ${run_id +1} ----------`)
+        const inputs = await getInputs(state, config, run_id);
+        const tracer = await initializeTracer(config);
+
+        const output = await runEvaluation(tracer, config, inputs);
+
+        await addTraceMetadata(tracer, state, config, inputs, output, run_id);
+
+        if (tracer.sessionId)
+            state.evaluation_session_ids.push(tracer.sessionId);
+    }
+
+    await windupEvaluation(state);
+}
+
+export { evaluate };

package/src/sdk/index.ts

@@ -5,3 +5,4 @@
 export * from "./sdk.js";
 export * from "./tracer.js";
 export * from "./lc_tracer.js";
+export * from "./evaluation.js";

package/src/sdk/runs.ts

@@ -58,14 +58,14 @@ export class Runs extends ClientSDK {
    * Update an evaluation run
    */
   async updateRun(
-    runId: string,
     updateRunRequest: components.UpdateRunRequest,
+    runId: string,
     options?: RequestOptions,
   ): Promise<components.UpdateRunResponse> {
     return unwrapAsync(runsUpdateRun(
       this,
-      runId,
       updateRunRequest,
+      runId,
       options,
     ));
   }

package/src/sdk/tracer.ts

@@ -11,7 +11,6 @@ import OpenAI from "openai";
 import * as anthropic from "@anthropic-ai/sdk";
 import * as cohere from "cohere-ai";
 import * as bedrock from "@aws-sdk/client-bedrock-runtime";
-import * as google_aiplatform from "@google-cloud/aiplatform";
 import * as pinecone from "@pinecone-database/pinecone";
 import * as ChainsModule from "langchain/chains";
 import * as AgentsModule from "langchain/agents";
@@ -114,7 +113,6 @@ export class HoneyHiveTracer {
             azureOpenAI: azureOpenAI,
             cohere: cohere,
             bedrock: bedrock,
-            google_aiplatform: google_aiplatform,
             pinecone: pinecone,
             langchain: {
               chainsModule: ChainsModule,
@@ -146,7 +144,6 @@ export class HoneyHiveTracer {
         azureOpenAI: azureOpenAI,
         cohere: cohere,
         bedrock: bedrock,
-        google_aiplatform: google_aiplatform,
         pinecone: pinecone,
         langchain: {
           chainsModule: ChainsModule,