honeyhive 1.0.21 → 1.0.23

package/src/sdk/evaluation.ts

@@ -1,20 +1,37 @@
 import { HoneyHive } from "./sdk";
 import { CreateRunResponse, Status } from '../models/components';
-import { HoneyHiveTracer } from './tracer';
+import { HoneyHiveTracer} from './tracer';
 import * as crypto from 'crypto';
+import * as fs from 'fs';
 
 type Dict<T> = { [key: string]: T };
 type Any = any;
 
 interface EvaluationConfig {
-    evaluationFunction: Function,
+    evaluationFunction: (...args: any[]) => any,
     hh_api_key: string;
     hh_project: string;
     name: string;
+    suite?: string;
     dataset_id?: string;
-    query_list?: Dict<Any>[];
-    runs?: number;
-    evaluators?: Function[];
+    dataset?: Dict<Any>[];
+    evaluators?: ((...args: any[]) => any)[];
+    server_url?: string;
+    runId?: string;
+    datasetId?: string;
+    datapointId?: string;
+}
+
+interface EvaluationData {
+    run_id: string;
+    dataset_id: string | undefined;
+    session_ids: string[];
+    status: Status;
+}
+
+interface EvaluationResult extends EvaluationData {
+    toJson(): void;
+    suite: string;
 }
 
 interface EvaluationState {
@@ -36,8 +53,8 @@ function validateRequirements(config: EvaluationConfig): void {
     if (!config.name) {
         throw new Error("Evaluation name not found. Please set 'name' to initiate Honeyhive Evaluation.");
     }
-    if (!config.dataset_id && !config.query_list) {
-        throw new Error("No valid 'dataset_id' or 'query_list' found. Please provide one to iterate the evaluation over.");
+    if (!config.dataset_id && !config.dataset) {
+        throw new Error("No valid 'dataset_id' or 'dataset' found. Please provide one to iterate the evaluation over.");
     }
 }
 
@@ -77,8 +94,8 @@ async function getInputs(state: EvaluationState, config: EvaluationConfig, run_i
         } catch (error) {
             console.error(`Error getting datapoint: ${error}`);
         }
-    } else if (config.query_list) {
-        return config.query_list[run_id];
+    } else if (config.dataset) {
+        return config.dataset[run_id];
     }
     return null;
 }
@@ -91,7 +108,11 @@ async function initializeTracer(config: EvaluationConfig, inputs: any): Promise<
             source: 'evaluation',
             sessionName: config.name,
             inputs: inputs ? inputs : {},
-            isEvaluation: true
+            isEvaluation: true,
+            // runId: config.runId,
+            // datasetId: config.datasetId,
+            // datapointId: config.datapointId,
+            ...(config.server_url && { serverUrl: config.server_url })
         });
     } catch (error) {
         throw new Error("Unable to initiate Honeyhive Tracer. Cannot run Evaluation");
@@ -128,7 +149,7 @@ async function runEvaluation(tracer: HoneyHiveTracer, evalconfig: EvaluationConf
     }
 }
 
-async function runEvaluators(inputs: any, evaluation_output: any, evaluators?: Function[]): Promise<Dict<Any>> {
+async function runEvaluators(inputs: any, evaluation_output: any, evaluators?: ((...args: any[]) => any)[]): Promise<Dict<Any>> {
     const metrics: Dict<Any> = {};
     if (evaluators) {
         for (let index = 0; index < evaluators.length; index++) {
@@ -161,8 +182,8 @@ async function addTraceMetadata(tracer: HoneyHiveTracer, state: EvaluationState,
                 tracing_metadata['datapoint_id'] = state.hh_dataset.datapoints[run_id];
                 tracing_metadata['dataset_id'] = config.dataset_id;
             }
-            if (state.external_dataset_id && config.query_list) {
-                tracing_metadata['datapoint_id'] = _generateHash(JSON.stringify(config.query_list[run_id]));
+            if (state.external_dataset_id && config.dataset) {
+                tracing_metadata['datapoint_id'] = _generateHash(JSON.stringify(config.dataset[run_id]));
                 tracing_metadata['dataset_id'] = state.external_dataset_id;
             }
             if (typeof evaluation_output !== 'object') {
@@ -210,25 +231,22 @@ async function windupEvaluation(state: EvaluationState): Promise<void> {
 
 async function evaluate(
     config: EvaluationConfig
-): Promise<{
-    run_id: string;
-    dataset_id: string | undefined;
-    session_ids: string[];
-    status: Status;
-}> {
+): Promise<EvaluationResult> {
     validateRequirements(config);
 
+    const suite = config.suite || "default";
+
     const state: EvaluationState = {
         hhai: new HoneyHive({ bearerAuth: config.hh_api_key }),
         hh_dataset: null,
-        external_dataset_id: config.query_list ? _generateHash(JSON.stringify(config.query_list)) : undefined,
+        external_dataset_id: config.dataset ? _generateHash(JSON.stringify(config.dataset)) : undefined,
         evaluation_session_ids: [],
         eval_run: null,
         state: Status.Pending
     };
 
     state.hh_dataset = await loadDataset(state.hhai, config);
-    const runs = config.runs || (state.hh_dataset ? state.hh_dataset.datapoints.length : (config.query_list ? config.query_list.length : 0));
+    const runs = state.hh_dataset ? state.hh_dataset.datapoints.length : (config.dataset ? config.dataset.length : 0);
 
     await setupEvaluation(state, config);
 
@@ -248,12 +266,23 @@ async function evaluate(
 
     await windupEvaluation(state);
 
-    return {
+    const result: EvaluationResult = {
         run_id: state.eval_run?.runId || '',
         dataset_id: config.dataset_id || state.external_dataset_id,
         session_ids: state.evaluation_session_ids,
-        status: state.state
+        status: state.state,
+        suite: suite,  // Changed from _suite to suite
+        toJson(): void {
+            const data: EvaluationData = {
+                run_id: this.run_id,
+                dataset_id: this.dataset_id,
+                session_ids: this.session_ids,
+                status: this.status
+            };
+            fs.writeFileSync(`${this.suite}.json`, JSON.stringify(data, null, 4));
+        }
     };
+    return result;
 }
 
 export { evaluate };

package/src/sdk/index.ts

@@ -1,8 +1,5 @@
 /*
- * Code generated by Speakeasy (https://speakeasyapi.dev). DO NOT EDIT.
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
  */
 
 export * from "./sdk.js";
-export * from "./tracer.js";
-export * from "./lc_tracer.js";
-export * from "./evaluation.js";

package/src/sdk/lc_tracer.ts

@@ -1,12 +1,33 @@
 import { HoneyHive } from "./sdk";
-import { DocumentInterface } from '@langchain/core/documents';
-import { BaseCallbackHandler } from '@langchain/core/callbacks/base';
-import { AgentAction, AgentFinish } from '@langchain/core/agents';
-import { ChainValues } from '@langchain/core/utils/types';
-import { LLMResult } from '@langchain/core/outputs';
-import { Serialized } from '@langchain/core/load/serializable';
 import { v4 as uuidv4 } from 'uuid';
 
+// Type declarations for TypeScript
+type Serialized = any;
+type LLMResult = any;
+type ChainValues = any;
+type AgentAction = any;
+type AgentFinish = any;
+type DocumentInterface = any;
+
+// Define a base class that will be used if LangChain is not available
+class DefaultBaseCallbackHandler {
+  constructor() {}
+}
+
+// Use the imported BaseCallbackHandler if available, otherwise use the default
+let BaseCallbackHandler: any = DefaultBaseCallbackHandler;
+
+// We'll try to import LangChain in a safer way
+try {
+  // Try to import LangChain's BaseCallbackHandler
+  // eslint-disable-next-line @typescript-eslint/no-var-requires, @typescript-eslint/no-require-imports
+  const { BaseCallbackHandler: LangChainBaseCallbackHandler } = require('@langchain/core/callbacks/base');
+  BaseCallbackHandler = LangChainBaseCallbackHandler;
+} catch (error) {
+  // If the import fails, we'll use the default handler
+  console.warn('LangChain not found, please install @langchain/core/callbacks/base');
+  // BaseCallbackHandler remains as DefaultBaseCallbackHandler
+}
 interface HoneyHiveTracerInput {
   project: string;
   sessionName: string;
@@ -137,7 +158,7 @@ class HoneyHiveLangChainTracer extends BaseCallbackHandler {
     this.logMap[log.event_id] = log; // Store in the logMap by event_id
   }
 
-  override async handleLLMStart(
+  async handleLLMStart(
     llm: Serialized,
     prompts: string[],
     runId: string,
@@ -164,43 +185,42 @@ class HoneyHiveLangChainTracer extends BaseCallbackHandler {
     this.addLogToParent(log);
   }
 
-  override async handleLLMEnd(
+  async handleLLMEnd(
     output: LLMResult,
     runId: string,
     // @ts-expect-error: Not used
     parentstring?: string
   ): Promise<void> {
-    const endTime = 1000 * Date.now();
     const log = this.logMap[runId];
-    if (log) {
-      log.end_time = endTime;
-      log.duration = (endTime - log.start_time) / 1000;
-      // Extract text and generationInfo from the output
-      const generations = output.generations;
+    if (!log) return;
+
+    const generations = output.generations;
+    const texts = generations.flat().map((gen: any) => gen.text);
+    const metadataArray = generations.flat().map((gen: any) => {
+      return {
+        finish_reason: gen.generationInfo?.finish_reason,
+        logprobs: gen.generationInfo?.logprobs,
+      };
+    });
 
-      const texts = generations.flat().map((gen) => gen.text);
-      const metadataArray = generations.flat().map((gen) => {
-        const { text, ...rest } = gen;
-        return rest;
-      });
+    const metadata = metadataArray.reduce((acc: any, curr: any, index: number) => {
+      acc[`generation_${index}`] = curr;
+      return acc;
+    }, {});
 
-      // Convert the metadata array to an object with index keys
-      const metadata = metadataArray.reduce((acc, curr, index) => {
-        acc[index] = curr;
-        return acc;
-      }, {} as Record<string, unknown>);
-      log.outputs = { generations: texts };
-      if (log.metadata) {
-        log.metadata['generationInfo'] = metadata;
-      }
-      // If this is the top-level chain, post the entire log structure
-      if (log.parent_id === undefined) {
-        await this.postTrace(this.rootLogs);
-      }
+    log.end_time = 1000 * Date.now();
+    log.duration = (log.end_time - log.start_time) / 1000;
+    log.outputs = { generations: texts };
+    if (log.metadata) {
+      log.metadata['generationInfo'] = metadata;
+    }
+    // If this is the top-level chain, post the entire log structure
+    if (log.parent_id === undefined) {
+      await this.postTrace(this.rootLogs);
     }
   }
 
-  override async handleChainStart(
+  async handleChainStart(
     chain: Serialized,
     inputs: ChainValues,
     runId: string,
@@ -226,7 +246,7 @@ class HoneyHiveLangChainTracer extends BaseCallbackHandler {
     this.addLogToParent(log);
   }
 
-  override async handleChainEnd(
+  async handleChainEnd(
     outputs: ChainValues,
     runId: string,
     // @ts-expect-error: Not used
@@ -246,7 +266,7 @@ class HoneyHiveLangChainTracer extends BaseCallbackHandler {
     }
   }
 
-  override async handleToolStart(
+  async handleToolStart(
     tool: Serialized,
     input: string,
     runId: string,
@@ -272,7 +292,7 @@ class HoneyHiveLangChainTracer extends BaseCallbackHandler {
     this.addLogToParent(log);
   }
 
-  override async handleToolEnd(
+  async handleToolEnd(
     output: string,
     runId: string,
     // @ts-expect-error: Not used
@@ -291,7 +311,7 @@ class HoneyHiveLangChainTracer extends BaseCallbackHandler {
     }
   }
 
-  override async handleAgentAction(
+  async handleAgentAction(
     action: AgentAction,
     runId: string,
     parentstring?: string
@@ -317,7 +337,7 @@ class HoneyHiveLangChainTracer extends BaseCallbackHandler {
     }
   }
 
-  override async handleAgentEnd(
+  async handleAgentEnd(
     finish: AgentFinish,
     runId: string,
     parentstring?: string
@@ -388,7 +408,7 @@ class HoneyHiveLangChainTracer extends BaseCallbackHandler {
     }
   }
 
-  override async handleChainError(
+  async handleChainError(
     error: Error,
     runId: string,
     // @ts-expect-error: Not used
@@ -407,7 +427,7 @@ class HoneyHiveLangChainTracer extends BaseCallbackHandler {
     }
   }
 
-  override async handleLLMError(
+  async handleLLMError(
     error: Error,
     runId: string,
     // @ts-expect-error: Not used
@@ -426,7 +446,7 @@ class HoneyHiveLangChainTracer extends BaseCallbackHandler {
     }
   }
 
-  override async handleToolError(
+  async handleToolError(
     error: Error,
     runId: string,
     // @ts-expect-error: Not used
@@ -445,7 +465,7 @@ class HoneyHiveLangChainTracer extends BaseCallbackHandler {
     }
   }
 
-  override async handleRetrieverStart(
+  async handleRetrieverStart(
     retriever: Serialized,
     query: string,
     runId: string,
@@ -471,7 +491,7 @@ class HoneyHiveLangChainTracer extends BaseCallbackHandler {
     this.addLogToParent(log);
   }
   
-  override async handleRetrieverEnd(
+  async handleRetrieverEnd(
     documents: DocumentInterface[],
     runId: string,
     // @ts-expect-error: Not used
@@ -491,7 +511,7 @@ class HoneyHiveLangChainTracer extends BaseCallbackHandler {
     }
   }
   
-  override async handleRetrieverError(
+  async handleRetrieverError(
     error: Error,
     runId: string,
     // @ts-expect-error: Not used

package/src/sdk/telemetry.ts

@@ -43,14 +43,14 @@ export class Telemetry {
         fs.mkdirSync(path.dirname(Telemetry.ANON_ID_PATH), { recursive: true });
         const anonIdFile = fs.openSync(Telemetry.ANON_ID_PATH, "w");
         this.anonId = uuid();
-        fs.writeSync(anonIdFile, this.anonId);
+        fs.writeSync(anonIdFile, this.anonId as string);
         fs.closeSync(anonIdFile);
       } else {
         const anonIdFile = fs.openSync(Telemetry.ANON_ID_PATH, "r");
         this.anonId = fs.readFileSync(anonIdFile, "utf8");
         fs.closeSync(anonIdFile);
       }
-      return this.anonId;
+      return this.anonId || Telemetry.UNKNOWN_ANON_ID;
     } catch (e) {
       return Telemetry.UNKNOWN_ANON_ID;
     }
@@ -160,7 +160,6 @@ export class Telemetry {
         },
       };
       this.posthog.capture(posthogPayload);
-      this.posthog.flush();
       await this.posthog.shutdown();
     }
   }