@lmnr-ai/lmnr 0.6.10 → 0.6.11

package/dist/index.js

@@ -31,6 +31,7 @@ var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: tru
 var index_exports = {};
 __export(index_exports, {
   Dataset: () => EvaluationDataset,
+  HumanEvaluator: () => HumanEvaluator,
   Laminar: () => Laminar,
   LaminarAttributes: () => LaminarAttributes,
   LaminarClient: () => LaminarClient,
@@ -485,7 +486,7 @@ var AgentResource = class extends BaseResource {
 };
 
 // package.json
-var version = "0.6.10";
+var version = "0.6.11";
 
 // src/version.ts
 var getLangVersion = () => {
@@ -553,6 +554,81 @@ var EvalsResource = class extends BaseResource {
     }
     return response.json();
   }
+  /**
+   * Create a new evaluation and return its ID.
+   *
+   * @param {string} [name] - Optional name of the evaluation
+   * @param {string} [groupName] - An identifier to group evaluations
+   * @returns {Promise<StringUUID>} The evaluation ID
+   */
+  async createEvaluation(name, groupName) {
+    const evaluation = await this.init(name, groupName);
+    return evaluation.id;
+  }
+  /**
+   * Create a datapoint for an evaluation.
+   *
+   * @param {Object} options - Create datapoint options
+   * @param {string} options.evalId - The evaluation ID
+   * @param {D} options.data - The input data for the executor
+   * @param {T} [options.target] - The target/expected output for evaluators
+   * @param {Record<string, any>} [options.metadata] - Optional metadata
+   * @param {number} [options.index] - Optional index of the datapoint
+   * @param {string} [options.traceId] - Optional trace ID
+   * @returns {Promise<StringUUID>} The datapoint ID
+   */
+  async createDatapoint({
+    evalId,
+    data,
+    target,
+    metadata,
+    index,
+    traceId
+  }) {
+    const datapointId = newUUID();
+    const partialDatapoint = {
+      id: datapointId,
+      data,
+      target,
+      index: index ?? 0,
+      traceId: traceId ?? newUUID(),
+      executorSpanId: newUUID(),
+      metadata
+    };
+    await this.saveDatapoints({
+      evalId,
+      datapoints: [partialDatapoint]
+    });
+    return datapointId;
+  }
+  /**
+   * Update a datapoint with evaluation results.
+   *
+   * @param {Object} options - Update datapoint options
+   * @param {string} options.evalId - The evaluation ID
+   * @param {string} options.datapointId - The datapoint ID
+   * @param {Record<string, number>} options.scores - The scores
+   * @param {O} [options.executorOutput] - The executor output
+   * @returns {Promise<void>}
+   */
+  async updateDatapoint({
+    evalId,
+    datapointId,
+    scores,
+    executorOutput
+  }) {
+    const response = await fetch(this.baseHttpUrl + `/v1/evals/${evalId}/datapoints/${datapointId}`, {
+      method: "POST",
+      headers: this.headers(),
+      body: JSON.stringify({
+        executorOutput,
+        scores
+      })
+    });
+    if (!response.ok) {
+      await this.handleError(response);
+    }
+  }
   /**
    * Save evaluation datapoints.
    *
@@ -1475,6 +1551,39 @@ var prettyPrintZodSchema = (schema, indent = 2) => {
 ${reconstructed.join("\n")}
 })`;
 };
+var modelToProviderMap = {
+  "gpt-4.1": "openai",
+  "gpt-4.1-mini": "openai",
+  "gpt-4.1-nano": "openai",
+  "o4-mini": "openai",
+  //prettier-ignore
+  "o3": "openai",
+  "o3-mini": "openai",
+  //prettier-ignore
+  "o1": "openai",
+  "o1-mini": "openai",
+  "gpt-4o": "openai",
+  "gpt-4o-mini": "openai",
+  "gpt-4o-2024-08-06": "openai",
+  "gpt-4.5-preview": "openai",
+  "o1-preview": "openai",
+  "claude-3-5-sonnet-latest": "anthropic",
+  "claude-3-5-sonnet-20240620": "anthropic",
+  "claude-3-5-sonnet-20241022": "anthropic",
+  "claude-3-7-sonnet-20250219": "anthropic",
+  "claude-3-7-sonnet-latest": "anthropic",
+  "cerebras-llama-3.3-70b": "cerebras",
+  "cerebras-llama-3.1-8b": "cerebras",
+  "groq-llama-3.3-70b-versatile": "groq",
+  "groq-llama-3.3-70b-specdec": "groq",
+  "gemini-1.5-flash": "google",
+  "gemini-1.5-pro": "google",
+  "gemini-1.5-flash-8b": "google",
+  "gemini-2.0-flash-lite": "google",
+  "gemini-2.0-flash": "google",
+  "gemini-2.5-flash-preview-04-17": "google",
+  "gemini-2.5-pro-preview-03-25": "google"
+};
 
 // src/browser/playwright.ts
 var logger5 = initializeLogger();
@@ -1978,6 +2087,15 @@ var StagehandInstrumentation = class extends import_instrumentation2.Instrumenta
       const result = await original.bind(this).apply(this, args);
       await instrumentation.playwrightInstrumentation.patchPage(this.page);
       instrumentation.patchStagehandPage(this.stagehandPage);
+      instrumentation.globalLLMClientOptions = {
+        provider: this.llmClient.type,
+        model: this.llmClient.modelName
+      };
+      instrumentation._wrap(
+        this.llmClient,
+        "createChatCompletion",
+        instrumentation.patchStagehandLLMClientCreateChatCompletion()
+      );
       return result;
     };
   }
@@ -2025,16 +2143,20 @@ var StagehandInstrumentation = class extends import_instrumentation2.Instrumenta
     }
     const extractHandler = page.extractHandler;
     if (extractHandler) {
-      this._wrap(
-        extractHandler,
-        "textExtract",
-        this.patchStagehandExtractHandlerTextExtract()
-      );
-      this._wrap(
-        extractHandler,
-        "domExtract",
-        this.patchStagehandExtractHandlerDomExtract()
-      );
+      if (extractHandler.textExtract) {
+        this._wrap(
+          extractHandler,
+          "textExtract",
+          this.patchStagehandExtractHandlerTextExtract()
+        );
+      }
+      if (extractHandler.domExtract) {
+        this._wrap(
+          extractHandler,
+          "domExtract",
+          this.patchStagehandExtractHandlerDomExtract()
+        );
+      }
     }
     this._wrap(
       page,
@@ -2196,6 +2318,92 @@ var StagehandInstrumentation = class extends import_instrumentation2.Instrumenta
       );
     };
   }
+  patchStagehandLLMClientCreateChatCompletion() {
+    const instrumentation = this;
+    return (original) => async function createChatCompletion(...args) {
+      const options = args[0];
+      return await observe({
+        name: "createChatCompletion",
+        // input and output are set as gen_ai.prompt and gen_ai.completion
+        ignoreInput: true,
+        ignoreOutput: true
+      }, async () => {
+        const span = import_api6.trace.getActiveSpan();
+        const innerOptions = options.options;
+        const recordedProvider = instrumentation.globalLLMClientOptions?.provider;
+        const provider = recordedProvider === "aisdk" && instrumentation.globalLLMClientOptions?.model ? modelToProviderMap[instrumentation.globalLLMClientOptions.model] ?? "aisdk" : recordedProvider;
+        span.setAttributes({
+          [SPAN_TYPE]: "LLM",
+          ...innerOptions.temperature ? {
+            "gen_ai.request.temperature": innerOptions.temperature
+          } : {},
+          ...innerOptions.top_p ? {
+            "gen_ai.request.top_p": innerOptions.top_p
+          } : {},
+          ...innerOptions.frequency_penalty ? {
+            "gen_ai.request.frequency_penalty": innerOptions.frequency_penalty
+          } : {},
+          ...innerOptions.presence_penalty ? {
+            "gen_ai.request.presence_penalty": innerOptions.presence_penalty
+          } : {},
+          ...innerOptions.maxTokens !== void 0 ? {
+            "gen_ai.request.max_tokens": innerOptions.maxTokens
+          } : {},
+          ...instrumentation.globalLLMClientOptions ? {
+            "gen_ai.request.model": instrumentation.globalLLMClientOptions.model,
+            "gen_ai.system": provider
+          } : {}
+        });
+        innerOptions.messages?.forEach((message, index) => {
+          span.setAttributes({
+            [`gen_ai.prompt.${index}.role`]: message.role,
+            [`gen_ai.prompt.${index}.content`]: JSON.stringify(message.content)
+          });
+        });
+        innerOptions.tools?.forEach((tool, index) => {
+          span.setAttributes({
+            [`llm.request.functions.${index}.name`]: tool.name,
+            [`llm.request.functions.${index}.description`]: tool.description,
+            [`llm.request.functions.${index}.parameters`]: JSON.stringify(tool.parameters)
+          });
+        });
+        const result = await original.bind(this).apply(this, args);
+        span.setAttributes({
+          "gen_ai.response.model": result.model,
+          "gen_ai.usage.input_tokens": result.usage.prompt_tokens,
+          "gen_ai.usage.output_tokens": result.usage.completion_tokens,
+          "llm.usage.total_tokens": result.usage.total_tokens
+        });
+        result.choices?.forEach((choice) => {
+          const index = choice.index;
+          span.setAttributes({
+            [`gen_ai.completion.${index}.finish_reason`]: choice.finish_reason,
+            [`gen_ai.completion.${index}.role`]: choice.message.role
+          });
+          if (choice.message.content) {
+            span.setAttribute(`gen_ai.completion.${index}.content`, JSON.stringify(choice.message.content));
+          }
+          choice.message.tool_calls?.forEach((toolCall, toolCallIndex) => {
+            span.setAttributes({
+              [`gen_ai.completion.${index}.message.tool_calls.${toolCallIndex}.id`]: toolCall.id,
+              [`gen_ai.completion.${index}.message.tool_calls.${toolCallIndex}.name`]: toolCall.function.name,
+              [`gen_ai.completion.${index}.message.tool_calls.${toolCallIndex}.arguments`]: JSON.stringify(toolCall.function.arguments)
+            });
+          });
+        });
+        if (!result.choices || result.choices.length === 0) {
+          const data = result.data;
+          if (data) {
+            span.setAttributes({
+              "gen_ai.completion.0.role": "assistant",
+              "gen_ai.completion.0.content": typeof data === "string" ? data : JSON.stringify(data)
+            });
+          }
+        }
+        return result;
+      });
+    };
+  }
 };
 
 // src/browser/puppeteer.ts
@@ -3298,10 +3506,11 @@ var getAverageScores = (results) => {
   const perScoreValues = {};
   for (const result of results) {
     for (const key in result.scores) {
-      if (perScoreValues[key]) {
-        perScoreValues[key].push(result.scores[key]);
+      const score = result.scores[key];
+      if (perScoreValues[key] && score !== null) {
+        perScoreValues[key].push(score);
       } else {
-        perScoreValues[key] = [result.scores[key]];
+        perScoreValues[key] = score !== null ? [score] : [];
       }
     }
   }
@@ -3311,6 +3520,8 @@ var getAverageScores = (results) => {
   }
   return averageScores;
 };
+var HumanEvaluator = class {
+};
 var EvaluationReporter = class {
   constructor(baseUrl) {
     this.cliProgress = new import_cli_progress.default.SingleBar(
@@ -3517,18 +3728,27 @@ var Evaluation = class {
         const value = await observe(
           { name: evaluatorName },
           async (output2, target2) => {
-            import_api13.trace.getActiveSpan().setAttribute(SPAN_TYPE, "EVALUATOR");
-            return await evaluator(output2, target2);
+            if (evaluator instanceof HumanEvaluator) {
+              import_api13.trace.getActiveSpan().setAttribute(SPAN_TYPE, "HUMAN_EVALUATOR");
+              return null;
+            } else {
+              import_api13.trace.getActiveSpan().setAttribute(SPAN_TYPE, "EVALUATOR");
+              return await evaluator(output2, target2);
+            }
           },
           output,
           target
         );
+        if (evaluator instanceof HumanEvaluator) {
+          scores[evaluatorName] = null;
+          continue;
+        }
         if (typeof value === "number") {
           if (isNaN(value)) {
             throw new Error(`Evaluator ${evaluatorName} returned NaN`);
           }
           scores[evaluatorName] = value;
-        } else {
+        } else if (value !== null) {
           scores = { ...scores, ...value };
         }
       }
@@ -3592,6 +3812,7 @@ var import_api14 = require("@opentelemetry/api");
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {
   Dataset,
+  HumanEvaluator,
   Laminar,
   LaminarAttributes,
   LaminarClient,