@lmnr-ai/lmnr 0.4.10 → 0.4.11-alpha

package/dist/index.js

@@ -35,14 +35,17 @@ var require_package = __commonJS({
   "package.json"(exports2, module2) {
     module2.exports = {
       name: "@lmnr-ai/lmnr",
-      version: "0.4.10",
+      version: "0.4.11-alpha",
       description: "TypeScript SDK for Laminar AI",
       main: "dist/index.js",
       types: "dist/index.d.ts",
       scripts: {
-        build: "tsup src/index.ts --format esm,cjs --dts",
+        build: "tsup src/index.ts src/cli.ts --format esm,cjs --dts",
         test: 'echo "Error: no test specified" && exit 1'
       },
+      bin: {
+        lmnr: "./dist/cli.js"
+      },
       repository: {
         type: "git",
         url: "git+https://github.com/lmnr-ai/lmnr-ts.git"
@@ -68,6 +71,8 @@ var require_package = __commonJS({
         "@langchain/core": ">=0.0.11",
         "@pinecone-database/pinecone": ">=2.0.1",
         "@qdrant/js-client-rest": ">=1.5.0",
+        "@types/argparse": "^2.0.16",
+        "@types/cli-progress": "^3.11.6",
         "@types/node": ">=14.18.63",
         "@types/semver": ">=7.5.3",
         "@types/uuid": ">=9.0.6",
@@ -105,6 +110,9 @@ var require_package = __commonJS({
         "@traceloop/instrumentation-pinecone": "^0.11.1",
         "@traceloop/instrumentation-qdrant": "^0.11.1",
         "@traceloop/instrumentation-vertexai": "^0.11.1",
+        argparse: "^2.0.1",
+        "cli-progress": "^3.12.0",
+        esbuild: "^0.23.1",
         "posthog-node": ">=3.1.3",
         uuid: ">=9.0.0"
       }
@@ -116,8 +124,8 @@ var require_package = __commonJS({
 var src_exports = {};
 __export(src_exports, {
   Dataset: () => Dataset,
-  Evaluation: () => Evaluation,
   Laminar: () => Laminar,
+  evaluate: () => evaluate,
   observe: () => observe
 });
 module.exports = __toCommonJS(src_exports);
@@ -661,6 +669,9 @@ var otelTraceIdToUUID = (traceId) => {
   }
   return id.replace(/^(.{8})(.{4})(.{4})(.{4})(.{12})$/, "$1-$2-$3-$4-$5");
 };
+function isNumber(value) {
+  return typeof value === "number";
+}
 
 // src/laminar.ts
 var import_exporter_trace_otlp_grpc = require("@opentelemetry/exporter-trace-otlp-grpc");
@@ -675,9 +686,8 @@ var Laminar = class {
    * If not specified, it will try to read from the LMNR_PROJECT_API_KEY environment variable.
    * @param env - Default environment passed to `run` and `evaluateEvent` requests,
    * unless overriden at request time. Usually, model provider keys are stored here.
-   * @param baseUrl - Url of Laminar endpoint, or the custom open telemetry ingester.
-   * If not specified, defaults to https://api.lmnr.ai:8443. For locally hosted Laminar,
-   * default setting must be http://localhost:8001.
+   * @param baseUrl - Laminar API url.
+   * If not specified, defaults to https://api.lmnr.ai.
    * @param instrumentModules - List of modules to instrument.
    * If not specified, all auto-instrumentable modules will be instrumented, which include
    * LLM calls (OpenAI, Anthropic, etc), Langchain, VectorDB calls (Pinecone, Qdrant, etc).
@@ -722,7 +732,7 @@ var Laminar = class {
     const metadata = new import_grpc_js.Metadata();
     metadata.set("authorization", `Bearer ${this.projectApiKey}`);
     const exporter = new import_exporter_trace_otlp_grpc.OTLPTraceExporter({
-      url: this.baseUrl,
+      url: `${this.baseUrl}:8443`,
       metadata
     });
     initialize({
@@ -926,11 +936,14 @@ var Laminar = class {
         name
       })
     });
+    if (!response.ok) {
+      throw new Error(`Failed to create evaluation ${name}. Response: ${response.statusText}`);
+    }
     return await response.json();
   }
-  static async postEvaluationResults(evaluationName, data) {
+  static async postEvaluationResults(evaluationId, data) {
     const body = JSON.stringify({
-      name: evaluationName,
+      evaluationId,
       points: data
     });
     const headers = this.getHeaders();
@@ -950,26 +963,27 @@ var Laminar = class {
     }
     ;
   }
-  static async updateEvaluationStatus(evaluationName, status) {
+  /**
+   * Updates the status of an evaluation. Returns the updated evaluation object.
+   * 
+   * @param evaluationId - The ID of the evaluation to update.
+   * @param status - The status to set for the evaluation.
+   */
+  static async updateEvaluationStatus(evaluationId, status) {
     const body = JSON.stringify({
-      name: evaluationName,
       status
     });
     const headers = this.getHeaders();
-    const url = `${this.baseUrl}/v1/evaluations`;
-    try {
-      const response = await fetch(url, {
-        method: "PUT",
-        headers,
-        body
-      });
-      if (!response.ok) {
-        console.error("Failed to update evaluation status. Response: ");
-        response.text().then((text) => console.error(text));
-      }
-    } catch (error) {
-      console.error("Failed to update evaluation status. Error: ", error);
+    const url = `${this.baseUrl}/v1/evaluations/${evaluationId}`;
+    const response = await fetch(url, {
+      method: "POST",
+      headers,
+      body
+    });
+    if (!response.ok) {
+      throw new Error(`Failed to update evaluation status ${evaluationId}. Response: ${await response.text()}`);
     }
+    return await response.json();
   }
   static getHeaders() {
     return {
@@ -978,12 +992,16 @@ var Laminar = class {
     };
   }
 };
-Laminar.baseUrl = "https://api.lmnr.ai:8443";
+Laminar.baseUrl = "https://api.lmnr.ai";
 Laminar.env = {};
 Laminar.isInitialized = false;
 
 // src/evaluations.ts
+var import_cli_progress = __toESM(require("cli-progress"));
 var DEFAULT_BATCH_SIZE = 5;
+var getEvaluationUrl = (projectId, evaluationId) => {
+  return `https://www.lmnr.ai/project/${projectId}/evaluations/${evaluationId}`;
+};
 var Dataset = class {
   slice(start, end) {
     const result = [];
@@ -993,6 +1011,41 @@ var Dataset = class {
     return result;
   }
 };
+var EvaluationReporter = class {
+  constructor() {
+    this.cliProgress = new import_cli_progress.default.SingleBar({}, import_cli_progress.default.Presets.shades_classic);
+  }
+  start({ name, projectId, id, length }) {
+    process.stdout.write(`
+Running evaluation ${name}...
+
+`);
+    process.stdout.write(`Check progress and results at ${getEvaluationUrl(projectId, id)}
+
+`);
+    this.cliProgress.start(length, 0);
+  }
+  update(batchLength) {
+    this.cliProgress.update(batchLength);
+  }
+  // Call either error or stop, not both
+  stopWithError(error) {
+    this.cliProgress.stop();
+    process.stdout.write(`
+Error: ${error.message}
+`);
+  }
+  // Call either error or stop, not both
+  stop(averageScores) {
+    this.cliProgress.stop();
+    process.stdout.write("\nAverage scores:\n");
+    for (const key in averageScores) {
+      process.stdout.write(`${key}: ${averageScores[key]}
+`);
+    }
+    process.stdout.write("\n");
+  }
+};
 var Evaluation = class {
   /**
    * Create a new evaluation and prepare data.
@@ -1000,6 +1053,7 @@ var Evaluation = class {
    * @param props.data List of data points to evaluate. `data` is the input to the executor function, `target` is the input to the evaluator function.
    * @param props.executor The executor function. Takes the data point + any additional arguments and returns the output to evaluate.
    * @param props.evaluators List of evaluator functions. Each evaluator function takes the output of the executor and the target data, and returns.
+   * @param props.config Optional override configurations for the evaluator.
    */
   constructor(name, {
     data,
@@ -1007,9 +1061,11 @@ var Evaluation = class {
     evaluators,
     config
   }) {
+    this.isFinished = false;
     this.batchSize = DEFAULT_BATCH_SIZE;
     var _a;
     this.name = name;
+    this.progressReporter = new EvaluationReporter();
     this.data = data;
     this.executor = executor;
     this.evaluators = Object.fromEntries(evaluators.map((e, i) => [e.name.length > 0 ? e.name : `evaluator_${i + 1}`, e]));
@@ -1022,26 +1078,42 @@ var Evaluation = class {
   /** 
    * Runs the evaluation.
    *
-   * Creates a new evaluation if no evaluation with such name exists, or adds data to an existing one otherwise.
+   * Creates a new evaluation.
    * Evaluates data points in batches of `batchSize`. The executor function is called on each data point
    * to get the output, and then evaluate it by each evaluator function.
    */
   async run() {
-    const response = await Laminar.createEvaluation(this.name);
-    const length = this.data instanceof Dataset ? this.data.size() : this.data.length;
-    for (let i = 0; i < length; i += this.batchSize) {
+    if (this.isFinished) {
+      throw new Error("Evaluation is already finished");
+    }
+    const evaluation = await Laminar.createEvaluation(this.name);
+    this.progressReporter.start({ name: evaluation.name, projectId: evaluation.projectId, id: evaluation.id, length: this.getLength() });
+    try {
+      await this.evaluateInBatches(evaluation);
+    } catch (e) {
+      await Laminar.updateEvaluationStatus(evaluation.id, "Error");
+      this.progressReporter.stopWithError(e);
+      this.isFinished = true;
+    }
+    if (!this.isFinished) {
+      const updatedEvaluation = await Laminar.updateEvaluationStatus(evaluation.id, "Finished");
+      this.progressReporter.stop(updatedEvaluation.averageScores);
+      this.isFinished = true;
+    }
+  }
+  // TODO: Calculate duration of the evaluation and add it to the summary
+  async evaluateInBatches(evaluation) {
+    for (let i = 0; i < this.getLength(); i += this.batchSize) {
       const batch = this.data.slice(i, i + this.batchSize);
       try {
-        await this.evaluateBatch(batch);
+        const results = await this.evaluateBatch(batch);
+        await Laminar.postEvaluationResults(evaluation.id, results);
       } catch (e) {
         console.error(`Error evaluating batch: ${e}`);
+      } finally {
+        this.progressReporter.update(batch.length);
       }
     }
-    try {
-      await Laminar.updateEvaluationStatus(response.name, "Finished");
-    } catch (e) {
-      console.error(`Error updating evaluation status: ${e}`);
-    }
   }
   async evaluateBatch(batch) {
     const batchPromises = batch.map(async (datapoint) => {
@@ -1051,7 +1123,10 @@ var Evaluation = class {
       for (const evaluatorName of this.evaluatorNames) {
         const evaluator = this.evaluators[evaluatorName];
         const value = await evaluator(output, target);
-        if (typeof value === "number") {
+        if (isNumber(value)) {
+          if (isNaN(value)) {
+            throw new Error(`Evaluator ${evaluatorName} returned NaN`);
+          }
           scores[evaluatorName] = value;
         } else {
           scores = { ...scores, ...value };
@@ -1065,9 +1140,27 @@ var Evaluation = class {
       };
     });
     const results = await Promise.all(batchPromises);
-    return Laminar.postEvaluationResults(this.name, results);
+    return results;
+  }
+  getLength() {
+    return this.data instanceof Dataset ? this.data.size() : this.data.length;
   }
 };
+function evaluate(name, {
+  data,
+  executor,
+  evaluators,
+  config
+}) {
+  (async () => {
+    const evaluation = new Evaluation(name, { data, executor, evaluators, config });
+    if (globalThis._set_global_evaluation) {
+      globalThis._evaluation = evaluation;
+    } else {
+      await evaluation.run();
+    }
+  })();
+}
 
 // src/decorators.ts
 async function observe({
@@ -1087,8 +1180,8 @@ async function observe({
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {
   Dataset,
-  Evaluation,
   Laminar,
+  evaluate,
   observe
 });
 //# sourceMappingURL=index.js.map