@lmnr-ai/lmnr 0.7.5-alpha.2 → 0.7.6

package/dist/cli.js

@@ -6,10 +6,6 @@ var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
 var __getOwnPropNames = Object.getOwnPropertyNames;
 var __getProtoOf = Object.getPrototypeOf;
 var __hasOwnProp = Object.prototype.hasOwnProperty;
-var __export = (target, all) => {
-  for (var name in all)
-    __defProp(target, name, { get: all[name], enumerable: true });
-};
 var __copyProps = (to, from, except, desc) => {
   if (from && typeof from === "object" || typeof from === "function") {
     for (let key of __getOwnPropNames(from))
@@ -26,21 +22,12 @@ var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__ge
   isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
   mod
 ));
-var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
 
-// src/cli.ts
-var cli_exports = {};
-__export(cli_exports, {
-  loadModule: () => loadModule
-});
-module.exports = __toCommonJS(cli_exports);
+// src/cli/index.ts
 var import_commander = require("commander");
-var esbuild = __toESM(require("esbuild"));
-var fs = __toESM(require("fs"));
-var glob = __toESM(require("glob"));
 
 // package.json
-var version = "0.7.5-alpha.2";
+var version = "0.7.5";
 
 // src/utils.ts
 var import_api = require("@opentelemetry/api");
@@ -85,6 +72,48 @@ function initializeLogger(options) {
   }));
 }
 var logger = initializeLogger();
+var isStringUUID = (id) => /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/.test(id);
+var newUUID = () => {
+  if (typeof crypto !== "undefined" && typeof crypto.randomUUID === "function") {
+    return crypto.randomUUID();
+  } else {
+    return (0, import_uuid.v4)();
+  }
+};
+var otelSpanIdToUUID = (spanId) => {
+  let id = spanId.toLowerCase();
+  if (id.startsWith("0x")) {
+    id = id.slice(2);
+  }
+  if (id.length !== 16) {
+    logger.warn(`Span ID ${spanId} is not 16 hex chars long. This is not a valid OpenTelemetry span ID.`);
+  }
+  if (!/^[0-9a-f]+$/.test(id)) {
+    logger.error(`Span ID ${spanId} is not a valid hex string. Generating a random UUID instead.`);
+    return newUUID();
+  }
+  return id.padStart(32, "0").replace(
+    /^([0-9a-f]{8})([0-9a-f]{4})([0-9a-f]{4})([0-9a-f]{4})([0-9a-f]{12})$/,
+    "$1-$2-$3-$4-$5"
+  );
+};
+var otelTraceIdToUUID = (traceId) => {
+  let id = traceId.toLowerCase();
+  if (id.startsWith("0x")) {
+    id = id.slice(2);
+  }
+  if (id.length !== 32) {
+    logger.warn(`Trace ID ${traceId} is not 32 hex chars long. This is not a valid OpenTelemetry trace ID.`);
+  }
+  if (!/^[0-9a-f]+$/.test(id)) {
+    logger.error(`Trace ID ${traceId} is not a valid hex string. Generating a random UUID instead.`);
+    return newUUID();
+  }
+  return id.replace(
+    /^([0-9a-f]{8})([0-9a-f]{4})([0-9a-f]{4})([0-9a-f]{4})([0-9a-f]{12})$/,
+    "$1-$2-$3-$4-$5"
+  );
+};
 var getDirname = () => {
   if (typeof __dirname !== "undefined") {
     return __dirname;
@@ -94,16 +123,1283 @@ var getDirname = () => {
   }
   return process.cwd();
 };
+var slicePayload = (value, length) => {
+  if (value === null || value === void 0) {
+    return value;
+  }
+  const str = JSON.stringify(value);
+  if (str.length <= length) {
+    return value;
+  }
+  return str.slice(0, length) + "...";
+};
+
+// src/client/index.ts
+var import_dotenv = require("dotenv");
+
+// src/client/resources/agent.ts
+var import_api3 = require("@opentelemetry/api");
 
-// src/cli.ts
+// src/opentelemetry-lib/tracing/context.ts
+var import_api2 = require("@opentelemetry/api");
+var import_async_hooks = require("async_hooks");
+var LaminarContextManager = class {
+  static getContext() {
+    const contexts = this._asyncLocalStorage.getStore() || [];
+    for (let i = contexts.length - 1; i >= 0; i--) {
+      const context = contexts[i];
+      const span = import_api2.trace.getSpan(context);
+      if (!span) {
+        return context;
+      }
+      try {
+        const isActive = this._activeSpans.has(span.spanContext().spanId);
+        if (isActive) {
+          return context;
+        }
+      } catch {
+        return context;
+      }
+    }
+    return import_api2.ROOT_CONTEXT;
+  }
+  static pushContext(context) {
+    const contexts = this._asyncLocalStorage.getStore() || [];
+    const newContexts = [...contexts, context];
+    this._asyncLocalStorage.enterWith(newContexts);
+  }
+  static popContext() {
+    const contexts = this._asyncLocalStorage.getStore() || [];
+    if (contexts.length > 0) {
+      const newContexts = contexts.slice(0, -1).filter((context) => {
+        const span = import_api2.trace.getSpan(context);
+        if (!span) {
+          return true;
+        }
+        try {
+          return this._activeSpans.has(span.spanContext().spanId);
+        } catch {
+          return true;
+        }
+      });
+      this._asyncLocalStorage.enterWith(newContexts);
+    }
+  }
+  static clearContexts() {
+    this._asyncLocalStorage.enterWith([]);
+  }
+  static getContextStack() {
+    return this._asyncLocalStorage.getStore() || [];
+  }
+  /**
+   * Run a function with an isolated context stack.
+   * This ensures that parallel executions don't interfere with each other.
+   */
+  static runWithIsolatedContext(initialStack, fn) {
+    return this._asyncLocalStorage.run(initialStack, fn);
+  }
+  static addActiveSpan(spanId) {
+    this._activeSpans.add(spanId);
+  }
+  static removeActiveSpan(spanId) {
+    this._activeSpans.delete(spanId);
+  }
+};
+LaminarContextManager._asyncLocalStorage = new import_async_hooks.AsyncLocalStorage();
+// Static registry for cross-async span management
+// We're keeping track of spans have started (and running) here for the cases when span
+// is started in one async context and ended in another
+// We use this registry to ignore the context of spans that were already ended in
+// another async context.
+// LaminarSpan adds and removes itself to and from this registry in start()
+// and end() methods respectively.
+LaminarContextManager._activeSpans = /* @__PURE__ */ new Set();
+
+// src/client/resources/index.ts
+var BaseResource = class {
+  constructor(baseHttpUrl, projectApiKey) {
+    this.baseHttpUrl = baseHttpUrl;
+    this.projectApiKey = projectApiKey;
+  }
+  headers() {
+    return {
+      Authorization: `Bearer ${this.projectApiKey}`,
+      "Content-Type": "application/json",
+      Accept: "application/json"
+    };
+  }
+  async handleError(response) {
+    const errorMsg = await response.text();
+    throw new Error(`${response.status} ${errorMsg}`);
+  }
+};
+
+// src/client/resources/agent.ts
 var logger2 = initializeLogger();
+var AgentResource = class extends BaseResource {
+  constructor(baseHttpUrl, projectApiKey) {
+    super(baseHttpUrl, projectApiKey);
+  }
+  /**
+   * Run Laminar index agent
+   *
+   * @param { RunAgentOptions } options - The options for running the agent
+   * @param { string } options.prompt - The prompt for the agent
+   * @param { string } [options.parentSpanContext] - The parent span context for tracing
+   * @param { ModelProvider } [options.modelProvider] - LLM provider to use
+   * @param { string } [options.model] - The model name as specified in the provider API
+   * @param { boolean } [options.stream] - Whether to stream the response. Defaults to false.
+   * @param { boolean } [options.enableThinking] - Whether to enable thinking in the underlying
+   * LLM. Defaults to true.
+   * @param { number } [options.timeout] - The timeout in seconds for the agent.
+   * Note: This is a soft timeout. The agent will finish a step even after the timeout has
+   * been reached.
+   * @param { string } [options.cdpUrl] - The URL of an existing Chrome DevTools Protocol
+   * (CDP) browser instance.
+   * @param { number } [options.maxSteps] - The maximum number of steps the agent can take.
+   * Defaults to 100.
+   * @param { number } [options.thinkingTokenBudget] - The maximum number of tokens the underlying
+   * LLM can spend on thinking per step, if supported by the LLM provider.
+   * @param { string } [options.startUrl] - The URL to start the agent on.
+   * Make sure it's a valid URL - refer to https://playwright.dev/docs/api/class-page#page-goto
+   * If not specified, the agent will infer this from the prompt.
+   * @param { string } [options.userAgent] - The user agent to set in the browser.
+   * If not specified, Laminar will use the default user agent.
+   * @param { boolean } [options.returnScreenshots] - Whether to return screenshots with
+   * each step. Defaults to false.
+   * @param { boolean } [options.returnAgentState] - Whether to return the agent state.
+   * Agent state can be used to resume the agent in subsequent runs.
+   * CAUTION: Agent state is a very large object. Defaults to false.
+   * @param { boolean } [options.returnStorageState] - Whether to return the storage state.
+   * Storage state includes browser cookies, auth, etc.
+   * CAUTION: Storage state is a relatively large object. Defaults to false.
+   * @param { boolean } [options.disableGiveControl] - Whether to NOT direct the agent
+   * to give control back to the user for tasks such as logging in. Defaults to false.
+   * @returns { Promise<AgentOutput | ReadableStream<RunAgentResponseChunk>> }
+   *    The agent output or a stream of response chunks
+   */
+  async run({
+    prompt,
+    parentSpanContext,
+    modelProvider,
+    model,
+    stream,
+    enableThinking,
+    timeout,
+    cdpUrl,
+    agentState,
+    storageState,
+    maxSteps,
+    thinkingTokenBudget,
+    startUrl,
+    userAgent,
+    returnScreenshots,
+    returnAgentState,
+    returnStorageState,
+    disableGiveControl
+  }) {
+    let requestParentSpanContext = parentSpanContext;
+    if (!requestParentSpanContext) {
+      const currentSpan = import_api3.trace.getSpan(LaminarContextManager.getContext()) ?? import_api3.trace.getActiveSpan();
+      if (currentSpan && currentSpan.isRecording()) {
+        const traceId = otelTraceIdToUUID(currentSpan.spanContext().traceId);
+        const spanId = otelSpanIdToUUID(currentSpan.spanContext().spanId);
+        requestParentSpanContext = JSON.stringify({
+          trace_id: traceId,
+          span_id: spanId,
+          is_remote: currentSpan.spanContext().isRemote
+        });
+      }
+    }
+    const request = {
+      prompt,
+      parentSpanContext: requestParentSpanContext,
+      modelProvider,
+      model,
+      stream: true,
+      enableThinking: enableThinking ?? true,
+      timeout,
+      cdpUrl,
+      agentState,
+      storageState,
+      maxSteps,
+      thinkingTokenBudget,
+      startUrl,
+      userAgent,
+      returnScreenshots: returnScreenshots ?? false,
+      returnAgentState: returnAgentState ?? false,
+      returnStorageState: returnStorageState ?? false,
+      disableGiveControl: disableGiveControl ?? false
+    };
+    if (stream) {
+      return this.runStreaming(request);
+    } else {
+      return await this.runNonStreaming(request);
+    }
+  }
+  /**
+   * Run agent in streaming mode
+   */
+  async runStreaming(request) {
+    const response = await fetch(this.baseHttpUrl + "/v1/agent/run", {
+      method: "POST",
+      headers: this.headers(),
+      body: JSON.stringify(request)
+    });
+    if (!response.ok) {
+      await this.handleError(response);
+    }
+    if (!response.body) {
+      throw new Error("Response body is null");
+    }
+    return response.body.pipeThrough(new TextDecoderStream()).pipeThrough(new TransformStream({
+      start() {
+        this.buffer = "";
+      },
+      transform(chunk, controller) {
+        this.buffer += chunk;
+        const lines = this.buffer.split("\n");
+        this.buffer = lines.pop() || "";
+        for (const line of lines) {
+          if (line.startsWith("[DONE]")) {
+            controller.terminate();
+            return;
+          }
+          if (!line.startsWith("data: ")) {
+            continue;
+          }
+          const jsonStr = line.substring(6);
+          if (jsonStr) {
+            try {
+              const parsed = JSON.parse(jsonStr);
+              controller.enqueue(parsed);
+            } catch (error) {
+              logger2.error(`Error parsing JSON: ${error instanceof Error ? error.message : String(error)}`);
+            }
+          }
+        }
+      },
+      flush(controller) {
+        if (this.buffer) {
+          if (this.buffer.startsWith("data: ")) {
+            const jsonStr = this.buffer.substring(6);
+            if (jsonStr) {
+              try {
+                const parsed = JSON.parse(jsonStr);
+                controller.enqueue(parsed);
+              } catch (error) {
+                logger2.error(`Error parsing JSON: ${error instanceof Error ? error.message : String(error)}`);
+              }
+            }
+          }
+        }
+      }
+    }));
+  }
+  /**
+   * Run agent in non-streaming mode
+   */
+  async runNonStreaming(request) {
+    const stream = await this.runStreaming(request);
+    const reader = stream.getReader();
+    let finalChunk = null;
+    let errorChunk = null;
+    try {
+      while (true) {
+        const { done, value } = await reader.read();
+        if (done) break;
+        if (value.chunkType === "finalOutput") {
+          finalChunk = value;
+          break;
+        } else if (value.chunkType === "error") {
+          errorChunk = value;
+          break;
+        } else if (value.chunkType === "timeout") {
+          errorChunk = {
+            chunkType: "error",
+            messageId: value.messageId,
+            error: "Timeout"
+          };
+          break;
+        }
+      }
+    } finally {
+      reader.releaseLock();
+    }
+    if (errorChunk) {
+      throw new Error(errorChunk.error);
+    }
+    return finalChunk?.content || { result: { isDone: true } };
+  }
+};
+
+// src/version.ts
+var getLangVersion = () => {
+  if (process?.versions?.node) {
+    return `node@${process.versions.node}`;
+  }
+};
+
+// src/client/resources/browser-events.ts
+var BrowserEventsResource = class extends BaseResource {
+  constructor(baseHttpUrl, projectApiKey) {
+    super(baseHttpUrl, projectApiKey);
+  }
+  async send({
+    sessionId,
+    traceId,
+    events
+  }) {
+    const payload = {
+      sessionId,
+      traceId,
+      events,
+      source: getLangVersion() ?? "javascript",
+      sdkVersion: version
+    };
+    const jsonString = JSON.stringify(payload);
+    const blob = new Blob([jsonString], { type: "application/json" });
+    const compressedStream = blob.stream().pipeThrough(new CompressionStream("gzip"));
+    const compressedResponse = new Response(compressedStream);
+    const compressedData = await compressedResponse.arrayBuffer();
+    const response = await fetch(this.baseHttpUrl + "/v1/browser-sessions/events", {
+      method: "POST",
+      headers: {
+        ...this.headers(),
+        "Content-Encoding": "gzip"
+      },
+      body: compressedData
+    });
+    if (!response.ok) {
+      await this.handleError(response);
+    }
+  }
+};
+
+// src/client/resources/datasets.ts
+var logger3 = initializeLogger();
+var DEFAULT_DATASET_PULL_LIMIT = 100;
+var DEFAULT_DATASET_PUSH_BATCH_SIZE = 100;
+var DatasetsResource = class extends BaseResource {
+  constructor(baseHttpUrl, projectApiKey) {
+    super(baseHttpUrl, projectApiKey);
+  }
+  /**
+   * List all datasets.
+   *
+   * @returns {Promise<Dataset[]>} Array of datasets
+   */
+  async listDatasets() {
+    const response = await fetch(this.baseHttpUrl + "/v1/datasets", {
+      method: "GET",
+      headers: this.headers()
+    });
+    if (!response.ok) {
+      await this.handleError(response);
+    }
+    return response.json();
+  }
+  /**
+   * Get a dataset by name.
+   *
+   * @param {string} name - Name of the dataset
+   * @returns {Promise<Dataset[]>} Array of datasets with matching name
+   */
+  async getDatasetByName(name) {
+    const params = new URLSearchParams({ name });
+    const response = await fetch(
+      this.baseHttpUrl + `/v1/datasets?${params.toString()}`,
+      {
+        method: "GET",
+        headers: this.headers()
+      }
+    );
+    if (!response.ok) {
+      await this.handleError(response);
+    }
+    return response.json();
+  }
+  /**
+   * Push datapoints to a dataset.
+   *
+   * @param {Object} options - Push options
+   * @param {Datapoint<D, T>[]} options.points - Datapoints to push
+   * @param {string} [options.name] - Name of the dataset (either name or id must be provided)
+   * @param {StringUUID} [options.id] - ID of the dataset (either name or id must be provided)
+   * @param {number} [options.batchSize] - Batch size for pushing (default: 100)
+   * @param {boolean} [options.createDataset] - Whether to create the dataset if it doesn't exist
+   * @returns {Promise<PushDatapointsResponse | undefined>}
+   */
+  async push({
+    points,
+    name,
+    id,
+    batchSize = DEFAULT_DATASET_PUSH_BATCH_SIZE,
+    createDataset = false
+  }) {
+    if (!name && !id) {
+      throw new Error("Either name or id must be provided");
+    }
+    if (name && id) {
+      throw new Error("Only one of name or id must be provided");
+    }
+    if (createDataset && !name) {
+      throw new Error("Name must be provided when creating a new dataset");
+    }
+    const identifier = name ? { name } : { datasetId: id };
+    const totalBatches = Math.ceil(points.length / batchSize);
+    let response;
+    for (let i = 0; i < points.length; i += batchSize) {
+      const batchNum = Math.floor(i / batchSize) + 1;
+      logger3.debug(`Pushing batch ${batchNum} of ${totalBatches}`);
+      const batch = points.slice(i, i + batchSize);
+      const fetchResponse = await fetch(
+        this.baseHttpUrl + "/v1/datasets/datapoints",
+        {
+          method: "POST",
+          headers: this.headers(),
+          body: JSON.stringify({
+            ...identifier,
+            datapoints: batch.map((point) => ({
+              data: point.data,
+              target: point.target ?? {},
+              metadata: point.metadata ?? {}
+            })),
+            createDataset
+          })
+        }
+      );
+      if (fetchResponse.status !== 200 && fetchResponse.status !== 201) {
+        await this.handleError(fetchResponse);
+      }
+      response = await fetchResponse.json();
+    }
+    return response;
+  }
+  /**
+   * Pull datapoints from a dataset.
+   *
+   * @param {Object} options - Pull options
+   * @param {string} [options.name] - Name of the dataset (either name or id must be provided)
+   * @param {StringUUID} [options.id] - ID of the dataset (either name or id must be provided)
+   * @param {number} [options.limit] - Maximum number of datapoints to return (default: 100)
+   * @param {number} [options.offset] - Offset for pagination (default: 0)
+   * @returns {Promise<GetDatapointsResponse<D, T>>}
+   */
+  async pull({
+    name,
+    id,
+    limit = DEFAULT_DATASET_PULL_LIMIT,
+    offset = 0
+  }) {
+    if (!name && !id) {
+      throw new Error("Either name or id must be provided");
+    }
+    if (name && id) {
+      throw new Error("Only one of name or id must be provided");
+    }
+    const paramsObj = {
+      offset: offset.toString(),
+      limit: limit.toString()
+    };
+    if (name) {
+      paramsObj.name = name;
+    } else {
+      paramsObj.datasetId = id;
+    }
+    const params = new URLSearchParams(paramsObj);
+    const response = await fetch(
+      this.baseHttpUrl + `/v1/datasets/datapoints?${params.toString()}`,
+      {
+        method: "GET",
+        headers: this.headers()
+      }
+    );
+    if (!response.ok) {
+      await this.handleError(response);
+    }
+    return response.json();
+  }
+};
+
+// src/client/resources/evals.ts
+var logger4 = initializeLogger();
+var INITIAL_EVALUATION_DATAPOINT_MAX_DATA_LENGTH = 16e6;
+var EvalsResource = class extends BaseResource {
+  constructor(baseHttpUrl, projectApiKey) {
+    super(baseHttpUrl, projectApiKey);
+  }
+  /**
+   * Initialize an evaluation.
+   *
+   * @param {string} name - Name of the evaluation
+   * @param {string} groupName - Group name of the evaluation
+   * @param {Record<string, any>} metadata - Optional metadata
+   * @returns {Promise<InitEvaluationResponse>} Response from the evaluation initialization
+   */
+  async init(name, groupName, metadata) {
+    const response = await fetch(this.baseHttpUrl + "/v1/evals", {
+      method: "POST",
+      headers: this.headers(),
+      body: JSON.stringify({
+        name: name ?? null,
+        groupName: groupName ?? null,
+        metadata: metadata ?? null
+      })
+    });
+    if (!response.ok) {
+      await this.handleError(response);
+    }
+    return response.json();
+  }
+  /**
+   * Create a new evaluation and return its ID.
+   *
+   * @param {string} [name] - Optional name of the evaluation
+   * @param {string} [groupName] - An identifier to group evaluations
+   * @param {Record<string, any>} [metadata] - Optional metadata
+   * @returns {Promise<StringUUID>} The evaluation ID
+   */
+  async create(args) {
+    const evaluation = await this.init(args?.name, args?.groupName, args?.metadata);
+    return evaluation.id;
+  }
+  /**
+   * Create a new evaluation and return its ID.
+   * @deprecated use `create` instead.
+   */
+  async createEvaluation(name, groupName, metadata) {
+    const evaluation = await this.init(name, groupName, metadata);
+    return evaluation.id;
+  }
+  /**
+   * Create a datapoint for an evaluation.
+   *
+   * @param {Object} options - Create datapoint options
+   * @param {string} options.evalId - The evaluation ID
+   * @param {D} options.data - The input data for the executor
+   * @param {T} [options.target] - The target/expected output for evaluators
+   * @param {Record<string, any>} [options.metadata] - Optional metadata
+   * @param {number} [options.index] - Optional index of the datapoint
+   * @param {string} [options.traceId] - Optional trace ID
+   * @returns {Promise<StringUUID>} The datapoint ID
+   */
+  async createDatapoint({
+    evalId,
+    data,
+    target,
+    metadata,
+    index,
+    traceId
+  }) {
+    const datapointId = newUUID();
+    const partialDatapoint = {
+      id: datapointId,
+      data,
+      target,
+      index: index ?? 0,
+      traceId: traceId ?? newUUID(),
+      executorSpanId: newUUID(),
+      metadata
+    };
+    await this.saveDatapoints({
+      evalId,
+      datapoints: [partialDatapoint]
+    });
+    return datapointId;
+  }
+  /**
+   * Update a datapoint with evaluation results.
+   *
+   * @param {Object} options - Update datapoint options
+   * @param {string} options.evalId - The evaluation ID
+   * @param {string} options.datapointId - The datapoint ID
+   * @param {Record<string, number>} options.scores - The scores
+   * @param {O} [options.executorOutput] - The executor output
+   * @returns {Promise<void>}
+   */
+  async updateDatapoint({
+    evalId,
+    datapointId,
+    scores,
+    executorOutput
+  }) {
+    const response = await fetch(
+      this.baseHttpUrl + `/v1/evals/${evalId}/datapoints/${datapointId}`,
+      {
+        method: "POST",
+        headers: this.headers(),
+        body: JSON.stringify({
+          executorOutput,
+          scores
+        })
+      }
+    );
+    if (!response.ok) {
+      await this.handleError(response);
+    }
+  }
+  /**
+   * Save evaluation datapoints.
+   *
+   * @param {Object} options - Save datapoints options
+   * @param {string} options.evalId - ID of the evaluation
+   * @param {EvaluationDatapoint<D, T, O>[]} options.datapoints - Datapoint to add
+   * @param {string} [options.groupName] - Group name of the evaluation
+   * @returns {Promise<void>} Response from the datapoint addition
+   */
+  async saveDatapoints({
+    evalId,
+    datapoints,
+    groupName
+  }) {
+    const response = await fetch(this.baseHttpUrl + `/v1/evals/${evalId}/datapoints`, {
+      method: "POST",
+      headers: this.headers(),
+      body: JSON.stringify({
+        points: datapoints.map((d) => ({
+          ...d,
+          data: slicePayload(d.data, INITIAL_EVALUATION_DATAPOINT_MAX_DATA_LENGTH),
+          target: slicePayload(d.target, INITIAL_EVALUATION_DATAPOINT_MAX_DATA_LENGTH),
+          executorOutput: slicePayload(
+            d.executorOutput,
+            INITIAL_EVALUATION_DATAPOINT_MAX_DATA_LENGTH
+          )
+        })),
+        groupName: groupName ?? null
+      })
+    });
+    if (response.status === 413) {
+      return await this.retrySaveDatapoints({
+        evalId,
+        datapoints,
+        groupName
+      });
+    }
+    if (!response.ok) {
+      await this.handleError(response);
+    }
+  }
+  /**
+   * Get evaluation datapoints.
+   *
+   * @deprecated Use `client.datasets.pull()` instead.
+   * @param {Object} options - Get datapoints options
+   * @param {string} options.datasetName - Name of the dataset
+   * @param {number} options.offset - Offset at which to start the query
+   * @param {number} options.limit - Maximum number of datapoints to return
+   * @returns {Promise<GetDatapointsResponse>} Response from the datapoint retrieval
+   */
+  async getDatapoints({
+    datasetName,
+    offset,
+    limit
+  }) {
+    logger4.warn(
+      "evals.getDatapoints() is deprecated. Use client.datasets.pull() instead."
+    );
+    const params = new URLSearchParams({
+      name: datasetName,
+      offset: offset.toString(),
+      limit: limit.toString()
+    });
+    const response = await fetch(
+      this.baseHttpUrl + `/v1/datasets/datapoints?${params.toString()}`,
+      {
+        method: "GET",
+        headers: this.headers()
+      }
+    );
+    if (!response.ok) {
+      await this.handleError(response);
+    }
+    return await response.json();
+  }
+  async retrySaveDatapoints({
+    evalId,
+    datapoints,
+    groupName,
+    maxRetries = 25,
+    initialLength = INITIAL_EVALUATION_DATAPOINT_MAX_DATA_LENGTH
+  }) {
+    let length = initialLength;
+    let lastResponse = null;
+    for (let i = 0; i < maxRetries; i++) {
+      logger4.debug(`Retrying save datapoints... ${i + 1} of ${maxRetries}, length: ${length}`);
+      const response = await fetch(this.baseHttpUrl + `/v1/evals/${evalId}/datapoints`, {
+        method: "POST",
+        headers: this.headers(),
+        body: JSON.stringify({
+          points: datapoints.map((d) => ({
+            ...d,
+            data: slicePayload(d.data, length),
+            target: slicePayload(d.target, length),
+            executorOutput: slicePayload(d.executorOutput, length)
+          })),
+          groupName: groupName ?? null
+        })
+      });
+      lastResponse = response;
+      length = Math.floor(length / 2);
+      if (response.status !== 413) {
+        break;
+      }
+    }
+    if (lastResponse && !lastResponse.ok) {
+      await this.handleError(lastResponse);
+    }
+  }
+};
+
+// src/client/resources/evaluators.ts
+var EvaluatorsResource = class extends BaseResource {
+  constructor(baseHttpUrl, projectApiKey) {
+    super(baseHttpUrl, projectApiKey);
+  }
+  /**
+   * Create a score for a span or trace
+   *
+   * @param {ScoreOptions} options - Score creation options
+   * @param {string} options.name - Name of the score
+   * @param {string} [options.traceId] - The trace ID to score (will be attached to top-level span)
+   * @param {string} [options.spanId] - The span ID to score
+   * @param {Record<string, any>} [options.metadata] - Additional metadata
+   * @param {number} options.score - The score value (float)
+   * @returns {Promise<void>}
+   *
+   * @example
+   * // Score by trace ID (will attach to root span)
+   * await evaluators.score({
+   *   name: "quality",
+   *   traceId: "trace-id-here",
+   *   score: 0.95,
+   *   metadata: { model: "gpt-4" }
+   * });
+   *
+   * @example
+   * // Score by span ID
+   * await evaluators.score({
+   *   name: "relevance",
+   *   spanId: "span-id-here",
+   *   score: 0.87
+   * });
+   */
+  async score(options) {
+    const { name, metadata, score } = options;
+    let payload;
+    if ("traceId" in options && options.traceId) {
+      const formattedTraceId = isStringUUID(options.traceId) ? options.traceId : otelTraceIdToUUID(options.traceId);
+      payload = {
+        name,
+        metadata,
+        score,
+        source: "Code" /* Code */,
+        traceId: formattedTraceId
+      };
+    } else if ("spanId" in options && options.spanId) {
+      const formattedSpanId = isStringUUID(options.spanId) ? options.spanId : otelSpanIdToUUID(options.spanId);
+      payload = {
+        name,
+        metadata,
+        score,
+        source: "Code" /* Code */,
+        spanId: formattedSpanId
+      };
+    } else {
+      throw new Error("Either 'traceId' or 'spanId' must be provided.");
+    }
+    const response = await fetch(this.baseHttpUrl + "/v1/evaluators/score", {
+      method: "POST",
+      headers: this.headers(),
+      body: JSON.stringify(payload)
+    });
+    if (!response.ok) {
+      await this.handleError(response);
+    }
+  }
+};
+
+// src/client/resources/tags.ts
+var TagsResource = class extends BaseResource {
+  /** Resource for tagging traces. */
+  constructor(baseHttpUrl, projectApiKey) {
+    super(baseHttpUrl, projectApiKey);
+  }
+  /**
+   * Tag a trace with a list of tags. Note that the trace must be ended before
+   * tagging it. You may want to call `await Laminar.flush()` after the trace
+   * that you want to tag.
+   *
+   * @param {string | StringUUID} trace_id - The trace id to tag.
+   * @param {string[] | string} tags - The tag or list of tags to add to the trace.
+   * @returns {Promise<any>} The response from the server.
+   * @example
+   * ```javascript
+   * import { Laminar, observe, LaminarClient } from "@lmnr-ai/lmnr";
+   * Laminar.initialize();
+   * const client = new LaminarClient();
+   * let traceId: StringUUID | null = null;
+   * // Make sure this is called outside of traced context.
+   * await observe(
+   *   {
+   *     name: "my-trace",
+   *   },
+   *   async () => {
+   *     traceId = await Laminar.getTraceId();
+   *     await foo();
+   *   },
+   * );
+   *
+   * // or make sure the trace is ended by this point.
+   * await Laminar.flush();
+   * if (traceId) {
+   *   await client.tags.tag(traceId, ["tag1", "tag2"]);
+   * }
+   * ```
+   */
+  async tag(trace_id, tags) {
+    const traceTags = Array.isArray(tags) ? tags : [tags];
+    const formattedTraceId = isStringUUID(trace_id) ? trace_id : otelTraceIdToUUID(trace_id);
+    const url = this.baseHttpUrl + "/v1/tag";
+    const payload = {
+      "traceId": formattedTraceId,
+      "names": traceTags
+    };
+    const response = await fetch(
+      url,
+      {
+        method: "POST",
+        headers: this.headers(),
+        body: JSON.stringify(payload)
+      }
+    );
+    if (!response.ok) {
+      await this.handleError(response);
+    }
+    return response.json();
+  }
+};
+
+// src/client/index.ts
+var LaminarClient = class {
+  constructor({
+    baseUrl,
+    projectApiKey,
+    port
+  } = {}) {
+    (0, import_dotenv.config)({
+      quiet: true
+    });
+    this.projectApiKey = projectApiKey ?? process.env.LMNR_PROJECT_API_KEY;
+    const httpPort = port ?? (baseUrl?.match(/:\d{1,5}$/g) ? parseInt(baseUrl.match(/:\d{1,5}$/g)[0].slice(1)) : 443);
+    const baseUrlNoPort = (baseUrl ?? process.env.LMNR_BASE_URL)?.replace(/\/$/, "").replace(/:\d{1,5}$/g, "");
+    this.baseUrl = `${baseUrlNoPort ?? "https://api.lmnr.ai"}:${httpPort}`;
+    this._agent = new AgentResource(this.baseUrl, this.projectApiKey);
+    this._browserEvents = new BrowserEventsResource(this.baseUrl, this.projectApiKey);
+    this._datasets = new DatasetsResource(this.baseUrl, this.projectApiKey);
+    this._evals = new EvalsResource(this.baseUrl, this.projectApiKey);
+    this._evaluators = new EvaluatorsResource(this.baseUrl, this.projectApiKey);
+    this._tags = new TagsResource(this.baseUrl, this.projectApiKey);
+  }
+  get agent() {
+    return this._agent;
+  }
+  get browserEvents() {
+    return this._browserEvents;
+  }
+  get datasets() {
+    return this._datasets;
+  }
+  get evals() {
+    return this._evals;
+  }
+  get evaluators() {
+    return this._evaluators;
+  }
+  get tags() {
+    return this._tags;
+  }
+};
+
+// src/cli/file-utils.ts
+var import_csv_parser = __toESM(require("csv-parser"));
+var import_export_to_csv = require("export-to-csv");
+var import_fs = require("fs");
+var fs = __toESM(require("fs/promises"));
+var path2 = __toESM(require("path"));
+var logger5 = initializeLogger();
+var isSupportedFile = (file) => {
+  const ext = path2.extname(file).toLowerCase();
+  return [".json", ".csv", ".jsonl"].includes(ext);
+};
+var collectFiles = async (paths, recursive = false) => {
+  const collectedFiles = [];
+  for (const filepath of paths) {
+    try {
+      const stats = await fs.stat(filepath);
+      if (stats.isFile()) {
+        if (isSupportedFile(filepath)) {
+          collectedFiles.push(filepath);
+        } else {
+          logger5.warn(`Skipping unsupported file type: ${filepath}`);
+        }
+      } else if (stats.isDirectory()) {
+        const entries = await fs.readdir(filepath);
+        for (const entry of entries) {
+          const fullPath = path2.join(filepath, entry);
+          const entryStats = await fs.stat(fullPath);
+          if (entryStats.isFile() && isSupportedFile(fullPath)) {
+            collectedFiles.push(fullPath);
+          } else if (recursive && entryStats.isDirectory()) {
+            const subFiles = await collectFiles([fullPath], true);
+            collectedFiles.push(...subFiles);
+          }
+        }
+      }
+    } catch (error) {
+      logger5.warn(
+        `Path does not exist or is not accessible: ${filepath}. Error: ${error instanceof Error ? error.message : String(error)}`
+      );
+    }
+  }
+  return collectedFiles;
+};
+var readJsonFile = async (filepath) => {
+  const content = await fs.readFile(filepath, "utf-8");
+  const parsed = JSON.parse(content);
+  return Array.isArray(parsed) ? parsed : [parsed];
+};
+var tryParseJson = (content) => {
+  if (typeof content !== "string") {
+    return content;
+  }
+  const trimmed = content.trim();
+  if (!trimmed.startsWith("{") && !trimmed.startsWith("[")) {
+    return content;
+  }
+  try {
+    return JSON.parse(content);
+  } catch (error) {
+    logger5.debug(
+      `Error parsing JSON: ${error instanceof Error ? error.message : String(error)}`
+    );
+    return content;
+  }
+};
+var parseCsvRow = (row) => {
+  const parsed = {};
+  for (const [key, value] of Object.entries(row)) {
+    parsed[key] = tryParseJson(value);
+  }
+  return parsed;
+};
+var readCsvFile = async (filepath) => new Promise((resolve, reject) => {
+  const results = [];
+  (0, import_fs.createReadStream)(filepath).pipe((0, import_csv_parser.default)()).on("data", (data) => results.push(parseCsvRow(data))).on("end", () => resolve(results)).on("error", reject);
+});
+async function readJsonlFile(filepath) {
+  const content = await fs.readFile(filepath, "utf-8");
+  const lines = content.split("\n").filter((line) => line.trim());
+  return lines.map((line) => JSON.parse(line));
+}
+async function readFile2(filepath) {
+  const ext = path2.extname(filepath).toLowerCase();
+  if (ext === ".json") {
+    return readJsonFile(filepath);
+  } else if (ext === ".csv") {
+    return readCsvFile(filepath);
+  } else if (ext === ".jsonl") {
+    return readJsonlFile(filepath);
+  } else {
+    throw new Error(`Unsupported file type: ${ext}`);
+  }
+}
+var loadFromPaths = async (paths, recursive = false) => {
+  const files = await collectFiles(paths, recursive);
+  if (files.length === 0) {
+    logger5.warn("No supported files found in the specified paths");
+    return [];
+  }
+  logger5.info(`Found ${files.length} file(s) to read`);
+  const result = [];
+  for (const file of files) {
+    try {
+      const data = await readFile2(file);
+      result.push(...data);
+      logger5.info(`Read ${data.length} record(s) from ${file}`);
+    } catch (error) {
+      logger5.error(
+        `Error reading file ${file}: ${error instanceof Error ? error.message : String(error)}`
+      );
+      throw error;
+    }
+  }
+  return result;
+};
+var writeJsonFile = async (filepath, data) => {
+  const content = JSON.stringify(data, null, 2);
+  await fs.writeFile(filepath, content, "utf-8");
+};
+var writeCsvFile = async (filepath, data) => {
+  if (data.length === 0) {
+    throw new Error("No data to write to CSV");
+  }
+  const formattedData = data.map((item) => Object.fromEntries(
+    Object.entries(item).map(([key, value]) => [key, stringifyForCsv(value)])
+  ));
+  const csvConfig = (0, import_export_to_csv.mkConfig)({ useKeysAsHeaders: true });
+  const csvOutput = (0, import_export_to_csv.generateCsv)(csvConfig)(formattedData);
+  const csvString = (0, import_export_to_csv.asString)(csvOutput);
+  await fs.writeFile(filepath, csvString, "utf-8");
+};
+var writeJsonlFile = async (filepath, data) => {
+  const lines = data.map((item) => JSON.stringify(item)).join("\n");
+  await fs.writeFile(filepath, lines + "\n", "utf-8");
+};
+var writeToFile = async (filepath, data, format) => {
+  const dir = path2.dirname(filepath);
+  await fs.mkdir(dir, { recursive: true });
+  const ext = format ?? path2.extname(filepath).slice(1);
+  if (format && format !== path2.extname(filepath).slice(1)) {
+    logger5.warn(
+      `Output format ${format} does not match file extension ${path2.extname(filepath).slice(1)}`
+    );
+  }
+  if (ext === "json") {
+    await writeJsonFile(filepath, data);
+  } else if (ext === "csv") {
+    await writeCsvFile(filepath, data);
+  } else if (ext === "jsonl") {
+    await writeJsonlFile(filepath, data);
+  } else {
+    throw new Error(`Unsupported output format: ${ext}`);
+  }
+};
+var stringifyForCsv = (value) => {
+  if (value === null || value === void 0) {
+    return "";
+  }
+  if (typeof value === "string") {
+    return value;
+  }
+  if (typeof value === "number" || typeof value === "boolean") {
+    return String(value);
+  }
+  return JSON.stringify(value);
+};
+var printToConsole = (data, format = "json") => {
+  if (format === "json") {
+    console.log(JSON.stringify(data, null, 2));
+  } else if (format === "csv") {
+    if (data.length === 0) {
+      logger5.error("No data to print");
+      return;
+    }
+    const formattedData = data.map((item) => Object.fromEntries(
+      Object.entries(item).map(([key, value]) => [key, stringifyForCsv(value)])
+    ));
+    const csvConfig = (0, import_export_to_csv.mkConfig)({ useKeysAsHeaders: true });
+    const csvOutput = (0, import_export_to_csv.generateCsv)(csvConfig)(formattedData);
+    const csvString = (0, import_export_to_csv.asString)(csvOutput);
+    console.log(csvString);
+  } else if (format === "jsonl") {
+    data.forEach((item) => console.log(JSON.stringify(item)));
+  } else {
+    throw new Error(
+      `Unsupported output format: ${String(format)}. (supported formats: json, csv, jsonl)`
+    );
+  }
+};
+
+// src/cli/datasets.ts
+var logger6 = initializeLogger();
+var DEFAULT_DATASET_PULL_BATCH_SIZE = 100;
+var DEFAULT_DATASET_PUSH_BATCH_SIZE2 = 100;
+var pullAllData = async (client, identifier, batchSize = DEFAULT_DATASET_PULL_BATCH_SIZE, offset = 0, limit) => {
+  let hasMore = true;
+  let currentOffset = offset;
+  const stopAt = limit !== void 0 ? offset + limit : void 0;
+  const result = [];
+  while (hasMore && (stopAt === void 0 || currentOffset < stopAt)) {
+    const data = await client.datasets.pull({
+      ...identifier,
+      offset: currentOffset,
+      limit: batchSize
+    });
+    result.push(...data.items);
+    if (data.items.length === 0 || data.items.length < batchSize) {
+      hasMore = false;
+    } else if (stopAt !== void 0 && currentOffset + batchSize >= stopAt) {
+      hasMore = false;
+    } else if (data.totalCount !== void 0 && currentOffset + batchSize >= data.totalCount) {
+      hasMore = false;
+    }
+    currentOffset += batchSize;
+  }
+  if (limit !== void 0) {
+    return result.slice(0, limit);
+  }
+  return result;
+};
+var handleDatasetsList = async (options) => {
+  const client = new LaminarClient({
+    projectApiKey: options.projectApiKey,
+    baseUrl: options.baseUrl,
+    port: options.port
+  });
+  try {
+    const datasets = await client.datasets.listDatasets();
+    if (datasets.length === 0) {
+      console.log("No datasets found.");
+      return;
+    }
+    const idWidth = 36;
+    const createdAtWidth = 19;
+    console.log(`
+${"ID".padEnd(idWidth)}  ${"Created At".padEnd(createdAtWidth)}  Name`);
+    console.log(`${"-".repeat(idWidth)}  ${"-".repeat(createdAtWidth)}  ${"-".repeat(20)}`);
+    for (const dataset of datasets) {
+      const createdAt = new Date(dataset.createdAt);
+      const createdAtStr = createdAt.toISOString().replace("T", " ").substring(0, 19);
+      console.log(
+        `${dataset.id.padEnd(idWidth)}  ${createdAtStr.padEnd(createdAtWidth)}  ${dataset.name}`
+      );
+    }
+    console.log(`
+Total: ${datasets.length} dataset(s)
+`);
+  } catch (error) {
+    logger6.error(
+      `Failed to list datasets: ${error instanceof Error ? error.message : String(error)}`
+    );
+  }
+};
+var handleDatasetsPush = async (paths, options) => {
+  if (!options.name && !options.id) {
+    logger6.error("Either name or id must be provided");
+    return;
+  }
+  if (options.name && options.id) {
+    logger6.error("Only one of name or id must be provided");
+    return;
+  }
+  const client = new LaminarClient({
+    projectApiKey: options.projectApiKey,
+    baseUrl: options.baseUrl,
+    port: options.port
+  });
+  const data = await loadFromPaths(paths, options.recursive);
+  if (data.length === 0) {
+    logger6.warn("No data to push. Skipping");
+    return;
+  }
+  const identifier = options.name ? { name: options.name } : { id: options.id };
+  try {
+    await client.datasets.push({
+      points: data,
+      ...identifier,
+      batchSize: options.batchSize ?? DEFAULT_DATASET_PUSH_BATCH_SIZE2
+    });
+    logger6.info(`Pushed ${data.length} data points to dataset ${options.name || options.id}`);
+  } catch (error) {
+    logger6.error(
+      `Failed to push dataset: ${error instanceof Error ? error.message : String(error)}`
+    );
+  }
+};
+var handleDatasetsPull = async (outputPath, options) => {
+  if (!options.name && !options.id) {
+    logger6.error("Either name or id must be provided");
+    return;
+  }
+  if (options.name && options.id) {
+    logger6.error("Only one of name or id must be provided");
+    return;
+  }
+  const client = new LaminarClient({
+    projectApiKey: options.projectApiKey,
+    baseUrl: options.baseUrl,
+    port: options.port
+  });
+  const identifier = options.name ? { name: options.name } : { id: options.id };
+  try {
+    const result = await pullAllData(
+      client,
+      identifier,
+      options.batchSize ?? DEFAULT_DATASET_PULL_BATCH_SIZE,
+      options.offset ?? 0,
+      options.limit
+    );
+    if (outputPath) {
+      await writeToFile(outputPath, result, options.outputFormat);
+      logger6.info(`Successfully pulled ${result.length} data points to ${outputPath}`);
+    } else {
+      printToConsole(result, options.outputFormat ?? "json");
+    }
+  } catch (error) {
+    logger6.error(
+      `Failed to pull dataset: ${error instanceof Error ? error.message : String(error)}`
+    );
+  }
+};
+var handleDatasetsCreate = async (name, paths, options) => {
+  const client = new LaminarClient({
+    projectApiKey: options.projectApiKey,
+    baseUrl: options.baseUrl,
+    port: options.port
+  });
+  const data = await loadFromPaths(paths, options.recursive);
+  if (data.length === 0) {
+    logger6.warn("No data to push. Skipping");
+    return;
+  }
+  logger6.info(`Pushing ${data.length} data points to dataset '${name}'...`);
+  try {
+    await client.datasets.push({
+      points: data,
+      name,
+      batchSize: options.batchSize ?? DEFAULT_DATASET_PUSH_BATCH_SIZE2,
+      createDataset: true
+    });
+    logger6.info(`Successfully pushed ${data.length} data points to dataset '${name}'`);
+  } catch (error) {
+    logger6.error(
+      `Failed to create dataset: ${error instanceof Error ? error.message : String(error)}`
+    );
+    return;
+  }
+  logger6.info(`Pulling data from dataset '${name}'...`);
+  try {
+    const result = await pullAllData(
+      client,
+      { name },
+      options.batchSize ?? DEFAULT_DATASET_PULL_BATCH_SIZE,
+      0,
+      void 0
+    );
+    await writeToFile(options.outputFile, result, options.outputFormat);
+    logger6.info(
+      `Successfully created dataset '${name}' and saved ${result.length} datapoints to ${options.outputFile}`
+    );
+  } catch (error) {
+    logger6.error(
+      `Failed to pull dataset after creation: ${error instanceof Error ? error.message : String(error)}`
+    );
+  }
+};
+
+// src/cli/evals.ts
+var esbuild = __toESM(require("esbuild"));
+var fs2 = __toESM(require("fs"));
+var glob = __toESM(require("glob"));
+var logger7 = initializeLogger();
 var createSkipDynamicImportsPlugin = (skipModules) => ({
   name: "skip-dynamic-imports",
   setup(build2) {
     if (!skipModules || skipModules.length === 0) return;
     build2.onResolve({ filter: /.*/ }, (args) => {
       if (args.kind === "dynamic-import" && skipModules.includes(args.path)) {
-        logger2.warn(`Skipping dynamic import: ${args.path}`);
+        logger7.warn(`Skipping dynamic import: ${args.path}`);
         return {
           path: args.path,
           namespace: "lmnr-skip-dynamic-import"
@@ -138,6 +1434,85 @@ function loadModule({
   );
   return globalThis._evaluations;
 }
+async function runEvaluation(files, options) {
+  let evalFiles;
+  if (files && files.length > 0) {
+    evalFiles = files.flatMap((file) => glob.sync(file));
+  } else {
+    evalFiles = glob.sync("evals/**/*.eval.{ts,js}");
+  }
+  evalFiles.sort();
+  if (evalFiles.length === 0) {
+    logger7.error("No evaluation files found. Please provide a file or ensure there are eval files that are named like `*.eval.{ts,js}` in the `evals` directory or its subdirectories.");
+    process.exit(1);
+  }
+  if (files.length === 0) {
+    logger7.info(`Located ${evalFiles.length} evaluation files in evals/`);
+  } else {
+    logger7.info(`Running ${evalFiles.length} evaluation files.`);
+  }
+  const scores = [];
+  for (const file of evalFiles) {
+    logger7.info(`Loading ${file}...`);
+    const buildOptions = {
+      bundle: true,
+      platform: "node",
+      entryPoints: [file],
+      outfile: `tmp_out_${file}.js`,
+      write: false,
+      // will be loaded in memory as a temp file
+      external: [
+        "node_modules/*",
+        "playwright",
+        "puppeteer",
+        "puppeteer-core",
+        "playwright-core",
+        "fsevents",
+        ...options.externalPackages ? options.externalPackages : []
+      ],
+      plugins: [
+        createSkipDynamicImportsPlugin(options.dynamicImportsToSkip || [])
+      ],
+      treeShaking: true
+    };
+    const result = await esbuild.build(buildOptions);
+    if (!result.outputFiles) {
+      logger7.error("Error when building: No output files found it is likely that all eval files are not valid TypeScript or JavaScript files.");
+      if (options.failOnError) {
+        process.exit(1);
+      }
+      continue;
+    }
+    const outputFileText = result.outputFiles[0].text;
+    const evaluations = loadModule({
+      filename: file,
+      moduleText: outputFileText
+    });
+    logger7.info(`Loaded ${evaluations.length} evaluations from ${file}`);
+    for (const evaluation of evaluations) {
+      if (!evaluation?.run) {
+        logger7.error(`Evaluation ${file} does not properly call evaluate()`);
+        if (options.failOnError) {
+          process.exit(1);
+        }
+        continue;
+      }
+      const evalResult = await evaluation.run();
+      scores.push({
+        file,
+        scores: evalResult?.averageScores ?? {},
+        url: evalResult?.url ?? "",
+        evaluationId: evalResult?.evaluationId ?? ""
+      });
+    }
+  }
+  if (options.outputFile) {
+    fs2.writeFileSync(options.outputFile, JSON.stringify(scores, null, 2));
+  }
+}
+
+// src/cli/index.ts
+var logger8 = initializeLogger();
 async function cli() {
   const program = new import_commander.Command();
   program.name("lmnr").description("CLI for Laminar. Use `lmnr <subcommand> --help` for more information.").version(version, "-v, --version", "display version number");
@@ -157,75 +1532,55 @@ async function cli() {
     "--dynamic-imports-to-skip <modules...>",
     "[ADVANCED] List of module names to skip when encountered as dynamic imports. These dynamic imports will resolve to an empty module to prevent build failures. This is meant to skip the imports that are not used in the evaluation itself."
   ).action(async (files, options) => {
-    let evalFiles;
-    if (files && files.length > 0) {
-      evalFiles = files.flatMap((file) => glob.sync(file));
-    } else {
-      evalFiles = glob.sync("evals/**/*.eval.{ts,js}");
-    }
-    evalFiles.sort();
-    if (evalFiles.length === 0) {
-      logger2.error("No evaluation files found. Please provide a file or ensure there are eval files that are named like `*.eval.{ts,js}` in the `evals` directory or its subdirectories.");
-      process.exit(1);
-    }
-    if (files.length === 0) {
-      logger2.info(`Located ${evalFiles.length} evaluation files in evals/`);
-    } else {
-      logger2.info(`Running ${evalFiles.length} evaluation files.`);
-    }
-    const scores = [];
-    for (const file of evalFiles) {
-      logger2.info(`Loading ${file}...`);
-      const buildOptions = {
-        bundle: true,
-        platform: "node",
-        entryPoints: [file],
-        outfile: `tmp_out_${file}.js`,
-        write: false,
-        // will be loaded in memory as a temp file
-        external: [
-          "node_modules/*",
-          "playwright",
-          "puppeteer",
-          "puppeteer-core",
-          "playwright-core",
-          "fsevents",
-          ...options.externalPackages ? options.externalPackages : []
-        ],
-        plugins: [
-          createSkipDynamicImportsPlugin(options.dynamicImportsToSkip || [])
-        ],
-        treeShaking: true
-      };
-      const result = await esbuild.build(buildOptions);
-      if (!result.outputFiles) {
-        logger2.error("Error when building: No output files found it is likely that all eval files are not valid TypeScript or JavaScript files.");
-        if (options.failOnError) {
-          process.exit(1);
-        }
-        continue;
-      }
-      const outputFileText = result.outputFiles[0].text;
-      const evaluations = loadModule({
-        filename: file,
-        moduleText: outputFileText
-      });
-      logger2.info(`Loaded ${evaluations.length} evaluations from ${file}`);
-      for (const evaluation of evaluations) {
-        if (!evaluation?.run) {
-          logger2.error(`Evaluation ${file} does not properly call evaluate()`);
-          if (options.failOnError) {
-            process.exit(1);
-          }
-          continue;
-        }
-        const evalScores = await evaluation.run();
-        scores.push({ file, scores: evalScores });
-      }
-    }
-    if (options.outputFile) {
-      fs.writeFileSync(options.outputFile, JSON.stringify(scores, null, 2));
-    }
+    await runEvaluation(files, options);
+  });
+  const datasetsCmd = program.command("datasets").description("Manage datasets").option(
+    "--project-api-key <key>",
+    "Project API key. If not provided, reads from LMNR_PROJECT_API_KEY env variable"
+  ).option(
+    "--base-url <url>",
+    "Base URL for the Laminar API. Defaults to https://api.lmnr.ai or LMNR_BASE_URL env variable"
+  ).option(
+    "--port <port>",
+    "Port for the Laminar API. Defaults to 443",
+    (val) => parseInt(val, 10)
+  );
+  datasetsCmd.command("list").description("List all datasets").action(async (options, cmd) => {
+    const parentOpts = cmd.parent?.opts() || {};
+    await handleDatasetsList({ ...parentOpts, ...options });
+  });
+  datasetsCmd.command("push").description("Push datapoints to an existing dataset").argument("<paths...>", "Paths to files or directories containing data to push").option("-n, --name <name>", "Name of the dataset (either name or id must be provided)").option("--id <id>", "ID of the dataset (either name or id must be provided)").option("-r, --recursive", "Recursively read files in directories", false).option(
+    "--batch-size <size>",
+    "Batch size for pushing data",
+    (val) => parseInt(val, 10),
+    100
+  ).action(async (paths, options, cmd) => {
+    const parentOpts = cmd.parent?.opts() || {};
+    await handleDatasetsPush(paths, { ...parentOpts, ...options });
+  });
+  datasetsCmd.command("pull").description("Pull data from a dataset").argument("[output-path]", "Path to save the data. If not provided, prints to console").option("-n, --name <name>", "Name of the dataset (either name or id must be provided)").option("--id <id>", "ID of the dataset (either name or id must be provided)").option(
+    "--output-format <format>",
+    "Output format (json, csv, jsonl). Inferred from file extension if not provided"
+  ).option(
+    "--batch-size <size>",
+    "Batch size for pulling data",
+    (val) => parseInt(val, 10),
+    100
+  ).option("--limit <limit>", "Limit number of datapoints to pull", (val) => parseInt(val, 10)).option("--offset <offset>", "Offset for pagination", (val) => parseInt(val, 10), 0).action(async (outputPath, options, cmd) => {
+    const parentOpts = cmd.parent?.opts() || {};
+    await handleDatasetsPull(outputPath, { ...parentOpts, ...options });
+  });
+  datasetsCmd.command("create").description("Create a dataset from input files").argument("<name>", "Name of the dataset to create").argument("<paths...>", "Paths to files or directories containing data to push").requiredOption("-o, --output-file <file>", "Path to save the pulled data").option(
+    "--output-format <format>",
+    "Output format (json, csv, jsonl). Inferred from file extension if not provided"
+  ).option("-r, --recursive", "Recursively read files in directories", false).option(
+    "--batch-size <size>",
+    "Batch size for pushing/pulling data",
+    (val) => parseInt(val, 10),
+    100
+  ).action(async (name, paths, options, cmd) => {
+    const parentOpts = cmd.parent?.opts() || {};
+    await handleDatasetsCreate(name, paths, { ...parentOpts, ...options });
   });
   if (!process.argv.slice(2).length) {
     program.help();
@@ -234,11 +1589,7 @@ async function cli() {
   await program.parseAsync();
 }
 cli().catch((err) => {
-  logger2.error(err instanceof Error ? err.message : err);
+  logger8.error(err instanceof Error ? err.message : err);
   throw err;
 });
-// Annotate the CommonJS export names for ESM import in node:
-0 && (module.exports = {
-  loadModule
-});
 //# sourceMappingURL=cli.js.map