@lmnr-ai/lmnr 0.4.5 → 0.4.7-alpha

package/src/laminar.ts

@@ -1,8 +1,10 @@
 import { PipelineRunResponse, PipelineRunRequest, EvaluationDatapoint, EvaluationStatus } from './types';
 import { Attributes, AttributeValue, context, createContextKey, isSpanContextValid, TimeInput, trace } from '@opentelemetry/api';
-import { InitializeOptions, initialize as traceloopInitialize } from '@traceloop/node-server-sdk'
+import { InitializeOptions, initialize as traceloopInitialize } from './sdk/node-server-sdk'
 import { otelSpanIdToUUID, otelTraceIdToUUID } from './utils';
-
+import { OTLPTraceExporter } from '@opentelemetry/exporter-trace-otlp-grpc';
+import { Metadata } from '@grpc/grpc-js';
+import { ASSOCIATION_PROPERTIES_KEY } from './sdk/tracing/tracing';
 
 // quick patch to get the traceloop's default tracer, since their 
 // `getTracer` function is not exported.
@@ -19,7 +21,7 @@ interface LaminarInitializeProps {
 }
 
 export class Laminar {
-    private static baseUrl: string = 'https://api.lmnr.ai';
+    private static baseUrl: string = 'https://api.lmnr.ai:8443';
     private static projectApiKey: string;
     private static env: Record<string, string> = {};
     private static isInitialized: boolean = false;
@@ -34,8 +36,28 @@ export class Laminar {
      * @param env - Default environment passed to `run` and `evaluateEvent` requests,
      * unless overriden at request time. Usually, model provider keys are stored here.
      * @param baseUrl - Url of Laminar endpoint, or the custom open telemetry ingester.
-     * If not specified, defaults to https://api.lmnr.ai. For locally hosted Laminar,
-     * default setting must be http://localhost:8000.
+     * If not specified, defaults to https://api.lmnr.ai:8443. For locally hosted Laminar,
+     * default setting must be http://localhost:8001.
+     * @param instrumentModules - List of modules to instrument.
+     * If not specified, all auto-instrumentable modules will be instrumented, which include
+     * LLM calls (OpenAI, Anthropic, etc), Langchain, VectorDB calls (Pinecone, Qdrant, etc).
+     * Pass an empty object {} to disable any kind of automatic instrumentation.
+     * If you only want to auto-instrument specific modules, then pass them in the object.
+     * 
+     * Example:
+     * ```typescript
+     * import { Laminar as L } from '@lmnr-ai/lmnr';
+     * import { OpenAI } from 'openai';
+     * import * as ChainsModule from "langchain/chains";
+     * 
+     * // Initialize Laminar while auto-instrumenting Langchain and OpenAI modules.
+     * L.initialize({ projectApiKey: "<LMNR_PROJECT_API_KEY>", instrumentModules: {
+     *  langchain: {
+     *      chainsModule: ChainsModule
+     *  },
+     *  openAI: OpenAI
+     * } });
+     * ```
      *
      * @throws {Error} - If project API key is not set
      */
@@ -59,11 +81,19 @@ export class Laminar {
         }
         this.isInitialized = true;
         this.env = env ?? {};
+
+        const metadata = new Metadata();
+        metadata.set('authorization', `Bearer ${this.projectApiKey}`);
+        const exporter = new OTLPTraceExporter({
+            url: this.baseUrl,
+            metadata,
+        });
+
         traceloopInitialize({
-            apiKey: this.projectApiKey,
-            baseUrl: this.baseUrl,
+            exporter,
             silenceInitializationMessage: true,
             instrumentModules,
+            disableBatch: false,
         });
     }
 
@@ -267,7 +297,17 @@ export class Laminar {
         if (userId) {
             associationProperties = { ...associationProperties, "user_id": userId };
         }
-        return context.active().setValue(createContextKey("association_properites"), associationProperties);
+
+        let entityContext = context.active();
+        const currentAssociationProperties = entityContext.getValue(ASSOCIATION_PROPERTIES_KEY);
+        if (associationProperties) {
+            entityContext = entityContext.setValue(
+                ASSOCIATION_PROPERTIES_KEY,
+                { ...(currentAssociationProperties ?? {}), ...associationProperties },
+            );
+        }
+
+        return entityContext;
     }
 
     public static async createEvaluation(name: string) {

package/src/sdk/configuration/index.ts

@@ -0,0 +1,68 @@
+import { InitializeOptions } from "../interfaces";
+import { startTracing } from "../tracing";
+import { diag, DiagConsoleLogger, DiagLogLevel } from "@opentelemetry/api";
+import { InitializationError } from "../errors";
+
+export let _configuration: InitializeOptions | undefined;
+
+/**
+ * Initializes the SDK.
+ * Must be called once before any other SDK methods.
+ *
+ * @param options - The options to initialize the SDK. See the {@link InitializeOptions} for details.
+ * @throws {InitializationError} if the configuration is invalid or if failed to fetch feature data.
+ */
+export const initialize = (options: InitializeOptions) => {
+  if (_configuration) {
+    return;
+  }
+
+  if (!options.baseUrl) {
+    options.baseUrl =
+      process.env.LMNR_BASE_URL || "https://api.lmnr.ai:8443";
+  }
+  if (!options.apiKey) {
+    options.apiKey = process.env.LMNR_PROJECT_API_KEY;
+  }
+  if (options.apiKey && typeof options.apiKey !== "string") {
+    throw new InitializationError('"apiKey" must be a string');
+  }
+
+  if (!options.appName) {
+    options.appName = process.env.npm_package_name;
+  }
+
+  _configuration = Object.freeze(options);
+
+  if (options.logLevel) {
+    diag.setLogger(
+      new DiagConsoleLogger(),
+      logLevelToOtelLogLevel(options.logLevel),
+    );
+  }
+
+  if (!options.silenceInitializationMessage) {
+    console.log(
+      `Laminar exporting traces to ${
+        _configuration.exporter ? "a custom exporter" : _configuration.baseUrl
+      }`,
+    );
+  }
+
+  startTracing(_configuration);
+};
+
+const logLevelToOtelLogLevel = (
+  logLevel: "debug" | "info" | "warn" | "error",
+) => {
+  switch (logLevel) {
+    case "debug":
+      return DiagLogLevel.DEBUG;
+    case "info":
+      return DiagLogLevel.INFO;
+    case "warn":
+      return DiagLogLevel.WARN;
+    case "error":
+      return DiagLogLevel.ERROR;
+  }
+};

package/src/sdk/errors/index.ts

@@ -0,0 +1,57 @@
+/**
+ * The severity of an error.
+ */
+export const SEVERITY = {
+  Warning: "Warning",
+  Error: "Error",
+  Critical: "Critical",
+} as const;
+
+export type Severity = (typeof SEVERITY)[keyof typeof SEVERITY];
+
+/**
+ * Base class for all Traceloop errors.
+ */
+export class TraceloopError extends Error {
+  /**
+   * The severity of the error.
+   */
+  severity: Severity;
+  /**
+   * The underlying cause of the error.
+   */
+  underlyingCause?: Error;
+
+  constructor(message: string, severity: Severity = SEVERITY.Error) {
+    super(message);
+    this.severity = severity;
+  }
+}
+
+export class NotInitializedError extends TraceloopError {
+  constructor() {
+    super(
+      `The Traceloop SDK must be initialized by calling the "initialize" function prior to use.`,
+      SEVERITY.Critical,
+    );
+  }
+}
+
+export class InitializationError extends TraceloopError {
+  constructor(message?: string, cause?: Error) {
+    super(message ?? "Failed to initialize Traceloop SDK", SEVERITY.Critical);
+    this.underlyingCause = cause;
+  }
+}
+
+export class ArgumentNotProvidedError extends TraceloopError {
+  constructor(argumentName: string) {
+    super(`The "${argumentName}" argument is required and must be a string.`);
+  }
+}
+
+export class PromptNotFoundError extends TraceloopError {
+  constructor(key: string) {
+    super(`The prompt "${key}" was not found in the registry.`);
+  }
+}

package/src/sdk/interfaces/index.ts

@@ -0,0 +1 @@
+export * from "./initialize-options.interface";

package/src/sdk/interfaces/initialize-options.interface.ts

@@ -0,0 +1,114 @@
+import { SpanExporter, SpanProcessor } from "@opentelemetry/sdk-trace-base";
+import { TextMapPropagator, ContextManager } from "@opentelemetry/api";
+import type * as openai from "openai";
+import type * as anthropic from "@anthropic-ai/sdk";
+import type * as azure from "@azure/openai";
+import type * as cohere from "cohere-ai";
+import type * as bedrock from "@aws-sdk/client-bedrock-runtime";
+import type * as aiplatform from "@google-cloud/aiplatform";
+import type * as vertexAI from "@google-cloud/vertexai";
+import type * as pinecone from "@pinecone-database/pinecone";
+import type * as ChainsModule from "langchain/chains";
+import type * as AgentsModule from "langchain/agents";
+import type * as ToolsModule from "langchain/tools";
+import type * as RunnableModule from "@langchain/core/runnables";
+import type * as VectorStoreModule from "@langchain/core/vectorstores";
+import type * as llamaindex from "llamaindex";
+import type * as chromadb from "chromadb";
+import type * as qdrant from "@qdrant/js-client-rest";
+
+/**
+ * Options for initializing the Traceloop SDK.
+ */
+export interface InitializeOptions {
+  /**
+   * The app name to be used when reporting traces. Optional.
+   * Defaults to the package name.
+   */
+  appName?: string;
+
+  /**
+   * The API Key for sending traces data. Optional.
+   * Defaults to the TRACELOOP_API_KEY environment variable.
+   */
+  apiKey?: string;
+
+  /**
+   * The OTLP endpoint for sending traces data. Optional.
+   * Defaults to TRACELOOP_BASE_URL environment variable or https://api.traceloop.com/
+   */
+  baseUrl?: string;
+
+  /**
+   * Sends traces and spans without batching, for local developement. Optional.
+   * Defaults to false.
+   */
+  disableBatch?: boolean;
+
+  /**
+   * Defines default log level for SDK and all instrumentations. Optional.
+   * Defaults to error.
+   */
+  logLevel?: "debug" | "info" | "warn" | "error";
+
+  /**
+   * Whether to log prompts, completions and embeddings on traces. Optional.
+   * Defaults to true.
+   */
+  traceContent?: boolean;
+
+  /**
+   * The OpenTelemetry SpanExporter to be used for sending traces data. Optional.
+   * Defaults to the OTLP exporter.
+   */
+  exporter?: SpanExporter;
+
+  /**
+   * The OpenTelemetry SpanProcessor to be used for processing traces data. Optional.
+   * Defaults to the BatchSpanProcessor.
+   */
+  processor?: SpanProcessor;
+
+  /**
+   * The OpenTelemetry Propagator to use. Optional.
+   * Defaults to OpenTelemetry SDK defaults.
+   */
+  propagator?: TextMapPropagator;
+
+  /**
+   * The OpenTelemetry ContextManager to use. Optional.
+   * Defaults to OpenTelemetry SDK defaults.
+   */
+  contextManager?: ContextManager;
+
+  /**
+   * Explicitly specify modules to instrument. Optional.
+   * This is a workaround specific to Next.js, see https://www.traceloop.com/docs/openllmetry/getting-started-nextjs
+   */
+  instrumentModules?: {
+    openAI?: typeof openai.OpenAI;
+    anthropic?: typeof anthropic;
+    azureOpenAI?: typeof azure;
+    cohere?: typeof cohere;
+    bedrock?: typeof bedrock;
+    google_vertexai?: typeof vertexAI;
+    google_aiplatform?: typeof aiplatform;
+    pinecone?: typeof pinecone;
+    langchain?: {
+      chainsModule?: typeof ChainsModule;
+      agentsModule?: typeof AgentsModule;
+      toolsModule?: typeof ToolsModule;
+      runnablesModule?: typeof RunnableModule;
+      vectorStoreModule?: typeof VectorStoreModule;
+    };
+    llamaIndex?: typeof llamaindex;
+    chromadb?: typeof chromadb;
+    qdrant?: typeof qdrant;
+  };
+
+  /**
+   * Whether to silence the initialization message. Optional.
+   * Defaults to false.
+   */
+  silenceInitializationMessage?: boolean;
+}

package/src/sdk/node-server-sdk.ts

@@ -0,0 +1,7 @@
+export * from "./errors";
+export { InitializeOptions } from "./interfaces";
+export { initialize } from "./configuration";
+export { forceFlush } from "./tracing";
+export * from "./tracing/decorators";
+export * from "./tracing/manual";
+export * from "./tracing/association";

package/src/sdk/telemetry/telemetry.ts

@@ -0,0 +1,86 @@
+import * as os from "os";
+import * as fs from "fs";
+import * as path from "path";
+import { v4 as uuid } from "uuid";
+import { PostHog } from "posthog-node";
+const { version } = require('../../../package.json');
+
+export class Telemetry {
+  private static instance: Telemetry;
+
+  private static readonly ANON_ID_PATH = `${os.homedir()}/.cache/lmnr/telemetry_anon_id`;
+  private static readonly UNKNOWN_ANON_ID = "UNKNOWN";
+
+  private telemetryEnabled: boolean;
+  private posthog: PostHog | undefined;
+  private anonId: string | undefined;
+
+  public static getInstance(): Telemetry {
+    if (!Telemetry.instance) {
+      Telemetry.instance = new Telemetry();
+    }
+    return Telemetry.instance;
+  }
+
+  private constructor() {
+    this.telemetryEnabled =
+      !process.env.LMNR_TELEMETRY ||
+      process.env.LMNR_TELEMETRY.toLowerCase() === "true";
+
+    if (this.telemetryEnabled) {
+      this.posthog = new PostHog(
+        "phc_dUMdjfNKf11jcHgtn7juSnT4P1pO0tafsPUWt4PuwG7",
+      );
+    }
+  }
+
+  private getAnonId() {
+    if (this.anonId) {
+      return this.anonId;
+    }
+
+    try {
+      if (!fs.existsSync(Telemetry.ANON_ID_PATH)) {
+        fs.mkdirSync(path.dirname(Telemetry.ANON_ID_PATH), { recursive: true });
+        const anonIdFile = fs.openSync(Telemetry.ANON_ID_PATH, "w");
+        this.anonId = uuid();
+        fs.writeSync(anonIdFile, this.anonId);
+        fs.closeSync(anonIdFile);
+      } else {
+        const anonIdFile = fs.openSync(Telemetry.ANON_ID_PATH, "r");
+        this.anonId = fs.readFileSync(anonIdFile, "utf8");
+        fs.closeSync(anonIdFile);
+      }
+      return this.anonId;
+    } catch (e) {
+      return Telemetry.UNKNOWN_ANON_ID;
+    }
+  }
+
+  private getContext() {
+    return {
+      sdk: "typescript",
+      sdk_version: version,
+    };
+  }
+
+  public capture(event: string, properties?: Record<string, string>) {
+    if (this.telemetryEnabled && this.posthog) {
+      this.posthog.capture({
+        distinctId: this.getAnonId(),
+        event,
+        properties: {
+          ...properties,
+          ...this.getContext(),
+        },
+      });
+      this.posthog.flush();
+    }
+  }
+
+  public logException(error: Error) {
+    if (this.telemetryEnabled) {
+      this.capture("error", { error: error.message, stack: error.stack || "" });
+    }
+  }
+}

package/src/sdk/tracing/association.ts

@@ -0,0 +1,21 @@
+import { context } from "@opentelemetry/api";
+import { ASSOCIATION_PROPERTIES_KEY } from "./tracing";
+
+export function withAssociationProperties<
+  A extends unknown[],
+  F extends (...args: A) => ReturnType<F>,
+>(
+  properties: { [name: string]: string },
+  fn: F,
+  thisArg?: ThisParameterType<F>,
+  ...args: A
+) {
+  if (Object.keys(properties).length === 0) {
+    return fn.apply(thisArg, args);
+  }
+
+  const newContext = context
+    .active()
+    .setValue(ASSOCIATION_PROPERTIES_KEY, properties);
+  return context.with(newContext, fn, thisArg, ...args);
+}

package/src/sdk/tracing/decorators.ts

@@ -0,0 +1,140 @@
+import { Span, context } from "@opentelemetry/api";
+import { suppressTracing } from "@opentelemetry/core";
+import {
+  ASSOCIATION_PROPERTIES_KEY,
+  getTracer,
+} from "./tracing";
+import {
+  SpanAttributes,
+} from "@traceloop/ai-semantic-conventions";
+import { shouldSendTraces } from ".";
+import { Telemetry } from "../telemetry/telemetry";
+
+export type DecoratorConfig = {
+  name: string;
+  associationProperties?: { [name: string]: string };
+  inputParameters?: unknown[];
+  suppressTracing?: boolean;
+};
+
+export function withEntity<
+  A extends unknown[],
+  F extends (...args: A) => ReturnType<F>,
+>(
+  {
+    name,
+    associationProperties,
+    inputParameters,
+    suppressTracing: shouldSuppressTracing,
+  }: DecoratorConfig,
+  fn: F,
+  thisArg?: ThisParameterType<F>,
+  ...args: A
+) {
+  let entityContext = context.active();
+
+  const currentAssociationProperties = entityContext.getValue(ASSOCIATION_PROPERTIES_KEY);
+
+  if (associationProperties) {
+    entityContext = entityContext.setValue(
+      ASSOCIATION_PROPERTIES_KEY,
+      { ...(currentAssociationProperties ?? {}), ...associationProperties },
+    );
+  }
+
+  if (shouldSuppressTracing) {
+    entityContext = suppressTracing(entityContext);
+  }
+
+  return context.with(entityContext, () =>
+    getTracer().startActiveSpan(
+      name,
+      {},
+      entityContext,
+      async (span: Span) => {
+        if (shouldSendTraces()) {
+          try {
+            const input = inputParameters ?? args;
+            if (
+              input.length === 1 &&
+              typeof input[0] === "object" &&
+              !(input[0] instanceof Map)
+            ) {
+              span.setAttribute(
+                SpanAttributes.TRACELOOP_ENTITY_INPUT,
+                serialize({ args: [], kwargs: input[0] }),
+              );
+            } else {
+              span.setAttribute(
+                SpanAttributes.TRACELOOP_ENTITY_INPUT,
+                serialize({
+                  args: input,
+                  kwargs: {},
+                }),
+              );
+            }
+          } catch (error) {
+            Telemetry.getInstance().logException(error as any);
+          }
+        }
+
+        const res = fn.apply(thisArg, args);
+        if (res instanceof Promise) {
+          return res.then((resolvedRes) => {
+            try {
+              if (shouldSendTraces()) {
+                span.setAttribute(
+                  SpanAttributes.TRACELOOP_ENTITY_OUTPUT,
+                  serialize(resolvedRes),
+                );
+              }
+            } catch (error) {
+              Telemetry.getInstance().logException(error as any);
+            } finally {
+              span.end();
+            }
+
+            return resolvedRes;
+          });
+        }
+        try {
+          if (shouldSendTraces()) {
+            span.setAttribute(
+              SpanAttributes.TRACELOOP_ENTITY_OUTPUT,
+              serialize(res),
+            );
+          }
+        } catch (error) {
+          Telemetry.getInstance().logException(error as any);
+        } finally {
+          span.end();
+        }
+
+        return res;
+      },
+    ),
+  );
+}
+
+function cleanInput(input: unknown): unknown {
+  if (input instanceof Map) {
+    return Array.from(input.entries());
+  } else if (Array.isArray(input)) {
+    return input.map((value) => cleanInput(value));
+  } else if (!input) {
+    return input;
+  } else if (typeof input === "object") {
+    // serialize object one by one
+    const output: any = {};
+    Object.entries(input as any).forEach(([key, value]) => {
+      output[key] = cleanInput(value);
+    });
+    return output;
+  }
+
+  return input;
+}
+
+function serialize(input: unknown): string {
+  return JSON.stringify(cleanInput(input));
+}