@garrix82/reactgenie-dsl 1.0.1 → 1.0.2

package/lib/nl/llm-monitoring.test.ts

@@ -1,136 +0,0 @@
-const mockCreateRun = jest.fn();
-const mockTraceable = jest.fn((fn) => fn);
-const mockGetCurrentRunTree = jest.fn();
-
-jest.mock("langsmith", () => ({
-  Client: jest.fn().mockImplementation(() => ({
-    createRun: mockCreateRun,
-  })),
-}));
-
-jest.mock("langsmith/traceable", () => ({
-  traceable: mockTraceable,
-  getCurrentRunTree: mockGetCurrentRunTree,
-}));
-
-function loadMonitor() {
-  return require("./llm-monitoring") as typeof import("./llm-monitoring");
-}
-
-describe("LLMMonitor tracing", () => {
-  beforeEach(() => {
-    jest.clearAllMocks();
-    jest.resetModules();
-  });
-
-  it("wraps traced calls with langsmith traceable chain runs", async () => {
-    const { getLLMMonitor } = loadMonitor();
-    const monitor = getLLMMonitor({
-      apiKey: "ls-test-key",
-      project: "reactgenie-dsl",
-    });
-
-    await monitor.traceCall(
-      "parse-command",
-      { model: "gpt-4o-mini", provider: "openai" },
-      async () => "ok"
-    );
-
-    expect(mockTraceable).toHaveBeenCalledWith(
-      expect.any(Function),
-      expect.objectContaining({
-        name: "parse-command",
-        run_type: "chain",
-        project_name: "reactgenie-dsl",
-      })
-    );
-  });
-
-  it("logs llm child runs with token and cost metadata", async () => {
-    const childRun = {
-      postRun: jest.fn().mockResolvedValue(undefined),
-      end: jest.fn().mockResolvedValue(undefined),
-      patchRun: jest.fn().mockResolvedValue(undefined),
-    };
-    const createChild = jest.fn().mockReturnValue(childRun);
-    mockGetCurrentRunTree.mockReturnValue({ createChild });
-
-    const { getLLMMonitor } = loadMonitor();
-    const monitor = getLLMMonitor({
-      apiKey: "ls-test-key",
-      project: "reactgenie-dsl",
-    });
-
-    await monitor.logCall({
-      model: "gpt-4o-mini",
-      provider: "openai",
-      promptTokens: 12,
-      completionTokens: 4,
-      totalTokens: 16,
-      inputCost: 0.00003,
-      outputCost: 0.00008,
-      totalCost: 0.00011,
-      prompt: "hello",
-      completion: "world",
-      latency: 42,
-      timestamp: new Date("2026-03-11T10:00:00.000Z"),
-      error: "upstream failed",
-    });
-
-    expect(createChild).toHaveBeenCalledWith(
-      expect.objectContaining({
-        run_type: "llm",
-        inputs: { prompt: "hello" },
-      })
-    );
-    expect(childRun.end).toHaveBeenCalledWith(
-      expect.objectContaining({
-        completion: "world",
-        usage_metadata: expect.objectContaining({
-          input_tokens: 12,
-          output_tokens: 4,
-          total_tokens: 16,
-          input_cost: 0.00003,
-          output_cost: 0.00008,
-          total_cost: 0.00011,
-        }),
-      }),
-      "upstream failed",
-      expect.any(Number)
-    );
-    expect(childRun.patchRun).toHaveBeenCalled();
-  });
-
-  it("does not emit synthetic failed llm runs from traceCall failures", async () => {
-    const captureException = jest.fn();
-    const { getLLMMonitor } = loadMonitor();
-    const monitor = getLLMMonitor({
-      apiKey: "ls-test-key",
-      project: "reactgenie-dsl",
-      telemetryBridge: {
-        captureException,
-      },
-    });
-
-    await expect(
-      monitor.traceCall(
-        "parse-command",
-        { model: "gpt-4o-mini", provider: "openai" },
-        async () => {
-          throw new Error("trace failed");
-        }
-      )
-    ).rejects.toThrow("trace failed");
-
-    expect(captureException).toHaveBeenCalledWith(
-      expect.objectContaining({ message: "trace failed" }),
-      expect.objectContaining({
-        source: "llm-monitor",
-        name: "parse-command",
-        model: "gpt-4o-mini",
-        provider: "openai",
-      })
-    );
-    expect(mockCreateRun).not.toHaveBeenCalled();
-  });
-});

package/lib/nl/llm-monitoring.ts

@@ -1,339 +0,0 @@
-import { Client as LangSmithClient, RunTree } from "langsmith";
-
-/**
- * Interface for LLM call metrics
- */
-export interface LLMCallMetrics {
-  model: string;
-  provider: "openai" | "groq";
-  promptTokens?: number;
-  completionTokens?: number;
-  totalTokens?: number;
-  inputCost?: number;
-  outputCost?: number;
-  totalCost?: number;
-  prompt: string;
-  completion: string | null;
-  latency: number;
-  timestamp: Date;
-  error?: string;
-}
-
-export type LLMMonitorOptions = {
-  apiKey?: string;
-  project?: string;
-  endpoint?: string;
-  telemetryBridge?: LLMMonitorTelemetryBridge;
-};
-
-export type LLMMonitorTelemetryBridge = {
-  captureException?: (error: Error, context?: Record<string, unknown>) => void;
-  captureMessage?: (
-    message: string,
-    level?: "error" | "warn" | "info" | "debug",
-    context?: Record<string, unknown>
-  ) => void;
-};
-
-const VOICE_PIPELINE_NAME = "voice.pipeline";
-const VOICE_PIPELINE_VERSION = "v1";
-const VOICE_PIPELINE_FLOW = "speech_to_response";
-
-type LangSmithTraceableModule = {
-  traceable?: <T extends (...args: any[]) => any>(
-    wrappedFunc: T,
-    config?: Record<string, unknown>
-  ) => T;
-  getCurrentRunTree?: (
-    permitAbsentRunTree?: boolean
-  ) => (RunTree & { createChild?: (config: Record<string, unknown>) => RunTree }) | undefined;
-};
-
-let langSmithTraceableModuleCache: LangSmithTraceableModule | null | undefined;
-
-function loadLangSmithTraceableModule(): LangSmithTraceableModule | null {
-  if (langSmithTraceableModuleCache !== undefined) {
-    return langSmithTraceableModuleCache;
-  }
-
-  try {
-    const dynamicRequire: ((id: string) => unknown) | undefined =
-      typeof require === "function"
-        ? require
-        : (globalThis as { require?: (id: string) => unknown }).require;
-
-    if (!dynamicRequire) {
-      langSmithTraceableModuleCache = null;
-      return null;
-    }
-
-    // Keep this out of static dependency analysis so React Native doesn't bundle Node-only async_hooks.
-    const traceableModuleName = ["langsmith", "traceable"].join("/");
-    const loaded = dynamicRequire(traceableModuleName) as LangSmithTraceableModule;
-    langSmithTraceableModuleCache = loaded;
-    return loaded;
-  } catch {
-    langSmithTraceableModuleCache = null;
-    return null;
-  }
-}
-
-function isReactNativeRuntime(): boolean {
-  const maybeNavigator = (globalThis as { navigator?: { product?: string } }).navigator;
-  return maybeNavigator?.product === "ReactNative";
-}
-
-function maskKey(s?: string) {
-  if (!s) return s;
-  if (s.length <= 10) return '****';
-  return `${s.slice(0,4)}...${s.slice(-4)}`;
-}
-
-export class LLMMonitor {
-  private static instance: LLMMonitor | null = null;
-  private langsmith: LangSmithClient | null = null;
-  private enabled: boolean = false;
-  private apiKey?: string;
-  private project?: string;
-  private endpoint?: string;
-  private currentThread?: string | number | null = null;
-  private telemetryBridge: LLMMonitorTelemetryBridge | null = null;
-  private authErrorLogged: boolean = false;
-
-  private constructor(options?: LLMMonitorOptions) {
-    this.configure(options);
-  }
-
-  static getInstance(options?: LLMMonitorOptions): LLMMonitor {
-    if (!LLMMonitor.instance) {
-      LLMMonitor.instance = new LLMMonitor(options);
-      return LLMMonitor.instance;
-    }
-
-    const shouldReconfigure =
-      (typeof options?.apiKey === "string" &&
-        options.apiKey !== LLMMonitor.instance.apiKey) ||
-      (typeof options?.endpoint === "string" &&
-        options.endpoint !== LLMMonitor.instance.endpoint);
-
-    if (shouldReconfigure) {
-      LLMMonitor.instance.configure(options);
-    } else if (options?.project) {
-      LLMMonitor.instance.project = options.project;
-      if (options.telemetryBridge) {
-        LLMMonitor.instance.telemetryBridge = options.telemetryBridge;
-      }
-    } else if (options?.telemetryBridge) {
-      LLMMonitor.instance.telemetryBridge = options.telemetryBridge;
-    }
-
-    return LLMMonitor.instance;
-  }
-
-  private configure(options?: LLMMonitorOptions) {
-    this.apiKey = options?.apiKey ?? this.apiKey;
-    this.project = options?.project ?? this.project;
-    this.endpoint = options?.endpoint ?? this.endpoint;
-    this.telemetryBridge = options?.telemetryBridge || this.telemetryBridge;
-
-    if (this.apiKey) {
-      this.langsmith = new LangSmithClient({
-        apiKey: this.apiKey,
-        ...(this.endpoint ? { apiUrl: this.endpoint } : {}),
-      });
-      this.enabled = true;
-      this.authErrorLogged = false;
-      console.log(`LLMMonitor: LangSmith initialized (key: ${maskKey(this.apiKey)})`);
-    } else {
-      this.langsmith = null;
-      this.enabled = false;
-      this.authErrorLogged = false;
-      console.log("LLMMonitor: LangSmith API key not found, monitoring disabled.");
-      console.log(
-        "Provide a LangSmith key via app config and pass it into the LLM monitor."
-      );
-    }
-  }
-
-  /**
-   * Set the current thread id for grouping runs/traces.
-   * Accepts a string or number. Pass `null`/`undefined` to clear.
-   */
-  public setThread(threadId?: string | number | null) {
-    this.currentThread = threadId ?? null;
-  }
-
-  /**
-   * Get the project name from environment or use default
-   */
-  private getProjectName(): string {
-    if (this.project) return this.project;
-    console.warn("LLMMonitor: No project specified, using 'default'.");
-    return "default";
-  }
-
-  private buildPipelineMetadata(timestamp: Date): Record<string, unknown> {
-    return {
-      timestamp: timestamp.toISOString(),
-      thread_id: this.currentThread ?? null,
-      pipeline_name: this.currentThread ? VOICE_PIPELINE_NAME : null,
-      pipeline_version: this.currentThread ? VOICE_PIPELINE_VERSION : null,
-      pipeline_flow: this.currentThread ? VOICE_PIPELINE_FLOW : null,
-    };
-  }
-
-  private buildUsageMetadata(metrics: LLMCallMetrics): Record<string, number> | undefined {
-    const usageMetadata: Record<string, number> = {};
-
-    if (typeof metrics.promptTokens === "number") {
-      usageMetadata.input_tokens = metrics.promptTokens;
-    }
-    if (typeof metrics.completionTokens === "number") {
-      usageMetadata.output_tokens = metrics.completionTokens;
-    }
-    if (typeof metrics.totalTokens === "number") {
-      usageMetadata.total_tokens = metrics.totalTokens;
-    }
-    if (typeof metrics.inputCost === "number") {
-      usageMetadata.input_cost = metrics.inputCost;
-    }
-    if (typeof metrics.outputCost === "number") {
-      usageMetadata.output_cost = metrics.outputCost;
-    }
-    if (typeof metrics.totalCost === "number") {
-      usageMetadata.total_cost = metrics.totalCost;
-    }
-
-    return Object.keys(usageMetadata).length > 0 ? usageMetadata : undefined;
-  }
-
-  /**
-   * Log an LLM call with metrics
-   */
-  async logCall(metrics: LLMCallMetrics): Promise<void> {
-    // Only send to LangSmith if configured
-    if (!this.enabled || !this.langsmith) {
-      return;
-    }
-
-    try {
-      const usageMetadata = this.buildUsageMetadata(metrics);
-      const outputs: Record<string, unknown> = {
-        completion: metrics.completion,
-      };
-      if (usageMetadata) {
-        outputs.usage_metadata = usageMetadata;
-      }
-
-      const runExtraMetadata: Record<string, unknown> = {
-        provider: metrics.provider,
-        model: metrics.model,
-        ...this.buildPipelineMetadata(metrics.timestamp),
-      };
-      if (usageMetadata) {
-        runExtraMetadata.usage_metadata = usageMetadata;
-      }
-
-      const runConfig = {
-        name: `${metrics.provider}_${metrics.model}`,
-        run_type: "llm",
-        project_name: this.getProjectName(),
-        start_time: metrics.timestamp.getTime(),
-        end_time: metrics.timestamp.getTime() + metrics.latency,
-        inputs: {
-          prompt: metrics.prompt,
-        },
-        extra: {
-          metadata: runExtraMetadata,
-          invocation_params: {
-            ls_provider: metrics.provider,
-            ls_model_name: metrics.model,
-            ls_model_type: "chat",
-          },
-        },
-        outputs,
-        error: metrics.error,
-      } as const;
-
-      // React Native runtime does not support UUID random source required by RunTree internals.
-      // Use direct client.createRun there.
-      if (isReactNativeRuntime()) {
-        await this.langsmith.createRun(runConfig);
-        return;
-      }
-
-      const traceableModule = loadLangSmithTraceableModule();
-      const parentRun = traceableModule?.getCurrentRunTree?.(true);
-      if (parentRun && typeof parentRun.createChild === "function") {
-        const runTree = parentRun.createChild(runConfig);
-        await runTree.postRun();
-        await runTree.end(outputs, metrics.error, runConfig.end_time);
-        await runTree.patchRun();
-        return;
-      }
-
-      await this.langsmith.createRun(runConfig);
-    } catch (error) {
-      const message = error instanceof Error ? error.message : String(error);
-      if (message.includes("status [403]")) {
-        if (!this.authErrorLogged) {
-          this.authErrorLogged = true;
-          console.error(
-            "LangSmith rejected tracing with 403. Check API key scope, endpoint region, and workspace ID."
-          );
-        }
-        return;
-      }
-      console.error("Failed to log to LangSmith:", error);
-    }
-  }
-
-  /**
-   * Create a traced wrapper for async functions
-   */
-  async traceCall<T>(
-    name: string,
-    metadata: { model: string; provider: "openai" | "groq" },
-    fn: () => Promise<T>
-  ): Promise<T> {
-    try {
-      const traced =
-        this.enabled && this.langsmith && !isReactNativeRuntime()
-          ? loadLangSmithTraceableModule()?.traceable?.(fn, {
-              name,
-              run_type: "chain",
-              project_name: this.getProjectName(),
-              client: this.langsmith,
-              metadata: {
-                model: metadata.model,
-                provider: metadata.provider,
-                ...this.buildPipelineMetadata(new Date()),
-              },
-            }) || fn
-          : fn;
-
-      const result = await traced();
-      return result;
-    } catch (error) {
-      const normalizedError =
-        error instanceof Error ? error : new Error(String(error));
-
-      this.telemetryBridge?.captureException?.(normalizedError, {
-        source: "llm-monitor",
-        name,
-        model: metadata.model,
-        provider: metadata.provider,
-        thread_id: this.currentThread ?? null,
-        pipeline_name: this.currentThread ? VOICE_PIPELINE_NAME : undefined,
-        pipeline_version: this.currentThread ? VOICE_PIPELINE_VERSION : undefined,
-        pipeline_flow: this.currentThread ? VOICE_PIPELINE_FLOW : undefined,
-      });
-
-      throw error;
-    }
-  }
-}
-
-export function getLLMMonitor(options?: LLMMonitorOptions): LLMMonitor {
-  return LLMMonitor.getInstance(options);
-}