@copilotkit/runtime 1.50.0-beta.9 → 1.50.0

package/src/lib/runtime/copilot-runtime.ts

@@ -13,47 +13,50 @@
  */
 
 import {
-  Action,
-  CopilotErrorHandler,
+  type Action,
+  type CopilotErrorHandler,
   CopilotKitMisuseError,
-  MaybePromise,
-  NonEmptyRecord,
-  Parameter,
+  type MaybePromise,
+  type NonEmptyRecord,
+  type Parameter,
   readBody,
   getZodParameters,
-  PartialBy,
+  type PartialBy,
+  isTelemetryDisabled,
 } from "@copilotkit/shared";
-import { type RunAgentInput } from "@ag-ui/core";
+import type { RunAgentInput } from "@ag-ui/core";
 import { aguiToGQL } from "../../graphql/message-conversion/agui-to-gql";
-import { CopilotServiceAdapter, RemoteChainParameters } from "../../service-adapters";
+import type { CopilotServiceAdapter, RemoteChainParameters } from "../../service-adapters";
 import {
   CopilotRuntime as CopilotRuntimeVNext,
-  CopilotRuntimeOptions,
-  CopilotRuntimeOptions as CopilotRuntimeOptionsVNext,
-  InMemoryAgentRunner as InMemoryAgentRunnerVNext,
+  type CopilotRuntimeOptions,
+  type CopilotRuntimeOptions as CopilotRuntimeOptionsVNext,
+  InMemoryAgentRunner,
 } from "@copilotkitnext/runtime";
+import { TelemetryAgentRunner } from "./telemetry-agent-runner";
+import telemetry from "../telemetry-client";
 
-import { MessageInput } from "../../graphql/inputs/message.input";
-import { Message } from "../../graphql/types/converted";
+import type { MessageInput } from "../../graphql/inputs/message.input";
+import type { Message } from "../../graphql/types/converted";
 
 import {
   EndpointType,
-  EndpointDefinition,
-  CopilotKitEndpoint,
-  LangGraphPlatformEndpoint,
+  type EndpointDefinition,
+  type CopilotKitEndpoint,
+  type LangGraphPlatformEndpoint,
 } from "./types";
 
-import { CopilotObservabilityConfig, LLMRequestData, LLMResponseData } from "../observability";
-import { AbstractAgent } from "@ag-ui/client";
+import type { CopilotObservabilityConfig, LLMRequestData, LLMResponseData } from "../observability";
+import type { AbstractAgent } from "@ag-ui/client";
 
 // +++ MCP Imports +++
 import {
-  MCPClient,
-  MCPEndpointConfig,
-  MCPTool,
+  type MCPClient,
+  type MCPEndpointConfig,
+  type MCPTool,
   extractParametersFromSchema,
 } from "./mcp-tools-utils";
-import { BasicAgent, BasicAgentConfiguration } from "@copilotkitnext/agent";
+import { BasicAgent, type BasicAgentConfiguration } from "@copilotkitnext/agent";
 // Define the function type alias here or import if defined elsewhere
 type CreateMCPClientFunction = (config: MCPEndpointConfig) => Promise<MCPClient>;
 
@@ -308,9 +311,19 @@ export class CopilotRuntime<const T extends Parameter[] | [] = []> {
     const agents = params?.agents ?? {};
     const endpointAgents = this.assignEndpointsToAgents(params?.remoteEndpoints ?? []);
 
+    // Determine the base runner (user-provided or default)
+    const baseRunner = params?.runner ?? new InMemoryAgentRunner();
+
+    // Wrap with TelemetryAgentRunner unless telemetry is disabled
+    // This ensures we always capture agent execution telemetry when enabled,
+    // even if the user provides their own custom runner
+    const runner = isTelemetryDisabled()
+      ? baseRunner
+      : new TelemetryAgentRunner({ runner: baseRunner });
+
     this.runtimeArgs = {
       agents: { ...endpointAgents, ...agents },
-      runner: params?.runner ?? new InMemoryAgentRunnerVNext(),
+      runner,
       // TODO: add support for transcriptionService from CopilotRuntimeOptionsVNext once it is ready
       // transcriptionService: params?.transcriptionService,
 
@@ -372,10 +385,11 @@ export class CopilotRuntime<const T extends Parameter[] | [] = []> {
           });
         }
 
-        if (this.params.actions) {
+        const actions = this.params?.actions;
+        if (actions) {
           const mcpTools = await this.getToolsFromMCP();
           agentsList = this.assignToolsToAgents(agents, [
-            ...this.getToolsFromActions(this.params.actions),
+            ...this.getToolsFromActions(actions),
             ...mcpTools,
           ]);
         }
@@ -437,6 +451,29 @@ export class CopilotRuntime<const T extends Parameter[] | [] = []> {
     params?: CopilotRuntimeConstructorParams<T> & PartialBy<CopilotRuntimeOptions, "agents">,
   ) {
     return async (hookParams: BeforeRequestMiddlewareFnParameters[0]) => {
+      const { request } = hookParams;
+
+      // Capture telemetry for copilot request creation
+      const publicApiKey = request.headers.get("x-copilotcloud-public-api-key");
+      const body = (await readBody(request)) as RunAgentInput;
+      const forwardedProps = body.forwardedProps as
+        | {
+            cloud?: { guardrails?: unknown };
+            metadata?: { requestType?: string };
+          }
+        | undefined;
+
+      // Get cloud base URL from environment or default
+      const cloudBaseUrl = process.env.COPILOT_CLOUD_BASE_URL || "https://api.cloud.copilotkit.ai";
+
+      telemetry.capture("oss.runtime.copilot_request_created", {
+        "cloud.guardrails.enabled": forwardedProps?.cloud?.guardrails !== undefined,
+        requestType: forwardedProps?.metadata?.requestType ?? "unknown",
+        "cloud.api_key_provided": !!publicApiKey,
+        ...(publicApiKey ? { "cloud.public_api_key": publicApiKey } : {}),
+        "cloud.base_url": cloudBaseUrl,
+      });
+
       // TODO: get public api key and run with expected data
       // if (this.observability?.enabled && this.params.publicApiKey) {
       //   this.logObservabilityBeforeRequest()
@@ -447,7 +484,6 @@ export class CopilotRuntime<const T extends Parameter[] | [] = []> {
 
       if (params?.middleware?.onBeforeRequest) {
         const { request, runtime, path } = hookParams;
-        const body = (await readBody(request)) as RunAgentInput;
         const gqlMessages = (aguiToGQL(body.messages) as Message[]).reduce(
           (acc, msg) => {
             if ("role" in msg && msg.role === "user") {

package/src/lib/runtime/telemetry-agent-runner.ts

@@ -0,0 +1,139 @@
+/**
+ * TelemetryAgentRunner - A wrapper around AgentRunner that adds telemetry
+ * for agent execution streams.
+ *
+ * This captures the following telemetry events:
+ * - oss.runtime.agent_execution_stream_started - when an agent execution starts
+ * - oss.runtime.agent_execution_stream_ended - when an agent execution completes
+ * - oss.runtime.agent_execution_stream_errored - when an agent execution fails
+ */
+
+import { type AgentRunner, InMemoryAgentRunner } from "@copilotkitnext/runtime";
+import { createHash } from "node:crypto";
+import { tap, catchError, finalize } from "rxjs";
+import telemetry from "../telemetry-client";
+import type { AgentExecutionResponseInfo } from "@copilotkit/shared/src/telemetry/events";
+
+/**
+ * Configuration options for TelemetryAgentRunner
+ */
+export interface TelemetryAgentRunnerConfig {
+  /**
+   * The underlying runner to delegate to
+   * If not provided, defaults to InMemoryAgentRunner
+   */
+  runner?: AgentRunner;
+
+  /**
+   * Optional LangSmith API key (will be hashed for telemetry)
+   */
+  langsmithApiKey?: string;
+}
+
+/**
+ * An AgentRunner wrapper that adds telemetry tracking for agent executions.
+ *
+ * Usage:
+ * ```ts
+ * const runtime = new CopilotRuntime({
+ *   runner: new TelemetryAgentRunner(),
+ *   // or with custom runner:
+ *   runner: new TelemetryAgentRunner({ runner: customRunner }),
+ * });
+ * ```
+ */
+export class TelemetryAgentRunner implements AgentRunner {
+  private readonly _runner: AgentRunner;
+  private readonly hashedLgcKey: string | undefined;
+
+  constructor(config?: TelemetryAgentRunnerConfig) {
+    this._runner = config?.runner ?? new InMemoryAgentRunner();
+    this.hashedLgcKey = config?.langsmithApiKey
+      ? createHash("sha256").update(config.langsmithApiKey).digest("hex")
+      : undefined;
+  }
+
+  /**
+   * Runs an agent with telemetry tracking.
+   * Wraps the underlying runner's Observable stream with telemetry events.
+   */
+  run(...args: Parameters<AgentRunner["run"]>): ReturnType<AgentRunner["run"]> {
+    const streamInfo: AgentExecutionResponseInfo = {
+      hashedLgcKey: this.hashedLgcKey,
+    };
+    let streamErrored = false;
+
+    // Capture stream started event
+    telemetry.capture("oss.runtime.agent_execution_stream_started", {
+      hashedLgcKey: this.hashedLgcKey,
+    });
+
+    // Delegate to the underlying runner and wrap with telemetry
+    return this._runner.run(...args).pipe(
+      // Extract metadata from events if available
+      tap((event) => {
+        // Try to extract provider/model info from raw events
+        const rawEvent = (
+          event as {
+            rawEvent?: { metadata?: Record<string, unknown>; data?: Record<string, unknown> };
+          }
+        ).rawEvent;
+        if (rawEvent?.data) {
+          const data = rawEvent.data as { output?: { model?: string } };
+          if (data?.output?.model) {
+            streamInfo.model = data.output.model;
+            streamInfo.provider = data.output.model;
+          }
+        }
+        if (rawEvent?.metadata) {
+          const metadata = rawEvent.metadata as {
+            langgraph_host?: string;
+            langgraph_version?: string;
+          };
+          if (metadata?.langgraph_host) {
+            streamInfo.langGraphHost = metadata.langgraph_host;
+          }
+          if (metadata?.langgraph_version) {
+            streamInfo.langGraphVersion = metadata.langgraph_version;
+          }
+        }
+      }),
+      catchError((error) => {
+        // Capture stream error event
+        streamErrored = true;
+        telemetry.capture("oss.runtime.agent_execution_stream_errored", {
+          ...streamInfo,
+          error: error instanceof Error ? error.message : String(error),
+        });
+        throw error;
+      }),
+      finalize(() => {
+        // Capture stream ended event (only if not errored)
+        if (!streamErrored) {
+          telemetry.capture("oss.runtime.agent_execution_stream_ended", streamInfo);
+        }
+      }),
+    );
+  }
+
+  /**
+   * Delegates to the underlying runner's connect method
+   */
+  connect(...args: Parameters<AgentRunner["connect"]>): ReturnType<AgentRunner["connect"]> {
+    return this._runner.connect(...args);
+  }
+
+  /**
+   * Delegates to the underlying runner's isRunning method
+   */
+  isRunning(...args: Parameters<AgentRunner["isRunning"]>): ReturnType<AgentRunner["isRunning"]> {
+    return this._runner.isRunning(...args);
+  }
+
+  /**
+   * Delegates to the underlying runner's stop method
+   */
+  stop(...args: Parameters<AgentRunner["stop"]>): ReturnType<AgentRunner["stop"]> {
+    return this._runner.stop(...args);
+  }
+}