@copilotkit/runtime 1.56.3 → 1.56.4-canary.1777531098

package/src/v2/runtime/__tests__/hono-telemetry.integration.test.ts

@@ -0,0 +1,99 @@
+/**
+ * Integration test: Hono adapter + telemetry.
+ *
+ * Asserts both ends of the path-to-production chain that the v2 refactor
+ * previously broke (2ac4a40b5, 2026-03-29):
+ *   1. `oss.runtime.instance_created` fires once per handler factory.
+ *   2. `oss.runtime.copilot_request_created` fires when a real HTTP request
+ *      flows through the handler.
+ *
+ * If either regresses, this test fails.
+ */
+import { describe, it, expect, vi, beforeEach, afterEach } from "vitest";
+import type { AbstractAgent } from "@ag-ui/client";
+
+import { telemetry } from "../telemetry";
+import { createCopilotHonoHandler } from "../endpoints/hono";
+import { CopilotRuntime } from "../core/runtime";
+
+function makeAgent(): AbstractAgent {
+  const a: unknown = {
+    execute: async () => ({ events: [] }),
+  };
+  (a as { clone: () => unknown }).clone = () => makeAgent();
+  return a as AbstractAgent;
+}
+
+describe("Hono adapter — telemetry firing (integration)", () => {
+  let captureSpy: ReturnType<typeof vi.spyOn>;
+
+  beforeEach(() => {
+    captureSpy = vi.spyOn(telemetry, "capture").mockResolvedValue(undefined);
+  });
+
+  afterEach(() => {
+    captureSpy.mockRestore();
+  });
+
+  it("fires instance_created on handler creation (multi-route)", async () => {
+    const runtime = new CopilotRuntime({ agents: { default: makeAgent() } });
+    createCopilotHonoHandler({ runtime, basePath: "/" });
+
+    await vi.waitFor(() => {
+      expect(captureSpy).toHaveBeenCalledWith(
+        "oss.runtime.instance_created",
+        expect.objectContaining({
+          agentsAmount: 1,
+          actionsAmount: 0,
+          endpointsAmount: 0,
+          "cloud.api_key_provided": false,
+        }),
+      );
+    });
+  });
+
+  it("fires copilot_request_created when a real HTTP request hits the handler", async () => {
+    const runtime = new CopilotRuntime({ agents: { default: makeAgent() } });
+    const endpoint = createCopilotHonoHandler({ runtime, basePath: "/" });
+
+    await endpoint.fetch(
+      new Request("https://example.com/agent/default/run", {
+        method: "POST",
+        headers: { "Content-Type": "application/json" },
+        body: JSON.stringify({ messages: [], state: {}, threadId: "t1" }),
+      }),
+    );
+
+    expect(captureSpy).toHaveBeenCalledWith(
+      "oss.runtime.copilot_request_created",
+      expect.objectContaining({
+        requestType: "run",
+        "cloud.api_key_provided": false,
+      }),
+    );
+  });
+
+  it("includes cloud.public_api_key on request when header is present", async () => {
+    const runtime = new CopilotRuntime({ agents: { default: makeAgent() } });
+    const endpoint = createCopilotHonoHandler({ runtime, basePath: "/" });
+
+    await endpoint.fetch(
+      new Request("https://example.com/agent/default/run", {
+        method: "POST",
+        headers: {
+          "Content-Type": "application/json",
+          "x-copilotcloud-public-api-key": "ck_pub_test_123",
+        },
+        body: JSON.stringify({ messages: [], state: {}, threadId: "t1" }),
+      }),
+    );
+
+    expect(captureSpy).toHaveBeenCalledWith(
+      "oss.runtime.copilot_request_created",
+      expect.objectContaining({
+        "cloud.api_key_provided": true,
+        "cloud.public_api_key": "ck_pub_test_123",
+      }),
+    );
+  });
+});

package/src/v2/runtime/__tests__/intelligence-run-telemetry.test.ts

@@ -0,0 +1,194 @@
+/**
+ * Telemetry lifecycle tests for `packages/runtime/src/v2/runtime/handlers/intelligence/run.ts`.
+ *
+ * intelligence/run.ts fires three events across the agent run lifecycle:
+ *   - oss.runtime.agent_execution_stream_started  (line 126, after thread lock)
+ *   - oss.runtime.agent_execution_stream_errored  (inside runner subscribe's error handler)
+ *   - oss.runtime.agent_execution_stream_ended    (inside runner subscribe's complete handler)
+ *
+ * This test verifies each fires under the expected condition. It's paired
+ * with sse-response-telemetry.test.ts which covers the SSE path of the
+ * same event names — kept separate so a regression in one file fails only
+ * its own test.
+ */
+import type { AbstractAgent, BaseEvent } from "@ag-ui/client";
+import { Observable } from "rxjs";
+import { describe, it, expect, vi, beforeEach, afterEach } from "vitest";
+
+import { handleRunAgent } from "../handlers/handle-run";
+import { IntelligenceAgentRunner } from "../runner/intelligence";
+import { telemetry } from "../telemetry";
+import type { CopilotRuntime } from "../core/runtime";
+
+// --- Minimal helpers (mirroring handle-run.test.ts's intelligence block) ---
+
+interface MockIntelligencePlatform {
+  [key: string]: ((...args: any[]) => any) | undefined;
+}
+
+function makeAgent(): AbstractAgent {
+  const makeClone = () =>
+    ({
+      clone: vi.fn(() => makeClone()),
+      setMessages: vi.fn(),
+      setState: vi.fn(),
+      threadId: undefined,
+      headers: {},
+      runAgent: vi.fn().mockResolvedValue(undefined),
+    }) as unknown as AbstractAgent;
+  const agent: any = {
+    clone: vi.fn(() => makeClone()),
+    setMessages: vi.fn(),
+    setState: vi.fn(),
+    threadId: undefined,
+    headers: {},
+    runAgent: vi.fn().mockResolvedValue(undefined),
+  };
+  return agent as AbstractAgent;
+}
+
+function makeIntelligenceRuntime(
+  runObservable: Observable<BaseEvent>,
+  extraPlatform: MockIntelligencePlatform = {},
+): CopilotRuntime {
+  const runner = Object.create(IntelligenceAgentRunner.prototype);
+  runner.run = vi.fn(() => runObservable);
+
+  const platform: MockIntelligencePlatform = {
+    getOrCreateThread: vi.fn().mockResolvedValue({
+      thread: { id: "thread-1", name: null },
+      created: false,
+    }),
+    getThreadMessages: vi.fn().mockResolvedValue({ messages: [] }),
+    ɵacquireThreadLock: vi.fn().mockResolvedValue({
+      threadId: "thread-1",
+      runId: "run-1",
+      joinToken: "jt-1",
+    }),
+    ɵcleanupThreadLock: vi.fn().mockResolvedValue(undefined),
+    ɵgetClientWsUrl: vi.fn(() => "wss://runtime.example/client"),
+    ɵrenewThreadLock: vi.fn().mockResolvedValue(undefined),
+    ...extraPlatform,
+  };
+
+  return {
+    agents: Promise.resolve({ "my-agent": makeAgent() }),
+    transcriptionService: undefined,
+    beforeRequestMiddleware: undefined,
+    afterRequestMiddleware: undefined,
+    runner,
+    mode: "intelligence",
+    generateThreadNames: false,
+    intelligence: platform,
+    identifyUser: vi.fn().mockResolvedValue({ id: "user-1", name: "User One" }),
+    lockTtlSeconds: 20,
+    lockHeartbeatIntervalSeconds: 15,
+  } as unknown as CopilotRuntime;
+}
+
+function makeRunRequest(): Request {
+  return new Request("https://example.com/agent/my-agent/run", {
+    method: "POST",
+    headers: { "Content-Type": "application/json" },
+    body: JSON.stringify({
+      threadId: "thread-1",
+      runId: "run-1",
+      state: {},
+      messages: [],
+      tools: [],
+      context: [],
+      forwardedProps: {},
+    }),
+  });
+}
+
+// --- Tests ---
+
+describe("intelligence/run.ts — telemetry lifecycle", () => {
+  let captureSpy: ReturnType<typeof vi.spyOn>;
+  let errorSpy: ReturnType<typeof vi.spyOn>;
+
+  beforeEach(() => {
+    vi.useFakeTimers();
+    captureSpy = vi.spyOn(telemetry, "capture").mockResolvedValue(undefined);
+    // Swallow the logger.error that fires on simulated agent errors.
+    errorSpy = vi.spyOn(console, "error").mockImplementation(() => {});
+  });
+
+  afterEach(() => {
+    vi.useRealTimers();
+    captureSpy.mockRestore();
+    errorSpy.mockRestore();
+  });
+
+  it("fires agent_execution_stream_started once thread lock is acquired", async () => {
+    // Observable that never completes — so only stream_started should fire.
+    const never = new Observable<BaseEvent>(() => {});
+    const runtime = makeIntelligenceRuntime(never);
+
+    await handleRunAgent({
+      runtime,
+      request: makeRunRequest(),
+      agentId: "my-agent",
+    });
+
+    expect(captureSpy).toHaveBeenCalledWith(
+      "oss.runtime.agent_execution_stream_started",
+      {},
+    );
+    // And NOT the other two
+    expect(captureSpy).not.toHaveBeenCalledWith(
+      "oss.runtime.agent_execution_stream_errored",
+      expect.anything(),
+    );
+    expect(captureSpy).not.toHaveBeenCalledWith(
+      "oss.runtime.agent_execution_stream_ended",
+      expect.anything(),
+    );
+  });
+
+  it("fires agent_execution_stream_ended when runner.run observable completes", async () => {
+    // Observable that completes immediately.
+    const completing = new Observable<BaseEvent>((subscriber) => {
+      subscriber.complete();
+    });
+    const runtime = makeIntelligenceRuntime(completing);
+
+    await handleRunAgent({
+      runtime,
+      request: makeRunRequest(),
+      agentId: "my-agent",
+    });
+
+    expect(captureSpy).toHaveBeenCalledWith(
+      "oss.runtime.agent_execution_stream_started",
+      {},
+    );
+    expect(captureSpy).toHaveBeenCalledWith(
+      "oss.runtime.agent_execution_stream_ended",
+      {},
+    );
+  });
+
+  it("fires agent_execution_stream_errored when runner.run observable errors", async () => {
+    const failing = new Observable<BaseEvent>((subscriber) => {
+      subscriber.error(new Error("agent exploded"));
+    });
+    const runtime = makeIntelligenceRuntime(failing);
+
+    await handleRunAgent({
+      runtime,
+      request: makeRunRequest(),
+      agentId: "my-agent",
+    });
+
+    expect(captureSpy).toHaveBeenCalledWith(
+      "oss.runtime.agent_execution_stream_started",
+      {},
+    );
+    expect(captureSpy).toHaveBeenCalledWith(
+      "oss.runtime.agent_execution_stream_errored",
+      expect.objectContaining({ error: "agent exploded" }),
+    );
+  });
+});

package/src/v2/runtime/__tests__/sse-response-telemetry.test.ts

@@ -0,0 +1,108 @@
+/**
+ * Telemetry lifecycle tests for `packages/runtime/src/v2/runtime/handlers/shared/sse-response.ts`.
+ *
+ * sse-response.ts fires three events across the SSE stream lifecycle:
+ *   - oss.runtime.agent_execution_stream_started  (line 73, right after observableFactory resolves)
+ *   - oss.runtime.agent_execution_stream_errored  (inside subscribe's error handler)
+ *   - oss.runtime.agent_execution_stream_ended    (inside subscribe's complete handler)
+ *
+ * Paired with intelligence-run-telemetry.test.ts which covers the
+ * intelligence/run path of the same event names — kept separate so a
+ * regression in one source file fails only its own test.
+ */
+import { BaseEvent } from "@ag-ui/client";
+import { Observable } from "rxjs";
+import { describe, it, expect, vi, beforeEach, afterEach } from "vitest";
+
+import { createSseEventResponse } from "../handlers/shared/sse-response";
+import { telemetry } from "../telemetry";
+
+function makeRequest(): Request {
+  return new Request("https://example.com/agent/test/run", { method: "POST" });
+}
+
+describe("sse-response.ts — telemetry lifecycle", () => {
+  let captureSpy: ReturnType<typeof vi.spyOn>;
+  let errorSpy: ReturnType<typeof vi.spyOn>;
+
+  beforeEach(() => {
+    captureSpy = vi.spyOn(telemetry, "capture").mockResolvedValue(undefined);
+    // Swallow the console.error from SSE logError on simulated failures.
+    errorSpy = vi.spyOn(console, "error").mockImplementation(() => {});
+  });
+
+  afterEach(() => {
+    captureSpy.mockRestore();
+    errorSpy.mockRestore();
+  });
+
+  it("fires agent_execution_stream_started once the observable factory resolves", async () => {
+    // Observable that never completes — only started should fire.
+    const never = new Observable<BaseEvent>(() => {});
+    createSseEventResponse({
+      request: makeRequest(),
+      observableFactory: () => never,
+    });
+
+    await vi.waitFor(() => {
+      expect(captureSpy).toHaveBeenCalledWith(
+        "oss.runtime.agent_execution_stream_started",
+        {},
+      );
+    });
+
+    expect(captureSpy).not.toHaveBeenCalledWith(
+      "oss.runtime.agent_execution_stream_errored",
+      expect.anything(),
+    );
+    expect(captureSpy).not.toHaveBeenCalledWith(
+      "oss.runtime.agent_execution_stream_ended",
+      expect.anything(),
+    );
+  });
+
+  it("fires agent_execution_stream_ended when the observable completes", async () => {
+    const completing = new Observable<BaseEvent>((subscriber) => {
+      subscriber.complete();
+    });
+    createSseEventResponse({
+      request: makeRequest(),
+      observableFactory: () => completing,
+    });
+
+    await vi.waitFor(() => {
+      expect(captureSpy).toHaveBeenCalledWith(
+        "oss.runtime.agent_execution_stream_ended",
+        {},
+      );
+    });
+
+    // started should also have fired before ended
+    expect(captureSpy).toHaveBeenCalledWith(
+      "oss.runtime.agent_execution_stream_started",
+      {},
+    );
+  });
+
+  it("fires agent_execution_stream_errored with the error message when the observable errors", async () => {
+    const failing = new Observable<BaseEvent>((subscriber) => {
+      subscriber.error(new Error("stream blew up"));
+    });
+    createSseEventResponse({
+      request: makeRequest(),
+      observableFactory: () => failing,
+    });
+
+    await vi.waitFor(() => {
+      expect(captureSpy).toHaveBeenCalledWith(
+        "oss.runtime.agent_execution_stream_errored",
+        expect.objectContaining({ error: "stream blew up" }),
+      );
+    });
+
+    expect(captureSpy).toHaveBeenCalledWith(
+      "oss.runtime.agent_execution_stream_started",
+      {},
+    );
+  });
+});

package/src/v2/runtime/__tests__/telemetry.test.ts

@@ -2,27 +2,15 @@ import { describe, it, expect, vi, beforeEach, afterEach } from "vitest";
 import { TelemetryClient } from "../telemetry/telemetry-client";
 import scarfClient from "../telemetry/scarf-client";
 
-const { segmentTrackMock } = vi.hoisted(() => ({
-  segmentTrackMock: vi.fn(),
-}));
-
-vi.mock("@segment/analytics-node", () => ({
-  Analytics: vi.fn().mockImplementation(() => ({
-    track: segmentTrackMock,
-  })),
-}));
-
 describe("TelemetryClient", () => {
   let scarfSpy: ReturnType<typeof vi.spyOn>;
 
   beforeEach(() => {
     scarfSpy = vi.spyOn(scarfClient, "logEvent").mockResolvedValue(undefined);
-    segmentTrackMock.mockClear();
   });
 
   afterEach(() => {
     scarfSpy.mockRestore();
-    segmentTrackMock.mockClear();
   });
 
   it("sends event to scarf when sampled in", async () => {
@@ -44,33 +32,6 @@ describe("TelemetryClient", () => {
     });
   });
 
-  it("sends event with properties to segment when sampled in", async () => {
-    vi.spyOn(Math, "random").mockReturnValue(0);
-    const client = new TelemetryClient({
-      telemetryDisabled: false,
-      sampleRate: 1,
-    });
-
-    await client.capture("oss.runtime.instance_created", {
-      actionsAmount: 5,
-      endpointTypes: ["rest"],
-      endpointsAmount: 2,
-      "cloud.api_key_provided": true,
-      "cloud.public_api_key": "pk_test_123",
-    });
-
-    expect(segmentTrackMock).toHaveBeenCalledTimes(1);
-    const call = segmentTrackMock.mock.calls[0][0];
-    expect(call.event).toBe("oss.runtime.instance_created");
-    expect(call.anonymousId).toMatch(/^anon_/);
-    expect(call.properties).toMatchObject({
-      actionsAmount: 5,
-      endpointsAmount: 2,
-      "cloud.api_key_provided": true,
-      "cloud.public_api_key": "pk_test_123",
-    });
-  });
-
   it("only sends event name to scarf, not properties", async () => {
     const client = new TelemetryClient({
       telemetryDisabled: false,
@@ -106,7 +67,6 @@ describe("TelemetryClient", () => {
     });
 
     expect(scarfSpy).not.toHaveBeenCalled();
-    expect(segmentTrackMock).not.toHaveBeenCalled();
   });
 
   it("does not send events when sampled out", async () => {
@@ -124,7 +84,6 @@ describe("TelemetryClient", () => {
     });
 
     expect(scarfSpy).not.toHaveBeenCalled();
-    expect(segmentTrackMock).not.toHaveBeenCalled();
   });
 
   it("respects sample rate boundary", async () => {
@@ -139,26 +98,6 @@ describe("TelemetryClient", () => {
     expect(scarfSpy).toHaveBeenCalled();
   });
 
-  it("includes global properties in segment track call", async () => {
-    vi.spyOn(Math, "random").mockReturnValue(0);
-    const client = new TelemetryClient({
-      telemetryDisabled: false,
-      sampleRate: 1,
-    });
-
-    client.setGlobalProperties({ "copilotkit.package.name": "test-pkg" });
-
-    await client.capture("oss.runtime.instance_created", {
-      actionsAmount: 1,
-      endpointTypes: [],
-      endpointsAmount: 0,
-      "cloud.api_key_provided": false,
-    });
-
-    const call = segmentTrackMock.mock.calls[0][0];
-    expect(call.properties["copilotkit.package.name"]).toBe("test-pkg");
-  });
-
   it("throws when sample rate is out of range", () => {
     expect(() => new TelemetryClient({ sampleRate: 1.5 })).toThrow(
       "Sample rate must be between 0 and 1",

package/src/v2/runtime/core/fetch-handler.ts

@@ -62,6 +62,7 @@ import {
   type MethodCall,
 } from "../endpoints/single-route-helpers";
 import { logger } from "@copilotkit/shared";
+import { fireInstanceCreatedTelemetry } from "../telemetry/instance-created";
 
 /* ------------------------------------------------------------------------------------------------
  * Public types
@@ -114,6 +115,8 @@ export function createCopilotRuntimeHandler(
 ): CopilotRuntimeFetchHandler {
   const { runtime, basePath, mode = "multi-route", cors, hooks } = options;
 
+  fireInstanceCreatedTelemetry({ runtime });
+
   const corsConfig = resolveCorsConfig(cors);
 
   return async (request: Request): Promise<Response> => {

package/src/v2/runtime/endpoints/express.ts

@@ -147,15 +147,21 @@ export function createCopilotExpressHandler({
     router.post(normalizedBase, expressHandler);
     router.options(normalizedBase, expressHandler);
   } else if (normalizedBase === "/") {
-    router.all("*", expressHandler);
+    router.all(/.*/, expressHandler);
   } else {
-    router.all(`${normalizedBase}/*`, expressHandler);
-    router.all(normalizedBase, expressHandler);
+    router.all(
+      new RegExp(`^${escapeRegExp(normalizedBase)}(\\/.*)?$`),
+      expressHandler,
+    );
   }
 
   return router;
 }
 
+function escapeRegExp(s: string): string {
+  return s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+}
+
 function normalizeBasePath(path: string): string {
   if (!path) {
     throw new Error("basePath must be provided for Express endpoint");

package/src/v2/runtime/handlers/handle-connect.ts

@@ -43,9 +43,8 @@ export async function handleConnectAgent({
       return handleIntelligenceConnect({
         runtime,
         request,
+        agentId,
         threadId: connectRequest.input.threadId,
-        runId: connectRequest.input.runId,
-        lastSeenEventId: connectRequest.lastSeenEventId,
       });
     }