@copilotkit/runtime 1.54.1 → 1.55.0-next.7

package/src/v2/runtime/runner/intelligence.ts

@@ -0,0 +1,429 @@
+import {
+  AgentRunner,
+  AgentRunnerConnectRequest,
+  AgentRunnerIsRunningRequest,
+  AgentRunnerRunRequest,
+  type AgentRunnerStopRequest,
+} from "./agent-runner";
+import { EMPTY, Observable, from } from "rxjs";
+import { catchError, finalize } from "rxjs/operators";
+import {
+  AbstractAgent,
+  BaseEvent,
+  EventType,
+  RunStartedEvent,
+} from "@ag-ui/client";
+import {
+  finalizeRunEvents,
+  AG_UI_CHANNEL_EVENT,
+  phoenixExponentialBackoff,
+} from "@copilotkit/shared";
+import { Socket, Channel } from "phoenix";
+import { randomUUID } from "node:crypto";
+
+export interface IntelligenceAgentRunnerOptions {
+  /** Phoenix runner websocket URL, e.g. "ws://localhost:4000/runner" */
+  url: string;
+  /** Optional Phoenix socket auth token used during websocket connect. */
+  authToken?: string;
+}
+
+interface ThreadState {
+  socket: Socket;
+  channel: Channel;
+  isRunning: boolean;
+  stopRequested: boolean;
+  agent: AbstractAgent | null;
+  currentEvents: BaseEvent[];
+  nextEventSeq: number;
+  hasRunStarted: boolean;
+}
+
+export class IntelligenceAgentRunner extends AgentRunner {
+  private options: IntelligenceAgentRunnerOptions;
+  private threads = new Map<string, ThreadState>();
+
+  constructor(options: IntelligenceAgentRunnerOptions) {
+    super();
+    // Store config — sockets are created per-run, not eagerly.
+    this.options = options;
+  }
+
+  /**
+   * Create a new Phoenix socket with explicit exponential backoff.
+   *
+   * Each run/connect gets its own socket so that:
+   *  - A socket failure only affects a single thread, not all threads.
+   *  - Cleanup is simple: channel.leave() + socket.disconnect() tears
+   *    down everything for that run with no shared-state concerns.
+   *  - Each run gets its own independent retry budget.
+   *
+   * reconnectAfterMs — delay before Phoenix reconnects the WebSocket
+   *   after an unclean close. 100ms base, doubling up to a 10s cap.
+   *
+   * rejoinAfterMs — delay before Phoenix re-joins a channel that
+   *   entered the "errored" state. 1s base, doubling up to 30s cap.
+   *
+   * These are set explicitly because Phoenix's default schedule is a
+   * fixed stepped array (not exponential), and any code that calls
+   * socket.disconnect() in an onError handler will set
+   * closeWasClean = true and reset the reconnect timer — permanently
+   * killing retries.
+   */
+  private createSocket(): Socket {
+    const socket = new Socket(this.options.url, {
+      ...(this.options.authToken ? { authToken: this.options.authToken } : {}),
+      reconnectAfterMs: phoenixExponentialBackoff(100, 10_000),
+      rejoinAfterMs: phoenixExponentialBackoff(1_000, 30_000),
+    });
+    socket.connect();
+    return socket;
+  }
+
+  private createRunnerEventPayload(
+    event: BaseEvent,
+    request: AgentRunnerRunRequest,
+    state: ThreadState,
+  ): Record<string, unknown> {
+    const canonicalEvent = this.stampRunnerMetadata(event, state);
+    const payload = {
+      ...(canonicalEvent as Record<string, unknown>),
+    };
+
+    payload.thread_id ??= request.threadId;
+
+    const runId = payload.runId ?? payload.run_id ?? request.input.runId;
+
+    if (runId) {
+      payload.run_id = runId;
+    }
+
+    return payload;
+  }
+
+  private stampRunnerMetadata(event: BaseEvent, state: ThreadState): BaseEvent {
+    const eventRecord = event as BaseEvent & {
+      metadata?: Record<string, unknown>;
+    };
+
+    const existingMetadata = eventRecord.metadata ?? {};
+    const hasEventId = typeof existingMetadata.cpki_event_id === "string";
+    const hasEventSeq = typeof existingMetadata.cpki_event_seq === "number";
+
+    if (hasEventId && hasEventSeq) {
+      const eventSeq = existingMetadata.cpki_event_seq as number;
+      state.nextEventSeq = Math.max(state.nextEventSeq, eventSeq + 1);
+      return eventRecord;
+    }
+
+    const eventSeq = state.nextEventSeq++;
+
+    return {
+      ...eventRecord,
+      metadata: {
+        ...existingMetadata,
+        cpki_event_id:
+          typeof existingMetadata.cpki_event_id === "string"
+            ? existingMetadata.cpki_event_id
+            : randomUUID(),
+        cpki_event_seq: eventSeq,
+      },
+    };
+  }
+
+  run(request: AgentRunnerRunRequest): Observable<BaseEvent> {
+    const { threadId, agent, input, joinCode } = request;
+
+    const existing = this.threads.get(threadId);
+    if (existing?.isRunning) {
+      throw new Error("Thread already running");
+    }
+
+    return new Observable((observer) => {
+      const socket = this.createSocket();
+
+      const channelTopic = joinCode ?? threadId;
+      const channel = socket.channel(`ingestion:${channelTopic}`, {
+        runId: input.runId,
+      });
+
+      const state: ThreadState = {
+        socket,
+        channel,
+        isRunning: true,
+        stopRequested: false,
+        agent,
+        currentEvents: [],
+        nextEventSeq: 1,
+        hasRunStarted: false,
+      };
+      this.threads.set(threadId, state);
+
+      // Track consecutive socket errors for this run. Phoenix retries
+      // automatically via reconnectAfterMs, but if the connection fails
+      // repeatedly we abort the agent — otherwise runAgent() completes
+      // normally, finalization events buffer silently on the dead
+      // channel, and the client never receives them.
+      //
+      // Aborting the agent is the single trigger that cascades through
+      // the existing error pipeline: runAgent() rejects → catchError
+      // pushes RUN_ERROR → finalize calls finalizeRunEvents +
+      // removeThread → channel.leave() + socket.disconnect().
+      const MAX_CONSECUTIVE_ERRORS = 5;
+      let consecutiveErrors = 0;
+
+      socket.onError(() => {
+        consecutiveErrors++;
+        if (consecutiveErrors >= MAX_CONSECUTIVE_ERRORS && state.agent) {
+          try {
+            state.agent.abortRun();
+          } catch {
+            // Ignore abort errors.
+          }
+        }
+        // Otherwise: Phoenix retries automatically using the exponential
+        // backoff schedule configured in createSocket().
+      });
+
+      socket.onOpen(() => {
+        // A successful (re)connection resets the counter so transient
+        // network blips don't accumulate across recoveries.
+        consecutiveErrors = 0;
+      });
+
+      // Listen for custom "stop" events pushed by the client over the
+      // channel. This must be registered before channel.join() so the
+      // handler is in place by the time the server starts relaying messages.
+      // The client sends the stop event before leaving the channel, so the
+      // runner is guaranteed to receive it while still joined.
+      channel.on(AG_UI_CHANNEL_EVENT, (payload: BaseEvent) => {
+        if (
+          payload.type === EventType.CUSTOM &&
+          (payload as BaseEvent & { name?: string }).name === "stop"
+        ) {
+          this.stop({ threadId });
+        }
+      });
+
+      channel
+        .join()
+        .receive("ok", () => {
+          this.executeAgentRun(request, state, threadId).subscribe({
+            complete: () => observer.complete(),
+          });
+        })
+        .receive("error", (resp) => {
+          const errorEvent = {
+            type: EventType.RUN_ERROR,
+            message: `Failed to join channel: ${JSON.stringify(resp)}`,
+            code: "CHANNEL_JOIN_ERROR",
+          } as BaseEvent;
+          observer.next(errorEvent);
+          state.currentEvents.push(errorEvent);
+          this.removeThread(threadId);
+          observer.complete();
+        })
+        .receive("timeout", () => {
+          const errorEvent = {
+            type: EventType.RUN_ERROR,
+            message: "Timed out joining channel",
+            code: "CHANNEL_JOIN_TIMEOUT",
+          } as BaseEvent;
+          observer.next(errorEvent);
+          state.currentEvents.push(errorEvent);
+          this.removeThread(threadId);
+          observer.complete();
+        });
+
+      return () => {
+        this.removeThread(threadId);
+      };
+    });
+  }
+
+  connect(request: AgentRunnerConnectRequest): Observable<BaseEvent> {
+    const { threadId } = request;
+
+    return new Observable((observer) => {
+      const socket = this.createSocket();
+
+      const channel = socket.channel(`thread:${threadId}`);
+
+      channel.on("ag_ui_event", (payload: BaseEvent) => {
+        observer.next(payload);
+
+        if (
+          payload.type === EventType.RUN_FINISHED ||
+          payload.type === EventType.RUN_ERROR
+        ) {
+          observer.complete();
+        }
+      });
+
+      const cleanup = () => {
+        channel.leave();
+        socket.disconnect();
+      };
+
+      channel
+        .join()
+        .receive("ok", () => undefined)
+        .receive("error", (resp) => {
+          observer.error(
+            new Error(`Failed to join channel: ${JSON.stringify(resp)}`),
+          );
+          cleanup();
+        })
+        .receive("timeout", () => {
+          observer.error(new Error("Timed out joining channel"));
+          cleanup();
+        });
+
+      return () => {
+        cleanup();
+      };
+    });
+  }
+
+  isRunning(request: AgentRunnerIsRunningRequest): Promise<boolean> {
+    const state = this.threads.get(request.threadId);
+    return Promise.resolve(state?.isRunning ?? false);
+  }
+
+  stop(request: AgentRunnerStopRequest): Promise<boolean | undefined> {
+    const state = this.threads.get(request.threadId);
+    if (!state || !state.isRunning || state.stopRequested) {
+      return Promise.resolve(false);
+    }
+
+    state.stopRequested = true;
+
+    // Direct local abort — the runtime is the authority.
+    if (state.agent) {
+      try {
+        state.agent.abortRun();
+      } catch {
+        // Ignore abort errors.
+      }
+    }
+
+    return Promise.resolve(true);
+  }
+
+  private executeAgentRun(
+    request: AgentRunnerRunRequest,
+    state: ThreadState,
+    threadId: string,
+  ): Observable<void> {
+    const { currentEvents, channel } = state;
+    const pushCanonicalEvent = (event: BaseEvent): void => {
+      const canonicalEvent = this.stampRunnerMetadata(event, state);
+      currentEvents.push(canonicalEvent);
+
+      if (canonicalEvent.type === EventType.RUN_STARTED) {
+        state.hasRunStarted = true;
+      }
+
+      channel.push(
+        "event",
+        this.createRunnerEventPayload(canonicalEvent, request, state),
+      );
+    };
+
+    const getPersistedInputMessages = () =>
+      request.persistedInputMessages ?? request.input.messages;
+
+    const buildRunStartedEvent = (
+      source?: RunStartedEvent,
+    ): RunStartedEvent => {
+      const baseInput = source?.input ?? request.input;
+      const persistedInputMessages = getPersistedInputMessages();
+
+      return {
+        ...(source ?? {
+          type: EventType.RUN_STARTED,
+          threadId: request.threadId,
+          runId: request.input.runId,
+        }),
+        input: {
+          ...baseInput,
+          ...(persistedInputMessages !== undefined
+            ? { messages: persistedInputMessages }
+            : {}),
+        },
+      } as RunStartedEvent;
+    };
+
+    const ensureRunStarted = (): void => {
+      if (!state.hasRunStarted) {
+        state.hasRunStarted = true;
+        pushCanonicalEvent(buildRunStartedEvent());
+      }
+    };
+
+    return from(
+      request.agent.runAgent(request.input, {
+        onEvent: ({ event }: { event: BaseEvent }) => {
+          if (event.type === EventType.RUN_STARTED) {
+            pushCanonicalEvent(buildRunStartedEvent(event as RunStartedEvent));
+            return;
+          }
+
+          ensureRunStarted();
+          pushCanonicalEvent(event);
+        },
+      }),
+    ).pipe(
+      catchError((error) => {
+        ensureRunStarted();
+        const errorEvent = {
+          type: EventType.RUN_ERROR,
+          message: error instanceof Error ? error.message : String(error),
+        } as BaseEvent;
+        pushCanonicalEvent(errorEvent);
+        return EMPTY;
+      }),
+      finalize(() => {
+        ensureRunStarted();
+        const appended = finalizeRunEvents(currentEvents, {
+          stopRequested: state.stopRequested,
+        });
+        for (const event of appended) {
+          channel.push(
+            "event",
+            this.createRunnerEventPayload(event, request, state),
+          );
+        }
+        this.removeThread(threadId);
+      }),
+    );
+  }
+
+  /**
+   * Tear down all resources for a thread: leave the channel,
+   * disconnect the per-run socket, and remove the thread state.
+   *
+   * Idempotent — safe to call multiple times for the same threadId
+   * (e.g. from join error handlers, finalize, and Observable teardown).
+   */
+  private removeThread(threadId: string): void {
+    const state = this.threads.get(threadId);
+    if (!state) {
+      return;
+    }
+
+    // Delete first so concurrent calls see the entry as already removed.
+    this.threads.delete(threadId);
+
+    try {
+      state.channel.leave();
+    } catch {
+      // Channel may already be closed/left.
+    }
+    try {
+      state.socket.disconnect();
+    } catch {
+      // Socket may already be disconnected.
+    }
+  }
+}

package/src/v2/runtime/runtime.ts

@@ -0,0 +1,260 @@
+import {
+  MaybePromise,
+  NonEmptyRecord,
+  RuntimeMode,
+  RUNTIME_MODE_SSE,
+  RUNTIME_MODE_INTELLIGENCE,
+  createLicenseChecker,
+  type LicenseChecker,
+} from "@copilotkit/shared";
+import { AbstractAgent } from "@ag-ui/client";
+import type { MCPClientConfig } from "@ag-ui/mcp-apps-middleware";
+import { A2UIMiddlewareConfig } from "@ag-ui/a2ui-middleware";
+import pkg from "../../../package.json";
+import type {
+  BeforeRequestMiddleware,
+  AfterRequestMiddleware,
+} from "./middleware";
+import { TranscriptionService } from "./transcription-service/transcription-service";
+import { AgentRunner } from "./runner/agent-runner";
+import { InMemoryAgentRunner } from "./runner/in-memory";
+import { IntelligenceAgentRunner } from "./runner/intelligence";
+import { CopilotKitIntelligence } from "./intelligence-platform";
+
+export const VERSION = pkg.version;
+
+interface BaseCopilotRuntimeMiddlewareOptions {
+  /** If set, middleware only applies to these named agents. Applies to all agents if omitted. */
+  agents?: string[];
+}
+
+export type McpAppsServerConfig = MCPClientConfig & {
+  /** Agent to bind this server to. If omitted, the server is available to all agents. */
+  agentId?: string;
+};
+
+export interface McpAppsConfig {
+  /** List of MCP server configurations. */
+  servers: McpAppsServerConfig[];
+}
+
+interface CopilotRuntimeMiddlewares {
+  /** Auto-apply A2UIMiddleware to agents at run time. */
+  a2ui?: BaseCopilotRuntimeMiddlewareOptions & A2UIMiddlewareConfig;
+  /** Auto-apply MCPAppsMiddleware to agents at run time. */
+  mcpApps?: McpAppsConfig;
+}
+
+interface BaseCopilotRuntimeOptions extends CopilotRuntimeMiddlewares {
+  /** Map of available agents (loaded lazily is fine). */
+  agents: MaybePromise<NonEmptyRecord<Record<string, AbstractAgent>>>;
+  /** Optional transcription service for audio processing. */
+  transcriptionService?: TranscriptionService;
+  /** Optional *before* middleware – callback function or webhook URL. */
+  beforeRequestMiddleware?: BeforeRequestMiddleware;
+  /** Optional *after* middleware – callback function or webhook URL. */
+  afterRequestMiddleware?: AfterRequestMiddleware;
+  /** Signed license token for server-side feature verification. Falls back to COPILOTKIT_LICENSE_TOKEN env var. */
+  licenseToken?: string;
+}
+
+export interface CopilotRuntimeUser {
+  id: string;
+}
+
+export type IdentifyUserCallback = (
+  request: Request,
+) => MaybePromise<CopilotRuntimeUser>;
+
+export interface CopilotSseRuntimeOptions extends BaseCopilotRuntimeOptions {
+  /** The runner to use for running agents in SSE mode. */
+  runner?: AgentRunner;
+  intelligence?: undefined;
+  generateThreadNames?: undefined;
+}
+
+export interface CopilotIntelligenceRuntimeOptions extends BaseCopilotRuntimeOptions {
+  /** Configures Intelligence mode for durable threads and realtime events. */
+  intelligence: CopilotKitIntelligence;
+  /** Resolves the authenticated user for intelligence requests. */
+  identifyUser: IdentifyUserCallback;
+  /** Auto-generate short names for newly created threads. */
+  generateThreadNames?: boolean;
+}
+
+export type CopilotRuntimeOptions =
+  | CopilotSseRuntimeOptions
+  | CopilotIntelligenceRuntimeOptions;
+
+export interface CopilotRuntimeLike {
+  agents: CopilotRuntimeOptions["agents"];
+  transcriptionService: CopilotRuntimeOptions["transcriptionService"];
+  beforeRequestMiddleware: CopilotRuntimeOptions["beforeRequestMiddleware"];
+  afterRequestMiddleware: CopilotRuntimeOptions["afterRequestMiddleware"];
+  runner: AgentRunner;
+  a2ui: CopilotRuntimeOptions["a2ui"];
+  mcpApps: CopilotRuntimeOptions["mcpApps"];
+  intelligence?: CopilotKitIntelligence;
+  identifyUser?: IdentifyUserCallback;
+  mode: RuntimeMode;
+  licenseChecker?: LicenseChecker;
+}
+
+export interface CopilotSseRuntimeLike extends CopilotRuntimeLike {
+  intelligence?: undefined;
+  mode: RUNTIME_MODE_SSE;
+}
+
+export interface CopilotIntelligenceRuntimeLike extends CopilotRuntimeLike {
+  intelligence: CopilotKitIntelligence;
+  identifyUser: IdentifyUserCallback;
+  generateThreadNames: boolean;
+  mode: RUNTIME_MODE_INTELLIGENCE;
+}
+
+abstract class BaseCopilotRuntime implements CopilotRuntimeLike {
+  public agents: CopilotRuntimeOptions["agents"];
+  public transcriptionService: CopilotRuntimeOptions["transcriptionService"];
+  public beforeRequestMiddleware: CopilotRuntimeOptions["beforeRequestMiddleware"];
+  public afterRequestMiddleware: CopilotRuntimeOptions["afterRequestMiddleware"];
+  public runner: AgentRunner;
+  public a2ui: CopilotRuntimeOptions["a2ui"];
+  public mcpApps: CopilotRuntimeOptions["mcpApps"];
+  public licenseChecker?: LicenseChecker;
+
+  abstract readonly intelligence?: CopilotKitIntelligence;
+  abstract readonly mode: RuntimeMode;
+
+  constructor(options: BaseCopilotRuntimeOptions, runner: AgentRunner) {
+    const {
+      agents,
+      transcriptionService,
+      beforeRequestMiddleware,
+      afterRequestMiddleware,
+      a2ui,
+      mcpApps,
+    } = options;
+
+    this.agents = agents;
+    this.transcriptionService = transcriptionService;
+    this.beforeRequestMiddleware = beforeRequestMiddleware;
+    this.afterRequestMiddleware = afterRequestMiddleware;
+    this.a2ui = a2ui;
+    this.mcpApps = mcpApps;
+    this.runner = runner;
+    this.licenseChecker = createLicenseChecker(options.licenseToken);
+  }
+}
+
+export class CopilotSseRuntime
+  extends BaseCopilotRuntime
+  implements CopilotSseRuntimeLike
+{
+  readonly intelligence = undefined;
+  readonly mode = RUNTIME_MODE_SSE;
+
+  constructor(options: CopilotSseRuntimeOptions) {
+    super(options, options.runner ?? new InMemoryAgentRunner());
+  }
+}
+
+export class CopilotIntelligenceRuntime
+  extends BaseCopilotRuntime
+  implements CopilotIntelligenceRuntimeLike
+{
+  readonly intelligence: CopilotKitIntelligence;
+  readonly identifyUser: IdentifyUserCallback;
+  readonly generateThreadNames: boolean;
+  readonly mode = RUNTIME_MODE_INTELLIGENCE;
+
+  constructor(options: CopilotIntelligenceRuntimeOptions) {
+    super(
+      options,
+      new IntelligenceAgentRunner({
+        url: options.intelligence.ɵgetRunnerWsUrl(),
+        authToken: options.intelligence.ɵgetRunnerAuthToken(),
+      }),
+    );
+    this.intelligence = options.intelligence;
+    this.identifyUser = options.identifyUser;
+    this.generateThreadNames = options.generateThreadNames ?? true;
+  }
+}
+
+function hasIntelligenceOptions(
+  options: CopilotRuntimeOptions,
+): options is CopilotIntelligenceRuntimeOptions {
+  return "intelligence" in options && !!options.intelligence;
+}
+
+export function isIntelligenceRuntime(
+  runtime: CopilotRuntimeLike,
+): runtime is CopilotIntelligenceRuntimeLike {
+  return runtime.mode === RUNTIME_MODE_INTELLIGENCE && !!runtime.intelligence;
+}
+
+/**
+ * Compatibility shim that preserves the legacy `CopilotRuntime` entrypoint.
+ * New code should prefer `CopilotSseRuntime` or `CopilotIntelligenceRuntime`.
+ */
+export class CopilotRuntime implements CopilotRuntimeLike {
+  private delegate: CopilotRuntimeLike;
+
+  constructor(options: CopilotRuntimeOptions) {
+    this.delegate = hasIntelligenceOptions(options)
+      ? new CopilotIntelligenceRuntime(options)
+      : new CopilotSseRuntime(options);
+  }
+
+  get agents(): CopilotRuntimeOptions["agents"] {
+    return this.delegate.agents;
+  }
+
+  get transcriptionService(): CopilotRuntimeOptions["transcriptionService"] {
+    return this.delegate.transcriptionService;
+  }
+
+  get beforeRequestMiddleware(): CopilotRuntimeOptions["beforeRequestMiddleware"] {
+    return this.delegate.beforeRequestMiddleware;
+  }
+
+  get afterRequestMiddleware(): CopilotRuntimeOptions["afterRequestMiddleware"] {
+    return this.delegate.afterRequestMiddleware;
+  }
+
+  get runner(): AgentRunner {
+    return this.delegate.runner;
+  }
+
+  get a2ui(): CopilotRuntimeOptions["a2ui"] {
+    return this.delegate.a2ui;
+  }
+
+  get mcpApps(): CopilotRuntimeOptions["mcpApps"] {
+    return this.delegate.mcpApps;
+  }
+
+  get intelligence(): CopilotKitIntelligence | undefined {
+    return this.delegate.intelligence;
+  }
+
+  get generateThreadNames(): boolean | undefined {
+    return isIntelligenceRuntime(this.delegate)
+      ? this.delegate.generateThreadNames
+      : undefined;
+  }
+
+  get identifyUser(): IdentifyUserCallback | undefined {
+    return isIntelligenceRuntime(this.delegate)
+      ? this.delegate.identifyUser
+      : undefined;
+  }
+
+  get mode(): RuntimeMode {
+    return this.delegate.mode;
+  }
+
+  get licenseChecker() {
+    return this.delegate.licenseChecker;
+  }
+}