@pipedog/core 0.0.0 → 0.1.0

package/dist/pipeline/llm_switcher.js

@@ -0,0 +1,107 @@
+// Copyright (c) 2024-2026, Daily
+// Copyright (c) 2026, pipedog contributors
+// SPDX-License-Identifier: BSD-2-Clause
+//
+// Ported from pipecat: src/pipecat/pipeline/llm_switcher.py
+/** LLM switcher for switching between different LLMs at runtime, with different switching strategies. */
+import { LLMContextFrame } from "../frames/frames.js";
+import { ServiceSwitcher, } from "./service_switcher.js";
+/**
+ * A pipeline that switches between different LLMs at runtime.
+ *
+ * @example
+ * const llmSwitcher = new LLMSwitcher([openaiLlm, anthropicLlm]);
+ */
+export class LLMSwitcher extends ServiceSwitcher {
+    /**
+     * Initialize the service switcher with a list of LLMs and a switching strategy.
+     *
+     * @param llms List of LLM services to switch between.
+     * @param options.strategyType The strategy class to use for switching
+     *   between LLMs. Defaults to `ServiceSwitcherStrategyManual`.
+     */
+    constructor(llms, options = {}) {
+        super(llms, options);
+    }
+    /**
+     * Process a frame, syncing context tool handlers on all member LLMs.
+     *
+     * On an `LLMContextFrame`, the handlers advertised in the context are synced
+     * on every member LLM — active or not — so that tools listed via
+     * `new LLMContext({ tools: [...] })` keep working across service switches.
+     *
+     * This is needed because member LLMs sit behind per-branch filters: only the
+     * active LLM receives the context frame and would otherwise sync its
+     * handlers, leaving inactive LLMs out of step with the advertised tools.
+     *
+     * @param frame The frame to process.
+     * @param direction The direction of frame flow.
+     */
+    async processFrame(frame, direction) {
+        await super.processFrame(frame, direction);
+        if (frame instanceof LLMContextFrame) {
+            this._syncRegisteredToolHandlers(frame.context.tools);
+        }
+    }
+    /**
+     * Sync the context's tool handlers on every member LLM.
+     *
+     * @param tools The advertised tools whose handlers should be synced on all
+     *   member LLMs.
+     */
+    _syncRegisteredToolHandlers(tools) {
+        for (const llm of this.llms) {
+            // Protected on LLMService; the switcher drives its members' sync, as upstream does.
+            llm._syncRegisteredToolHandlers(tools);
+        }
+    }
+    /** The list of LLMs managed by this switcher. */
+    get llms() {
+        return this.services;
+    }
+    /** The currently active LLM. */
+    get activeLlm() {
+        return this.strategy.activeService;
+    }
+    /**
+     * Run a one-shot, out-of-band (i.e. out-of-pipeline) inference with the
+     * given LLM context, using the currently active LLM.
+     *
+     * @param context The LLM context containing conversation history.
+     * @param options Additional options forwarded to the active LLM's
+     *   `runInference` (e.g. maxTokens, systemInstruction).
+     * @returns The LLM's response as a string, or null if no response is generated.
+     */
+    async runInference(context, options = {}) {
+        if (this.activeLlm) {
+            return await this.activeLlm.runInference(context, options);
+        }
+        return null;
+    }
+    /**
+     * Register a function handler for LLM function calls, on all LLMs, active or not.
+     *
+     * @param functionName The name of the function to handle. Use null to handle
+     *   all function calls with a catch-all handler.
+     * @param handler The function handler. Should accept a single
+     *   FunctionCallParams parameter.
+     * @param options.cancelOnInterruption Whether to cancel this function call
+     *   when an interruption occurs. Defaults to null (fall back to the
+     *   `toolOptions()` value on the handler, then to true).
+     * @param options.timeoutSecs Optional timeout in seconds for the function call.
+     * @param options.cancellableByLlm Whether the LLM may cancel this call while
+     *   it runs, through the `cancel_<name>` tool advertised alongside it. Pair
+     *   it with `cancelOnInterruption: false`. Defaults to null (fall back to the
+     *   `toolOptions()` value, then to false).
+     */
+    registerFunction(functionName, handler, options = {}) {
+        for (const llm of this.llms) {
+            llm.registerFunction(functionName, handler, {
+                cancelOnInterruption: options.cancelOnInterruption ?? null,
+                timeoutSecs: options.timeoutSecs ?? null,
+                cancellableByLlm: options.cancellableByLlm ?? null,
+            });
+        }
+    }
+}
+//# sourceMappingURL=llm_switcher.js.map

package/dist/pipeline/llm_with_backend.d.ts

@@ -0,0 +1,435 @@
+/**
+ * An LLM with a backend: a conversational frontend delegating the rest.
+ *
+ * The frontend is any LLM service, text or speech-to-speech, and holds the
+ * conversation. The backend is a `BackendLLMWorker` running a heavier model
+ * with the tools, and does the work the frontend hands off. `LLMWithBackend`
+ * wraps the frontend so the pair drops into a pipeline where an LLM goes, and
+ * installs the tools that join them.
+ *
+ * The two exchange messages. The frontend's `delegate` tool puts a message to
+ * the backend and returns at once; what the backend has to say comes back as a
+ * stream of outputs, each appended to the frontend's conversation as a message
+ * marked `Backend:`, spoken or silent as the backend's flag says. A second
+ * request joins the first, and a correction changes or stops it. How a request
+ * is worded is the `BackendRequestStrategy`'s business, and the
+ * `BackendConnector` owns the tools, the session with the backend, and how
+ * each output is rendered into the conversation.
+ */
+import { Event, type Task } from "@pipedog/asyncio";
+import { FunctionSchema } from "../adapters/schemas/function_schema.ts";
+import { LLMMessagesAppendFrame } from "../frames/frames.ts";
+import { type LLMContextMessage } from "../processors/aggregators/llm_context.ts";
+import type { FrameProcessorSetup } from "../processors/frame_processor.ts";
+import { type FunctionCallParams, type LLMService } from "../services/llm_service.ts";
+import type { BaseWorker } from "../workers/base_worker.ts";
+import { _BackendSession, type BackendEvent, BackendLLMWorker, BackendOutput, BackendToolCall } from "../workers/llm/backend_llm_worker.ts";
+import { Pipeline } from "./pipeline.ts";
+type AnyLLMService = LLMService<any>;
+/** Name of the tool the frontend calls to delegate. */
+export declare const DELEGATE_TOOL_NAME = "delegate";
+/** How a backend output is marked in the frontend's conversation. */
+export declare const BACKEND_MESSAGE_PREFIX = "Backend: ";
+/** How a backend reasoning summary is marked in the frontend's conversation. */
+export declare const BACKEND_THOUGHT_PREFIX = "Backend (thinking): ";
+/**
+ * How a backend note, text its model wrote without the mark that asks for the
+ * user to be told it, is marked in the frontend's conversation.
+ */
+export declare const BACKEND_NOTE_PREFIX = "Backend (note): ";
+/** How a function call the backend is making is marked in the frontend's conversation. */
+export declare const BACKEND_WORKING_PREFIX = "Backend (working): ";
+/** Every mark a rendered backend message can open with. */
+export declare const BACKEND_PREFIXES: readonly string[];
+/**
+ * Carried on each spoken backend message, so the frontend relays it on
+ * whatever turn it lands on, a user's turn included, where the standing
+ * instruction alone leaves it unsaid.
+ */
+export declare const RELAY_NOTE: string;
+/** Where the request strategy's own guidance goes in `FRONTEND_PAIRING_INSTRUCTION`. */
+export declare const STRATEGY_PLACEHOLDER = "{strategy}";
+/**
+ * How the frontend's model works with a backend, appended to its system
+ * instruction: when and how to delegate, that to the user there is one
+ * assistant, what to do after delegating, and what to do with the backend's
+ * messages. The request strategy's guidance (how the request is worded) goes
+ * where `STRATEGY_PLACEHOLDER` stands, inside the delegating paragraph. The
+ * frontend's own system instruction says what the backend is for.
+ * `new BackendConnector({ frontendPairingInstruction })` replaces it.
+ *
+ * The order of the paragraphs matters, and the placeholder's place inside the
+ * first one: with the one-assistant paragraph first, or the strategy's
+ * guidance in a paragraph of its own, gpt-4.1 answers a user's "never mind"
+ * with "I won't make any changes" and no delegate call in nearly every run,
+ * where this order delegates the stop every time.
+ */
+export declare const FRONTEND_PAIRING_INSTRUCTION: string;
+/** The transcript strategy's guidance, in the pairing instruction's delegating paragraph. */
+export declare const TRANSCRIPT_FRONTEND_INSTRUCTION: string;
+/** How the transcript strategy describes the `delegate` tool. */
+export declare const TRANSCRIPT_TOOL_DESCRIPTION: string;
+/** The explicit strategy's guidance, in the pairing instruction's delegating paragraph. */
+export declare const EXPLICIT_FRONTEND_INSTRUCTION: string;
+/** How the explicit strategy describes the `delegate` tool. */
+export declare const EXPLICIT_TOOL_DESCRIPTION: string;
+export interface ConnectorContextOptions {
+    /** Name of the backend worker, local or registered elsewhere. */
+    backendName: string;
+    /** Whether the frontend is a speech-to-speech service. */
+    frontendIsRealtime: boolean;
+}
+/**
+ * What a connector needs to know about the layers it joins.
+ *
+ * - `backendName`: name of the backend worker, local or registered elsewhere.
+ * - `frontendIsRealtime`: whether the frontend is a speech-to-speech service,
+ *   whose context can lag the audio, which changes how a delegation words its
+ *   request.
+ */
+export declare class ConnectorContext {
+    backendName: string;
+    frontendIsRealtime: boolean;
+    constructor(options: ConnectorContextOptions);
+}
+/**
+ * Defines the `delegate` tool the frontend calls, and turns a call into the
+ * request the backend receives.
+ *
+ * The tool's parameters are this strategy's to declare, so the connector's
+ * handler has one signature and reads whatever was declared back from
+ * `params.arguments` in `composeRequest`.
+ */
+export declare class BackendRequestStrategy {
+    /** JSON-schema properties of the `delegate` tool; empty when it takes none. */
+    toolParameters: Record<string, unknown>;
+    /** Which of the properties are required. */
+    toolRequired: string[];
+    /**
+     * The strategy's guidance to the frontend model, if any: how the request is
+     * worded. It goes where the pairing instruction's `STRATEGY_PLACEHOLDER`
+     * stands.
+     */
+    frontendInstruction: string | null;
+    /** How the `delegate` tool is described to the frontend model. */
+    toolDescription: string;
+    /**
+     * Whether the strategy reads the conversation from the frontend's context,
+     * so it needs that context to be current when the `delegate` tool runs. A
+     * speech-to-speech frontend's context lags the audio, so the connector
+     * refuses such a strategy there.
+     */
+    requiresContext: boolean;
+    /**
+     * Turn a `delegate` call into the text put to the backend.
+     *
+     * @param params The call, carrying the arguments the tool declared and the
+     *   frontend's context.
+     * @returns The request text, or null when the call carries nothing new for
+     *   the backend, in which case nothing is sent.
+     */
+    composeRequest(_params: FunctionCallParams): Promise<string | null>;
+}
+export interface TranscriptBackendRequestStrategyOptions {
+    /**
+     * The strategy's guidance to the frontend model, placed in the pairing
+     * instruction: that a delegation hands over the whole conversation, unworded.
+     */
+    frontendInstruction?: string | null;
+    /** How the `delegate` tool is described. */
+    toolDescription?: string;
+    /** What the backend should do with the transcript, placed after it. */
+    requestInstruction?: string;
+}
+/**
+ * Hands the backend the conversation and lets it work out the request.
+ *
+ * The frontend model calls the tool with no arguments; the request is the
+ * transcript of what was said since the previous delegation (the whole
+ * conversation the first time), rendered by `_renderTranscriptRequest`. The
+ * backend's own messages in the conversation are left out: the backend
+ * already has what it said. The default for a text frontend, whose context is
+ * current when the tool runs.
+ *
+ * The cursor assumes the conversation accrues. A rewritten context or a failed
+ * delegation's turns are not re-sent; the user's next request carries what
+ * matters. A call that finds no new turn since the previous delegation sends
+ * nothing.
+ */
+export declare class TranscriptBackendRequestStrategy extends BackendRequestStrategy {
+    requiresContext: boolean;
+    protected _requestInstruction: string;
+    protected _delegatedThrough: number;
+    /** Initialize the strategy. */
+    constructor(options?: TranscriptBackendRequestStrategyOptions);
+    /** Render the turns since the previous delegation as the request. */
+    composeRequest(params: FunctionCallParams): Promise<string | null>;
+}
+export interface ExplicitBackendRequestStrategyOptions {
+    /**
+     * The strategy's guidance to the frontend model, placed in the pairing
+     * instruction: how to word the request.
+     */
+    frontendInstruction?: string | null;
+    /** How the `delegate` tool is described. */
+    toolDescription?: string;
+    /** What the backend should do with the request, placed after it. */
+    requestInstruction?: string;
+}
+/**
+ * Has the frontend model word the request itself.
+ *
+ * The tool takes a `request` argument, which is sent to the backend as it
+ * stands. The default for a speech-to-speech frontend, which responds to audio
+ * before the transcript of that audio reaches its context, so the turns that
+ * prompted a handoff may not be recorded when the tool runs.
+ */
+export declare class ExplicitBackendRequestStrategy extends BackendRequestStrategy {
+    toolParameters: Record<string, unknown>;
+    toolRequired: string[];
+    protected _requestInstruction: string;
+    /** Initialize the strategy. */
+    constructor(options?: ExplicitBackendRequestStrategyOptions);
+    /** Send the model's request as it stands, with the reporting expectation after it. */
+    composeRequest(params: FunctionCallParams): Promise<string | null>;
+}
+/** Whether a context message is one the connector rendered from a backend output. */
+export declare function _isBackendMessage(message: LLMContextMessage): boolean;
+export interface BackendConnectorOptions {
+    /** How a `delegate` call becomes the backend's request. Picked by frontend kind when omitted. */
+    requestStrategy?: BackendRequestStrategy | null;
+    /**
+     * How the model works with a backend, appended to its system instruction:
+     * `FRONTEND_PAIRING_INSTRUCTION` unless given. One of its own has to say
+     * what that one says: when and how to delegate, that to the user there is
+     * one assistant, what to do after delegating, and what to do with the
+     * backend's messages. The request strategy's guidance goes where
+     * `STRATEGY_PLACEHOLDER` stands, or after the text when it has none.
+     */
+    frontendPairingInstruction?: string;
+    /**
+     * Whether the frontend runs again as soon as a delegation is sent, which is
+     * where its acknowledgement comes from: a text model's tool-call turn
+     * carries no prose. Off for a frontend that announces a handoff itself.
+     * Defaults to true.
+     */
+    respondOnDelegate?: boolean;
+    /**
+     * How long the backend may take to acknowledge a message, including the
+     * wait for it to become ready. Defaults to 30.
+     */
+    timeoutSecs?: number | null;
+    /**
+     * Whether to send every exchange with the backend to the client as an RTVI
+     * server message (`type` `"llm-with-backend"`): each request, output,
+     * tool-call phase, error and idle. For debugging; a client's event log shows
+     * them. Defaults to false.
+     */
+    clientTrace?: boolean;
+}
+/**
+ * Joins the two layers: the tools the frontend calls, the session with the
+ * backend, and what each output becomes.
+ *
+ * A request strategy defines the `delegate` tool and composes the request. It
+ * may be given, or left to the connector to pick by frontend kind once bound:
+ * `TranscriptBackendRequestStrategy` for a text frontend,
+ * `ExplicitBackendRequestStrategy` for a speech-to-speech one, whose context
+ * lags the audio. A given strategy that reads the frontend's context
+ * (`BackendRequestStrategy.requiresContext`) is refused for a speech-to-speech
+ * frontend.
+ *
+ * The connector holds the session with the backend for the frontend's life,
+ * and turns each output the backend sends into a message appended to the
+ * frontend's conversation with `renderOutput`, run or not as the output's
+ * `prefersSpoken` flag says. The function calls the backend makes on the way
+ * are reported in the frontend's pipeline for clients to show; nothing else in
+ * the pipeline sees them.
+ *
+ * @example
+ * const connector = new BackendConnector({ respondOnDelegate: false });
+ */
+export declare class BackendConnector {
+    protected _requestStrategy: BackendRequestStrategy | null;
+    protected _frontendPairingInstruction: string;
+    protected _respondOnDelegate: boolean;
+    protected _timeoutSecs: number | null;
+    protected _clientTrace: boolean;
+    protected _context: ConnectorContext | null;
+    protected _tools: FunctionSchema[];
+    protected _session: _BackendSession | null;
+    protected _sessionOpen: Event;
+    /** Initialize the connector. */
+    constructor(options?: BackendConnectorOptions);
+    /** The request strategy in use. Available once bound. */
+    get requestStrategy(): BackendRequestStrategy;
+    /** The tools to install on the frontend. Available once bound. */
+    get tools(): FunctionSchema[];
+    /** The `delegate` tool. Available once bound. */
+    get tool(): FunctionSchema;
+    /**
+     * What is appended to the frontend's system instruction: the pairing
+     * instruction with the strategy's guidance in place. Available once bound.
+     */
+    get frontendInstruction(): string | null;
+    /** The session with the backend, while one is open. */
+    get session(): _BackendSession | null;
+    /**
+     * Settle the strategy for the layers being joined and build the tools.
+     *
+     * @param context The layers.
+     * @throws Error If the frontend is speech-to-speech and the strategy reads
+     *   the frontend's context, which lags the audio there.
+     */
+    bind(context: ConnectorContext): void;
+    /**
+     * Build the tools to install on the frontend: `delegate`.
+     *
+     * Override to install tools of another shape entirely; the service reads
+     * nothing else from the connector but `frontendInstruction`.
+     *
+     * @returns The tools, each carrying its handler.
+     */
+    buildTools(): FunctionSchema[];
+    /**
+     * Hold the session with the backend, delivering what it produces, until cancelled.
+     *
+     * @param worker The pipeline worker the frontend runs in.
+     * @param frontend The frontend service, which takes the deliveries.
+     */
+    runSession(worker: BaseWorker, frontend: AnyLLMService): Promise<void>;
+    /** The session, once it is open; waits for it up to the timeout. */
+    protected _openSession(): Promise<_BackendSession>;
+    /**
+     * Put the request a `delegate` call carries to the backend, and settle the call at once.
+     *
+     * @param params The `delegate` call.
+     */
+    delegate(params: FunctionCallParams): Promise<void>;
+    /**
+     * Deliver one thing the backend produced to the frontend.
+     *
+     * An output becomes a message appended to the frontend's conversation, which
+     * runs the frontend when the output asks to be spoken. A function call phase
+     * is reported as the `ExternalFunctionCall*Frame` for it, which the RTVI
+     * observer turns into function-call events. An error becomes a spoken
+     * message, so the user hears the work stopped. The appended messages are
+     * uninterruptible: one queued while the bot speaks waits behind that speech,
+     * and an interruption clears the speech but must leave the message to be
+     * recorded.
+     *
+     * @param frontend The frontend service.
+     * @param event What the backend produced.
+     */
+    deliver(frontend: AnyLLMService, event: BackendEvent): Promise<void>;
+    /** Send one exchange with the backend to the client, when tracing is on. */
+    protected _trace(frontend: AnyLLMService, data: Record<string, unknown>): Promise<void>;
+    /** Queue a message into the frontend's conversation, past any interruption on the way. */
+    protected _append(frontend: AnyLLMService, frame: LLMMessagesAppendFrame): Promise<void>;
+    /**
+     * Render a backend output as the message the frontend's conversation takes in.
+     *
+     * Override for another wording, role, or to leave some outputs out by
+     * returning null. The default marks the message so the frontend can tell it
+     * from the user's and its own, and so the transcript request strategy can
+     * leave it out of what it sends the backend: a spoken output as `Backend:`
+     * with `RELAY_NOTE` after it, a silent one as `Backend (note):`, a reasoning
+     * summary as `Backend (thinking):`.
+     *
+     * @param output The output.
+     * @returns The message, or null to deliver nothing for this output.
+     */
+    renderOutput(output: BackendOutput): LLMContextMessage | null;
+    /**
+     * Render a phase of a backend function call as a message the frontend's
+     * conversation takes in, silently.
+     *
+     * The default records each call as it starts, with its arguments, so the
+     * frontend can say what the backend is doing if asked; the other phases are
+     * not recorded. Override to record more, less, or nothing.
+     *
+     * @param call The phase of the call.
+     * @returns The message, or null to record nothing for this phase.
+     */
+    renderToolCall(call: BackendToolCall): LLMContextMessage | null;
+}
+export interface LLMWithBackendOptions {
+    /** The conversational LLM service, text or speech-to-speech. */
+    frontend: AnyLLMService;
+    /**
+     * The backend worker, or the name of one registered elsewhere (in the app,
+     * or in another process on a shared bus). A worker given here is added to
+     * the pipeline worker at setup.
+     */
+    backend: BackendLLMWorker | string;
+    /** How delegation works. A default `BackendConnector` when omitted. */
+    connector?: BackendConnector | null;
+}
+/**
+ * A conversational LLM, kept fast and light, with a backend that does the
+ * work it hands off.
+ *
+ * **Alpha.** The API is unstable: names, arguments, the strategies and the job
+ * contract are likely to change between releases.
+ *
+ * Put it where the LLM goes in a pipeline. It is a `Pipeline`, not an
+ * `LLMService`: functions, settings updates, event handlers and a
+ * `FlowManager` go on `frontend`. It wraps the frontend service, installs the
+ * connector's tools on it as built-in tools, appends the connector's guidance
+ * to the frontend's system instruction, adds a local backend worker to the
+ * pipeline worker so the app never wires it up, and holds the session with
+ * the backend for the pipeline's life. A built-in tool is sent on every
+ * inference beside whatever tools the frontend has, and never enters the
+ * context's tool set, so a tool change announced to the model never mentions
+ * it. The tools are the backend's; the frontend has `delegate` and no more,
+ * though a tool it must keep, in its context or configured on the service,
+ * stays.
+ *
+ * The guidance is the connector's pairing instruction (how to work with a
+ * backend: when and how to delegate, and what to do with the backend's
+ * messages) with the strategy's (how the request is worded) in place. The
+ * frontend's own system instruction is the place to say what the backend is
+ * for, in plain words, as the backend's own instruction does: no tool names,
+ * nothing to update when a tool changes.
+ *
+ * @example
+ * const llm = new LLMWithBackend({
+ *   frontend: new OpenAILLMService({ ... }),
+ *   backend: new BackendLLMWorker({
+ *     llm: new AnthropicLLMService({ ... }),
+ *     context: new LLMContext({ tools: [getWeather] }),
+ *   }),
+ * });
+ * const pipeline = new Pipeline([transport.input(), stt, userAgg, llm, tts, transport.output(), assistantAgg]);
+ */
+export declare class LLMWithBackend extends Pipeline {
+    protected _frontend: AnyLLMService;
+    protected _backend: BackendLLMWorker | string;
+    protected _connector: BackendConnector;
+    protected _sessionTask: Task<unknown> | null;
+    /**
+     * Initialize the service.
+     *
+     * @throws Error If the frontend service declines the role, or the
+     *   connector's request strategy reads the frontend's context and the
+     *   frontend is speech-to-speech.
+     */
+    constructor(options: LLMWithBackendOptions);
+    /** The frontend LLM service. */
+    get frontend(): AnyLLMService;
+    /** The backend worker, or its name. */
+    get backend(): BackendLLMWorker | string;
+    /** The connector joining the two layers. */
+    get connector(): BackendConnector;
+    /** The `delegate` tool installed on the frontend. */
+    get tool(): FunctionSchema;
+    /**
+     * Register a local backend worker as a child of the pipeline worker, set up
+     * the frontend, and open the session.
+     */
+    setup(setup: FrameProcessorSetup): Promise<void>;
+    /** Close the session with the backend, then clean up the frontend. */
+    cleanup(): Promise<void>;
+}
+export {};
+//# sourceMappingURL=llm_with_backend.d.ts.map