@pipedog/core 0.0.0 → 0.1.0

package/dist/processors/aggregators/gated_llm_context.js

@@ -0,0 +1,86 @@
+// Copyright (c) 2024-2026, Daily
+// Copyright (c) 2026, pipedog contributors
+// SPDX-License-Identifier: BSD-2-Clause
+//
+// Ported from pipecat: src/pipecat/processors/aggregators/gated_llm_context.py
+import { CancelFrame, EndFrame, LLMContextFrame, StartFrame, } from "../../frames/frames.js";
+import { FrameProcessor, } from "../frame_processor.js";
+/**
+ * Aggregator that gates LLM context frames until notified.
+ *
+ * This aggregator captures LLM context frames and holds them until a notifier
+ * signals that they can be released. This is useful for controlling the flow
+ * of context frames based on external conditions or timing.
+ */
+export class GatedLLMContextAggregator extends FrameProcessor {
+    _notifier;
+    _startOpen;
+    _lastContextFrame = null;
+    _gateTask = null;
+    /** Initialize the gated context aggregator. */
+    constructor(options) {
+        super(options);
+        this._notifier = options.notifier;
+        this._startOpen = options.startOpen ?? false;
+    }
+    /**
+     * Process incoming frames, gating LLM context frames.
+     *
+     * @param frame The frame to process.
+     * @param direction The direction of frame flow in the pipeline.
+     */
+    async processFrame(frame, direction) {
+        await super.processFrame(frame, direction);
+        if (frame instanceof StartFrame) {
+            await this.pushFrame(frame);
+            await this._start();
+        }
+        if (frame instanceof EndFrame || frame instanceof CancelFrame) {
+            await this._stop();
+            await this.pushFrame(frame);
+        }
+        else if (frame instanceof LLMContextFrame) {
+            if (this._startOpen) {
+                this._startOpen = false;
+                await this.pushFrame(frame, direction);
+            }
+            else {
+                this._lastContextFrame = frame;
+            }
+        }
+        else {
+            // NOTE: as upstream, a StartFrame also reaches this branch and is pushed
+            // a second time.
+            await this.pushFrame(frame, direction);
+        }
+    }
+    /** Release resources held by the aggregator. */
+    async cleanup() {
+        await super.cleanup();
+        await this._stop();
+    }
+    /** Start the gate task handler. */
+    async _start() {
+        if (!this._gateTask) {
+            this._gateTask = this.createTask(() => this._gateTaskHandler(), "_gate_task_handler");
+        }
+    }
+    /** Stop the gate task handler. */
+    async _stop() {
+        if (this._gateTask) {
+            await this.cancelTask(this._gateTask);
+            this._gateTask = null;
+        }
+    }
+    /** Handle the gating logic by waiting for notifications and releasing frames. */
+    async _gateTaskHandler() {
+        while (true) {
+            await this._notifier.wait();
+            if (this._lastContextFrame) {
+                await this.pushFrame(this._lastContextFrame);
+                this._lastContextFrame = null;
+            }
+        }
+    }
+}
+//# sourceMappingURL=gated_llm_context.js.map

package/dist/processors/aggregators/index.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=index.d.ts.map

package/dist/processors/aggregators/index.js

@@ -0,0 +1,8 @@
+// Copyright (c) 2024-2026, Daily
+// Copyright (c) 2026, pipedog contributors
+// SPDX-License-Identifier: BSD-2-Clause
+//
+// Ported from pipecat: src/pipecat/processors/aggregators/__init__.py
+// The upstream package initializer is empty; import the modules directly.
+export {};
+//# sourceMappingURL=index.js.map

package/dist/processors/aggregators/llm_context.d.ts

@@ -0,0 +1,370 @@
+/**
+ * Universal LLM context management for LLM services in Pipecat.
+ *
+ * Context contents are represented in a universal format (based on OpenAI)
+ * that supports a union of known Pipecat LLM service functionality.
+ *
+ * Whenever an LLM service needs to access context, it does a just-in-time
+ * translation from this universal context into whatever format it needs,
+ * using a service-specific adapter.
+ */
+import type { DirectFunction } from "../../adapters/schemas/direct_function.ts";
+import { FunctionSchema } from "../../adapters/schemas/function_schema.ts";
+import { ToolsSchema } from "../../adapters/schemas/tools_schema.ts";
+import type { AudioRawFrame, FileSourceType } from "../../frames/frames.ts";
+import { type NotGiven } from "../../utils/types.ts";
+export { isGiven, NOT_GIVEN, NotGiven } from "../../utils/types.ts";
+/**
+ * A message in the universal (OpenAI-based) context format: an object with a
+ * `role` and, usually, a `content` that is a string or a list of content parts.
+ * Keys are the wire keys (`tool_call_id`, `tool_calls`, ...), unchanged.
+ */
+export interface LLMStandardMessage {
+    role: string;
+    content?: any;
+    [key: string]: any;
+}
+/** The tool choice strategy, in OpenAI's `tool_choice` format. */
+export type LLMContextToolChoice = "none" | "auto" | "required" | {
+    type: "function";
+    function: {
+        name: string;
+    };
+} | Record<string, any>;
+/**
+ * A container for a context message that is specific to a particular LLM service.
+ *
+ * Enables the use of service-specific message types while maintaining
+ * compatibility with the universal LLM context format.
+ */
+export declare class LLMSpecificMessage {
+    llm: string;
+    message: unknown;
+    constructor(options: {
+        llm: string;
+        message: unknown;
+    });
+}
+export type LLMContextMessage = LLMStandardMessage | LLMSpecificMessage;
+/** The tools accepted by LLMContext: a ToolsSchema, a plain list, or NOT_GIVEN. */
+export type LLMContextTools = ToolsSchema | (FunctionSchema | DirectFunction)[] | NotGiven;
+/**
+ * Return the text of a standard context message, joining the text parts of
+ * list content.
+ *
+ * @param message A context message.
+ * @returns The text, or `""` for a message in a service's own format or one
+ *   that carries no text.
+ */
+export declare function standardMessageText(message: LLMContextMessage): string;
+export interface LLMContextOptions {
+    /** Initial list of conversation messages. */
+    messages?: LLMContextMessage[] | null;
+    /**
+     * Available tools for the LLM to use. May be a `ToolsSchema` or a plain
+     * list of direct functions and/or `FunctionSchema` objects (normalized to a
+     * `ToolsSchema` internally). Any tool that carries a handler (a direct
+     * function, or a `FunctionSchema` with its `handler` set) is registered
+     * with the LLM service automatically, so no separate `registerFunction`
+     * call is needed.
+     */
+    tools?: LLMContextTools;
+    /** Tool selection strategy for the LLM. */
+    toolChoice?: LLMContextToolChoice | NotGiven;
+}
+/**
+ * Manages conversation context for LLM interactions.
+ *
+ * Handles message history, tool definitions, tool choices, and multimedia
+ * content for LLM conversations. Provides methods for message manipulation,
+ * and content formatting.
+ */
+export declare class LLMContext {
+    protected _messages: LLMContextMessage[];
+    protected _tools: ToolsSchema | NotGiven;
+    protected _toolChoice: LLMContextToolChoice | NotGiven;
+    constructor(options?: LLMContextOptions);
+    /**
+     * Create a context message containing an image URL.
+     *
+     * @param options.role The role of this message (defaults to "user").
+     * @param options.url The URL of the image.
+     * @param options.text Optional text to include with the image.
+     */
+    static createImageUrlMessage(options: {
+        role?: string;
+        url: string;
+        text?: string | null;
+    }): LLMContextMessage;
+    /**
+     * Create a context message containing an image.
+     *
+     * @param options.role The role of this message (defaults to "user").
+     * @param options.format Image format (e.g. 'RGB', 'RGBA', or, if already
+     *   encoded, the MIME type like 'image/jpeg').
+     * @param options.size Image dimensions as [width, height].
+     * @param options.image Raw image bytes.
+     * @param options.text Optional text to include with the image.
+     */
+    static createImageMessage(options: {
+        role?: string;
+        format: string;
+        size: [number, number];
+        image: Uint8Array;
+        text?: string | null;
+    }): Promise<LLMContextMessage>;
+    /**
+     * Create a context message containing a file.
+     *
+     * @param options.role The role of this message (defaults to "user").
+     * @param options.type The type of the file.
+     * @param options.format MIME type of the file (e.g. "application/pdf").
+     * @param options.file Base64 data URL (`data:<mime>;base64,...`) or plain URL string.
+     * @param options.name Optional name of the file.
+     * @param options.text Optional text to include with the file.
+     */
+    static createFileMessage(options: {
+        role?: string;
+        type: FileSourceType;
+        format: string;
+        file: string;
+        name?: string | null;
+        text?: string | null;
+    }): Promise<LLMContextMessage>;
+    /**
+     * Create a context message containing an image or file URL.
+     *
+     * @param options.role The role of this message (defaults to "user").
+     * @param options.format The MIME type of the file (e.g. 'application/pdf').
+     * @param options.url The URL of the image or file.
+     * @param options.filename Optional name of the file.
+     * @param options.text Optional text to include with the image or file.
+     */
+    static createFileUrlMessage(options: {
+        role?: string;
+        format: string;
+        url: string;
+        filename?: string | null;
+        text?: string | null;
+    }): LLMContextMessage;
+    /**
+     * Create a context message containing audio.
+     *
+     * @param options.role The role of this message (defaults to "user").
+     * @param options.audioFrames List of audio frame objects to include.
+     * @param options.text Optional text to include with the audio.
+     */
+    static createAudioMessage(options: {
+        role?: string;
+        audioFrames: AudioRawFrame[];
+        text?: string;
+    }): Promise<LLMContextMessage>;
+    /**
+     * Get the current messages list.
+     *
+     * NOTE: This is equivalent to calling `getMessages()` with no filter. If
+     * you want to filter out LLM-specific messages that don't pertain to your
+     * LLM, use `getMessages()` directly.
+     *
+     * @returns List of conversation messages.
+     */
+    get messages(): LLMContextMessage[];
+    /**
+     * Get the current messages list.
+     *
+     * @param llmSpecificFilter Optional filter to return LLM-specific messages
+     *   for the given LLM, in addition to the standard messages. If messages end
+     *   up being filtered, an error will be logged; this is intended to catch
+     *   accidental use of incompatible LLM-specific messages.
+     * @param options.truncateLargeValues If true, return deep copies of
+     *   messages with large values shortened. For standard messages, known
+     *   binary data (base64-encoded images, audio) is replaced with short
+     *   placeholders. For LLM-specific messages, long string values are
+     *   truncated.
+     * @returns List of conversation messages.
+     */
+    getMessages(llmSpecificFilter?: string | null, options?: {
+        truncateLargeValues?: boolean;
+    }): LLMContextMessage[];
+    /**
+     * Return deep copies of messages with large values replaced by placeholders.
+     *
+     * For standard (universal-format) messages, the following known binary
+     * patterns are replaced with short placeholders:
+     *
+     * - `image_url` items with `data:image/...` base64 URLs
+     * - `input_audio` items with `input_audio.data` or `audio` fields
+     * - `audio` items with an `audio` field
+     * - Top-level messages with a `mime_type` starting with `image/`
+     *
+     * For `LLMSpecificMessage` instances, long string values are truncated
+     * since the internal structure is provider-specific.
+     */
+    protected static _truncateLargeValuesFromMessages(messages: LLMContextMessage[]): LLMContextMessage[];
+    /**
+     * Recursively truncate long strings in a nested structure.
+     *
+     * Preserves the structure of objects and lists while truncating any string
+     * values that exceed `maxLength`.
+     *
+     * @param value The value to process (object, list, string, or other).
+     * @param options.maxLength Strings longer than this are truncated.
+     * @returns A copy of the structure with long strings truncated.
+     */
+    protected static _truncateLongStrings(value: unknown, options?: {
+        maxLength?: number;
+    }): unknown;
+    /**
+     * Get the tools list.
+     *
+     * @returns Tools list.
+     */
+    get tools(): ToolsSchema | NotGiven;
+    /**
+     * Get the current tool choice setting.
+     *
+     * @returns The tool choice configuration.
+     */
+    get toolChoice(): LLMContextToolChoice | NotGiven;
+    /**
+     * Add a single message to the context.
+     *
+     * @param message The message to add to the conversation history.
+     */
+    addMessage(message: LLMContextMessage): void;
+    /**
+     * Add multiple messages to the context.
+     *
+     * @param messages List of messages to add to the conversation history.
+     */
+    addMessages(messages: LLMContextMessage[]): void;
+    /**
+     * Replace all messages in the context.
+     *
+     * @param messages New list of messages to replace the current history.
+     */
+    setMessages(messages: LLMContextMessage[]): void;
+    /**
+     * Transform the current messages using the provided function.
+     *
+     * @param transform A function that takes the current list of messages and
+     *   returns a modified list of messages to set in the context.
+     */
+    transformMessages(transform: (messages: LLMContextMessage[]) => LLMContextMessage[]): void;
+    /**
+     * Set the available tools for the LLM.
+     *
+     * @param tools A ToolsSchema, a plain list of direct functions and/or
+     *   `FunctionSchema` objects, or NOT_GIVEN to disable tools.
+     */
+    setTools(tools?: LLMContextTools): void;
+    /**
+     * Set the tool choice configuration.
+     *
+     * @param toolChoice Tool selection strategy for the LLM.
+     */
+    setToolChoice(toolChoice: LLMContextToolChoice | NotGiven): void;
+    /**
+     * Add a message containing an image frame.
+     *
+     * @param options.format Image format (e.g. 'RGB', 'RGBA', or, if already
+     *   encoded, the MIME type like 'image/jpeg').
+     * @param options.size Image dimensions as [width, height].
+     * @param options.image Raw image bytes.
+     * @param options.text Optional text to include with the image.
+     * @param options.role The role of this message (defaults to "user").
+     */
+    addImageFrameMessage(options: {
+        format: string;
+        size: [number, number];
+        image: Uint8Array;
+        text?: string | null;
+        role?: string;
+    }): Promise<void>;
+    /**
+     * Add a message containing a file frame.
+     *
+     * @param options.type File source type.
+     * @param options.format MIME type of the file (e.g. 'application/pdf').
+     * @param options.file Base64 data URL (`data:<mime>;base64,...`) or plain
+     *   URL string. URL fetching for providers that require bytes is handled by
+     *   the provider adapter during message conversion.
+     * @param options.text Optional text to accompany the file.
+     * @param options.name Optional file name.
+     * @param options.role Message role (defaults to 'user').
+     */
+    addFileFrameMessage(options: {
+        type: FileSourceType;
+        format: string;
+        file: string;
+        text?: string | null;
+        name?: string | null;
+        role?: string;
+    }): Promise<void>;
+    /**
+     * Add a message containing audio frames.
+     *
+     * @param options.audioFrames List of audio frame objects to include.
+     * @param options.text Optional text to include with the audio.
+     */
+    addAudioFramesMessage(options: {
+        audioFrames: AudioRawFrame[];
+        text?: string;
+    }): Promise<void>;
+    /**
+     * Remove the oldest file or image message added since the model's last
+     * successful reply.
+     *
+     * Best-effort cleanup for an LLM service to call once it's determined (from
+     * a provider-specific error) that a request was rejected due to unsupported
+     * or malformed file content, e.g. an unsupported MIME type, corrupt bytes,
+     * or an image whose bytes don't match its declared media type. Removing it
+     * prevents the context from getting permanently stuck retrying the same
+     * failure. Images count as files here: both are discrete client-supplied
+     * attachments a provider can reject the same way. Audio content
+     * deliberately doesn't: audio-in-context flows append audio as ordinary
+     * conversation every turn, so removing the oldest audio message on an
+     * unrelated rejection would discard real conversation rather than a bad
+     * attachment.
+     *
+     * Providers don't say which message or content item triggered the
+     * rejection, so there's no way to identify the exact culprit; this is a
+     * heuristic, not a precise fix. An assistant message proves everything
+     * before it was already accepted by a prior successful completion
+     * (assistant messages are only ever appended after a completion succeeds),
+     * so only file messages added since the last one are candidates; a file
+     * from an earlier, already-successful turn is never at risk. If several
+     * files are candidates (e.g. sent in quick succession before either went
+     * through a completion), only the oldest is removed: if it wasn't the actual
+     * culprit, the next retry fails again and removes the next-oldest,
+     * converging on the bad file within a few retries without ever knowing
+     * which one the provider meant.
+     *
+     * @returns True if a file or image message was found and removed.
+     */
+    removeInvalidFileMessage(): boolean;
+    /**
+     * Normalize and validate the given tools.
+     *
+     * A plain list of direct functions and/or `FunctionSchema` objects is
+     * wrapped in a `ToolsSchema`.
+     *
+     * @param tools The tools to normalize: a `ToolsSchema`, a list of direct
+     *   functions and/or `FunctionSchema` objects, or `NOT_GIVEN`.
+     * @param options.allowProviderTools If true, a list that isn't entirely
+     *   standard tools (direct functions / `FunctionSchema` objects) is taken to
+     *   be already-formatted, provider-native tools and returned unchanged
+     *   rather than throwing. For callers whose tools parameter accepts
+     *   provider-native tools alongside standard ones.
+     * @throws TypeError If tools aren't a `ToolsSchema`, list, or `NOT_GIVEN`,
+     *   or, unless `allowProviderTools` is set, if a list contains anything
+     *   other than standard tools.
+     */
+    static _normalizeAndValidateTools(tools: LLMContextTools, options?: {
+        allowProviderTools?: false;
+    }): ToolsSchema | NotGiven;
+    static _normalizeAndValidateTools(tools: ToolsSchema | unknown[] | NotGiven, options: {
+        allowProviderTools: true;
+    }): ToolsSchema | unknown[] | NotGiven;
+}
+//# sourceMappingURL=llm_context.d.ts.map