@pipedog/core 0.0.0 → 0.2.0

package/dist/pipeline/parallel_pipeline.d.ts

@@ -0,0 +1,83 @@
+/**
+ * Parallel pipeline implementation for concurrent frame processing.
+ *
+ * This module provides a parallel pipeline that processes frames through
+ * multiple sub-pipelines concurrently, with coordination for system frames and
+ * proper handling of pipeline lifecycle events.
+ */
+import { type Frame } from "../frames/frames.ts";
+import type { FrameDirection, FrameProcessor, FrameProcessorSetup } from "../processors/frame_processor.ts";
+import { BasePipeline } from "./base_pipeline.ts";
+import { Pipeline } from "./pipeline.ts";
+/**
+ * Pipeline that processes frames through multiple sub-pipelines concurrently.
+ *
+ * Creates multiple parallel processing branches from the provided processor
+ * lists, coordinating frame flow and ensuring proper synchronization of
+ * lifecycle events like EndFrames. Each branch runs independently while system
+ * frames are handled specially to maintain pipeline coordination.
+ */
+export declare class ParallelPipeline extends BasePipeline {
+    protected _pipelines: Pipeline[];
+    protected _seenIds: Set<number>;
+    protected _frameCounter: Map<number, number>;
+    protected _synchronizing: boolean;
+    protected _bufferedFrames: [Frame, FrameDirection][];
+    /**
+     * Initialize the parallel pipeline with processor lists.
+     *
+     * @param args Variable number of processor lists, each becoming a parallel branch.
+     * @throws Error If no processor lists are provided.
+     * @throws TypeError If any argument is not a list of processors.
+     */
+    constructor(...args: FrameProcessor[][]);
+    /**
+     * The list of sub-processors contained within this processor.
+     *
+     * Only compound processors (e.g. pipelines and parallel pipelines) have
+     * sub-processors. Non-compound processors will return an empty list.
+     */
+    get processors(): FrameProcessor[];
+    /**
+     * The list of entry processors for this processor.
+     *
+     * Entry processors are the first processors in a compound processor (e.g.
+     * pipelines, parallel pipelines). Note that pipelines can also be an entry
+     * processor as pipelines are processors themselves. Non-compound processors
+     * will simply return an empty list.
+     */
+    get entryProcessors(): FrameProcessor[];
+    /**
+     * Collect processors that can generate metrics from all parallel branches.
+     *
+     * @returns Frame processors that support metrics collection from all branches.
+     */
+    processorsWithMetrics(): FrameProcessor[];
+    /**
+     * Set up the parallel pipeline and all its branches.
+     *
+     * @param setup Configuration for frame processor setup.
+     */
+    setup(setup: FrameProcessorSetup): Promise<void>;
+    /** Clean up the parallel pipeline and all its branches. */
+    cleanup(): Promise<void>;
+    /**
+     * Process frames through all parallel branches with lifecycle coordination.
+     *
+     * @param frame The frame to process.
+     * @param direction The direction of frame flow.
+     */
+    processFrame(frame: Frame, direction: FrameDirection): Promise<void>;
+    /**
+     * Push frames while avoiding duplicates using frame ID tracking.
+     *
+     * During lifecycle frame synchronization, non-lifecycle frames are buffered
+     * to prevent them from escaping the parallel pipeline before all branches
+     * have finished processing the lifecycle frame.
+     */
+    protected _parallelPushFrame(frame: Frame, direction: FrameDirection): Promise<void>;
+    protected _pipelineSinkPushFrame(frame: Frame, direction: FrameDirection): Promise<void>;
+    /** Flush frames that were buffered during lifecycle frame synchronization. */
+    protected _flushBufferedFrames(): Promise<void>;
+}
+//# sourceMappingURL=parallel_pipeline.d.ts.map

package/dist/pipeline/parallel_pipeline.js

@@ -0,0 +1,200 @@
+// Copyright (c) 2024-2026, Daily
+// Copyright (c) 2026, pipedog contributors
+// SPDX-License-Identifier: BSD-2-Clause
+//
+// Ported from pipecat: src/pipecat/pipeline/parallel_pipeline.py
+/**
+ * Parallel pipeline implementation for concurrent frame processing.
+ *
+ * This module provides a parallel pipeline that processes frames through
+ * multiple sub-pipelines concurrently, with coordination for system frames and
+ * proper handling of pipeline lifecycle events.
+ */
+import { CancelFrame, EndFrame, StartFrame } from "../frames/frames.js";
+import { logger } from "../utils/logger.js";
+import { BasePipeline } from "./base_pipeline.js";
+import { Pipeline, PipelineSink, PipelineSource } from "./pipeline.js";
+/**
+ * Pipeline that processes frames through multiple sub-pipelines concurrently.
+ *
+ * Creates multiple parallel processing branches from the provided processor
+ * lists, coordinating frame flow and ensuring proper synchronization of
+ * lifecycle events like EndFrames. Each branch runs independently while system
+ * frames are handled specially to maintain pipeline coordination.
+ */
+export class ParallelPipeline extends BasePipeline {
+    _pipelines = [];
+    _seenIds = new Set();
+    _frameCounter = new Map();
+    _synchronizing = false;
+    _bufferedFrames = [];
+    /**
+     * Initialize the parallel pipeline with processor lists.
+     *
+     * @param args Variable number of processor lists, each becoming a parallel branch.
+     * @throws Error If no processor lists are provided.
+     * @throws TypeError If any argument is not a list of processors.
+     */
+    constructor(...args) {
+        // We don't set it to direct mode because we use frame pausing and that
+        // requires queues.
+        super();
+        if (args.length === 0) {
+            throw new Error("ParallelPipeline needs at least one argument");
+        }
+        logger.debug(`Creating ${this} pipelines`);
+        for (const processors of args) {
+            if (!Array.isArray(processors)) {
+                throw new TypeError(`ParallelPipeline argument ${processors} is not a list`);
+            }
+            const numPipelines = this._pipelines.length;
+            // We add a source before the pipeline and a sink after so we control
+            // the frames that are pushed upstream and downstream.
+            const source = new PipelineSource((frame, direction) => this._parallelPushFrame(frame, direction), { name: `${this}::Source${numPipelines}` });
+            const sink = new PipelineSink((frame, direction) => this._pipelineSinkPushFrame(frame, direction), { name: `${this}::Sink${numPipelines}` });
+            // Create pipeline
+            const pipeline = new Pipeline(processors, { source, sink });
+            this._pipelines.push(pipeline);
+        }
+        logger.debug(`Finished creating ${this} pipelines`);
+    }
+    //
+    // Frame processor
+    //
+    /**
+     * The list of sub-processors contained within this processor.
+     *
+     * Only compound processors (e.g. pipelines and parallel pipelines) have
+     * sub-processors. Non-compound processors will return an empty list.
+     */
+    get processors() {
+        return this._pipelines;
+    }
+    /**
+     * The list of entry processors for this processor.
+     *
+     * Entry processors are the first processors in a compound processor (e.g.
+     * pipelines, parallel pipelines). Note that pipelines can also be an entry
+     * processor as pipelines are processors themselves. Non-compound processors
+     * will simply return an empty list.
+     */
+    get entryProcessors() {
+        return this._pipelines;
+    }
+    /**
+     * Collect processors that can generate metrics from all parallel branches.
+     *
+     * @returns Frame processors that support metrics collection from all branches.
+     */
+    processorsWithMetrics() {
+        return this._pipelines.flatMap((p) => p.processorsWithMetrics());
+    }
+    /**
+     * Set up the parallel pipeline and all its branches.
+     *
+     * @param setup Configuration for frame processor setup.
+     */
+    async setup(setup) {
+        await super.setup(setup);
+        // This pipeline is set up, so let the StartFrame in: each branch holds it
+        // until its own setup is done.
+        this._setupFinished();
+        await this._setupProcessors(this._pipelines, setup);
+    }
+    /** Clean up the parallel pipeline and all its branches. */
+    async cleanup() {
+        await super.cleanup();
+        await this._cleanupProcessors(this._pipelines);
+    }
+    /**
+     * Process frames through all parallel branches with lifecycle coordination.
+     *
+     * @param frame The frame to process.
+     * @param direction The direction of frame flow.
+     */
+    async processFrame(frame, direction) {
+        await super.processFrame(frame, direction);
+        // Parallel pipeline synchronized frames.
+        //
+        // - StartFrame: If a fast branch completes first, processors in other
+        //   branches that haven't received StartFrame yet could receive other
+        //   frames before it, causing errors.
+        //
+        // - EndFrame: If EndFrame escapes from a fast branch, downstream
+        //   processors (e.g. output transport) begin shutting down while other
+        //   branches still have frames to flush, causing lost output.
+        //
+        // - CancelFrame: PipelineWorker waits for CancelFrame to reach the
+        //   pipeline sink. If it escapes from a fast branch while slower branches
+        //   are still running, the task considers cancellation complete
+        //   prematurely.
+        if (isSynchronizedFrame(frame)) {
+            this._frameCounter.set(frame.id, this._pipelines.length);
+            this._synchronizing = true;
+            await this.pauseProcessingSystemFrames();
+            await this.pauseProcessingFrames();
+        }
+        // Process frames in each of the sub-pipelines.
+        for (const p of this._pipelines) {
+            await p.queueFrame(frame, direction);
+        }
+    }
+    /**
+     * Push frames while avoiding duplicates using frame ID tracking.
+     *
+     * During lifecycle frame synchronization, non-lifecycle frames are buffered
+     * to prevent them from escaping the parallel pipeline before all branches
+     * have finished processing the lifecycle frame.
+     */
+    async _parallelPushFrame(frame, direction) {
+        if (!this._seenIds.has(frame.id)) {
+            this._seenIds.add(frame.id);
+            if (this._synchronizing) {
+                this._bufferedFrames.push([frame, direction]);
+            }
+            else {
+                await this.pushFrame(frame, direction);
+            }
+        }
+    }
+    async _pipelineSinkPushFrame(frame, direction) {
+        // Parallel pipeline synchronized frames.
+        if (isSynchronizedFrame(frame)) {
+            // Decrement counter.
+            let frameCounter = this._frameCounter.get(frame.id) ?? 0;
+            if (frameCounter > 0) {
+                frameCounter -= 1;
+                this._frameCounter.set(frame.id, frameCounter);
+            }
+            // Only push the frame when all pipelines have processed it.
+            if (frameCounter === 0) {
+                this._synchronizing = false;
+                // StartFrame should always go before any other frame.
+                if (frame instanceof StartFrame) {
+                    await this._parallelPushFrame(frame, direction);
+                    await this._flushBufferedFrames();
+                }
+                else {
+                    await this._flushBufferedFrames();
+                    await this._parallelPushFrame(frame, direction);
+                }
+                await this.resumeProcessingSystemFrames();
+                await this.resumeProcessingFrames();
+            }
+        }
+        else {
+            await this._parallelPushFrame(frame, direction);
+        }
+    }
+    /** Flush frames that were buffered during lifecycle frame synchronization. */
+    async _flushBufferedFrames() {
+        while (this._bufferedFrames.length > 0) {
+            const [frame, direction] = this._bufferedFrames.shift();
+            await this.pushFrame(frame, direction);
+        }
+    }
+}
+function isSynchronizedFrame(frame) {
+    return frame instanceof StartFrame || frame instanceof EndFrame || frame instanceof CancelFrame;
+}
+//# sourceMappingURL=parallel_pipeline.js.map

package/dist/pipeline/pipeline.d.ts

@@ -0,0 +1,126 @@
+/**
+ * Pipeline implementation for connecting and managing frame processors.
+ *
+ * Provides the main Pipeline class that connects frame processors in sequence
+ * and manages frame flow between them, along with helper classes for pipeline
+ * source and sink operations.
+ */
+import type { Frame } from "../frames/frames.ts";
+import { FrameDirection, FrameProcessor, type FrameProcessorOptions, type FrameProcessorSetup } from "../processors/frame_processor.ts";
+import { BasePipeline } from "./base_pipeline.ts";
+/** Function receiving the frames that leave a pipeline at one of its ends. */
+export type PushFrameFn = (frame: Frame, direction: FrameDirection) => Promise<void>;
+/**
+ * Source processor that forwards frames to an upstream handler.
+ *
+ * Acts as the entry point for a pipeline, forwarding downstream frames to the
+ * next processor and upstream frames to a provided upstream handler function.
+ */
+export declare class PipelineSource extends FrameProcessor {
+    protected _upstreamPushFrame: PushFrameFn;
+    /**
+     * Initialize the pipeline source.
+     *
+     * @param upstreamPushFrame Async function to handle upstream frames.
+     * @param options Additional options passed to the parent class.
+     */
+    constructor(upstreamPushFrame: PushFrameFn, options?: FrameProcessorOptions);
+    /**
+     * Process frames and route them based on direction.
+     *
+     * @param frame The frame to process.
+     * @param direction The direction of frame flow.
+     */
+    processFrame(frame: Frame, direction: FrameDirection): Promise<void>;
+}
+/**
+ * Sink processor that forwards frames to a downstream handler.
+ *
+ * Acts as the exit point for a pipeline, forwarding upstream frames to the
+ * previous processor and downstream frames to a provided downstream handler
+ * function.
+ */
+export declare class PipelineSink extends FrameProcessor {
+    protected _downstreamPushFrame: PushFrameFn;
+    /**
+     * Initialize the pipeline sink.
+     *
+     * @param downstreamPushFrame Async function to handle downstream frames.
+     * @param options Additional options passed to the parent class.
+     */
+    constructor(downstreamPushFrame: PushFrameFn, options?: FrameProcessorOptions);
+    /**
+     * Process frames and route them based on direction.
+     *
+     * @param frame The frame to process.
+     * @param direction The direction of frame flow.
+     */
+    processFrame(frame: Frame, direction: FrameDirection): Promise<void>;
+}
+export interface PipelineOptions {
+    /** An optional pipeline source processor. */
+    source?: FrameProcessor | null;
+    /** An optional pipeline sink processor. */
+    sink?: FrameProcessor | null;
+}
+/**
+ * Main pipeline implementation that connects frame processors in sequence.
+ *
+ * Creates a linear chain of frame processors with automatic source and sink
+ * processors for external frame handling. Manages processor lifecycle and
+ * provides metrics collection from contained processors.
+ */
+export declare class Pipeline extends BasePipeline {
+    protected _source: FrameProcessor;
+    protected _sink: FrameProcessor;
+    protected _processors: FrameProcessor[];
+    /**
+     * Initialize the pipeline with a list of processors.
+     *
+     * @param processors Sequence of frame processors to connect in sequence.
+     * @param options.source An optional pipeline source processor.
+     * @param options.sink An optional pipeline sink processor.
+     */
+    constructor(processors: readonly FrameProcessor[], options?: PipelineOptions);
+    /**
+     * The list of sub-processors contained within this processor.
+     *
+     * Only compound processors (e.g. pipelines and parallel pipelines) have
+     * sub-processors. Non-compound processors return an empty list.
+     */
+    get processors(): FrameProcessor[];
+    /**
+     * The list of entry processors for this processor.
+     *
+     * Entry processors are the first processors in a compound processor (e.g.
+     * pipelines, parallel pipelines). Note that pipelines can also be an entry
+     * processor as pipelines are processors themselves. Non-compound processors
+     * simply return an empty list.
+     */
+    get entryProcessors(): FrameProcessor[];
+    /**
+     * Processors that can generate metrics.
+     *
+     * Recursively collects all processors that support metrics generation,
+     * including those from nested pipelines.
+     */
+    processorsWithMetrics(): FrameProcessor[];
+    /**
+     * Set up the pipeline and all contained processors.
+     *
+     * @param setup Configuration for frame processor setup.
+     */
+    setup(setup: FrameProcessorSetup): Promise<void>;
+    /** Clean up the pipeline and all contained processors. */
+    cleanup(): Promise<void>;
+    /**
+     * Process frames by routing them through the pipeline.
+     *
+     * @param frame The frame to process.
+     * @param direction The direction of frame flow.
+     */
+    processFrame(frame: Frame, direction: FrameDirection): Promise<void>;
+    /** Link all processors in sequence and set their parent. */
+    protected _linkProcessors(): void;
+}
+//# sourceMappingURL=pipeline.d.ts.map

package/dist/pipeline/pipeline.js

@@ -0,0 +1,196 @@
+// Copyright (c) 2024-2026, Daily
+// Copyright (c) 2026, pipedog contributors
+// SPDX-License-Identifier: BSD-2-Clause
+//
+// Ported from pipecat: src/pipecat/pipeline/pipeline.py
+import { FrameDirection, FrameProcessor, } from "../processors/frame_processor.js";
+import { BasePipeline } from "./base_pipeline.js";
+/**
+ * Source processor that forwards frames to an upstream handler.
+ *
+ * Acts as the entry point for a pipeline, forwarding downstream frames to the
+ * next processor and upstream frames to a provided upstream handler function.
+ */
+export class PipelineSource extends FrameProcessor {
+    _upstreamPushFrame;
+    /**
+     * Initialize the pipeline source.
+     *
+     * @param upstreamPushFrame Async function to handle upstream frames.
+     * @param options Additional options passed to the parent class.
+     */
+    constructor(upstreamPushFrame, options = {}) {
+        super({ ...options, enableDirectMode: true });
+        this._upstreamPushFrame = upstreamPushFrame;
+    }
+    /**
+     * Process frames and route them based on direction.
+     *
+     * @param frame The frame to process.
+     * @param direction The direction of frame flow.
+     */
+    async processFrame(frame, direction) {
+        await super.processFrame(frame, direction);
+        switch (direction) {
+            case FrameDirection.UPSTREAM:
+                await this._upstreamPushFrame(frame, direction);
+                break;
+            case FrameDirection.DOWNSTREAM:
+                await this.pushFrame(frame, direction);
+                break;
+        }
+    }
+}
+/**
+ * Sink processor that forwards frames to a downstream handler.
+ *
+ * Acts as the exit point for a pipeline, forwarding upstream frames to the
+ * previous processor and downstream frames to a provided downstream handler
+ * function.
+ */
+export class PipelineSink extends FrameProcessor {
+    _downstreamPushFrame;
+    /**
+     * Initialize the pipeline sink.
+     *
+     * @param downstreamPushFrame Async function to handle downstream frames.
+     * @param options Additional options passed to the parent class.
+     */
+    constructor(downstreamPushFrame, options = {}) {
+        super({ ...options, enableDirectMode: true });
+        this._downstreamPushFrame = downstreamPushFrame;
+    }
+    /**
+     * Process frames and route them based on direction.
+     *
+     * @param frame The frame to process.
+     * @param direction The direction of frame flow.
+     */
+    async processFrame(frame, direction) {
+        await super.processFrame(frame, direction);
+        switch (direction) {
+            case FrameDirection.UPSTREAM:
+                await this.pushFrame(frame, direction);
+                break;
+            case FrameDirection.DOWNSTREAM:
+                await this._downstreamPushFrame(frame, direction);
+                break;
+        }
+    }
+}
+/**
+ * Main pipeline implementation that connects frame processors in sequence.
+ *
+ * Creates a linear chain of frame processors with automatic source and sink
+ * processors for external frame handling. Manages processor lifecycle and
+ * provides metrics collection from contained processors.
+ */
+export class Pipeline extends BasePipeline {
+    _source;
+    _sink;
+    _processors;
+    /**
+     * Initialize the pipeline with a list of processors.
+     *
+     * @param processors Sequence of frame processors to connect in sequence.
+     * @param options.source An optional pipeline source processor.
+     * @param options.sink An optional pipeline sink processor.
+     */
+    constructor(processors, options = {}) {
+        super({ enableDirectMode: true });
+        // Add a source and a sink queue so we can forward frames upstream and
+        // downstream outside of the pipeline.
+        this._source =
+            options.source ||
+                new PipelineSource((frame, direction) => this.pushFrame(frame, direction), {
+                    name: `${this}::Source`,
+                });
+        this._sink =
+            options.sink ||
+                new PipelineSink((frame, direction) => this.pushFrame(frame, direction), {
+                    name: `${this}::Sink`,
+                });
+        this._processors = [this._source, ...processors, this._sink];
+        this._linkProcessors();
+    }
+    //
+    // Frame processor
+    //
+    /**
+     * The list of sub-processors contained within this processor.
+     *
+     * Only compound processors (e.g. pipelines and parallel pipelines) have
+     * sub-processors. Non-compound processors return an empty list.
+     */
+    get processors() {
+        return this._processors;
+    }
+    /**
+     * The list of entry processors for this processor.
+     *
+     * Entry processors are the first processors in a compound processor (e.g.
+     * pipelines, parallel pipelines). Note that pipelines can also be an entry
+     * processor as pipelines are processors themselves. Non-compound processors
+     * simply return an empty list.
+     */
+    get entryProcessors() {
+        return [this._source];
+    }
+    /**
+     * Processors that can generate metrics.
+     *
+     * Recursively collects all processors that support metrics generation,
+     * including those from nested pipelines.
+     */
+    processorsWithMetrics() {
+        const services = [];
+        for (const p of this.processors) {
+            if (p.canGenerateMetrics()) {
+                services.push(p);
+            }
+            services.push(...p.processorsWithMetrics());
+        }
+        return services;
+    }
+    /**
+     * Set up the pipeline and all contained processors.
+     *
+     * @param setup Configuration for frame processor setup.
+     */
+    async setup(setup) {
+        await super.setup(setup);
+        // This pipeline is set up, so let the StartFrame in: each processor holds
+        // it until its own setup is done.
+        this._setupFinished();
+        await this._setupProcessors(this._processors, setup);
+    }
+    /** Clean up the pipeline and all contained processors. */
+    async cleanup() {
+        await super.cleanup();
+        await this._cleanupProcessors(this._processors);
+    }
+    /**
+     * Process frames by routing them through the pipeline.
+     *
+     * @param frame The frame to process.
+     * @param direction The direction of frame flow.
+     */
+    async processFrame(frame, direction) {
+        await super.processFrame(frame, direction);
+        if (direction === FrameDirection.DOWNSTREAM) {
+            await this._source.queueFrame(frame, FrameDirection.DOWNSTREAM);
+        }
+        else if (direction === FrameDirection.UPSTREAM) {
+            await this._sink.queueFrame(frame, FrameDirection.UPSTREAM);
+        }
+    }
+    /** Link all processors in sequence and set their parent. */
+    _linkProcessors() {
+        let prev = this._processors[0];
+        for (const curr of this._processors.slice(1)) {
+            prev.link(curr);
+            prev = curr;
+        }
+    }
+}
+//# sourceMappingURL=pipeline.js.map