@pipedog/core 0.0.0 → 0.1.0

package/dist/utils/text/markup_utils.js

@@ -0,0 +1,156 @@
+// Copyright (c) 2024-2026, Daily
+// Copyright (c) 2026, pipedog contributors
+// SPDX-License-Identifier: BSD-2-Clause
+//
+// Ported from pipecat: src/pipecat/utils/text/markup_utils.py
+/**
+ * Shared XML/SSML markup utilities for text matching and cursor advancement.
+ *
+ * Two notions of "what is markup" live here, and they differ on a single point
+ * -- a `'<'` with no later `'>'`:
+ *
+ * - `stripMarkup` treats it as an open tag and swallows the rest of the
+ *   string. Correct for a *fragment* that may have been cut mid-tag, such as a
+ *   single token from a TTS word-timestamp stream.
+ * - `stripCompleteMarkup` treats it as content. Correct for a *complete*
+ *   text, where a lone `'<'` is real (`"5 < 10"`, `"<3"`).
+ *
+ * Every markup decision in the word-timestamp path routes through one of the
+ * two, so the callers can't disagree about which characters are a tag.
+ *
+ * Positions are UTF-16 code unit offsets, as everywhere in JavaScript.
+ */
+/**
+ * Yield `[rawIndex, char]` for each character of `text` outside markup.
+ *
+ * The fragment-tolerant definition of markup -- anything between '<' and '>',
+ * syntax-based and tag-name independent -- shared by `stripMarkup` and
+ * `rawOffsetAfterCleanChars` so the two can't disagree. An unclosed '<'
+ * swallows the rest of the string.
+ */
+function* iterCleanChars(text) {
+    let inTag = false;
+    for (let i = 0; i < text.length; i++) {
+        const ch = text[i];
+        if (inTag) {
+            if (ch === ">") {
+                inTag = false;
+            }
+        }
+        else if (ch === "<") {
+            inTag = true;
+        }
+        else {
+            yield [i, ch];
+        }
+    }
+}
+/**
+ * Remove XML/SSML-like markup from a possibly-truncated text fragment.
+ *
+ * Syntax-based, not tag-name based: treats anything between '<' and '>' as
+ * markup and preserves text outside it. An unclosed '<' swallows the rest of
+ * `text`, matching how a raw word-timestamp token can arrive mid-tag.
+ *
+ * For a *complete* text, use `stripCompleteMarkup` instead.
+ *
+ * Used by `TextSegmentMap` to match a word against a segment, where the
+ * incoming word may be a fragment of a still-open tag.
+ */
+export function stripMarkup(text) {
+    let out = "";
+    for (const [, ch] of iterCleanChars(text))
+        out += ch;
+    return out;
+}
+/**
+ * Matched '<...>' pairs in a complete text.
+ *
+ * The definition of markup for a static text, shared by `stripCompleteMarkup`
+ * and `splitMarkupRuns` so the two can't disagree about which characters a
+ * segment split may treat as a tag.
+ */
+const COMPLETE_MARKUP_RE = /<[^>]+>/g;
+/**
+ * Remove well-formed '<...>' markup from a complete, static text.
+ *
+ * Unlike `stripMarkup`, only strips matched '<...>' pairs -- a lone '<' with
+ * no later '>' is left in place as real content rather than swallowing the
+ * rest of `text`, since there is no streamed fragment here that could be
+ * mid-tag.
+ *
+ * Used by `TextSegment.isTransformed`, by `alnumOnly`, and by
+ * `WordCompletionTracker` to default `userFacingText` to a tag-free string.
+ */
+export function stripCompleteMarkup(text) {
+    return text.replace(COMPLETE_MARKUP_RE, "");
+}
+/**
+ * Return the raw offset into `text` just past its `n`-th markup-stripped char.
+ *
+ * Inverse of `stripMarkup` for a prefix: where `stripMarkup` collects every
+ * non-markup char, this finds the raw index one past the `n`-th of them --
+ * converting a match measured in markup-stripped space back to a raw offset.
+ * Returns `text.length` when `text` has fewer than `n` non-markup chars.
+ */
+export function rawOffsetAfterCleanChars(text, n) {
+    if (n <= 0) {
+        return 0;
+    }
+    let seen = 0;
+    for (const [i] of iterCleanChars(text)) {
+        seen += 1;
+        if (seen === n) {
+            return i + 1;
+        }
+    }
+    return text.length;
+}
+/**
+ * Split `text` into alternating runs of tagged and untagged words.
+ *
+ * A word is considered tagged if it overlaps a complete `'<...>'` pair. A lone
+ * `'<'` is treated as content, not as the start of a tag (see
+ * `stripCompleteMarkup`). Consecutive words with the same classification form
+ * a single run, so whitespace inside a tag such as `<phoneme alphabet="ipa">`
+ * never splits the words it spans across runs.
+ *
+ * Text with no markup yields a single run, unchanged.
+ *
+ * Used by `TextSegmentMap` when it builds its segments, to give a tag one of
+ * its own.
+ *
+ * @example
+ * splitMarkupRuns("I love to count <spell>1234</spell>.");
+ * // -> ["I love to count ", "<spell>1234</spell>."]
+ */
+export function splitMarkupRuns(text) {
+    const tagSpans = [];
+    for (const m of text.matchAll(COMPLETE_MARKUP_RE)) {
+        tagSpans.push([m.index, m.index + m[0].length]);
+    }
+    if (tagSpans.length === 0) {
+        return text ? [text] : [];
+    }
+    const runs = [];
+    let runIsTagged = null;
+    let pos = 0;
+    for (const token of text.split(/(\s+)/)) {
+        if (!token) {
+            continue;
+        }
+        const start = pos;
+        const end = pos + token.length;
+        pos = end;
+        const isTagged = tagSpans.some(([tagStart, tagEnd]) => tagStart < end && start < tagEnd);
+        if (isTagged === runIsTagged) {
+            runs[runs.length - 1] += token;
+        }
+        else {
+            runs.push(token);
+            runIsTagged = isTagged;
+        }
+    }
+    return runs;
+}
+//# sourceMappingURL=markup_utils.js.map

package/dist/utils/text/pattern_pair_aggregator.d.ts

@@ -0,0 +1,251 @@
+import { Aggregation } from "./base_text_aggregator.ts";
+import { SimpleTextAggregator, type SimpleTextAggregatorOptions } from "./simple_text_aggregator.ts";
+/**
+ * Actions to take when a pattern pair is matched.
+ *
+ * - `REMOVE`: The text along with its delimiters will be removed from the streaming text.
+ *   Sentence aggregation will continue on as if this text did not exist.
+ * - `KEEP`: The delimiters will be removed, but the content between them will be kept.
+ *   Sentence aggregation will continue on with the internal text included.
+ * - `AGGREGATE`: The delimiters will be removed and the content between will be treated
+ *   as a separate aggregation. Any text before the start of the pattern will be
+ *   returned early, whether or not a complete sentence was found. Then the pattern
+ *   will be returned. Then the aggregation will continue on sentence matching after
+ *   the closing delimiter is found. The content between the delimiters is not
+ *   aggregated by sentence. It is aggregated as one single block of text.
+ */
+export declare const MatchAction: {
+    readonly REMOVE: "remove";
+    readonly KEEP: "keep";
+    readonly AGGREGATE: "aggregate";
+};
+export type MatchAction = (typeof MatchAction)[keyof typeof MatchAction];
+export interface PatternMatchOptions {
+    /** The text content between the start and end patterns. */
+    content: string;
+    /**
+     * The type of the matched pattern pair. It should be representative of the
+     * content type (e.g., 'sentence', 'code', 'speaker', 'custom').
+     */
+    type: string;
+    /** The complete text including start and end patterns. */
+    fullMatch: string;
+}
+/**
+ * Represents a matched pattern pair with its content.
+ *
+ * A PatternMatch object is created when a complete pattern pair is found
+ * in the text. It contains information about which pattern was matched,
+ * the full matched text (including start and end patterns), and the
+ * content between the patterns.
+ */
+export declare class PatternMatch extends Aggregation {
+    fullMatch: string;
+    /** Initialize a pattern match. */
+    constructor(options: PatternMatchOptions);
+    /**
+     * Return a string representation of the pattern match.
+     *
+     * @returns A descriptive string showing the pattern type and content.
+     */
+    toString(): string;
+}
+/** A registered pattern pair. */
+interface PatternInfo {
+    start: string;
+    end: string;
+    type: string;
+    action: MatchAction;
+}
+/** Handler called when a pattern pair is matched. */
+export type PatternMatchHandler = (match: PatternMatch) => Promise<void> | void;
+export interface PatternPairAggregatorOptions extends SimpleTextAggregatorOptions {
+}
+/**
+ * Aggregator that identifies and processes content between pattern pairs.
+ *
+ * This aggregator buffers text until it can identify complete pattern pairs
+ * (defined by start and end patterns), processes the content between these
+ * patterns using registered handlers. By default, its aggregation method
+ * returns text at sentence boundaries, and remove the content found between
+ * any matched patterns. However, matched patterns can also be configured to
+ * returned as a separate aggregation object containing the content between
+ * their start and end patterns or left in, so that only the delimiters are
+ * removed and a callback can be triggered.
+ *
+ * This aggregator is particularly useful for processing structured content in
+ * streaming text, such as XML tags, markdown formatting, or custom delimiters.
+ *
+ * The aggregator ensures that patterns spanning multiple text chunks are
+ * correctly identified.
+ */
+export declare class PatternPairAggregator extends SimpleTextAggregator {
+    protected _patterns: Map<string, PatternInfo>;
+    protected _handlers: Map<string, PatternMatchHandler>;
+    protected _lastProcessedPosition: number;
+    /**
+     * Initialize the pattern pair aggregator.
+     *
+     * Creates an empty aggregator with no patterns or handlers registered.
+     * Text buffering and pattern detection will begin when text is aggregated.
+     *
+     * @param options Additional arguments passed to SimpleTextAggregator (e.g. aggregationType).
+     */
+    constructor(options?: PatternPairAggregatorOptions);
+    /**
+     * Get the currently aggregated text.
+     *
+     * @returns The text that has been accumulated in the buffer.
+     */
+    get text(): Aggregation;
+    /**
+     * Add a pattern pair to detect in the text.
+     *
+     * Registers a new pattern pair with a unique identifier. The aggregator
+     * will look for text that starts with the start pattern and ends with
+     * the end pattern, and treat the content between them as a match.
+     *
+     * @param type Identifier for this pattern pair. Should be unique and ideally descriptive.
+     *   (e.g., 'code', 'speaker', 'custom'). type can not be 'sentence' or 'word' as
+     *   those are reserved for the default behavior.
+     * @param startPattern Pattern that marks the beginning of content.
+     * @param endPattern Pattern that marks the end of content.
+     * @param action What to do when a complete pattern is matched.
+     *
+     *   - `MatchAction.REMOVE`: Remove the matched pattern from the text.
+     *   - `MatchAction.KEEP`: Keep the matched pattern in the text and treat it as
+     *     normal text. This allows you to register handlers for the pattern without
+     *     affecting the aggregation logic.
+     *   - `MatchAction.AGGREGATE`: Return the matched pattern as a separate aggregation object.
+     * @returns Self for method chaining.
+     * @throws Error If `type` is reserved (upstream: ValueError).
+     */
+    addPattern(type: string, startPattern: string, endPattern: string, action?: MatchAction): this;
+    /**
+     * Register a handler for when a pattern pair is matched.
+     *
+     * The handler will be called whenever a complete match for the
+     * specified type is found in the text.
+     *
+     * @param type The type of the pattern pair to trigger the handler.
+     * @param handler Async function to call when pattern is matched.
+     *   The function should accept a PatternMatch object.
+     * @returns Self for method chaining.
+     */
+    onPatternMatch(type: string, handler: PatternMatchHandler): this;
+    /**
+     * Process newly complete pattern pairs in the text.
+     *
+     * Searches for pattern pairs that have been completed since lastProcessedPosition,
+     * calls the appropriate handlers, and optionally removes the matches.
+     *
+     * @param text The text to process for pattern matches.
+     * @param lastProcessedPosition The position in text that was already processed.
+     *   Only patterns that end at or after this position will be processed.
+     * @returns Tuple of [allMatches, processedText] where:
+     *
+     *   - allMatches is a list of all pattern matches found. Note: There really should only ever be 1.
+     *   - processedText is the text after processing patterns. If no patterns are
+     *     found, it will be the same as input text.
+     */
+    protected _processCompletePatterns(text: string, lastProcessedPosition?: number): Promise<[PatternMatch[], string]>;
+    /**
+     * Check if text contains incomplete pattern pairs.
+     *
+     * Determines whether the text contains any start patterns without
+     * matching end patterns, which would indicate incomplete content.
+     *
+     * @param text The text to check for incomplete patterns.
+     * @returns A tuple of [startIndex, patternInfo] if an incomplete pattern is found,
+     *   or null if no patterns are found or all patterns are complete.
+     */
+    protected _matchStartOfPattern(text: string): [number, PatternInfo] | null;
+    /**
+     * Aggregate text and process pattern pairs.
+     *
+     * Processes the input text character-by-character, handles pattern pairs,
+     * and uses the parent's lookahead logic for sentence detection when no
+     * patterns are active.
+     *
+     * In TOKEN mode, pattern detection still works but non-pattern text is
+     * yielded as TOKEN aggregations instead of waiting for sentence boundaries.
+     * Text ending in a partial start delimiter (e.g. `<thin` of `<think>`) is
+     * held back until a later chunk determines whether it begins a pattern, so
+     * a delimiter split across chunks is still recognized.
+     *
+     * @param text Text to aggregate.
+     * @returns PatternMatch objects as patterns complete or sentences are detected.
+     */
+    aggregate(text: string): AsyncGenerator<PatternMatch>;
+    /**
+     * Flush any remaining text in the buffer.
+     *
+     * Degrades incomplete pattern state the same way a closed pair's
+     * action would, without knowing whether the pair would ever have
+     * closed:
+     *
+     * - REMOVE and AGGREGATE: an unclosed start delimiter and everything
+     *   from it onward is dropped (their content is either stripped or
+     *   treated as a side channel once closed, so it's never spoken
+     *   incomplete). The pattern handler is not invoked. Text before the
+     *   earliest such unclosed start delimiter is kept.
+     * - KEEP: a closed KEEP pair is spoken verbatim, delimiters included
+     *   (see `aggregate`), so an unclosed one degrades to the same
+     *   thing rather than being dropped.
+     *
+     * If the buffer ends with a proper prefix of a REMOVE/AGGREGATE start
+     * delimiter (the stream cut off mid-delimiter), that trailing partial
+     * text is trimmed too. This accepts an inherent ambiguity: a legitimate
+     * trailing character sequence that happens to match a delimiter prefix
+     * (e.g. a stray `<`) is trimmed the same way.
+     *
+     * @returns Any remaining text with incomplete REMOVE/AGGREGATE pattern
+     *   content dropped, or null if there is nothing left to return.
+     */
+    flush(): Promise<Aggregation | null>;
+    /**
+     * Find the earliest unmatched start delimiter that flush() should cut at.
+     *
+     * For each registered pattern, replicates the same left-to-right,
+     * non-overlapping pairing that `_processCompletePatterns` uses
+     * (earliest start paired with earliest following end) to determine
+     * which start-delimiter occurrences are still unmatched. Only
+     * REMOVE/AGGREGATE patterns' unmatched starts are candidates: KEEP
+     * patterns are excluded because unclosed KEEP content is kept, not cut.
+     *
+     * @param text The buffered text to scan.
+     * @returns The index of the earliest droppable unmatched start delimiter,
+     *   or null if there isn't one.
+     */
+    protected _findEarliestDroppableStart(text: string): number | null;
+    /**
+     * Trim a trailing partial REMOVE/AGGREGATE start delimiter, if any.
+     *
+     * The stream can end mid-delimiter (e.g. buffered text ending in
+     * `<te` of `<test>`). That partial text isn't a complete start
+     * delimiter, so `_findEarliestDroppableStart` won't catch it,
+     * but it also isn't safe to speak. KEEP patterns are excluded since a
+     * KEEP delimiter is kept whether or not it's complete.
+     *
+     * @param text The text to check for a trailing partial delimiter.
+     * @returns The text with a trailing partial start delimiter removed, or the
+     *   original text if there isn't one.
+     */
+    protected _trimTrailingPartialStart(text: string): string;
+    /**
+     * Handle interruptions by clearing the buffer and pattern state.
+     *
+     * Called when an interruption occurs in the processing pipeline,
+     * to reset the state and discard any partially aggregated text.
+     */
+    handleInterruption(): Promise<void>;
+    /**
+     * Clear the internally aggregated text.
+     *
+     * Resets the aggregator to its initial state, discarding any
+     * buffered text and clearing pattern tracking state.
+     */
+    reset(): Promise<void>;
+}
+export {};
+//# sourceMappingURL=pattern_pair_aggregator.d.ts.map