@pipedog/core 0.0.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +25 -0
- package/NOTICE +9 -0
- package/dist/adapters/base_llm_adapter.d.ts +266 -0
- package/dist/adapters/base_llm_adapter.js +489 -0
- package/dist/adapters/index.d.ts +2 -0
- package/dist/adapters/index.js +8 -0
- package/dist/adapters/schemas/direct_function.d.ts +202 -0
- package/dist/adapters/schemas/direct_function.js +302 -0
- package/dist/adapters/schemas/function_schema.d.ts +67 -0
- package/dist/adapters/schemas/function_schema.js +72 -0
- package/dist/adapters/schemas/index.d.ts +2 -0
- package/dist/adapters/schemas/index.js +8 -0
- package/dist/adapters/schemas/tools_schema.d.ts +57 -0
- package/dist/adapters/schemas/tools_schema.js +79 -0
- package/dist/adapters/services/index.d.ts +2 -0
- package/dist/adapters/services/index.js +8 -0
- package/dist/adapters/services/open_ai_adapter.d.ts +147 -0
- package/dist/adapters/services/open_ai_adapter.js +250 -0
- package/dist/audio/dtmf/dtmf_data.d.ts +3 -0
- package/dist/audio/dtmf/dtmf_data.js +22 -0
- package/dist/audio/dtmf/index.d.ts +2 -0
- package/dist/audio/dtmf/index.js +8 -0
- package/dist/audio/dtmf/types.d.ts +31 -0
- package/dist/audio/dtmf/types.js +35 -0
- package/dist/audio/dtmf/utils.d.ts +13 -0
- package/dist/audio/dtmf/utils.js +62 -0
- package/dist/audio/filters/base_audio_filter.d.ts +51 -0
- package/dist/audio/filters/base_audio_filter.js +17 -0
- package/dist/audio/filters/index.d.ts +2 -0
- package/dist/audio/filters/index.js +8 -0
- package/dist/audio/index.d.ts +2 -0
- package/dist/audio/index.js +8 -0
- package/dist/audio/krisp_instance.d.ts +95 -0
- package/dist/audio/krisp_instance.js +229 -0
- package/dist/audio/mixers/base_audio_mixer.d.ts +56 -0
- package/dist/audio/mixers/base_audio_mixer.js +16 -0
- package/dist/audio/mixers/index.d.ts +2 -0
- package/dist/audio/mixers/index.js +8 -0
- package/dist/audio/resamplers/base_audio_resampler.d.ts +46 -0
- package/dist/audio/resamplers/base_audio_resampler.js +35 -0
- package/dist/audio/resamplers/engine.d.ts +60 -0
- package/dist/audio/resamplers/engine.js +89 -0
- package/dist/audio/resamplers/index.d.ts +2 -0
- package/dist/audio/resamplers/index.js +8 -0
- package/dist/audio/resamplers/resampy_resampler.d.ts +10 -0
- package/dist/audio/resamplers/resampy_resampler.js +15 -0
- package/dist/audio/resamplers/sinc_engine.d.ts +10 -0
- package/dist/audio/resamplers/sinc_engine.js +235 -0
- package/dist/audio/resamplers/soxr_resampler.d.ts +58 -0
- package/dist/audio/resamplers/soxr_resampler.js +79 -0
- package/dist/audio/resamplers/soxr_stream_resampler.d.ts +78 -0
- package/dist/audio/resamplers/soxr_stream_resampler.js +159 -0
- package/dist/audio/turn/base_turn_analyzer.d.ts +84 -0
- package/dist/audio/turn/base_turn_analyzer.js +61 -0
- package/dist/audio/turn/index.d.ts +2 -0
- package/dist/audio/turn/index.js +8 -0
- package/dist/audio/turn/smart_turn/_whisper_features.d.ts +30 -0
- package/dist/audio/turn/smart_turn/_whisper_features.js +225 -0
- package/dist/audio/turn/smart_turn/base_smart_turn.d.ts +94 -0
- package/dist/audio/turn/smart_turn/base_smart_turn.js +239 -0
- package/dist/audio/turn/smart_turn/local_smart_turn_v3.d.ts +76 -0
- package/dist/audio/turn/smart_turn/local_smart_turn_v3.js +151 -0
- package/dist/audio/utils.d.ts +224 -0
- package/dist/audio/utils.js +634 -0
- package/dist/audio/vad/data/index.d.ts +2 -0
- package/dist/audio/vad/data/index.js +8 -0
- package/dist/audio/vad/index.d.ts +2 -0
- package/dist/audio/vad/index.js +8 -0
- package/dist/audio/vad/vad_analyzer.d.ts +134 -0
- package/dist/audio/vad/vad_analyzer.js +249 -0
- package/dist/audio/vad/vad_controller.d.ts +143 -0
- package/dist/audio/vad/vad_controller.js +228 -0
- package/dist/audio/volume.d.ts +38 -0
- package/dist/audio/volume.js +75 -0
- package/dist/bus/adapters/base.d.ts +35 -0
- package/dist/bus/adapters/base.js +19 -0
- package/dist/bus/adapters/index.d.ts +10 -0
- package/dist/bus/adapters/index.js +15 -0
- package/dist/bus/adapters/llm_context_adapter.d.ts +36 -0
- package/dist/bus/adapters/llm_context_adapter.js +82 -0
- package/dist/bus/adapters/tools_schema_adapter.d.ts +25 -0
- package/dist/bus/adapters/tools_schema_adapter.js +70 -0
- package/dist/bus/bridge_processor.d.ts +111 -0
- package/dist/bus/bridge_processor.js +166 -0
- package/dist/bus/bus.d.ts +110 -0
- package/dist/bus/bus.js +200 -0
- package/dist/bus/index.d.ts +22 -0
- package/dist/bus/index.js +27 -0
- package/dist/bus/local/async_queue.d.ts +12 -0
- package/dist/bus/local/async_queue.js +21 -0
- package/dist/bus/local/index.d.ts +3 -0
- package/dist/bus/local/index.js +8 -0
- package/dist/bus/messages.d.ts +378 -0
- package/dist/bus/messages.js +357 -0
- package/dist/bus/network/index.d.ts +16 -0
- package/dist/bus/network/index.js +21 -0
- package/dist/bus/queue.d.ts +27 -0
- package/dist/bus/queue.js +51 -0
- package/dist/bus/serializers/base.d.ts +25 -0
- package/dist/bus/serializers/base.js +14 -0
- package/dist/bus/serializers/index.d.ts +9 -0
- package/dist/bus/serializers/index.js +14 -0
- package/dist/bus/serializers/json.d.ts +77 -0
- package/dist/bus/serializers/json.js +367 -0
- package/dist/bus/subscriber.d.ts +46 -0
- package/dist/bus/subscriber.js +27 -0
- package/dist/bus/ui/index.d.ts +3 -0
- package/dist/bus/ui/index.js +8 -0
- package/dist/bus/ui/messages.d.ts +186 -0
- package/dist/bus/ui/messages.js +171 -0
- package/dist/classifiers/base_classifier.d.ts +263 -0
- package/dist/classifiers/base_classifier.js +346 -0
- package/dist/classifiers/cloudflare/clef/classifier.d.ts +70 -0
- package/dist/classifiers/cloudflare/clef/classifier.js +204 -0
- package/dist/classifiers/cloudflare/clef/client.d.ts +104 -0
- package/dist/classifiers/cloudflare/clef/client.js +234 -0
- package/dist/classifiers/cloudflare/clef/index.d.ts +2 -0
- package/dist/classifiers/cloudflare/clef/index.js +8 -0
- package/dist/classifiers/cloudflare/index.d.ts +2 -0
- package/dist/classifiers/cloudflare/index.js +8 -0
- package/dist/classifiers/http_client.d.ts +70 -0
- package/dist/classifiers/http_client.js +161 -0
- package/dist/classifiers/index.d.ts +2 -0
- package/dist/classifiers/index.js +8 -0
- package/dist/classifiers/jev/classifier.d.ts +8 -0
- package/dist/classifiers/jev/classifier.js +13 -0
- package/dist/classifiers/jev/client.d.ts +8 -0
- package/dist/classifiers/jev/client.js +13 -0
- package/dist/classifiers/jev/index.d.ts +2 -0
- package/dist/classifiers/jev/index.js +8 -0
- package/dist/classifiers/llm/classifier.d.ts +72 -0
- package/dist/classifiers/llm/classifier.js +304 -0
- package/dist/classifiers/llm/index.d.ts +2 -0
- package/dist/classifiers/llm/index.js +8 -0
- package/dist/classifiers/openai/decisions/classifier.d.ts +77 -0
- package/dist/classifiers/openai/decisions/classifier.js +241 -0
- package/dist/classifiers/openai/decisions/client.d.ts +108 -0
- package/dist/classifiers/openai/decisions/client.js +234 -0
- package/dist/classifiers/openai/decisions/index.d.ts +2 -0
- package/dist/classifiers/openai/decisions/index.js +8 -0
- package/dist/classifiers/openai/index.d.ts +2 -0
- package/dist/classifiers/openai/index.js +8 -0
- package/dist/classifiers/typesafe/index.d.ts +2 -0
- package/dist/classifiers/typesafe/index.js +8 -0
- package/dist/classifiers/typesafe/jev/classifier.d.ts +70 -0
- package/dist/classifiers/typesafe/jev/classifier.js +220 -0
- package/dist/classifiers/typesafe/jev/client.d.ts +105 -0
- package/dist/classifiers/typesafe/jev/client.js +233 -0
- package/dist/classifiers/typesafe/jev/index.d.ts +2 -0
- package/dist/classifiers/typesafe/jev/index.js +8 -0
- package/dist/clocks/base_clock.d.ts +13 -0
- package/dist/clocks/base_clock.js +14 -0
- package/dist/clocks/index.d.ts +2 -0
- package/dist/clocks/index.js +8 -0
- package/dist/clocks/system_clock.d.ts +15 -0
- package/dist/clocks/system_clock.js +25 -0
- package/dist/frames/frames.d.ts +2533 -0
- package/dist/frames/frames.js +2485 -0
- package/dist/frames/index.d.ts +2 -0
- package/dist/frames/index.js +8 -0
- package/dist/frames/protobufs/frames_pb2.d.ts +229 -0
- package/dist/frames/protobufs/frames_pb2.js +466 -0
- package/dist/index.d.ts +11 -0
- package/dist/index.js +56 -0
- package/dist/metrics/index.d.ts +2 -0
- package/dist/metrics/index.js +8 -0
- package/dist/metrics/metrics.d.ts +276 -0
- package/dist/metrics/metrics.js +254 -0
- package/dist/observers/base_observer.d.ts +173 -0
- package/dist/observers/base_observer.js +151 -0
- package/dist/observers/error_observer.d.ts +75 -0
- package/dist/observers/error_observer.js +87 -0
- package/dist/observers/function_call_observer.d.ts +157 -0
- package/dist/observers/function_call_observer.js +222 -0
- package/dist/observers/index.d.ts +2 -0
- package/dist/observers/index.js +8 -0
- package/dist/observers/loggers/debug_log_observer.d.ts +91 -0
- package/dist/observers/loggers/debug_log_observer.js +228 -0
- package/dist/observers/loggers/index.d.ts +2 -0
- package/dist/observers/loggers/index.js +8 -0
- package/dist/observers/loggers/llm_log_observer.d.ts +24 -0
- package/dist/observers/loggers/llm_log_observer.js +110 -0
- package/dist/observers/loggers/metrics_log_observer.d.ts +85 -0
- package/dist/observers/loggers/metrics_log_observer.js +186 -0
- package/dist/observers/loggers/transcription_log_observer.d.ts +23 -0
- package/dist/observers/loggers/transcription_log_observer.js +74 -0
- package/dist/observers/service_metrics_observer.d.ts +168 -0
- package/dist/observers/service_metrics_observer.js +236 -0
- package/dist/observers/speaking_observer.d.ts +104 -0
- package/dist/observers/speaking_observer.js +150 -0
- package/dist/observers/startup_timing_observer.d.ts +221 -0
- package/dist/observers/startup_timing_observer.js +365 -0
- package/dist/observers/turn_tracking_observer.d.ts +78 -0
- package/dist/observers/turn_tracking_observer.js +189 -0
- package/dist/observers/user_bot_latency_observer.d.ts +411 -0
- package/dist/observers/user_bot_latency_observer.js +974 -0
- package/dist/pipeline/base_pipeline.d.ts +49 -0
- package/dist/pipeline/base_pipeline.js +101 -0
- package/dist/pipeline/capabilities.d.ts +76 -0
- package/dist/pipeline/capabilities.js +97 -0
- package/dist/pipeline/index.d.ts +2 -0
- package/dist/pipeline/index.js +8 -0
- package/dist/pipeline/job_context.d.ts +317 -0
- package/dist/pipeline/job_context.js +450 -0
- package/dist/pipeline/job_decorator.d.ts +54 -0
- package/dist/pipeline/job_decorator.js +92 -0
- package/dist/pipeline/llm_switcher.d.ts +84 -0
- package/dist/pipeline/llm_switcher.js +107 -0
- package/dist/pipeline/llm_with_backend.d.ts +435 -0
- package/dist/pipeline/llm_with_backend.js +806 -0
- package/dist/pipeline/parallel_pipeline.d.ts +83 -0
- package/dist/pipeline/parallel_pipeline.js +200 -0
- package/dist/pipeline/pipeline.d.ts +126 -0
- package/dist/pipeline/pipeline.js +196 -0
- package/dist/pipeline/service_switcher.d.ts +287 -0
- package/dist/pipeline/service_switcher.js +509 -0
- package/dist/pipeline/sync_parallel_pipeline.d.ts +192 -0
- package/dist/pipeline/sync_parallel_pipeline.js +387 -0
- package/dist/pipeline/worker.d.ts +821 -0
- package/dist/pipeline/worker.js +1656 -0
- package/dist/pipeline/worker_observer.d.ts +131 -0
- package/dist/pipeline/worker_observer.js +274 -0
- package/dist/pipeline/worker_ready_decorator.d.ts +45 -0
- package/dist/pipeline/worker_ready_decorator.js +76 -0
- package/dist/processors/aggregators/async_tool_messages.d.ts +146 -0
- package/dist/processors/aggregators/async_tool_messages.js +291 -0
- package/dist/processors/aggregators/dtmf_aggregator.d.ts +61 -0
- package/dist/processors/aggregators/dtmf_aggregator.js +144 -0
- package/dist/processors/aggregators/gated.d.ts +43 -0
- package/dist/processors/aggregators/gated.js +86 -0
- package/dist/processors/aggregators/gated_llm_context.d.ts +42 -0
- package/dist/processors/aggregators/gated_llm_context.js +86 -0
- package/dist/processors/aggregators/index.d.ts +2 -0
- package/dist/processors/aggregators/index.js +8 -0
- package/dist/processors/aggregators/llm_context.d.ts +370 -0
- package/dist/processors/aggregators/llm_context.js +580 -0
- package/dist/processors/aggregators/llm_context_summarizer.d.ts +219 -0
- package/dist/processors/aggregators/llm_context_summarizer.js +447 -0
- package/dist/processors/aggregators/llm_response.d.ts +37 -0
- package/dist/processors/aggregators/llm_response.js +71 -0
- package/dist/processors/aggregators/llm_response_universal.d.ts +966 -0
- package/dist/processors/aggregators/llm_response_universal.js +2394 -0
- package/dist/processors/aggregators/llm_text_processor.d.ts +53 -0
- package/dist/processors/aggregators/llm_text_processor.js +102 -0
- package/dist/processors/aggregators/sentence.d.ts +39 -0
- package/dist/processors/aggregators/sentence.js +69 -0
- package/dist/processors/async_generator.d.ts +42 -0
- package/dist/processors/async_generator.js +76 -0
- package/dist/processors/audio/audio_buffer_processor.d.ts +241 -0
- package/dist/processors/audio/audio_buffer_processor.js +565 -0
- package/dist/processors/audio/index.d.ts +2 -0
- package/dist/processors/audio/index.js +8 -0
- package/dist/processors/audio/vad_processor.d.ts +67 -0
- package/dist/processors/audio/vad_processor.js +103 -0
- package/dist/processors/consumer_processor.d.ts +49 -0
- package/dist/processors/consumer_processor.js +85 -0
- package/dist/processors/filters/frame_filter.d.ts +39 -0
- package/dist/processors/filters/frame_filter.js +52 -0
- package/dist/processors/filters/function_filter.d.ts +59 -0
- package/dist/processors/filters/function_filter.js +78 -0
- package/dist/processors/filters/identity_filter.d.ts +33 -0
- package/dist/processors/filters/identity_filter.js +37 -0
- package/dist/processors/filters/index.d.ts +2 -0
- package/dist/processors/filters/index.js +8 -0
- package/dist/processors/filters/null_filter.d.ts +34 -0
- package/dist/processors/filters/null_filter.js +47 -0
- package/dist/processors/filters/wake_notifier_filter.d.ts +48 -0
- package/dist/processors/filters/wake_notifier_filter.js +49 -0
- package/dist/processors/frame_processor.d.ts +360 -0
- package/dist/processors/frame_processor.js +932 -0
- package/dist/processors/frameworks/index.d.ts +2 -0
- package/dist/processors/frameworks/index.js +8 -0
- package/dist/processors/frameworks/rtvi/frames.d.ts +241 -0
- package/dist/processors/frameworks/rtvi/frames.js +259 -0
- package/dist/processors/frameworks/rtvi/index.d.ts +6 -0
- package/dist/processors/frameworks/rtvi/index.js +11 -0
- package/dist/processors/frameworks/rtvi/models.d.ts +1741 -0
- package/dist/processors/frameworks/rtvi/models.js +1807 -0
- package/dist/processors/frameworks/rtvi/observer.d.ts +340 -0
- package/dist/processors/frameworks/rtvi/observer.js +1113 -0
- package/dist/processors/frameworks/rtvi/processor.d.ts +182 -0
- package/dist/processors/frameworks/rtvi/processor.js +676 -0
- package/dist/processors/idle_frame_processor.d.ts +44 -0
- package/dist/processors/idle_frame_processor.js +86 -0
- package/dist/processors/index.d.ts +2 -0
- package/dist/processors/index.js +8 -0
- package/dist/processors/logger.d.ts +42 -0
- package/dist/processors/logger.js +87 -0
- package/dist/processors/metrics/frame_processor_metrics.d.ts +172 -0
- package/dist/processors/metrics/frame_processor_metrics.js +372 -0
- package/dist/processors/metrics/index.d.ts +2 -0
- package/dist/processors/metrics/index.js +8 -0
- package/dist/processors/producer_processor.d.ts +68 -0
- package/dist/processors/producer_processor.js +78 -0
- package/dist/processors/text_transformer.d.ts +30 -0
- package/dist/processors/text_transformer.js +48 -0
- package/dist/registry/index.d.ts +3 -0
- package/dist/registry/index.js +8 -0
- package/dist/registry/registry.d.ts +69 -0
- package/dist/registry/registry.js +149 -0
- package/dist/registry/types.d.ts +47 -0
- package/dist/registry/types.js +39 -0
- package/dist/serializers/base_serializer.d.ts +94 -0
- package/dist/serializers/base_serializer.js +82 -0
- package/dist/serializers/index.d.ts +2 -0
- package/dist/serializers/index.js +8 -0
- package/dist/services/ai_service.d.ts +145 -0
- package/dist/services/ai_service.js +262 -0
- package/dist/services/anthropic/index.d.ts +2 -0
- package/dist/services/anthropic/index.js +7 -0
- package/dist/services/assemblyai/index.d.ts +2 -0
- package/dist/services/assemblyai/index.js +7 -0
- package/dist/services/asyncai/index.d.ts +2 -0
- package/dist/services/asyncai/index.js +7 -0
- package/dist/services/aws/index.d.ts +2 -0
- package/dist/services/aws/index.js +7 -0
- package/dist/services/azure/index.d.ts +2 -0
- package/dist/services/azure/index.js +7 -0
- package/dist/services/baseten/index.d.ts +2 -0
- package/dist/services/baseten/index.js +7 -0
- package/dist/services/bland/index.d.ts +2 -0
- package/dist/services/bland/index.js +7 -0
- package/dist/services/camb/index.d.ts +2 -0
- package/dist/services/camb/index.js +7 -0
- package/dist/services/cartesia/index.d.ts +2 -0
- package/dist/services/cartesia/index.js +7 -0
- package/dist/services/cerebras/index.d.ts +2 -0
- package/dist/services/cerebras/index.js +7 -0
- package/dist/services/crusoe/index.d.ts +2 -0
- package/dist/services/crusoe/index.js +7 -0
- package/dist/services/deepgram/index.d.ts +2 -0
- package/dist/services/deepgram/index.js +7 -0
- package/dist/services/deepseek/index.d.ts +2 -0
- package/dist/services/deepseek/index.js +7 -0
- package/dist/services/elevenlabs/index.d.ts +2 -0
- package/dist/services/elevenlabs/index.js +7 -0
- package/dist/services/fal/index.d.ts +2 -0
- package/dist/services/fal/index.js +7 -0
- package/dist/services/fireworks/index.d.ts +2 -0
- package/dist/services/fireworks/index.js +7 -0
- package/dist/services/fish/index.d.ts +2 -0
- package/dist/services/fish/index.js +7 -0
- package/dist/services/funasr/index.d.ts +3 -0
- package/dist/services/funasr/index.js +8 -0
- package/dist/services/gladia/index.d.ts +2 -0
- package/dist/services/gladia/index.js +7 -0
- package/dist/services/google/index.d.ts +2 -0
- package/dist/services/google/index.js +7 -0
- package/dist/services/gradium/index.d.ts +2 -0
- package/dist/services/gradium/index.js +7 -0
- package/dist/services/grok/index.d.ts +2 -0
- package/dist/services/grok/index.js +7 -0
- package/dist/services/groq/index.d.ts +2 -0
- package/dist/services/groq/index.js +7 -0
- package/dist/services/heygen/index.d.ts +2 -0
- package/dist/services/heygen/index.js +7 -0
- package/dist/services/hume/index.d.ts +2 -0
- package/dist/services/hume/index.js +7 -0
- package/dist/services/image_service.d.ts +52 -0
- package/dist/services/image_service.js +60 -0
- package/dist/services/inception/index.d.ts +2 -0
- package/dist/services/inception/index.js +7 -0
- package/dist/services/index.d.ts +2 -0
- package/dist/services/index.js +8 -0
- package/dist/services/inworld/index.d.ts +2 -0
- package/dist/services/inworld/index.js +7 -0
- package/dist/services/keenable/index.d.ts +2 -0
- package/dist/services/keenable/index.js +7 -0
- package/dist/services/kokoro/index.d.ts +2 -0
- package/dist/services/kokoro/index.js +7 -0
- package/dist/services/llm_service.d.ts +1106 -0
- package/dist/services/llm_service.js +2231 -0
- package/dist/services/mem0/index.d.ts +2 -0
- package/dist/services/mem0/index.js +7 -0
- package/dist/services/meta/index.d.ts +2 -0
- package/dist/services/meta/index.js +7 -0
- package/dist/services/minimax/index.d.ts +2 -0
- package/dist/services/minimax/index.js +7 -0
- package/dist/services/mistral/index.d.ts +2 -0
- package/dist/services/mistral/index.js +7 -0
- package/dist/services/moondream/index.d.ts +2 -0
- package/dist/services/moondream/index.js +7 -0
- package/dist/services/moonshine/index.d.ts +2 -0
- package/dist/services/moonshine/index.js +7 -0
- package/dist/services/nebius/index.d.ts +2 -0
- package/dist/services/nebius/index.js +7 -0
- package/dist/services/neuphonic/index.d.ts +2 -0
- package/dist/services/neuphonic/index.js +7 -0
- package/dist/services/novita/index.d.ts +2 -0
- package/dist/services/novita/index.js +7 -0
- package/dist/services/nvidia/index.d.ts +2 -0
- package/dist/services/nvidia/index.js +11 -0
- package/dist/services/ollama/index.d.ts +2 -0
- package/dist/services/ollama/index.js +7 -0
- package/dist/services/openai/index.d.ts +2 -0
- package/dist/services/openai/index.js +7 -0
- package/dist/services/openclaw/index.d.ts +14 -0
- package/dist/services/openclaw/index.js +19 -0
- package/dist/services/openrouter/index.d.ts +2 -0
- package/dist/services/openrouter/index.js +7 -0
- package/dist/services/perplexity/index.d.ts +2 -0
- package/dist/services/perplexity/index.js +7 -0
- package/dist/services/piper/index.d.ts +2 -0
- package/dist/services/piper/index.js +7 -0
- package/dist/services/pocket_tts/index.d.ts +2 -0
- package/dist/services/pocket_tts/index.js +7 -0
- package/dist/services/qwen/index.d.ts +2 -0
- package/dist/services/qwen/index.js +7 -0
- package/dist/services/resembleai/index.d.ts +2 -0
- package/dist/services/resembleai/index.js +7 -0
- package/dist/services/rime/index.d.ts +2 -0
- package/dist/services/rime/index.js +7 -0
- package/dist/services/sambanova/index.d.ts +2 -0
- package/dist/services/sambanova/index.js +7 -0
- package/dist/services/sarvam/index.d.ts +2 -0
- package/dist/services/sarvam/index.js +7 -0
- package/dist/services/settings.d.ts +356 -0
- package/dist/services/settings.js +437 -0
- package/dist/services/simli/index.d.ts +2 -0
- package/dist/services/simli/index.js +7 -0
- package/dist/services/smallest/index.d.ts +2 -0
- package/dist/services/smallest/index.js +7 -0
- package/dist/services/soniox/index.d.ts +2 -0
- package/dist/services/soniox/index.js +7 -0
- package/dist/services/speechify/index.d.ts +2 -0
- package/dist/services/speechify/index.js +7 -0
- package/dist/services/speechmatics/index.d.ts +2 -0
- package/dist/services/speechmatics/index.js +7 -0
- package/dist/services/stt_latency.d.ts +62 -0
- package/dist/services/stt_latency.js +71 -0
- package/dist/services/stt_service.d.ts +601 -0
- package/dist/services/stt_service.js +1044 -0
- package/dist/services/tavus/index.d.ts +2 -0
- package/dist/services/tavus/index.js +7 -0
- package/dist/services/together/index.d.ts +2 -0
- package/dist/services/together/index.js +7 -0
- package/dist/services/tts_service.d.ts +909 -0
- package/dist/services/tts_service.js +2018 -0
- package/dist/services/ultravox/index.d.ts +2 -0
- package/dist/services/ultravox/index.js +9 -0
- package/dist/services/vision_service.d.ts +56 -0
- package/dist/services/vision_service.js +62 -0
- package/dist/services/websocket_service.d.ts +213 -0
- package/dist/services/websocket_service.js +654 -0
- package/dist/services/whisper/index.d.ts +2 -0
- package/dist/services/whisper/index.js +9 -0
- package/dist/services/xai/index.d.ts +2 -0
- package/dist/services/xai/index.js +9 -0
- package/dist/services/xtts/index.d.ts +2 -0
- package/dist/services/xtts/index.js +9 -0
- package/dist/tests/index.d.ts +2 -0
- package/dist/tests/index.js +8 -0
- package/dist/tests/utils.d.ts +123 -0
- package/dist/tests/utils.js +212 -0
- package/dist/transcriptions/index.d.ts +2 -0
- package/dist/transcriptions/index.js +8 -0
- package/dist/transcriptions/language.d.ts +376 -0
- package/dist/transcriptions/language.js +543 -0
- package/dist/transports/base_input.d.ts +149 -0
- package/dist/transports/base_input.js +338 -0
- package/dist/transports/base_output.d.ts +432 -0
- package/dist/transports/base_output.js +1231 -0
- package/dist/transports/base_transport.d.ts +223 -0
- package/dist/transports/base_transport.js +167 -0
- package/dist/transports/daily/index.d.ts +2 -0
- package/dist/transports/daily/index.js +9 -0
- package/dist/transports/heygen/index.d.ts +2 -0
- package/dist/transports/heygen/index.js +9 -0
- package/dist/transports/index.d.ts +2 -0
- package/dist/transports/index.js +8 -0
- package/dist/transports/lemonslice/index.d.ts +2 -0
- package/dist/transports/lemonslice/index.js +9 -0
- package/dist/transports/livekit/index.d.ts +2 -0
- package/dist/transports/livekit/index.js +9 -0
- package/dist/transports/local/index.d.ts +2 -0
- package/dist/transports/local/index.js +9 -0
- package/dist/transports/moq/index.d.ts +2 -0
- package/dist/transports/moq/index.js +9 -0
- package/dist/transports/smallwebrtc/index.d.ts +2 -0
- package/dist/transports/smallwebrtc/index.js +9 -0
- package/dist/transports/tavus/index.d.ts +2 -0
- package/dist/transports/tavus/index.js +9 -0
- package/dist/transports/video_in_sampler.d.ts +157 -0
- package/dist/transports/video_in_sampler.js +258 -0
- package/dist/transports/vonage/index.d.ts +2 -0
- package/dist/transports/vonage/index.js +9 -0
- package/dist/transports/websocket/index.d.ts +2 -0
- package/dist/transports/websocket/index.js +9 -0
- package/dist/transports/whatsapp/index.d.ts +2 -0
- package/dist/transports/whatsapp/index.js +9 -0
- package/dist/turns/eager_end_of_turn_mixin.d.ts +70 -0
- package/dist/turns/eager_end_of_turn_mixin.js +122 -0
- package/dist/turns/empty_user_turn.d.ts +55 -0
- package/dist/turns/empty_user_turn.js +53 -0
- package/dist/turns/index.d.ts +2 -0
- package/dist/turns/index.js +8 -0
- package/dist/turns/speculation_gate.d.ts +169 -0
- package/dist/turns/speculation_gate.js +294 -0
- package/dist/turns/types.d.ts +33 -0
- package/dist/turns/types.js +35 -0
- package/dist/turns/user_idle_controller.d.ts +94 -0
- package/dist/turns/user_idle_controller.js +187 -0
- package/dist/turns/user_mute/always_user_mute_strategy.d.ts +15 -0
- package/dist/turns/user_mute/always_user_mute_strategy.js +29 -0
- package/dist/turns/user_mute/base_user_mute_strategy.d.ts +39 -0
- package/dist/turns/user_mute/base_user_mute_strategy.js +45 -0
- package/dist/turns/user_mute/first_speech_user_mute_strategy.d.ts +28 -0
- package/dist/turns/user_mute/first_speech_user_mute_strategy.js +52 -0
- package/dist/turns/user_mute/function_call_user_mute_strategy.d.ts +25 -0
- package/dist/turns/user_mute/function_call_user_mute_strategy.js +50 -0
- package/dist/turns/user_mute/index.d.ts +6 -0
- package/dist/turns/user_mute/index.js +11 -0
- package/dist/turns/user_mute/mute_until_first_bot_complete_user_mute_strategy.d.ts +43 -0
- package/dist/turns/user_mute/mute_until_first_bot_complete_user_mute_strategy.js +76 -0
- package/dist/turns/user_start/base_user_turn_start_strategy.d.ts +135 -0
- package/dist/turns/user_start/base_user_turn_start_strategy.js +134 -0
- package/dist/turns/user_start/external_user_turn_start_strategy.d.ts +45 -0
- package/dist/turns/user_start/external_user_turn_start_strategy.js +64 -0
- package/dist/turns/user_start/index.d.ts +8 -0
- package/dist/turns/user_start/index.js +16 -0
- package/dist/turns/user_start/krisp_viva_ip_user_turn_start_strategy.d.ts +133 -0
- package/dist/turns/user_start/krisp_viva_ip_user_turn_start_strategy.js +243 -0
- package/dist/turns/user_start/min_words_user_turn_start_strategy.d.ts +60 -0
- package/dist/turns/user_start/min_words_user_turn_start_strategy.js +102 -0
- package/dist/turns/user_start/transcription_user_turn_start_strategy.d.ts +29 -0
- package/dist/turns/user_start/transcription_user_turn_start_strategy.js +43 -0
- package/dist/turns/user_start/vad_user_turn_start_strategy.d.ts +20 -0
- package/dist/turns/user_start/vad_user_turn_start_strategy.js +31 -0
- package/dist/turns/user_start/wake_phrase_user_turn_start_strategy.d.ts +141 -0
- package/dist/turns/user_start/wake_phrase_user_turn_start_strategy.js +238 -0
- package/dist/turns/user_stop/base_user_turn_stop_strategy.d.ts +174 -0
- package/dist/turns/user_stop/base_user_turn_stop_strategy.js +170 -0
- package/dist/turns/user_stop/deferred_user_turn_stop_strategy.d.ts +81 -0
- package/dist/turns/user_stop/deferred_user_turn_stop_strategy.js +107 -0
- package/dist/turns/user_stop/eager_match_policy.d.ts +36 -0
- package/dist/turns/user_stop/eager_match_policy.js +50 -0
- package/dist/turns/user_stop/eager_user_turn_stop_strategy.d.ts +84 -0
- package/dist/turns/user_stop/eager_user_turn_stop_strategy.js +154 -0
- package/dist/turns/user_stop/external_user_turn_completion_stop_strategy.d.ts +45 -0
- package/dist/turns/user_stop/external_user_turn_completion_stop_strategy.js +55 -0
- package/dist/turns/user_stop/external_user_turn_stop_strategy.d.ts +115 -0
- package/dist/turns/user_stop/external_user_turn_stop_strategy.js +205 -0
- package/dist/turns/user_stop/index.d.ts +10 -0
- package/dist/turns/user_stop/index.js +15 -0
- package/dist/turns/user_stop/llm_turn_completion_user_turn_stop_strategy.d.ts +44 -0
- package/dist/turns/user_stop/llm_turn_completion_user_turn_stop_strategy.js +59 -0
- package/dist/turns/user_stop/speech_timeout_user_turn_stop_strategy.d.ts +140 -0
- package/dist/turns/user_stop/speech_timeout_user_turn_stop_strategy.js +303 -0
- package/dist/turns/user_stop/turn_analyzer_user_turn_stop_strategy.d.ts +135 -0
- package/dist/turns/user_stop/turn_analyzer_user_turn_stop_strategy.js +332 -0
- package/dist/turns/user_turn_completion_mixin.d.ts +147 -0
- package/dist/turns/user_turn_completion_mixin.js +591 -0
- package/dist/turns/user_turn_controller.d.ts +134 -0
- package/dist/turns/user_turn_controller.js +335 -0
- package/dist/turns/user_turn_processor.d.ts +87 -0
- package/dist/turns/user_turn_processor.js +180 -0
- package/dist/turns/user_turn_strategies.d.ts +187 -0
- package/dist/turns/user_turn_strategies.js +222 -0
- package/dist/utils/async_tool_cancellation.d.ts +35 -0
- package/dist/utils/async_tool_cancellation.js +78 -0
- package/dist/utils/asyncio/index.d.ts +2 -0
- package/dist/utils/asyncio/index.js +7 -0
- package/dist/utils/asyncio/task_manager.d.ts +49 -0
- package/dist/utils/asyncio/task_manager.js +105 -0
- package/dist/utils/base_object.d.ts +76 -0
- package/dist/utils/base_object.js +154 -0
- package/dist/utils/context/aggregated_frame_sequencer.d.ts +491 -0
- package/dist/utils/context/aggregated_frame_sequencer.js +803 -0
- package/dist/utils/context/audio_context_manager.d.ts +131 -0
- package/dist/utils/context/audio_context_manager.js +233 -0
- package/dist/utils/context/index.d.ts +2 -0
- package/dist/utils/context/index.js +7 -0
- package/dist/utils/context/llm_context_summarization.d.ts +258 -0
- package/dist/utils/context/llm_context_summarization.js +511 -0
- package/dist/utils/context/text_segment_map.d.ts +611 -0
- package/dist/utils/context/text_segment_map.js +1132 -0
- package/dist/utils/context/word_completion_tracker.d.ts +273 -0
- package/dist/utils/context/word_completion_tracker.js +441 -0
- package/dist/utils/context/word_timestamp_tracker.d.ts +72 -0
- package/dist/utils/context/word_timestamp_tracker.js +98 -0
- package/dist/utils/deprecation.d.ts +28 -0
- package/dist/utils/deprecation.js +75 -0
- package/dist/utils/env.d.ts +38 -0
- package/dist/utils/env.js +62 -0
- package/dist/utils/errors.d.ts +75 -0
- package/dist/utils/errors.js +202 -0
- package/dist/utils/file_resolver.d.ts +126 -0
- package/dist/utils/file_resolver.js +314 -0
- package/dist/utils/file_storage.d.ts +115 -0
- package/dist/utils/file_storage.js +178 -0
- package/dist/utils/frame_queue.d.ts +36 -0
- package/dist/utils/frame_queue.js +66 -0
- package/dist/utils/http.d.ts +39 -0
- package/dist/utils/http.js +53 -0
- package/dist/utils/image.d.ts +14 -0
- package/dist/utils/image.js +63 -0
- package/dist/utils/index.d.ts +2 -0
- package/dist/utils/index.js +8 -0
- package/dist/utils/jpeg_encoder.d.ts +14 -0
- package/dist/utils/jpeg_encoder.js +315 -0
- package/dist/utils/log_config.d.ts +34 -0
- package/dist/utils/log_config.js +76 -0
- package/dist/utils/logger.d.ts +40 -0
- package/dist/utils/logger.js +91 -0
- package/dist/utils/mixin.d.ts +14 -0
- package/dist/utils/mixin.js +63 -0
- package/dist/utils/network.d.ts +73 -0
- package/dist/utils/network.js +90 -0
- package/dist/utils/security/allowed_origins.d.ts +16 -0
- package/dist/utils/security/allowed_origins.js +34 -0
- package/dist/utils/security/index.d.ts +2 -0
- package/dist/utils/security/index.js +7 -0
- package/dist/utils/security/redaction.d.ts +27 -0
- package/dist/utils/security/redaction.js +123 -0
- package/dist/utils/security/ssrf.d.ts +79 -0
- package/dist/utils/security/ssrf.js +327 -0
- package/dist/utils/shared.d.ts +35 -0
- package/dist/utils/shared.js +136 -0
- package/dist/utils/startup.d.ts +17 -0
- package/dist/utils/startup.js +130 -0
- package/dist/utils/string.d.ts +116 -0
- package/dist/utils/string.js +460 -0
- package/dist/utils/sync/base_notifier.d.ts +25 -0
- package/dist/utils/sync/base_notifier.js +16 -0
- package/dist/utils/sync/event_notifier.d.ts +38 -0
- package/dist/utils/sync/event_notifier.js +51 -0
- package/dist/utils/sync/index.d.ts +2 -0
- package/dist/utils/sync/index.js +7 -0
- package/dist/utils/text/alnum_utils.d.ts +109 -0
- package/dist/utils/text/alnum_utils.js +262 -0
- package/dist/utils/text/base_text_aggregator.d.ts +146 -0
- package/dist/utils/text/base_text_aggregator.js +102 -0
- package/dist/utils/text/base_text_filter.d.ts +54 -0
- package/dist/utils/text/base_text_filter.js +50 -0
- package/dist/utils/text/index.d.ts +2 -0
- package/dist/utils/text/index.js +7 -0
- package/dist/utils/text/markdown_text_filter.d.ts +147 -0
- package/dist/utils/text/markdown_text_filter.js +522 -0
- package/dist/utils/text/markup_utils.d.ts +54 -0
- package/dist/utils/text/markup_utils.js +156 -0
- package/dist/utils/text/pattern_pair_aggregator.d.ts +251 -0
- package/dist/utils/text/pattern_pair_aggregator.js +478 -0
- package/dist/utils/text/phonemes.d.ts +60 -0
- package/dist/utils/text/phonemes.js +172 -0
- package/dist/utils/text/simple_text_aggregator.d.ts +135 -0
- package/dist/utils/text/simple_text_aggregator.js +254 -0
- package/dist/utils/text/skip_tags_aggregator.d.ts +85 -0
- package/dist/utils/text/skip_tags_aggregator.js +147 -0
- package/dist/utils/text/transforms/acronyms.d.ts +21 -0
- package/dist/utils/text/transforms/acronyms.js +31 -0
- package/dist/utils/text/transforms/currency.d.ts +16 -0
- package/dist/utils/text/transforms/currency.js +56 -0
- package/dist/utils/text/transforms/dates.d.ts +18 -0
- package/dist/utils/text/transforms/dates.js +102 -0
- package/dist/utils/text/transforms/email.d.ts +17 -0
- package/dist/utils/text/transforms/email.js +35 -0
- package/dist/utils/text/transforms/index.d.ts +37 -0
- package/dist/utils/text/transforms/index.js +42 -0
- package/dist/utils/text/transforms/numbers.d.ts +25 -0
- package/dist/utils/text/transforms/numbers.js +59 -0
- package/dist/utils/text/transforms/percentages.d.ts +16 -0
- package/dist/utils/text/transforms/percentages.js +32 -0
- package/dist/utils/text/transforms/phone.d.ts +21 -0
- package/dist/utils/text/transforms/phone.js +39 -0
- package/dist/utils/text/transforms/pronunciations.d.ts +65 -0
- package/dist/utils/text/transforms/pronunciations.js +112 -0
- package/dist/utils/text/transforms/replacements.d.ts +39 -0
- package/dist/utils/text/transforms/replacements.js +46 -0
- package/dist/utils/text/transforms/strip_markdown.d.ts +25 -0
- package/dist/utils/text/transforms/strip_markdown.js +60 -0
- package/dist/utils/text/transforms/units.d.ts +22 -0
- package/dist/utils/text/transforms/units.js +81 -0
- package/dist/utils/text/transforms/voice_formatter.d.ts +86 -0
- package/dist/utils/text/transforms/voice_formatter.js +98 -0
- package/dist/utils/text/unicode_digits.d.ts +3 -0
- package/dist/utils/text/unicode_digits.js +34 -0
- package/dist/utils/text/word_timestamp_utils.d.ts +26 -0
- package/dist/utils/text/word_timestamp_utils.js +52 -0
- package/dist/utils/time.d.ts +13 -0
- package/dist/utils/time.js +31 -0
- package/dist/utils/tracing/index.d.ts +3 -0
- package/dist/utils/tracing/index.js +8 -0
- package/dist/utils/tracing/service_attributes.d.ts +198 -0
- package/dist/utils/tracing/service_attributes.js +434 -0
- package/dist/utils/tracing/service_decorators.d.ts +214 -0
- package/dist/utils/tracing/service_decorators.js +1662 -0
- package/dist/utils/tracing/setup.d.ts +107 -0
- package/dist/utils/tracing/setup.js +160 -0
- package/dist/utils/tracing/tracing_context.d.ts +55 -0
- package/dist/utils/tracing/tracing_context.js +97 -0
- package/dist/utils/tracing/turn_trace_observer.d.ts +95 -0
- package/dist/utils/tracing/turn_trace_observer.js +233 -0
- package/dist/utils/types.d.ts +80 -0
- package/dist/utils/types.js +108 -0
- package/dist/utils/utils.d.ts +5 -0
- package/dist/utils/utils.js +22 -0
- package/dist/utils/yaml.d.ts +179 -0
- package/dist/utils/yaml.js +556 -0
- package/dist/version.d.ts +3 -0
- package/dist/version.js +7 -0
- package/dist/workers/base_worker.d.ts +772 -0
- package/dist/workers/base_worker.js +1298 -0
- package/dist/workers/index.d.ts +2 -0
- package/dist/workers/index.js +8 -0
- package/dist/workers/llm/backend_llm_worker.d.ts +552 -0
- package/dist/workers/llm/backend_llm_worker.js +1055 -0
- package/dist/workers/llm/index.d.ts +6 -0
- package/dist/workers/llm/index.js +11 -0
- package/dist/workers/llm/llm_context_worker.d.ts +66 -0
- package/dist/workers/llm/llm_context_worker.js +72 -0
- package/dist/workers/llm/llm_worker.d.ts +180 -0
- package/dist/workers/llm/llm_worker.js +323 -0
- package/dist/workers/llm/tool_decorator.d.ts +68 -0
- package/dist/workers/llm/tool_decorator.js +60 -0
- package/dist/workers/proxy/index.d.ts +3 -0
- package/dist/workers/proxy/index.js +9 -0
- package/dist/workers/proxy/websocket/client.d.ts +110 -0
- package/dist/workers/proxy/websocket/client.js +196 -0
- package/dist/workers/proxy/websocket/index.d.ts +3 -0
- package/dist/workers/proxy/websocket/index.js +10 -0
- package/dist/workers/runner.d.ts +268 -0
- package/dist/workers/runner.js +489 -0
- package/dist/workers/ui/index.d.ts +22 -0
- package/dist/workers/ui/index.js +27 -0
- package/dist/workers/ui/ui_event_decorator.d.ts +43 -0
- package/dist/workers/ui/ui_event_decorator.js +75 -0
- package/dist/workers/ui/ui_job_context.d.ts +12 -0
- package/dist/workers/ui/ui_job_context.js +17 -0
- package/dist/workers/ui/ui_prompts.d.ts +21 -0
- package/dist/workers/ui/ui_prompts.js +41 -0
- package/dist/workers/ui/ui_tools.d.ts +32 -0
- package/dist/workers/ui/ui_tools.js +112 -0
- package/dist/workers/ui/ui_worker.d.ts +571 -0
- package/dist/workers/ui/ui_worker.js +1156 -0
- package/package.json +61 -3
- package/README.md +0 -5
|
@@ -0,0 +1,441 @@
|
|
|
1
|
+
// Copyright (c) 2024-2026, Daily
|
|
2
|
+
// Copyright (c) 2026, pipedog contributors
|
|
3
|
+
// SPDX-License-Identifier: BSD-2-Clause
|
|
4
|
+
//
|
|
5
|
+
// Ported from pipecat: src/pipecat/utils/context/word_completion_tracker.py
|
|
6
|
+
/** Per-frame bookkeeping for the words a TTS provider reports speaking. */
|
|
7
|
+
import { logger } from "../logger.js";
|
|
8
|
+
import { alnumOnly, isSpace } from "../text/alnum_utils.js";
|
|
9
|
+
import { stripCompleteMarkup, stripMarkup } from "../text/markup_utils.js";
|
|
10
|
+
import { TextSegmentMap } from "./text_segment_map.js";
|
|
11
|
+
/**
|
|
12
|
+
* Follows one AggregatedTextFrame from dispatch until it is fully spoken.
|
|
13
|
+
*
|
|
14
|
+
* A TTS provider reports the words it speaks one event at a time. This class
|
|
15
|
+
* consumes those events for a single frame and answers, for each one:
|
|
16
|
+
*
|
|
17
|
+
* - What text should the emitted `TTSTextFrame` carry? -- `getWordForFrame`
|
|
18
|
+
* - Which of the LLM's own text does that stand for? -- `getLlmConsumed`
|
|
19
|
+
* - How much of the frame has been spoken? -- `getAccumulatedUserFacingText`
|
|
20
|
+
* and `getRemainingUserFacingText`
|
|
21
|
+
* - Is the frame finished? -- the return of `addWordAndCheckComplete`
|
|
22
|
+
* - Did the word run past this frame? -- `getOverflowWord`
|
|
23
|
+
*
|
|
24
|
+
* Three texts describe the same frame, and each answer above is phrased in
|
|
25
|
+
* one of them:
|
|
26
|
+
*
|
|
27
|
+
* | Text | Example | Answers about |
|
|
28
|
+
* |------------------|------------------------------|-------------------------|
|
|
29
|
+
* | `ttsText` | `<spell>4111 1111</spell>` | what was spoken |
|
|
30
|
+
* | `userFacingText` | `4111 1111` | what a UI displays |
|
|
31
|
+
* | `llmText` | `<card>4111 1111</card>` | what the context stores |
|
|
32
|
+
*
|
|
33
|
+
* Keeping a position in all three is the job of `TextSegmentMap`, which this
|
|
34
|
+
* class owns one of and defers to for every question of *where*. What the
|
|
35
|
+
* tracker adds is the handful of decisions a position alone cannot express:
|
|
36
|
+
*
|
|
37
|
+
* - **Providers drop events.** A word the provider never reported has no
|
|
38
|
+
* event coming, so the next word that does arrive is matched a little
|
|
39
|
+
* further on and the text passed over is emitted with it. When nothing
|
|
40
|
+
* matches at all, waiting would stall this frame and everything queued
|
|
41
|
+
* behind it, so the frame is force-completed instead: the unspoken remainder
|
|
42
|
+
* is emitted so the context still gets it, and the stray word is handed
|
|
43
|
+
* back for the next frame to try.
|
|
44
|
+
* - **Some text is never spoken.** A closing `</card>`, or a tag sitting
|
|
45
|
+
* between the last word and its punctuation, never arrives as its own
|
|
46
|
+
* event. Whatever is left once everything speakable is done belongs to this
|
|
47
|
+
* frame, so the word that finishes it claims the rest.
|
|
48
|
+
* - **A word can belong to two frames.** A provider may merge across the
|
|
49
|
+
* boundary (`"1111And"`). The part that fits stays here; the rest is
|
|
50
|
+
* exposed as overflow for the caller to feed to the next frame.
|
|
51
|
+
*
|
|
52
|
+
* @example
|
|
53
|
+
* const tracker = new WordCompletionTracker("Hello, world!");
|
|
54
|
+
* tracker.addWordAndCheckComplete("Hello"); // false
|
|
55
|
+
* tracker.addWordAndCheckComplete("world"); // true -- nothing left to speak
|
|
56
|
+
*/
|
|
57
|
+
export class WordCompletionTracker {
|
|
58
|
+
// --- The three texts ---
|
|
59
|
+
_ttsText;
|
|
60
|
+
_userFacingText;
|
|
61
|
+
_llmText;
|
|
62
|
+
// --- Cursors into two of them ---
|
|
63
|
+
// The map owns the authoritative positions; these mirror it, and only
|
|
64
|
+
// diverge where the tracker deliberately moves further (see
|
|
65
|
+
// _recordLlmSpan and _forceComplete). The position in ttsText is not
|
|
66
|
+
// mirrored -- the map's rawPos is read directly.
|
|
67
|
+
_userFacingPos = 0;
|
|
68
|
+
_llmPos = 0;
|
|
69
|
+
_llmSpokenPos = 0;
|
|
70
|
+
// --- Answers about the most recent word ---
|
|
71
|
+
// Rewritten by every addWordAndCheckComplete call and read back through
|
|
72
|
+
// the get* accessors.
|
|
73
|
+
_frameWord = null; // this frame's share of the word
|
|
74
|
+
_overflowWord = null; // the next frame's share
|
|
75
|
+
_llmConsumed = null; // LLM text the word stands for
|
|
76
|
+
// Set by _forceComplete. The map is never advanced there, so its own
|
|
77
|
+
// isComplete would keep saying false; this flag is the answer instead.
|
|
78
|
+
_forceCompleted = false;
|
|
79
|
+
_segmentMap;
|
|
80
|
+
/**
|
|
81
|
+
* Initialize the tracker with the frame's three texts.
|
|
82
|
+
*
|
|
83
|
+
* Only `ttsText` is required; the other two default to it, which is exactly
|
|
84
|
+
* right for a frame nothing rewrote.
|
|
85
|
+
*
|
|
86
|
+
* @param ttsText What was sent to the TTS, and so what the incoming words
|
|
87
|
+
* are matched against. May carry synthesis tags (`<spell>...`).
|
|
88
|
+
* @param options.llmText What the LLM wrote (see `WordCompletionTrackerOptions`).
|
|
89
|
+
* @param options.userFacingText What a client displays.
|
|
90
|
+
*/
|
|
91
|
+
constructor(ttsText, options = {}) {
|
|
92
|
+
this._ttsText = ttsText;
|
|
93
|
+
// Stripping markup from the fallback keeps synthesis tags out of what a UI
|
|
94
|
+
// shows, and gives the map a plain run to split around any tag rather than
|
|
95
|
+
// one atomic string.
|
|
96
|
+
this._userFacingText =
|
|
97
|
+
options.userFacingText !== null && options.userFacingText !== undefined
|
|
98
|
+
? options.userFacingText
|
|
99
|
+
: stripCompleteMarkup(ttsText);
|
|
100
|
+
this._llmText = options.llmText ?? null;
|
|
101
|
+
this._segmentMap = new TextSegmentMap(ttsText, this._userFacingText, this._llmText);
|
|
102
|
+
}
|
|
103
|
+
/**
|
|
104
|
+
* Record one word the TTS provider reported speaking.
|
|
105
|
+
*
|
|
106
|
+
* Three things can happen, in this order:
|
|
107
|
+
*
|
|
108
|
+
* 1. The frame is already finished -- the word is ignored.
|
|
109
|
+
* 2. The word does not match what is left to speak, so the provider must
|
|
110
|
+
* have dropped an event: the frame is force-completed (see
|
|
111
|
+
* `_forceComplete`) and this word is handed back as overflow.
|
|
112
|
+
* 3. Otherwise the word advances the frame. Afterwards the `get*` accessors
|
|
113
|
+
* describe it: this frame's share of the word, the LLM text it stands
|
|
114
|
+
* for, and how much of the frame is now spoken.
|
|
115
|
+
*
|
|
116
|
+
* @param word One token from the provider's word-timestamp stream. It may
|
|
117
|
+
* be a plain word, a word carrying its own spacing or punctuation, or a
|
|
118
|
+
* fragment of a still-open SSML tag -- matching is textual, so none of
|
|
119
|
+
* those need special handling from the caller. Services that report
|
|
120
|
+
* spaces and punctuation as separate tokens (e.g. Inworld) must merge
|
|
121
|
+
* them into the preceding word first, via `mergePunctTokens`.
|
|
122
|
+
* @returns True once nothing is left for this frame to speak.
|
|
123
|
+
*/
|
|
124
|
+
addWordAndCheckComplete(word) {
|
|
125
|
+
this._frameWord = null;
|
|
126
|
+
this._overflowWord = null;
|
|
127
|
+
this._llmConsumed = null;
|
|
128
|
+
// Every raw character consumed, not isComplete: a frame ending in an
|
|
129
|
+
// emoji contributes no alphanumeric content, so it reads as complete
|
|
130
|
+
// before that emoji's own event arrives, and that event is still wanted.
|
|
131
|
+
if (this._forceCompleted || this._segmentMap.rawPos >= this._ttsText.length) {
|
|
132
|
+
logger.warning(`${this}, trying to add a word in an already complete frame`);
|
|
133
|
+
return true;
|
|
134
|
+
}
|
|
135
|
+
if (!this.wordBelongsHere(word)) {
|
|
136
|
+
return this._forceComplete(word);
|
|
137
|
+
}
|
|
138
|
+
const llmPosBefore = this._llmPos;
|
|
139
|
+
const spokenBefore = this._llmSpokenPos;
|
|
140
|
+
const rawPosBefore = this._segmentMap.rawPos;
|
|
141
|
+
this._segmentMap.advanceWord(word);
|
|
142
|
+
const overflow = this._segmentMap.lastOverflow;
|
|
143
|
+
const spoken = this._ttsText.slice(rawPosBefore, this._segmentMap.rawPos);
|
|
144
|
+
// If the spoken text is longer than the word reported by the provider, the
|
|
145
|
+
// provider skipped some text. Markup does not count because providers don't
|
|
146
|
+
// report tags, and we compare only letters and digits so differences in case,
|
|
147
|
+
// accents, punctuation, or spacing don't affect the check.
|
|
148
|
+
const coversSkippedText = alnumOnly(stripMarkup(spoken)).length > alnumOnly(word).length;
|
|
149
|
+
if (coversSkippedText) {
|
|
150
|
+
// The skipped text will never be reported by the provider, so include it
|
|
151
|
+
// in this frame together with the word that brings the tracker back in sync.
|
|
152
|
+
this._frameWord = spoken;
|
|
153
|
+
}
|
|
154
|
+
else {
|
|
155
|
+
// Keep only the part of the word that belongs to this frame.
|
|
156
|
+
// The beginning may overlap with text already emitted by the previous
|
|
157
|
+
// frame, while the end may overlap with the next frame.
|
|
158
|
+
const head = this._llmText !== null ? this._segmentMap.lastLeadingDuplicate : 0;
|
|
159
|
+
const tail = overflow ? word.length - overflow.length : word.length;
|
|
160
|
+
this._frameWord = word.slice(head, tail);
|
|
161
|
+
}
|
|
162
|
+
this._overflowWord = overflow;
|
|
163
|
+
this._userFacingPos = this._segmentMap.userFacingPos;
|
|
164
|
+
this._llmPos = this._segmentMap.llmPos;
|
|
165
|
+
this._llmSpokenPos = this._segmentMap.llmSpokenPos;
|
|
166
|
+
if (this._llmText !== null) {
|
|
167
|
+
this._recordLlmSpan(word, llmPosBefore, spokenBefore);
|
|
168
|
+
}
|
|
169
|
+
const complete = this.isComplete;
|
|
170
|
+
if (complete) {
|
|
171
|
+
// Everything speakable has been spoken, so anything still left is
|
|
172
|
+
// text no word will ever arrive for -- a closing tag, or one sitting
|
|
173
|
+
// between the last word and its punctuation. It belongs to this
|
|
174
|
+
// frame, so take it rather than leave it out of the turn.
|
|
175
|
+
this._userFacingPos = this._userFacingText.length;
|
|
176
|
+
}
|
|
177
|
+
return complete;
|
|
178
|
+
}
|
|
179
|
+
/**
|
|
180
|
+
* End this frame early because `word` does not belong to it.
|
|
181
|
+
*
|
|
182
|
+
* The provider dropped one or more events, so the rest of this frame will
|
|
183
|
+
* never be reported. Rather than stall, emit the unspoken remainder as this
|
|
184
|
+
* frame's word -- the conversation context still receives the full text --
|
|
185
|
+
* and hand `word` back as overflow for the next frame to try.
|
|
186
|
+
*
|
|
187
|
+
* The segment map is deliberately left where it is, since nothing here was
|
|
188
|
+
* actually spoken; `_forceCompleted` answers for it from now on.
|
|
189
|
+
*
|
|
190
|
+
* @returns Always true -- the frame is finished.
|
|
191
|
+
*/
|
|
192
|
+
_forceComplete(word) {
|
|
193
|
+
this._frameWord = this._ttsText.slice(this._segmentMap.rawPos);
|
|
194
|
+
this._userFacingPos = this._userFacingText.length;
|
|
195
|
+
if (this._llmText !== null) {
|
|
196
|
+
// The whole remainder is this frame's by definition, tags included.
|
|
197
|
+
this._llmConsumed = this._llmText.slice(this._llmSpokenPos);
|
|
198
|
+
this._llmPos = this._llmText.length;
|
|
199
|
+
this._llmSpokenPos = this._llmText.length;
|
|
200
|
+
}
|
|
201
|
+
this._forceCompleted = true;
|
|
202
|
+
this._overflowWord = word;
|
|
203
|
+
return true;
|
|
204
|
+
}
|
|
205
|
+
/**
|
|
206
|
+
* Move the cursors to the end, for a caller that emits the remainder itself.
|
|
207
|
+
*
|
|
208
|
+
* The accumulated and remaining views then agree with the text that went out.
|
|
209
|
+
*/
|
|
210
|
+
takeRemainingAsSpoken() {
|
|
211
|
+
this._userFacingPos = this._userFacingText.length;
|
|
212
|
+
if (this._llmText !== null) {
|
|
213
|
+
this._llmPos = this._llmText.length;
|
|
214
|
+
this._llmSpokenPos = this._llmText.length;
|
|
215
|
+
}
|
|
216
|
+
}
|
|
217
|
+
/**
|
|
218
|
+
* Record which part of `llmText` the word just added stands for.
|
|
219
|
+
*
|
|
220
|
+
* Usually that is simply the span the attributed cursor moved over. Two
|
|
221
|
+
* cases reach further, and both leave `_llmPos` ahead of the map's:
|
|
222
|
+
*
|
|
223
|
+
* - **The word finished the frame**: take everything to the end of
|
|
224
|
+
* `llmText`. The map stops at the last spoken character, so a closing tag
|
|
225
|
+
* -- which never arrives as its own event -- is still outstanding and
|
|
226
|
+
* belongs to this word.
|
|
227
|
+
* - **Neither cursor moved**: the map accepted the word without matching it
|
|
228
|
+
* to anything, as it does for a symbol the source text spells differently
|
|
229
|
+
* (a provider reporting `->` for `→`). Nothing else will ever report that
|
|
230
|
+
* character, so the word's own length is spent on it instead.
|
|
231
|
+
*
|
|
232
|
+
* A word inside a transformed segment records nothing, and is checked
|
|
233
|
+
* before that second case: the cursors are held there on purpose, so "did
|
|
234
|
+
* not move" would be misread as "stood for something unmatched" and would
|
|
235
|
+
* walk through text the transform covers. Only the word completing the
|
|
236
|
+
* segment carries its original span.
|
|
237
|
+
*
|
|
238
|
+
* An empty span is not the same as an unmatched word -- it means an earlier
|
|
239
|
+
* word already covered this one, which `suppressInContext` reports so the
|
|
240
|
+
* context does not record it twice.
|
|
241
|
+
*/
|
|
242
|
+
_recordLlmSpan(word, llmPosBefore, spokenBefore) {
|
|
243
|
+
const llmText = this._llmText;
|
|
244
|
+
if (llmText === null) {
|
|
245
|
+
throw new Error("_recordLlmSpan requires an llmText");
|
|
246
|
+
}
|
|
247
|
+
if (this.isComplete) {
|
|
248
|
+
this._llmConsumed = llmText.slice(llmPosBefore);
|
|
249
|
+
this._llmPos = llmText.length;
|
|
250
|
+
this._llmSpokenPos = llmText.length;
|
|
251
|
+
}
|
|
252
|
+
else if (this._segmentMap.inTransformedSegment) {
|
|
253
|
+
this._llmConsumed = null;
|
|
254
|
+
}
|
|
255
|
+
else if (this._llmSpokenPos === spokenBefore &&
|
|
256
|
+
this._llmPos === llmPosBefore &&
|
|
257
|
+
this._segmentMap.lastCompletedSegment === null) {
|
|
258
|
+
// Neither cursor moved, so the map accepted the word without matching
|
|
259
|
+
// it to anything -- a symbol the source text spells differently, which
|
|
260
|
+
// no other event will report. Spend the word's own length so the
|
|
261
|
+
// character it stands for still reaches the context.
|
|
262
|
+
let start = this._llmPos;
|
|
263
|
+
while (start < llmText.length && isSpace(llmText[start])) {
|
|
264
|
+
start += 1;
|
|
265
|
+
}
|
|
266
|
+
const end = start + word.length;
|
|
267
|
+
this._llmConsumed = llmText.slice(start, end);
|
|
268
|
+
this._llmPos = end;
|
|
269
|
+
}
|
|
270
|
+
else {
|
|
271
|
+
this._llmConsumed = llmText.slice(llmPosBefore, this._llmPos);
|
|
272
|
+
}
|
|
273
|
+
}
|
|
274
|
+
/**
|
|
275
|
+
* Return true if `word` plausibly continues what this frame has left to say.
|
|
276
|
+
*
|
|
277
|
+
* A false answer means the provider dropped an event. Callers ask this
|
|
278
|
+
* before adding a word so they can offer it to the next frame instead;
|
|
279
|
+
* adding it anyway force-completes this one.
|
|
280
|
+
*/
|
|
281
|
+
wordBelongsHere(word) {
|
|
282
|
+
return this._segmentMap.wordBelongsCurrentSegment(word);
|
|
283
|
+
}
|
|
284
|
+
/**
|
|
285
|
+
* True when the last word must not be written to the conversation context.
|
|
286
|
+
*
|
|
287
|
+
* Two kinds of word answer to this, both of which the context already has
|
|
288
|
+
* covered, or will have:
|
|
289
|
+
*
|
|
290
|
+
* - **One step inside a rewritten span.** `"$42.50"` is spoken as five
|
|
291
|
+
* words, none of which the transcript should contain; the word that
|
|
292
|
+
* finishes the span carries `"$42.50"` for all of them.
|
|
293
|
+
* - **A word with no span of its own.** A provider reporting `","` on its
|
|
294
|
+
* own, after `"Yeah"` took the comma into its span, has nothing left to
|
|
295
|
+
* record. The context falls back to the spoken text when a word carries
|
|
296
|
+
* no span, which would store the mark a second time.
|
|
297
|
+
*
|
|
298
|
+
* The word is still emitted either way: the provider spoke it, and a
|
|
299
|
+
* consumer reading the word stream should see it. Without an `llmText`
|
|
300
|
+
* there are no spans at all, so nothing is suppressed and every word is
|
|
301
|
+
* recorded from its spoken text as usual.
|
|
302
|
+
*/
|
|
303
|
+
suppressInContext() {
|
|
304
|
+
return (this._segmentMap.inTransformedSegment ||
|
|
305
|
+
(this._llmText !== null && !(this._llmConsumed ?? "").trim()));
|
|
306
|
+
}
|
|
307
|
+
/**
|
|
308
|
+
* Return this frame's share of the last word -- the text to emit for it.
|
|
309
|
+
*
|
|
310
|
+
* Usually the whole word. A word straddling the boundary gives up its tail
|
|
311
|
+
* (`"1111"` out of `"1111And"`), and a word that repeats the previous word's
|
|
312
|
+
* punctuation gives up that mark. A word matched past an event the provider
|
|
313
|
+
* never sent brings along the text it passed over, and after a
|
|
314
|
+
* force-complete this is the frame's unspoken remainder instead -- either
|
|
315
|
+
* way, nothing is missing from the turn.
|
|
316
|
+
*/
|
|
317
|
+
getWordForFrame() {
|
|
318
|
+
return this._frameWord ? this._frameWord.trim() : this._frameWord;
|
|
319
|
+
}
|
|
320
|
+
/**
|
|
321
|
+
* Return the part of the last word that belongs to the *next* frame.
|
|
322
|
+
*
|
|
323
|
+
* Feed it to that frame's tracker as if the provider had sent it there.
|
|
324
|
+
* Casing and punctuation are untouched so it still reads as a real word.
|
|
325
|
+
* `null` when the word fit entirely within this frame.
|
|
326
|
+
*/
|
|
327
|
+
getOverflowWord() {
|
|
328
|
+
return this._overflowWord ? this._overflowWord.trim() : this._overflowWord;
|
|
329
|
+
}
|
|
330
|
+
/**
|
|
331
|
+
* Return the LLM's own text that the last word stands for.
|
|
332
|
+
*
|
|
333
|
+
* This is what the conversation context records, so it keeps the tags and
|
|
334
|
+
* spellings the LLM wrote (`"<card>4111"`) rather than what the provider
|
|
335
|
+
* reported speaking (`"4111"`).
|
|
336
|
+
*
|
|
337
|
+
* `null` when there is nothing to attribute: no `llmText` was given, the
|
|
338
|
+
* word is mid-rewrite, or `llmText` is already exhausted (a trailing emoji
|
|
339
|
+
* it never carried).
|
|
340
|
+
*/
|
|
341
|
+
getLlmConsumed() {
|
|
342
|
+
if (!this._llmConsumed) {
|
|
343
|
+
return null;
|
|
344
|
+
}
|
|
345
|
+
return this._llmConsumed.trim() || null;
|
|
346
|
+
}
|
|
347
|
+
/**
|
|
348
|
+
* Return the part of the frame spoken so far, as the user sees it.
|
|
349
|
+
*
|
|
350
|
+
* With `getRemainingUserFacingText` this splits the frame's text in two,
|
|
351
|
+
* which is what lets a client highlight speech as it happens.
|
|
352
|
+
*/
|
|
353
|
+
getAccumulatedUserFacingText() {
|
|
354
|
+
return this._userFacingText.slice(0, this._userFacingPos);
|
|
355
|
+
}
|
|
356
|
+
/**
|
|
357
|
+
* Return the part of the frame not yet spoken, as the user sees it.
|
|
358
|
+
*
|
|
359
|
+
* @param strip Whether to trim surrounding whitespace. Pass false to keep
|
|
360
|
+
* the leading space, so accumulated + remaining reproduces the frame's
|
|
361
|
+
* text exactly -- callers that index into that text rely on it.
|
|
362
|
+
*/
|
|
363
|
+
getRemainingUserFacingText(strip = true) {
|
|
364
|
+
const remaining = this._userFacingText.slice(this._userFacingPos);
|
|
365
|
+
return strip ? remaining.trim() : remaining;
|
|
366
|
+
}
|
|
367
|
+
/**
|
|
368
|
+
* Return everything spoken so far, as it was sent to the TTS.
|
|
369
|
+
*
|
|
370
|
+
* The whole frame up to the cursor, where `getWordForFrame` describes only
|
|
371
|
+
* the most recent word.
|
|
372
|
+
*/
|
|
373
|
+
getAccumulatedTtsText() {
|
|
374
|
+
return this._ttsText.slice(0, this._segmentMap.rawPos);
|
|
375
|
+
}
|
|
376
|
+
/**
|
|
377
|
+
* Return everything spoken so far, as the LLM wrote it.
|
|
378
|
+
*
|
|
379
|
+
* The whole frame up to the cursor, where `getLlmConsumed` describes only
|
|
380
|
+
* the most recent word. `null` without an `llmText`.
|
|
381
|
+
*/
|
|
382
|
+
getAccumulatedLlmText() {
|
|
383
|
+
if (this._llmText === null) {
|
|
384
|
+
return null;
|
|
385
|
+
}
|
|
386
|
+
return this._llmText.slice(0, this._llmSpokenPos);
|
|
387
|
+
}
|
|
388
|
+
/**
|
|
389
|
+
* Return what this frame still has left to speak.
|
|
390
|
+
*
|
|
391
|
+
* Callers ending a frame early emit this, so text the provider never
|
|
392
|
+
* reported still reaches the conversation context.
|
|
393
|
+
*
|
|
394
|
+
* @param strip Whether to trim surrounding whitespace. Pass false to keep
|
|
395
|
+
* the leading space, so accumulated + remaining reproduces `ttsText`
|
|
396
|
+
* exactly.
|
|
397
|
+
*/
|
|
398
|
+
getRemainingTtsText(strip = true) {
|
|
399
|
+
const remaining = this._ttsText.slice(this._segmentMap.rawPos);
|
|
400
|
+
return strip ? remaining.trim() : remaining;
|
|
401
|
+
}
|
|
402
|
+
/**
|
|
403
|
+
* Return what this frame still has left to speak, as the LLM wrote it.
|
|
404
|
+
*
|
|
405
|
+
* The companion to `getRemainingTtsText` when ending a frame early: that
|
|
406
|
+
* supplies the text to emit, this the text to record. `null` without an
|
|
407
|
+
* `llmText`, or when nothing is left.
|
|
408
|
+
*/
|
|
409
|
+
getRemainingLlmText() {
|
|
410
|
+
if (this._llmText === null) {
|
|
411
|
+
return null;
|
|
412
|
+
}
|
|
413
|
+
const remaining = this._llmText.slice(this._llmSpokenPos).trim();
|
|
414
|
+
return remaining ? remaining : null;
|
|
415
|
+
}
|
|
416
|
+
/**
|
|
417
|
+
* True when this frame has nothing left to speak.
|
|
418
|
+
*
|
|
419
|
+
* Alphanumeric content is what counts, so a frame whose remainder is only
|
|
420
|
+
* punctuation or a closing tag is already finished -- no word will ever
|
|
421
|
+
* arrive for those.
|
|
422
|
+
*/
|
|
423
|
+
get isComplete() {
|
|
424
|
+
return this._forceCompleted || this._segmentMap.isComplete;
|
|
425
|
+
}
|
|
426
|
+
/** Rewind to the start of the frame, keeping the three texts. */
|
|
427
|
+
reset() {
|
|
428
|
+
this._userFacingPos = 0;
|
|
429
|
+
this._llmPos = 0;
|
|
430
|
+
this._llmSpokenPos = 0;
|
|
431
|
+
this._overflowWord = null;
|
|
432
|
+
this._llmConsumed = null;
|
|
433
|
+
this._frameWord = null;
|
|
434
|
+
this._forceCompleted = false;
|
|
435
|
+
this._segmentMap.reset();
|
|
436
|
+
}
|
|
437
|
+
toString() {
|
|
438
|
+
return `${this.constructor.name}(${JSON.stringify(this._ttsText)})`;
|
|
439
|
+
}
|
|
440
|
+
}
|
|
441
|
+
//# sourceMappingURL=word_completion_tracker.js.map
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
export interface WordTimestampOptions {
|
|
2
|
+
/** The word. */
|
|
3
|
+
word: string;
|
|
4
|
+
/** Seconds from the start of the word's audio. */
|
|
5
|
+
timestamp: number;
|
|
6
|
+
/** The audio context the word belongs to. */
|
|
7
|
+
contextId: string | null;
|
|
8
|
+
/** Whether the word carries its own spacing. */
|
|
9
|
+
includesInterFrameSpaces?: boolean;
|
|
10
|
+
}
|
|
11
|
+
/**
|
|
12
|
+
* A word a TTS provider reported, and when it is spoken.
|
|
13
|
+
*
|
|
14
|
+
* - `word`: the word.
|
|
15
|
+
* - `timestamp`: seconds from the start of the word's audio.
|
|
16
|
+
* - `contextId`: the audio context the word belongs to.
|
|
17
|
+
* - `includesInterFrameSpaces`: whether the word carries its own spacing.
|
|
18
|
+
*/
|
|
19
|
+
export declare class WordTimestamp {
|
|
20
|
+
word: string;
|
|
21
|
+
timestamp: number;
|
|
22
|
+
contextId: string | null;
|
|
23
|
+
includesInterFrameSpaces: boolean;
|
|
24
|
+
constructor(options: WordTimestampOptions);
|
|
25
|
+
}
|
|
26
|
+
/**
|
|
27
|
+
* Turns word times reported from the start of their audio into PTS.
|
|
28
|
+
*
|
|
29
|
+
* Timing starts when a context's audio starts playing, or after the last word
|
|
30
|
+
* if that is later, so words keep their order across contexts. Words reported
|
|
31
|
+
* before then are cached and handed back by `start`.
|
|
32
|
+
*
|
|
33
|
+
* `lastPts` is the PTS of the last word frame, and `uninterruptibleLastPts`
|
|
34
|
+
* that of the last uninterruptible one. `replyLastPts` is that of the last
|
|
35
|
+
* word since `startReply`.
|
|
36
|
+
*/
|
|
37
|
+
export declare class WordTimestampTracker {
|
|
38
|
+
protected _baseline: number | null;
|
|
39
|
+
protected _cached: WordTimestamp[];
|
|
40
|
+
protected _replyStartPts: number;
|
|
41
|
+
lastPts: number;
|
|
42
|
+
uninterruptibleLastPts: number;
|
|
43
|
+
/** Whether words are being timed. */
|
|
44
|
+
get started(): boolean;
|
|
45
|
+
/** The PTS of the reply's last word, or `null` if the reply timed no words. */
|
|
46
|
+
get replyLastPts(): number | null;
|
|
47
|
+
/** Note where a reply's words start, for `replyLastPts`. */
|
|
48
|
+
startReply(): void;
|
|
49
|
+
/**
|
|
50
|
+
* Start timing words, if not started yet.
|
|
51
|
+
*
|
|
52
|
+
* @param now The current clock time, in nanoseconds.
|
|
53
|
+
* @returns The words cached until now, to time in order.
|
|
54
|
+
*/
|
|
55
|
+
start(now: number): WordTimestamp[];
|
|
56
|
+
/**
|
|
57
|
+
* Hold a word until timing starts.
|
|
58
|
+
*
|
|
59
|
+
* @param word The word reported before timing started.
|
|
60
|
+
*/
|
|
61
|
+
cache(word: WordTimestamp): void;
|
|
62
|
+
/**
|
|
63
|
+
* Get the PTS of a word.
|
|
64
|
+
*
|
|
65
|
+
* @param timestamp Seconds from the start of the word's audio.
|
|
66
|
+
* @returns The word's PTS, in nanoseconds.
|
|
67
|
+
*/
|
|
68
|
+
pts(timestamp: number): number;
|
|
69
|
+
/** Stop timing words and drop the cached ones. */
|
|
70
|
+
reset(): void;
|
|
71
|
+
}
|
|
72
|
+
//# sourceMappingURL=word_timestamp_tracker.d.ts.map
|
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
// Copyright (c) 2024-2026, Daily
|
|
2
|
+
// Copyright (c) 2026, pipedog contributors
|
|
3
|
+
// SPDX-License-Identifier: BSD-2-Clause
|
|
4
|
+
//
|
|
5
|
+
// Ported from pipecat: src/pipecat/utils/context/word_timestamp_tracker.py
|
|
6
|
+
/** Timestamps for the words a TTS service speaks. */
|
|
7
|
+
import { secondsToNanoseconds } from "../time.js";
|
|
8
|
+
/**
|
|
9
|
+
* A word a TTS provider reported, and when it is spoken.
|
|
10
|
+
*
|
|
11
|
+
* - `word`: the word.
|
|
12
|
+
* - `timestamp`: seconds from the start of the word's audio.
|
|
13
|
+
* - `contextId`: the audio context the word belongs to.
|
|
14
|
+
* - `includesInterFrameSpaces`: whether the word carries its own spacing.
|
|
15
|
+
*/
|
|
16
|
+
export class WordTimestamp {
|
|
17
|
+
word;
|
|
18
|
+
timestamp;
|
|
19
|
+
contextId;
|
|
20
|
+
includesInterFrameSpaces;
|
|
21
|
+
constructor(options) {
|
|
22
|
+
this.word = options.word;
|
|
23
|
+
this.timestamp = options.timestamp;
|
|
24
|
+
this.contextId = options.contextId;
|
|
25
|
+
this.includesInterFrameSpaces = options.includesInterFrameSpaces ?? false;
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
/**
|
|
29
|
+
* Turns word times reported from the start of their audio into PTS.
|
|
30
|
+
*
|
|
31
|
+
* Timing starts when a context's audio starts playing, or after the last word
|
|
32
|
+
* if that is later, so words keep their order across contexts. Words reported
|
|
33
|
+
* before then are cached and handed back by `start`.
|
|
34
|
+
*
|
|
35
|
+
* `lastPts` is the PTS of the last word frame, and `uninterruptibleLastPts`
|
|
36
|
+
* that of the last uninterruptible one. `replyLastPts` is that of the last
|
|
37
|
+
* word since `startReply`.
|
|
38
|
+
*/
|
|
39
|
+
export class WordTimestampTracker {
|
|
40
|
+
_baseline = null;
|
|
41
|
+
_cached = [];
|
|
42
|
+
_replyStartPts = 0;
|
|
43
|
+
lastPts = 0;
|
|
44
|
+
uninterruptibleLastPts = 0;
|
|
45
|
+
/** Whether words are being timed. */
|
|
46
|
+
get started() {
|
|
47
|
+
return this._baseline !== null;
|
|
48
|
+
}
|
|
49
|
+
/** The PTS of the reply's last word, or `null` if the reply timed no words. */
|
|
50
|
+
get replyLastPts() {
|
|
51
|
+
return this.lastPts !== this._replyStartPts ? this.lastPts : null;
|
|
52
|
+
}
|
|
53
|
+
/** Note where a reply's words start, for `replyLastPts`. */
|
|
54
|
+
startReply() {
|
|
55
|
+
this._replyStartPts = this.lastPts;
|
|
56
|
+
}
|
|
57
|
+
/**
|
|
58
|
+
* Start timing words, if not started yet.
|
|
59
|
+
*
|
|
60
|
+
* @param now The current clock time, in nanoseconds.
|
|
61
|
+
* @returns The words cached until now, to time in order.
|
|
62
|
+
*/
|
|
63
|
+
start(now) {
|
|
64
|
+
if (this._baseline !== null) {
|
|
65
|
+
return [];
|
|
66
|
+
}
|
|
67
|
+
this._baseline = Math.max(this.lastPts, now);
|
|
68
|
+
const cached = this._cached;
|
|
69
|
+
this._cached = [];
|
|
70
|
+
return cached;
|
|
71
|
+
}
|
|
72
|
+
/**
|
|
73
|
+
* Hold a word until timing starts.
|
|
74
|
+
*
|
|
75
|
+
* @param word The word reported before timing started.
|
|
76
|
+
*/
|
|
77
|
+
cache(word) {
|
|
78
|
+
this._cached.push(word);
|
|
79
|
+
}
|
|
80
|
+
/**
|
|
81
|
+
* Get the PTS of a word.
|
|
82
|
+
*
|
|
83
|
+
* @param timestamp Seconds from the start of the word's audio.
|
|
84
|
+
* @returns The word's PTS, in nanoseconds.
|
|
85
|
+
*/
|
|
86
|
+
pts(timestamp) {
|
|
87
|
+
if (this._baseline === null) {
|
|
88
|
+
throw new Error("WordTimestampTracker.pts() called before start()");
|
|
89
|
+
}
|
|
90
|
+
return this._baseline + secondsToNanoseconds(timestamp);
|
|
91
|
+
}
|
|
92
|
+
/** Stop timing words and drop the cached ones. */
|
|
93
|
+
reset() {
|
|
94
|
+
this._baseline = null;
|
|
95
|
+
this._cached = [];
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
//# sourceMappingURL=word_timestamp_tracker.js.map
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The canonical deprecation message, for symbol and hand-written warnings
|
|
3
|
+
* alike. Kept consistent and parseable so the developer-facing message agrees
|
|
4
|
+
* with the docstring directive.
|
|
5
|
+
*/
|
|
6
|
+
export declare const DEPRECATION_MESSAGE_RE: RegExp;
|
|
7
|
+
/**
|
|
8
|
+
* Call sites already warned about by `warnDeprecated`, identified by message
|
|
9
|
+
* and source location (`message\u0000location`).
|
|
10
|
+
*
|
|
11
|
+
* The record lasts for the life of the process, so a test asserting on one of
|
|
12
|
+
* these warnings clears it first.
|
|
13
|
+
*/
|
|
14
|
+
export declare const _warnedSites: Set<string>;
|
|
15
|
+
/**
|
|
16
|
+
* Warn once per call site that something deprecated was used.
|
|
17
|
+
*
|
|
18
|
+
* Many of Pipecat's deprecations are detected inside Pipecat, as a pipeline
|
|
19
|
+
* runs, so the warning is always logged, and a per-site record keeps it to one
|
|
20
|
+
* warning per call site: a deprecated field read wherever its object travels,
|
|
21
|
+
* or a check on every frame, would otherwise repeat without bound.
|
|
22
|
+
*
|
|
23
|
+
* @param message The warning message, following `DEPRECATION_MESSAGE_RE`.
|
|
24
|
+
* @param stacklevel Counted from the caller: `1` names the line that calls this
|
|
25
|
+
* function, `2` its caller.
|
|
26
|
+
*/
|
|
27
|
+
export declare function warnDeprecated(message: string, stacklevel?: number): void;
|
|
28
|
+
//# sourceMappingURL=deprecation.d.ts.map
|