@pipedog/core 0.0.0 → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +25 -0
- package/NOTICE +9 -0
- package/dist/adapters/base_llm_adapter.d.ts +266 -0
- package/dist/adapters/base_llm_adapter.js +489 -0
- package/dist/adapters/index.d.ts +2 -0
- package/dist/adapters/index.js +8 -0
- package/dist/adapters/schemas/direct_function.d.ts +202 -0
- package/dist/adapters/schemas/direct_function.js +302 -0
- package/dist/adapters/schemas/function_schema.d.ts +67 -0
- package/dist/adapters/schemas/function_schema.js +72 -0
- package/dist/adapters/schemas/index.d.ts +2 -0
- package/dist/adapters/schemas/index.js +8 -0
- package/dist/adapters/schemas/tools_schema.d.ts +57 -0
- package/dist/adapters/schemas/tools_schema.js +79 -0
- package/dist/adapters/services/index.d.ts +2 -0
- package/dist/adapters/services/index.js +8 -0
- package/dist/adapters/services/open_ai_adapter.d.ts +147 -0
- package/dist/adapters/services/open_ai_adapter.js +250 -0
- package/dist/audio/dtmf/dtmf_data.d.ts +3 -0
- package/dist/audio/dtmf/dtmf_data.js +22 -0
- package/dist/audio/dtmf/index.d.ts +2 -0
- package/dist/audio/dtmf/index.js +8 -0
- package/dist/audio/dtmf/types.d.ts +31 -0
- package/dist/audio/dtmf/types.js +35 -0
- package/dist/audio/dtmf/utils.d.ts +13 -0
- package/dist/audio/dtmf/utils.js +62 -0
- package/dist/audio/filters/base_audio_filter.d.ts +51 -0
- package/dist/audio/filters/base_audio_filter.js +17 -0
- package/dist/audio/filters/index.d.ts +2 -0
- package/dist/audio/filters/index.js +8 -0
- package/dist/audio/index.d.ts +2 -0
- package/dist/audio/index.js +8 -0
- package/dist/audio/krisp_instance.d.ts +95 -0
- package/dist/audio/krisp_instance.js +229 -0
- package/dist/audio/mixers/base_audio_mixer.d.ts +56 -0
- package/dist/audio/mixers/base_audio_mixer.js +16 -0
- package/dist/audio/mixers/index.d.ts +2 -0
- package/dist/audio/mixers/index.js +8 -0
- package/dist/audio/resamplers/base_audio_resampler.d.ts +46 -0
- package/dist/audio/resamplers/base_audio_resampler.js +35 -0
- package/dist/audio/resamplers/index.d.ts +2 -0
- package/dist/audio/resamplers/index.js +8 -0
- package/dist/audio/resamplers/resampy_resampler.d.ts +10 -0
- package/dist/audio/resamplers/resampy_resampler.js +15 -0
- package/dist/audio/resamplers/soxr_resampler.d.ts +54 -0
- package/dist/audio/resamplers/soxr_resampler.js +69 -0
- package/dist/audio/resamplers/soxr_stream_resampler.d.ts +75 -0
- package/dist/audio/resamplers/soxr_stream_resampler.js +146 -0
- package/dist/audio/turn/base_turn_analyzer.d.ts +84 -0
- package/dist/audio/turn/base_turn_analyzer.js +61 -0
- package/dist/audio/turn/index.d.ts +2 -0
- package/dist/audio/turn/index.js +8 -0
- package/dist/audio/turn/smart_turn/_whisper_features.d.ts +30 -0
- package/dist/audio/turn/smart_turn/_whisper_features.js +225 -0
- package/dist/audio/turn/smart_turn/base_smart_turn.d.ts +94 -0
- package/dist/audio/turn/smart_turn/base_smart_turn.js +239 -0
- package/dist/audio/turn/smart_turn/local_smart_turn_v3.d.ts +76 -0
- package/dist/audio/turn/smart_turn/local_smart_turn_v3.js +143 -0
- package/dist/audio/utils.d.ts +224 -0
- package/dist/audio/utils.js +634 -0
- package/dist/audio/vad/data/index.d.ts +2 -0
- package/dist/audio/vad/data/index.js +8 -0
- package/dist/audio/vad/index.d.ts +2 -0
- package/dist/audio/vad/index.js +8 -0
- package/dist/audio/vad/vad_analyzer.d.ts +126 -0
- package/dist/audio/vad/vad_analyzer.js +211 -0
- package/dist/audio/vad/vad_controller.d.ts +143 -0
- package/dist/audio/vad/vad_controller.js +228 -0
- package/dist/audio/volume.d.ts +38 -0
- package/dist/audio/volume.js +75 -0
- package/dist/bus/adapters/base.d.ts +35 -0
- package/dist/bus/adapters/base.js +19 -0
- package/dist/bus/adapters/index.d.ts +10 -0
- package/dist/bus/adapters/index.js +15 -0
- package/dist/bus/adapters/llm_context_adapter.d.ts +36 -0
- package/dist/bus/adapters/llm_context_adapter.js +82 -0
- package/dist/bus/adapters/tools_schema_adapter.d.ts +25 -0
- package/dist/bus/adapters/tools_schema_adapter.js +70 -0
- package/dist/bus/bridge_processor.d.ts +111 -0
- package/dist/bus/bridge_processor.js +166 -0
- package/dist/bus/bus.d.ts +110 -0
- package/dist/bus/bus.js +200 -0
- package/dist/bus/index.d.ts +22 -0
- package/dist/bus/index.js +27 -0
- package/dist/bus/local/async_queue.d.ts +12 -0
- package/dist/bus/local/async_queue.js +21 -0
- package/dist/bus/local/index.d.ts +3 -0
- package/dist/bus/local/index.js +8 -0
- package/dist/bus/messages.d.ts +378 -0
- package/dist/bus/messages.js +357 -0
- package/dist/bus/network/index.d.ts +16 -0
- package/dist/bus/network/index.js +21 -0
- package/dist/bus/queue.d.ts +27 -0
- package/dist/bus/queue.js +51 -0
- package/dist/bus/serializers/base.d.ts +25 -0
- package/dist/bus/serializers/base.js +14 -0
- package/dist/bus/serializers/index.d.ts +9 -0
- package/dist/bus/serializers/index.js +14 -0
- package/dist/bus/serializers/json.d.ts +77 -0
- package/dist/bus/serializers/json.js +367 -0
- package/dist/bus/subscriber.d.ts +46 -0
- package/dist/bus/subscriber.js +27 -0
- package/dist/bus/ui/index.d.ts +3 -0
- package/dist/bus/ui/index.js +8 -0
- package/dist/bus/ui/messages.d.ts +186 -0
- package/dist/bus/ui/messages.js +171 -0
- package/dist/classifiers/base_classifier.d.ts +263 -0
- package/dist/classifiers/base_classifier.js +346 -0
- package/dist/classifiers/cloudflare/clef/classifier.d.ts +70 -0
- package/dist/classifiers/cloudflare/clef/classifier.js +204 -0
- package/dist/classifiers/cloudflare/clef/client.d.ts +104 -0
- package/dist/classifiers/cloudflare/clef/client.js +234 -0
- package/dist/classifiers/cloudflare/clef/index.d.ts +2 -0
- package/dist/classifiers/cloudflare/clef/index.js +8 -0
- package/dist/classifiers/cloudflare/index.d.ts +2 -0
- package/dist/classifiers/cloudflare/index.js +8 -0
- package/dist/classifiers/http_client.d.ts +70 -0
- package/dist/classifiers/http_client.js +161 -0
- package/dist/classifiers/index.d.ts +2 -0
- package/dist/classifiers/index.js +8 -0
- package/dist/classifiers/jev/classifier.d.ts +8 -0
- package/dist/classifiers/jev/classifier.js +13 -0
- package/dist/classifiers/jev/client.d.ts +8 -0
- package/dist/classifiers/jev/client.js +13 -0
- package/dist/classifiers/jev/index.d.ts +2 -0
- package/dist/classifiers/jev/index.js +8 -0
- package/dist/classifiers/llm/classifier.d.ts +72 -0
- package/dist/classifiers/llm/classifier.js +304 -0
- package/dist/classifiers/llm/index.d.ts +2 -0
- package/dist/classifiers/llm/index.js +8 -0
- package/dist/classifiers/openai/decisions/classifier.d.ts +77 -0
- package/dist/classifiers/openai/decisions/classifier.js +241 -0
- package/dist/classifiers/openai/decisions/client.d.ts +108 -0
- package/dist/classifiers/openai/decisions/client.js +234 -0
- package/dist/classifiers/openai/decisions/index.d.ts +2 -0
- package/dist/classifiers/openai/decisions/index.js +8 -0
- package/dist/classifiers/openai/index.d.ts +2 -0
- package/dist/classifiers/openai/index.js +8 -0
- package/dist/classifiers/typesafe/index.d.ts +2 -0
- package/dist/classifiers/typesafe/index.js +8 -0
- package/dist/classifiers/typesafe/jev/classifier.d.ts +70 -0
- package/dist/classifiers/typesafe/jev/classifier.js +220 -0
- package/dist/classifiers/typesafe/jev/client.d.ts +105 -0
- package/dist/classifiers/typesafe/jev/client.js +233 -0
- package/dist/classifiers/typesafe/jev/index.d.ts +2 -0
- package/dist/classifiers/typesafe/jev/index.js +8 -0
- package/dist/clocks/base_clock.d.ts +13 -0
- package/dist/clocks/base_clock.js +14 -0
- package/dist/clocks/index.d.ts +2 -0
- package/dist/clocks/index.js +8 -0
- package/dist/clocks/system_clock.d.ts +15 -0
- package/dist/clocks/system_clock.js +25 -0
- package/dist/frames/frames.d.ts +2533 -0
- package/dist/frames/frames.js +2485 -0
- package/dist/frames/index.d.ts +2 -0
- package/dist/frames/index.js +8 -0
- package/dist/frames/protobufs/frames_pb2.d.ts +229 -0
- package/dist/frames/protobufs/frames_pb2.js +466 -0
- package/dist/index.d.ts +11 -0
- package/dist/index.js +56 -0
- package/dist/metrics/index.d.ts +2 -0
- package/dist/metrics/index.js +8 -0
- package/dist/metrics/metrics.d.ts +276 -0
- package/dist/metrics/metrics.js +254 -0
- package/dist/observers/base_observer.d.ts +173 -0
- package/dist/observers/base_observer.js +151 -0
- package/dist/observers/error_observer.d.ts +75 -0
- package/dist/observers/error_observer.js +87 -0
- package/dist/observers/function_call_observer.d.ts +157 -0
- package/dist/observers/function_call_observer.js +222 -0
- package/dist/observers/index.d.ts +2 -0
- package/dist/observers/index.js +8 -0
- package/dist/observers/loggers/debug_log_observer.d.ts +91 -0
- package/dist/observers/loggers/debug_log_observer.js +228 -0
- package/dist/observers/loggers/index.d.ts +2 -0
- package/dist/observers/loggers/index.js +8 -0
- package/dist/observers/loggers/llm_log_observer.d.ts +24 -0
- package/dist/observers/loggers/llm_log_observer.js +110 -0
- package/dist/observers/loggers/metrics_log_observer.d.ts +85 -0
- package/dist/observers/loggers/metrics_log_observer.js +186 -0
- package/dist/observers/loggers/transcription_log_observer.d.ts +23 -0
- package/dist/observers/loggers/transcription_log_observer.js +74 -0
- package/dist/observers/service_metrics_observer.d.ts +168 -0
- package/dist/observers/service_metrics_observer.js +236 -0
- package/dist/observers/speaking_observer.d.ts +104 -0
- package/dist/observers/speaking_observer.js +150 -0
- package/dist/observers/startup_timing_observer.d.ts +221 -0
- package/dist/observers/startup_timing_observer.js +365 -0
- package/dist/observers/turn_tracking_observer.d.ts +78 -0
- package/dist/observers/turn_tracking_observer.js +189 -0
- package/dist/observers/user_bot_latency_observer.d.ts +411 -0
- package/dist/observers/user_bot_latency_observer.js +974 -0
- package/dist/pipeline/base_pipeline.d.ts +49 -0
- package/dist/pipeline/base_pipeline.js +101 -0
- package/dist/pipeline/capabilities.d.ts +76 -0
- package/dist/pipeline/capabilities.js +97 -0
- package/dist/pipeline/index.d.ts +2 -0
- package/dist/pipeline/index.js +8 -0
- package/dist/pipeline/job_context.d.ts +317 -0
- package/dist/pipeline/job_context.js +450 -0
- package/dist/pipeline/job_decorator.d.ts +54 -0
- package/dist/pipeline/job_decorator.js +92 -0
- package/dist/pipeline/llm_switcher.d.ts +84 -0
- package/dist/pipeline/llm_switcher.js +107 -0
- package/dist/pipeline/llm_with_backend.d.ts +435 -0
- package/dist/pipeline/llm_with_backend.js +806 -0
- package/dist/pipeline/parallel_pipeline.d.ts +83 -0
- package/dist/pipeline/parallel_pipeline.js +200 -0
- package/dist/pipeline/pipeline.d.ts +126 -0
- package/dist/pipeline/pipeline.js +196 -0
- package/dist/pipeline/service_switcher.d.ts +287 -0
- package/dist/pipeline/service_switcher.js +509 -0
- package/dist/pipeline/sync_parallel_pipeline.d.ts +192 -0
- package/dist/pipeline/sync_parallel_pipeline.js +387 -0
- package/dist/pipeline/worker.d.ts +814 -0
- package/dist/pipeline/worker.js +1639 -0
- package/dist/pipeline/worker_observer.d.ts +131 -0
- package/dist/pipeline/worker_observer.js +274 -0
- package/dist/pipeline/worker_ready_decorator.d.ts +45 -0
- package/dist/pipeline/worker_ready_decorator.js +76 -0
- package/dist/processors/aggregators/async_tool_messages.d.ts +146 -0
- package/dist/processors/aggregators/async_tool_messages.js +291 -0
- package/dist/processors/aggregators/dtmf_aggregator.d.ts +61 -0
- package/dist/processors/aggregators/dtmf_aggregator.js +144 -0
- package/dist/processors/aggregators/gated.d.ts +43 -0
- package/dist/processors/aggregators/gated.js +86 -0
- package/dist/processors/aggregators/gated_llm_context.d.ts +42 -0
- package/dist/processors/aggregators/gated_llm_context.js +86 -0
- package/dist/processors/aggregators/index.d.ts +2 -0
- package/dist/processors/aggregators/index.js +8 -0
- package/dist/processors/aggregators/llm_context.d.ts +370 -0
- package/dist/processors/aggregators/llm_context.js +580 -0
- package/dist/processors/aggregators/llm_context_summarizer.d.ts +219 -0
- package/dist/processors/aggregators/llm_context_summarizer.js +447 -0
- package/dist/processors/aggregators/llm_response.d.ts +37 -0
- package/dist/processors/aggregators/llm_response.js +71 -0
- package/dist/processors/aggregators/llm_response_universal.d.ts +966 -0
- package/dist/processors/aggregators/llm_response_universal.js +2394 -0
- package/dist/processors/aggregators/llm_text_processor.d.ts +53 -0
- package/dist/processors/aggregators/llm_text_processor.js +102 -0
- package/dist/processors/aggregators/sentence.d.ts +39 -0
- package/dist/processors/aggregators/sentence.js +69 -0
- package/dist/processors/async_generator.d.ts +42 -0
- package/dist/processors/async_generator.js +76 -0
- package/dist/processors/audio/audio_buffer_processor.d.ts +241 -0
- package/dist/processors/audio/audio_buffer_processor.js +565 -0
- package/dist/processors/audio/index.d.ts +2 -0
- package/dist/processors/audio/index.js +8 -0
- package/dist/processors/audio/vad_processor.d.ts +67 -0
- package/dist/processors/audio/vad_processor.js +103 -0
- package/dist/processors/consumer_processor.d.ts +49 -0
- package/dist/processors/consumer_processor.js +85 -0
- package/dist/processors/filters/frame_filter.d.ts +39 -0
- package/dist/processors/filters/frame_filter.js +52 -0
- package/dist/processors/filters/function_filter.d.ts +59 -0
- package/dist/processors/filters/function_filter.js +78 -0
- package/dist/processors/filters/identity_filter.d.ts +33 -0
- package/dist/processors/filters/identity_filter.js +37 -0
- package/dist/processors/filters/index.d.ts +2 -0
- package/dist/processors/filters/index.js +8 -0
- package/dist/processors/filters/null_filter.d.ts +34 -0
- package/dist/processors/filters/null_filter.js +47 -0
- package/dist/processors/filters/wake_notifier_filter.d.ts +48 -0
- package/dist/processors/filters/wake_notifier_filter.js +49 -0
- package/dist/processors/frame_processor.d.ts +360 -0
- package/dist/processors/frame_processor.js +932 -0
- package/dist/processors/frameworks/index.d.ts +2 -0
- package/dist/processors/frameworks/index.js +8 -0
- package/dist/processors/frameworks/rtvi/frames.d.ts +241 -0
- package/dist/processors/frameworks/rtvi/frames.js +259 -0
- package/dist/processors/frameworks/rtvi/index.d.ts +6 -0
- package/dist/processors/frameworks/rtvi/index.js +11 -0
- package/dist/processors/frameworks/rtvi/models.d.ts +1741 -0
- package/dist/processors/frameworks/rtvi/models.js +1807 -0
- package/dist/processors/frameworks/rtvi/observer.d.ts +340 -0
- package/dist/processors/frameworks/rtvi/observer.js +1113 -0
- package/dist/processors/frameworks/rtvi/processor.d.ts +182 -0
- package/dist/processors/frameworks/rtvi/processor.js +676 -0
- package/dist/processors/idle_frame_processor.d.ts +44 -0
- package/dist/processors/idle_frame_processor.js +86 -0
- package/dist/processors/index.d.ts +2 -0
- package/dist/processors/index.js +8 -0
- package/dist/processors/logger.d.ts +42 -0
- package/dist/processors/logger.js +87 -0
- package/dist/processors/metrics/frame_processor_metrics.d.ts +172 -0
- package/dist/processors/metrics/frame_processor_metrics.js +372 -0
- package/dist/processors/metrics/index.d.ts +2 -0
- package/dist/processors/metrics/index.js +8 -0
- package/dist/processors/producer_processor.d.ts +68 -0
- package/dist/processors/producer_processor.js +78 -0
- package/dist/processors/text_transformer.d.ts +30 -0
- package/dist/processors/text_transformer.js +48 -0
- package/dist/registry/index.d.ts +3 -0
- package/dist/registry/index.js +8 -0
- package/dist/registry/registry.d.ts +69 -0
- package/dist/registry/registry.js +149 -0
- package/dist/registry/types.d.ts +47 -0
- package/dist/registry/types.js +39 -0
- package/dist/serializers/base_serializer.d.ts +94 -0
- package/dist/serializers/base_serializer.js +82 -0
- package/dist/serializers/index.d.ts +2 -0
- package/dist/serializers/index.js +8 -0
- package/dist/services/ai_service.d.ts +145 -0
- package/dist/services/ai_service.js +262 -0
- package/dist/services/anthropic/index.d.ts +2 -0
- package/dist/services/anthropic/index.js +7 -0
- package/dist/services/assemblyai/index.d.ts +2 -0
- package/dist/services/assemblyai/index.js +7 -0
- package/dist/services/asyncai/index.d.ts +2 -0
- package/dist/services/asyncai/index.js +7 -0
- package/dist/services/aws/index.d.ts +2 -0
- package/dist/services/aws/index.js +7 -0
- package/dist/services/azure/index.d.ts +2 -0
- package/dist/services/azure/index.js +7 -0
- package/dist/services/baseten/index.d.ts +2 -0
- package/dist/services/baseten/index.js +7 -0
- package/dist/services/bland/index.d.ts +2 -0
- package/dist/services/bland/index.js +7 -0
- package/dist/services/camb/index.d.ts +2 -0
- package/dist/services/camb/index.js +7 -0
- package/dist/services/cartesia/index.d.ts +2 -0
- package/dist/services/cartesia/index.js +7 -0
- package/dist/services/cerebras/index.d.ts +2 -0
- package/dist/services/cerebras/index.js +7 -0
- package/dist/services/crusoe/index.d.ts +2 -0
- package/dist/services/crusoe/index.js +7 -0
- package/dist/services/deepgram/index.d.ts +2 -0
- package/dist/services/deepgram/index.js +7 -0
- package/dist/services/deepseek/index.d.ts +2 -0
- package/dist/services/deepseek/index.js +7 -0
- package/dist/services/elevenlabs/index.d.ts +2 -0
- package/dist/services/elevenlabs/index.js +7 -0
- package/dist/services/fal/index.d.ts +2 -0
- package/dist/services/fal/index.js +7 -0
- package/dist/services/fireworks/index.d.ts +2 -0
- package/dist/services/fireworks/index.js +7 -0
- package/dist/services/fish/index.d.ts +2 -0
- package/dist/services/fish/index.js +7 -0
- package/dist/services/funasr/index.d.ts +3 -0
- package/dist/services/funasr/index.js +8 -0
- package/dist/services/gladia/index.d.ts +2 -0
- package/dist/services/gladia/index.js +7 -0
- package/dist/services/google/index.d.ts +2 -0
- package/dist/services/google/index.js +7 -0
- package/dist/services/gradium/index.d.ts +2 -0
- package/dist/services/gradium/index.js +7 -0
- package/dist/services/grok/index.d.ts +2 -0
- package/dist/services/grok/index.js +7 -0
- package/dist/services/groq/index.d.ts +2 -0
- package/dist/services/groq/index.js +7 -0
- package/dist/services/heygen/index.d.ts +2 -0
- package/dist/services/heygen/index.js +7 -0
- package/dist/services/hume/index.d.ts +2 -0
- package/dist/services/hume/index.js +7 -0
- package/dist/services/image_service.d.ts +52 -0
- package/dist/services/image_service.js +60 -0
- package/dist/services/inception/index.d.ts +2 -0
- package/dist/services/inception/index.js +7 -0
- package/dist/services/index.d.ts +2 -0
- package/dist/services/index.js +8 -0
- package/dist/services/inworld/index.d.ts +2 -0
- package/dist/services/inworld/index.js +7 -0
- package/dist/services/keenable/index.d.ts +2 -0
- package/dist/services/keenable/index.js +7 -0
- package/dist/services/kokoro/index.d.ts +2 -0
- package/dist/services/kokoro/index.js +7 -0
- package/dist/services/llm_service.d.ts +1106 -0
- package/dist/services/llm_service.js +2231 -0
- package/dist/services/mem0/index.d.ts +2 -0
- package/dist/services/mem0/index.js +7 -0
- package/dist/services/meta/index.d.ts +2 -0
- package/dist/services/meta/index.js +7 -0
- package/dist/services/minimax/index.d.ts +2 -0
- package/dist/services/minimax/index.js +7 -0
- package/dist/services/mistral/index.d.ts +2 -0
- package/dist/services/mistral/index.js +7 -0
- package/dist/services/moondream/index.d.ts +2 -0
- package/dist/services/moondream/index.js +7 -0
- package/dist/services/moonshine/index.d.ts +2 -0
- package/dist/services/moonshine/index.js +7 -0
- package/dist/services/nebius/index.d.ts +2 -0
- package/dist/services/nebius/index.js +7 -0
- package/dist/services/neuphonic/index.d.ts +2 -0
- package/dist/services/neuphonic/index.js +7 -0
- package/dist/services/novita/index.d.ts +2 -0
- package/dist/services/novita/index.js +7 -0
- package/dist/services/nvidia/index.d.ts +2 -0
- package/dist/services/nvidia/index.js +11 -0
- package/dist/services/ollama/index.d.ts +2 -0
- package/dist/services/ollama/index.js +7 -0
- package/dist/services/openai/index.d.ts +2 -0
- package/dist/services/openai/index.js +7 -0
- package/dist/services/openclaw/index.d.ts +14 -0
- package/dist/services/openclaw/index.js +19 -0
- package/dist/services/openrouter/index.d.ts +2 -0
- package/dist/services/openrouter/index.js +7 -0
- package/dist/services/perplexity/index.d.ts +2 -0
- package/dist/services/perplexity/index.js +7 -0
- package/dist/services/piper/index.d.ts +2 -0
- package/dist/services/piper/index.js +7 -0
- package/dist/services/pocket_tts/index.d.ts +2 -0
- package/dist/services/pocket_tts/index.js +7 -0
- package/dist/services/qwen/index.d.ts +2 -0
- package/dist/services/qwen/index.js +7 -0
- package/dist/services/resembleai/index.d.ts +2 -0
- package/dist/services/resembleai/index.js +7 -0
- package/dist/services/rime/index.d.ts +2 -0
- package/dist/services/rime/index.js +7 -0
- package/dist/services/sambanova/index.d.ts +2 -0
- package/dist/services/sambanova/index.js +7 -0
- package/dist/services/sarvam/index.d.ts +2 -0
- package/dist/services/sarvam/index.js +7 -0
- package/dist/services/settings.d.ts +356 -0
- package/dist/services/settings.js +437 -0
- package/dist/services/simli/index.d.ts +2 -0
- package/dist/services/simli/index.js +7 -0
- package/dist/services/smallest/index.d.ts +2 -0
- package/dist/services/smallest/index.js +7 -0
- package/dist/services/soniox/index.d.ts +2 -0
- package/dist/services/soniox/index.js +7 -0
- package/dist/services/speechify/index.d.ts +2 -0
- package/dist/services/speechify/index.js +7 -0
- package/dist/services/speechmatics/index.d.ts +2 -0
- package/dist/services/speechmatics/index.js +7 -0
- package/dist/services/stt_latency.d.ts +62 -0
- package/dist/services/stt_latency.js +71 -0
- package/dist/services/stt_service.d.ts +601 -0
- package/dist/services/stt_service.js +1044 -0
- package/dist/services/tavus/index.d.ts +2 -0
- package/dist/services/tavus/index.js +7 -0
- package/dist/services/together/index.d.ts +2 -0
- package/dist/services/together/index.js +7 -0
- package/dist/services/tts_service.d.ts +909 -0
- package/dist/services/tts_service.js +2018 -0
- package/dist/services/ultravox/index.d.ts +2 -0
- package/dist/services/ultravox/index.js +9 -0
- package/dist/services/vision_service.d.ts +56 -0
- package/dist/services/vision_service.js +62 -0
- package/dist/services/websocket_service.d.ts +213 -0
- package/dist/services/websocket_service.js +654 -0
- package/dist/services/whisper/index.d.ts +2 -0
- package/dist/services/whisper/index.js +9 -0
- package/dist/services/xai/index.d.ts +2 -0
- package/dist/services/xai/index.js +9 -0
- package/dist/services/xtts/index.d.ts +2 -0
- package/dist/services/xtts/index.js +9 -0
- package/dist/tests/index.d.ts +2 -0
- package/dist/tests/index.js +8 -0
- package/dist/tests/utils.d.ts +123 -0
- package/dist/tests/utils.js +212 -0
- package/dist/transcriptions/index.d.ts +2 -0
- package/dist/transcriptions/index.js +8 -0
- package/dist/transcriptions/language.d.ts +376 -0
- package/dist/transcriptions/language.js +543 -0
- package/dist/transports/base_input.d.ts +149 -0
- package/dist/transports/base_input.js +338 -0
- package/dist/transports/base_output.d.ts +432 -0
- package/dist/transports/base_output.js +1231 -0
- package/dist/transports/base_transport.d.ts +223 -0
- package/dist/transports/base_transport.js +167 -0
- package/dist/transports/daily/index.d.ts +2 -0
- package/dist/transports/daily/index.js +9 -0
- package/dist/transports/heygen/index.d.ts +2 -0
- package/dist/transports/heygen/index.js +9 -0
- package/dist/transports/index.d.ts +2 -0
- package/dist/transports/index.js +8 -0
- package/dist/transports/lemonslice/index.d.ts +2 -0
- package/dist/transports/lemonslice/index.js +9 -0
- package/dist/transports/livekit/index.d.ts +2 -0
- package/dist/transports/livekit/index.js +9 -0
- package/dist/transports/local/index.d.ts +2 -0
- package/dist/transports/local/index.js +9 -0
- package/dist/transports/moq/index.d.ts +2 -0
- package/dist/transports/moq/index.js +9 -0
- package/dist/transports/smallwebrtc/index.d.ts +2 -0
- package/dist/transports/smallwebrtc/index.js +9 -0
- package/dist/transports/tavus/index.d.ts +2 -0
- package/dist/transports/tavus/index.js +9 -0
- package/dist/transports/video_in_sampler.d.ts +157 -0
- package/dist/transports/video_in_sampler.js +258 -0
- package/dist/transports/vonage/index.d.ts +2 -0
- package/dist/transports/vonage/index.js +9 -0
- package/dist/transports/websocket/index.d.ts +2 -0
- package/dist/transports/websocket/index.js +9 -0
- package/dist/transports/whatsapp/index.d.ts +2 -0
- package/dist/transports/whatsapp/index.js +9 -0
- package/dist/turns/eager_end_of_turn_mixin.d.ts +70 -0
- package/dist/turns/eager_end_of_turn_mixin.js +122 -0
- package/dist/turns/empty_user_turn.d.ts +55 -0
- package/dist/turns/empty_user_turn.js +53 -0
- package/dist/turns/index.d.ts +2 -0
- package/dist/turns/index.js +8 -0
- package/dist/turns/speculation_gate.d.ts +169 -0
- package/dist/turns/speculation_gate.js +294 -0
- package/dist/turns/types.d.ts +33 -0
- package/dist/turns/types.js +35 -0
- package/dist/turns/user_idle_controller.d.ts +94 -0
- package/dist/turns/user_idle_controller.js +187 -0
- package/dist/turns/user_mute/always_user_mute_strategy.d.ts +15 -0
- package/dist/turns/user_mute/always_user_mute_strategy.js +29 -0
- package/dist/turns/user_mute/base_user_mute_strategy.d.ts +39 -0
- package/dist/turns/user_mute/base_user_mute_strategy.js +45 -0
- package/dist/turns/user_mute/first_speech_user_mute_strategy.d.ts +28 -0
- package/dist/turns/user_mute/first_speech_user_mute_strategy.js +52 -0
- package/dist/turns/user_mute/function_call_user_mute_strategy.d.ts +25 -0
- package/dist/turns/user_mute/function_call_user_mute_strategy.js +50 -0
- package/dist/turns/user_mute/index.d.ts +6 -0
- package/dist/turns/user_mute/index.js +11 -0
- package/dist/turns/user_mute/mute_until_first_bot_complete_user_mute_strategy.d.ts +43 -0
- package/dist/turns/user_mute/mute_until_first_bot_complete_user_mute_strategy.js +76 -0
- package/dist/turns/user_start/base_user_turn_start_strategy.d.ts +135 -0
- package/dist/turns/user_start/base_user_turn_start_strategy.js +134 -0
- package/dist/turns/user_start/external_user_turn_start_strategy.d.ts +45 -0
- package/dist/turns/user_start/external_user_turn_start_strategy.js +64 -0
- package/dist/turns/user_start/index.d.ts +8 -0
- package/dist/turns/user_start/index.js +16 -0
- package/dist/turns/user_start/krisp_viva_ip_user_turn_start_strategy.d.ts +133 -0
- package/dist/turns/user_start/krisp_viva_ip_user_turn_start_strategy.js +243 -0
- package/dist/turns/user_start/min_words_user_turn_start_strategy.d.ts +60 -0
- package/dist/turns/user_start/min_words_user_turn_start_strategy.js +102 -0
- package/dist/turns/user_start/transcription_user_turn_start_strategy.d.ts +29 -0
- package/dist/turns/user_start/transcription_user_turn_start_strategy.js +43 -0
- package/dist/turns/user_start/vad_user_turn_start_strategy.d.ts +20 -0
- package/dist/turns/user_start/vad_user_turn_start_strategy.js +31 -0
- package/dist/turns/user_start/wake_phrase_user_turn_start_strategy.d.ts +141 -0
- package/dist/turns/user_start/wake_phrase_user_turn_start_strategy.js +238 -0
- package/dist/turns/user_stop/base_user_turn_stop_strategy.d.ts +174 -0
- package/dist/turns/user_stop/base_user_turn_stop_strategy.js +170 -0
- package/dist/turns/user_stop/deferred_user_turn_stop_strategy.d.ts +81 -0
- package/dist/turns/user_stop/deferred_user_turn_stop_strategy.js +107 -0
- package/dist/turns/user_stop/eager_match_policy.d.ts +36 -0
- package/dist/turns/user_stop/eager_match_policy.js +50 -0
- package/dist/turns/user_stop/eager_user_turn_stop_strategy.d.ts +84 -0
- package/dist/turns/user_stop/eager_user_turn_stop_strategy.js +154 -0
- package/dist/turns/user_stop/external_user_turn_completion_stop_strategy.d.ts +45 -0
- package/dist/turns/user_stop/external_user_turn_completion_stop_strategy.js +55 -0
- package/dist/turns/user_stop/external_user_turn_stop_strategy.d.ts +115 -0
- package/dist/turns/user_stop/external_user_turn_stop_strategy.js +205 -0
- package/dist/turns/user_stop/index.d.ts +10 -0
- package/dist/turns/user_stop/index.js +15 -0
- package/dist/turns/user_stop/llm_turn_completion_user_turn_stop_strategy.d.ts +44 -0
- package/dist/turns/user_stop/llm_turn_completion_user_turn_stop_strategy.js +59 -0
- package/dist/turns/user_stop/speech_timeout_user_turn_stop_strategy.d.ts +140 -0
- package/dist/turns/user_stop/speech_timeout_user_turn_stop_strategy.js +303 -0
- package/dist/turns/user_stop/turn_analyzer_user_turn_stop_strategy.d.ts +135 -0
- package/dist/turns/user_stop/turn_analyzer_user_turn_stop_strategy.js +332 -0
- package/dist/turns/user_turn_completion_mixin.d.ts +147 -0
- package/dist/turns/user_turn_completion_mixin.js +591 -0
- package/dist/turns/user_turn_controller.d.ts +134 -0
- package/dist/turns/user_turn_controller.js +335 -0
- package/dist/turns/user_turn_processor.d.ts +87 -0
- package/dist/turns/user_turn_processor.js +180 -0
- package/dist/turns/user_turn_strategies.d.ts +187 -0
- package/dist/turns/user_turn_strategies.js +222 -0
- package/dist/utils/async_tool_cancellation.d.ts +35 -0
- package/dist/utils/async_tool_cancellation.js +78 -0
- package/dist/utils/asyncio/index.d.ts +2 -0
- package/dist/utils/asyncio/index.js +7 -0
- package/dist/utils/asyncio/task_manager.d.ts +49 -0
- package/dist/utils/asyncio/task_manager.js +105 -0
- package/dist/utils/base_object.d.ts +76 -0
- package/dist/utils/base_object.js +154 -0
- package/dist/utils/context/aggregated_frame_sequencer.d.ts +491 -0
- package/dist/utils/context/aggregated_frame_sequencer.js +803 -0
- package/dist/utils/context/audio_context_manager.d.ts +131 -0
- package/dist/utils/context/audio_context_manager.js +233 -0
- package/dist/utils/context/index.d.ts +2 -0
- package/dist/utils/context/index.js +7 -0
- package/dist/utils/context/llm_context_summarization.d.ts +258 -0
- package/dist/utils/context/llm_context_summarization.js +511 -0
- package/dist/utils/context/text_segment_map.d.ts +611 -0
- package/dist/utils/context/text_segment_map.js +1132 -0
- package/dist/utils/context/word_completion_tracker.d.ts +273 -0
- package/dist/utils/context/word_completion_tracker.js +441 -0
- package/dist/utils/context/word_timestamp_tracker.d.ts +72 -0
- package/dist/utils/context/word_timestamp_tracker.js +98 -0
- package/dist/utils/deprecation.d.ts +28 -0
- package/dist/utils/deprecation.js +75 -0
- package/dist/utils/env.d.ts +38 -0
- package/dist/utils/env.js +62 -0
- package/dist/utils/errors.d.ts +75 -0
- package/dist/utils/errors.js +202 -0
- package/dist/utils/file_resolver.d.ts +126 -0
- package/dist/utils/file_resolver.js +314 -0
- package/dist/utils/file_storage.d.ts +99 -0
- package/dist/utils/file_storage.js +100 -0
- package/dist/utils/frame_queue.d.ts +36 -0
- package/dist/utils/frame_queue.js +66 -0
- package/dist/utils/http.d.ts +39 -0
- package/dist/utils/http.js +53 -0
- package/dist/utils/image.d.ts +15 -0
- package/dist/utils/image.js +30 -0
- package/dist/utils/index.d.ts +2 -0
- package/dist/utils/index.js +8 -0
- package/dist/utils/log_config.d.ts +34 -0
- package/dist/utils/log_config.js +76 -0
- package/dist/utils/logger.d.ts +40 -0
- package/dist/utils/logger.js +91 -0
- package/dist/utils/mixin.d.ts +14 -0
- package/dist/utils/mixin.js +63 -0
- package/dist/utils/network.d.ts +73 -0
- package/dist/utils/network.js +90 -0
- package/dist/utils/security/allowed_origins.d.ts +16 -0
- package/dist/utils/security/allowed_origins.js +34 -0
- package/dist/utils/security/index.d.ts +2 -0
- package/dist/utils/security/index.js +7 -0
- package/dist/utils/security/redaction.d.ts +27 -0
- package/dist/utils/security/redaction.js +123 -0
- package/dist/utils/security/ssrf.d.ts +79 -0
- package/dist/utils/security/ssrf.js +327 -0
- package/dist/utils/shared.d.ts +35 -0
- package/dist/utils/shared.js +136 -0
- package/dist/utils/startup.d.ts +17 -0
- package/dist/utils/startup.js +130 -0
- package/dist/utils/string.d.ts +116 -0
- package/dist/utils/string.js +460 -0
- package/dist/utils/sync/base_notifier.d.ts +25 -0
- package/dist/utils/sync/base_notifier.js +16 -0
- package/dist/utils/sync/event_notifier.d.ts +38 -0
- package/dist/utils/sync/event_notifier.js +51 -0
- package/dist/utils/sync/index.d.ts +2 -0
- package/dist/utils/sync/index.js +7 -0
- package/dist/utils/text/alnum_utils.d.ts +109 -0
- package/dist/utils/text/alnum_utils.js +262 -0
- package/dist/utils/text/base_text_aggregator.d.ts +146 -0
- package/dist/utils/text/base_text_aggregator.js +102 -0
- package/dist/utils/text/base_text_filter.d.ts +54 -0
- package/dist/utils/text/base_text_filter.js +50 -0
- package/dist/utils/text/index.d.ts +2 -0
- package/dist/utils/text/index.js +7 -0
- package/dist/utils/text/markdown_text_filter.d.ts +147 -0
- package/dist/utils/text/markdown_text_filter.js +522 -0
- package/dist/utils/text/markup_utils.d.ts +54 -0
- package/dist/utils/text/markup_utils.js +156 -0
- package/dist/utils/text/pattern_pair_aggregator.d.ts +251 -0
- package/dist/utils/text/pattern_pair_aggregator.js +478 -0
- package/dist/utils/text/phonemes.d.ts +60 -0
- package/dist/utils/text/phonemes.js +172 -0
- package/dist/utils/text/simple_text_aggregator.d.ts +135 -0
- package/dist/utils/text/simple_text_aggregator.js +254 -0
- package/dist/utils/text/skip_tags_aggregator.d.ts +85 -0
- package/dist/utils/text/skip_tags_aggregator.js +147 -0
- package/dist/utils/text/transforms/pronunciations.d.ts +65 -0
- package/dist/utils/text/transforms/pronunciations.js +111 -0
- package/dist/utils/text/word_timestamp_utils.d.ts +26 -0
- package/dist/utils/text/word_timestamp_utils.js +52 -0
- package/dist/utils/time.d.ts +13 -0
- package/dist/utils/time.js +31 -0
- package/dist/utils/tracing/index.d.ts +3 -0
- package/dist/utils/tracing/index.js +8 -0
- package/dist/utils/tracing/service_attributes.d.ts +198 -0
- package/dist/utils/tracing/service_attributes.js +434 -0
- package/dist/utils/tracing/service_decorators.d.ts +214 -0
- package/dist/utils/tracing/service_decorators.js +1662 -0
- package/dist/utils/tracing/setup.d.ts +95 -0
- package/dist/utils/tracing/setup.js +110 -0
- package/dist/utils/tracing/tracing_context.d.ts +55 -0
- package/dist/utils/tracing/tracing_context.js +97 -0
- package/dist/utils/tracing/turn_trace_observer.d.ts +90 -0
- package/dist/utils/tracing/turn_trace_observer.js +219 -0
- package/dist/utils/types.d.ts +80 -0
- package/dist/utils/types.js +108 -0
- package/dist/utils/utils.d.ts +5 -0
- package/dist/utils/utils.js +22 -0
- package/dist/utils/yaml.d.ts +64 -0
- package/dist/utils/yaml.js +41 -0
- package/dist/version.d.ts +3 -0
- package/dist/version.js +7 -0
- package/dist/workers/base_worker.d.ts +772 -0
- package/dist/workers/base_worker.js +1298 -0
- package/dist/workers/index.d.ts +2 -0
- package/dist/workers/index.js +8 -0
- package/dist/workers/llm/backend_llm_worker.d.ts +552 -0
- package/dist/workers/llm/backend_llm_worker.js +1055 -0
- package/dist/workers/llm/index.d.ts +6 -0
- package/dist/workers/llm/index.js +11 -0
- package/dist/workers/llm/llm_context_worker.d.ts +66 -0
- package/dist/workers/llm/llm_context_worker.js +72 -0
- package/dist/workers/llm/llm_worker.d.ts +180 -0
- package/dist/workers/llm/llm_worker.js +323 -0
- package/dist/workers/llm/tool_decorator.d.ts +68 -0
- package/dist/workers/llm/tool_decorator.js +60 -0
- package/dist/workers/runner.d.ts +268 -0
- package/dist/workers/runner.js +489 -0
- package/dist/workers/ui/index.d.ts +22 -0
- package/dist/workers/ui/index.js +27 -0
- package/dist/workers/ui/ui_event_decorator.d.ts +43 -0
- package/dist/workers/ui/ui_event_decorator.js +75 -0
- package/dist/workers/ui/ui_job_context.d.ts +12 -0
- package/dist/workers/ui/ui_job_context.js +17 -0
- package/dist/workers/ui/ui_prompts.d.ts +21 -0
- package/dist/workers/ui/ui_prompts.js +41 -0
- package/dist/workers/ui/ui_tools.d.ts +32 -0
- package/dist/workers/ui/ui_tools.js +112 -0
- package/dist/workers/ui/ui_worker.d.ts +571 -0
- package/dist/workers/ui/ui_worker.js +1156 -0
- package/package.json +46 -3
- package/README.md +0 -5
|
@@ -0,0 +1,1113 @@
|
|
|
1
|
+
// Copyright (c) 2024-2026, Daily
|
|
2
|
+
// Copyright (c) 2026, pipedog contributors
|
|
3
|
+
// SPDX-License-Identifier: BSD-2-Clause
|
|
4
|
+
//
|
|
5
|
+
// Ported from pipecat: src/pipecat/processors/frameworks/rtvi/observer.py
|
|
6
|
+
/**
|
|
7
|
+
* RTVI observer for converting pipeline frames to outgoing RTVI messages.
|
|
8
|
+
*
|
|
9
|
+
* Porting notes:
|
|
10
|
+
* - Deprecated upstream APIs are not ported: `RTVIObserverParams.userTranscriptionEnabled`
|
|
11
|
+
* and `skipAggregatorTypes`, the `aggregationType` argument of
|
|
12
|
+
* `addBotOutputTransformer()` / `removeBotOutputTransformer()`, and the
|
|
13
|
+
* 2-parameter `(text, textType) => string` transform signature. Every
|
|
14
|
+
* transform is called with the 4-parameter, progress-aware signature.
|
|
15
|
+
* - The deprecated `bot-llm-text`, `bot-tts-text`, `bot-transcription` and
|
|
16
|
+
* `user-transcription` messages are still sent, because they are part of the
|
|
17
|
+
* wire protocol current clients consume. With `userTranscriptionEnabled`
|
|
18
|
+
* gone, `user-transcription` is always sent (upstream's default).
|
|
19
|
+
* - RTVI models are dumped with `modelDump({ excludeNone })` (see `processor.ts`);
|
|
20
|
+
* metrics models are plain classes, dumped to their snake_case wire form here.
|
|
21
|
+
*/
|
|
22
|
+
import { createTask, rethrowIfCancelled, time } from "@pipedog/asyncio";
|
|
23
|
+
import { AudioVolumeTracker } from "../../../audio/volume.js";
|
|
24
|
+
import { AggregatedTextFrame, AggregatedTextProgressFrame, AudioRawFrame, BotStartedSpeakingFrame, BotStoppedSpeakingFrame, ExternalFunctionCallCancelFrame, ExternalFunctionCallInProgressFrame, ExternalFunctionCallResultFrame, ExternalFunctionCallsStartedFrame, FunctionCallCancelFrame, FunctionCallInProgressFrame, FunctionCallResultFrame, FunctionCallsStartedFrame, InputAudioRawFrame, InterimTranscriptionFrame, InterruptionFrame, LLMContextFrame, LLMFullResponseEndFrame, LLMFullResponseStartFrame, LLMMarkerResponseFrame, LLMTextFrame, MetricsFrame, TextType, TranscriptionFrame, TTSAudioRawFrame, TTSStartedFrame, TTSStoppedFrame, TTSTextFrame, UserBackchannelFrame, UserMuteStartedFrame, UserMuteStoppedFrame, UserStartedSpeakingFrame, UserStoppedSpeakingFrame, VADUserStartedSpeakingFrame, VADUserStoppedSpeakingFrame, } from "../../../frames/frames.js";
|
|
25
|
+
import { LLMUsageMetricsData, ProcessingMetricsData, STTUsageMetricsData, TTFAMetricsData, TTFATMetricsData, TTFBMetricsData, TTSUsageMetricsData, } from "../../../metrics/metrics.js";
|
|
26
|
+
import { BaseObserver, } from "../../../observers/base_observer.js";
|
|
27
|
+
import { BaseOutputTransport } from "../../../transports/base_output.js";
|
|
28
|
+
import { logger } from "../../../utils/logger.js";
|
|
29
|
+
import { matchEndofsentence } from "../../../utils/string.js";
|
|
30
|
+
import { timeNowIso8601 } from "../../../utils/time.js";
|
|
31
|
+
import { FrameDirection } from "../../frame_processor.js";
|
|
32
|
+
import { RTVIConfigureObserverFrame, RTVISendTextFrame, RTVIServerMessageFrame, RTVIServerResponseFrame, RTVIUICommandFrame, RTVIUIJobGroupFrame, } from "./frames.js";
|
|
33
|
+
import * as RTVI from "./models.js";
|
|
34
|
+
import { BotOutputTransformResult } from "./models.js";
|
|
35
|
+
/**
|
|
36
|
+
* Level of detail to include in function call RTVI events.
|
|
37
|
+
*
|
|
38
|
+
* Controls what information is exposed in function call events for security.
|
|
39
|
+
*
|
|
40
|
+
* - DISABLED: No events emitted for this function call.
|
|
41
|
+
* - NONE: Events only with tool_call_id, no function name or metadata (most secure).
|
|
42
|
+
* - NAME: Events with function name, no arguments or results.
|
|
43
|
+
* - ARGUMENTS: Events with function name and arguments, no results.
|
|
44
|
+
* - FULL: Events with function name, arguments, and results.
|
|
45
|
+
*/
|
|
46
|
+
export const RTVIFunctionCallReportLevel = {
|
|
47
|
+
DISABLED: "disabled",
|
|
48
|
+
NONE: "none",
|
|
49
|
+
NAME: "name",
|
|
50
|
+
ARGUMENTS: "arguments",
|
|
51
|
+
FULL: "full",
|
|
52
|
+
};
|
|
53
|
+
/**
|
|
54
|
+
* Parameters for configuring RTVI Observer behavior.
|
|
55
|
+
*
|
|
56
|
+
* - `botOutputEnabled`: Indicates if bot output messages should be sent.
|
|
57
|
+
* - `botLlmEnabled`: Indicates if the bot's LLM messages should be sent. Of
|
|
58
|
+
* these, the `bot-llm-text` message is deprecated in favor of `llm-raw-text`
|
|
59
|
+
* (`llmRawTextEnabled`) and will be removed in 2.0.0. This flag stays, for
|
|
60
|
+
* `bot-llm-started` and `bot-llm-stopped`.
|
|
61
|
+
* - `botLlmMarkerEnabled`: Indicates if the bot's LLM marker reports
|
|
62
|
+
* (`LLMMarkerResponseFrame`, e.g. from `filterIncompleteUserTurns`) should
|
|
63
|
+
* be sent. A report carries the marker the LLM emitted and the raw text of
|
|
64
|
+
* the whole response, which is meant for evaluation, not for clients, so
|
|
65
|
+
* this is off by default. Defaults to false.
|
|
66
|
+
* - `botTtsEnabled`: Indicates if the bot's TTS messages should be sent. Of
|
|
67
|
+
* these, the `bot-tts-text` message is deprecated in favor of
|
|
68
|
+
* `tts-raw-text` (`ttsRawTextEnabled`) and will be removed in 2.0.0. This
|
|
69
|
+
* flag stays, for `bot-tts-started` and `bot-tts-stopped`.
|
|
70
|
+
* - `botSpeakingEnabled`: Indicates if the bot's started/stopped speaking and
|
|
71
|
+
* interrupted messages should be sent.
|
|
72
|
+
* - `botAudioLevelEnabled`: Indicates if bot's audio level messages should be sent.
|
|
73
|
+
* - `userLlmEnabled`: Indicates if the user's LLM input messages should be sent.
|
|
74
|
+
* - `userSpeakingEnabled`: Indicates if the user's started/stopped speaking
|
|
75
|
+
* messages should be sent.
|
|
76
|
+
* - `vadUserSpeakingEnabled`: Indicates if raw VAD user started/stopped
|
|
77
|
+
* speaking messages should be sent. These reflect the VAD signal directly,
|
|
78
|
+
* independent of turn finalization (unlike `userSpeakingEnabled`, which a
|
|
79
|
+
* turn strategy may gate or defer). Off by default. Defaults to false.
|
|
80
|
+
* - `userMuteEnabled`: Indicates if user mute started/stopped messages should be sent.
|
|
81
|
+
* - `userInputEnabled`: Indicates if the user's input messages should be sent.
|
|
82
|
+
* They report what the user said or typed: transcriptions, including
|
|
83
|
+
* interim ones, backchannels, and the text the client sent.
|
|
84
|
+
* - `userAudioLevelEnabled`: Indicates if user's audio level messages should be sent.
|
|
85
|
+
* - `metricsEnabled`: Indicates if metrics messages should be sent.
|
|
86
|
+
* - `systemLogsEnabled`: Indicates if system logs should be sent.
|
|
87
|
+
* - `sttRawTextEnabled`: Indicates if the STT's raw text messages should be
|
|
88
|
+
* sent. They report each transcription, interim ones included, as the STT
|
|
89
|
+
* produced it. Only sent to clients of protocol 2.2.0 or later. Defaults to false.
|
|
90
|
+
* - `llmRawTextEnabled`: Indicates if the LLM's raw text messages should be
|
|
91
|
+
* sent. They report each chunk of text the LLM streams. Only sent to
|
|
92
|
+
* clients of protocol 2.2.0 or later. Defaults to false.
|
|
93
|
+
* - `ttsRawTextEnabled`: Indicates if the TTS's raw text messages should be
|
|
94
|
+
* sent. They report the text the TTS speaks, as it is spoken, and leave out
|
|
95
|
+
* the same text types as bot output messages. Only sent to clients of
|
|
96
|
+
* protocol 2.2.0 or later. Defaults to false.
|
|
97
|
+
* - `ignoredSources`: List of frame processors whose frames should be
|
|
98
|
+
* silently ignored by this observer. Useful for suppressing RTVI messages
|
|
99
|
+
* from secondary pipeline branches (e.g. a silent evaluation LLM) that
|
|
100
|
+
* should not be visible to clients. Sources can also be added and removed
|
|
101
|
+
* dynamically via `addIgnoredSource()` and `removeIgnoredSource()`.
|
|
102
|
+
* - `skipTextTypes`: List of text types to skip sending as tts/output
|
|
103
|
+
* messages. Text of type `backchannel` is only sent to clients of protocol
|
|
104
|
+
* 2.2.0 or later; add `"backchannel"` here to keep it from every client.
|
|
105
|
+
* Note: if using this to avoid sending secure information, be sure to also
|
|
106
|
+
* disable botLlmEnabled, and leave llmRawTextEnabled off, to avoid leaking
|
|
107
|
+
* through LLM messages.
|
|
108
|
+
* - `botOutputTransforms`: A list of `[textType | "*", transform]` tuples to
|
|
109
|
+
* transform text before sending it to the client. See `BotOutputTransform`.
|
|
110
|
+
* - `audioLevelPeriodSecs`: How often audio levels should be sent if enabled.
|
|
111
|
+
* - `functionCallReportLevel`: Controls what information is exposed in
|
|
112
|
+
* function call events for security. A record mapping function names to
|
|
113
|
+
* levels, where `"*"` sets the default level for unlisted functions.
|
|
114
|
+
* Defaults to `{ "*": RTVIFunctionCallReportLevel.NONE }`.
|
|
115
|
+
*
|
|
116
|
+
* @example
|
|
117
|
+
* new RTVIObserverParams({
|
|
118
|
+
* functionCallReportLevel: {
|
|
119
|
+
* "*": RTVIFunctionCallReportLevel.NONE, // Default: events with no metadata
|
|
120
|
+
* get_weather: RTVIFunctionCallReportLevel.FULL, // Expose everything
|
|
121
|
+
* },
|
|
122
|
+
* });
|
|
123
|
+
*/
|
|
124
|
+
export class RTVIObserverParams {
|
|
125
|
+
botOutputEnabled;
|
|
126
|
+
botLlmEnabled;
|
|
127
|
+
botLlmMarkerEnabled;
|
|
128
|
+
botTtsEnabled;
|
|
129
|
+
botSpeakingEnabled;
|
|
130
|
+
botAudioLevelEnabled;
|
|
131
|
+
userLlmEnabled;
|
|
132
|
+
userSpeakingEnabled;
|
|
133
|
+
vadUserSpeakingEnabled;
|
|
134
|
+
userMuteEnabled;
|
|
135
|
+
userInputEnabled;
|
|
136
|
+
userAudioLevelEnabled;
|
|
137
|
+
metricsEnabled;
|
|
138
|
+
systemLogsEnabled;
|
|
139
|
+
sttRawTextEnabled;
|
|
140
|
+
llmRawTextEnabled;
|
|
141
|
+
ttsRawTextEnabled;
|
|
142
|
+
ignoredSources;
|
|
143
|
+
skipTextTypes;
|
|
144
|
+
botOutputTransforms;
|
|
145
|
+
audioLevelPeriodSecs;
|
|
146
|
+
functionCallReportLevel;
|
|
147
|
+
constructor(options = {}) {
|
|
148
|
+
this.botOutputEnabled = options.botOutputEnabled ?? true;
|
|
149
|
+
this.botLlmEnabled = options.botLlmEnabled ?? true;
|
|
150
|
+
this.botLlmMarkerEnabled = options.botLlmMarkerEnabled ?? false;
|
|
151
|
+
this.botTtsEnabled = options.botTtsEnabled ?? true;
|
|
152
|
+
this.botSpeakingEnabled = options.botSpeakingEnabled ?? true;
|
|
153
|
+
this.botAudioLevelEnabled = options.botAudioLevelEnabled ?? false;
|
|
154
|
+
this.userLlmEnabled = options.userLlmEnabled ?? true;
|
|
155
|
+
this.userSpeakingEnabled = options.userSpeakingEnabled ?? true;
|
|
156
|
+
this.vadUserSpeakingEnabled = options.vadUserSpeakingEnabled ?? false;
|
|
157
|
+
this.userMuteEnabled = options.userMuteEnabled ?? true;
|
|
158
|
+
this.userInputEnabled = options.userInputEnabled ?? true;
|
|
159
|
+
this.userAudioLevelEnabled = options.userAudioLevelEnabled ?? false;
|
|
160
|
+
this.metricsEnabled = options.metricsEnabled ?? true;
|
|
161
|
+
this.systemLogsEnabled = options.systemLogsEnabled ?? false;
|
|
162
|
+
this.sttRawTextEnabled = options.sttRawTextEnabled ?? false;
|
|
163
|
+
this.llmRawTextEnabled = options.llmRawTextEnabled ?? false;
|
|
164
|
+
this.ttsRawTextEnabled = options.ttsRawTextEnabled ?? false;
|
|
165
|
+
this.ignoredSources = options.ignoredSources ?? [];
|
|
166
|
+
this.skipTextTypes = options.skipTextTypes ?? null;
|
|
167
|
+
this.botOutputTransforms = options.botOutputTransforms ?? null;
|
|
168
|
+
this.audioLevelPeriodSecs = options.audioLevelPeriodSecs ?? 0.15;
|
|
169
|
+
this.functionCallReportLevel = options.functionCallReportLevel ?? {
|
|
170
|
+
"*": RTVIFunctionCallReportLevel.NONE,
|
|
171
|
+
};
|
|
172
|
+
}
|
|
173
|
+
}
|
|
174
|
+
// Python truthiness, for `x if x else None`.
|
|
175
|
+
function pyTruthy(value) {
|
|
176
|
+
if (value === null || value === undefined || value === false || value === 0 || value === "") {
|
|
177
|
+
return false;
|
|
178
|
+
}
|
|
179
|
+
if (Array.isArray(value))
|
|
180
|
+
return value.length > 0;
|
|
181
|
+
if (value instanceof Map || value instanceof Set)
|
|
182
|
+
return value.size > 0;
|
|
183
|
+
if (typeof value === "object" && Object.getPrototypeOf(value) === Object.prototype) {
|
|
184
|
+
return Object.keys(value).length > 0;
|
|
185
|
+
}
|
|
186
|
+
return true;
|
|
187
|
+
}
|
|
188
|
+
// Pydantic's `model_dump(exclude_none=True)` for the metrics models, which
|
|
189
|
+
// are plain classes here: camelCase fields become their snake_case wire keys
|
|
190
|
+
// and null fields are dropped, recursively.
|
|
191
|
+
function dumpExcludeNone(value) {
|
|
192
|
+
if (Array.isArray(value)) {
|
|
193
|
+
return value.map(dumpExcludeNone);
|
|
194
|
+
}
|
|
195
|
+
if (value !== null && typeof value === "object") {
|
|
196
|
+
const out = {};
|
|
197
|
+
for (const [key, field] of Object.entries(value)) {
|
|
198
|
+
if (field === null || field === undefined)
|
|
199
|
+
continue;
|
|
200
|
+
out[key.replace(/[A-Z]/g, (c) => `_${c.toLowerCase()}`)] = dumpExcludeNone(field);
|
|
201
|
+
}
|
|
202
|
+
return out;
|
|
203
|
+
}
|
|
204
|
+
return value;
|
|
205
|
+
}
|
|
206
|
+
function dumpMetrics(model) {
|
|
207
|
+
return dumpExcludeNone(model);
|
|
208
|
+
}
|
|
209
|
+
// Python's list comparison `a >= b`.
|
|
210
|
+
function versionAtLeast(a, b) {
|
|
211
|
+
const n = Math.min(a.length, b.length);
|
|
212
|
+
for (let i = 0; i < n; i++) {
|
|
213
|
+
const x = a[i];
|
|
214
|
+
const y = b[i];
|
|
215
|
+
if (x !== y)
|
|
216
|
+
return x > y;
|
|
217
|
+
}
|
|
218
|
+
return a.length >= b.length;
|
|
219
|
+
}
|
|
220
|
+
/**
|
|
221
|
+
* Pipeline frame observer for RTVI server message handling.
|
|
222
|
+
*
|
|
223
|
+
* This observer monitors pipeline frames and converts them into appropriate
|
|
224
|
+
* RTVI messages for client communication. It handles various frame types
|
|
225
|
+
* including speech events, transcriptions, LLM responses, and TTS events.
|
|
226
|
+
*
|
|
227
|
+
* Note: this observer only handles outgoing messages. Incoming RTVI client
|
|
228
|
+
* messages are handled by the RTVIProcessor.
|
|
229
|
+
*/
|
|
230
|
+
export class RTVIObserver extends BaseObserver {
|
|
231
|
+
_rtvi;
|
|
232
|
+
_params;
|
|
233
|
+
_ignoredSources;
|
|
234
|
+
_botTranscription = "";
|
|
235
|
+
_lastUserAudioLevel = 0;
|
|
236
|
+
_lastBotAudioLevel = 0;
|
|
237
|
+
_userVolumeTracker = new AudioVolumeTracker();
|
|
238
|
+
_botVolumeTracker = new AudioVolumeTracker();
|
|
239
|
+
// Track bot speaking state for queuing aggregated text frames
|
|
240
|
+
_botIsSpeaking = false;
|
|
241
|
+
_queuedAggregatedTextFrames = [];
|
|
242
|
+
// Skipped segments whose spoken text may still come (see _isSkipped)
|
|
243
|
+
_skippedSegmentIds = new Set();
|
|
244
|
+
// Whether an interruption cut off the skipped segments still remembered
|
|
245
|
+
_skippedSegmentsInterrupted = false;
|
|
246
|
+
// Removes the system logs sink (loguru's `logger.remove(id)`).
|
|
247
|
+
_systemLoggerRemove = null;
|
|
248
|
+
// Tasks sending system logs; loguru schedules coroutine sinks as tasks.
|
|
249
|
+
#systemLogTasks = new Set();
|
|
250
|
+
_aggregationTransforms = [];
|
|
251
|
+
/**
|
|
252
|
+
* Initialize the RTVI observer.
|
|
253
|
+
*
|
|
254
|
+
* @param rtvi The RTVI processor to push frames to.
|
|
255
|
+
* @param options.params Settings to enable/disable specific messages.
|
|
256
|
+
* Other options are passed to the parent class.
|
|
257
|
+
*/
|
|
258
|
+
constructor(rtvi = null, options = {}) {
|
|
259
|
+
super(options);
|
|
260
|
+
this._rtvi = rtvi;
|
|
261
|
+
this._params = options.params || new RTVIObserverParams();
|
|
262
|
+
this._ignoredSources = new Set(this._params.ignoredSources);
|
|
263
|
+
if (this._params.systemLogsEnabled) {
|
|
264
|
+
this._systemLoggerRemove = logger.addSink((record) => this.#scheduleLoggerSink(record));
|
|
265
|
+
}
|
|
266
|
+
for (const [textType, fn] of this._params.botOutputTransforms ?? []) {
|
|
267
|
+
this.addBotOutputTransformer(fn, textType);
|
|
268
|
+
}
|
|
269
|
+
}
|
|
270
|
+
/** Whether the connected client uses a deprecated 1.x protocol. */
|
|
271
|
+
get _isLegacyClient() {
|
|
272
|
+
if (!this._rtvi) {
|
|
273
|
+
return false;
|
|
274
|
+
}
|
|
275
|
+
return this._rtvi.clientVersion[0] === RTVI.LEGACY_SUPPORTED_MAJOR;
|
|
276
|
+
}
|
|
277
|
+
/** Whether the connected client's protocol is `version` or later. */
|
|
278
|
+
_clientIsAtLeast(version) {
|
|
279
|
+
if (!this._rtvi) {
|
|
280
|
+
// No client has negotiated a version, so assume the current protocol, as
|
|
281
|
+
// _isLegacyClient does.
|
|
282
|
+
return true;
|
|
283
|
+
}
|
|
284
|
+
return versionAtLeast(this._rtvi.clientVersion, version);
|
|
285
|
+
}
|
|
286
|
+
/** Whether the connected client supports the backchannel text type (2.2.0+). */
|
|
287
|
+
get _clientSupportsBackchannels() {
|
|
288
|
+
return this._clientIsAtLeast([2, 2, 0]);
|
|
289
|
+
}
|
|
290
|
+
/** Whether the connected client supports the raw text messages (2.2.0+). */
|
|
291
|
+
get _clientSupportsRawText() {
|
|
292
|
+
return this._clientIsAtLeast([2, 2, 0]);
|
|
293
|
+
}
|
|
294
|
+
/**
|
|
295
|
+
* Register a text transformer for a specific text type.
|
|
296
|
+
*
|
|
297
|
+
* When `accumulatedText` and `remainingText` are null the transform is
|
|
298
|
+
* being called for the full segment text (`bot-output` message). When they
|
|
299
|
+
* are provided it is being called for a progress event; return a
|
|
300
|
+
* `BotOutputTransformResult` with `accumulatedText` and `remainingText`
|
|
301
|
+
* populated so the client can display word-level progress on the
|
|
302
|
+
* transformed text.
|
|
303
|
+
*
|
|
304
|
+
* @param transformFunction The transform callable.
|
|
305
|
+
* @param textType Text type to match, or `"*"` for all types.
|
|
306
|
+
*/
|
|
307
|
+
addBotOutputTransformer(transformFunction, textType = "*") {
|
|
308
|
+
this._aggregationTransforms.push([textType, transformFunction]);
|
|
309
|
+
}
|
|
310
|
+
/**
|
|
311
|
+
* Remove a text transformer for a specific text type.
|
|
312
|
+
*
|
|
313
|
+
* @param transformFunction The function to remove.
|
|
314
|
+
* @param textType The type of text to remove the transformer for.
|
|
315
|
+
*/
|
|
316
|
+
removeBotOutputTransformer(transformFunction, textType = "*") {
|
|
317
|
+
this._aggregationTransforms = this._aggregationTransforms.filter(([transformType, func]) => !(transformType === textType && func === transformFunction));
|
|
318
|
+
}
|
|
319
|
+
/**
|
|
320
|
+
* Ignore all frames pushed by the given processor.
|
|
321
|
+
*
|
|
322
|
+
* Any frame whose source matches `source` will be silently skipped,
|
|
323
|
+
* preventing RTVI messages from being emitted for activity in that
|
|
324
|
+
* processor. Useful for suppressing events from secondary pipeline
|
|
325
|
+
* branches (e.g. a silent evaluation LLM) that should not be visible
|
|
326
|
+
* to clients.
|
|
327
|
+
*
|
|
328
|
+
* @param source The frame processor to ignore.
|
|
329
|
+
*/
|
|
330
|
+
addIgnoredSource(source) {
|
|
331
|
+
this._ignoredSources.add(source);
|
|
332
|
+
}
|
|
333
|
+
/**
|
|
334
|
+
* Stop ignoring frames pushed by the given processor.
|
|
335
|
+
*
|
|
336
|
+
* Reverses a previous call to `addIgnoredSource()`. If `source` was not
|
|
337
|
+
* previously ignored this is a no-op.
|
|
338
|
+
*
|
|
339
|
+
* @param source The frame processor to stop ignoring.
|
|
340
|
+
*/
|
|
341
|
+
removeIgnoredSource(source) {
|
|
342
|
+
this._ignoredSources.delete(source);
|
|
343
|
+
}
|
|
344
|
+
/**
|
|
345
|
+
* Get the report level for a specific function call.
|
|
346
|
+
*
|
|
347
|
+
* @param functionName The name of the function to get the report level for.
|
|
348
|
+
* @returns The report level for the function. Looks up the function name
|
|
349
|
+
* first, then falls back to "*" key, then NONE.
|
|
350
|
+
*/
|
|
351
|
+
_getFunctionCallReportLevel(functionName) {
|
|
352
|
+
const levels = this._params.functionCallReportLevel;
|
|
353
|
+
if (Object.hasOwn(levels, functionName)) {
|
|
354
|
+
return levels[functionName];
|
|
355
|
+
}
|
|
356
|
+
return Object.hasOwn(levels, "*")
|
|
357
|
+
? levels["*"]
|
|
358
|
+
: RTVIFunctionCallReportLevel.NONE;
|
|
359
|
+
}
|
|
360
|
+
/**
|
|
361
|
+
* Apply a dynamic observer-config update (only the fields that are set).
|
|
362
|
+
*
|
|
363
|
+
* @param frame The config frame; null fields leave the current value unchanged.
|
|
364
|
+
*/
|
|
365
|
+
_applyConfig(frame) {
|
|
366
|
+
if (frame.functionCallReportLevel !== null && frame.functionCallReportLevel !== undefined) {
|
|
367
|
+
this._params.functionCallReportLevel = frame.functionCallReportLevel;
|
|
368
|
+
logger.debug(`${this}: function_call_report_level set to ${JSON.stringify(frame.functionCallReportLevel)}`);
|
|
369
|
+
}
|
|
370
|
+
if (frame.vadUserSpeakingEnabled !== null && frame.vadUserSpeakingEnabled !== undefined) {
|
|
371
|
+
this._params.vadUserSpeakingEnabled = frame.vadUserSpeakingEnabled;
|
|
372
|
+
logger.debug(`${this}: vad_user_speaking_enabled set to ${frame.vadUserSpeakingEnabled}`);
|
|
373
|
+
}
|
|
374
|
+
if (frame.botLlmMarkerEnabled !== null && frame.botLlmMarkerEnabled !== undefined) {
|
|
375
|
+
this._params.botLlmMarkerEnabled = frame.botLlmMarkerEnabled;
|
|
376
|
+
logger.debug(`${this}: bot_llm_marker_enabled set to ${frame.botLlmMarkerEnabled}`);
|
|
377
|
+
}
|
|
378
|
+
}
|
|
379
|
+
// Log sinks are synchronous here; run the async sink as a tracked task, as
|
|
380
|
+
// loguru does with coroutine sinks.
|
|
381
|
+
#scheduleLoggerSink(record) {
|
|
382
|
+
const message = `${record.time.toISOString()} | ${record.level.padEnd(8)} | ${record.message}`;
|
|
383
|
+
const task = createTask(() => this._loggerSink(message));
|
|
384
|
+
this.#systemLogTasks.add(task);
|
|
385
|
+
task.addDoneCallback(() => this.#systemLogTasks.delete(task));
|
|
386
|
+
}
|
|
387
|
+
/** Logger sink so we can send system logs to RTVI clients. */
|
|
388
|
+
async _loggerSink(message) {
|
|
389
|
+
const rtviMessage = new RTVI.SystemLogMessage({
|
|
390
|
+
data: new RTVI.TextMessageData({ text: message }),
|
|
391
|
+
});
|
|
392
|
+
await this.sendRtviMessage(rtviMessage);
|
|
393
|
+
}
|
|
394
|
+
/** Cleanup RTVI observer resources. */
|
|
395
|
+
async cleanup() {
|
|
396
|
+
await super.cleanup();
|
|
397
|
+
if (this._systemLoggerRemove !== null) {
|
|
398
|
+
this._systemLoggerRemove();
|
|
399
|
+
this._systemLoggerRemove = null;
|
|
400
|
+
}
|
|
401
|
+
}
|
|
402
|
+
/**
|
|
403
|
+
* Send an RTVI message.
|
|
404
|
+
*
|
|
405
|
+
* By default, we push a transport frame. But this function can be
|
|
406
|
+
* overridden by subclass to send RTVI messages in different ways.
|
|
407
|
+
*
|
|
408
|
+
* @param model The message to send.
|
|
409
|
+
* @param excludeNone Whether to exclude null values from the model dump.
|
|
410
|
+
*/
|
|
411
|
+
async sendRtviMessage(model, excludeNone = true) {
|
|
412
|
+
if (this._rtvi) {
|
|
413
|
+
await this._rtvi.pushTransportMessage(model, excludeNone);
|
|
414
|
+
}
|
|
415
|
+
}
|
|
416
|
+
/** Report a function-call frame, the pipeline's own or an external one. */
|
|
417
|
+
async _reportFunctionCallFrame(frame) {
|
|
418
|
+
if (frame instanceof FunctionCallsStartedFrame) {
|
|
419
|
+
for (const functionCall of frame.functionCalls) {
|
|
420
|
+
await this._reportFunctionCall("started", functionCall.functionName, functionCall.toolCallId);
|
|
421
|
+
}
|
|
422
|
+
}
|
|
423
|
+
else if (frame instanceof FunctionCallInProgressFrame) {
|
|
424
|
+
await this._reportFunctionCall("in_progress", frame.functionName, frame.toolCallId, {
|
|
425
|
+
arguments: frame.arguments,
|
|
426
|
+
});
|
|
427
|
+
}
|
|
428
|
+
else if (frame instanceof FunctionCallCancelFrame) {
|
|
429
|
+
await this._reportFunctionCall("stopped", frame.functionName, frame.toolCallId, {
|
|
430
|
+
cancelled: true,
|
|
431
|
+
});
|
|
432
|
+
}
|
|
433
|
+
else if (frame instanceof FunctionCallResultFrame) {
|
|
434
|
+
// An intermediate result leaves the call running.
|
|
435
|
+
if (frame.properties === null || frame.properties === undefined || frame.properties.isFinal) {
|
|
436
|
+
await this._reportFunctionCall("stopped", frame.functionName, frame.toolCallId, {
|
|
437
|
+
result: frame.result,
|
|
438
|
+
});
|
|
439
|
+
}
|
|
440
|
+
}
|
|
441
|
+
else if (frame instanceof ExternalFunctionCallsStartedFrame) {
|
|
442
|
+
for (const call of frame.functionCalls) {
|
|
443
|
+
await this._reportFunctionCall("started", call.functionName, call.toolCallId);
|
|
444
|
+
}
|
|
445
|
+
}
|
|
446
|
+
else if (frame instanceof ExternalFunctionCallInProgressFrame) {
|
|
447
|
+
await this._reportFunctionCall("in_progress", frame.functionName, frame.toolCallId, {
|
|
448
|
+
arguments: frame.arguments,
|
|
449
|
+
});
|
|
450
|
+
}
|
|
451
|
+
else if (frame instanceof ExternalFunctionCallResultFrame) {
|
|
452
|
+
if (frame.isFinal) {
|
|
453
|
+
await this._reportFunctionCall("stopped", frame.functionName, frame.toolCallId, {
|
|
454
|
+
result: frame.result,
|
|
455
|
+
});
|
|
456
|
+
}
|
|
457
|
+
}
|
|
458
|
+
else if (frame instanceof ExternalFunctionCallCancelFrame) {
|
|
459
|
+
await this._reportFunctionCall("stopped", frame.functionName, frame.toolCallId, {
|
|
460
|
+
cancelled: true,
|
|
461
|
+
});
|
|
462
|
+
}
|
|
463
|
+
}
|
|
464
|
+
/**
|
|
465
|
+
* Send the function-call message for a phase, as the report level allows.
|
|
466
|
+
*
|
|
467
|
+
* The pipeline's own calls and calls reported from elsewhere (the
|
|
468
|
+
* `ExternalFunctionCall*Frame` family) go through here alike, so both obey
|
|
469
|
+
* the per-function report level.
|
|
470
|
+
*/
|
|
471
|
+
async _reportFunctionCall(phase, functionName, toolCallId, options = {}) {
|
|
472
|
+
const args = options.arguments ?? null;
|
|
473
|
+
const result = options.result ?? null;
|
|
474
|
+
const cancelled = options.cancelled ?? false;
|
|
475
|
+
const reportLevel = this._getFunctionCallReportLevel(functionName);
|
|
476
|
+
if (reportLevel === RTVIFunctionCallReportLevel.DISABLED) {
|
|
477
|
+
return;
|
|
478
|
+
}
|
|
479
|
+
const named = reportLevel === RTVIFunctionCallReportLevel.NAME ||
|
|
480
|
+
reportLevel === RTVIFunctionCallReportLevel.ARGUMENTS ||
|
|
481
|
+
reportLevel === RTVIFunctionCallReportLevel.FULL;
|
|
482
|
+
const withArguments = reportLevel === RTVIFunctionCallReportLevel.ARGUMENTS ||
|
|
483
|
+
reportLevel === RTVIFunctionCallReportLevel.FULL;
|
|
484
|
+
const full = reportLevel === RTVIFunctionCallReportLevel.FULL;
|
|
485
|
+
let message;
|
|
486
|
+
if (phase === "started") {
|
|
487
|
+
message = new RTVI.LLMFunctionCallStartMessage({
|
|
488
|
+
data: new RTVI.LLMFunctionCallStartMessageData({
|
|
489
|
+
functionName: named ? functionName : null,
|
|
490
|
+
}),
|
|
491
|
+
});
|
|
492
|
+
}
|
|
493
|
+
else if (phase === "in_progress") {
|
|
494
|
+
message = new RTVI.LLMFunctionCallInProgressMessage({
|
|
495
|
+
data: new RTVI.LLMFunctionCallInProgressMessageData({
|
|
496
|
+
toolCallId,
|
|
497
|
+
functionName: named ? functionName : null,
|
|
498
|
+
arguments: withArguments ? args : null,
|
|
499
|
+
}),
|
|
500
|
+
});
|
|
501
|
+
}
|
|
502
|
+
else {
|
|
503
|
+
message = new RTVI.LLMFunctionCallStoppedMessage({
|
|
504
|
+
data: new RTVI.LLMFunctionCallStoppedMessageData({
|
|
505
|
+
toolCallId,
|
|
506
|
+
cancelled,
|
|
507
|
+
functionName: named ? functionName : null,
|
|
508
|
+
result: full && !cancelled ? (pyTruthy(result) ? result : null) : null,
|
|
509
|
+
}),
|
|
510
|
+
});
|
|
511
|
+
}
|
|
512
|
+
await this.sendRtviMessage(message);
|
|
513
|
+
}
|
|
514
|
+
/**
|
|
515
|
+
* Process a frame being pushed through the pipeline.
|
|
516
|
+
*
|
|
517
|
+
* @param data Frame push event data containing source, frame, direction,
|
|
518
|
+
* and timestamp.
|
|
519
|
+
*/
|
|
520
|
+
async onPushFrame(data) {
|
|
521
|
+
const src = data.source;
|
|
522
|
+
const frame = data.frame;
|
|
523
|
+
const direction = data.direction;
|
|
524
|
+
// Frames from explicitly ignored sources are always skipped.
|
|
525
|
+
if (this._ignoredSources.size > 0 && this._ignoredSources.has(src)) {
|
|
526
|
+
return;
|
|
527
|
+
}
|
|
528
|
+
// For broadcast frames (pushed in both directions), only process
|
|
529
|
+
// the downstream copy to avoid sending duplicate RTVI messages.
|
|
530
|
+
if (frame.broadcastSiblingId !== null && direction !== FrameDirection.DOWNSTREAM) {
|
|
531
|
+
return;
|
|
532
|
+
}
|
|
533
|
+
// Audio frames are the bulk of what a pipeline pushes, and they are
|
|
534
|
+
// only handled when audio levels are reported.
|
|
535
|
+
if (frame instanceof AudioRawFrame &&
|
|
536
|
+
!(this._params.userAudioLevelEnabled || this._params.botAudioLevelEnabled)) {
|
|
537
|
+
return;
|
|
538
|
+
}
|
|
539
|
+
// A frame is handled the first time it is pushed, except aggregated
|
|
540
|
+
// text, which is handled once it has gone through the output
|
|
541
|
+
// transport and has the right timing.
|
|
542
|
+
if (!data.firstPush &&
|
|
543
|
+
!(frame instanceof AggregatedTextFrame || frame instanceof AggregatedTextProgressFrame)) {
|
|
544
|
+
return;
|
|
545
|
+
}
|
|
546
|
+
if ((frame instanceof UserStartedSpeakingFrame || frame instanceof UserStoppedSpeakingFrame) &&
|
|
547
|
+
this._params.userSpeakingEnabled) {
|
|
548
|
+
await this._handleInterruptions(frame);
|
|
549
|
+
}
|
|
550
|
+
else if ((frame instanceof VADUserStartedSpeakingFrame ||
|
|
551
|
+
frame instanceof VADUserStoppedSpeakingFrame) &&
|
|
552
|
+
this._params.vadUserSpeakingEnabled) {
|
|
553
|
+
await this._handleVadSpeaking(frame);
|
|
554
|
+
}
|
|
555
|
+
else if ((frame instanceof UserMuteStartedFrame || frame instanceof UserMuteStoppedFrame) &&
|
|
556
|
+
this._params.userMuteEnabled) {
|
|
557
|
+
await this._handleUserMute(frame);
|
|
558
|
+
}
|
|
559
|
+
else if (frame instanceof BotStartedSpeakingFrame ||
|
|
560
|
+
frame instanceof BotStoppedSpeakingFrame) {
|
|
561
|
+
await this._handleBotSpeaking(frame);
|
|
562
|
+
}
|
|
563
|
+
else if (frame instanceof InterruptionFrame) {
|
|
564
|
+
await this._handleBotInterrupted();
|
|
565
|
+
}
|
|
566
|
+
else if (frame instanceof TranscriptionFrame || frame instanceof InterimTranscriptionFrame) {
|
|
567
|
+
await this._handleUserTranscriptions(frame);
|
|
568
|
+
}
|
|
569
|
+
else if (frame instanceof UserBackchannelFrame && this._params.userInputEnabled) {
|
|
570
|
+
await this._sendUserInput(frame.text, "backchannel", {
|
|
571
|
+
timestamp: frame.timestamp,
|
|
572
|
+
final: true,
|
|
573
|
+
userId: frame.userId,
|
|
574
|
+
});
|
|
575
|
+
}
|
|
576
|
+
else if (frame instanceof RTVISendTextFrame && this._params.userInputEnabled) {
|
|
577
|
+
await this._sendUserInput(frame.text, "chat", {
|
|
578
|
+
timestamp: timeNowIso8601(),
|
|
579
|
+
final: true,
|
|
580
|
+
userId: frame.userId,
|
|
581
|
+
msgId: frame.msgId,
|
|
582
|
+
});
|
|
583
|
+
}
|
|
584
|
+
else if (frame instanceof LLMContextFrame && this._params.userLlmEnabled) {
|
|
585
|
+
await this._handleContext(frame);
|
|
586
|
+
}
|
|
587
|
+
else if (frame instanceof LLMFullResponseStartFrame && this._params.botLlmEnabled) {
|
|
588
|
+
await this.sendRtviMessage(new RTVI.BotLLMStartedMessage());
|
|
589
|
+
}
|
|
590
|
+
else if (frame instanceof LLMFullResponseEndFrame && this._params.botLlmEnabled) {
|
|
591
|
+
await this.sendRtviMessage(new RTVI.BotLLMStoppedMessage());
|
|
592
|
+
}
|
|
593
|
+
else if (frame instanceof LLMTextFrame &&
|
|
594
|
+
(this._params.botLlmEnabled || this._params.llmRawTextEnabled)) {
|
|
595
|
+
await this._handleLlmTextFrame(frame);
|
|
596
|
+
}
|
|
597
|
+
else if (frame instanceof LLMMarkerResponseFrame && this._params.botLlmMarkerEnabled) {
|
|
598
|
+
await this.sendRtviMessage(new RTVI.BotLLMMarkerMessage({
|
|
599
|
+
data: new RTVI.BotLLMMarkerMessageData({
|
|
600
|
+
text: frame.marker || "",
|
|
601
|
+
kind: frame.kind,
|
|
602
|
+
raw: frame.raw,
|
|
603
|
+
markers: [...frame.markers],
|
|
604
|
+
}),
|
|
605
|
+
}));
|
|
606
|
+
}
|
|
607
|
+
else if (frame instanceof TTSStartedFrame && this._params.botTtsEnabled) {
|
|
608
|
+
await this.sendRtviMessage(new RTVI.BotTTSStartedMessage());
|
|
609
|
+
}
|
|
610
|
+
else if (frame instanceof TTSStoppedFrame && this._params.botTtsEnabled) {
|
|
611
|
+
await this.sendRtviMessage(new RTVI.BotTTSStoppedMessage());
|
|
612
|
+
}
|
|
613
|
+
else if (frame instanceof AggregatedTextProgressFrame) {
|
|
614
|
+
if (src instanceof BaseOutputTransport) {
|
|
615
|
+
await this._handleAggregatedProgress(frame);
|
|
616
|
+
}
|
|
617
|
+
}
|
|
618
|
+
else if (frame instanceof AggregatedTextFrame &&
|
|
619
|
+
(this._params.botOutputEnabled ||
|
|
620
|
+
this._params.botTtsEnabled ||
|
|
621
|
+
this._params.ttsRawTextEnabled)) {
|
|
622
|
+
if (src instanceof BaseOutputTransport) {
|
|
623
|
+
await this._handleAggregatedLlmText(frame);
|
|
624
|
+
}
|
|
625
|
+
}
|
|
626
|
+
else if (frame instanceof MetricsFrame && this._params.metricsEnabled) {
|
|
627
|
+
await this._handleMetrics(frame);
|
|
628
|
+
}
|
|
629
|
+
else if (frame instanceof FunctionCallsStartedFrame ||
|
|
630
|
+
frame instanceof FunctionCallInProgressFrame ||
|
|
631
|
+
frame instanceof FunctionCallCancelFrame ||
|
|
632
|
+
frame instanceof FunctionCallResultFrame ||
|
|
633
|
+
frame instanceof ExternalFunctionCallsStartedFrame ||
|
|
634
|
+
frame instanceof ExternalFunctionCallInProgressFrame ||
|
|
635
|
+
frame instanceof ExternalFunctionCallCancelFrame ||
|
|
636
|
+
frame instanceof ExternalFunctionCallResultFrame) {
|
|
637
|
+
await this._reportFunctionCallFrame(frame);
|
|
638
|
+
}
|
|
639
|
+
else if (frame instanceof RTVIServerMessageFrame) {
|
|
640
|
+
const message = new RTVI.ServerMessage({ data: frame.data });
|
|
641
|
+
await this.sendRtviMessage(message);
|
|
642
|
+
}
|
|
643
|
+
else if (frame instanceof RTVIUICommandFrame) {
|
|
644
|
+
const message = new RTVI.UICommandMessage({
|
|
645
|
+
data: new RTVI.UICommandData({ command: frame.command, payload: frame.payload }),
|
|
646
|
+
});
|
|
647
|
+
await this.sendRtviMessage(message);
|
|
648
|
+
}
|
|
649
|
+
else if (frame instanceof RTVIUIJobGroupFrame) {
|
|
650
|
+
if (frame.data !== null && frame.data !== undefined) {
|
|
651
|
+
const message = new RTVI.UIJobGroupMessage({ data: frame.data });
|
|
652
|
+
await this.sendRtviMessage(message);
|
|
653
|
+
}
|
|
654
|
+
}
|
|
655
|
+
else if (frame instanceof RTVIConfigureObserverFrame) {
|
|
656
|
+
this._applyConfig(frame);
|
|
657
|
+
}
|
|
658
|
+
else if (frame instanceof RTVIServerResponseFrame) {
|
|
659
|
+
if (frame.error !== null && frame.error !== undefined) {
|
|
660
|
+
await this._sendErrorResponse(frame);
|
|
661
|
+
}
|
|
662
|
+
else {
|
|
663
|
+
await this._sendServerResponse(frame);
|
|
664
|
+
}
|
|
665
|
+
}
|
|
666
|
+
else if (frame instanceof InputAudioRawFrame && this._params.userAudioLevelEnabled) {
|
|
667
|
+
// Every frame feeds the rolling window, but the window is only
|
|
668
|
+
// measured when a level is due to be reported.
|
|
669
|
+
this._userVolumeTracker.update(frame.audio, frame.sampleRate);
|
|
670
|
+
const currTime = time();
|
|
671
|
+
const diffTime = currTime - this._lastUserAudioLevel;
|
|
672
|
+
if (diffTime > this._params.audioLevelPeriodSecs) {
|
|
673
|
+
const level = this._userVolumeTracker.volume;
|
|
674
|
+
const message = new RTVI.UserAudioLevelMessage({
|
|
675
|
+
data: new RTVI.AudioLevelMessageData({ value: level }),
|
|
676
|
+
});
|
|
677
|
+
await this.sendRtviMessage(message);
|
|
678
|
+
this._lastUserAudioLevel = currTime;
|
|
679
|
+
}
|
|
680
|
+
}
|
|
681
|
+
else if (frame instanceof TTSAudioRawFrame && this._params.botAudioLevelEnabled) {
|
|
682
|
+
this._botVolumeTracker.update(frame.audio, frame.sampleRate);
|
|
683
|
+
const currTime = time();
|
|
684
|
+
const diffTime = currTime - this._lastBotAudioLevel;
|
|
685
|
+
if (diffTime > this._params.audioLevelPeriodSecs) {
|
|
686
|
+
const level = this._botVolumeTracker.volume;
|
|
687
|
+
const message = new RTVI.BotAudioLevelMessage({
|
|
688
|
+
data: new RTVI.AudioLevelMessageData({ value: level }),
|
|
689
|
+
});
|
|
690
|
+
await this.sendRtviMessage(message);
|
|
691
|
+
this._lastBotAudioLevel = currTime;
|
|
692
|
+
}
|
|
693
|
+
}
|
|
694
|
+
}
|
|
695
|
+
/** Handle user speaking interruption frames. */
|
|
696
|
+
async _handleInterruptions(frame) {
|
|
697
|
+
let message = null;
|
|
698
|
+
if (frame instanceof UserStartedSpeakingFrame) {
|
|
699
|
+
message = new RTVI.UserStartedSpeakingMessage();
|
|
700
|
+
}
|
|
701
|
+
else if (frame instanceof UserStoppedSpeakingFrame) {
|
|
702
|
+
message = new RTVI.UserStoppedSpeakingMessage();
|
|
703
|
+
}
|
|
704
|
+
if (message) {
|
|
705
|
+
await this.sendRtviMessage(message);
|
|
706
|
+
}
|
|
707
|
+
}
|
|
708
|
+
/** Emit raw VAD user started/stopped speaking messages. */
|
|
709
|
+
async _handleVadSpeaking(frame) {
|
|
710
|
+
let message = null;
|
|
711
|
+
if (frame instanceof VADUserStartedSpeakingFrame) {
|
|
712
|
+
message = new RTVI.VADUserStartedSpeakingMessage();
|
|
713
|
+
}
|
|
714
|
+
else if (frame instanceof VADUserStoppedSpeakingFrame) {
|
|
715
|
+
message = new RTVI.VADUserStoppedSpeakingMessage();
|
|
716
|
+
}
|
|
717
|
+
if (message) {
|
|
718
|
+
await this.sendRtviMessage(message);
|
|
719
|
+
}
|
|
720
|
+
}
|
|
721
|
+
/** Handle user mute/unmute frames. */
|
|
722
|
+
async _handleUserMute(frame) {
|
|
723
|
+
let message = null;
|
|
724
|
+
if (frame instanceof UserMuteStartedFrame) {
|
|
725
|
+
message = new RTVI.UserMuteStartedMessage();
|
|
726
|
+
}
|
|
727
|
+
else if (frame instanceof UserMuteStoppedFrame) {
|
|
728
|
+
message = new RTVI.UserMuteStoppedMessage();
|
|
729
|
+
}
|
|
730
|
+
if (message) {
|
|
731
|
+
await this.sendRtviMessage(message);
|
|
732
|
+
}
|
|
733
|
+
}
|
|
734
|
+
/** Track whether the bot is speaking, and tell clients if they want to know. */
|
|
735
|
+
async _handleBotSpeaking(frame) {
|
|
736
|
+
if (frame instanceof BotStartedSpeakingFrame) {
|
|
737
|
+
if (this._params.botSpeakingEnabled) {
|
|
738
|
+
await this.sendRtviMessage(new RTVI.BotStartedSpeakingMessage());
|
|
739
|
+
}
|
|
740
|
+
this._forgetInterruptedSegments();
|
|
741
|
+
// Flush any queued aggregated text frames
|
|
742
|
+
for (const queuedFrame of this._queuedAggregatedTextFrames) {
|
|
743
|
+
await this._sendAggregatedLlmText(queuedFrame);
|
|
744
|
+
}
|
|
745
|
+
this._queuedAggregatedTextFrames.length = 0;
|
|
746
|
+
this._botIsSpeaking = true;
|
|
747
|
+
}
|
|
748
|
+
else if (frame instanceof BotStoppedSpeakingFrame) {
|
|
749
|
+
if (this._params.botSpeakingEnabled) {
|
|
750
|
+
await this.sendRtviMessage(new RTVI.BotStoppedSpeakingMessage());
|
|
751
|
+
}
|
|
752
|
+
this._forgetInterruptedSegments();
|
|
753
|
+
this._botIsSpeaking = false;
|
|
754
|
+
}
|
|
755
|
+
}
|
|
756
|
+
/** Tell clients the bot's output was cut off, and drop what it won't say. */
|
|
757
|
+
async _handleBotInterrupted() {
|
|
758
|
+
// The bot's in-flight output was cut off (VAD barge-in or a programmatic
|
|
759
|
+
// runImmediately interrupt). Let clients drop what it was mid-saying.
|
|
760
|
+
if (this._params.botSpeakingEnabled) {
|
|
761
|
+
await this.sendRtviMessage(new RTVI.BotInterruptedMessage());
|
|
762
|
+
}
|
|
763
|
+
this._skippedSegmentsInterrupted = true;
|
|
764
|
+
// Queued text whose speech was cut off would otherwise go out with the
|
|
765
|
+
// next speech, and hold back text that won't be spoken.
|
|
766
|
+
this._queuedAggregatedTextFrames = this._queuedAggregatedTextFrames.filter((f) => !f.interruptible);
|
|
767
|
+
}
|
|
768
|
+
/** Handle aggregated LLM text output frames. */
|
|
769
|
+
async _handleAggregatedLlmText(frame) {
|
|
770
|
+
// Text waits for the bot to start speaking, including text spoken without
|
|
771
|
+
// a TTS service (a speech-to-speech service's transcript). Text the TTS
|
|
772
|
+
// left out (skipTts) has nothing to wait for, except text queued before it.
|
|
773
|
+
if (this._botIsSpeaking || (frame.skipTts && this._queuedAggregatedTextFrames.length === 0)) {
|
|
774
|
+
await this._sendAggregatedLlmText(frame);
|
|
775
|
+
}
|
|
776
|
+
else {
|
|
777
|
+
this._queuedAggregatedTextFrames.push(frame);
|
|
778
|
+
}
|
|
779
|
+
}
|
|
780
|
+
/** Handle progress frames. */
|
|
781
|
+
async _handleAggregatedProgress(frame) {
|
|
782
|
+
if (this._isSkippedType(frame.textType)) {
|
|
783
|
+
if (frame.remainingText === "") {
|
|
784
|
+
// The skipped segment's last word has been spoken.
|
|
785
|
+
this._skippedSegmentIds.delete(frame.segmentId);
|
|
786
|
+
}
|
|
787
|
+
return;
|
|
788
|
+
}
|
|
789
|
+
// 1.4.x clients use the old separate bot-output-progress event which was never
|
|
790
|
+
// released, so progress events are simply not sent to legacy clients.
|
|
791
|
+
if (this._isLegacyClient) {
|
|
792
|
+
return;
|
|
793
|
+
}
|
|
794
|
+
logger.trace(`${this} TTS progress: context_id=${frame.contextId} ` +
|
|
795
|
+
`source_segment_id=${frame.segmentId} ` +
|
|
796
|
+
`accumulated=${JSON.stringify(frame.accumulatedText)} ` +
|
|
797
|
+
`remaining=${JSON.stringify(frame.remainingText)}`);
|
|
798
|
+
let accumulated = frame.accumulatedText;
|
|
799
|
+
let remaining = frame.remainingText;
|
|
800
|
+
let text = frame.text;
|
|
801
|
+
const textType = frame.textType;
|
|
802
|
+
for (const [transformType, transform] of this._aggregationTransforms) {
|
|
803
|
+
if (transformType === textType || transformType === "*") {
|
|
804
|
+
const result = await transform(text, textType, accumulated, remaining);
|
|
805
|
+
if (result instanceof BotOutputTransformResult) {
|
|
806
|
+
accumulated = result.accumulatedText || accumulated;
|
|
807
|
+
remaining = result.remainingText || remaining;
|
|
808
|
+
text = result.text || text;
|
|
809
|
+
}
|
|
810
|
+
}
|
|
811
|
+
}
|
|
812
|
+
if (this._params.botOutputEnabled) {
|
|
813
|
+
const spokenStatus = remaining === "" ? "completed" : "in-progress";
|
|
814
|
+
const message = new RTVI.BotOutputMessage({
|
|
815
|
+
data: new RTVI.BotOutputMessageData({
|
|
816
|
+
text,
|
|
817
|
+
willBeSpoken: true,
|
|
818
|
+
textType,
|
|
819
|
+
segmentId: frame.segmentId,
|
|
820
|
+
spokenStatus,
|
|
821
|
+
spokenProgress: new RTVI.SpokenProgressData({
|
|
822
|
+
accumulatedText: accumulated,
|
|
823
|
+
remainingText: remaining,
|
|
824
|
+
}),
|
|
825
|
+
}),
|
|
826
|
+
});
|
|
827
|
+
await this.sendRtviMessage(message);
|
|
828
|
+
}
|
|
829
|
+
}
|
|
830
|
+
/** Forget the skipped segments an interruption cut off. */
|
|
831
|
+
_forgetInterruptedSegments() {
|
|
832
|
+
// By the time the bot next starts or stops speaking, the output transport
|
|
833
|
+
// has dropped the interrupted words, so none of them can still come.
|
|
834
|
+
if (this._skippedSegmentsInterrupted) {
|
|
835
|
+
this._skippedSegmentIds.clear();
|
|
836
|
+
this._skippedSegmentsInterrupted = false;
|
|
837
|
+
}
|
|
838
|
+
}
|
|
839
|
+
/** Whether text of this type is kept out of bot output and TTS text messages. */
|
|
840
|
+
_isSkippedType(textType) {
|
|
841
|
+
// A client that doesn't support backchannels would show one as part of the
|
|
842
|
+
// bot's response.
|
|
843
|
+
if (textType === TextType.BACKCHANNEL && !this._clientSupportsBackchannels) {
|
|
844
|
+
return true;
|
|
845
|
+
}
|
|
846
|
+
const skipTypes = this._params.skipTextTypes;
|
|
847
|
+
return Boolean(skipTypes && skipTypes.length > 0 && skipTypes.includes(textType));
|
|
848
|
+
}
|
|
849
|
+
/** Whether the frame's text type, or its segment's, keeps it from bot output. */
|
|
850
|
+
_isSkipped(frame) {
|
|
851
|
+
const segmentId = frame instanceof TTSTextFrame ? frame.segmentId : null;
|
|
852
|
+
if (this._isSkippedType(frame.textType)) {
|
|
853
|
+
if (segmentId !== null && segmentId !== undefined) {
|
|
854
|
+
// The segment spoken in one piece: no words of it follow.
|
|
855
|
+
this._skippedSegmentIds.delete(segmentId);
|
|
856
|
+
}
|
|
857
|
+
else if (frame.willBeSpoken) {
|
|
858
|
+
// A segment the TTS will speak: skip the words it is spoken in too.
|
|
859
|
+
this._skippedSegmentIds.add(frame.id);
|
|
860
|
+
}
|
|
861
|
+
return true;
|
|
862
|
+
}
|
|
863
|
+
// A word is skipped with the segment it was spoken in.
|
|
864
|
+
return segmentId !== null && segmentId !== undefined && this._skippedSegmentIds.has(segmentId);
|
|
865
|
+
}
|
|
866
|
+
/** Send aggregated LLM text messages. */
|
|
867
|
+
async _sendAggregatedLlmText(frame) {
|
|
868
|
+
if (this._isSkipped(frame)) {
|
|
869
|
+
return;
|
|
870
|
+
}
|
|
871
|
+
const textType = frame.textType;
|
|
872
|
+
// For 2.0.0+ clients, word and token types are not emitted as bot-output events;
|
|
873
|
+
// word-level progress is covered by the spoken_status/spoken_progress fields.
|
|
874
|
+
// bot-tts-text is a separate channel and is NOT suppressed here.
|
|
875
|
+
const suppressBotOutput = !this._isLegacyClient && (textType === TextType.WORD || textType === TextType.TOKEN);
|
|
876
|
+
let text = frame.text;
|
|
877
|
+
for (const [transformType, transform] of this._aggregationTransforms) {
|
|
878
|
+
if (transformType === textType || transformType === "*") {
|
|
879
|
+
const result = await transform(text, textType, null, null);
|
|
880
|
+
text = result instanceof BotOutputTransformResult ? result.text : result;
|
|
881
|
+
}
|
|
882
|
+
}
|
|
883
|
+
const isTTS = frame instanceof TTSTextFrame;
|
|
884
|
+
const willBeSpoken = frame.willBeSpoken;
|
|
885
|
+
// Text the TTS spoke for a segment is reported as that segment.
|
|
886
|
+
let segmentId = frame.id;
|
|
887
|
+
if (frame instanceof TTSTextFrame &&
|
|
888
|
+
frame.segmentId !== null &&
|
|
889
|
+
frame.segmentId !== undefined) {
|
|
890
|
+
segmentId = frame.segmentId;
|
|
891
|
+
}
|
|
892
|
+
if (this._params.botOutputEnabled && !suppressBotOutput) {
|
|
893
|
+
let spokenStatus;
|
|
894
|
+
let progress;
|
|
895
|
+
if (willBeSpoken) {
|
|
896
|
+
if (isTTS) {
|
|
897
|
+
// pushTextFrames path: TTSTextFrame arrives after synthesis completes
|
|
898
|
+
spokenStatus = "completed";
|
|
899
|
+
progress = new RTVI.SpokenProgressData({ accumulatedText: text, remainingText: "" });
|
|
900
|
+
}
|
|
901
|
+
else {
|
|
902
|
+
// word-timestamp path: AggregatedTextFrame arrives before synthesis starts
|
|
903
|
+
spokenStatus = "new";
|
|
904
|
+
progress = new RTVI.SpokenProgressData({ accumulatedText: "", remainingText: text });
|
|
905
|
+
}
|
|
906
|
+
}
|
|
907
|
+
else {
|
|
908
|
+
// Text that isn't spoken gets this one message, which starts its
|
|
909
|
+
// segment as spoken text's first message does.
|
|
910
|
+
spokenStatus = "new";
|
|
911
|
+
progress = null;
|
|
912
|
+
}
|
|
913
|
+
const data = new RTVI.BotOutputMessageData({
|
|
914
|
+
text,
|
|
915
|
+
spoken: isTTS,
|
|
916
|
+
willBeSpoken,
|
|
917
|
+
textType,
|
|
918
|
+
segmentId,
|
|
919
|
+
spokenStatus,
|
|
920
|
+
spokenProgress: progress,
|
|
921
|
+
});
|
|
922
|
+
const message = new RTVI.BotOutputMessage({ data });
|
|
923
|
+
await this.sendRtviMessage(message);
|
|
924
|
+
}
|
|
925
|
+
if (isTTS && this._params.ttsRawTextEnabled && this._clientSupportsRawText) {
|
|
926
|
+
await this.sendRtviMessage(new RTVI.TTSRawTextMessage({ data: new RTVI.TextMessageData({ text }) }));
|
|
927
|
+
}
|
|
928
|
+
if (isTTS && this._params.botTtsEnabled) {
|
|
929
|
+
const ttsMessage = new RTVI.BotTTSTextMessage({ data: new RTVI.TextMessageData({ text }) });
|
|
930
|
+
await this.sendRtviMessage(ttsMessage);
|
|
931
|
+
}
|
|
932
|
+
}
|
|
933
|
+
/** Handle LLM text output frames. */
|
|
934
|
+
async _handleLlmTextFrame(frame) {
|
|
935
|
+
if (this._params.llmRawTextEnabled && this._clientSupportsRawText) {
|
|
936
|
+
await this.sendRtviMessage(new RTVI.LLMRawTextMessage({ data: new RTVI.TextMessageData({ text: frame.text }) }));
|
|
937
|
+
}
|
|
938
|
+
if (!this._params.botLlmEnabled) {
|
|
939
|
+
return;
|
|
940
|
+
}
|
|
941
|
+
const message = new RTVI.BotLLMTextMessage({
|
|
942
|
+
data: new RTVI.TextMessageData({ text: frame.text }),
|
|
943
|
+
});
|
|
944
|
+
await this.sendRtviMessage(message);
|
|
945
|
+
// TODO (mrkb): Remove all this logic when we fully deprecate bot-transcription messages.
|
|
946
|
+
this._botTranscription += frame.text;
|
|
947
|
+
if (matchEndofsentence(this._botTranscription) && this._botTranscription.length > 0) {
|
|
948
|
+
await this.sendRtviMessage(new RTVI.BotTranscriptionMessage({
|
|
949
|
+
data: new RTVI.TextMessageData({ text: this._botTranscription }),
|
|
950
|
+
}));
|
|
951
|
+
this._botTranscription = "";
|
|
952
|
+
}
|
|
953
|
+
}
|
|
954
|
+
/** Handle user transcription frames. */
|
|
955
|
+
async _handleUserTranscriptions(frame) {
|
|
956
|
+
const final = frame instanceof TranscriptionFrame;
|
|
957
|
+
if (this._params.userInputEnabled) {
|
|
958
|
+
await this._sendUserInput(frame.text, "transcription", {
|
|
959
|
+
timestamp: frame.timestamp,
|
|
960
|
+
final,
|
|
961
|
+
userId: frame.userId,
|
|
962
|
+
});
|
|
963
|
+
}
|
|
964
|
+
if (this._params.sttRawTextEnabled && this._clientSupportsRawText) {
|
|
965
|
+
await this.sendRtviMessage(new RTVI.STTRawTextMessage({
|
|
966
|
+
data: new RTVI.STTRawTextMessageData({
|
|
967
|
+
text: frame.text,
|
|
968
|
+
userId: frame.userId,
|
|
969
|
+
timestamp: frame.timestamp,
|
|
970
|
+
final,
|
|
971
|
+
}),
|
|
972
|
+
}));
|
|
973
|
+
}
|
|
974
|
+
// `user-transcription` is deprecated upstream, but clients still consume
|
|
975
|
+
// it; the deprecated switch to turn it off is not ported.
|
|
976
|
+
const message = new RTVI.UserTranscriptionMessage({
|
|
977
|
+
data: new RTVI.UserTranscriptionMessageData({
|
|
978
|
+
text: frame.text,
|
|
979
|
+
userId: frame.userId,
|
|
980
|
+
timestamp: frame.timestamp,
|
|
981
|
+
final,
|
|
982
|
+
}),
|
|
983
|
+
});
|
|
984
|
+
await this.sendRtviMessage(message);
|
|
985
|
+
}
|
|
986
|
+
/** Send a user input message. */
|
|
987
|
+
async _sendUserInput(text, inputType, options) {
|
|
988
|
+
const message = new RTVI.UserInputMessage({
|
|
989
|
+
data: new RTVI.UserInputMessageData({
|
|
990
|
+
text,
|
|
991
|
+
inputType,
|
|
992
|
+
timestamp: options.timestamp,
|
|
993
|
+
final: options.final,
|
|
994
|
+
userId: options.userId ?? null,
|
|
995
|
+
msgId: options.msgId ?? null,
|
|
996
|
+
}),
|
|
997
|
+
});
|
|
998
|
+
await this.sendRtviMessage(message);
|
|
999
|
+
}
|
|
1000
|
+
/** Process LLM context frames to extract user messages for the RTVI client. */
|
|
1001
|
+
async _handleContext(frame) {
|
|
1002
|
+
try {
|
|
1003
|
+
const messages = frame.context.getMessages();
|
|
1004
|
+
if (!messages || messages.length === 0) {
|
|
1005
|
+
return;
|
|
1006
|
+
}
|
|
1007
|
+
const message = messages[messages.length - 1];
|
|
1008
|
+
// Upstream tells the two formats apart by `getattr` (never true for a
|
|
1009
|
+
// dict) vs `isinstance(message, dict)`: a plain object here is a dict.
|
|
1010
|
+
const isObject = message !== null && typeof message === "object";
|
|
1011
|
+
const proto = isObject ? Object.getPrototypeOf(message) : undefined;
|
|
1012
|
+
const isDict = isObject && (proto === Object.prototype || proto === null);
|
|
1013
|
+
// Handle Google LLM format (objects with a `parts` attribute)
|
|
1014
|
+
// Note: not possible if frame is a universal LLMContextFrame
|
|
1015
|
+
if (isObject && !isDict && message.role === "user" && "parts" in message) {
|
|
1016
|
+
const parts = (message.parts ?? []);
|
|
1017
|
+
const text = parts
|
|
1018
|
+
.filter((part) => part !== null && typeof part === "object" && "text" in part)
|
|
1019
|
+
.map((part) => part.text)
|
|
1020
|
+
.join("");
|
|
1021
|
+
if (text) {
|
|
1022
|
+
const rtviMessage = new RTVI.UserLLMTextMessage({
|
|
1023
|
+
data: new RTVI.TextMessageData({ text }),
|
|
1024
|
+
});
|
|
1025
|
+
await this.sendRtviMessage(rtviMessage);
|
|
1026
|
+
}
|
|
1027
|
+
}
|
|
1028
|
+
// Handle OpenAI format (original implementation)
|
|
1029
|
+
else if (isDict) {
|
|
1030
|
+
if (message.role === "user") {
|
|
1031
|
+
const content = message.content;
|
|
1032
|
+
if (content === null || content === undefined) {
|
|
1033
|
+
return;
|
|
1034
|
+
}
|
|
1035
|
+
let text;
|
|
1036
|
+
if (typeof content === "string") {
|
|
1037
|
+
text = content;
|
|
1038
|
+
}
|
|
1039
|
+
else {
|
|
1040
|
+
// Anything else is a sequence of content parts.
|
|
1041
|
+
text = content
|
|
1042
|
+
.filter((item) => "text" in item)
|
|
1043
|
+
.map((item) => item.text)
|
|
1044
|
+
.join(" ");
|
|
1045
|
+
}
|
|
1046
|
+
const rtviMessage = new RTVI.UserLLMTextMessage({
|
|
1047
|
+
data: new RTVI.TextMessageData({ text }),
|
|
1048
|
+
});
|
|
1049
|
+
await this.sendRtviMessage(rtviMessage);
|
|
1050
|
+
}
|
|
1051
|
+
}
|
|
1052
|
+
}
|
|
1053
|
+
catch (e) {
|
|
1054
|
+
rethrowIfCancelled(e);
|
|
1055
|
+
logger.warning(`Caught an error while trying to handle context: ${e instanceof Error ? e.message : e}`);
|
|
1056
|
+
}
|
|
1057
|
+
}
|
|
1058
|
+
/** Handle metrics frames and convert to RTVI metrics messages. */
|
|
1059
|
+
async _handleMetrics(frame) {
|
|
1060
|
+
const metrics = {};
|
|
1061
|
+
const append = (key, value) => {
|
|
1062
|
+
if (!(key in metrics)) {
|
|
1063
|
+
metrics[key] = [];
|
|
1064
|
+
}
|
|
1065
|
+
metrics[key].push(value);
|
|
1066
|
+
};
|
|
1067
|
+
for (const d of frame.data) {
|
|
1068
|
+
if (d instanceof TTFBMetricsData) {
|
|
1069
|
+
append("ttfb", dumpMetrics(d));
|
|
1070
|
+
}
|
|
1071
|
+
else if (d instanceof TTFAMetricsData) {
|
|
1072
|
+
append("ttfa", dumpMetrics(d));
|
|
1073
|
+
}
|
|
1074
|
+
else if (d instanceof TTFATMetricsData) {
|
|
1075
|
+
append("ttfat", dumpMetrics(d));
|
|
1076
|
+
}
|
|
1077
|
+
else if (d instanceof ProcessingMetricsData) {
|
|
1078
|
+
append("processing", dumpMetrics(d));
|
|
1079
|
+
}
|
|
1080
|
+
else if (d instanceof LLMUsageMetricsData) {
|
|
1081
|
+
append("tokens", dumpMetrics(d.value));
|
|
1082
|
+
}
|
|
1083
|
+
else if (d instanceof STTUsageMetricsData) {
|
|
1084
|
+
append("stt_usage", dumpMetrics(d));
|
|
1085
|
+
}
|
|
1086
|
+
else if (d instanceof TTSUsageMetricsData) {
|
|
1087
|
+
append("characters", dumpMetrics(d));
|
|
1088
|
+
}
|
|
1089
|
+
}
|
|
1090
|
+
const message = new RTVI.MetricsMessage({ data: metrics });
|
|
1091
|
+
await this.sendRtviMessage(message);
|
|
1092
|
+
}
|
|
1093
|
+
/** Send a response to the client for a specific request. */
|
|
1094
|
+
async _sendServerResponse(frame) {
|
|
1095
|
+
const message = new RTVI.ServerResponse({
|
|
1096
|
+
id: String(frame.clientMsg.msgId),
|
|
1097
|
+
data: new RTVI.RawServerResponseData({ t: frame.clientMsg.type, d: frame.data }),
|
|
1098
|
+
});
|
|
1099
|
+
await this.sendRtviMessage(message);
|
|
1100
|
+
}
|
|
1101
|
+
/** Send a response to the client for a specific request. */
|
|
1102
|
+
async _sendErrorResponse(frame) {
|
|
1103
|
+
if (frame.error === null || frame.error === undefined) {
|
|
1104
|
+
throw new Error("assertion failed: frame.error is not None");
|
|
1105
|
+
}
|
|
1106
|
+
const message = new RTVI.ErrorResponse({
|
|
1107
|
+
id: String(frame.clientMsg.msgId),
|
|
1108
|
+
data: new RTVI.ErrorResponseData({ error: frame.error }),
|
|
1109
|
+
});
|
|
1110
|
+
await this.sendRtviMessage(message);
|
|
1111
|
+
}
|
|
1112
|
+
}
|
|
1113
|
+
//# sourceMappingURL=observer.js.map
|