@pipedog/core 0.0.0 → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +25 -0
- package/NOTICE +9 -0
- package/dist/adapters/base_llm_adapter.d.ts +266 -0
- package/dist/adapters/base_llm_adapter.js +489 -0
- package/dist/adapters/index.d.ts +2 -0
- package/dist/adapters/index.js +8 -0
- package/dist/adapters/schemas/direct_function.d.ts +202 -0
- package/dist/adapters/schemas/direct_function.js +302 -0
- package/dist/adapters/schemas/function_schema.d.ts +67 -0
- package/dist/adapters/schemas/function_schema.js +72 -0
- package/dist/adapters/schemas/index.d.ts +2 -0
- package/dist/adapters/schemas/index.js +8 -0
- package/dist/adapters/schemas/tools_schema.d.ts +57 -0
- package/dist/adapters/schemas/tools_schema.js +79 -0
- package/dist/adapters/services/index.d.ts +2 -0
- package/dist/adapters/services/index.js +8 -0
- package/dist/adapters/services/open_ai_adapter.d.ts +147 -0
- package/dist/adapters/services/open_ai_adapter.js +250 -0
- package/dist/audio/dtmf/dtmf_data.d.ts +3 -0
- package/dist/audio/dtmf/dtmf_data.js +22 -0
- package/dist/audio/dtmf/index.d.ts +2 -0
- package/dist/audio/dtmf/index.js +8 -0
- package/dist/audio/dtmf/types.d.ts +31 -0
- package/dist/audio/dtmf/types.js +35 -0
- package/dist/audio/dtmf/utils.d.ts +13 -0
- package/dist/audio/dtmf/utils.js +62 -0
- package/dist/audio/filters/base_audio_filter.d.ts +51 -0
- package/dist/audio/filters/base_audio_filter.js +17 -0
- package/dist/audio/filters/index.d.ts +2 -0
- package/dist/audio/filters/index.js +8 -0
- package/dist/audio/index.d.ts +2 -0
- package/dist/audio/index.js +8 -0
- package/dist/audio/krisp_instance.d.ts +95 -0
- package/dist/audio/krisp_instance.js +229 -0
- package/dist/audio/mixers/base_audio_mixer.d.ts +56 -0
- package/dist/audio/mixers/base_audio_mixer.js +16 -0
- package/dist/audio/mixers/index.d.ts +2 -0
- package/dist/audio/mixers/index.js +8 -0
- package/dist/audio/resamplers/base_audio_resampler.d.ts +46 -0
- package/dist/audio/resamplers/base_audio_resampler.js +35 -0
- package/dist/audio/resamplers/index.d.ts +2 -0
- package/dist/audio/resamplers/index.js +8 -0
- package/dist/audio/resamplers/resampy_resampler.d.ts +10 -0
- package/dist/audio/resamplers/resampy_resampler.js +15 -0
- package/dist/audio/resamplers/soxr_resampler.d.ts +54 -0
- package/dist/audio/resamplers/soxr_resampler.js +69 -0
- package/dist/audio/resamplers/soxr_stream_resampler.d.ts +75 -0
- package/dist/audio/resamplers/soxr_stream_resampler.js +146 -0
- package/dist/audio/turn/base_turn_analyzer.d.ts +84 -0
- package/dist/audio/turn/base_turn_analyzer.js +61 -0
- package/dist/audio/turn/index.d.ts +2 -0
- package/dist/audio/turn/index.js +8 -0
- package/dist/audio/turn/smart_turn/_whisper_features.d.ts +30 -0
- package/dist/audio/turn/smart_turn/_whisper_features.js +225 -0
- package/dist/audio/turn/smart_turn/base_smart_turn.d.ts +94 -0
- package/dist/audio/turn/smart_turn/base_smart_turn.js +239 -0
- package/dist/audio/turn/smart_turn/local_smart_turn_v3.d.ts +76 -0
- package/dist/audio/turn/smart_turn/local_smart_turn_v3.js +143 -0
- package/dist/audio/utils.d.ts +224 -0
- package/dist/audio/utils.js +634 -0
- package/dist/audio/vad/data/index.d.ts +2 -0
- package/dist/audio/vad/data/index.js +8 -0
- package/dist/audio/vad/index.d.ts +2 -0
- package/dist/audio/vad/index.js +8 -0
- package/dist/audio/vad/vad_analyzer.d.ts +126 -0
- package/dist/audio/vad/vad_analyzer.js +211 -0
- package/dist/audio/vad/vad_controller.d.ts +143 -0
- package/dist/audio/vad/vad_controller.js +228 -0
- package/dist/audio/volume.d.ts +38 -0
- package/dist/audio/volume.js +75 -0
- package/dist/bus/adapters/base.d.ts +35 -0
- package/dist/bus/adapters/base.js +19 -0
- package/dist/bus/adapters/index.d.ts +10 -0
- package/dist/bus/adapters/index.js +15 -0
- package/dist/bus/adapters/llm_context_adapter.d.ts +36 -0
- package/dist/bus/adapters/llm_context_adapter.js +82 -0
- package/dist/bus/adapters/tools_schema_adapter.d.ts +25 -0
- package/dist/bus/adapters/tools_schema_adapter.js +70 -0
- package/dist/bus/bridge_processor.d.ts +111 -0
- package/dist/bus/bridge_processor.js +166 -0
- package/dist/bus/bus.d.ts +110 -0
- package/dist/bus/bus.js +200 -0
- package/dist/bus/index.d.ts +22 -0
- package/dist/bus/index.js +27 -0
- package/dist/bus/local/async_queue.d.ts +12 -0
- package/dist/bus/local/async_queue.js +21 -0
- package/dist/bus/local/index.d.ts +3 -0
- package/dist/bus/local/index.js +8 -0
- package/dist/bus/messages.d.ts +378 -0
- package/dist/bus/messages.js +357 -0
- package/dist/bus/network/index.d.ts +16 -0
- package/dist/bus/network/index.js +21 -0
- package/dist/bus/queue.d.ts +27 -0
- package/dist/bus/queue.js +51 -0
- package/dist/bus/serializers/base.d.ts +25 -0
- package/dist/bus/serializers/base.js +14 -0
- package/dist/bus/serializers/index.d.ts +9 -0
- package/dist/bus/serializers/index.js +14 -0
- package/dist/bus/serializers/json.d.ts +77 -0
- package/dist/bus/serializers/json.js +367 -0
- package/dist/bus/subscriber.d.ts +46 -0
- package/dist/bus/subscriber.js +27 -0
- package/dist/bus/ui/index.d.ts +3 -0
- package/dist/bus/ui/index.js +8 -0
- package/dist/bus/ui/messages.d.ts +186 -0
- package/dist/bus/ui/messages.js +171 -0
- package/dist/classifiers/base_classifier.d.ts +263 -0
- package/dist/classifiers/base_classifier.js +346 -0
- package/dist/classifiers/cloudflare/clef/classifier.d.ts +70 -0
- package/dist/classifiers/cloudflare/clef/classifier.js +204 -0
- package/dist/classifiers/cloudflare/clef/client.d.ts +104 -0
- package/dist/classifiers/cloudflare/clef/client.js +234 -0
- package/dist/classifiers/cloudflare/clef/index.d.ts +2 -0
- package/dist/classifiers/cloudflare/clef/index.js +8 -0
- package/dist/classifiers/cloudflare/index.d.ts +2 -0
- package/dist/classifiers/cloudflare/index.js +8 -0
- package/dist/classifiers/http_client.d.ts +70 -0
- package/dist/classifiers/http_client.js +161 -0
- package/dist/classifiers/index.d.ts +2 -0
- package/dist/classifiers/index.js +8 -0
- package/dist/classifiers/jev/classifier.d.ts +8 -0
- package/dist/classifiers/jev/classifier.js +13 -0
- package/dist/classifiers/jev/client.d.ts +8 -0
- package/dist/classifiers/jev/client.js +13 -0
- package/dist/classifiers/jev/index.d.ts +2 -0
- package/dist/classifiers/jev/index.js +8 -0
- package/dist/classifiers/llm/classifier.d.ts +72 -0
- package/dist/classifiers/llm/classifier.js +304 -0
- package/dist/classifiers/llm/index.d.ts +2 -0
- package/dist/classifiers/llm/index.js +8 -0
- package/dist/classifiers/openai/decisions/classifier.d.ts +77 -0
- package/dist/classifiers/openai/decisions/classifier.js +241 -0
- package/dist/classifiers/openai/decisions/client.d.ts +108 -0
- package/dist/classifiers/openai/decisions/client.js +234 -0
- package/dist/classifiers/openai/decisions/index.d.ts +2 -0
- package/dist/classifiers/openai/decisions/index.js +8 -0
- package/dist/classifiers/openai/index.d.ts +2 -0
- package/dist/classifiers/openai/index.js +8 -0
- package/dist/classifiers/typesafe/index.d.ts +2 -0
- package/dist/classifiers/typesafe/index.js +8 -0
- package/dist/classifiers/typesafe/jev/classifier.d.ts +70 -0
- package/dist/classifiers/typesafe/jev/classifier.js +220 -0
- package/dist/classifiers/typesafe/jev/client.d.ts +105 -0
- package/dist/classifiers/typesafe/jev/client.js +233 -0
- package/dist/classifiers/typesafe/jev/index.d.ts +2 -0
- package/dist/classifiers/typesafe/jev/index.js +8 -0
- package/dist/clocks/base_clock.d.ts +13 -0
- package/dist/clocks/base_clock.js +14 -0
- package/dist/clocks/index.d.ts +2 -0
- package/dist/clocks/index.js +8 -0
- package/dist/clocks/system_clock.d.ts +15 -0
- package/dist/clocks/system_clock.js +25 -0
- package/dist/frames/frames.d.ts +2533 -0
- package/dist/frames/frames.js +2485 -0
- package/dist/frames/index.d.ts +2 -0
- package/dist/frames/index.js +8 -0
- package/dist/frames/protobufs/frames_pb2.d.ts +229 -0
- package/dist/frames/protobufs/frames_pb2.js +466 -0
- package/dist/index.d.ts +11 -0
- package/dist/index.js +56 -0
- package/dist/metrics/index.d.ts +2 -0
- package/dist/metrics/index.js +8 -0
- package/dist/metrics/metrics.d.ts +276 -0
- package/dist/metrics/metrics.js +254 -0
- package/dist/observers/base_observer.d.ts +173 -0
- package/dist/observers/base_observer.js +151 -0
- package/dist/observers/error_observer.d.ts +75 -0
- package/dist/observers/error_observer.js +87 -0
- package/dist/observers/function_call_observer.d.ts +157 -0
- package/dist/observers/function_call_observer.js +222 -0
- package/dist/observers/index.d.ts +2 -0
- package/dist/observers/index.js +8 -0
- package/dist/observers/loggers/debug_log_observer.d.ts +91 -0
- package/dist/observers/loggers/debug_log_observer.js +228 -0
- package/dist/observers/loggers/index.d.ts +2 -0
- package/dist/observers/loggers/index.js +8 -0
- package/dist/observers/loggers/llm_log_observer.d.ts +24 -0
- package/dist/observers/loggers/llm_log_observer.js +110 -0
- package/dist/observers/loggers/metrics_log_observer.d.ts +85 -0
- package/dist/observers/loggers/metrics_log_observer.js +186 -0
- package/dist/observers/loggers/transcription_log_observer.d.ts +23 -0
- package/dist/observers/loggers/transcription_log_observer.js +74 -0
- package/dist/observers/service_metrics_observer.d.ts +168 -0
- package/dist/observers/service_metrics_observer.js +236 -0
- package/dist/observers/speaking_observer.d.ts +104 -0
- package/dist/observers/speaking_observer.js +150 -0
- package/dist/observers/startup_timing_observer.d.ts +221 -0
- package/dist/observers/startup_timing_observer.js +365 -0
- package/dist/observers/turn_tracking_observer.d.ts +78 -0
- package/dist/observers/turn_tracking_observer.js +189 -0
- package/dist/observers/user_bot_latency_observer.d.ts +411 -0
- package/dist/observers/user_bot_latency_observer.js +974 -0
- package/dist/pipeline/base_pipeline.d.ts +49 -0
- package/dist/pipeline/base_pipeline.js +101 -0
- package/dist/pipeline/capabilities.d.ts +76 -0
- package/dist/pipeline/capabilities.js +97 -0
- package/dist/pipeline/index.d.ts +2 -0
- package/dist/pipeline/index.js +8 -0
- package/dist/pipeline/job_context.d.ts +317 -0
- package/dist/pipeline/job_context.js +450 -0
- package/dist/pipeline/job_decorator.d.ts +54 -0
- package/dist/pipeline/job_decorator.js +92 -0
- package/dist/pipeline/llm_switcher.d.ts +84 -0
- package/dist/pipeline/llm_switcher.js +107 -0
- package/dist/pipeline/llm_with_backend.d.ts +435 -0
- package/dist/pipeline/llm_with_backend.js +806 -0
- package/dist/pipeline/parallel_pipeline.d.ts +83 -0
- package/dist/pipeline/parallel_pipeline.js +200 -0
- package/dist/pipeline/pipeline.d.ts +126 -0
- package/dist/pipeline/pipeline.js +196 -0
- package/dist/pipeline/service_switcher.d.ts +287 -0
- package/dist/pipeline/service_switcher.js +509 -0
- package/dist/pipeline/sync_parallel_pipeline.d.ts +192 -0
- package/dist/pipeline/sync_parallel_pipeline.js +387 -0
- package/dist/pipeline/worker.d.ts +814 -0
- package/dist/pipeline/worker.js +1639 -0
- package/dist/pipeline/worker_observer.d.ts +131 -0
- package/dist/pipeline/worker_observer.js +274 -0
- package/dist/pipeline/worker_ready_decorator.d.ts +45 -0
- package/dist/pipeline/worker_ready_decorator.js +76 -0
- package/dist/processors/aggregators/async_tool_messages.d.ts +146 -0
- package/dist/processors/aggregators/async_tool_messages.js +291 -0
- package/dist/processors/aggregators/dtmf_aggregator.d.ts +61 -0
- package/dist/processors/aggregators/dtmf_aggregator.js +144 -0
- package/dist/processors/aggregators/gated.d.ts +43 -0
- package/dist/processors/aggregators/gated.js +86 -0
- package/dist/processors/aggregators/gated_llm_context.d.ts +42 -0
- package/dist/processors/aggregators/gated_llm_context.js +86 -0
- package/dist/processors/aggregators/index.d.ts +2 -0
- package/dist/processors/aggregators/index.js +8 -0
- package/dist/processors/aggregators/llm_context.d.ts +370 -0
- package/dist/processors/aggregators/llm_context.js +580 -0
- package/dist/processors/aggregators/llm_context_summarizer.d.ts +219 -0
- package/dist/processors/aggregators/llm_context_summarizer.js +447 -0
- package/dist/processors/aggregators/llm_response.d.ts +37 -0
- package/dist/processors/aggregators/llm_response.js +71 -0
- package/dist/processors/aggregators/llm_response_universal.d.ts +966 -0
- package/dist/processors/aggregators/llm_response_universal.js +2394 -0
- package/dist/processors/aggregators/llm_text_processor.d.ts +53 -0
- package/dist/processors/aggregators/llm_text_processor.js +102 -0
- package/dist/processors/aggregators/sentence.d.ts +39 -0
- package/dist/processors/aggregators/sentence.js +69 -0
- package/dist/processors/async_generator.d.ts +42 -0
- package/dist/processors/async_generator.js +76 -0
- package/dist/processors/audio/audio_buffer_processor.d.ts +241 -0
- package/dist/processors/audio/audio_buffer_processor.js +565 -0
- package/dist/processors/audio/index.d.ts +2 -0
- package/dist/processors/audio/index.js +8 -0
- package/dist/processors/audio/vad_processor.d.ts +67 -0
- package/dist/processors/audio/vad_processor.js +103 -0
- package/dist/processors/consumer_processor.d.ts +49 -0
- package/dist/processors/consumer_processor.js +85 -0
- package/dist/processors/filters/frame_filter.d.ts +39 -0
- package/dist/processors/filters/frame_filter.js +52 -0
- package/dist/processors/filters/function_filter.d.ts +59 -0
- package/dist/processors/filters/function_filter.js +78 -0
- package/dist/processors/filters/identity_filter.d.ts +33 -0
- package/dist/processors/filters/identity_filter.js +37 -0
- package/dist/processors/filters/index.d.ts +2 -0
- package/dist/processors/filters/index.js +8 -0
- package/dist/processors/filters/null_filter.d.ts +34 -0
- package/dist/processors/filters/null_filter.js +47 -0
- package/dist/processors/filters/wake_notifier_filter.d.ts +48 -0
- package/dist/processors/filters/wake_notifier_filter.js +49 -0
- package/dist/processors/frame_processor.d.ts +360 -0
- package/dist/processors/frame_processor.js +932 -0
- package/dist/processors/frameworks/index.d.ts +2 -0
- package/dist/processors/frameworks/index.js +8 -0
- package/dist/processors/frameworks/rtvi/frames.d.ts +241 -0
- package/dist/processors/frameworks/rtvi/frames.js +259 -0
- package/dist/processors/frameworks/rtvi/index.d.ts +6 -0
- package/dist/processors/frameworks/rtvi/index.js +11 -0
- package/dist/processors/frameworks/rtvi/models.d.ts +1741 -0
- package/dist/processors/frameworks/rtvi/models.js +1807 -0
- package/dist/processors/frameworks/rtvi/observer.d.ts +340 -0
- package/dist/processors/frameworks/rtvi/observer.js +1113 -0
- package/dist/processors/frameworks/rtvi/processor.d.ts +182 -0
- package/dist/processors/frameworks/rtvi/processor.js +676 -0
- package/dist/processors/idle_frame_processor.d.ts +44 -0
- package/dist/processors/idle_frame_processor.js +86 -0
- package/dist/processors/index.d.ts +2 -0
- package/dist/processors/index.js +8 -0
- package/dist/processors/logger.d.ts +42 -0
- package/dist/processors/logger.js +87 -0
- package/dist/processors/metrics/frame_processor_metrics.d.ts +172 -0
- package/dist/processors/metrics/frame_processor_metrics.js +372 -0
- package/dist/processors/metrics/index.d.ts +2 -0
- package/dist/processors/metrics/index.js +8 -0
- package/dist/processors/producer_processor.d.ts +68 -0
- package/dist/processors/producer_processor.js +78 -0
- package/dist/processors/text_transformer.d.ts +30 -0
- package/dist/processors/text_transformer.js +48 -0
- package/dist/registry/index.d.ts +3 -0
- package/dist/registry/index.js +8 -0
- package/dist/registry/registry.d.ts +69 -0
- package/dist/registry/registry.js +149 -0
- package/dist/registry/types.d.ts +47 -0
- package/dist/registry/types.js +39 -0
- package/dist/serializers/base_serializer.d.ts +94 -0
- package/dist/serializers/base_serializer.js +82 -0
- package/dist/serializers/index.d.ts +2 -0
- package/dist/serializers/index.js +8 -0
- package/dist/services/ai_service.d.ts +145 -0
- package/dist/services/ai_service.js +262 -0
- package/dist/services/anthropic/index.d.ts +2 -0
- package/dist/services/anthropic/index.js +7 -0
- package/dist/services/assemblyai/index.d.ts +2 -0
- package/dist/services/assemblyai/index.js +7 -0
- package/dist/services/asyncai/index.d.ts +2 -0
- package/dist/services/asyncai/index.js +7 -0
- package/dist/services/aws/index.d.ts +2 -0
- package/dist/services/aws/index.js +7 -0
- package/dist/services/azure/index.d.ts +2 -0
- package/dist/services/azure/index.js +7 -0
- package/dist/services/baseten/index.d.ts +2 -0
- package/dist/services/baseten/index.js +7 -0
- package/dist/services/bland/index.d.ts +2 -0
- package/dist/services/bland/index.js +7 -0
- package/dist/services/camb/index.d.ts +2 -0
- package/dist/services/camb/index.js +7 -0
- package/dist/services/cartesia/index.d.ts +2 -0
- package/dist/services/cartesia/index.js +7 -0
- package/dist/services/cerebras/index.d.ts +2 -0
- package/dist/services/cerebras/index.js +7 -0
- package/dist/services/crusoe/index.d.ts +2 -0
- package/dist/services/crusoe/index.js +7 -0
- package/dist/services/deepgram/index.d.ts +2 -0
- package/dist/services/deepgram/index.js +7 -0
- package/dist/services/deepseek/index.d.ts +2 -0
- package/dist/services/deepseek/index.js +7 -0
- package/dist/services/elevenlabs/index.d.ts +2 -0
- package/dist/services/elevenlabs/index.js +7 -0
- package/dist/services/fal/index.d.ts +2 -0
- package/dist/services/fal/index.js +7 -0
- package/dist/services/fireworks/index.d.ts +2 -0
- package/dist/services/fireworks/index.js +7 -0
- package/dist/services/fish/index.d.ts +2 -0
- package/dist/services/fish/index.js +7 -0
- package/dist/services/funasr/index.d.ts +3 -0
- package/dist/services/funasr/index.js +8 -0
- package/dist/services/gladia/index.d.ts +2 -0
- package/dist/services/gladia/index.js +7 -0
- package/dist/services/google/index.d.ts +2 -0
- package/dist/services/google/index.js +7 -0
- package/dist/services/gradium/index.d.ts +2 -0
- package/dist/services/gradium/index.js +7 -0
- package/dist/services/grok/index.d.ts +2 -0
- package/dist/services/grok/index.js +7 -0
- package/dist/services/groq/index.d.ts +2 -0
- package/dist/services/groq/index.js +7 -0
- package/dist/services/heygen/index.d.ts +2 -0
- package/dist/services/heygen/index.js +7 -0
- package/dist/services/hume/index.d.ts +2 -0
- package/dist/services/hume/index.js +7 -0
- package/dist/services/image_service.d.ts +52 -0
- package/dist/services/image_service.js +60 -0
- package/dist/services/inception/index.d.ts +2 -0
- package/dist/services/inception/index.js +7 -0
- package/dist/services/index.d.ts +2 -0
- package/dist/services/index.js +8 -0
- package/dist/services/inworld/index.d.ts +2 -0
- package/dist/services/inworld/index.js +7 -0
- package/dist/services/keenable/index.d.ts +2 -0
- package/dist/services/keenable/index.js +7 -0
- package/dist/services/kokoro/index.d.ts +2 -0
- package/dist/services/kokoro/index.js +7 -0
- package/dist/services/llm_service.d.ts +1106 -0
- package/dist/services/llm_service.js +2231 -0
- package/dist/services/mem0/index.d.ts +2 -0
- package/dist/services/mem0/index.js +7 -0
- package/dist/services/meta/index.d.ts +2 -0
- package/dist/services/meta/index.js +7 -0
- package/dist/services/minimax/index.d.ts +2 -0
- package/dist/services/minimax/index.js +7 -0
- package/dist/services/mistral/index.d.ts +2 -0
- package/dist/services/mistral/index.js +7 -0
- package/dist/services/moondream/index.d.ts +2 -0
- package/dist/services/moondream/index.js +7 -0
- package/dist/services/moonshine/index.d.ts +2 -0
- package/dist/services/moonshine/index.js +7 -0
- package/dist/services/nebius/index.d.ts +2 -0
- package/dist/services/nebius/index.js +7 -0
- package/dist/services/neuphonic/index.d.ts +2 -0
- package/dist/services/neuphonic/index.js +7 -0
- package/dist/services/novita/index.d.ts +2 -0
- package/dist/services/novita/index.js +7 -0
- package/dist/services/nvidia/index.d.ts +2 -0
- package/dist/services/nvidia/index.js +11 -0
- package/dist/services/ollama/index.d.ts +2 -0
- package/dist/services/ollama/index.js +7 -0
- package/dist/services/openai/index.d.ts +2 -0
- package/dist/services/openai/index.js +7 -0
- package/dist/services/openclaw/index.d.ts +14 -0
- package/dist/services/openclaw/index.js +19 -0
- package/dist/services/openrouter/index.d.ts +2 -0
- package/dist/services/openrouter/index.js +7 -0
- package/dist/services/perplexity/index.d.ts +2 -0
- package/dist/services/perplexity/index.js +7 -0
- package/dist/services/piper/index.d.ts +2 -0
- package/dist/services/piper/index.js +7 -0
- package/dist/services/pocket_tts/index.d.ts +2 -0
- package/dist/services/pocket_tts/index.js +7 -0
- package/dist/services/qwen/index.d.ts +2 -0
- package/dist/services/qwen/index.js +7 -0
- package/dist/services/resembleai/index.d.ts +2 -0
- package/dist/services/resembleai/index.js +7 -0
- package/dist/services/rime/index.d.ts +2 -0
- package/dist/services/rime/index.js +7 -0
- package/dist/services/sambanova/index.d.ts +2 -0
- package/dist/services/sambanova/index.js +7 -0
- package/dist/services/sarvam/index.d.ts +2 -0
- package/dist/services/sarvam/index.js +7 -0
- package/dist/services/settings.d.ts +356 -0
- package/dist/services/settings.js +437 -0
- package/dist/services/simli/index.d.ts +2 -0
- package/dist/services/simli/index.js +7 -0
- package/dist/services/smallest/index.d.ts +2 -0
- package/dist/services/smallest/index.js +7 -0
- package/dist/services/soniox/index.d.ts +2 -0
- package/dist/services/soniox/index.js +7 -0
- package/dist/services/speechify/index.d.ts +2 -0
- package/dist/services/speechify/index.js +7 -0
- package/dist/services/speechmatics/index.d.ts +2 -0
- package/dist/services/speechmatics/index.js +7 -0
- package/dist/services/stt_latency.d.ts +62 -0
- package/dist/services/stt_latency.js +71 -0
- package/dist/services/stt_service.d.ts +601 -0
- package/dist/services/stt_service.js +1044 -0
- package/dist/services/tavus/index.d.ts +2 -0
- package/dist/services/tavus/index.js +7 -0
- package/dist/services/together/index.d.ts +2 -0
- package/dist/services/together/index.js +7 -0
- package/dist/services/tts_service.d.ts +909 -0
- package/dist/services/tts_service.js +2018 -0
- package/dist/services/ultravox/index.d.ts +2 -0
- package/dist/services/ultravox/index.js +9 -0
- package/dist/services/vision_service.d.ts +56 -0
- package/dist/services/vision_service.js +62 -0
- package/dist/services/websocket_service.d.ts +213 -0
- package/dist/services/websocket_service.js +654 -0
- package/dist/services/whisper/index.d.ts +2 -0
- package/dist/services/whisper/index.js +9 -0
- package/dist/services/xai/index.d.ts +2 -0
- package/dist/services/xai/index.js +9 -0
- package/dist/services/xtts/index.d.ts +2 -0
- package/dist/services/xtts/index.js +9 -0
- package/dist/tests/index.d.ts +2 -0
- package/dist/tests/index.js +8 -0
- package/dist/tests/utils.d.ts +123 -0
- package/dist/tests/utils.js +212 -0
- package/dist/transcriptions/index.d.ts +2 -0
- package/dist/transcriptions/index.js +8 -0
- package/dist/transcriptions/language.d.ts +376 -0
- package/dist/transcriptions/language.js +543 -0
- package/dist/transports/base_input.d.ts +149 -0
- package/dist/transports/base_input.js +338 -0
- package/dist/transports/base_output.d.ts +432 -0
- package/dist/transports/base_output.js +1231 -0
- package/dist/transports/base_transport.d.ts +223 -0
- package/dist/transports/base_transport.js +167 -0
- package/dist/transports/daily/index.d.ts +2 -0
- package/dist/transports/daily/index.js +9 -0
- package/dist/transports/heygen/index.d.ts +2 -0
- package/dist/transports/heygen/index.js +9 -0
- package/dist/transports/index.d.ts +2 -0
- package/dist/transports/index.js +8 -0
- package/dist/transports/lemonslice/index.d.ts +2 -0
- package/dist/transports/lemonslice/index.js +9 -0
- package/dist/transports/livekit/index.d.ts +2 -0
- package/dist/transports/livekit/index.js +9 -0
- package/dist/transports/local/index.d.ts +2 -0
- package/dist/transports/local/index.js +9 -0
- package/dist/transports/moq/index.d.ts +2 -0
- package/dist/transports/moq/index.js +9 -0
- package/dist/transports/smallwebrtc/index.d.ts +2 -0
- package/dist/transports/smallwebrtc/index.js +9 -0
- package/dist/transports/tavus/index.d.ts +2 -0
- package/dist/transports/tavus/index.js +9 -0
- package/dist/transports/video_in_sampler.d.ts +157 -0
- package/dist/transports/video_in_sampler.js +258 -0
- package/dist/transports/vonage/index.d.ts +2 -0
- package/dist/transports/vonage/index.js +9 -0
- package/dist/transports/websocket/index.d.ts +2 -0
- package/dist/transports/websocket/index.js +9 -0
- package/dist/transports/whatsapp/index.d.ts +2 -0
- package/dist/transports/whatsapp/index.js +9 -0
- package/dist/turns/eager_end_of_turn_mixin.d.ts +70 -0
- package/dist/turns/eager_end_of_turn_mixin.js +122 -0
- package/dist/turns/empty_user_turn.d.ts +55 -0
- package/dist/turns/empty_user_turn.js +53 -0
- package/dist/turns/index.d.ts +2 -0
- package/dist/turns/index.js +8 -0
- package/dist/turns/speculation_gate.d.ts +169 -0
- package/dist/turns/speculation_gate.js +294 -0
- package/dist/turns/types.d.ts +33 -0
- package/dist/turns/types.js +35 -0
- package/dist/turns/user_idle_controller.d.ts +94 -0
- package/dist/turns/user_idle_controller.js +187 -0
- package/dist/turns/user_mute/always_user_mute_strategy.d.ts +15 -0
- package/dist/turns/user_mute/always_user_mute_strategy.js +29 -0
- package/dist/turns/user_mute/base_user_mute_strategy.d.ts +39 -0
- package/dist/turns/user_mute/base_user_mute_strategy.js +45 -0
- package/dist/turns/user_mute/first_speech_user_mute_strategy.d.ts +28 -0
- package/dist/turns/user_mute/first_speech_user_mute_strategy.js +52 -0
- package/dist/turns/user_mute/function_call_user_mute_strategy.d.ts +25 -0
- package/dist/turns/user_mute/function_call_user_mute_strategy.js +50 -0
- package/dist/turns/user_mute/index.d.ts +6 -0
- package/dist/turns/user_mute/index.js +11 -0
- package/dist/turns/user_mute/mute_until_first_bot_complete_user_mute_strategy.d.ts +43 -0
- package/dist/turns/user_mute/mute_until_first_bot_complete_user_mute_strategy.js +76 -0
- package/dist/turns/user_start/base_user_turn_start_strategy.d.ts +135 -0
- package/dist/turns/user_start/base_user_turn_start_strategy.js +134 -0
- package/dist/turns/user_start/external_user_turn_start_strategy.d.ts +45 -0
- package/dist/turns/user_start/external_user_turn_start_strategy.js +64 -0
- package/dist/turns/user_start/index.d.ts +8 -0
- package/dist/turns/user_start/index.js +16 -0
- package/dist/turns/user_start/krisp_viva_ip_user_turn_start_strategy.d.ts +133 -0
- package/dist/turns/user_start/krisp_viva_ip_user_turn_start_strategy.js +243 -0
- package/dist/turns/user_start/min_words_user_turn_start_strategy.d.ts +60 -0
- package/dist/turns/user_start/min_words_user_turn_start_strategy.js +102 -0
- package/dist/turns/user_start/transcription_user_turn_start_strategy.d.ts +29 -0
- package/dist/turns/user_start/transcription_user_turn_start_strategy.js +43 -0
- package/dist/turns/user_start/vad_user_turn_start_strategy.d.ts +20 -0
- package/dist/turns/user_start/vad_user_turn_start_strategy.js +31 -0
- package/dist/turns/user_start/wake_phrase_user_turn_start_strategy.d.ts +141 -0
- package/dist/turns/user_start/wake_phrase_user_turn_start_strategy.js +238 -0
- package/dist/turns/user_stop/base_user_turn_stop_strategy.d.ts +174 -0
- package/dist/turns/user_stop/base_user_turn_stop_strategy.js +170 -0
- package/dist/turns/user_stop/deferred_user_turn_stop_strategy.d.ts +81 -0
- package/dist/turns/user_stop/deferred_user_turn_stop_strategy.js +107 -0
- package/dist/turns/user_stop/eager_match_policy.d.ts +36 -0
- package/dist/turns/user_stop/eager_match_policy.js +50 -0
- package/dist/turns/user_stop/eager_user_turn_stop_strategy.d.ts +84 -0
- package/dist/turns/user_stop/eager_user_turn_stop_strategy.js +154 -0
- package/dist/turns/user_stop/external_user_turn_completion_stop_strategy.d.ts +45 -0
- package/dist/turns/user_stop/external_user_turn_completion_stop_strategy.js +55 -0
- package/dist/turns/user_stop/external_user_turn_stop_strategy.d.ts +115 -0
- package/dist/turns/user_stop/external_user_turn_stop_strategy.js +205 -0
- package/dist/turns/user_stop/index.d.ts +10 -0
- package/dist/turns/user_stop/index.js +15 -0
- package/dist/turns/user_stop/llm_turn_completion_user_turn_stop_strategy.d.ts +44 -0
- package/dist/turns/user_stop/llm_turn_completion_user_turn_stop_strategy.js +59 -0
- package/dist/turns/user_stop/speech_timeout_user_turn_stop_strategy.d.ts +140 -0
- package/dist/turns/user_stop/speech_timeout_user_turn_stop_strategy.js +303 -0
- package/dist/turns/user_stop/turn_analyzer_user_turn_stop_strategy.d.ts +135 -0
- package/dist/turns/user_stop/turn_analyzer_user_turn_stop_strategy.js +332 -0
- package/dist/turns/user_turn_completion_mixin.d.ts +147 -0
- package/dist/turns/user_turn_completion_mixin.js +591 -0
- package/dist/turns/user_turn_controller.d.ts +134 -0
- package/dist/turns/user_turn_controller.js +335 -0
- package/dist/turns/user_turn_processor.d.ts +87 -0
- package/dist/turns/user_turn_processor.js +180 -0
- package/dist/turns/user_turn_strategies.d.ts +187 -0
- package/dist/turns/user_turn_strategies.js +222 -0
- package/dist/utils/async_tool_cancellation.d.ts +35 -0
- package/dist/utils/async_tool_cancellation.js +78 -0
- package/dist/utils/asyncio/index.d.ts +2 -0
- package/dist/utils/asyncio/index.js +7 -0
- package/dist/utils/asyncio/task_manager.d.ts +49 -0
- package/dist/utils/asyncio/task_manager.js +105 -0
- package/dist/utils/base_object.d.ts +76 -0
- package/dist/utils/base_object.js +154 -0
- package/dist/utils/context/aggregated_frame_sequencer.d.ts +491 -0
- package/dist/utils/context/aggregated_frame_sequencer.js +803 -0
- package/dist/utils/context/audio_context_manager.d.ts +131 -0
- package/dist/utils/context/audio_context_manager.js +233 -0
- package/dist/utils/context/index.d.ts +2 -0
- package/dist/utils/context/index.js +7 -0
- package/dist/utils/context/llm_context_summarization.d.ts +258 -0
- package/dist/utils/context/llm_context_summarization.js +511 -0
- package/dist/utils/context/text_segment_map.d.ts +611 -0
- package/dist/utils/context/text_segment_map.js +1132 -0
- package/dist/utils/context/word_completion_tracker.d.ts +273 -0
- package/dist/utils/context/word_completion_tracker.js +441 -0
- package/dist/utils/context/word_timestamp_tracker.d.ts +72 -0
- package/dist/utils/context/word_timestamp_tracker.js +98 -0
- package/dist/utils/deprecation.d.ts +28 -0
- package/dist/utils/deprecation.js +75 -0
- package/dist/utils/env.d.ts +38 -0
- package/dist/utils/env.js +62 -0
- package/dist/utils/errors.d.ts +75 -0
- package/dist/utils/errors.js +202 -0
- package/dist/utils/file_resolver.d.ts +126 -0
- package/dist/utils/file_resolver.js +314 -0
- package/dist/utils/file_storage.d.ts +99 -0
- package/dist/utils/file_storage.js +100 -0
- package/dist/utils/frame_queue.d.ts +36 -0
- package/dist/utils/frame_queue.js +66 -0
- package/dist/utils/http.d.ts +39 -0
- package/dist/utils/http.js +53 -0
- package/dist/utils/image.d.ts +15 -0
- package/dist/utils/image.js +30 -0
- package/dist/utils/index.d.ts +2 -0
- package/dist/utils/index.js +8 -0
- package/dist/utils/log_config.d.ts +34 -0
- package/dist/utils/log_config.js +76 -0
- package/dist/utils/logger.d.ts +40 -0
- package/dist/utils/logger.js +91 -0
- package/dist/utils/mixin.d.ts +14 -0
- package/dist/utils/mixin.js +63 -0
- package/dist/utils/network.d.ts +73 -0
- package/dist/utils/network.js +90 -0
- package/dist/utils/security/allowed_origins.d.ts +16 -0
- package/dist/utils/security/allowed_origins.js +34 -0
- package/dist/utils/security/index.d.ts +2 -0
- package/dist/utils/security/index.js +7 -0
- package/dist/utils/security/redaction.d.ts +27 -0
- package/dist/utils/security/redaction.js +123 -0
- package/dist/utils/security/ssrf.d.ts +79 -0
- package/dist/utils/security/ssrf.js +327 -0
- package/dist/utils/shared.d.ts +35 -0
- package/dist/utils/shared.js +136 -0
- package/dist/utils/startup.d.ts +17 -0
- package/dist/utils/startup.js +130 -0
- package/dist/utils/string.d.ts +116 -0
- package/dist/utils/string.js +460 -0
- package/dist/utils/sync/base_notifier.d.ts +25 -0
- package/dist/utils/sync/base_notifier.js +16 -0
- package/dist/utils/sync/event_notifier.d.ts +38 -0
- package/dist/utils/sync/event_notifier.js +51 -0
- package/dist/utils/sync/index.d.ts +2 -0
- package/dist/utils/sync/index.js +7 -0
- package/dist/utils/text/alnum_utils.d.ts +109 -0
- package/dist/utils/text/alnum_utils.js +262 -0
- package/dist/utils/text/base_text_aggregator.d.ts +146 -0
- package/dist/utils/text/base_text_aggregator.js +102 -0
- package/dist/utils/text/base_text_filter.d.ts +54 -0
- package/dist/utils/text/base_text_filter.js +50 -0
- package/dist/utils/text/index.d.ts +2 -0
- package/dist/utils/text/index.js +7 -0
- package/dist/utils/text/markdown_text_filter.d.ts +147 -0
- package/dist/utils/text/markdown_text_filter.js +522 -0
- package/dist/utils/text/markup_utils.d.ts +54 -0
- package/dist/utils/text/markup_utils.js +156 -0
- package/dist/utils/text/pattern_pair_aggregator.d.ts +251 -0
- package/dist/utils/text/pattern_pair_aggregator.js +478 -0
- package/dist/utils/text/phonemes.d.ts +60 -0
- package/dist/utils/text/phonemes.js +172 -0
- package/dist/utils/text/simple_text_aggregator.d.ts +135 -0
- package/dist/utils/text/simple_text_aggregator.js +254 -0
- package/dist/utils/text/skip_tags_aggregator.d.ts +85 -0
- package/dist/utils/text/skip_tags_aggregator.js +147 -0
- package/dist/utils/text/transforms/pronunciations.d.ts +65 -0
- package/dist/utils/text/transforms/pronunciations.js +111 -0
- package/dist/utils/text/word_timestamp_utils.d.ts +26 -0
- package/dist/utils/text/word_timestamp_utils.js +52 -0
- package/dist/utils/time.d.ts +13 -0
- package/dist/utils/time.js +31 -0
- package/dist/utils/tracing/index.d.ts +3 -0
- package/dist/utils/tracing/index.js +8 -0
- package/dist/utils/tracing/service_attributes.d.ts +198 -0
- package/dist/utils/tracing/service_attributes.js +434 -0
- package/dist/utils/tracing/service_decorators.d.ts +214 -0
- package/dist/utils/tracing/service_decorators.js +1662 -0
- package/dist/utils/tracing/setup.d.ts +95 -0
- package/dist/utils/tracing/setup.js +110 -0
- package/dist/utils/tracing/tracing_context.d.ts +55 -0
- package/dist/utils/tracing/tracing_context.js +97 -0
- package/dist/utils/tracing/turn_trace_observer.d.ts +90 -0
- package/dist/utils/tracing/turn_trace_observer.js +219 -0
- package/dist/utils/types.d.ts +80 -0
- package/dist/utils/types.js +108 -0
- package/dist/utils/utils.d.ts +5 -0
- package/dist/utils/utils.js +22 -0
- package/dist/utils/yaml.d.ts +64 -0
- package/dist/utils/yaml.js +41 -0
- package/dist/version.d.ts +3 -0
- package/dist/version.js +7 -0
- package/dist/workers/base_worker.d.ts +772 -0
- package/dist/workers/base_worker.js +1298 -0
- package/dist/workers/index.d.ts +2 -0
- package/dist/workers/index.js +8 -0
- package/dist/workers/llm/backend_llm_worker.d.ts +552 -0
- package/dist/workers/llm/backend_llm_worker.js +1055 -0
- package/dist/workers/llm/index.d.ts +6 -0
- package/dist/workers/llm/index.js +11 -0
- package/dist/workers/llm/llm_context_worker.d.ts +66 -0
- package/dist/workers/llm/llm_context_worker.js +72 -0
- package/dist/workers/llm/llm_worker.d.ts +180 -0
- package/dist/workers/llm/llm_worker.js +323 -0
- package/dist/workers/llm/tool_decorator.d.ts +68 -0
- package/dist/workers/llm/tool_decorator.js +60 -0
- package/dist/workers/runner.d.ts +268 -0
- package/dist/workers/runner.js +489 -0
- package/dist/workers/ui/index.d.ts +22 -0
- package/dist/workers/ui/index.js +27 -0
- package/dist/workers/ui/ui_event_decorator.d.ts +43 -0
- package/dist/workers/ui/ui_event_decorator.js +75 -0
- package/dist/workers/ui/ui_job_context.d.ts +12 -0
- package/dist/workers/ui/ui_job_context.js +17 -0
- package/dist/workers/ui/ui_prompts.d.ts +21 -0
- package/dist/workers/ui/ui_prompts.js +41 -0
- package/dist/workers/ui/ui_tools.d.ts +32 -0
- package/dist/workers/ui/ui_tools.js +112 -0
- package/dist/workers/ui/ui_worker.d.ts +571 -0
- package/dist/workers/ui/ui_worker.js +1156 -0
- package/package.json +46 -3
- package/README.md +0 -5
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
/** This module defines a controller for managing user turn lifecycle. */
|
|
2
|
+
import { Event, type Task } from "@pipedog/asyncio";
|
|
3
|
+
import { type Frame, InterimTranscriptionFrame, ProposedUserStartedSpeakingFrame, ProposedUserStoppedSpeakingFrame, TranscriptionFrame, UserStartedSpeakingFrame, UserStoppedSpeakingFrame, VADUserStartedSpeakingFrame, VADUserStoppedSpeakingFrame } from "../frames/frames.ts";
|
|
4
|
+
import { FrameDirection, type FrameProcessorSetup } from "../processors/frame_processor.ts";
|
|
5
|
+
import { BaseObject } from "../utils/base_object.ts";
|
|
6
|
+
import { type UserTurnSpeculation } from "./types.ts";
|
|
7
|
+
import type { BaseUserTurnStartStrategy, UserTurnStartedParams } from "./user_start/index.ts";
|
|
8
|
+
import { type BaseUserTurnStopStrategy, UserTurnStoppedParams } from "./user_stop/index.ts";
|
|
9
|
+
import type { UserTurnStrategies } from "./user_turn_strategies.ts";
|
|
10
|
+
export interface UserTurnControllerOptions {
|
|
11
|
+
/** Configured strategies for starting and stopping user turns. */
|
|
12
|
+
userTurnStrategies: UserTurnStrategies;
|
|
13
|
+
/**
|
|
14
|
+
* Timeout in seconds to automatically stop a user turn if no activity is
|
|
15
|
+
* detected.
|
|
16
|
+
*/
|
|
17
|
+
userTurnStopTimeout?: number;
|
|
18
|
+
}
|
|
19
|
+
type AnyStrategy = BaseUserTurnStartStrategy | BaseUserTurnStopStrategy;
|
|
20
|
+
/**
|
|
21
|
+
* Controller for managing user turn lifecycle.
|
|
22
|
+
*
|
|
23
|
+
* This class manages user turn state (active/inactive), handles start and
|
|
24
|
+
* stop strategies, and emits events when user turns begin, end, or timeout
|
|
25
|
+
* occurs.
|
|
26
|
+
*
|
|
27
|
+
* Event handlers available:
|
|
28
|
+
*
|
|
29
|
+
* - on_user_turn_started: Emitted when a user turn starts.
|
|
30
|
+
* - on_user_turn_inference_triggered: Emitted when enough signal exists to
|
|
31
|
+
* start LLM inference, carrying a `UserTurnSpeculation` when the turn it
|
|
32
|
+
* answers hasn't ended yet. Fires together with `on_user_turn_stopped` for
|
|
33
|
+
* most strategies; fires alone when a downstream strategy gates
|
|
34
|
+
* finalization on the LLM's verdict.
|
|
35
|
+
* - on_user_turn_stopped: Emitted when a user turn is semantically final.
|
|
36
|
+
* - on_user_turn_stop_timeout: Emitted if no stop strategy triggers before timeout.
|
|
37
|
+
* - on_push_frame: Emitted when a strategy wants to push a frame.
|
|
38
|
+
* - on_broadcast_frame: Emitted when a strategy wants to broadcast a frame.
|
|
39
|
+
*
|
|
40
|
+
* @example
|
|
41
|
+
* controller.eventHandler("on_user_turn_started", async (controller, strategy, params) => { ... });
|
|
42
|
+
* controller.eventHandler(
|
|
43
|
+
* "on_user_turn_inference_triggered",
|
|
44
|
+
* async (controller, strategy, speculation) => { ... },
|
|
45
|
+
* );
|
|
46
|
+
* controller.eventHandler("on_user_turn_stopped", async (controller, strategy, params) => { ... });
|
|
47
|
+
* controller.eventHandler("on_user_turn_stop_timeout", async (controller) => { ... });
|
|
48
|
+
* controller.eventHandler("on_push_frame", async (controller, frame, direction) => { ... });
|
|
49
|
+
* controller.eventHandler("on_broadcast_frame", async (controller, frameCls, options) => { ... });
|
|
50
|
+
*/
|
|
51
|
+
export declare class UserTurnController extends BaseObject {
|
|
52
|
+
protected _userTurnStrategies: UserTurnStrategies;
|
|
53
|
+
protected _userTurnStopTimeout: number;
|
|
54
|
+
protected _setup: FrameProcessorSetup | null;
|
|
55
|
+
protected _userSpeaking: boolean;
|
|
56
|
+
protected _userTurn: boolean;
|
|
57
|
+
protected _userTurnStopTimeoutEvent: Event;
|
|
58
|
+
protected _userTurnStopTimeoutTask: Task<unknown> | null;
|
|
59
|
+
constructor(options: UserTurnControllerOptions);
|
|
60
|
+
/** The currently active user turn strategies. */
|
|
61
|
+
get userTurnStrategies(): UserTurnStrategies;
|
|
62
|
+
/**
|
|
63
|
+
* Set up the controller.
|
|
64
|
+
*
|
|
65
|
+
* @param setup Configuration object containing setup parameters.
|
|
66
|
+
*/
|
|
67
|
+
setup(setup: FrameProcessorSetup): Promise<void>;
|
|
68
|
+
/**
|
|
69
|
+
* Start watching for a turn that stops without the user stopping.
|
|
70
|
+
*
|
|
71
|
+
* Paired with `stop()`.
|
|
72
|
+
*/
|
|
73
|
+
start(): Promise<void>;
|
|
74
|
+
/**
|
|
75
|
+
* Stop the turn stop timeout, leaving the strategies alone.
|
|
76
|
+
*
|
|
77
|
+
* Called at session end. The strategies may be shared, so cleaning them up
|
|
78
|
+
* waits for `cleanup()`.
|
|
79
|
+
*/
|
|
80
|
+
stop(): Promise<void>;
|
|
81
|
+
/** Cleanup the controller. */
|
|
82
|
+
cleanup(): Promise<void>;
|
|
83
|
+
/**
|
|
84
|
+
* Replace the current strategies with the given ones.
|
|
85
|
+
*
|
|
86
|
+
* @param strategies The new user turn strategies the controller should use.
|
|
87
|
+
*/
|
|
88
|
+
updateStrategies(strategies: UserTurnStrategies): Promise<void>;
|
|
89
|
+
/**
|
|
90
|
+
* Whether any active start strategy resolves proposed turn starts.
|
|
91
|
+
*
|
|
92
|
+
* A proposal is resolved once, so a caller holding this controller should
|
|
93
|
+
* stop forwarding `ProposedUserStartedSpeakingFrame` when this is true —
|
|
94
|
+
* passing it along would let a resolver further down the pipeline decide
|
|
95
|
+
* the same turn a second time.
|
|
96
|
+
*/
|
|
97
|
+
get resolvesProposedTurnStartFrames(): boolean;
|
|
98
|
+
/**
|
|
99
|
+
* Whether any active stop strategy resolves proposed turn stops.
|
|
100
|
+
*
|
|
101
|
+
* The end-of-turn counterpart to `resolvesProposedTurnStartFrames`.
|
|
102
|
+
*/
|
|
103
|
+
get resolvesProposedTurnStopFrames(): boolean;
|
|
104
|
+
/**
|
|
105
|
+
* Process an incoming frame to detect user turn start or stop.
|
|
106
|
+
*
|
|
107
|
+
* The frame is passed to the configured user turn strategies, which are
|
|
108
|
+
* responsible for deciding when a user turn starts or stops and emitting
|
|
109
|
+
* the corresponding events.
|
|
110
|
+
*
|
|
111
|
+
* @param frame The frame to be processed.
|
|
112
|
+
*/
|
|
113
|
+
processFrame(frame: Frame): Promise<void>;
|
|
114
|
+
protected _setupStrategies(): Promise<void>;
|
|
115
|
+
protected _cleanupStrategies(): Promise<void>;
|
|
116
|
+
protected _handleUserStartedSpeaking(_frame: UserStartedSpeakingFrame | ProposedUserStartedSpeakingFrame): Promise<void>;
|
|
117
|
+
protected _handleUserStoppedSpeaking(_frame: UserStoppedSpeakingFrame | ProposedUserStoppedSpeakingFrame): Promise<void>;
|
|
118
|
+
protected _handleVadUserStartedSpeaking(_frame: VADUserStartedSpeakingFrame): Promise<void>;
|
|
119
|
+
protected _handleVadUserStoppedSpeaking(_frame: VADUserStoppedSpeakingFrame): Promise<void>;
|
|
120
|
+
protected _handleTranscription(_frame: TranscriptionFrame | InterimTranscriptionFrame): Promise<void>;
|
|
121
|
+
protected _onPushFrame: (_strategy: AnyStrategy, frame: Frame, direction?: FrameDirection) => Promise<void>;
|
|
122
|
+
protected _onBroadcastFrame: (_strategy: AnyStrategy, frameCls: new (options: never) => Frame, options?: unknown) => Promise<void>;
|
|
123
|
+
protected _onUserTurnSpeculationCancelled: (_strategy: BaseUserTurnStopStrategy) => Promise<void>;
|
|
124
|
+
protected _onUserTurnStarted: (strategy: BaseUserTurnStartStrategy, params: UserTurnStartedParams) => Promise<void>;
|
|
125
|
+
protected _onUserTurnInferenceTriggered: (strategy: BaseUserTurnStopStrategy, speculation: UserTurnSpeculation | null) => Promise<void>;
|
|
126
|
+
protected _onUserTurnStopped: (strategy: BaseUserTurnStopStrategy, params: UserTurnStoppedParams) => Promise<void>;
|
|
127
|
+
protected _onResetAggregation: (strategy: BaseUserTurnStartStrategy) => Promise<void>;
|
|
128
|
+
protected _triggerUserTurnStart(strategy: BaseUserTurnStartStrategy | null, params: UserTurnStartedParams): Promise<void>;
|
|
129
|
+
protected _triggerUserTurnInferenceTriggered(strategy: BaseUserTurnStopStrategy | null, speculation?: UserTurnSpeculation | null): Promise<void>;
|
|
130
|
+
protected _triggerUserTurnStop(strategy: BaseUserTurnStopStrategy | null, params: UserTurnStoppedParams): Promise<void>;
|
|
131
|
+
protected _userTurnStopTimeoutTaskHandler(): Promise<void>;
|
|
132
|
+
}
|
|
133
|
+
export {};
|
|
134
|
+
//# sourceMappingURL=user_turn_controller.d.ts.map
|
|
@@ -0,0 +1,335 @@
|
|
|
1
|
+
// Copyright (c) 2024-2026, Daily
|
|
2
|
+
// Copyright (c) 2026, pipedog contributors
|
|
3
|
+
// SPDX-License-Identifier: BSD-2-Clause
|
|
4
|
+
//
|
|
5
|
+
// Ported from pipecat: src/pipecat/turns/user_turn_controller.py
|
|
6
|
+
/** This module defines a controller for managing user turn lifecycle. */
|
|
7
|
+
import { Event, TimeoutError, waitFor } from "@pipedog/asyncio";
|
|
8
|
+
import { InterimTranscriptionFrame, ProposedUserStartedSpeakingFrame, ProposedUserStoppedSpeakingFrame, TranscriptionFrame, UserStartedSpeakingFrame, UserStoppedSpeakingFrame, VADUserStartedSpeakingFrame, VADUserStoppedSpeakingFrame, } from "../frames/frames.js";
|
|
9
|
+
import { FrameDirection } from "../processors/frame_processor.js";
|
|
10
|
+
import { BaseObject } from "../utils/base_object.js";
|
|
11
|
+
import { ProcessFrameResult } from "./types.js";
|
|
12
|
+
import { UserTurnStoppedParams } from "./user_stop/index.js";
|
|
13
|
+
/**
|
|
14
|
+
* Controller for managing user turn lifecycle.
|
|
15
|
+
*
|
|
16
|
+
* This class manages user turn state (active/inactive), handles start and
|
|
17
|
+
* stop strategies, and emits events when user turns begin, end, or timeout
|
|
18
|
+
* occurs.
|
|
19
|
+
*
|
|
20
|
+
* Event handlers available:
|
|
21
|
+
*
|
|
22
|
+
* - on_user_turn_started: Emitted when a user turn starts.
|
|
23
|
+
* - on_user_turn_inference_triggered: Emitted when enough signal exists to
|
|
24
|
+
* start LLM inference, carrying a `UserTurnSpeculation` when the turn it
|
|
25
|
+
* answers hasn't ended yet. Fires together with `on_user_turn_stopped` for
|
|
26
|
+
* most strategies; fires alone when a downstream strategy gates
|
|
27
|
+
* finalization on the LLM's verdict.
|
|
28
|
+
* - on_user_turn_stopped: Emitted when a user turn is semantically final.
|
|
29
|
+
* - on_user_turn_stop_timeout: Emitted if no stop strategy triggers before timeout.
|
|
30
|
+
* - on_push_frame: Emitted when a strategy wants to push a frame.
|
|
31
|
+
* - on_broadcast_frame: Emitted when a strategy wants to broadcast a frame.
|
|
32
|
+
*
|
|
33
|
+
* @example
|
|
34
|
+
* controller.eventHandler("on_user_turn_started", async (controller, strategy, params) => { ... });
|
|
35
|
+
* controller.eventHandler(
|
|
36
|
+
* "on_user_turn_inference_triggered",
|
|
37
|
+
* async (controller, strategy, speculation) => { ... },
|
|
38
|
+
* );
|
|
39
|
+
* controller.eventHandler("on_user_turn_stopped", async (controller, strategy, params) => { ... });
|
|
40
|
+
* controller.eventHandler("on_user_turn_stop_timeout", async (controller) => { ... });
|
|
41
|
+
* controller.eventHandler("on_push_frame", async (controller, frame, direction) => { ... });
|
|
42
|
+
* controller.eventHandler("on_broadcast_frame", async (controller, frameCls, options) => { ... });
|
|
43
|
+
*/
|
|
44
|
+
export class UserTurnController extends BaseObject {
|
|
45
|
+
_userTurnStrategies;
|
|
46
|
+
_userTurnStopTimeout;
|
|
47
|
+
_setup = null;
|
|
48
|
+
_userSpeaking = false;
|
|
49
|
+
_userTurn = false;
|
|
50
|
+
_userTurnStopTimeoutEvent = new Event();
|
|
51
|
+
_userTurnStopTimeoutTask = null;
|
|
52
|
+
constructor(options) {
|
|
53
|
+
super();
|
|
54
|
+
this._userTurnStrategies = options.userTurnStrategies;
|
|
55
|
+
this._userTurnStopTimeout = options.userTurnStopTimeout ?? 5.0;
|
|
56
|
+
this._registerEventHandler("on_push_frame", true);
|
|
57
|
+
this._registerEventHandler("on_broadcast_frame", true);
|
|
58
|
+
this._registerEventHandler("on_user_turn_started", true);
|
|
59
|
+
this._registerEventHandler("on_user_turn_inference_triggered", true);
|
|
60
|
+
this._registerEventHandler("on_user_turn_speculation_cancelled", true);
|
|
61
|
+
this._registerEventHandler("on_user_turn_stopped", true);
|
|
62
|
+
this._registerEventHandler("on_user_turn_stop_timeout", true);
|
|
63
|
+
this._registerEventHandler("on_reset_aggregation", true);
|
|
64
|
+
}
|
|
65
|
+
/** The currently active user turn strategies. */
|
|
66
|
+
get userTurnStrategies() {
|
|
67
|
+
return this._userTurnStrategies;
|
|
68
|
+
}
|
|
69
|
+
/**
|
|
70
|
+
* Set up the controller.
|
|
71
|
+
*
|
|
72
|
+
* @param setup Configuration object containing setup parameters.
|
|
73
|
+
*/
|
|
74
|
+
async setup(setup) {
|
|
75
|
+
await super.setup(setup.taskManager);
|
|
76
|
+
// Kept so updateStrategies() can set up new strategies without the caller
|
|
77
|
+
// having to hand us the setup again.
|
|
78
|
+
this._setup = setup;
|
|
79
|
+
await this._setupStrategies();
|
|
80
|
+
}
|
|
81
|
+
/**
|
|
82
|
+
* Start watching for a turn that stops without the user stopping.
|
|
83
|
+
*
|
|
84
|
+
* Paired with `stop()`.
|
|
85
|
+
*/
|
|
86
|
+
async start() {
|
|
87
|
+
if (!this._userTurnStopTimeoutTask) {
|
|
88
|
+
this._userTurnStopTimeoutTask = this.createTask(() => this._userTurnStopTimeoutTaskHandler(), "_userTurnStopTimeoutTaskHandler");
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
/**
|
|
92
|
+
* Stop the turn stop timeout, leaving the strategies alone.
|
|
93
|
+
*
|
|
94
|
+
* Called at session end. The strategies may be shared, so cleaning them up
|
|
95
|
+
* waits for `cleanup()`.
|
|
96
|
+
*/
|
|
97
|
+
async stop() {
|
|
98
|
+
if (this._userTurnStopTimeoutTask) {
|
|
99
|
+
await this.cancelTask(this._userTurnStopTimeoutTask);
|
|
100
|
+
this._userTurnStopTimeoutTask = null;
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
/** Cleanup the controller. */
|
|
104
|
+
async cleanup() {
|
|
105
|
+
await super.cleanup();
|
|
106
|
+
await this.stop();
|
|
107
|
+
await this._cleanupStrategies();
|
|
108
|
+
}
|
|
109
|
+
/**
|
|
110
|
+
* Replace the current strategies with the given ones.
|
|
111
|
+
*
|
|
112
|
+
* @param strategies The new user turn strategies the controller should use.
|
|
113
|
+
*/
|
|
114
|
+
async updateStrategies(strategies) {
|
|
115
|
+
await this._cleanupStrategies();
|
|
116
|
+
this._userTurnStrategies = strategies;
|
|
117
|
+
await this._setupStrategies();
|
|
118
|
+
}
|
|
119
|
+
/**
|
|
120
|
+
* Whether any active start strategy resolves proposed turn starts.
|
|
121
|
+
*
|
|
122
|
+
* A proposal is resolved once, so a caller holding this controller should
|
|
123
|
+
* stop forwarding `ProposedUserStartedSpeakingFrame` when this is true —
|
|
124
|
+
* passing it along would let a resolver further down the pipeline decide
|
|
125
|
+
* the same turn a second time.
|
|
126
|
+
*/
|
|
127
|
+
get resolvesProposedTurnStartFrames() {
|
|
128
|
+
return (this._userTurnStrategies.start ?? []).some((s) => s.resolvesProposedTurnStartFrames);
|
|
129
|
+
}
|
|
130
|
+
/**
|
|
131
|
+
* Whether any active stop strategy resolves proposed turn stops.
|
|
132
|
+
*
|
|
133
|
+
* The end-of-turn counterpart to `resolvesProposedTurnStartFrames`.
|
|
134
|
+
*/
|
|
135
|
+
get resolvesProposedTurnStopFrames() {
|
|
136
|
+
return (this._userTurnStrategies.stop ?? []).some((s) => s.resolvesProposedTurnStopFrames);
|
|
137
|
+
}
|
|
138
|
+
/**
|
|
139
|
+
* Process an incoming frame to detect user turn start or stop.
|
|
140
|
+
*
|
|
141
|
+
* The frame is passed to the configured user turn strategies, which are
|
|
142
|
+
* responsible for deciding when a user turn starts or stops and emitting
|
|
143
|
+
* the corresponding events.
|
|
144
|
+
*
|
|
145
|
+
* @param frame The frame to be processed.
|
|
146
|
+
*/
|
|
147
|
+
async processFrame(frame) {
|
|
148
|
+
if (frame instanceof UserStartedSpeakingFrame ||
|
|
149
|
+
frame instanceof ProposedUserStartedSpeakingFrame) {
|
|
150
|
+
await this._handleUserStartedSpeaking(frame);
|
|
151
|
+
}
|
|
152
|
+
else if (frame instanceof UserStoppedSpeakingFrame ||
|
|
153
|
+
frame instanceof ProposedUserStoppedSpeakingFrame) {
|
|
154
|
+
await this._handleUserStoppedSpeaking(frame);
|
|
155
|
+
}
|
|
156
|
+
else if (frame instanceof VADUserStartedSpeakingFrame) {
|
|
157
|
+
await this._handleVadUserStartedSpeaking(frame);
|
|
158
|
+
}
|
|
159
|
+
else if (frame instanceof VADUserStoppedSpeakingFrame) {
|
|
160
|
+
await this._handleVadUserStoppedSpeaking(frame);
|
|
161
|
+
}
|
|
162
|
+
else if (frame instanceof TranscriptionFrame || frame instanceof InterimTranscriptionFrame) {
|
|
163
|
+
await this._handleTranscription(frame);
|
|
164
|
+
}
|
|
165
|
+
for (const strategy of this._userTurnStrategies.start ?? []) {
|
|
166
|
+
const result = await strategy.processFrame(frame);
|
|
167
|
+
if (result === ProcessFrameResult.STOP)
|
|
168
|
+
break;
|
|
169
|
+
}
|
|
170
|
+
for (const strategy of this._userTurnStrategies.stop ?? []) {
|
|
171
|
+
const result = await strategy.processFrame(frame);
|
|
172
|
+
if (result === ProcessFrameResult.STOP)
|
|
173
|
+
break;
|
|
174
|
+
}
|
|
175
|
+
}
|
|
176
|
+
async _setupStrategies() {
|
|
177
|
+
if (!this._setup) {
|
|
178
|
+
throw new Error(`${this} was not properly set up`);
|
|
179
|
+
}
|
|
180
|
+
for (const s of this._userTurnStrategies.start ?? []) {
|
|
181
|
+
await s.setup(this._setup);
|
|
182
|
+
s.addEventHandler("on_push_frame", this._onPushFrame);
|
|
183
|
+
s.addEventHandler("on_broadcast_frame", this._onBroadcastFrame);
|
|
184
|
+
s.addEventHandler("on_user_turn_started", this._onUserTurnStarted);
|
|
185
|
+
s.addEventHandler("on_reset_aggregation", this._onResetAggregation);
|
|
186
|
+
}
|
|
187
|
+
for (const s of this._userTurnStrategies.stop ?? []) {
|
|
188
|
+
await s.setup(this._setup);
|
|
189
|
+
s.addEventHandler("on_push_frame", this._onPushFrame);
|
|
190
|
+
s.addEventHandler("on_broadcast_frame", this._onBroadcastFrame);
|
|
191
|
+
s.addEventHandler("on_user_turn_inference_triggered", this._onUserTurnInferenceTriggered);
|
|
192
|
+
s.addEventHandler("on_user_turn_speculation_cancelled", this._onUserTurnSpeculationCancelled);
|
|
193
|
+
s.addEventHandler("on_user_turn_stopped", this._onUserTurnStopped);
|
|
194
|
+
}
|
|
195
|
+
}
|
|
196
|
+
async _cleanupStrategies() {
|
|
197
|
+
// Remove the handlers _setupStrategies added (symmetric), so re-applying
|
|
198
|
+
// strategies via updateStrategies — possibly reusing the same strategy
|
|
199
|
+
// instances — doesn't accumulate duplicate handler registrations.
|
|
200
|
+
for (const s of this._userTurnStrategies.start ?? []) {
|
|
201
|
+
await s.cleanup();
|
|
202
|
+
s.removeEventHandler("on_push_frame", this._onPushFrame);
|
|
203
|
+
s.removeEventHandler("on_broadcast_frame", this._onBroadcastFrame);
|
|
204
|
+
s.removeEventHandler("on_user_turn_started", this._onUserTurnStarted);
|
|
205
|
+
s.removeEventHandler("on_reset_aggregation", this._onResetAggregation);
|
|
206
|
+
}
|
|
207
|
+
for (const s of this._userTurnStrategies.stop ?? []) {
|
|
208
|
+
await s.cleanup();
|
|
209
|
+
s.removeEventHandler("on_push_frame", this._onPushFrame);
|
|
210
|
+
s.removeEventHandler("on_broadcast_frame", this._onBroadcastFrame);
|
|
211
|
+
s.removeEventHandler("on_user_turn_inference_triggered", this._onUserTurnInferenceTriggered);
|
|
212
|
+
s.removeEventHandler("on_user_turn_speculation_cancelled", this._onUserTurnSpeculationCancelled);
|
|
213
|
+
s.removeEventHandler("on_user_turn_stopped", this._onUserTurnStopped);
|
|
214
|
+
}
|
|
215
|
+
}
|
|
216
|
+
async _handleUserStartedSpeaking(_frame) {
|
|
217
|
+
this._userSpeaking = true;
|
|
218
|
+
// The user started talking, let's reset the user turn timeout.
|
|
219
|
+
this._userTurnStopTimeoutEvent.set();
|
|
220
|
+
}
|
|
221
|
+
async _handleUserStoppedSpeaking(_frame) {
|
|
222
|
+
this._userSpeaking = false;
|
|
223
|
+
// The user stopped talking, let's reset the user turn timeout.
|
|
224
|
+
this._userTurnStopTimeoutEvent.set();
|
|
225
|
+
}
|
|
226
|
+
async _handleVadUserStartedSpeaking(_frame) {
|
|
227
|
+
this._userSpeaking = true;
|
|
228
|
+
// The user started talking, let's reset the user turn timeout.
|
|
229
|
+
this._userTurnStopTimeoutEvent.set();
|
|
230
|
+
}
|
|
231
|
+
async _handleVadUserStoppedSpeaking(_frame) {
|
|
232
|
+
this._userSpeaking = false;
|
|
233
|
+
// The user stopped talking, let's reset the user turn timeout.
|
|
234
|
+
this._userTurnStopTimeoutEvent.set();
|
|
235
|
+
}
|
|
236
|
+
async _handleTranscription(_frame) {
|
|
237
|
+
// We have received a transcription, let's reset the user turn timeout.
|
|
238
|
+
this._userTurnStopTimeoutEvent.set();
|
|
239
|
+
}
|
|
240
|
+
// Strategy event handlers are arrow functions so the same reference can be
|
|
241
|
+
// added in _setupStrategies() and removed in _cleanupStrategies().
|
|
242
|
+
_onPushFrame = async (_strategy, frame, direction = FrameDirection.DOWNSTREAM) => {
|
|
243
|
+
await this._callEventHandler("on_push_frame", frame, direction);
|
|
244
|
+
};
|
|
245
|
+
_onBroadcastFrame = async (_strategy, frameCls, options) => {
|
|
246
|
+
await this._callEventHandler("on_broadcast_frame", frameCls, options);
|
|
247
|
+
};
|
|
248
|
+
_onUserTurnSpeculationCancelled = async (_strategy) => {
|
|
249
|
+
await this._callEventHandler("on_user_turn_speculation_cancelled");
|
|
250
|
+
};
|
|
251
|
+
_onUserTurnStarted = async (strategy, params) => {
|
|
252
|
+
await this._triggerUserTurnStart(strategy, params);
|
|
253
|
+
};
|
|
254
|
+
_onUserTurnInferenceTriggered = async (strategy, speculation) => {
|
|
255
|
+
await this._triggerUserTurnInferenceTriggered(strategy, speculation);
|
|
256
|
+
};
|
|
257
|
+
_onUserTurnStopped = async (strategy, params) => {
|
|
258
|
+
await this._triggerUserTurnStop(strategy, params);
|
|
259
|
+
};
|
|
260
|
+
_onResetAggregation = async (strategy) => {
|
|
261
|
+
await this._callEventHandler("on_reset_aggregation", strategy);
|
|
262
|
+
};
|
|
263
|
+
async _triggerUserTurnStart(strategy, params) {
|
|
264
|
+
// Prevent two consecutive user turn starts.
|
|
265
|
+
if (this._userTurn)
|
|
266
|
+
return;
|
|
267
|
+
this._userTurn = true;
|
|
268
|
+
this._userTurnStopTimeoutEvent.set();
|
|
269
|
+
// Notify every strategy that the turn has started. Start strategies ready
|
|
270
|
+
// themselves for the next detection; stop strategies arm to detect this
|
|
271
|
+
// turn's end. A strategy resets whatever per-turn state it keeps inside
|
|
272
|
+
// its own handleUserTurnStarted.
|
|
273
|
+
for (const s of this._userTurnStrategies.start ?? []) {
|
|
274
|
+
await s.handleUserTurnStarted();
|
|
275
|
+
}
|
|
276
|
+
for (const s of this._userTurnStrategies.stop ?? []) {
|
|
277
|
+
await s.handleUserTurnStarted();
|
|
278
|
+
}
|
|
279
|
+
await this._callEventHandler("on_user_turn_started", strategy, params);
|
|
280
|
+
}
|
|
281
|
+
async _triggerUserTurnInferenceTriggered(strategy, speculation = null) {
|
|
282
|
+
// Inference-triggered fires only while a turn is active. The turn remains
|
|
283
|
+
// active afterward — only `on_user_turn_stopped` flips state.
|
|
284
|
+
if (!this._userTurn)
|
|
285
|
+
return;
|
|
286
|
+
// Re-arm the stop watchdog so a stuck turn (inference fired but
|
|
287
|
+
// finalization never arrives) still times out and finalizes.
|
|
288
|
+
this._userTurnStopTimeoutEvent.set();
|
|
289
|
+
await this._callEventHandler("on_user_turn_inference_triggered", strategy, speculation);
|
|
290
|
+
}
|
|
291
|
+
async _triggerUserTurnStop(strategy, params) {
|
|
292
|
+
// Prevent two consecutive user turn stops.
|
|
293
|
+
if (!this._userTurn)
|
|
294
|
+
return;
|
|
295
|
+
// Never finalize while the user is audibly speaking. A stop strategy can
|
|
296
|
+
// finalize on a latent signal (e.g. an LLM ● that resolves after the user
|
|
297
|
+
// resumed), which is stale by the time it arrives. Keep the turn open so
|
|
298
|
+
// the next inference re-evaluates; the watchdog still finalizes if the
|
|
299
|
+
// user then falls silent. Detector strategies only finalize once the user
|
|
300
|
+
// has stopped, so this is a no-op for them.
|
|
301
|
+
if (this._userSpeaking)
|
|
302
|
+
return;
|
|
303
|
+
this._userTurn = false;
|
|
304
|
+
this._userTurnStopTimeoutEvent.set();
|
|
305
|
+
// Notify every strategy that the turn has ended. Stop strategies reset
|
|
306
|
+
// (and, e.g., drop a turn analyzer's buffered speech that must not survive
|
|
307
|
+
// an externally-ended turn). Start strategies get the same callback, but
|
|
308
|
+
// it's a no-op by default: their reset is turn-start semantic, so
|
|
309
|
+
// resetting them here would be wrong.
|
|
310
|
+
for (const s of this._userTurnStrategies.start ?? []) {
|
|
311
|
+
await s.handleUserTurnStopped();
|
|
312
|
+
}
|
|
313
|
+
for (const s of this._userTurnStrategies.stop ?? []) {
|
|
314
|
+
await s.handleUserTurnStopped();
|
|
315
|
+
}
|
|
316
|
+
await this._callEventHandler("on_user_turn_stopped", strategy, params);
|
|
317
|
+
}
|
|
318
|
+
async _userTurnStopTimeoutTaskHandler() {
|
|
319
|
+
while (true) {
|
|
320
|
+
try {
|
|
321
|
+
await waitFor(() => this._userTurnStopTimeoutEvent.wait(), this._userTurnStopTimeout);
|
|
322
|
+
this._userTurnStopTimeoutEvent.clear();
|
|
323
|
+
}
|
|
324
|
+
catch (e) {
|
|
325
|
+
if (!(e instanceof TimeoutError))
|
|
326
|
+
throw e;
|
|
327
|
+
if (this._userTurn && !this._userSpeaking) {
|
|
328
|
+
await this._callEventHandler("on_user_turn_stop_timeout");
|
|
329
|
+
await this._triggerUserTurnStop(null, new UserTurnStoppedParams({ enableUserSpeakingFrames: true }));
|
|
330
|
+
}
|
|
331
|
+
}
|
|
332
|
+
}
|
|
333
|
+
}
|
|
334
|
+
}
|
|
335
|
+
//# sourceMappingURL=user_turn_controller.js.map
|
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
/** Frame processor for managing the user turn lifecycle. */
|
|
2
|
+
import { CancelFrame, EndFrame, type Frame } from "../frames/frames.ts";
|
|
3
|
+
import { FrameDirection, FrameProcessor, type FrameProcessorOptions, type FrameProcessorSetup } from "../processors/frame_processor.ts";
|
|
4
|
+
import type { UserTurnSpeculation } from "./types.ts";
|
|
5
|
+
import { UserIdleController } from "./user_idle_controller.ts";
|
|
6
|
+
import type { BaseUserTurnStartStrategy, UserTurnStartedParams } from "./user_start/index.ts";
|
|
7
|
+
import type { BaseUserTurnStopStrategy, UserTurnStoppedParams } from "./user_stop/index.ts";
|
|
8
|
+
import { UserTurnController } from "./user_turn_controller.ts";
|
|
9
|
+
import { UserTurnStrategies } from "./user_turn_strategies.ts";
|
|
10
|
+
export interface UserTurnProcessorOptions extends FrameProcessorOptions {
|
|
11
|
+
/** Configured strategies for starting and stopping user turns. */
|
|
12
|
+
userTurnStrategies?: UserTurnStrategies | null;
|
|
13
|
+
/**
|
|
14
|
+
* Timeout in seconds to automatically stop a user turn if no activity is
|
|
15
|
+
* detected.
|
|
16
|
+
*/
|
|
17
|
+
userTurnStopTimeout?: number;
|
|
18
|
+
/**
|
|
19
|
+
* Timeout in seconds for detecting user idle state. The processor will emit
|
|
20
|
+
* an `on_user_turn_idle` event when the user has been idle (not speaking)
|
|
21
|
+
* for this duration. Set to 0 to disable idle detection.
|
|
22
|
+
*/
|
|
23
|
+
userIdleTimeout?: number;
|
|
24
|
+
}
|
|
25
|
+
/**
|
|
26
|
+
* Frame processor for managing the user turn lifecycle.
|
|
27
|
+
*
|
|
28
|
+
* This processor uses a turn controller to determine when a user turn starts
|
|
29
|
+
* or stops. The actual frames emitted (e.g., UserStartedSpeakingFrame,
|
|
30
|
+
* UserStoppedSpeakingFrame) or interruptions depend on the configured
|
|
31
|
+
* strategies.
|
|
32
|
+
*
|
|
33
|
+
* Event handlers available:
|
|
34
|
+
*
|
|
35
|
+
* - on_user_turn_started: Emitted when a user turn starts.
|
|
36
|
+
* - on_user_turn_inference_triggered: Emitted when enough signal exists to
|
|
37
|
+
* start LLM inference. Fires together with `on_user_turn_stopped` for most
|
|
38
|
+
* strategies; fires alone when a downstream strategy gates finalization on
|
|
39
|
+
* the LLM's verdict.
|
|
40
|
+
* - on_user_turn_stopped: Emitted when a user turn is semantically final.
|
|
41
|
+
* - on_user_turn_stop_timeout: Emitted if no stop strategy triggers before timeout.
|
|
42
|
+
* - on_user_turn_idle: Emitted when the user has been idle for the configured timeout.
|
|
43
|
+
*
|
|
44
|
+
* @example
|
|
45
|
+
* processor.eventHandler("on_user_turn_started", async (processor, strategy) => { ... });
|
|
46
|
+
* processor.eventHandler("on_user_turn_inference_triggered", async (processor, strategy) => { ... });
|
|
47
|
+
* processor.eventHandler("on_user_turn_stopped", async (processor, strategy) => { ... });
|
|
48
|
+
* processor.eventHandler("on_user_turn_stop_timeout", async (processor) => { ... });
|
|
49
|
+
* processor.eventHandler("on_user_turn_idle", async (processor) => { ... });
|
|
50
|
+
*/
|
|
51
|
+
export declare class UserTurnProcessor extends FrameProcessor {
|
|
52
|
+
protected _userTurnController: UserTurnController;
|
|
53
|
+
protected _userIdleController: UserIdleController;
|
|
54
|
+
constructor(options?: UserTurnProcessorOptions);
|
|
55
|
+
/**
|
|
56
|
+
* Set up the processor.
|
|
57
|
+
*
|
|
58
|
+
* @param setup Configuration object containing setup parameters.
|
|
59
|
+
*/
|
|
60
|
+
setup(setup: FrameProcessorSetup): Promise<void>;
|
|
61
|
+
/** Clean up processor resources. */
|
|
62
|
+
cleanup(): Promise<void>;
|
|
63
|
+
/**
|
|
64
|
+
* Process an incoming frame to detect user turn start or stop.
|
|
65
|
+
*
|
|
66
|
+
* The frame is passed to the user turn controller which is responsible for
|
|
67
|
+
* deciding when a user turn starts or stops and emitting the corresponding
|
|
68
|
+
* events.
|
|
69
|
+
*
|
|
70
|
+
* @param frame The frame to be processed.
|
|
71
|
+
* @param direction The direction of the incoming frame.
|
|
72
|
+
*/
|
|
73
|
+
processFrame(frame: Frame, direction: FrameDirection): Promise<void>;
|
|
74
|
+
protected _stop(_frame: EndFrame): Promise<void>;
|
|
75
|
+
protected _cancel(_frame: CancelFrame): Promise<void>;
|
|
76
|
+
protected _stopControllers(): Promise<void>;
|
|
77
|
+
protected _cleanup(): Promise<void>;
|
|
78
|
+
protected _onPushFrame(frame: Frame, direction?: FrameDirection): Promise<void>;
|
|
79
|
+
protected _onBroadcastFrame(frameCls: new (options: unknown) => Frame, options?: unknown): Promise<void>;
|
|
80
|
+
protected _onUserTurnSpeculationCancelled(_controller: UserTurnController): Promise<void>;
|
|
81
|
+
protected _onUserTurnStarted(_controller: UserTurnController, strategy: BaseUserTurnStartStrategy, params: UserTurnStartedParams): Promise<void>;
|
|
82
|
+
protected _onUserTurnStopped(_controller: UserTurnController, strategy: BaseUserTurnStopStrategy, params: UserTurnStoppedParams): Promise<void>;
|
|
83
|
+
protected _onUserTurnStopTimeout(_controller: UserTurnController): Promise<void>;
|
|
84
|
+
protected _onUserTurnIdle(_controller: UserIdleController): Promise<void>;
|
|
85
|
+
protected _onUserTurnInferenceTriggered(_controller: UserTurnController, strategy: BaseUserTurnStopStrategy, _speculation: UserTurnSpeculation | null): Promise<void>;
|
|
86
|
+
}
|
|
87
|
+
//# sourceMappingURL=user_turn_processor.d.ts.map
|