@pipedog/core 0.0.0 → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +25 -0
- package/NOTICE +9 -0
- package/dist/adapters/base_llm_adapter.d.ts +266 -0
- package/dist/adapters/base_llm_adapter.js +489 -0
- package/dist/adapters/index.d.ts +2 -0
- package/dist/adapters/index.js +8 -0
- package/dist/adapters/schemas/direct_function.d.ts +202 -0
- package/dist/adapters/schemas/direct_function.js +302 -0
- package/dist/adapters/schemas/function_schema.d.ts +67 -0
- package/dist/adapters/schemas/function_schema.js +72 -0
- package/dist/adapters/schemas/index.d.ts +2 -0
- package/dist/adapters/schemas/index.js +8 -0
- package/dist/adapters/schemas/tools_schema.d.ts +57 -0
- package/dist/adapters/schemas/tools_schema.js +79 -0
- package/dist/adapters/services/index.d.ts +2 -0
- package/dist/adapters/services/index.js +8 -0
- package/dist/adapters/services/open_ai_adapter.d.ts +147 -0
- package/dist/adapters/services/open_ai_adapter.js +250 -0
- package/dist/audio/dtmf/dtmf_data.d.ts +3 -0
- package/dist/audio/dtmf/dtmf_data.js +22 -0
- package/dist/audio/dtmf/index.d.ts +2 -0
- package/dist/audio/dtmf/index.js +8 -0
- package/dist/audio/dtmf/types.d.ts +31 -0
- package/dist/audio/dtmf/types.js +35 -0
- package/dist/audio/dtmf/utils.d.ts +13 -0
- package/dist/audio/dtmf/utils.js +62 -0
- package/dist/audio/filters/base_audio_filter.d.ts +51 -0
- package/dist/audio/filters/base_audio_filter.js +17 -0
- package/dist/audio/filters/index.d.ts +2 -0
- package/dist/audio/filters/index.js +8 -0
- package/dist/audio/index.d.ts +2 -0
- package/dist/audio/index.js +8 -0
- package/dist/audio/krisp_instance.d.ts +95 -0
- package/dist/audio/krisp_instance.js +229 -0
- package/dist/audio/mixers/base_audio_mixer.d.ts +56 -0
- package/dist/audio/mixers/base_audio_mixer.js +16 -0
- package/dist/audio/mixers/index.d.ts +2 -0
- package/dist/audio/mixers/index.js +8 -0
- package/dist/audio/resamplers/base_audio_resampler.d.ts +46 -0
- package/dist/audio/resamplers/base_audio_resampler.js +35 -0
- package/dist/audio/resamplers/index.d.ts +2 -0
- package/dist/audio/resamplers/index.js +8 -0
- package/dist/audio/resamplers/resampy_resampler.d.ts +10 -0
- package/dist/audio/resamplers/resampy_resampler.js +15 -0
- package/dist/audio/resamplers/soxr_resampler.d.ts +54 -0
- package/dist/audio/resamplers/soxr_resampler.js +69 -0
- package/dist/audio/resamplers/soxr_stream_resampler.d.ts +75 -0
- package/dist/audio/resamplers/soxr_stream_resampler.js +146 -0
- package/dist/audio/turn/base_turn_analyzer.d.ts +84 -0
- package/dist/audio/turn/base_turn_analyzer.js +61 -0
- package/dist/audio/turn/index.d.ts +2 -0
- package/dist/audio/turn/index.js +8 -0
- package/dist/audio/turn/smart_turn/_whisper_features.d.ts +30 -0
- package/dist/audio/turn/smart_turn/_whisper_features.js +225 -0
- package/dist/audio/turn/smart_turn/base_smart_turn.d.ts +94 -0
- package/dist/audio/turn/smart_turn/base_smart_turn.js +239 -0
- package/dist/audio/turn/smart_turn/local_smart_turn_v3.d.ts +76 -0
- package/dist/audio/turn/smart_turn/local_smart_turn_v3.js +143 -0
- package/dist/audio/utils.d.ts +224 -0
- package/dist/audio/utils.js +634 -0
- package/dist/audio/vad/data/index.d.ts +2 -0
- package/dist/audio/vad/data/index.js +8 -0
- package/dist/audio/vad/index.d.ts +2 -0
- package/dist/audio/vad/index.js +8 -0
- package/dist/audio/vad/vad_analyzer.d.ts +126 -0
- package/dist/audio/vad/vad_analyzer.js +211 -0
- package/dist/audio/vad/vad_controller.d.ts +143 -0
- package/dist/audio/vad/vad_controller.js +228 -0
- package/dist/audio/volume.d.ts +38 -0
- package/dist/audio/volume.js +75 -0
- package/dist/bus/adapters/base.d.ts +35 -0
- package/dist/bus/adapters/base.js +19 -0
- package/dist/bus/adapters/index.d.ts +10 -0
- package/dist/bus/adapters/index.js +15 -0
- package/dist/bus/adapters/llm_context_adapter.d.ts +36 -0
- package/dist/bus/adapters/llm_context_adapter.js +82 -0
- package/dist/bus/adapters/tools_schema_adapter.d.ts +25 -0
- package/dist/bus/adapters/tools_schema_adapter.js +70 -0
- package/dist/bus/bridge_processor.d.ts +111 -0
- package/dist/bus/bridge_processor.js +166 -0
- package/dist/bus/bus.d.ts +110 -0
- package/dist/bus/bus.js +200 -0
- package/dist/bus/index.d.ts +22 -0
- package/dist/bus/index.js +27 -0
- package/dist/bus/local/async_queue.d.ts +12 -0
- package/dist/bus/local/async_queue.js +21 -0
- package/dist/bus/local/index.d.ts +3 -0
- package/dist/bus/local/index.js +8 -0
- package/dist/bus/messages.d.ts +378 -0
- package/dist/bus/messages.js +357 -0
- package/dist/bus/network/index.d.ts +16 -0
- package/dist/bus/network/index.js +21 -0
- package/dist/bus/queue.d.ts +27 -0
- package/dist/bus/queue.js +51 -0
- package/dist/bus/serializers/base.d.ts +25 -0
- package/dist/bus/serializers/base.js +14 -0
- package/dist/bus/serializers/index.d.ts +9 -0
- package/dist/bus/serializers/index.js +14 -0
- package/dist/bus/serializers/json.d.ts +77 -0
- package/dist/bus/serializers/json.js +367 -0
- package/dist/bus/subscriber.d.ts +46 -0
- package/dist/bus/subscriber.js +27 -0
- package/dist/bus/ui/index.d.ts +3 -0
- package/dist/bus/ui/index.js +8 -0
- package/dist/bus/ui/messages.d.ts +186 -0
- package/dist/bus/ui/messages.js +171 -0
- package/dist/classifiers/base_classifier.d.ts +263 -0
- package/dist/classifiers/base_classifier.js +346 -0
- package/dist/classifiers/cloudflare/clef/classifier.d.ts +70 -0
- package/dist/classifiers/cloudflare/clef/classifier.js +204 -0
- package/dist/classifiers/cloudflare/clef/client.d.ts +104 -0
- package/dist/classifiers/cloudflare/clef/client.js +234 -0
- package/dist/classifiers/cloudflare/clef/index.d.ts +2 -0
- package/dist/classifiers/cloudflare/clef/index.js +8 -0
- package/dist/classifiers/cloudflare/index.d.ts +2 -0
- package/dist/classifiers/cloudflare/index.js +8 -0
- package/dist/classifiers/http_client.d.ts +70 -0
- package/dist/classifiers/http_client.js +161 -0
- package/dist/classifiers/index.d.ts +2 -0
- package/dist/classifiers/index.js +8 -0
- package/dist/classifiers/jev/classifier.d.ts +8 -0
- package/dist/classifiers/jev/classifier.js +13 -0
- package/dist/classifiers/jev/client.d.ts +8 -0
- package/dist/classifiers/jev/client.js +13 -0
- package/dist/classifiers/jev/index.d.ts +2 -0
- package/dist/classifiers/jev/index.js +8 -0
- package/dist/classifiers/llm/classifier.d.ts +72 -0
- package/dist/classifiers/llm/classifier.js +304 -0
- package/dist/classifiers/llm/index.d.ts +2 -0
- package/dist/classifiers/llm/index.js +8 -0
- package/dist/classifiers/openai/decisions/classifier.d.ts +77 -0
- package/dist/classifiers/openai/decisions/classifier.js +241 -0
- package/dist/classifiers/openai/decisions/client.d.ts +108 -0
- package/dist/classifiers/openai/decisions/client.js +234 -0
- package/dist/classifiers/openai/decisions/index.d.ts +2 -0
- package/dist/classifiers/openai/decisions/index.js +8 -0
- package/dist/classifiers/openai/index.d.ts +2 -0
- package/dist/classifiers/openai/index.js +8 -0
- package/dist/classifiers/typesafe/index.d.ts +2 -0
- package/dist/classifiers/typesafe/index.js +8 -0
- package/dist/classifiers/typesafe/jev/classifier.d.ts +70 -0
- package/dist/classifiers/typesafe/jev/classifier.js +220 -0
- package/dist/classifiers/typesafe/jev/client.d.ts +105 -0
- package/dist/classifiers/typesafe/jev/client.js +233 -0
- package/dist/classifiers/typesafe/jev/index.d.ts +2 -0
- package/dist/classifiers/typesafe/jev/index.js +8 -0
- package/dist/clocks/base_clock.d.ts +13 -0
- package/dist/clocks/base_clock.js +14 -0
- package/dist/clocks/index.d.ts +2 -0
- package/dist/clocks/index.js +8 -0
- package/dist/clocks/system_clock.d.ts +15 -0
- package/dist/clocks/system_clock.js +25 -0
- package/dist/frames/frames.d.ts +2533 -0
- package/dist/frames/frames.js +2485 -0
- package/dist/frames/index.d.ts +2 -0
- package/dist/frames/index.js +8 -0
- package/dist/frames/protobufs/frames_pb2.d.ts +229 -0
- package/dist/frames/protobufs/frames_pb2.js +466 -0
- package/dist/index.d.ts +11 -0
- package/dist/index.js +56 -0
- package/dist/metrics/index.d.ts +2 -0
- package/dist/metrics/index.js +8 -0
- package/dist/metrics/metrics.d.ts +276 -0
- package/dist/metrics/metrics.js +254 -0
- package/dist/observers/base_observer.d.ts +173 -0
- package/dist/observers/base_observer.js +151 -0
- package/dist/observers/error_observer.d.ts +75 -0
- package/dist/observers/error_observer.js +87 -0
- package/dist/observers/function_call_observer.d.ts +157 -0
- package/dist/observers/function_call_observer.js +222 -0
- package/dist/observers/index.d.ts +2 -0
- package/dist/observers/index.js +8 -0
- package/dist/observers/loggers/debug_log_observer.d.ts +91 -0
- package/dist/observers/loggers/debug_log_observer.js +228 -0
- package/dist/observers/loggers/index.d.ts +2 -0
- package/dist/observers/loggers/index.js +8 -0
- package/dist/observers/loggers/llm_log_observer.d.ts +24 -0
- package/dist/observers/loggers/llm_log_observer.js +110 -0
- package/dist/observers/loggers/metrics_log_observer.d.ts +85 -0
- package/dist/observers/loggers/metrics_log_observer.js +186 -0
- package/dist/observers/loggers/transcription_log_observer.d.ts +23 -0
- package/dist/observers/loggers/transcription_log_observer.js +74 -0
- package/dist/observers/service_metrics_observer.d.ts +168 -0
- package/dist/observers/service_metrics_observer.js +236 -0
- package/dist/observers/speaking_observer.d.ts +104 -0
- package/dist/observers/speaking_observer.js +150 -0
- package/dist/observers/startup_timing_observer.d.ts +221 -0
- package/dist/observers/startup_timing_observer.js +365 -0
- package/dist/observers/turn_tracking_observer.d.ts +78 -0
- package/dist/observers/turn_tracking_observer.js +189 -0
- package/dist/observers/user_bot_latency_observer.d.ts +411 -0
- package/dist/observers/user_bot_latency_observer.js +974 -0
- package/dist/pipeline/base_pipeline.d.ts +49 -0
- package/dist/pipeline/base_pipeline.js +101 -0
- package/dist/pipeline/capabilities.d.ts +76 -0
- package/dist/pipeline/capabilities.js +97 -0
- package/dist/pipeline/index.d.ts +2 -0
- package/dist/pipeline/index.js +8 -0
- package/dist/pipeline/job_context.d.ts +317 -0
- package/dist/pipeline/job_context.js +450 -0
- package/dist/pipeline/job_decorator.d.ts +54 -0
- package/dist/pipeline/job_decorator.js +92 -0
- package/dist/pipeline/llm_switcher.d.ts +84 -0
- package/dist/pipeline/llm_switcher.js +107 -0
- package/dist/pipeline/llm_with_backend.d.ts +435 -0
- package/dist/pipeline/llm_with_backend.js +806 -0
- package/dist/pipeline/parallel_pipeline.d.ts +83 -0
- package/dist/pipeline/parallel_pipeline.js +200 -0
- package/dist/pipeline/pipeline.d.ts +126 -0
- package/dist/pipeline/pipeline.js +196 -0
- package/dist/pipeline/service_switcher.d.ts +287 -0
- package/dist/pipeline/service_switcher.js +509 -0
- package/dist/pipeline/sync_parallel_pipeline.d.ts +192 -0
- package/dist/pipeline/sync_parallel_pipeline.js +387 -0
- package/dist/pipeline/worker.d.ts +814 -0
- package/dist/pipeline/worker.js +1639 -0
- package/dist/pipeline/worker_observer.d.ts +131 -0
- package/dist/pipeline/worker_observer.js +274 -0
- package/dist/pipeline/worker_ready_decorator.d.ts +45 -0
- package/dist/pipeline/worker_ready_decorator.js +76 -0
- package/dist/processors/aggregators/async_tool_messages.d.ts +146 -0
- package/dist/processors/aggregators/async_tool_messages.js +291 -0
- package/dist/processors/aggregators/dtmf_aggregator.d.ts +61 -0
- package/dist/processors/aggregators/dtmf_aggregator.js +144 -0
- package/dist/processors/aggregators/gated.d.ts +43 -0
- package/dist/processors/aggregators/gated.js +86 -0
- package/dist/processors/aggregators/gated_llm_context.d.ts +42 -0
- package/dist/processors/aggregators/gated_llm_context.js +86 -0
- package/dist/processors/aggregators/index.d.ts +2 -0
- package/dist/processors/aggregators/index.js +8 -0
- package/dist/processors/aggregators/llm_context.d.ts +370 -0
- package/dist/processors/aggregators/llm_context.js +580 -0
- package/dist/processors/aggregators/llm_context_summarizer.d.ts +219 -0
- package/dist/processors/aggregators/llm_context_summarizer.js +447 -0
- package/dist/processors/aggregators/llm_response.d.ts +37 -0
- package/dist/processors/aggregators/llm_response.js +71 -0
- package/dist/processors/aggregators/llm_response_universal.d.ts +966 -0
- package/dist/processors/aggregators/llm_response_universal.js +2394 -0
- package/dist/processors/aggregators/llm_text_processor.d.ts +53 -0
- package/dist/processors/aggregators/llm_text_processor.js +102 -0
- package/dist/processors/aggregators/sentence.d.ts +39 -0
- package/dist/processors/aggregators/sentence.js +69 -0
- package/dist/processors/async_generator.d.ts +42 -0
- package/dist/processors/async_generator.js +76 -0
- package/dist/processors/audio/audio_buffer_processor.d.ts +241 -0
- package/dist/processors/audio/audio_buffer_processor.js +565 -0
- package/dist/processors/audio/index.d.ts +2 -0
- package/dist/processors/audio/index.js +8 -0
- package/dist/processors/audio/vad_processor.d.ts +67 -0
- package/dist/processors/audio/vad_processor.js +103 -0
- package/dist/processors/consumer_processor.d.ts +49 -0
- package/dist/processors/consumer_processor.js +85 -0
- package/dist/processors/filters/frame_filter.d.ts +39 -0
- package/dist/processors/filters/frame_filter.js +52 -0
- package/dist/processors/filters/function_filter.d.ts +59 -0
- package/dist/processors/filters/function_filter.js +78 -0
- package/dist/processors/filters/identity_filter.d.ts +33 -0
- package/dist/processors/filters/identity_filter.js +37 -0
- package/dist/processors/filters/index.d.ts +2 -0
- package/dist/processors/filters/index.js +8 -0
- package/dist/processors/filters/null_filter.d.ts +34 -0
- package/dist/processors/filters/null_filter.js +47 -0
- package/dist/processors/filters/wake_notifier_filter.d.ts +48 -0
- package/dist/processors/filters/wake_notifier_filter.js +49 -0
- package/dist/processors/frame_processor.d.ts +360 -0
- package/dist/processors/frame_processor.js +932 -0
- package/dist/processors/frameworks/index.d.ts +2 -0
- package/dist/processors/frameworks/index.js +8 -0
- package/dist/processors/frameworks/rtvi/frames.d.ts +241 -0
- package/dist/processors/frameworks/rtvi/frames.js +259 -0
- package/dist/processors/frameworks/rtvi/index.d.ts +6 -0
- package/dist/processors/frameworks/rtvi/index.js +11 -0
- package/dist/processors/frameworks/rtvi/models.d.ts +1741 -0
- package/dist/processors/frameworks/rtvi/models.js +1807 -0
- package/dist/processors/frameworks/rtvi/observer.d.ts +340 -0
- package/dist/processors/frameworks/rtvi/observer.js +1113 -0
- package/dist/processors/frameworks/rtvi/processor.d.ts +182 -0
- package/dist/processors/frameworks/rtvi/processor.js +676 -0
- package/dist/processors/idle_frame_processor.d.ts +44 -0
- package/dist/processors/idle_frame_processor.js +86 -0
- package/dist/processors/index.d.ts +2 -0
- package/dist/processors/index.js +8 -0
- package/dist/processors/logger.d.ts +42 -0
- package/dist/processors/logger.js +87 -0
- package/dist/processors/metrics/frame_processor_metrics.d.ts +172 -0
- package/dist/processors/metrics/frame_processor_metrics.js +372 -0
- package/dist/processors/metrics/index.d.ts +2 -0
- package/dist/processors/metrics/index.js +8 -0
- package/dist/processors/producer_processor.d.ts +68 -0
- package/dist/processors/producer_processor.js +78 -0
- package/dist/processors/text_transformer.d.ts +30 -0
- package/dist/processors/text_transformer.js +48 -0
- package/dist/registry/index.d.ts +3 -0
- package/dist/registry/index.js +8 -0
- package/dist/registry/registry.d.ts +69 -0
- package/dist/registry/registry.js +149 -0
- package/dist/registry/types.d.ts +47 -0
- package/dist/registry/types.js +39 -0
- package/dist/serializers/base_serializer.d.ts +94 -0
- package/dist/serializers/base_serializer.js +82 -0
- package/dist/serializers/index.d.ts +2 -0
- package/dist/serializers/index.js +8 -0
- package/dist/services/ai_service.d.ts +145 -0
- package/dist/services/ai_service.js +262 -0
- package/dist/services/anthropic/index.d.ts +2 -0
- package/dist/services/anthropic/index.js +7 -0
- package/dist/services/assemblyai/index.d.ts +2 -0
- package/dist/services/assemblyai/index.js +7 -0
- package/dist/services/asyncai/index.d.ts +2 -0
- package/dist/services/asyncai/index.js +7 -0
- package/dist/services/aws/index.d.ts +2 -0
- package/dist/services/aws/index.js +7 -0
- package/dist/services/azure/index.d.ts +2 -0
- package/dist/services/azure/index.js +7 -0
- package/dist/services/baseten/index.d.ts +2 -0
- package/dist/services/baseten/index.js +7 -0
- package/dist/services/bland/index.d.ts +2 -0
- package/dist/services/bland/index.js +7 -0
- package/dist/services/camb/index.d.ts +2 -0
- package/dist/services/camb/index.js +7 -0
- package/dist/services/cartesia/index.d.ts +2 -0
- package/dist/services/cartesia/index.js +7 -0
- package/dist/services/cerebras/index.d.ts +2 -0
- package/dist/services/cerebras/index.js +7 -0
- package/dist/services/crusoe/index.d.ts +2 -0
- package/dist/services/crusoe/index.js +7 -0
- package/dist/services/deepgram/index.d.ts +2 -0
- package/dist/services/deepgram/index.js +7 -0
- package/dist/services/deepseek/index.d.ts +2 -0
- package/dist/services/deepseek/index.js +7 -0
- package/dist/services/elevenlabs/index.d.ts +2 -0
- package/dist/services/elevenlabs/index.js +7 -0
- package/dist/services/fal/index.d.ts +2 -0
- package/dist/services/fal/index.js +7 -0
- package/dist/services/fireworks/index.d.ts +2 -0
- package/dist/services/fireworks/index.js +7 -0
- package/dist/services/fish/index.d.ts +2 -0
- package/dist/services/fish/index.js +7 -0
- package/dist/services/funasr/index.d.ts +3 -0
- package/dist/services/funasr/index.js +8 -0
- package/dist/services/gladia/index.d.ts +2 -0
- package/dist/services/gladia/index.js +7 -0
- package/dist/services/google/index.d.ts +2 -0
- package/dist/services/google/index.js +7 -0
- package/dist/services/gradium/index.d.ts +2 -0
- package/dist/services/gradium/index.js +7 -0
- package/dist/services/grok/index.d.ts +2 -0
- package/dist/services/grok/index.js +7 -0
- package/dist/services/groq/index.d.ts +2 -0
- package/dist/services/groq/index.js +7 -0
- package/dist/services/heygen/index.d.ts +2 -0
- package/dist/services/heygen/index.js +7 -0
- package/dist/services/hume/index.d.ts +2 -0
- package/dist/services/hume/index.js +7 -0
- package/dist/services/image_service.d.ts +52 -0
- package/dist/services/image_service.js +60 -0
- package/dist/services/inception/index.d.ts +2 -0
- package/dist/services/inception/index.js +7 -0
- package/dist/services/index.d.ts +2 -0
- package/dist/services/index.js +8 -0
- package/dist/services/inworld/index.d.ts +2 -0
- package/dist/services/inworld/index.js +7 -0
- package/dist/services/keenable/index.d.ts +2 -0
- package/dist/services/keenable/index.js +7 -0
- package/dist/services/kokoro/index.d.ts +2 -0
- package/dist/services/kokoro/index.js +7 -0
- package/dist/services/llm_service.d.ts +1106 -0
- package/dist/services/llm_service.js +2231 -0
- package/dist/services/mem0/index.d.ts +2 -0
- package/dist/services/mem0/index.js +7 -0
- package/dist/services/meta/index.d.ts +2 -0
- package/dist/services/meta/index.js +7 -0
- package/dist/services/minimax/index.d.ts +2 -0
- package/dist/services/minimax/index.js +7 -0
- package/dist/services/mistral/index.d.ts +2 -0
- package/dist/services/mistral/index.js +7 -0
- package/dist/services/moondream/index.d.ts +2 -0
- package/dist/services/moondream/index.js +7 -0
- package/dist/services/moonshine/index.d.ts +2 -0
- package/dist/services/moonshine/index.js +7 -0
- package/dist/services/nebius/index.d.ts +2 -0
- package/dist/services/nebius/index.js +7 -0
- package/dist/services/neuphonic/index.d.ts +2 -0
- package/dist/services/neuphonic/index.js +7 -0
- package/dist/services/novita/index.d.ts +2 -0
- package/dist/services/novita/index.js +7 -0
- package/dist/services/nvidia/index.d.ts +2 -0
- package/dist/services/nvidia/index.js +11 -0
- package/dist/services/ollama/index.d.ts +2 -0
- package/dist/services/ollama/index.js +7 -0
- package/dist/services/openai/index.d.ts +2 -0
- package/dist/services/openai/index.js +7 -0
- package/dist/services/openclaw/index.d.ts +14 -0
- package/dist/services/openclaw/index.js +19 -0
- package/dist/services/openrouter/index.d.ts +2 -0
- package/dist/services/openrouter/index.js +7 -0
- package/dist/services/perplexity/index.d.ts +2 -0
- package/dist/services/perplexity/index.js +7 -0
- package/dist/services/piper/index.d.ts +2 -0
- package/dist/services/piper/index.js +7 -0
- package/dist/services/pocket_tts/index.d.ts +2 -0
- package/dist/services/pocket_tts/index.js +7 -0
- package/dist/services/qwen/index.d.ts +2 -0
- package/dist/services/qwen/index.js +7 -0
- package/dist/services/resembleai/index.d.ts +2 -0
- package/dist/services/resembleai/index.js +7 -0
- package/dist/services/rime/index.d.ts +2 -0
- package/dist/services/rime/index.js +7 -0
- package/dist/services/sambanova/index.d.ts +2 -0
- package/dist/services/sambanova/index.js +7 -0
- package/dist/services/sarvam/index.d.ts +2 -0
- package/dist/services/sarvam/index.js +7 -0
- package/dist/services/settings.d.ts +356 -0
- package/dist/services/settings.js +437 -0
- package/dist/services/simli/index.d.ts +2 -0
- package/dist/services/simli/index.js +7 -0
- package/dist/services/smallest/index.d.ts +2 -0
- package/dist/services/smallest/index.js +7 -0
- package/dist/services/soniox/index.d.ts +2 -0
- package/dist/services/soniox/index.js +7 -0
- package/dist/services/speechify/index.d.ts +2 -0
- package/dist/services/speechify/index.js +7 -0
- package/dist/services/speechmatics/index.d.ts +2 -0
- package/dist/services/speechmatics/index.js +7 -0
- package/dist/services/stt_latency.d.ts +62 -0
- package/dist/services/stt_latency.js +71 -0
- package/dist/services/stt_service.d.ts +601 -0
- package/dist/services/stt_service.js +1044 -0
- package/dist/services/tavus/index.d.ts +2 -0
- package/dist/services/tavus/index.js +7 -0
- package/dist/services/together/index.d.ts +2 -0
- package/dist/services/together/index.js +7 -0
- package/dist/services/tts_service.d.ts +909 -0
- package/dist/services/tts_service.js +2018 -0
- package/dist/services/ultravox/index.d.ts +2 -0
- package/dist/services/ultravox/index.js +9 -0
- package/dist/services/vision_service.d.ts +56 -0
- package/dist/services/vision_service.js +62 -0
- package/dist/services/websocket_service.d.ts +213 -0
- package/dist/services/websocket_service.js +654 -0
- package/dist/services/whisper/index.d.ts +2 -0
- package/dist/services/whisper/index.js +9 -0
- package/dist/services/xai/index.d.ts +2 -0
- package/dist/services/xai/index.js +9 -0
- package/dist/services/xtts/index.d.ts +2 -0
- package/dist/services/xtts/index.js +9 -0
- package/dist/tests/index.d.ts +2 -0
- package/dist/tests/index.js +8 -0
- package/dist/tests/utils.d.ts +123 -0
- package/dist/tests/utils.js +212 -0
- package/dist/transcriptions/index.d.ts +2 -0
- package/dist/transcriptions/index.js +8 -0
- package/dist/transcriptions/language.d.ts +376 -0
- package/dist/transcriptions/language.js +543 -0
- package/dist/transports/base_input.d.ts +149 -0
- package/dist/transports/base_input.js +338 -0
- package/dist/transports/base_output.d.ts +432 -0
- package/dist/transports/base_output.js +1231 -0
- package/dist/transports/base_transport.d.ts +223 -0
- package/dist/transports/base_transport.js +167 -0
- package/dist/transports/daily/index.d.ts +2 -0
- package/dist/transports/daily/index.js +9 -0
- package/dist/transports/heygen/index.d.ts +2 -0
- package/dist/transports/heygen/index.js +9 -0
- package/dist/transports/index.d.ts +2 -0
- package/dist/transports/index.js +8 -0
- package/dist/transports/lemonslice/index.d.ts +2 -0
- package/dist/transports/lemonslice/index.js +9 -0
- package/dist/transports/livekit/index.d.ts +2 -0
- package/dist/transports/livekit/index.js +9 -0
- package/dist/transports/local/index.d.ts +2 -0
- package/dist/transports/local/index.js +9 -0
- package/dist/transports/moq/index.d.ts +2 -0
- package/dist/transports/moq/index.js +9 -0
- package/dist/transports/smallwebrtc/index.d.ts +2 -0
- package/dist/transports/smallwebrtc/index.js +9 -0
- package/dist/transports/tavus/index.d.ts +2 -0
- package/dist/transports/tavus/index.js +9 -0
- package/dist/transports/video_in_sampler.d.ts +157 -0
- package/dist/transports/video_in_sampler.js +258 -0
- package/dist/transports/vonage/index.d.ts +2 -0
- package/dist/transports/vonage/index.js +9 -0
- package/dist/transports/websocket/index.d.ts +2 -0
- package/dist/transports/websocket/index.js +9 -0
- package/dist/transports/whatsapp/index.d.ts +2 -0
- package/dist/transports/whatsapp/index.js +9 -0
- package/dist/turns/eager_end_of_turn_mixin.d.ts +70 -0
- package/dist/turns/eager_end_of_turn_mixin.js +122 -0
- package/dist/turns/empty_user_turn.d.ts +55 -0
- package/dist/turns/empty_user_turn.js +53 -0
- package/dist/turns/index.d.ts +2 -0
- package/dist/turns/index.js +8 -0
- package/dist/turns/speculation_gate.d.ts +169 -0
- package/dist/turns/speculation_gate.js +294 -0
- package/dist/turns/types.d.ts +33 -0
- package/dist/turns/types.js +35 -0
- package/dist/turns/user_idle_controller.d.ts +94 -0
- package/dist/turns/user_idle_controller.js +187 -0
- package/dist/turns/user_mute/always_user_mute_strategy.d.ts +15 -0
- package/dist/turns/user_mute/always_user_mute_strategy.js +29 -0
- package/dist/turns/user_mute/base_user_mute_strategy.d.ts +39 -0
- package/dist/turns/user_mute/base_user_mute_strategy.js +45 -0
- package/dist/turns/user_mute/first_speech_user_mute_strategy.d.ts +28 -0
- package/dist/turns/user_mute/first_speech_user_mute_strategy.js +52 -0
- package/dist/turns/user_mute/function_call_user_mute_strategy.d.ts +25 -0
- package/dist/turns/user_mute/function_call_user_mute_strategy.js +50 -0
- package/dist/turns/user_mute/index.d.ts +6 -0
- package/dist/turns/user_mute/index.js +11 -0
- package/dist/turns/user_mute/mute_until_first_bot_complete_user_mute_strategy.d.ts +43 -0
- package/dist/turns/user_mute/mute_until_first_bot_complete_user_mute_strategy.js +76 -0
- package/dist/turns/user_start/base_user_turn_start_strategy.d.ts +135 -0
- package/dist/turns/user_start/base_user_turn_start_strategy.js +134 -0
- package/dist/turns/user_start/external_user_turn_start_strategy.d.ts +45 -0
- package/dist/turns/user_start/external_user_turn_start_strategy.js +64 -0
- package/dist/turns/user_start/index.d.ts +8 -0
- package/dist/turns/user_start/index.js +16 -0
- package/dist/turns/user_start/krisp_viva_ip_user_turn_start_strategy.d.ts +133 -0
- package/dist/turns/user_start/krisp_viva_ip_user_turn_start_strategy.js +243 -0
- package/dist/turns/user_start/min_words_user_turn_start_strategy.d.ts +60 -0
- package/dist/turns/user_start/min_words_user_turn_start_strategy.js +102 -0
- package/dist/turns/user_start/transcription_user_turn_start_strategy.d.ts +29 -0
- package/dist/turns/user_start/transcription_user_turn_start_strategy.js +43 -0
- package/dist/turns/user_start/vad_user_turn_start_strategy.d.ts +20 -0
- package/dist/turns/user_start/vad_user_turn_start_strategy.js +31 -0
- package/dist/turns/user_start/wake_phrase_user_turn_start_strategy.d.ts +141 -0
- package/dist/turns/user_start/wake_phrase_user_turn_start_strategy.js +238 -0
- package/dist/turns/user_stop/base_user_turn_stop_strategy.d.ts +174 -0
- package/dist/turns/user_stop/base_user_turn_stop_strategy.js +170 -0
- package/dist/turns/user_stop/deferred_user_turn_stop_strategy.d.ts +81 -0
- package/dist/turns/user_stop/deferred_user_turn_stop_strategy.js +107 -0
- package/dist/turns/user_stop/eager_match_policy.d.ts +36 -0
- package/dist/turns/user_stop/eager_match_policy.js +50 -0
- package/dist/turns/user_stop/eager_user_turn_stop_strategy.d.ts +84 -0
- package/dist/turns/user_stop/eager_user_turn_stop_strategy.js +154 -0
- package/dist/turns/user_stop/external_user_turn_completion_stop_strategy.d.ts +45 -0
- package/dist/turns/user_stop/external_user_turn_completion_stop_strategy.js +55 -0
- package/dist/turns/user_stop/external_user_turn_stop_strategy.d.ts +115 -0
- package/dist/turns/user_stop/external_user_turn_stop_strategy.js +205 -0
- package/dist/turns/user_stop/index.d.ts +10 -0
- package/dist/turns/user_stop/index.js +15 -0
- package/dist/turns/user_stop/llm_turn_completion_user_turn_stop_strategy.d.ts +44 -0
- package/dist/turns/user_stop/llm_turn_completion_user_turn_stop_strategy.js +59 -0
- package/dist/turns/user_stop/speech_timeout_user_turn_stop_strategy.d.ts +140 -0
- package/dist/turns/user_stop/speech_timeout_user_turn_stop_strategy.js +303 -0
- package/dist/turns/user_stop/turn_analyzer_user_turn_stop_strategy.d.ts +135 -0
- package/dist/turns/user_stop/turn_analyzer_user_turn_stop_strategy.js +332 -0
- package/dist/turns/user_turn_completion_mixin.d.ts +147 -0
- package/dist/turns/user_turn_completion_mixin.js +591 -0
- package/dist/turns/user_turn_controller.d.ts +134 -0
- package/dist/turns/user_turn_controller.js +335 -0
- package/dist/turns/user_turn_processor.d.ts +87 -0
- package/dist/turns/user_turn_processor.js +180 -0
- package/dist/turns/user_turn_strategies.d.ts +187 -0
- package/dist/turns/user_turn_strategies.js +222 -0
- package/dist/utils/async_tool_cancellation.d.ts +35 -0
- package/dist/utils/async_tool_cancellation.js +78 -0
- package/dist/utils/asyncio/index.d.ts +2 -0
- package/dist/utils/asyncio/index.js +7 -0
- package/dist/utils/asyncio/task_manager.d.ts +49 -0
- package/dist/utils/asyncio/task_manager.js +105 -0
- package/dist/utils/base_object.d.ts +76 -0
- package/dist/utils/base_object.js +154 -0
- package/dist/utils/context/aggregated_frame_sequencer.d.ts +491 -0
- package/dist/utils/context/aggregated_frame_sequencer.js +803 -0
- package/dist/utils/context/audio_context_manager.d.ts +131 -0
- package/dist/utils/context/audio_context_manager.js +233 -0
- package/dist/utils/context/index.d.ts +2 -0
- package/dist/utils/context/index.js +7 -0
- package/dist/utils/context/llm_context_summarization.d.ts +258 -0
- package/dist/utils/context/llm_context_summarization.js +511 -0
- package/dist/utils/context/text_segment_map.d.ts +611 -0
- package/dist/utils/context/text_segment_map.js +1132 -0
- package/dist/utils/context/word_completion_tracker.d.ts +273 -0
- package/dist/utils/context/word_completion_tracker.js +441 -0
- package/dist/utils/context/word_timestamp_tracker.d.ts +72 -0
- package/dist/utils/context/word_timestamp_tracker.js +98 -0
- package/dist/utils/deprecation.d.ts +28 -0
- package/dist/utils/deprecation.js +75 -0
- package/dist/utils/env.d.ts +38 -0
- package/dist/utils/env.js +62 -0
- package/dist/utils/errors.d.ts +75 -0
- package/dist/utils/errors.js +202 -0
- package/dist/utils/file_resolver.d.ts +126 -0
- package/dist/utils/file_resolver.js +314 -0
- package/dist/utils/file_storage.d.ts +99 -0
- package/dist/utils/file_storage.js +100 -0
- package/dist/utils/frame_queue.d.ts +36 -0
- package/dist/utils/frame_queue.js +66 -0
- package/dist/utils/http.d.ts +39 -0
- package/dist/utils/http.js +53 -0
- package/dist/utils/image.d.ts +15 -0
- package/dist/utils/image.js +30 -0
- package/dist/utils/index.d.ts +2 -0
- package/dist/utils/index.js +8 -0
- package/dist/utils/log_config.d.ts +34 -0
- package/dist/utils/log_config.js +76 -0
- package/dist/utils/logger.d.ts +40 -0
- package/dist/utils/logger.js +91 -0
- package/dist/utils/mixin.d.ts +14 -0
- package/dist/utils/mixin.js +63 -0
- package/dist/utils/network.d.ts +73 -0
- package/dist/utils/network.js +90 -0
- package/dist/utils/security/allowed_origins.d.ts +16 -0
- package/dist/utils/security/allowed_origins.js +34 -0
- package/dist/utils/security/index.d.ts +2 -0
- package/dist/utils/security/index.js +7 -0
- package/dist/utils/security/redaction.d.ts +27 -0
- package/dist/utils/security/redaction.js +123 -0
- package/dist/utils/security/ssrf.d.ts +79 -0
- package/dist/utils/security/ssrf.js +327 -0
- package/dist/utils/shared.d.ts +35 -0
- package/dist/utils/shared.js +136 -0
- package/dist/utils/startup.d.ts +17 -0
- package/dist/utils/startup.js +130 -0
- package/dist/utils/string.d.ts +116 -0
- package/dist/utils/string.js +460 -0
- package/dist/utils/sync/base_notifier.d.ts +25 -0
- package/dist/utils/sync/base_notifier.js +16 -0
- package/dist/utils/sync/event_notifier.d.ts +38 -0
- package/dist/utils/sync/event_notifier.js +51 -0
- package/dist/utils/sync/index.d.ts +2 -0
- package/dist/utils/sync/index.js +7 -0
- package/dist/utils/text/alnum_utils.d.ts +109 -0
- package/dist/utils/text/alnum_utils.js +262 -0
- package/dist/utils/text/base_text_aggregator.d.ts +146 -0
- package/dist/utils/text/base_text_aggregator.js +102 -0
- package/dist/utils/text/base_text_filter.d.ts +54 -0
- package/dist/utils/text/base_text_filter.js +50 -0
- package/dist/utils/text/index.d.ts +2 -0
- package/dist/utils/text/index.js +7 -0
- package/dist/utils/text/markdown_text_filter.d.ts +147 -0
- package/dist/utils/text/markdown_text_filter.js +522 -0
- package/dist/utils/text/markup_utils.d.ts +54 -0
- package/dist/utils/text/markup_utils.js +156 -0
- package/dist/utils/text/pattern_pair_aggregator.d.ts +251 -0
- package/dist/utils/text/pattern_pair_aggregator.js +478 -0
- package/dist/utils/text/phonemes.d.ts +60 -0
- package/dist/utils/text/phonemes.js +172 -0
- package/dist/utils/text/simple_text_aggregator.d.ts +135 -0
- package/dist/utils/text/simple_text_aggregator.js +254 -0
- package/dist/utils/text/skip_tags_aggregator.d.ts +85 -0
- package/dist/utils/text/skip_tags_aggregator.js +147 -0
- package/dist/utils/text/transforms/pronunciations.d.ts +65 -0
- package/dist/utils/text/transforms/pronunciations.js +111 -0
- package/dist/utils/text/word_timestamp_utils.d.ts +26 -0
- package/dist/utils/text/word_timestamp_utils.js +52 -0
- package/dist/utils/time.d.ts +13 -0
- package/dist/utils/time.js +31 -0
- package/dist/utils/tracing/index.d.ts +3 -0
- package/dist/utils/tracing/index.js +8 -0
- package/dist/utils/tracing/service_attributes.d.ts +198 -0
- package/dist/utils/tracing/service_attributes.js +434 -0
- package/dist/utils/tracing/service_decorators.d.ts +214 -0
- package/dist/utils/tracing/service_decorators.js +1662 -0
- package/dist/utils/tracing/setup.d.ts +95 -0
- package/dist/utils/tracing/setup.js +110 -0
- package/dist/utils/tracing/tracing_context.d.ts +55 -0
- package/dist/utils/tracing/tracing_context.js +97 -0
- package/dist/utils/tracing/turn_trace_observer.d.ts +90 -0
- package/dist/utils/tracing/turn_trace_observer.js +219 -0
- package/dist/utils/types.d.ts +80 -0
- package/dist/utils/types.js +108 -0
- package/dist/utils/utils.d.ts +5 -0
- package/dist/utils/utils.js +22 -0
- package/dist/utils/yaml.d.ts +64 -0
- package/dist/utils/yaml.js +41 -0
- package/dist/version.d.ts +3 -0
- package/dist/version.js +7 -0
- package/dist/workers/base_worker.d.ts +772 -0
- package/dist/workers/base_worker.js +1298 -0
- package/dist/workers/index.d.ts +2 -0
- package/dist/workers/index.js +8 -0
- package/dist/workers/llm/backend_llm_worker.d.ts +552 -0
- package/dist/workers/llm/backend_llm_worker.js +1055 -0
- package/dist/workers/llm/index.d.ts +6 -0
- package/dist/workers/llm/index.js +11 -0
- package/dist/workers/llm/llm_context_worker.d.ts +66 -0
- package/dist/workers/llm/llm_context_worker.js +72 -0
- package/dist/workers/llm/llm_worker.d.ts +180 -0
- package/dist/workers/llm/llm_worker.js +323 -0
- package/dist/workers/llm/tool_decorator.d.ts +68 -0
- package/dist/workers/llm/tool_decorator.js +60 -0
- package/dist/workers/runner.d.ts +268 -0
- package/dist/workers/runner.js +489 -0
- package/dist/workers/ui/index.d.ts +22 -0
- package/dist/workers/ui/index.js +27 -0
- package/dist/workers/ui/ui_event_decorator.d.ts +43 -0
- package/dist/workers/ui/ui_event_decorator.js +75 -0
- package/dist/workers/ui/ui_job_context.d.ts +12 -0
- package/dist/workers/ui/ui_job_context.js +17 -0
- package/dist/workers/ui/ui_prompts.d.ts +21 -0
- package/dist/workers/ui/ui_prompts.js +41 -0
- package/dist/workers/ui/ui_tools.d.ts +32 -0
- package/dist/workers/ui/ui_tools.js +112 -0
- package/dist/workers/ui/ui_worker.d.ts +571 -0
- package/dist/workers/ui/ui_worker.js +1156 -0
- package/package.json +46 -3
- package/README.md +0 -5
|
@@ -0,0 +1,157 @@
|
|
|
1
|
+
import type { UserImageRequestFrame } from "../frames/frames.ts";
|
|
2
|
+
/**
|
|
3
|
+
* How early a frame may arrive and still count as due, at most half an
|
|
4
|
+
* interval. It covers jitter in the incoming stream, which is about the same at
|
|
5
|
+
* any framerate.
|
|
6
|
+
*/
|
|
7
|
+
export declare const JITTER_TOLERANCE_SECS = 0.05;
|
|
8
|
+
/**
|
|
9
|
+
* The sampler framerate for a `captureParticipantVideo()` call's arguments.
|
|
10
|
+
*
|
|
11
|
+
* A sampler passes on only the frames that answer image requests at a
|
|
12
|
+
* framerate of `0`, which is how `onRequestOnly` is applied.
|
|
13
|
+
*
|
|
14
|
+
* Porting note: upstream warns that `framerate=0` is deprecated in favour of
|
|
15
|
+
* `on_request_only=True`; the deprecation warning is not ported (`method` is
|
|
16
|
+
* kept so callers have the same signature).
|
|
17
|
+
*/
|
|
18
|
+
export declare function _captureFramerate(framerate: number | null, onRequestOnly: boolean, _method: string): number | null;
|
|
19
|
+
/**
|
|
20
|
+
* Decides which frames of one incoming video source a transport passes on.
|
|
21
|
+
*
|
|
22
|
+
* A user's camera or screen share sends frames at its own rate, often 15-30
|
|
23
|
+
* per second, while a bot usually wants fewer: a vision bot may want one a
|
|
24
|
+
* second, or only a frame when it asks for one. An input transport keeps one
|
|
25
|
+
* sampler per video source and calls `sample()` for every frame that arrives;
|
|
26
|
+
* the sampler says whether to pass that frame on or drop it.
|
|
27
|
+
*
|
|
28
|
+
* The framerate decides which frames are passed on:
|
|
29
|
+
*
|
|
30
|
+
* - `null`: every frame.
|
|
31
|
+
* - `0`: only frames that answer an image request.
|
|
32
|
+
* - `N > 0`: `N` frames per second, on the schedule described below.
|
|
33
|
+
*
|
|
34
|
+
* Whatever the framerate, an image request queued with `addRequest()` is
|
|
35
|
+
* answered by the next frame that arrives, which is passed on with the
|
|
36
|
+
* request.
|
|
37
|
+
*
|
|
38
|
+
* For `N > 0`, frames are due on a fixed schedule, one interval (`1 / N`
|
|
39
|
+
* seconds) apart:
|
|
40
|
+
*
|
|
41
|
+
* - **On schedule.** A frame is due once its scheduled time comes, or up to a
|
|
42
|
+
* tolerance before it, since frames never arrive exactly on time. The next
|
|
43
|
+
* frame is scheduled one interval after this frame's *scheduled* time, not
|
|
44
|
+
* its arrival time, so the rate doesn't drift. A frame arriving late moves
|
|
45
|
+
* the schedule to its arrival, so a stream slower than the framerate passes
|
|
46
|
+
* every frame rather than falling further and further behind.
|
|
47
|
+
* - **Fresh start.** The first frame, a frame that answers a request, and the
|
|
48
|
+
* first frame after a stall (more than an interval past its scheduled time)
|
|
49
|
+
* start a new schedule from their arrival. After a stall this passes on one
|
|
50
|
+
* frame instead of a burst catching up on the missed ones.
|
|
51
|
+
*
|
|
52
|
+
* The tolerance is the smaller of `JITTER_TOLERANCE_SECS` and half an
|
|
53
|
+
* interval. At low framerates the next frame after any frame is therefore
|
|
54
|
+
* close to a full interval later; at framerates near the stream's own rate it
|
|
55
|
+
* covers the stream's jitter, so every frame passes.
|
|
56
|
+
*/
|
|
57
|
+
export declare class _VideoInSampler {
|
|
58
|
+
protected _framerate: number | null;
|
|
59
|
+
protected _nextTime: number;
|
|
60
|
+
protected _requests: UserImageRequestFrame[];
|
|
61
|
+
/**
|
|
62
|
+
* Initialize the sampler.
|
|
63
|
+
*
|
|
64
|
+
* @param framerate Frames per second to pass on. `0` passes on frames only
|
|
65
|
+
* to answer requests, and `null` passes on every frame.
|
|
66
|
+
*/
|
|
67
|
+
constructor(framerate: number | null);
|
|
68
|
+
/** The frames per second this sampler passes on. */
|
|
69
|
+
get framerate(): number | null;
|
|
70
|
+
set framerate(framerate: number | null);
|
|
71
|
+
/**
|
|
72
|
+
* Queue an image request for the next frame to answer.
|
|
73
|
+
*
|
|
74
|
+
* @param request The image request to answer.
|
|
75
|
+
*/
|
|
76
|
+
addRequest(request: UserImageRequestFrame): void;
|
|
77
|
+
/**
|
|
78
|
+
* Remove and return the image requests waiting for a frame.
|
|
79
|
+
*
|
|
80
|
+
* @returns The waiting requests, oldest first.
|
|
81
|
+
*/
|
|
82
|
+
takeRequests(): UserImageRequestFrame[];
|
|
83
|
+
/**
|
|
84
|
+
* Decide whether to pass on the frame that just arrived.
|
|
85
|
+
*
|
|
86
|
+
* @returns A tuple of `[due, request]`: `due` is whether to pass the frame
|
|
87
|
+
* on, and `request` is the `UserImageRequestFrame` it answers, or `null`.
|
|
88
|
+
*/
|
|
89
|
+
sample(): [boolean, UserImageRequestFrame | null];
|
|
90
|
+
}
|
|
91
|
+
/** An input transport's video samplers, by participant and video source. */
|
|
92
|
+
export declare class _VideoInSamplers {
|
|
93
|
+
#private;
|
|
94
|
+
protected _samplers: Map<string, [string, string, _VideoInSampler]>;
|
|
95
|
+
/**
|
|
96
|
+
* Start sampling a source, or change its framerate.
|
|
97
|
+
*
|
|
98
|
+
* Changing the framerate keeps the source's sampler, so image requests
|
|
99
|
+
* already waiting on it are still answered.
|
|
100
|
+
*
|
|
101
|
+
* @param participantId The participant whose video this is.
|
|
102
|
+
* @param videoSource The video source, e.g. `"camera"` or `"screenVideo"`.
|
|
103
|
+
* @param framerate Frames per second to pass on. `0` passes on frames only
|
|
104
|
+
* to answer requests, and `null` passes on every frame.
|
|
105
|
+
*/
|
|
106
|
+
capture(participantId: string, videoSource: string, framerate: number | null): void;
|
|
107
|
+
/**
|
|
108
|
+
* Whether a source is being sampled.
|
|
109
|
+
*
|
|
110
|
+
* @param participantId The participant whose video this is.
|
|
111
|
+
* @param videoSource The video source.
|
|
112
|
+
* @returns Whether `capture()` was called for the source.
|
|
113
|
+
*/
|
|
114
|
+
isCapturing(participantId: string, videoSource: string): boolean;
|
|
115
|
+
/**
|
|
116
|
+
* Queue an image request for the next frame of a source to answer.
|
|
117
|
+
*
|
|
118
|
+
* @param participantId The participant whose video is requested.
|
|
119
|
+
* @param videoSource The video source requested.
|
|
120
|
+
* @param request The image request to answer.
|
|
121
|
+
* @returns Whether the request was queued; `false` if the source isn't being
|
|
122
|
+
* sampled, so nothing would answer it.
|
|
123
|
+
*/
|
|
124
|
+
addRequest(participantId: string, videoSource: string, request: UserImageRequestFrame): boolean;
|
|
125
|
+
/**
|
|
126
|
+
* Decide whether to pass on the frame that just arrived from a source.
|
|
127
|
+
*
|
|
128
|
+
* @param participantId The participant the frame came from.
|
|
129
|
+
* @param videoSource The video source the frame came from.
|
|
130
|
+
* @returns A tuple of `[due, request]`, as from `_VideoInSampler.sample()`.
|
|
131
|
+
* A frame from a source that isn't being sampled is never due.
|
|
132
|
+
*/
|
|
133
|
+
sample(participantId: string, videoSource: string): [boolean, UserImageRequestFrame | null];
|
|
134
|
+
/**
|
|
135
|
+
* Remove and return image requests waiting for a frame, keeping the samplers.
|
|
136
|
+
*
|
|
137
|
+
* @param participantId The participant whose requests to take.
|
|
138
|
+
* @param videoSource The video source whose requests to take, or `null` for
|
|
139
|
+
* all of the participant's sources.
|
|
140
|
+
* @returns The waiting requests.
|
|
141
|
+
*/
|
|
142
|
+
takeRequests(participantId: string, videoSource?: string | null): UserImageRequestFrame[];
|
|
143
|
+
/**
|
|
144
|
+
* Stop sampling every video source of a participant.
|
|
145
|
+
*
|
|
146
|
+
* @param participantId The participant to forget, e.g. one who left.
|
|
147
|
+
* @returns The image requests that were waiting on the participant's video.
|
|
148
|
+
*/
|
|
149
|
+
removeParticipant(participantId: string): UserImageRequestFrame[];
|
|
150
|
+
/**
|
|
151
|
+
* Stop sampling every video source.
|
|
152
|
+
*
|
|
153
|
+
* @returns The image requests that were waiting on any of them.
|
|
154
|
+
*/
|
|
155
|
+
clear(): UserImageRequestFrame[];
|
|
156
|
+
}
|
|
157
|
+
//# sourceMappingURL=video_in_sampler.d.ts.map
|
|
@@ -0,0 +1,258 @@
|
|
|
1
|
+
// Copyright (c) 2024-2026, Daily
|
|
2
|
+
// Copyright (c) 2026, pipedog contributors
|
|
3
|
+
// SPDX-License-Identifier: BSD-2-Clause
|
|
4
|
+
//
|
|
5
|
+
// Ported from pipecat: src/pipecat/transports/video_in_sampler.py
|
|
6
|
+
var _a;
|
|
7
|
+
/** Selection of the incoming video frames an input transport passes on. */
|
|
8
|
+
import { time } from "@pipedog/asyncio";
|
|
9
|
+
/**
|
|
10
|
+
* How early a frame may arrive and still count as due, at most half an
|
|
11
|
+
* interval. It covers jitter in the incoming stream, which is about the same at
|
|
12
|
+
* any framerate.
|
|
13
|
+
*/
|
|
14
|
+
export const JITTER_TOLERANCE_SECS = 0.05;
|
|
15
|
+
/**
|
|
16
|
+
* The sampler framerate for a `captureParticipantVideo()` call's arguments.
|
|
17
|
+
*
|
|
18
|
+
* A sampler passes on only the frames that answer image requests at a
|
|
19
|
+
* framerate of `0`, which is how `onRequestOnly` is applied.
|
|
20
|
+
*
|
|
21
|
+
* Porting note: upstream warns that `framerate=0` is deprecated in favour of
|
|
22
|
+
* `on_request_only=True`; the deprecation warning is not ported (`method` is
|
|
23
|
+
* kept so callers have the same signature).
|
|
24
|
+
*/
|
|
25
|
+
export function _captureFramerate(framerate, onRequestOnly, _method) {
|
|
26
|
+
if (onRequestOnly)
|
|
27
|
+
return 0;
|
|
28
|
+
return framerate;
|
|
29
|
+
}
|
|
30
|
+
/**
|
|
31
|
+
* Decides which frames of one incoming video source a transport passes on.
|
|
32
|
+
*
|
|
33
|
+
* A user's camera or screen share sends frames at its own rate, often 15-30
|
|
34
|
+
* per second, while a bot usually wants fewer: a vision bot may want one a
|
|
35
|
+
* second, or only a frame when it asks for one. An input transport keeps one
|
|
36
|
+
* sampler per video source and calls `sample()` for every frame that arrives;
|
|
37
|
+
* the sampler says whether to pass that frame on or drop it.
|
|
38
|
+
*
|
|
39
|
+
* The framerate decides which frames are passed on:
|
|
40
|
+
*
|
|
41
|
+
* - `null`: every frame.
|
|
42
|
+
* - `0`: only frames that answer an image request.
|
|
43
|
+
* - `N > 0`: `N` frames per second, on the schedule described below.
|
|
44
|
+
*
|
|
45
|
+
* Whatever the framerate, an image request queued with `addRequest()` is
|
|
46
|
+
* answered by the next frame that arrives, which is passed on with the
|
|
47
|
+
* request.
|
|
48
|
+
*
|
|
49
|
+
* For `N > 0`, frames are due on a fixed schedule, one interval (`1 / N`
|
|
50
|
+
* seconds) apart:
|
|
51
|
+
*
|
|
52
|
+
* - **On schedule.** A frame is due once its scheduled time comes, or up to a
|
|
53
|
+
* tolerance before it, since frames never arrive exactly on time. The next
|
|
54
|
+
* frame is scheduled one interval after this frame's *scheduled* time, not
|
|
55
|
+
* its arrival time, so the rate doesn't drift. A frame arriving late moves
|
|
56
|
+
* the schedule to its arrival, so a stream slower than the framerate passes
|
|
57
|
+
* every frame rather than falling further and further behind.
|
|
58
|
+
* - **Fresh start.** The first frame, a frame that answers a request, and the
|
|
59
|
+
* first frame after a stall (more than an interval past its scheduled time)
|
|
60
|
+
* start a new schedule from their arrival. After a stall this passes on one
|
|
61
|
+
* frame instead of a burst catching up on the missed ones.
|
|
62
|
+
*
|
|
63
|
+
* The tolerance is the smaller of `JITTER_TOLERANCE_SECS` and half an
|
|
64
|
+
* interval. At low framerates the next frame after any frame is therefore
|
|
65
|
+
* close to a full interval later; at framerates near the stream's own rate it
|
|
66
|
+
* covers the stream's jitter, so every frame passes.
|
|
67
|
+
*/
|
|
68
|
+
export class _VideoInSampler {
|
|
69
|
+
_framerate;
|
|
70
|
+
// When the next frame is due. Zero before the first frame, which then
|
|
71
|
+
// counts as arriving after a stall and starts the schedule.
|
|
72
|
+
_nextTime = 0.0;
|
|
73
|
+
_requests = [];
|
|
74
|
+
/**
|
|
75
|
+
* Initialize the sampler.
|
|
76
|
+
*
|
|
77
|
+
* @param framerate Frames per second to pass on. `0` passes on frames only
|
|
78
|
+
* to answer requests, and `null` passes on every frame.
|
|
79
|
+
*/
|
|
80
|
+
constructor(framerate) {
|
|
81
|
+
this._framerate = framerate;
|
|
82
|
+
}
|
|
83
|
+
/** The frames per second this sampler passes on. */
|
|
84
|
+
get framerate() {
|
|
85
|
+
return this._framerate;
|
|
86
|
+
}
|
|
87
|
+
set framerate(framerate) {
|
|
88
|
+
this._framerate = framerate;
|
|
89
|
+
}
|
|
90
|
+
/**
|
|
91
|
+
* Queue an image request for the next frame to answer.
|
|
92
|
+
*
|
|
93
|
+
* @param request The image request to answer.
|
|
94
|
+
*/
|
|
95
|
+
addRequest(request) {
|
|
96
|
+
this._requests.push(request);
|
|
97
|
+
}
|
|
98
|
+
/**
|
|
99
|
+
* Remove and return the image requests waiting for a frame.
|
|
100
|
+
*
|
|
101
|
+
* @returns The waiting requests, oldest first.
|
|
102
|
+
*/
|
|
103
|
+
takeRequests() {
|
|
104
|
+
const requests = this._requests;
|
|
105
|
+
this._requests = [];
|
|
106
|
+
return requests;
|
|
107
|
+
}
|
|
108
|
+
/**
|
|
109
|
+
* Decide whether to pass on the frame that just arrived.
|
|
110
|
+
*
|
|
111
|
+
* @returns A tuple of `[due, request]`: `due` is whether to pass the frame
|
|
112
|
+
* on, and `request` is the `UserImageRequestFrame` it answers, or `null`.
|
|
113
|
+
*/
|
|
114
|
+
sample() {
|
|
115
|
+
// A waiting request is always answered by this frame.
|
|
116
|
+
const request = this._requests.length > 0 ? (this._requests.shift() ?? null) : null;
|
|
117
|
+
if (this._framerate === null)
|
|
118
|
+
return [true, request];
|
|
119
|
+
if (this._framerate === 0)
|
|
120
|
+
return [request !== null, request];
|
|
121
|
+
const now = time();
|
|
122
|
+
const interval = 1 / this._framerate;
|
|
123
|
+
const tolerance = Math.min(JITTER_TOLERANCE_SECS, interval / 2);
|
|
124
|
+
if (request === null && now < this._nextTime - tolerance) {
|
|
125
|
+
// Too early: the next frame isn't due yet.
|
|
126
|
+
return [false, null];
|
|
127
|
+
}
|
|
128
|
+
if (request || now - this._nextTime > interval) {
|
|
129
|
+
// Fresh start (a request, the first frame, or a stall): the next frame
|
|
130
|
+
// is due one interval from now.
|
|
131
|
+
this._nextTime = now + interval;
|
|
132
|
+
}
|
|
133
|
+
else {
|
|
134
|
+
// On schedule: one interval after this frame's scheduled time, or after
|
|
135
|
+
// its arrival if it came late.
|
|
136
|
+
this._nextTime = Math.max(this._nextTime, now - tolerance) + interval;
|
|
137
|
+
}
|
|
138
|
+
return [true, request];
|
|
139
|
+
}
|
|
140
|
+
}
|
|
141
|
+
/** An input transport's video samplers, by participant and video source. */
|
|
142
|
+
export class _VideoInSamplers {
|
|
143
|
+
// Python keys a dict by the (participant_id, video_source) tuple. A Map keeps
|
|
144
|
+
// the same insertion order; the entry carries both parts of the key.
|
|
145
|
+
_samplers = new Map();
|
|
146
|
+
static #key(participantId, videoSource) {
|
|
147
|
+
return JSON.stringify([participantId, videoSource]);
|
|
148
|
+
}
|
|
149
|
+
#get(participantId, videoSource) {
|
|
150
|
+
return this._samplers.get(_a.#key(participantId, videoSource))?.[2];
|
|
151
|
+
}
|
|
152
|
+
/**
|
|
153
|
+
* Start sampling a source, or change its framerate.
|
|
154
|
+
*
|
|
155
|
+
* Changing the framerate keeps the source's sampler, so image requests
|
|
156
|
+
* already waiting on it are still answered.
|
|
157
|
+
*
|
|
158
|
+
* @param participantId The participant whose video this is.
|
|
159
|
+
* @param videoSource The video source, e.g. `"camera"` or `"screenVideo"`.
|
|
160
|
+
* @param framerate Frames per second to pass on. `0` passes on frames only
|
|
161
|
+
* to answer requests, and `null` passes on every frame.
|
|
162
|
+
*/
|
|
163
|
+
capture(participantId, videoSource, framerate) {
|
|
164
|
+
const sampler = this.#get(participantId, videoSource);
|
|
165
|
+
if (sampler) {
|
|
166
|
+
sampler.framerate = framerate;
|
|
167
|
+
}
|
|
168
|
+
else {
|
|
169
|
+
this._samplers.set(_a.#key(participantId, videoSource), [
|
|
170
|
+
participantId,
|
|
171
|
+
videoSource,
|
|
172
|
+
new _VideoInSampler(framerate),
|
|
173
|
+
]);
|
|
174
|
+
}
|
|
175
|
+
}
|
|
176
|
+
/**
|
|
177
|
+
* Whether a source is being sampled.
|
|
178
|
+
*
|
|
179
|
+
* @param participantId The participant whose video this is.
|
|
180
|
+
* @param videoSource The video source.
|
|
181
|
+
* @returns Whether `capture()` was called for the source.
|
|
182
|
+
*/
|
|
183
|
+
isCapturing(participantId, videoSource) {
|
|
184
|
+
return this._samplers.has(_a.#key(participantId, videoSource));
|
|
185
|
+
}
|
|
186
|
+
/**
|
|
187
|
+
* Queue an image request for the next frame of a source to answer.
|
|
188
|
+
*
|
|
189
|
+
* @param participantId The participant whose video is requested.
|
|
190
|
+
* @param videoSource The video source requested.
|
|
191
|
+
* @param request The image request to answer.
|
|
192
|
+
* @returns Whether the request was queued; `false` if the source isn't being
|
|
193
|
+
* sampled, so nothing would answer it.
|
|
194
|
+
*/
|
|
195
|
+
addRequest(participantId, videoSource, request) {
|
|
196
|
+
const sampler = this.#get(participantId, videoSource);
|
|
197
|
+
if (sampler)
|
|
198
|
+
sampler.addRequest(request);
|
|
199
|
+
return sampler !== undefined;
|
|
200
|
+
}
|
|
201
|
+
/**
|
|
202
|
+
* Decide whether to pass on the frame that just arrived from a source.
|
|
203
|
+
*
|
|
204
|
+
* @param participantId The participant the frame came from.
|
|
205
|
+
* @param videoSource The video source the frame came from.
|
|
206
|
+
* @returns A tuple of `[due, request]`, as from `_VideoInSampler.sample()`.
|
|
207
|
+
* A frame from a source that isn't being sampled is never due.
|
|
208
|
+
*/
|
|
209
|
+
sample(participantId, videoSource) {
|
|
210
|
+
const sampler = this.#get(participantId, videoSource);
|
|
211
|
+
return sampler ? sampler.sample() : [false, null];
|
|
212
|
+
}
|
|
213
|
+
/**
|
|
214
|
+
* Remove and return image requests waiting for a frame, keeping the samplers.
|
|
215
|
+
*
|
|
216
|
+
* @param participantId The participant whose requests to take.
|
|
217
|
+
* @param videoSource The video source whose requests to take, or `null` for
|
|
218
|
+
* all of the participant's sources.
|
|
219
|
+
* @returns The waiting requests.
|
|
220
|
+
*/
|
|
221
|
+
takeRequests(participantId, videoSource = null) {
|
|
222
|
+
const requests = [];
|
|
223
|
+
for (const [participant, source, sampler] of this._samplers.values()) {
|
|
224
|
+
if (participant === participantId && (videoSource === null || videoSource === source)) {
|
|
225
|
+
requests.push(...sampler.takeRequests());
|
|
226
|
+
}
|
|
227
|
+
}
|
|
228
|
+
return requests;
|
|
229
|
+
}
|
|
230
|
+
/**
|
|
231
|
+
* Stop sampling every video source of a participant.
|
|
232
|
+
*
|
|
233
|
+
* @param participantId The participant to forget, e.g. one who left.
|
|
234
|
+
* @returns The image requests that were waiting on the participant's video.
|
|
235
|
+
*/
|
|
236
|
+
removeParticipant(participantId) {
|
|
237
|
+
const requests = this.takeRequests(participantId);
|
|
238
|
+
for (const [key, [participant]] of [...this._samplers]) {
|
|
239
|
+
if (participant === participantId)
|
|
240
|
+
this._samplers.delete(key);
|
|
241
|
+
}
|
|
242
|
+
return requests;
|
|
243
|
+
}
|
|
244
|
+
/**
|
|
245
|
+
* Stop sampling every video source.
|
|
246
|
+
*
|
|
247
|
+
* @returns The image requests that were waiting on any of them.
|
|
248
|
+
*/
|
|
249
|
+
clear() {
|
|
250
|
+
const requests = [];
|
|
251
|
+
for (const [, , sampler] of this._samplers.values())
|
|
252
|
+
requests.push(...sampler.takeRequests());
|
|
253
|
+
this._samplers.clear();
|
|
254
|
+
return requests;
|
|
255
|
+
}
|
|
256
|
+
}
|
|
257
|
+
_a = _VideoInSamplers;
|
|
258
|
+
//# sourceMappingURL=video_in_sampler.js.map
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
// Copyright (c) 2024-2026, Daily
|
|
2
|
+
// Copyright (c) 2026, pipedog contributors
|
|
3
|
+
// SPDX-License-Identifier: BSD-2-Clause
|
|
4
|
+
//
|
|
5
|
+
// Ported from pipecat: src/pipecat/transports/vonage/__init__.py
|
|
6
|
+
// Upstream's __init__.py is empty: it only marks the package. The provider
|
|
7
|
+
// modules themselves live in their own packages (see porting/manifest.json).
|
|
8
|
+
export {};
|
|
9
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
// Copyright (c) 2024-2026, Daily
|
|
2
|
+
// Copyright (c) 2026, pipedog contributors
|
|
3
|
+
// SPDX-License-Identifier: BSD-2-Clause
|
|
4
|
+
//
|
|
5
|
+
// Ported from pipecat: src/pipecat/transports/websocket/__init__.py
|
|
6
|
+
// Upstream's __init__.py is empty: it only marks the package. The provider
|
|
7
|
+
// modules themselves live in their own packages (see porting/manifest.json).
|
|
8
|
+
export {};
|
|
9
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
// Copyright (c) 2024-2026, Daily
|
|
2
|
+
// Copyright (c) 2026, pipedog contributors
|
|
3
|
+
// SPDX-License-Identifier: BSD-2-Clause
|
|
4
|
+
//
|
|
5
|
+
// Ported from pipecat: src/pipecat/transports/whatsapp/__init__.py
|
|
6
|
+
// Upstream's __init__.py is empty: it only marks the package. The provider
|
|
7
|
+
// modules themselves live in their own packages (see porting/manifest.json).
|
|
8
|
+
export {};
|
|
9
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
import type { Language } from "../transcriptions/language.ts";
|
|
2
|
+
import { type Ctor } from "../utils/mixin.ts";
|
|
3
|
+
import type { EagerMatchPolicy } from "./user_stop/index.ts";
|
|
4
|
+
import { type UserTurnStrategies } from "./user_turn_strategies.ts";
|
|
5
|
+
/** Options read by EagerEndOfTurnSTTServiceMixin from the constructor options. */
|
|
6
|
+
export interface EagerEndOfTurnSTTServiceMixinOptions {
|
|
7
|
+
/**
|
|
8
|
+
* Whether to answer a predicted end of turn ahead of the committed one.
|
|
9
|
+
* Off by default: it spends an inference on every prediction, including the
|
|
10
|
+
* ones the service withdraws.
|
|
11
|
+
*/
|
|
12
|
+
enableEagerEndOfTurn?: boolean;
|
|
13
|
+
/**
|
|
14
|
+
* Decides whether the committed transcript is close enough to the predicted
|
|
15
|
+
* one to keep the speculative response. Only consulted while eager end of
|
|
16
|
+
* turn is on. See EagerUserTurnStrategies.
|
|
17
|
+
*/
|
|
18
|
+
eagerMatchPolicy?: EagerMatchPolicy | null;
|
|
19
|
+
}
|
|
20
|
+
/** Members added by EagerEndOfTurnSTTServiceMixin. */
|
|
21
|
+
export interface EagerEndOfTurnSTTServiceMixin {
|
|
22
|
+
_enableEagerEndOfTurn: boolean;
|
|
23
|
+
_eagerMatchPolicy: EagerMatchPolicy | null;
|
|
24
|
+
_eagerEndOfTurnPending: boolean;
|
|
25
|
+
/** Whether predicted ends of turn are answered ahead of committed ones. */
|
|
26
|
+
readonly eagerEndOfTurnEnabled: boolean;
|
|
27
|
+
/** Whether a prediction is awaiting a committed end of turn. */
|
|
28
|
+
readonly eagerEndOfTurnPending: boolean;
|
|
29
|
+
recommendedUserTurnStrategies(options: {
|
|
30
|
+
enableInterruptions: boolean;
|
|
31
|
+
}): UserTurnStrategies;
|
|
32
|
+
_pushEagerEndOfTurn(transcript: string, options: {
|
|
33
|
+
userId: string;
|
|
34
|
+
language?: Language | null;
|
|
35
|
+
result?: unknown;
|
|
36
|
+
}): Promise<void>;
|
|
37
|
+
_cancelEagerEndOfTurn(): Promise<void>;
|
|
38
|
+
_clearEagerEndOfTurn(): void;
|
|
39
|
+
}
|
|
40
|
+
/**
|
|
41
|
+
* Adds eager end-of-turn signalling to an STT service that predicts one.
|
|
42
|
+
*
|
|
43
|
+
* Some services report that a turn has probably ended before committing to it,
|
|
44
|
+
* and withdraw the prediction if the user turns out to be mid-sentence. A
|
|
45
|
+
* response can be generated during that gap — see EagerUserTurnStopStrategy,
|
|
46
|
+
* which acts on the frames pushed here. At most one prediction is outstanding
|
|
47
|
+
* at a time, so the frames need not name which one they mean.
|
|
48
|
+
*
|
|
49
|
+
* A service drives the prediction through its own three lifecycle points:
|
|
50
|
+
*
|
|
51
|
+
* @example
|
|
52
|
+
* // the service predicts the turn has ended
|
|
53
|
+
* await this._pushEagerEndOfTurn(transcript, { userId: ..., language: ... });
|
|
54
|
+
*
|
|
55
|
+
* // the user resumed speaking, so the prediction is void
|
|
56
|
+
* await this._cancelEagerEndOfTurn();
|
|
57
|
+
*
|
|
58
|
+
* // the turn was committed, so the prediction is resolved
|
|
59
|
+
* this._clearEagerEndOfTurn();
|
|
60
|
+
*
|
|
61
|
+
* All three are safe to call when no prediction is outstanding, and all three
|
|
62
|
+
* are inert while `enableEagerEndOfTurn` is off, so a service can call them
|
|
63
|
+
* wherever its protocol reports a prediction without guarding each one.
|
|
64
|
+
*
|
|
65
|
+
* The base must be a FrameProcessor (upstream's mixin subclasses it).
|
|
66
|
+
* Upstream: `class XSTTService(EagerEndOfTurnSTTServiceMixin, STTService)`
|
|
67
|
+
* → `class XSTTService extends EagerEndOfTurnSTTServiceMixin(STTService)`.
|
|
68
|
+
*/
|
|
69
|
+
export declare const EagerEndOfTurnSTTServiceMixin: import("../utils/mixin.ts").Mixin<<B extends Ctor>(base: B) => B & Ctor<EagerEndOfTurnSTTServiceMixin>, EagerEndOfTurnSTTServiceMixin>;
|
|
70
|
+
//# sourceMappingURL=eager_end_of_turn_mixin.d.ts.map
|
|
@@ -0,0 +1,122 @@
|
|
|
1
|
+
// Copyright (c) 2024-2026, Daily
|
|
2
|
+
// Copyright (c) 2026, pipedog contributors
|
|
3
|
+
// SPDX-License-Identifier: BSD-2-Clause
|
|
4
|
+
//
|
|
5
|
+
// Ported from pipecat: src/pipecat/turns/eager_end_of_turn_mixin.py
|
|
6
|
+
/** Mixin for STT services that predict the end of a turn before committing to it. */
|
|
7
|
+
import { EagerEndOfTurnCancelFrame, EagerTranscriptionFrame } from "../frames/frames.js";
|
|
8
|
+
import { logger } from "../utils/logger.js";
|
|
9
|
+
import { defineMixin } from "../utils/mixin.js";
|
|
10
|
+
import { timeNowIso8601 } from "../utils/time.js";
|
|
11
|
+
import { EagerUserTurnStrategies, ExternalUserTurnStrategies, } from "./user_turn_strategies.js";
|
|
12
|
+
/**
|
|
13
|
+
* Adds eager end-of-turn signalling to an STT service that predicts one.
|
|
14
|
+
*
|
|
15
|
+
* Some services report that a turn has probably ended before committing to it,
|
|
16
|
+
* and withdraw the prediction if the user turns out to be mid-sentence. A
|
|
17
|
+
* response can be generated during that gap — see EagerUserTurnStopStrategy,
|
|
18
|
+
* which acts on the frames pushed here. At most one prediction is outstanding
|
|
19
|
+
* at a time, so the frames need not name which one they mean.
|
|
20
|
+
*
|
|
21
|
+
* A service drives the prediction through its own three lifecycle points:
|
|
22
|
+
*
|
|
23
|
+
* @example
|
|
24
|
+
* // the service predicts the turn has ended
|
|
25
|
+
* await this._pushEagerEndOfTurn(transcript, { userId: ..., language: ... });
|
|
26
|
+
*
|
|
27
|
+
* // the user resumed speaking, so the prediction is void
|
|
28
|
+
* await this._cancelEagerEndOfTurn();
|
|
29
|
+
*
|
|
30
|
+
* // the turn was committed, so the prediction is resolved
|
|
31
|
+
* this._clearEagerEndOfTurn();
|
|
32
|
+
*
|
|
33
|
+
* All three are safe to call when no prediction is outstanding, and all three
|
|
34
|
+
* are inert while `enableEagerEndOfTurn` is off, so a service can call them
|
|
35
|
+
* wherever its protocol reports a prediction without guarding each one.
|
|
36
|
+
*
|
|
37
|
+
* The base must be a FrameProcessor (upstream's mixin subclasses it).
|
|
38
|
+
* Upstream: `class XSTTService(EagerEndOfTurnSTTServiceMixin, STTService)`
|
|
39
|
+
* → `class XSTTService extends EagerEndOfTurnSTTServiceMixin(STTService)`.
|
|
40
|
+
*/
|
|
41
|
+
export const EagerEndOfTurnSTTServiceMixin = defineMixin("EagerEndOfTurnSTTServiceMixin", (Base) => {
|
|
42
|
+
const FrameProcessorBase = Base;
|
|
43
|
+
class EagerEndOfTurnSTTServiceMixinImpl extends FrameProcessorBase {
|
|
44
|
+
_enableEagerEndOfTurn;
|
|
45
|
+
_eagerMatchPolicy;
|
|
46
|
+
_eagerEndOfTurnPending = false;
|
|
47
|
+
constructor(...args) {
|
|
48
|
+
super(...args);
|
|
49
|
+
const options = (args[0] ?? {});
|
|
50
|
+
this._enableEagerEndOfTurn = options.enableEagerEndOfTurn ?? false;
|
|
51
|
+
this._eagerMatchPolicy = options.eagerMatchPolicy ?? null;
|
|
52
|
+
}
|
|
53
|
+
get eagerEndOfTurnEnabled() {
|
|
54
|
+
return this._enableEagerEndOfTurn;
|
|
55
|
+
}
|
|
56
|
+
get eagerEndOfTurnPending() {
|
|
57
|
+
return this._eagerEndOfTurnPending;
|
|
58
|
+
}
|
|
59
|
+
/**
|
|
60
|
+
* Build the turn strategies this service recommends.
|
|
61
|
+
*
|
|
62
|
+
* Both recommendations leave turn detection to the service. The eager one
|
|
63
|
+
* additionally answers a predicted end of turn, which only makes sense for
|
|
64
|
+
* a service configured to report one.
|
|
65
|
+
*
|
|
66
|
+
* @param options.enableInterruptions Whether a proposal starting a turn
|
|
67
|
+
* should interrupt the bot.
|
|
68
|
+
* @returns Strategies for the caller's `STTMetadataFrame`.
|
|
69
|
+
*/
|
|
70
|
+
recommendedUserTurnStrategies(options) {
|
|
71
|
+
if (!this._enableEagerEndOfTurn) {
|
|
72
|
+
return new ExternalUserTurnStrategies({
|
|
73
|
+
enableInterruptions: options.enableInterruptions,
|
|
74
|
+
});
|
|
75
|
+
}
|
|
76
|
+
return new EagerUserTurnStrategies({
|
|
77
|
+
matchPolicy: this._eagerMatchPolicy,
|
|
78
|
+
enableInterruptions: options.enableInterruptions,
|
|
79
|
+
});
|
|
80
|
+
}
|
|
81
|
+
/**
|
|
82
|
+
* Report that the turn has probably ended.
|
|
83
|
+
*
|
|
84
|
+
* @param transcript What the service heard for the turn so far.
|
|
85
|
+
* @param options.userId Identifier for the user who spoke.
|
|
86
|
+
* @param options.language Detected or specified language of the speech.
|
|
87
|
+
* @param options.result Raw result from the STT service.
|
|
88
|
+
*/
|
|
89
|
+
async _pushEagerEndOfTurn(transcript, options) {
|
|
90
|
+
if (!this._enableEagerEndOfTurn)
|
|
91
|
+
return;
|
|
92
|
+
this._eagerEndOfTurnPending = true;
|
|
93
|
+
logger.trace(`${this}: eager end of turn: [${transcript}]`);
|
|
94
|
+
await this.pushFrame(new EagerTranscriptionFrame({
|
|
95
|
+
text: transcript,
|
|
96
|
+
userId: options.userId,
|
|
97
|
+
timestamp: timeNowIso8601(),
|
|
98
|
+
language: options.language ?? null,
|
|
99
|
+
result: options.result ?? null,
|
|
100
|
+
}));
|
|
101
|
+
}
|
|
102
|
+
/** Withdraw the prediction. */
|
|
103
|
+
async _cancelEagerEndOfTurn() {
|
|
104
|
+
if (!this._eagerEndOfTurnPending)
|
|
105
|
+
return;
|
|
106
|
+
logger.trace(`${this}: eager end of turn withdrawn`);
|
|
107
|
+
this._eagerEndOfTurnPending = false;
|
|
108
|
+
await this.pushFrame(new EagerEndOfTurnCancelFrame());
|
|
109
|
+
}
|
|
110
|
+
/**
|
|
111
|
+
* Resolve the prediction without withdrawing it.
|
|
112
|
+
*
|
|
113
|
+
* Called when the turn is committed: whatever was generated from the
|
|
114
|
+
* prediction is settled by the committed transcript, not by this.
|
|
115
|
+
*/
|
|
116
|
+
_clearEagerEndOfTurn() {
|
|
117
|
+
this._eagerEndOfTurnPending = false;
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
return EagerEndOfTurnSTTServiceMixinImpl;
|
|
121
|
+
});
|
|
122
|
+
//# sourceMappingURL=eager_end_of_turn_mixin.js.map
|