@pipedog/core 0.0.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +25 -0
- package/NOTICE +9 -0
- package/dist/adapters/base_llm_adapter.d.ts +266 -0
- package/dist/adapters/base_llm_adapter.js +489 -0
- package/dist/adapters/index.d.ts +2 -0
- package/dist/adapters/index.js +8 -0
- package/dist/adapters/schemas/direct_function.d.ts +202 -0
- package/dist/adapters/schemas/direct_function.js +302 -0
- package/dist/adapters/schemas/function_schema.d.ts +67 -0
- package/dist/adapters/schemas/function_schema.js +72 -0
- package/dist/adapters/schemas/index.d.ts +2 -0
- package/dist/adapters/schemas/index.js +8 -0
- package/dist/adapters/schemas/tools_schema.d.ts +57 -0
- package/dist/adapters/schemas/tools_schema.js +79 -0
- package/dist/adapters/services/index.d.ts +2 -0
- package/dist/adapters/services/index.js +8 -0
- package/dist/adapters/services/open_ai_adapter.d.ts +147 -0
- package/dist/adapters/services/open_ai_adapter.js +250 -0
- package/dist/audio/dtmf/dtmf_data.d.ts +3 -0
- package/dist/audio/dtmf/dtmf_data.js +22 -0
- package/dist/audio/dtmf/index.d.ts +2 -0
- package/dist/audio/dtmf/index.js +8 -0
- package/dist/audio/dtmf/types.d.ts +31 -0
- package/dist/audio/dtmf/types.js +35 -0
- package/dist/audio/dtmf/utils.d.ts +13 -0
- package/dist/audio/dtmf/utils.js +62 -0
- package/dist/audio/filters/base_audio_filter.d.ts +51 -0
- package/dist/audio/filters/base_audio_filter.js +17 -0
- package/dist/audio/filters/index.d.ts +2 -0
- package/dist/audio/filters/index.js +8 -0
- package/dist/audio/index.d.ts +2 -0
- package/dist/audio/index.js +8 -0
- package/dist/audio/krisp_instance.d.ts +95 -0
- package/dist/audio/krisp_instance.js +229 -0
- package/dist/audio/mixers/base_audio_mixer.d.ts +56 -0
- package/dist/audio/mixers/base_audio_mixer.js +16 -0
- package/dist/audio/mixers/index.d.ts +2 -0
- package/dist/audio/mixers/index.js +8 -0
- package/dist/audio/resamplers/base_audio_resampler.d.ts +46 -0
- package/dist/audio/resamplers/base_audio_resampler.js +35 -0
- package/dist/audio/resamplers/engine.d.ts +60 -0
- package/dist/audio/resamplers/engine.js +89 -0
- package/dist/audio/resamplers/index.d.ts +2 -0
- package/dist/audio/resamplers/index.js +8 -0
- package/dist/audio/resamplers/resampy_resampler.d.ts +10 -0
- package/dist/audio/resamplers/resampy_resampler.js +15 -0
- package/dist/audio/resamplers/sinc_engine.d.ts +10 -0
- package/dist/audio/resamplers/sinc_engine.js +235 -0
- package/dist/audio/resamplers/soxr_resampler.d.ts +58 -0
- package/dist/audio/resamplers/soxr_resampler.js +79 -0
- package/dist/audio/resamplers/soxr_stream_resampler.d.ts +78 -0
- package/dist/audio/resamplers/soxr_stream_resampler.js +159 -0
- package/dist/audio/turn/base_turn_analyzer.d.ts +84 -0
- package/dist/audio/turn/base_turn_analyzer.js +61 -0
- package/dist/audio/turn/index.d.ts +2 -0
- package/dist/audio/turn/index.js +8 -0
- package/dist/audio/turn/smart_turn/_whisper_features.d.ts +30 -0
- package/dist/audio/turn/smart_turn/_whisper_features.js +225 -0
- package/dist/audio/turn/smart_turn/base_smart_turn.d.ts +94 -0
- package/dist/audio/turn/smart_turn/base_smart_turn.js +239 -0
- package/dist/audio/turn/smart_turn/local_smart_turn_v3.d.ts +76 -0
- package/dist/audio/turn/smart_turn/local_smart_turn_v3.js +151 -0
- package/dist/audio/utils.d.ts +224 -0
- package/dist/audio/utils.js +634 -0
- package/dist/audio/vad/data/index.d.ts +2 -0
- package/dist/audio/vad/data/index.js +8 -0
- package/dist/audio/vad/index.d.ts +2 -0
- package/dist/audio/vad/index.js +8 -0
- package/dist/audio/vad/vad_analyzer.d.ts +134 -0
- package/dist/audio/vad/vad_analyzer.js +249 -0
- package/dist/audio/vad/vad_controller.d.ts +143 -0
- package/dist/audio/vad/vad_controller.js +228 -0
- package/dist/audio/volume.d.ts +38 -0
- package/dist/audio/volume.js +75 -0
- package/dist/bus/adapters/base.d.ts +35 -0
- package/dist/bus/adapters/base.js +19 -0
- package/dist/bus/adapters/index.d.ts +10 -0
- package/dist/bus/adapters/index.js +15 -0
- package/dist/bus/adapters/llm_context_adapter.d.ts +36 -0
- package/dist/bus/adapters/llm_context_adapter.js +82 -0
- package/dist/bus/adapters/tools_schema_adapter.d.ts +25 -0
- package/dist/bus/adapters/tools_schema_adapter.js +70 -0
- package/dist/bus/bridge_processor.d.ts +111 -0
- package/dist/bus/bridge_processor.js +166 -0
- package/dist/bus/bus.d.ts +110 -0
- package/dist/bus/bus.js +200 -0
- package/dist/bus/index.d.ts +22 -0
- package/dist/bus/index.js +27 -0
- package/dist/bus/local/async_queue.d.ts +12 -0
- package/dist/bus/local/async_queue.js +21 -0
- package/dist/bus/local/index.d.ts +3 -0
- package/dist/bus/local/index.js +8 -0
- package/dist/bus/messages.d.ts +378 -0
- package/dist/bus/messages.js +357 -0
- package/dist/bus/network/index.d.ts +16 -0
- package/dist/bus/network/index.js +21 -0
- package/dist/bus/queue.d.ts +27 -0
- package/dist/bus/queue.js +51 -0
- package/dist/bus/serializers/base.d.ts +25 -0
- package/dist/bus/serializers/base.js +14 -0
- package/dist/bus/serializers/index.d.ts +9 -0
- package/dist/bus/serializers/index.js +14 -0
- package/dist/bus/serializers/json.d.ts +77 -0
- package/dist/bus/serializers/json.js +367 -0
- package/dist/bus/subscriber.d.ts +46 -0
- package/dist/bus/subscriber.js +27 -0
- package/dist/bus/ui/index.d.ts +3 -0
- package/dist/bus/ui/index.js +8 -0
- package/dist/bus/ui/messages.d.ts +186 -0
- package/dist/bus/ui/messages.js +171 -0
- package/dist/classifiers/base_classifier.d.ts +263 -0
- package/dist/classifiers/base_classifier.js +346 -0
- package/dist/classifiers/cloudflare/clef/classifier.d.ts +70 -0
- package/dist/classifiers/cloudflare/clef/classifier.js +204 -0
- package/dist/classifiers/cloudflare/clef/client.d.ts +104 -0
- package/dist/classifiers/cloudflare/clef/client.js +234 -0
- package/dist/classifiers/cloudflare/clef/index.d.ts +2 -0
- package/dist/classifiers/cloudflare/clef/index.js +8 -0
- package/dist/classifiers/cloudflare/index.d.ts +2 -0
- package/dist/classifiers/cloudflare/index.js +8 -0
- package/dist/classifiers/http_client.d.ts +70 -0
- package/dist/classifiers/http_client.js +161 -0
- package/dist/classifiers/index.d.ts +2 -0
- package/dist/classifiers/index.js +8 -0
- package/dist/classifiers/jev/classifier.d.ts +8 -0
- package/dist/classifiers/jev/classifier.js +13 -0
- package/dist/classifiers/jev/client.d.ts +8 -0
- package/dist/classifiers/jev/client.js +13 -0
- package/dist/classifiers/jev/index.d.ts +2 -0
- package/dist/classifiers/jev/index.js +8 -0
- package/dist/classifiers/llm/classifier.d.ts +72 -0
- package/dist/classifiers/llm/classifier.js +304 -0
- package/dist/classifiers/llm/index.d.ts +2 -0
- package/dist/classifiers/llm/index.js +8 -0
- package/dist/classifiers/openai/decisions/classifier.d.ts +77 -0
- package/dist/classifiers/openai/decisions/classifier.js +241 -0
- package/dist/classifiers/openai/decisions/client.d.ts +108 -0
- package/dist/classifiers/openai/decisions/client.js +234 -0
- package/dist/classifiers/openai/decisions/index.d.ts +2 -0
- package/dist/classifiers/openai/decisions/index.js +8 -0
- package/dist/classifiers/openai/index.d.ts +2 -0
- package/dist/classifiers/openai/index.js +8 -0
- package/dist/classifiers/typesafe/index.d.ts +2 -0
- package/dist/classifiers/typesafe/index.js +8 -0
- package/dist/classifiers/typesafe/jev/classifier.d.ts +70 -0
- package/dist/classifiers/typesafe/jev/classifier.js +220 -0
- package/dist/classifiers/typesafe/jev/client.d.ts +105 -0
- package/dist/classifiers/typesafe/jev/client.js +233 -0
- package/dist/classifiers/typesafe/jev/index.d.ts +2 -0
- package/dist/classifiers/typesafe/jev/index.js +8 -0
- package/dist/clocks/base_clock.d.ts +13 -0
- package/dist/clocks/base_clock.js +14 -0
- package/dist/clocks/index.d.ts +2 -0
- package/dist/clocks/index.js +8 -0
- package/dist/clocks/system_clock.d.ts +15 -0
- package/dist/clocks/system_clock.js +25 -0
- package/dist/frames/frames.d.ts +2533 -0
- package/dist/frames/frames.js +2485 -0
- package/dist/frames/index.d.ts +2 -0
- package/dist/frames/index.js +8 -0
- package/dist/frames/protobufs/frames_pb2.d.ts +229 -0
- package/dist/frames/protobufs/frames_pb2.js +466 -0
- package/dist/index.d.ts +11 -0
- package/dist/index.js +56 -0
- package/dist/metrics/index.d.ts +2 -0
- package/dist/metrics/index.js +8 -0
- package/dist/metrics/metrics.d.ts +276 -0
- package/dist/metrics/metrics.js +254 -0
- package/dist/observers/base_observer.d.ts +173 -0
- package/dist/observers/base_observer.js +151 -0
- package/dist/observers/error_observer.d.ts +75 -0
- package/dist/observers/error_observer.js +87 -0
- package/dist/observers/function_call_observer.d.ts +157 -0
- package/dist/observers/function_call_observer.js +222 -0
- package/dist/observers/index.d.ts +2 -0
- package/dist/observers/index.js +8 -0
- package/dist/observers/loggers/debug_log_observer.d.ts +91 -0
- package/dist/observers/loggers/debug_log_observer.js +228 -0
- package/dist/observers/loggers/index.d.ts +2 -0
- package/dist/observers/loggers/index.js +8 -0
- package/dist/observers/loggers/llm_log_observer.d.ts +24 -0
- package/dist/observers/loggers/llm_log_observer.js +110 -0
- package/dist/observers/loggers/metrics_log_observer.d.ts +85 -0
- package/dist/observers/loggers/metrics_log_observer.js +186 -0
- package/dist/observers/loggers/transcription_log_observer.d.ts +23 -0
- package/dist/observers/loggers/transcription_log_observer.js +74 -0
- package/dist/observers/service_metrics_observer.d.ts +168 -0
- package/dist/observers/service_metrics_observer.js +236 -0
- package/dist/observers/speaking_observer.d.ts +104 -0
- package/dist/observers/speaking_observer.js +150 -0
- package/dist/observers/startup_timing_observer.d.ts +221 -0
- package/dist/observers/startup_timing_observer.js +365 -0
- package/dist/observers/turn_tracking_observer.d.ts +78 -0
- package/dist/observers/turn_tracking_observer.js +189 -0
- package/dist/observers/user_bot_latency_observer.d.ts +411 -0
- package/dist/observers/user_bot_latency_observer.js +974 -0
- package/dist/pipeline/base_pipeline.d.ts +49 -0
- package/dist/pipeline/base_pipeline.js +101 -0
- package/dist/pipeline/capabilities.d.ts +76 -0
- package/dist/pipeline/capabilities.js +97 -0
- package/dist/pipeline/index.d.ts +2 -0
- package/dist/pipeline/index.js +8 -0
- package/dist/pipeline/job_context.d.ts +317 -0
- package/dist/pipeline/job_context.js +450 -0
- package/dist/pipeline/job_decorator.d.ts +54 -0
- package/dist/pipeline/job_decorator.js +92 -0
- package/dist/pipeline/llm_switcher.d.ts +84 -0
- package/dist/pipeline/llm_switcher.js +107 -0
- package/dist/pipeline/llm_with_backend.d.ts +435 -0
- package/dist/pipeline/llm_with_backend.js +806 -0
- package/dist/pipeline/parallel_pipeline.d.ts +83 -0
- package/dist/pipeline/parallel_pipeline.js +200 -0
- package/dist/pipeline/pipeline.d.ts +126 -0
- package/dist/pipeline/pipeline.js +196 -0
- package/dist/pipeline/service_switcher.d.ts +287 -0
- package/dist/pipeline/service_switcher.js +509 -0
- package/dist/pipeline/sync_parallel_pipeline.d.ts +192 -0
- package/dist/pipeline/sync_parallel_pipeline.js +387 -0
- package/dist/pipeline/worker.d.ts +821 -0
- package/dist/pipeline/worker.js +1656 -0
- package/dist/pipeline/worker_observer.d.ts +131 -0
- package/dist/pipeline/worker_observer.js +274 -0
- package/dist/pipeline/worker_ready_decorator.d.ts +45 -0
- package/dist/pipeline/worker_ready_decorator.js +76 -0
- package/dist/processors/aggregators/async_tool_messages.d.ts +146 -0
- package/dist/processors/aggregators/async_tool_messages.js +291 -0
- package/dist/processors/aggregators/dtmf_aggregator.d.ts +61 -0
- package/dist/processors/aggregators/dtmf_aggregator.js +144 -0
- package/dist/processors/aggregators/gated.d.ts +43 -0
- package/dist/processors/aggregators/gated.js +86 -0
- package/dist/processors/aggregators/gated_llm_context.d.ts +42 -0
- package/dist/processors/aggregators/gated_llm_context.js +86 -0
- package/dist/processors/aggregators/index.d.ts +2 -0
- package/dist/processors/aggregators/index.js +8 -0
- package/dist/processors/aggregators/llm_context.d.ts +370 -0
- package/dist/processors/aggregators/llm_context.js +580 -0
- package/dist/processors/aggregators/llm_context_summarizer.d.ts +219 -0
- package/dist/processors/aggregators/llm_context_summarizer.js +447 -0
- package/dist/processors/aggregators/llm_response.d.ts +37 -0
- package/dist/processors/aggregators/llm_response.js +71 -0
- package/dist/processors/aggregators/llm_response_universal.d.ts +966 -0
- package/dist/processors/aggregators/llm_response_universal.js +2394 -0
- package/dist/processors/aggregators/llm_text_processor.d.ts +53 -0
- package/dist/processors/aggregators/llm_text_processor.js +102 -0
- package/dist/processors/aggregators/sentence.d.ts +39 -0
- package/dist/processors/aggregators/sentence.js +69 -0
- package/dist/processors/async_generator.d.ts +42 -0
- package/dist/processors/async_generator.js +76 -0
- package/dist/processors/audio/audio_buffer_processor.d.ts +241 -0
- package/dist/processors/audio/audio_buffer_processor.js +565 -0
- package/dist/processors/audio/index.d.ts +2 -0
- package/dist/processors/audio/index.js +8 -0
- package/dist/processors/audio/vad_processor.d.ts +67 -0
- package/dist/processors/audio/vad_processor.js +103 -0
- package/dist/processors/consumer_processor.d.ts +49 -0
- package/dist/processors/consumer_processor.js +85 -0
- package/dist/processors/filters/frame_filter.d.ts +39 -0
- package/dist/processors/filters/frame_filter.js +52 -0
- package/dist/processors/filters/function_filter.d.ts +59 -0
- package/dist/processors/filters/function_filter.js +78 -0
- package/dist/processors/filters/identity_filter.d.ts +33 -0
- package/dist/processors/filters/identity_filter.js +37 -0
- package/dist/processors/filters/index.d.ts +2 -0
- package/dist/processors/filters/index.js +8 -0
- package/dist/processors/filters/null_filter.d.ts +34 -0
- package/dist/processors/filters/null_filter.js +47 -0
- package/dist/processors/filters/wake_notifier_filter.d.ts +48 -0
- package/dist/processors/filters/wake_notifier_filter.js +49 -0
- package/dist/processors/frame_processor.d.ts +360 -0
- package/dist/processors/frame_processor.js +932 -0
- package/dist/processors/frameworks/index.d.ts +2 -0
- package/dist/processors/frameworks/index.js +8 -0
- package/dist/processors/frameworks/rtvi/frames.d.ts +241 -0
- package/dist/processors/frameworks/rtvi/frames.js +259 -0
- package/dist/processors/frameworks/rtvi/index.d.ts +6 -0
- package/dist/processors/frameworks/rtvi/index.js +11 -0
- package/dist/processors/frameworks/rtvi/models.d.ts +1741 -0
- package/dist/processors/frameworks/rtvi/models.js +1807 -0
- package/dist/processors/frameworks/rtvi/observer.d.ts +340 -0
- package/dist/processors/frameworks/rtvi/observer.js +1113 -0
- package/dist/processors/frameworks/rtvi/processor.d.ts +182 -0
- package/dist/processors/frameworks/rtvi/processor.js +676 -0
- package/dist/processors/idle_frame_processor.d.ts +44 -0
- package/dist/processors/idle_frame_processor.js +86 -0
- package/dist/processors/index.d.ts +2 -0
- package/dist/processors/index.js +8 -0
- package/dist/processors/logger.d.ts +42 -0
- package/dist/processors/logger.js +87 -0
- package/dist/processors/metrics/frame_processor_metrics.d.ts +172 -0
- package/dist/processors/metrics/frame_processor_metrics.js +372 -0
- package/dist/processors/metrics/index.d.ts +2 -0
- package/dist/processors/metrics/index.js +8 -0
- package/dist/processors/producer_processor.d.ts +68 -0
- package/dist/processors/producer_processor.js +78 -0
- package/dist/processors/text_transformer.d.ts +30 -0
- package/dist/processors/text_transformer.js +48 -0
- package/dist/registry/index.d.ts +3 -0
- package/dist/registry/index.js +8 -0
- package/dist/registry/registry.d.ts +69 -0
- package/dist/registry/registry.js +149 -0
- package/dist/registry/types.d.ts +47 -0
- package/dist/registry/types.js +39 -0
- package/dist/serializers/base_serializer.d.ts +94 -0
- package/dist/serializers/base_serializer.js +82 -0
- package/dist/serializers/index.d.ts +2 -0
- package/dist/serializers/index.js +8 -0
- package/dist/services/ai_service.d.ts +145 -0
- package/dist/services/ai_service.js +262 -0
- package/dist/services/anthropic/index.d.ts +2 -0
- package/dist/services/anthropic/index.js +7 -0
- package/dist/services/assemblyai/index.d.ts +2 -0
- package/dist/services/assemblyai/index.js +7 -0
- package/dist/services/asyncai/index.d.ts +2 -0
- package/dist/services/asyncai/index.js +7 -0
- package/dist/services/aws/index.d.ts +2 -0
- package/dist/services/aws/index.js +7 -0
- package/dist/services/azure/index.d.ts +2 -0
- package/dist/services/azure/index.js +7 -0
- package/dist/services/baseten/index.d.ts +2 -0
- package/dist/services/baseten/index.js +7 -0
- package/dist/services/bland/index.d.ts +2 -0
- package/dist/services/bland/index.js +7 -0
- package/dist/services/camb/index.d.ts +2 -0
- package/dist/services/camb/index.js +7 -0
- package/dist/services/cartesia/index.d.ts +2 -0
- package/dist/services/cartesia/index.js +7 -0
- package/dist/services/cerebras/index.d.ts +2 -0
- package/dist/services/cerebras/index.js +7 -0
- package/dist/services/crusoe/index.d.ts +2 -0
- package/dist/services/crusoe/index.js +7 -0
- package/dist/services/deepgram/index.d.ts +2 -0
- package/dist/services/deepgram/index.js +7 -0
- package/dist/services/deepseek/index.d.ts +2 -0
- package/dist/services/deepseek/index.js +7 -0
- package/dist/services/elevenlabs/index.d.ts +2 -0
- package/dist/services/elevenlabs/index.js +7 -0
- package/dist/services/fal/index.d.ts +2 -0
- package/dist/services/fal/index.js +7 -0
- package/dist/services/fireworks/index.d.ts +2 -0
- package/dist/services/fireworks/index.js +7 -0
- package/dist/services/fish/index.d.ts +2 -0
- package/dist/services/fish/index.js +7 -0
- package/dist/services/funasr/index.d.ts +3 -0
- package/dist/services/funasr/index.js +8 -0
- package/dist/services/gladia/index.d.ts +2 -0
- package/dist/services/gladia/index.js +7 -0
- package/dist/services/google/index.d.ts +2 -0
- package/dist/services/google/index.js +7 -0
- package/dist/services/gradium/index.d.ts +2 -0
- package/dist/services/gradium/index.js +7 -0
- package/dist/services/grok/index.d.ts +2 -0
- package/dist/services/grok/index.js +7 -0
- package/dist/services/groq/index.d.ts +2 -0
- package/dist/services/groq/index.js +7 -0
- package/dist/services/heygen/index.d.ts +2 -0
- package/dist/services/heygen/index.js +7 -0
- package/dist/services/hume/index.d.ts +2 -0
- package/dist/services/hume/index.js +7 -0
- package/dist/services/image_service.d.ts +52 -0
- package/dist/services/image_service.js +60 -0
- package/dist/services/inception/index.d.ts +2 -0
- package/dist/services/inception/index.js +7 -0
- package/dist/services/index.d.ts +2 -0
- package/dist/services/index.js +8 -0
- package/dist/services/inworld/index.d.ts +2 -0
- package/dist/services/inworld/index.js +7 -0
- package/dist/services/keenable/index.d.ts +2 -0
- package/dist/services/keenable/index.js +7 -0
- package/dist/services/kokoro/index.d.ts +2 -0
- package/dist/services/kokoro/index.js +7 -0
- package/dist/services/llm_service.d.ts +1106 -0
- package/dist/services/llm_service.js +2231 -0
- package/dist/services/mem0/index.d.ts +2 -0
- package/dist/services/mem0/index.js +7 -0
- package/dist/services/meta/index.d.ts +2 -0
- package/dist/services/meta/index.js +7 -0
- package/dist/services/minimax/index.d.ts +2 -0
- package/dist/services/minimax/index.js +7 -0
- package/dist/services/mistral/index.d.ts +2 -0
- package/dist/services/mistral/index.js +7 -0
- package/dist/services/moondream/index.d.ts +2 -0
- package/dist/services/moondream/index.js +7 -0
- package/dist/services/moonshine/index.d.ts +2 -0
- package/dist/services/moonshine/index.js +7 -0
- package/dist/services/nebius/index.d.ts +2 -0
- package/dist/services/nebius/index.js +7 -0
- package/dist/services/neuphonic/index.d.ts +2 -0
- package/dist/services/neuphonic/index.js +7 -0
- package/dist/services/novita/index.d.ts +2 -0
- package/dist/services/novita/index.js +7 -0
- package/dist/services/nvidia/index.d.ts +2 -0
- package/dist/services/nvidia/index.js +11 -0
- package/dist/services/ollama/index.d.ts +2 -0
- package/dist/services/ollama/index.js +7 -0
- package/dist/services/openai/index.d.ts +2 -0
- package/dist/services/openai/index.js +7 -0
- package/dist/services/openclaw/index.d.ts +14 -0
- package/dist/services/openclaw/index.js +19 -0
- package/dist/services/openrouter/index.d.ts +2 -0
- package/dist/services/openrouter/index.js +7 -0
- package/dist/services/perplexity/index.d.ts +2 -0
- package/dist/services/perplexity/index.js +7 -0
- package/dist/services/piper/index.d.ts +2 -0
- package/dist/services/piper/index.js +7 -0
- package/dist/services/pocket_tts/index.d.ts +2 -0
- package/dist/services/pocket_tts/index.js +7 -0
- package/dist/services/qwen/index.d.ts +2 -0
- package/dist/services/qwen/index.js +7 -0
- package/dist/services/resembleai/index.d.ts +2 -0
- package/dist/services/resembleai/index.js +7 -0
- package/dist/services/rime/index.d.ts +2 -0
- package/dist/services/rime/index.js +7 -0
- package/dist/services/sambanova/index.d.ts +2 -0
- package/dist/services/sambanova/index.js +7 -0
- package/dist/services/sarvam/index.d.ts +2 -0
- package/dist/services/sarvam/index.js +7 -0
- package/dist/services/settings.d.ts +356 -0
- package/dist/services/settings.js +437 -0
- package/dist/services/simli/index.d.ts +2 -0
- package/dist/services/simli/index.js +7 -0
- package/dist/services/smallest/index.d.ts +2 -0
- package/dist/services/smallest/index.js +7 -0
- package/dist/services/soniox/index.d.ts +2 -0
- package/dist/services/soniox/index.js +7 -0
- package/dist/services/speechify/index.d.ts +2 -0
- package/dist/services/speechify/index.js +7 -0
- package/dist/services/speechmatics/index.d.ts +2 -0
- package/dist/services/speechmatics/index.js +7 -0
- package/dist/services/stt_latency.d.ts +62 -0
- package/dist/services/stt_latency.js +71 -0
- package/dist/services/stt_service.d.ts +601 -0
- package/dist/services/stt_service.js +1044 -0
- package/dist/services/tavus/index.d.ts +2 -0
- package/dist/services/tavus/index.js +7 -0
- package/dist/services/together/index.d.ts +2 -0
- package/dist/services/together/index.js +7 -0
- package/dist/services/tts_service.d.ts +909 -0
- package/dist/services/tts_service.js +2018 -0
- package/dist/services/ultravox/index.d.ts +2 -0
- package/dist/services/ultravox/index.js +9 -0
- package/dist/services/vision_service.d.ts +56 -0
- package/dist/services/vision_service.js +62 -0
- package/dist/services/websocket_service.d.ts +213 -0
- package/dist/services/websocket_service.js +654 -0
- package/dist/services/whisper/index.d.ts +2 -0
- package/dist/services/whisper/index.js +9 -0
- package/dist/services/xai/index.d.ts +2 -0
- package/dist/services/xai/index.js +9 -0
- package/dist/services/xtts/index.d.ts +2 -0
- package/dist/services/xtts/index.js +9 -0
- package/dist/tests/index.d.ts +2 -0
- package/dist/tests/index.js +8 -0
- package/dist/tests/utils.d.ts +123 -0
- package/dist/tests/utils.js +212 -0
- package/dist/transcriptions/index.d.ts +2 -0
- package/dist/transcriptions/index.js +8 -0
- package/dist/transcriptions/language.d.ts +376 -0
- package/dist/transcriptions/language.js +543 -0
- package/dist/transports/base_input.d.ts +149 -0
- package/dist/transports/base_input.js +338 -0
- package/dist/transports/base_output.d.ts +432 -0
- package/dist/transports/base_output.js +1231 -0
- package/dist/transports/base_transport.d.ts +223 -0
- package/dist/transports/base_transport.js +167 -0
- package/dist/transports/daily/index.d.ts +2 -0
- package/dist/transports/daily/index.js +9 -0
- package/dist/transports/heygen/index.d.ts +2 -0
- package/dist/transports/heygen/index.js +9 -0
- package/dist/transports/index.d.ts +2 -0
- package/dist/transports/index.js +8 -0
- package/dist/transports/lemonslice/index.d.ts +2 -0
- package/dist/transports/lemonslice/index.js +9 -0
- package/dist/transports/livekit/index.d.ts +2 -0
- package/dist/transports/livekit/index.js +9 -0
- package/dist/transports/local/index.d.ts +2 -0
- package/dist/transports/local/index.js +9 -0
- package/dist/transports/moq/index.d.ts +2 -0
- package/dist/transports/moq/index.js +9 -0
- package/dist/transports/smallwebrtc/index.d.ts +2 -0
- package/dist/transports/smallwebrtc/index.js +9 -0
- package/dist/transports/tavus/index.d.ts +2 -0
- package/dist/transports/tavus/index.js +9 -0
- package/dist/transports/video_in_sampler.d.ts +157 -0
- package/dist/transports/video_in_sampler.js +258 -0
- package/dist/transports/vonage/index.d.ts +2 -0
- package/dist/transports/vonage/index.js +9 -0
- package/dist/transports/websocket/index.d.ts +2 -0
- package/dist/transports/websocket/index.js +9 -0
- package/dist/transports/whatsapp/index.d.ts +2 -0
- package/dist/transports/whatsapp/index.js +9 -0
- package/dist/turns/eager_end_of_turn_mixin.d.ts +70 -0
- package/dist/turns/eager_end_of_turn_mixin.js +122 -0
- package/dist/turns/empty_user_turn.d.ts +55 -0
- package/dist/turns/empty_user_turn.js +53 -0
- package/dist/turns/index.d.ts +2 -0
- package/dist/turns/index.js +8 -0
- package/dist/turns/speculation_gate.d.ts +169 -0
- package/dist/turns/speculation_gate.js +294 -0
- package/dist/turns/types.d.ts +33 -0
- package/dist/turns/types.js +35 -0
- package/dist/turns/user_idle_controller.d.ts +94 -0
- package/dist/turns/user_idle_controller.js +187 -0
- package/dist/turns/user_mute/always_user_mute_strategy.d.ts +15 -0
- package/dist/turns/user_mute/always_user_mute_strategy.js +29 -0
- package/dist/turns/user_mute/base_user_mute_strategy.d.ts +39 -0
- package/dist/turns/user_mute/base_user_mute_strategy.js +45 -0
- package/dist/turns/user_mute/first_speech_user_mute_strategy.d.ts +28 -0
- package/dist/turns/user_mute/first_speech_user_mute_strategy.js +52 -0
- package/dist/turns/user_mute/function_call_user_mute_strategy.d.ts +25 -0
- package/dist/turns/user_mute/function_call_user_mute_strategy.js +50 -0
- package/dist/turns/user_mute/index.d.ts +6 -0
- package/dist/turns/user_mute/index.js +11 -0
- package/dist/turns/user_mute/mute_until_first_bot_complete_user_mute_strategy.d.ts +43 -0
- package/dist/turns/user_mute/mute_until_first_bot_complete_user_mute_strategy.js +76 -0
- package/dist/turns/user_start/base_user_turn_start_strategy.d.ts +135 -0
- package/dist/turns/user_start/base_user_turn_start_strategy.js +134 -0
- package/dist/turns/user_start/external_user_turn_start_strategy.d.ts +45 -0
- package/dist/turns/user_start/external_user_turn_start_strategy.js +64 -0
- package/dist/turns/user_start/index.d.ts +8 -0
- package/dist/turns/user_start/index.js +16 -0
- package/dist/turns/user_start/krisp_viva_ip_user_turn_start_strategy.d.ts +133 -0
- package/dist/turns/user_start/krisp_viva_ip_user_turn_start_strategy.js +243 -0
- package/dist/turns/user_start/min_words_user_turn_start_strategy.d.ts +60 -0
- package/dist/turns/user_start/min_words_user_turn_start_strategy.js +102 -0
- package/dist/turns/user_start/transcription_user_turn_start_strategy.d.ts +29 -0
- package/dist/turns/user_start/transcription_user_turn_start_strategy.js +43 -0
- package/dist/turns/user_start/vad_user_turn_start_strategy.d.ts +20 -0
- package/dist/turns/user_start/vad_user_turn_start_strategy.js +31 -0
- package/dist/turns/user_start/wake_phrase_user_turn_start_strategy.d.ts +141 -0
- package/dist/turns/user_start/wake_phrase_user_turn_start_strategy.js +238 -0
- package/dist/turns/user_stop/base_user_turn_stop_strategy.d.ts +174 -0
- package/dist/turns/user_stop/base_user_turn_stop_strategy.js +170 -0
- package/dist/turns/user_stop/deferred_user_turn_stop_strategy.d.ts +81 -0
- package/dist/turns/user_stop/deferred_user_turn_stop_strategy.js +107 -0
- package/dist/turns/user_stop/eager_match_policy.d.ts +36 -0
- package/dist/turns/user_stop/eager_match_policy.js +50 -0
- package/dist/turns/user_stop/eager_user_turn_stop_strategy.d.ts +84 -0
- package/dist/turns/user_stop/eager_user_turn_stop_strategy.js +154 -0
- package/dist/turns/user_stop/external_user_turn_completion_stop_strategy.d.ts +45 -0
- package/dist/turns/user_stop/external_user_turn_completion_stop_strategy.js +55 -0
- package/dist/turns/user_stop/external_user_turn_stop_strategy.d.ts +115 -0
- package/dist/turns/user_stop/external_user_turn_stop_strategy.js +205 -0
- package/dist/turns/user_stop/index.d.ts +10 -0
- package/dist/turns/user_stop/index.js +15 -0
- package/dist/turns/user_stop/llm_turn_completion_user_turn_stop_strategy.d.ts +44 -0
- package/dist/turns/user_stop/llm_turn_completion_user_turn_stop_strategy.js +59 -0
- package/dist/turns/user_stop/speech_timeout_user_turn_stop_strategy.d.ts +140 -0
- package/dist/turns/user_stop/speech_timeout_user_turn_stop_strategy.js +303 -0
- package/dist/turns/user_stop/turn_analyzer_user_turn_stop_strategy.d.ts +135 -0
- package/dist/turns/user_stop/turn_analyzer_user_turn_stop_strategy.js +332 -0
- package/dist/turns/user_turn_completion_mixin.d.ts +147 -0
- package/dist/turns/user_turn_completion_mixin.js +591 -0
- package/dist/turns/user_turn_controller.d.ts +134 -0
- package/dist/turns/user_turn_controller.js +335 -0
- package/dist/turns/user_turn_processor.d.ts +87 -0
- package/dist/turns/user_turn_processor.js +180 -0
- package/dist/turns/user_turn_strategies.d.ts +187 -0
- package/dist/turns/user_turn_strategies.js +222 -0
- package/dist/utils/async_tool_cancellation.d.ts +35 -0
- package/dist/utils/async_tool_cancellation.js +78 -0
- package/dist/utils/asyncio/index.d.ts +2 -0
- package/dist/utils/asyncio/index.js +7 -0
- package/dist/utils/asyncio/task_manager.d.ts +49 -0
- package/dist/utils/asyncio/task_manager.js +105 -0
- package/dist/utils/base_object.d.ts +76 -0
- package/dist/utils/base_object.js +154 -0
- package/dist/utils/context/aggregated_frame_sequencer.d.ts +491 -0
- package/dist/utils/context/aggregated_frame_sequencer.js +803 -0
- package/dist/utils/context/audio_context_manager.d.ts +131 -0
- package/dist/utils/context/audio_context_manager.js +233 -0
- package/dist/utils/context/index.d.ts +2 -0
- package/dist/utils/context/index.js +7 -0
- package/dist/utils/context/llm_context_summarization.d.ts +258 -0
- package/dist/utils/context/llm_context_summarization.js +511 -0
- package/dist/utils/context/text_segment_map.d.ts +611 -0
- package/dist/utils/context/text_segment_map.js +1132 -0
- package/dist/utils/context/word_completion_tracker.d.ts +273 -0
- package/dist/utils/context/word_completion_tracker.js +441 -0
- package/dist/utils/context/word_timestamp_tracker.d.ts +72 -0
- package/dist/utils/context/word_timestamp_tracker.js +98 -0
- package/dist/utils/deprecation.d.ts +28 -0
- package/dist/utils/deprecation.js +75 -0
- package/dist/utils/env.d.ts +38 -0
- package/dist/utils/env.js +62 -0
- package/dist/utils/errors.d.ts +75 -0
- package/dist/utils/errors.js +202 -0
- package/dist/utils/file_resolver.d.ts +126 -0
- package/dist/utils/file_resolver.js +314 -0
- package/dist/utils/file_storage.d.ts +115 -0
- package/dist/utils/file_storage.js +178 -0
- package/dist/utils/frame_queue.d.ts +36 -0
- package/dist/utils/frame_queue.js +66 -0
- package/dist/utils/http.d.ts +39 -0
- package/dist/utils/http.js +53 -0
- package/dist/utils/image.d.ts +14 -0
- package/dist/utils/image.js +63 -0
- package/dist/utils/index.d.ts +2 -0
- package/dist/utils/index.js +8 -0
- package/dist/utils/jpeg_encoder.d.ts +14 -0
- package/dist/utils/jpeg_encoder.js +315 -0
- package/dist/utils/log_config.d.ts +34 -0
- package/dist/utils/log_config.js +76 -0
- package/dist/utils/logger.d.ts +40 -0
- package/dist/utils/logger.js +91 -0
- package/dist/utils/mixin.d.ts +14 -0
- package/dist/utils/mixin.js +63 -0
- package/dist/utils/network.d.ts +73 -0
- package/dist/utils/network.js +90 -0
- package/dist/utils/security/allowed_origins.d.ts +16 -0
- package/dist/utils/security/allowed_origins.js +34 -0
- package/dist/utils/security/index.d.ts +2 -0
- package/dist/utils/security/index.js +7 -0
- package/dist/utils/security/redaction.d.ts +27 -0
- package/dist/utils/security/redaction.js +123 -0
- package/dist/utils/security/ssrf.d.ts +79 -0
- package/dist/utils/security/ssrf.js +327 -0
- package/dist/utils/shared.d.ts +35 -0
- package/dist/utils/shared.js +136 -0
- package/dist/utils/startup.d.ts +17 -0
- package/dist/utils/startup.js +130 -0
- package/dist/utils/string.d.ts +116 -0
- package/dist/utils/string.js +460 -0
- package/dist/utils/sync/base_notifier.d.ts +25 -0
- package/dist/utils/sync/base_notifier.js +16 -0
- package/dist/utils/sync/event_notifier.d.ts +38 -0
- package/dist/utils/sync/event_notifier.js +51 -0
- package/dist/utils/sync/index.d.ts +2 -0
- package/dist/utils/sync/index.js +7 -0
- package/dist/utils/text/alnum_utils.d.ts +109 -0
- package/dist/utils/text/alnum_utils.js +262 -0
- package/dist/utils/text/base_text_aggregator.d.ts +146 -0
- package/dist/utils/text/base_text_aggregator.js +102 -0
- package/dist/utils/text/base_text_filter.d.ts +54 -0
- package/dist/utils/text/base_text_filter.js +50 -0
- package/dist/utils/text/index.d.ts +2 -0
- package/dist/utils/text/index.js +7 -0
- package/dist/utils/text/markdown_text_filter.d.ts +147 -0
- package/dist/utils/text/markdown_text_filter.js +522 -0
- package/dist/utils/text/markup_utils.d.ts +54 -0
- package/dist/utils/text/markup_utils.js +156 -0
- package/dist/utils/text/pattern_pair_aggregator.d.ts +251 -0
- package/dist/utils/text/pattern_pair_aggregator.js +478 -0
- package/dist/utils/text/phonemes.d.ts +60 -0
- package/dist/utils/text/phonemes.js +172 -0
- package/dist/utils/text/simple_text_aggregator.d.ts +135 -0
- package/dist/utils/text/simple_text_aggregator.js +254 -0
- package/dist/utils/text/skip_tags_aggregator.d.ts +85 -0
- package/dist/utils/text/skip_tags_aggregator.js +147 -0
- package/dist/utils/text/transforms/acronyms.d.ts +21 -0
- package/dist/utils/text/transforms/acronyms.js +31 -0
- package/dist/utils/text/transforms/currency.d.ts +16 -0
- package/dist/utils/text/transforms/currency.js +56 -0
- package/dist/utils/text/transforms/dates.d.ts +18 -0
- package/dist/utils/text/transforms/dates.js +102 -0
- package/dist/utils/text/transforms/email.d.ts +17 -0
- package/dist/utils/text/transforms/email.js +35 -0
- package/dist/utils/text/transforms/index.d.ts +37 -0
- package/dist/utils/text/transforms/index.js +42 -0
- package/dist/utils/text/transforms/numbers.d.ts +25 -0
- package/dist/utils/text/transforms/numbers.js +59 -0
- package/dist/utils/text/transforms/percentages.d.ts +16 -0
- package/dist/utils/text/transforms/percentages.js +32 -0
- package/dist/utils/text/transforms/phone.d.ts +21 -0
- package/dist/utils/text/transforms/phone.js +39 -0
- package/dist/utils/text/transforms/pronunciations.d.ts +65 -0
- package/dist/utils/text/transforms/pronunciations.js +112 -0
- package/dist/utils/text/transforms/replacements.d.ts +39 -0
- package/dist/utils/text/transforms/replacements.js +46 -0
- package/dist/utils/text/transforms/strip_markdown.d.ts +25 -0
- package/dist/utils/text/transforms/strip_markdown.js +60 -0
- package/dist/utils/text/transforms/units.d.ts +22 -0
- package/dist/utils/text/transforms/units.js +81 -0
- package/dist/utils/text/transforms/voice_formatter.d.ts +86 -0
- package/dist/utils/text/transforms/voice_formatter.js +98 -0
- package/dist/utils/text/unicode_digits.d.ts +3 -0
- package/dist/utils/text/unicode_digits.js +34 -0
- package/dist/utils/text/word_timestamp_utils.d.ts +26 -0
- package/dist/utils/text/word_timestamp_utils.js +52 -0
- package/dist/utils/time.d.ts +13 -0
- package/dist/utils/time.js +31 -0
- package/dist/utils/tracing/index.d.ts +3 -0
- package/dist/utils/tracing/index.js +8 -0
- package/dist/utils/tracing/service_attributes.d.ts +198 -0
- package/dist/utils/tracing/service_attributes.js +434 -0
- package/dist/utils/tracing/service_decorators.d.ts +214 -0
- package/dist/utils/tracing/service_decorators.js +1662 -0
- package/dist/utils/tracing/setup.d.ts +107 -0
- package/dist/utils/tracing/setup.js +160 -0
- package/dist/utils/tracing/tracing_context.d.ts +55 -0
- package/dist/utils/tracing/tracing_context.js +97 -0
- package/dist/utils/tracing/turn_trace_observer.d.ts +95 -0
- package/dist/utils/tracing/turn_trace_observer.js +233 -0
- package/dist/utils/types.d.ts +80 -0
- package/dist/utils/types.js +108 -0
- package/dist/utils/utils.d.ts +5 -0
- package/dist/utils/utils.js +22 -0
- package/dist/utils/yaml.d.ts +179 -0
- package/dist/utils/yaml.js +556 -0
- package/dist/version.d.ts +3 -0
- package/dist/version.js +7 -0
- package/dist/workers/base_worker.d.ts +772 -0
- package/dist/workers/base_worker.js +1298 -0
- package/dist/workers/index.d.ts +2 -0
- package/dist/workers/index.js +8 -0
- package/dist/workers/llm/backend_llm_worker.d.ts +552 -0
- package/dist/workers/llm/backend_llm_worker.js +1055 -0
- package/dist/workers/llm/index.d.ts +6 -0
- package/dist/workers/llm/index.js +11 -0
- package/dist/workers/llm/llm_context_worker.d.ts +66 -0
- package/dist/workers/llm/llm_context_worker.js +72 -0
- package/dist/workers/llm/llm_worker.d.ts +180 -0
- package/dist/workers/llm/llm_worker.js +323 -0
- package/dist/workers/llm/tool_decorator.d.ts +68 -0
- package/dist/workers/llm/tool_decorator.js +60 -0
- package/dist/workers/proxy/index.d.ts +3 -0
- package/dist/workers/proxy/index.js +9 -0
- package/dist/workers/proxy/websocket/client.d.ts +110 -0
- package/dist/workers/proxy/websocket/client.js +196 -0
- package/dist/workers/proxy/websocket/index.d.ts +3 -0
- package/dist/workers/proxy/websocket/index.js +10 -0
- package/dist/workers/runner.d.ts +268 -0
- package/dist/workers/runner.js +489 -0
- package/dist/workers/ui/index.d.ts +22 -0
- package/dist/workers/ui/index.js +27 -0
- package/dist/workers/ui/ui_event_decorator.d.ts +43 -0
- package/dist/workers/ui/ui_event_decorator.js +75 -0
- package/dist/workers/ui/ui_job_context.d.ts +12 -0
- package/dist/workers/ui/ui_job_context.js +17 -0
- package/dist/workers/ui/ui_prompts.d.ts +21 -0
- package/dist/workers/ui/ui_prompts.js +41 -0
- package/dist/workers/ui/ui_tools.d.ts +32 -0
- package/dist/workers/ui/ui_tools.js +112 -0
- package/dist/workers/ui/ui_worker.d.ts +571 -0
- package/dist/workers/ui/ui_worker.js +1156 -0
- package/package.json +61 -3
- package/README.md +0 -5
|
@@ -0,0 +1,601 @@
|
|
|
1
|
+
/** Base classes for Speech-to-Text services with continuous and segmented processing. */
|
|
2
|
+
import { Queue, type Task } from "@pipedog/asyncio";
|
|
3
|
+
import { type CancelFrame, type EndFrame, type ErrorFrame, type Frame, InputAudioRawFrame, STTMetadataFrame, type StartFrame, VADUserStartedSpeakingFrame, VADUserStoppedSpeakingFrame } from "../frames/frames.ts";
|
|
4
|
+
import { FrameDirection, type FrameProcessorSetup } from "../processors/frame_processor.ts";
|
|
5
|
+
import { type Language } from "../transcriptions/language.ts";
|
|
6
|
+
import { AIService, type AIServiceOptions } from "./ai_service.ts";
|
|
7
|
+
import { STTSettings } from "./settings.ts";
|
|
8
|
+
import { WebsocketService, type WebsocketServiceOptions } from "./websocket_service.ts";
|
|
9
|
+
export interface STTServiceOptions extends AIServiceOptions {
|
|
10
|
+
/** Whether to pass audio frames downstream after processing. Defaults to true. */
|
|
11
|
+
audioPassthrough?: boolean;
|
|
12
|
+
/**
|
|
13
|
+
* The sample rate for audio input. If null, will be determined from the
|
|
14
|
+
* start frame.
|
|
15
|
+
*/
|
|
16
|
+
sampleRate?: number | null;
|
|
17
|
+
/**
|
|
18
|
+
* Time in seconds to wait after VAD stop before reporting TTFB. This delay
|
|
19
|
+
* allows the final transcription to arrive. Defaults to 2.0.
|
|
20
|
+
*/
|
|
21
|
+
sttTtfbTimeout?: number;
|
|
22
|
+
/**
|
|
23
|
+
* P99 latency from speech end to final transcript in seconds. Broadcast via
|
|
24
|
+
* STTMetadataFrame at pipeline start.
|
|
25
|
+
*/
|
|
26
|
+
ttfsP99Latency?: number | null;
|
|
27
|
+
/**
|
|
28
|
+
* Seconds of no audio before sending silence to keep the connection alive.
|
|
29
|
+
* null disables keepalive.
|
|
30
|
+
*/
|
|
31
|
+
keepaliveTimeout?: number | null;
|
|
32
|
+
/** Seconds between idle checks when keepalive is enabled. */
|
|
33
|
+
keepaliveInterval?: number;
|
|
34
|
+
/** The runtime-updatable settings for the STT service. */
|
|
35
|
+
settings?: STTSettings | null;
|
|
36
|
+
}
|
|
37
|
+
/**
|
|
38
|
+
* Base class for speech-to-text services.
|
|
39
|
+
*
|
|
40
|
+
* Provides common functionality for STT services including audio
|
|
41
|
+
* passthrough, muting, settings management, and audio processing. Subclasses
|
|
42
|
+
* must implement the `runStt` method to provide actual speech recognition.
|
|
43
|
+
*
|
|
44
|
+
* Includes an optional keepalive mechanism that sends silent audio when no
|
|
45
|
+
* real audio has been sent for a configurable timeout, preventing servers from
|
|
46
|
+
* closing idle connections (e.g. when behind a ServiceSwitcher). Subclasses
|
|
47
|
+
* that enable keepalive must override `_sendKeepalive()` to deliver the
|
|
48
|
+
* silence in the appropriate service-specific protocol.
|
|
49
|
+
*
|
|
50
|
+
* A streaming STT reports latency through TTFB — speech end to final
|
|
51
|
+
* transcript — and not through processing metrics. Audio arrives
|
|
52
|
+
* continuously, so there is no discrete request whose duration a
|
|
53
|
+
* `FrameProcessor.startProcessingMetrics()` window could measure; anchoring
|
|
54
|
+
* one to a speech or turn boundary measures how long the user talked.
|
|
55
|
+
* SegmentedSTTService does issue a discrete request per utterance, so its
|
|
56
|
+
* subclasses time that call and report both.
|
|
57
|
+
*
|
|
58
|
+
* Event handlers:
|
|
59
|
+
*
|
|
60
|
+
* - on_connected: called when connected to the STT service.
|
|
61
|
+
* - on_disconnected: called when disconnected from the STT service.
|
|
62
|
+
* - on_connection_error: called when a connection to the STT service error
|
|
63
|
+
* occurs.
|
|
64
|
+
*
|
|
65
|
+
* @example
|
|
66
|
+
* stt.addEventHandler("on_connected", async (stt: STTService) => {
|
|
67
|
+
* logger.debug("STT connected");
|
|
68
|
+
* });
|
|
69
|
+
*
|
|
70
|
+
* stt.addEventHandler("on_disconnected", async (stt: STTService) => {
|
|
71
|
+
* logger.debug("STT disconnected");
|
|
72
|
+
* });
|
|
73
|
+
*
|
|
74
|
+
* stt.addEventHandler("on_connection_error", async (stt: STTService, error: string) => {
|
|
75
|
+
* logger.error(`STT connection error: ${error}`);
|
|
76
|
+
* });
|
|
77
|
+
*/
|
|
78
|
+
export declare abstract class STTService extends AIService {
|
|
79
|
+
protected _settings: STTSettings;
|
|
80
|
+
protected _audioPassthrough: boolean;
|
|
81
|
+
protected _initSampleRate: number | null;
|
|
82
|
+
protected _sampleRate: number;
|
|
83
|
+
protected _muted: boolean;
|
|
84
|
+
protected _userId: string;
|
|
85
|
+
protected _ttfsP99Latency: number | null;
|
|
86
|
+
protected _sttTtfbTimeout: number;
|
|
87
|
+
protected _ttfbTimeoutTask: Task<unknown> | null;
|
|
88
|
+
protected _userSpeaking: boolean;
|
|
89
|
+
protected _finalizePending: boolean;
|
|
90
|
+
protected _finalizeRequested: boolean;
|
|
91
|
+
protected _lastTranscriptTime: number;
|
|
92
|
+
protected _keepaliveTimeout: number | null;
|
|
93
|
+
protected _keepaliveInterval: number;
|
|
94
|
+
protected _keepaliveTask: Task<unknown> | null;
|
|
95
|
+
protected _lastAudioTime: number;
|
|
96
|
+
protected _sttUsagePendingSeconds: number;
|
|
97
|
+
protected _canReconnect: boolean;
|
|
98
|
+
protected _needReconnect: boolean;
|
|
99
|
+
protected _reconnecting: boolean;
|
|
100
|
+
protected _reconnectAudioBuffer: [InputAudioRawFrame, FrameDirection][];
|
|
101
|
+
/**
|
|
102
|
+
* Initialize the STT service.
|
|
103
|
+
*
|
|
104
|
+
* @param options.audioPassthrough Whether to pass audio frames downstream
|
|
105
|
+
* after processing. Defaults to true.
|
|
106
|
+
* @param options.sampleRate The sample rate for audio input. If null, will
|
|
107
|
+
* be determined from the start frame.
|
|
108
|
+
* @param options.sttTtfbTimeout Time in seconds to wait after VAD stop
|
|
109
|
+
* before reporting TTFB. This delay allows the final transcription to
|
|
110
|
+
* arrive. Defaults to 2.0. Note: STT "TTFB" differs from traditional TTFB
|
|
111
|
+
* (which measures from a discrete request to first response byte). Since
|
|
112
|
+
* STT receives continuous audio, we measure from when the user stops
|
|
113
|
+
* speaking to when the final transcript arrives—capturing the latency that
|
|
114
|
+
* matters for voice AI applications.
|
|
115
|
+
* @param options.ttfsP99Latency P99 latency from speech end to final
|
|
116
|
+
* transcript in seconds. This is broadcast via STTMetadataFrame at
|
|
117
|
+
* pipeline start for downstream processors (e.g., turn strategies) to
|
|
118
|
+
* optimize timing. Subclasses provide measured defaults; pass a value here
|
|
119
|
+
* to override for your deployment. Turn-based services where the server
|
|
120
|
+
* defines turn boundaries should override `supportsTtfs` to return false
|
|
121
|
+
* instead of supplying a value.
|
|
122
|
+
* @param options.keepaliveTimeout Seconds of no audio before sending
|
|
123
|
+
* silence to keep the connection alive. null disables keepalive. Useful
|
|
124
|
+
* for services that close idle connections (e.g. behind a
|
|
125
|
+
* ServiceSwitcher).
|
|
126
|
+
* @param options.keepaliveInterval Seconds between idle checks when
|
|
127
|
+
* keepalive is enabled.
|
|
128
|
+
* @param options.settings The runtime-updatable settings for the STT
|
|
129
|
+
* service. Other options are passed to the parent AIService.
|
|
130
|
+
*/
|
|
131
|
+
constructor(options?: STTServiceOptions);
|
|
132
|
+
/** The service's current settings, for reading. */
|
|
133
|
+
get settings(): STTSettings;
|
|
134
|
+
/** Whether the STT service is currently muted and will not process audio. */
|
|
135
|
+
get isMuted(): boolean;
|
|
136
|
+
/**
|
|
137
|
+
* Mark that a finalize request has been sent, awaiting server confirmation.
|
|
138
|
+
*
|
|
139
|
+
* For providers that have explicit server confirmation of finalization
|
|
140
|
+
* (e.g., Deepgram's from_finalize field), call this when sending the
|
|
141
|
+
* finalize request. Then call `confirmFinalize()` when the server confirms.
|
|
142
|
+
*
|
|
143
|
+
* For providers without server confirmation, don't call this method - just
|
|
144
|
+
* send the finalize/flush/commit command and rely on the TTFB timeout.
|
|
145
|
+
*/
|
|
146
|
+
requestFinalize(): void;
|
|
147
|
+
/**
|
|
148
|
+
* Confirm that the server has acknowledged the finalize request.
|
|
149
|
+
*
|
|
150
|
+
* Call this when the server response confirms finalization (e.g.,
|
|
151
|
+
* Deepgram's from_finalize=True). The next TranscriptionFrame pushed will be
|
|
152
|
+
* marked as finalized.
|
|
153
|
+
*
|
|
154
|
+
* Only has effect if `requestFinalize()` was previously called.
|
|
155
|
+
*/
|
|
156
|
+
confirmFinalize(): void;
|
|
157
|
+
/** The current sample rate for audio processing, in Hz. */
|
|
158
|
+
get sampleRate(): number;
|
|
159
|
+
/**
|
|
160
|
+
* Accumulate client-measured audio seconds for usage metrics.
|
|
161
|
+
*
|
|
162
|
+
* @param audio Raw 16-bit mono PCM audio bytes submitted to the service.
|
|
163
|
+
*/
|
|
164
|
+
protected _recordSttAudioUsage(audio: Uint8Array): void;
|
|
165
|
+
/**
|
|
166
|
+
* Emit an incremental STT usage metrics event.
|
|
167
|
+
*
|
|
168
|
+
* Reports the audio seconds accumulated since the last report as an
|
|
169
|
+
* `STTUsageMetricsData` in a `MetricsFrame`. Services call this once per
|
|
170
|
+
* billing event (e.g. on each final transcript); the base class calls it a
|
|
171
|
+
* final time on stop/cancel to flush trailing audio. Does nothing if there
|
|
172
|
+
* is no usage to report.
|
|
173
|
+
*/
|
|
174
|
+
emitSttUsageMetrics(): Promise<void>;
|
|
175
|
+
/** Flush any trailing audio usage not yet reported by a billing event. */
|
|
176
|
+
protected _flushSttUsageMetrics(): Promise<void>;
|
|
177
|
+
/**
|
|
178
|
+
* Convert a language to the service-specific language format.
|
|
179
|
+
*
|
|
180
|
+
* @param language The language to convert.
|
|
181
|
+
* @returns The service-specific language identifier. Return `null` to
|
|
182
|
+
* indicate an unsupported language. This optional return is an extension
|
|
183
|
+
* hook for future or third-party subclasses; as of 2026-04-28,
|
|
184
|
+
* first-party services return a string.
|
|
185
|
+
*/
|
|
186
|
+
languageToServiceLanguage(language: Language): string | null;
|
|
187
|
+
/**
|
|
188
|
+
* Called when the assistant's turn completes with the aggregated reply text.
|
|
189
|
+
*
|
|
190
|
+
* Override in subclasses to react to each completed bot reply — for
|
|
191
|
+
* example, to feed the text to a provider-side context carryover API. The
|
|
192
|
+
* default implementation is a no-op.
|
|
193
|
+
*
|
|
194
|
+
* @param text The assistant's aggregated spoken text for this turn.
|
|
195
|
+
*/
|
|
196
|
+
protected _processAssistantTurn(_text: string): Promise<void>;
|
|
197
|
+
/**
|
|
198
|
+
* Run speech-to-text on the provided audio data.
|
|
199
|
+
*
|
|
200
|
+
* This method must be implemented by subclasses to provide actual speech
|
|
201
|
+
* recognition functionality.
|
|
202
|
+
*
|
|
203
|
+
* @param audio Raw audio bytes to transcribe.
|
|
204
|
+
* @returns An async generator yielding frames containing transcription
|
|
205
|
+
* results (typically TextFrame).
|
|
206
|
+
*/
|
|
207
|
+
abstract runStt(audio: Uint8Array): AsyncGenerator<Frame | null, void, unknown>;
|
|
208
|
+
/**
|
|
209
|
+
* Set up the service.
|
|
210
|
+
*
|
|
211
|
+
* @param setup Configuration object containing setup parameters.
|
|
212
|
+
*/
|
|
213
|
+
setup(setup: FrameProcessorSetup): Promise<void>;
|
|
214
|
+
/** Clean up STT service resources. */
|
|
215
|
+
cleanup(): Promise<void>;
|
|
216
|
+
/**
|
|
217
|
+
* Stop the STT service on a graceful end.
|
|
218
|
+
*
|
|
219
|
+
* @param frame The end frame.
|
|
220
|
+
*/
|
|
221
|
+
stop(frame: EndFrame): Promise<void>;
|
|
222
|
+
/**
|
|
223
|
+
* Cancel the STT service immediately.
|
|
224
|
+
*
|
|
225
|
+
* @param frame The cancel frame.
|
|
226
|
+
*/
|
|
227
|
+
cancel(frame: CancelFrame): Promise<void>;
|
|
228
|
+
/**
|
|
229
|
+
* Apply an STT settings delta.
|
|
230
|
+
*
|
|
231
|
+
* Handles `model` (via parent). Translates `Language` values before
|
|
232
|
+
* applying so the stored value is a service-specific string. Concrete
|
|
233
|
+
* services should override this method and handle language changes
|
|
234
|
+
* (including any reconnect logic) based on the returned changed-field
|
|
235
|
+
* record.
|
|
236
|
+
*
|
|
237
|
+
* @param delta An STT settings delta.
|
|
238
|
+
* @returns Record mapping changed field names to their previous values.
|
|
239
|
+
*/
|
|
240
|
+
protected _updateSettings(delta: STTSettings): Promise<Record<string, unknown>>;
|
|
241
|
+
/**
|
|
242
|
+
* Process an audio frame for speech recognition.
|
|
243
|
+
*
|
|
244
|
+
* If a reconnect is in progress, the frame is buffered and replayed once
|
|
245
|
+
* the connection is restored. If the service is muted, the frame is
|
|
246
|
+
* dropped. Otherwise the frame is sent to the STT service and, if a userId
|
|
247
|
+
* is present, it is stored for use in transcription results.
|
|
248
|
+
*
|
|
249
|
+
* @param frame The audio frame to process.
|
|
250
|
+
* @param direction The direction of frame processing.
|
|
251
|
+
*/
|
|
252
|
+
processAudioFrame(frame: InputAudioRawFrame, direction: FrameDirection): Promise<void>;
|
|
253
|
+
/**
|
|
254
|
+
* Process frames, handling VAD events and audio segmentation.
|
|
255
|
+
*
|
|
256
|
+
* @param frame The frame to process.
|
|
257
|
+
* @param direction The direction of frame processing.
|
|
258
|
+
*/
|
|
259
|
+
processFrame(frame: Frame, direction: FrameDirection): Promise<void>;
|
|
260
|
+
/**
|
|
261
|
+
* Push a frame downstream, tracking TranscriptionFrame timestamps for TTFB.
|
|
262
|
+
*
|
|
263
|
+
* Stores the timestamp of each TranscriptionFrame for TTFB calculation. If
|
|
264
|
+
* the frame is marked as finalized (via requestFinalize/confirmFinalize),
|
|
265
|
+
* reports TTFB immediately and cancels any pending timeout. Otherwise, TTFB
|
|
266
|
+
* is reported after a timeout.
|
|
267
|
+
*
|
|
268
|
+
* @param frame The frame to push.
|
|
269
|
+
* @param direction The direction to push the frame.
|
|
270
|
+
*/
|
|
271
|
+
pushFrame(frame: Frame, direction?: FrameDirection): Promise<void>;
|
|
272
|
+
/**
|
|
273
|
+
* Whether this STT service has a meaningful TTFS-to-final-transcript metric.
|
|
274
|
+
*
|
|
275
|
+
* Returns false for turn-based STTs where the server defines the turn
|
|
276
|
+
* boundary, so there is no separate "speech end → final transcript"
|
|
277
|
+
* interval to measure. Downstream turn-stop strategies that consume
|
|
278
|
+
* STTMetadataFrame treat a 0 latency as "no extra wait."
|
|
279
|
+
*/
|
|
280
|
+
get supportsTtfs(): boolean;
|
|
281
|
+
/**
|
|
282
|
+
* Build the STT metadata frame broadcast at start.
|
|
283
|
+
*
|
|
284
|
+
* Overrides `AIService.serviceMetadataFrame()` to return an
|
|
285
|
+
* STTMetadataFrame carrying the service's TTFS P99 latency. A service that
|
|
286
|
+
* does its own server-side end-of-turn detection overrides this (calling
|
|
287
|
+
* `super`) to set `userTurnStrategies` on the returned frame.
|
|
288
|
+
*/
|
|
289
|
+
serviceMetadataFrame(): STTMetadataFrame;
|
|
290
|
+
/** Cancel any pending TTFB timeout task. */
|
|
291
|
+
protected _cancelTtfbTimeout(): Promise<void>;
|
|
292
|
+
/**
|
|
293
|
+
* Reset STT TTFB measurement state.
|
|
294
|
+
*
|
|
295
|
+
* Called when starting a new utterance or on interruption to ensure we
|
|
296
|
+
* don't use stale state for TTFB calculations. This specifically guards
|
|
297
|
+
* against the case where a TranscriptionFrame is received without
|
|
298
|
+
* corresponding VADUserStartedSpeakingFrame and VADUserStoppedSpeakingFrame
|
|
299
|
+
* frames.
|
|
300
|
+
*
|
|
301
|
+
* Note: does not reset `_userSpeaking` since InterruptionFrame can arrive
|
|
302
|
+
* while user is still speaking.
|
|
303
|
+
*/
|
|
304
|
+
protected _resetSttTtfbState(): Promise<void>;
|
|
305
|
+
/**
|
|
306
|
+
* Handle VAD user started speaking frame to start tracking transcriptions.
|
|
307
|
+
*
|
|
308
|
+
* Cancels any pending TTFB timeout, resets TTFB tracking state, and marks
|
|
309
|
+
* user as speaking. Also resets finalization state to prevent stale
|
|
310
|
+
* finalization from a previous utterance.
|
|
311
|
+
*
|
|
312
|
+
* @param frame The VAD user started speaking frame.
|
|
313
|
+
*/
|
|
314
|
+
protected _handleVadUserStartedSpeaking(_frame: VADUserStartedSpeakingFrame): Promise<void>;
|
|
315
|
+
/**
|
|
316
|
+
* Check if reconnection is needed after the user stops speaking.
|
|
317
|
+
*
|
|
318
|
+
* Called when the user's full turn has ended and the transcription has been
|
|
319
|
+
* received. Re-enables reconnection and triggers any deferred reconnect
|
|
320
|
+
* that was requested while the user was speaking.
|
|
321
|
+
*/
|
|
322
|
+
protected _maybeReconnectOnUserStoppedSpeaking(): Promise<void>;
|
|
323
|
+
/**
|
|
324
|
+
* Handle VAD user stopped speaking frame.
|
|
325
|
+
*
|
|
326
|
+
* Calculates the actual speech end time and starts a timeout task to wait
|
|
327
|
+
* for the final transcription before reporting TTFB.
|
|
328
|
+
*
|
|
329
|
+
* @param frame The VAD user stopped speaking frame.
|
|
330
|
+
*/
|
|
331
|
+
protected _handleVadUserStoppedSpeaking(frame: VADUserStoppedSpeakingFrame): Promise<void>;
|
|
332
|
+
/**
|
|
333
|
+
* Wait for timeout then report TTFB using the last transcript timestamp.
|
|
334
|
+
*
|
|
335
|
+
* This timeout allows the final transcription to arrive before we
|
|
336
|
+
* calculate and report TTFB. Uses `_lastTranscriptTime` as the end time so
|
|
337
|
+
* we measure to when the transcript actually arrived, not when the timeout
|
|
338
|
+
* fired.
|
|
339
|
+
*
|
|
340
|
+
* A transcript that predates the end of speech belongs to an earlier
|
|
341
|
+
* segment the service finalized on its own endpointing; the metrics
|
|
342
|
+
* collector refuses it rather than report the negative interval it would
|
|
343
|
+
* produce.
|
|
344
|
+
*/
|
|
345
|
+
protected _ttfbTimeoutHandler(): Promise<void>;
|
|
346
|
+
/** Start the keepalive task if keepalive is enabled. */
|
|
347
|
+
protected _createKeepaliveTask(): void;
|
|
348
|
+
/** Stop the keepalive task if running. */
|
|
349
|
+
protected _cancelKeepaliveTask(): Promise<void>;
|
|
350
|
+
/**
|
|
351
|
+
* Perform a full reconnect cycle with audio buffering.
|
|
352
|
+
*
|
|
353
|
+
* Sets `_reconnecting` so incoming audio frames are buffered rather than
|
|
354
|
+
* sent to a dead connection. Delegates the actual connection reset to
|
|
355
|
+
* `_doReconnect()`. After the new connection is established all buffered
|
|
356
|
+
* frames are replayed. On failure the error is reported via `pushError`
|
|
357
|
+
* and the `on_connection_error` event handler.
|
|
358
|
+
*/
|
|
359
|
+
protected _reconnect(): Promise<void>;
|
|
360
|
+
/**
|
|
361
|
+
* Perform the service-specific connection reset.
|
|
362
|
+
*
|
|
363
|
+
* Called by `_reconnect()` inside the reconnecting guard. The default
|
|
364
|
+
* implementation is a no-op. Subclasses that support explicit reconnection
|
|
365
|
+
* (e.g. WebsocketSTTService) should override this to tear down and
|
|
366
|
+
* re-establish their connection.
|
|
367
|
+
*/
|
|
368
|
+
protected _doReconnect(): Promise<void>;
|
|
369
|
+
/**
|
|
370
|
+
* Reconnect immediately if safe, or defer until after the current user turn.
|
|
371
|
+
*
|
|
372
|
+
* Reconnection is unsafe while the user is speaking because the service is
|
|
373
|
+
* actively receiving audio. Calling this method while the user is speaking
|
|
374
|
+
* schedules a reconnect that fires as soon as `UserStoppedSpeakingFrame` is
|
|
375
|
+
* received.
|
|
376
|
+
*/
|
|
377
|
+
protected _requestReconnect(): Promise<void>;
|
|
378
|
+
/**
|
|
379
|
+
* Send periodic silent audio to prevent the server from closing the
|
|
380
|
+
* connection.
|
|
381
|
+
*
|
|
382
|
+
* When keepalive is enabled, this task checks periodically if the
|
|
383
|
+
* connection has been idle (no audio sent) for longer than
|
|
384
|
+
* keepaliveTimeout seconds. If so, it generates silent 16-bit mono PCM
|
|
385
|
+
* audio and passes it to `_sendKeepalive()` for service-specific formatting
|
|
386
|
+
* and sending. A failed send is logged and the loop keeps going.
|
|
387
|
+
*/
|
|
388
|
+
protected _keepaliveTaskHandler(): Promise<void>;
|
|
389
|
+
/**
|
|
390
|
+
* Check if the service is ready to send keepalive.
|
|
391
|
+
*
|
|
392
|
+
* Subclasses should override this to check their connection state.
|
|
393
|
+
*
|
|
394
|
+
* @returns True if keepalive can be sent.
|
|
395
|
+
*/
|
|
396
|
+
protected _isKeepaliveReady(): boolean;
|
|
397
|
+
/**
|
|
398
|
+
* Send silent audio to keep the connection alive.
|
|
399
|
+
*
|
|
400
|
+
* Subclasses that enable keepalive must override this to deliver silence in
|
|
401
|
+
* their service-specific protocol.
|
|
402
|
+
*
|
|
403
|
+
* @param silence Silent 16-bit mono PCM audio bytes.
|
|
404
|
+
*/
|
|
405
|
+
protected _sendKeepalive(_silence: Uint8Array): Promise<void>;
|
|
406
|
+
}
|
|
407
|
+
export interface SegmentedSTTServiceOptions extends STTServiceOptions {
|
|
408
|
+
/**
|
|
409
|
+
* Seconds of silence appended to each segment before it is transcribed, so
|
|
410
|
+
* the model hears the end of speech and can finish the last word. Set to 0
|
|
411
|
+
* to disable. Defaults to 0.5.
|
|
412
|
+
*/
|
|
413
|
+
trailingSilenceSecs?: number;
|
|
414
|
+
}
|
|
415
|
+
/**
|
|
416
|
+
* STT service that processes speech in segments using VAD events.
|
|
417
|
+
*
|
|
418
|
+
* Uses Voice Activity Detection (VAD) events to detect speech segments and
|
|
419
|
+
* runs speech-to-text only on those segments, rather than continuously.
|
|
420
|
+
*
|
|
421
|
+
* Requires VAD to be enabled in the pipeline to function properly. Maintains
|
|
422
|
+
* a small audio buffer to account for the delay between actual speech start
|
|
423
|
+
* and VAD detection.
|
|
424
|
+
*
|
|
425
|
+
* The buffered segment is passed to `runStt` as a WAV container by default,
|
|
426
|
+
* which is what cloud providers want for their upload APIs. Local models that
|
|
427
|
+
* consume raw 16-bit PCM directly override `wantsWavSegments` to return false
|
|
428
|
+
* so they receive the unwrapped buffer instead. This is a subclass-level
|
|
429
|
+
* contract, not a user-configurable option: the format is dictated by what
|
|
430
|
+
* the model expects.
|
|
431
|
+
*
|
|
432
|
+
* A segment ends right where the VAD stopped, and models tend to drop or
|
|
433
|
+
* garble the final word when the audio ends that abruptly, so each segment is
|
|
434
|
+
* padded with `trailingSilenceSecs` of silence before transcription.
|
|
435
|
+
*
|
|
436
|
+
* Transcription runs off the audio path. When the VAD stops, the segment is
|
|
437
|
+
* queued and one background task transcribes queued segments in order,
|
|
438
|
+
* pushing each transcript as it completes; meanwhile audio and other frames
|
|
439
|
+
* keep flowing through the service, as they do through a streaming STT whose
|
|
440
|
+
* transcripts arrive asynchronously. A graceful stop transcribes what is
|
|
441
|
+
* queued before the service stops; a cancel drops it.
|
|
442
|
+
*/
|
|
443
|
+
export declare abstract class SegmentedSTTService extends STTService {
|
|
444
|
+
protected _trailingSilenceSecs: number;
|
|
445
|
+
protected _content: unknown;
|
|
446
|
+
protected _wave: unknown;
|
|
447
|
+
protected _audioBuffer: Uint8Array;
|
|
448
|
+
protected _audioBufferSize1s: number;
|
|
449
|
+
protected _segmentQueue: Queue<Uint8Array<ArrayBufferLike> | null>;
|
|
450
|
+
protected _segmentTask: Task<unknown> | null;
|
|
451
|
+
/**
|
|
452
|
+
* Initialize the segmented STT service.
|
|
453
|
+
*
|
|
454
|
+
* @param options.sampleRate The sample rate for audio input. If null, will
|
|
455
|
+
* be determined from the start frame.
|
|
456
|
+
* @param options.trailingSilenceSecs Seconds of silence appended to each
|
|
457
|
+
* segment before it is transcribed, so the model hears the end of speech
|
|
458
|
+
* and can finish the last word. Set to 0 to disable. Other options are
|
|
459
|
+
* passed to the parent STTService.
|
|
460
|
+
*/
|
|
461
|
+
constructor(options?: SegmentedSTTServiceOptions);
|
|
462
|
+
/**
|
|
463
|
+
* Set up the service.
|
|
464
|
+
*
|
|
465
|
+
* @param setup Configuration object containing setup parameters.
|
|
466
|
+
*/
|
|
467
|
+
setup(setup: FrameProcessorSetup): Promise<void>;
|
|
468
|
+
/**
|
|
469
|
+
* Start the service and its transcription task.
|
|
470
|
+
*
|
|
471
|
+
* @param frame The start frame.
|
|
472
|
+
*/
|
|
473
|
+
start(frame: StartFrame): Promise<void>;
|
|
474
|
+
/**
|
|
475
|
+
* Transcribe the queued segments, then stop the service.
|
|
476
|
+
*
|
|
477
|
+
* @param frame The end frame.
|
|
478
|
+
*/
|
|
479
|
+
stop(frame: EndFrame): Promise<void>;
|
|
480
|
+
/**
|
|
481
|
+
* Stop the service at once, dropping any queued segments.
|
|
482
|
+
*
|
|
483
|
+
* @param frame The cancel frame.
|
|
484
|
+
*/
|
|
485
|
+
cancel(frame: CancelFrame): Promise<void>;
|
|
486
|
+
/**
|
|
487
|
+
* Whether segments are passed to `runStt` as a WAV container.
|
|
488
|
+
*
|
|
489
|
+
* Returns true (the default) for cloud providers whose upload APIs expect a
|
|
490
|
+
* WAV file. Local models that read the buffer as raw 16-bit PCM override
|
|
491
|
+
* this to return false; otherwise they would misinterpret the 44-byte WAV
|
|
492
|
+
* header as audio samples.
|
|
493
|
+
*/
|
|
494
|
+
get wantsWavSegments(): boolean;
|
|
495
|
+
/**
|
|
496
|
+
* Push a frame, marking TranscriptionFrames as finalized.
|
|
497
|
+
*
|
|
498
|
+
* Segmented STT services process complete speech segments and return a
|
|
499
|
+
* single TranscriptionFrame per segment, so every transcription is
|
|
500
|
+
* inherently finalized.
|
|
501
|
+
*
|
|
502
|
+
* @param frame The frame to push.
|
|
503
|
+
* @param direction The direction of frame flow in the pipeline.
|
|
504
|
+
*/
|
|
505
|
+
pushFrame(frame: Frame, direction?: FrameDirection): Promise<void>;
|
|
506
|
+
/** Process frames, handling VAD events and audio segmentation. */
|
|
507
|
+
processFrame(frame: Frame, direction: FrameDirection): Promise<void>;
|
|
508
|
+
protected _handleUserStartedSpeaking(_frame: VADUserStartedSpeakingFrame): Promise<void>;
|
|
509
|
+
protected _handleUserStoppedSpeaking(_frame: VADUserStoppedSpeakingFrame): Promise<void>;
|
|
510
|
+
/** Transcribe queued segments one at a time, in order, until told to finish. */
|
|
511
|
+
protected _segmentTaskHandler(): Promise<void>;
|
|
512
|
+
protected _cancelSegmentTask(): Promise<void>;
|
|
513
|
+
/** Silence appended to a segment, as 16-bit mono PCM at the service sample rate. */
|
|
514
|
+
protected _trailingSilence(): Uint8Array;
|
|
515
|
+
/**
|
|
516
|
+
* Process audio frames by buffering them for segmented transcription.
|
|
517
|
+
*
|
|
518
|
+
* Continuously buffers audio, growing the buffer while user is speaking and
|
|
519
|
+
* maintaining a small buffer when not speaking to account for VAD delay.
|
|
520
|
+
*
|
|
521
|
+
* If the frame has a userId, it is stored for later use in transcription.
|
|
522
|
+
*
|
|
523
|
+
* @param frame The audio frame to process.
|
|
524
|
+
* @param direction The direction of frame processing.
|
|
525
|
+
*/
|
|
526
|
+
processAudioFrame(frame: InputAudioRawFrame, _direction: FrameDirection): Promise<void>;
|
|
527
|
+
}
|
|
528
|
+
export interface WebsocketSTTServiceOptions extends STTServiceOptions, WebsocketServiceOptions {
|
|
529
|
+
/** Whether to automatically reconnect on websocket errors. Defaults to true. */
|
|
530
|
+
reconnectOnError?: boolean;
|
|
531
|
+
}
|
|
532
|
+
declare const WebsocketSTTService_base: typeof STTService & (abstract new (...args: any[]) => WebsocketService);
|
|
533
|
+
/**
|
|
534
|
+
* Base class for websocket-based STT services.
|
|
535
|
+
*
|
|
536
|
+
* Combines STT functionality with websocket connectivity, providing
|
|
537
|
+
* automatic error handling, reconnection capabilities, and optional
|
|
538
|
+
* silence-based keepalive.
|
|
539
|
+
*
|
|
540
|
+
* The keepalive feature (inherited from STTService) sends silent audio when
|
|
541
|
+
* no real audio has been sent for a configurable timeout, preventing servers
|
|
542
|
+
* from closing idle connections (e.g. when behind a ServiceSwitcher).
|
|
543
|
+
* Subclasses can override `_sendKeepalive()` to wrap the silence in a
|
|
544
|
+
* service-specific protocol.
|
|
545
|
+
*/
|
|
546
|
+
export declare abstract class WebsocketSTTService extends WebsocketSTTService_base {
|
|
547
|
+
/**
|
|
548
|
+
* Initialize the Websocket STT service.
|
|
549
|
+
*
|
|
550
|
+
* @param options.reconnectOnError Whether to automatically reconnect on
|
|
551
|
+
* websocket errors. Other options are passed to the parent classes
|
|
552
|
+
* (including keepaliveTimeout and keepaliveInterval for STTService).
|
|
553
|
+
*/
|
|
554
|
+
constructor(options?: WebsocketSTTServiceOptions);
|
|
555
|
+
/**
|
|
556
|
+
* Stop the websocket STT service on a graceful end.
|
|
557
|
+
*
|
|
558
|
+
* @param frame The end frame.
|
|
559
|
+
*/
|
|
560
|
+
stop(frame: EndFrame): Promise<void>;
|
|
561
|
+
/**
|
|
562
|
+
* Cancel the websocket STT service immediately.
|
|
563
|
+
*
|
|
564
|
+
* Disconnecting here is the prompt teardown: the websocket receive loop
|
|
565
|
+
* runs independently and keeps reading transcripts from the provider until
|
|
566
|
+
* the socket is closed.
|
|
567
|
+
*
|
|
568
|
+
* @param frame The cancel frame.
|
|
569
|
+
*/
|
|
570
|
+
cancel(frame: CancelFrame): Promise<void>;
|
|
571
|
+
/** Release websocket STT resources at teardown. */
|
|
572
|
+
cleanup(): Promise<void>;
|
|
573
|
+
/** Connect and start keepalive task if enabled. */
|
|
574
|
+
_connect(): Promise<void>;
|
|
575
|
+
/** Disconnect and cancel keepalive task. */
|
|
576
|
+
_disconnect(): Promise<void>;
|
|
577
|
+
/**
|
|
578
|
+
* Disconnect and reconnect the websocket.
|
|
579
|
+
*
|
|
580
|
+
* Called by `STTService._reconnect()` inside the reconnecting guard. Tears
|
|
581
|
+
* down the current websocket connection and re-establishes it. Keepalive
|
|
582
|
+
* management is handled by `_connect` / `_disconnect`.
|
|
583
|
+
*/
|
|
584
|
+
protected _doReconnect(): Promise<void>;
|
|
585
|
+
/** Check if the websocket is open and ready for keepalive. */
|
|
586
|
+
protected _isKeepaliveReady(): boolean;
|
|
587
|
+
/**
|
|
588
|
+
* Send silent audio over the websocket to keep the connection alive.
|
|
589
|
+
*
|
|
590
|
+
* The default implementation sends raw PCM bytes directly. Subclasses can
|
|
591
|
+
* override this to wrap the silence in a service-specific protocol.
|
|
592
|
+
*
|
|
593
|
+
* @param silence Silent 16-bit mono PCM audio bytes.
|
|
594
|
+
*/
|
|
595
|
+
protected _sendKeepalive(silence: Uint8Array): Promise<void>;
|
|
596
|
+
protected _reportError(error: ErrorFrame, options?: {
|
|
597
|
+
forceTreatAsPermanent?: boolean;
|
|
598
|
+
}): Promise<void>;
|
|
599
|
+
}
|
|
600
|
+
export {};
|
|
601
|
+
//# sourceMappingURL=stt_service.d.ts.map
|