@pipedog/core 0.0.0 → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +25 -0
- package/NOTICE +9 -0
- package/dist/adapters/base_llm_adapter.d.ts +266 -0
- package/dist/adapters/base_llm_adapter.js +489 -0
- package/dist/adapters/index.d.ts +2 -0
- package/dist/adapters/index.js +8 -0
- package/dist/adapters/schemas/direct_function.d.ts +202 -0
- package/dist/adapters/schemas/direct_function.js +302 -0
- package/dist/adapters/schemas/function_schema.d.ts +67 -0
- package/dist/adapters/schemas/function_schema.js +72 -0
- package/dist/adapters/schemas/index.d.ts +2 -0
- package/dist/adapters/schemas/index.js +8 -0
- package/dist/adapters/schemas/tools_schema.d.ts +57 -0
- package/dist/adapters/schemas/tools_schema.js +79 -0
- package/dist/adapters/services/index.d.ts +2 -0
- package/dist/adapters/services/index.js +8 -0
- package/dist/adapters/services/open_ai_adapter.d.ts +147 -0
- package/dist/adapters/services/open_ai_adapter.js +250 -0
- package/dist/audio/dtmf/dtmf_data.d.ts +3 -0
- package/dist/audio/dtmf/dtmf_data.js +22 -0
- package/dist/audio/dtmf/index.d.ts +2 -0
- package/dist/audio/dtmf/index.js +8 -0
- package/dist/audio/dtmf/types.d.ts +31 -0
- package/dist/audio/dtmf/types.js +35 -0
- package/dist/audio/dtmf/utils.d.ts +13 -0
- package/dist/audio/dtmf/utils.js +62 -0
- package/dist/audio/filters/base_audio_filter.d.ts +51 -0
- package/dist/audio/filters/base_audio_filter.js +17 -0
- package/dist/audio/filters/index.d.ts +2 -0
- package/dist/audio/filters/index.js +8 -0
- package/dist/audio/index.d.ts +2 -0
- package/dist/audio/index.js +8 -0
- package/dist/audio/krisp_instance.d.ts +95 -0
- package/dist/audio/krisp_instance.js +229 -0
- package/dist/audio/mixers/base_audio_mixer.d.ts +56 -0
- package/dist/audio/mixers/base_audio_mixer.js +16 -0
- package/dist/audio/mixers/index.d.ts +2 -0
- package/dist/audio/mixers/index.js +8 -0
- package/dist/audio/resamplers/base_audio_resampler.d.ts +46 -0
- package/dist/audio/resamplers/base_audio_resampler.js +35 -0
- package/dist/audio/resamplers/index.d.ts +2 -0
- package/dist/audio/resamplers/index.js +8 -0
- package/dist/audio/resamplers/resampy_resampler.d.ts +10 -0
- package/dist/audio/resamplers/resampy_resampler.js +15 -0
- package/dist/audio/resamplers/soxr_resampler.d.ts +54 -0
- package/dist/audio/resamplers/soxr_resampler.js +69 -0
- package/dist/audio/resamplers/soxr_stream_resampler.d.ts +75 -0
- package/dist/audio/resamplers/soxr_stream_resampler.js +146 -0
- package/dist/audio/turn/base_turn_analyzer.d.ts +84 -0
- package/dist/audio/turn/base_turn_analyzer.js +61 -0
- package/dist/audio/turn/index.d.ts +2 -0
- package/dist/audio/turn/index.js +8 -0
- package/dist/audio/turn/smart_turn/_whisper_features.d.ts +30 -0
- package/dist/audio/turn/smart_turn/_whisper_features.js +225 -0
- package/dist/audio/turn/smart_turn/base_smart_turn.d.ts +94 -0
- package/dist/audio/turn/smart_turn/base_smart_turn.js +239 -0
- package/dist/audio/turn/smart_turn/local_smart_turn_v3.d.ts +76 -0
- package/dist/audio/turn/smart_turn/local_smart_turn_v3.js +143 -0
- package/dist/audio/utils.d.ts +224 -0
- package/dist/audio/utils.js +634 -0
- package/dist/audio/vad/data/index.d.ts +2 -0
- package/dist/audio/vad/data/index.js +8 -0
- package/dist/audio/vad/index.d.ts +2 -0
- package/dist/audio/vad/index.js +8 -0
- package/dist/audio/vad/vad_analyzer.d.ts +126 -0
- package/dist/audio/vad/vad_analyzer.js +211 -0
- package/dist/audio/vad/vad_controller.d.ts +143 -0
- package/dist/audio/vad/vad_controller.js +228 -0
- package/dist/audio/volume.d.ts +38 -0
- package/dist/audio/volume.js +75 -0
- package/dist/bus/adapters/base.d.ts +35 -0
- package/dist/bus/adapters/base.js +19 -0
- package/dist/bus/adapters/index.d.ts +10 -0
- package/dist/bus/adapters/index.js +15 -0
- package/dist/bus/adapters/llm_context_adapter.d.ts +36 -0
- package/dist/bus/adapters/llm_context_adapter.js +82 -0
- package/dist/bus/adapters/tools_schema_adapter.d.ts +25 -0
- package/dist/bus/adapters/tools_schema_adapter.js +70 -0
- package/dist/bus/bridge_processor.d.ts +111 -0
- package/dist/bus/bridge_processor.js +166 -0
- package/dist/bus/bus.d.ts +110 -0
- package/dist/bus/bus.js +200 -0
- package/dist/bus/index.d.ts +22 -0
- package/dist/bus/index.js +27 -0
- package/dist/bus/local/async_queue.d.ts +12 -0
- package/dist/bus/local/async_queue.js +21 -0
- package/dist/bus/local/index.d.ts +3 -0
- package/dist/bus/local/index.js +8 -0
- package/dist/bus/messages.d.ts +378 -0
- package/dist/bus/messages.js +357 -0
- package/dist/bus/network/index.d.ts +16 -0
- package/dist/bus/network/index.js +21 -0
- package/dist/bus/queue.d.ts +27 -0
- package/dist/bus/queue.js +51 -0
- package/dist/bus/serializers/base.d.ts +25 -0
- package/dist/bus/serializers/base.js +14 -0
- package/dist/bus/serializers/index.d.ts +9 -0
- package/dist/bus/serializers/index.js +14 -0
- package/dist/bus/serializers/json.d.ts +77 -0
- package/dist/bus/serializers/json.js +367 -0
- package/dist/bus/subscriber.d.ts +46 -0
- package/dist/bus/subscriber.js +27 -0
- package/dist/bus/ui/index.d.ts +3 -0
- package/dist/bus/ui/index.js +8 -0
- package/dist/bus/ui/messages.d.ts +186 -0
- package/dist/bus/ui/messages.js +171 -0
- package/dist/classifiers/base_classifier.d.ts +263 -0
- package/dist/classifiers/base_classifier.js +346 -0
- package/dist/classifiers/cloudflare/clef/classifier.d.ts +70 -0
- package/dist/classifiers/cloudflare/clef/classifier.js +204 -0
- package/dist/classifiers/cloudflare/clef/client.d.ts +104 -0
- package/dist/classifiers/cloudflare/clef/client.js +234 -0
- package/dist/classifiers/cloudflare/clef/index.d.ts +2 -0
- package/dist/classifiers/cloudflare/clef/index.js +8 -0
- package/dist/classifiers/cloudflare/index.d.ts +2 -0
- package/dist/classifiers/cloudflare/index.js +8 -0
- package/dist/classifiers/http_client.d.ts +70 -0
- package/dist/classifiers/http_client.js +161 -0
- package/dist/classifiers/index.d.ts +2 -0
- package/dist/classifiers/index.js +8 -0
- package/dist/classifiers/jev/classifier.d.ts +8 -0
- package/dist/classifiers/jev/classifier.js +13 -0
- package/dist/classifiers/jev/client.d.ts +8 -0
- package/dist/classifiers/jev/client.js +13 -0
- package/dist/classifiers/jev/index.d.ts +2 -0
- package/dist/classifiers/jev/index.js +8 -0
- package/dist/classifiers/llm/classifier.d.ts +72 -0
- package/dist/classifiers/llm/classifier.js +304 -0
- package/dist/classifiers/llm/index.d.ts +2 -0
- package/dist/classifiers/llm/index.js +8 -0
- package/dist/classifiers/openai/decisions/classifier.d.ts +77 -0
- package/dist/classifiers/openai/decisions/classifier.js +241 -0
- package/dist/classifiers/openai/decisions/client.d.ts +108 -0
- package/dist/classifiers/openai/decisions/client.js +234 -0
- package/dist/classifiers/openai/decisions/index.d.ts +2 -0
- package/dist/classifiers/openai/decisions/index.js +8 -0
- package/dist/classifiers/openai/index.d.ts +2 -0
- package/dist/classifiers/openai/index.js +8 -0
- package/dist/classifiers/typesafe/index.d.ts +2 -0
- package/dist/classifiers/typesafe/index.js +8 -0
- package/dist/classifiers/typesafe/jev/classifier.d.ts +70 -0
- package/dist/classifiers/typesafe/jev/classifier.js +220 -0
- package/dist/classifiers/typesafe/jev/client.d.ts +105 -0
- package/dist/classifiers/typesafe/jev/client.js +233 -0
- package/dist/classifiers/typesafe/jev/index.d.ts +2 -0
- package/dist/classifiers/typesafe/jev/index.js +8 -0
- package/dist/clocks/base_clock.d.ts +13 -0
- package/dist/clocks/base_clock.js +14 -0
- package/dist/clocks/index.d.ts +2 -0
- package/dist/clocks/index.js +8 -0
- package/dist/clocks/system_clock.d.ts +15 -0
- package/dist/clocks/system_clock.js +25 -0
- package/dist/frames/frames.d.ts +2533 -0
- package/dist/frames/frames.js +2485 -0
- package/dist/frames/index.d.ts +2 -0
- package/dist/frames/index.js +8 -0
- package/dist/frames/protobufs/frames_pb2.d.ts +229 -0
- package/dist/frames/protobufs/frames_pb2.js +466 -0
- package/dist/index.d.ts +11 -0
- package/dist/index.js +56 -0
- package/dist/metrics/index.d.ts +2 -0
- package/dist/metrics/index.js +8 -0
- package/dist/metrics/metrics.d.ts +276 -0
- package/dist/metrics/metrics.js +254 -0
- package/dist/observers/base_observer.d.ts +173 -0
- package/dist/observers/base_observer.js +151 -0
- package/dist/observers/error_observer.d.ts +75 -0
- package/dist/observers/error_observer.js +87 -0
- package/dist/observers/function_call_observer.d.ts +157 -0
- package/dist/observers/function_call_observer.js +222 -0
- package/dist/observers/index.d.ts +2 -0
- package/dist/observers/index.js +8 -0
- package/dist/observers/loggers/debug_log_observer.d.ts +91 -0
- package/dist/observers/loggers/debug_log_observer.js +228 -0
- package/dist/observers/loggers/index.d.ts +2 -0
- package/dist/observers/loggers/index.js +8 -0
- package/dist/observers/loggers/llm_log_observer.d.ts +24 -0
- package/dist/observers/loggers/llm_log_observer.js +110 -0
- package/dist/observers/loggers/metrics_log_observer.d.ts +85 -0
- package/dist/observers/loggers/metrics_log_observer.js +186 -0
- package/dist/observers/loggers/transcription_log_observer.d.ts +23 -0
- package/dist/observers/loggers/transcription_log_observer.js +74 -0
- package/dist/observers/service_metrics_observer.d.ts +168 -0
- package/dist/observers/service_metrics_observer.js +236 -0
- package/dist/observers/speaking_observer.d.ts +104 -0
- package/dist/observers/speaking_observer.js +150 -0
- package/dist/observers/startup_timing_observer.d.ts +221 -0
- package/dist/observers/startup_timing_observer.js +365 -0
- package/dist/observers/turn_tracking_observer.d.ts +78 -0
- package/dist/observers/turn_tracking_observer.js +189 -0
- package/dist/observers/user_bot_latency_observer.d.ts +411 -0
- package/dist/observers/user_bot_latency_observer.js +974 -0
- package/dist/pipeline/base_pipeline.d.ts +49 -0
- package/dist/pipeline/base_pipeline.js +101 -0
- package/dist/pipeline/capabilities.d.ts +76 -0
- package/dist/pipeline/capabilities.js +97 -0
- package/dist/pipeline/index.d.ts +2 -0
- package/dist/pipeline/index.js +8 -0
- package/dist/pipeline/job_context.d.ts +317 -0
- package/dist/pipeline/job_context.js +450 -0
- package/dist/pipeline/job_decorator.d.ts +54 -0
- package/dist/pipeline/job_decorator.js +92 -0
- package/dist/pipeline/llm_switcher.d.ts +84 -0
- package/dist/pipeline/llm_switcher.js +107 -0
- package/dist/pipeline/llm_with_backend.d.ts +435 -0
- package/dist/pipeline/llm_with_backend.js +806 -0
- package/dist/pipeline/parallel_pipeline.d.ts +83 -0
- package/dist/pipeline/parallel_pipeline.js +200 -0
- package/dist/pipeline/pipeline.d.ts +126 -0
- package/dist/pipeline/pipeline.js +196 -0
- package/dist/pipeline/service_switcher.d.ts +287 -0
- package/dist/pipeline/service_switcher.js +509 -0
- package/dist/pipeline/sync_parallel_pipeline.d.ts +192 -0
- package/dist/pipeline/sync_parallel_pipeline.js +387 -0
- package/dist/pipeline/worker.d.ts +814 -0
- package/dist/pipeline/worker.js +1639 -0
- package/dist/pipeline/worker_observer.d.ts +131 -0
- package/dist/pipeline/worker_observer.js +274 -0
- package/dist/pipeline/worker_ready_decorator.d.ts +45 -0
- package/dist/pipeline/worker_ready_decorator.js +76 -0
- package/dist/processors/aggregators/async_tool_messages.d.ts +146 -0
- package/dist/processors/aggregators/async_tool_messages.js +291 -0
- package/dist/processors/aggregators/dtmf_aggregator.d.ts +61 -0
- package/dist/processors/aggregators/dtmf_aggregator.js +144 -0
- package/dist/processors/aggregators/gated.d.ts +43 -0
- package/dist/processors/aggregators/gated.js +86 -0
- package/dist/processors/aggregators/gated_llm_context.d.ts +42 -0
- package/dist/processors/aggregators/gated_llm_context.js +86 -0
- package/dist/processors/aggregators/index.d.ts +2 -0
- package/dist/processors/aggregators/index.js +8 -0
- package/dist/processors/aggregators/llm_context.d.ts +370 -0
- package/dist/processors/aggregators/llm_context.js +580 -0
- package/dist/processors/aggregators/llm_context_summarizer.d.ts +219 -0
- package/dist/processors/aggregators/llm_context_summarizer.js +447 -0
- package/dist/processors/aggregators/llm_response.d.ts +37 -0
- package/dist/processors/aggregators/llm_response.js +71 -0
- package/dist/processors/aggregators/llm_response_universal.d.ts +966 -0
- package/dist/processors/aggregators/llm_response_universal.js +2394 -0
- package/dist/processors/aggregators/llm_text_processor.d.ts +53 -0
- package/dist/processors/aggregators/llm_text_processor.js +102 -0
- package/dist/processors/aggregators/sentence.d.ts +39 -0
- package/dist/processors/aggregators/sentence.js +69 -0
- package/dist/processors/async_generator.d.ts +42 -0
- package/dist/processors/async_generator.js +76 -0
- package/dist/processors/audio/audio_buffer_processor.d.ts +241 -0
- package/dist/processors/audio/audio_buffer_processor.js +565 -0
- package/dist/processors/audio/index.d.ts +2 -0
- package/dist/processors/audio/index.js +8 -0
- package/dist/processors/audio/vad_processor.d.ts +67 -0
- package/dist/processors/audio/vad_processor.js +103 -0
- package/dist/processors/consumer_processor.d.ts +49 -0
- package/dist/processors/consumer_processor.js +85 -0
- package/dist/processors/filters/frame_filter.d.ts +39 -0
- package/dist/processors/filters/frame_filter.js +52 -0
- package/dist/processors/filters/function_filter.d.ts +59 -0
- package/dist/processors/filters/function_filter.js +78 -0
- package/dist/processors/filters/identity_filter.d.ts +33 -0
- package/dist/processors/filters/identity_filter.js +37 -0
- package/dist/processors/filters/index.d.ts +2 -0
- package/dist/processors/filters/index.js +8 -0
- package/dist/processors/filters/null_filter.d.ts +34 -0
- package/dist/processors/filters/null_filter.js +47 -0
- package/dist/processors/filters/wake_notifier_filter.d.ts +48 -0
- package/dist/processors/filters/wake_notifier_filter.js +49 -0
- package/dist/processors/frame_processor.d.ts +360 -0
- package/dist/processors/frame_processor.js +932 -0
- package/dist/processors/frameworks/index.d.ts +2 -0
- package/dist/processors/frameworks/index.js +8 -0
- package/dist/processors/frameworks/rtvi/frames.d.ts +241 -0
- package/dist/processors/frameworks/rtvi/frames.js +259 -0
- package/dist/processors/frameworks/rtvi/index.d.ts +6 -0
- package/dist/processors/frameworks/rtvi/index.js +11 -0
- package/dist/processors/frameworks/rtvi/models.d.ts +1741 -0
- package/dist/processors/frameworks/rtvi/models.js +1807 -0
- package/dist/processors/frameworks/rtvi/observer.d.ts +340 -0
- package/dist/processors/frameworks/rtvi/observer.js +1113 -0
- package/dist/processors/frameworks/rtvi/processor.d.ts +182 -0
- package/dist/processors/frameworks/rtvi/processor.js +676 -0
- package/dist/processors/idle_frame_processor.d.ts +44 -0
- package/dist/processors/idle_frame_processor.js +86 -0
- package/dist/processors/index.d.ts +2 -0
- package/dist/processors/index.js +8 -0
- package/dist/processors/logger.d.ts +42 -0
- package/dist/processors/logger.js +87 -0
- package/dist/processors/metrics/frame_processor_metrics.d.ts +172 -0
- package/dist/processors/metrics/frame_processor_metrics.js +372 -0
- package/dist/processors/metrics/index.d.ts +2 -0
- package/dist/processors/metrics/index.js +8 -0
- package/dist/processors/producer_processor.d.ts +68 -0
- package/dist/processors/producer_processor.js +78 -0
- package/dist/processors/text_transformer.d.ts +30 -0
- package/dist/processors/text_transformer.js +48 -0
- package/dist/registry/index.d.ts +3 -0
- package/dist/registry/index.js +8 -0
- package/dist/registry/registry.d.ts +69 -0
- package/dist/registry/registry.js +149 -0
- package/dist/registry/types.d.ts +47 -0
- package/dist/registry/types.js +39 -0
- package/dist/serializers/base_serializer.d.ts +94 -0
- package/dist/serializers/base_serializer.js +82 -0
- package/dist/serializers/index.d.ts +2 -0
- package/dist/serializers/index.js +8 -0
- package/dist/services/ai_service.d.ts +145 -0
- package/dist/services/ai_service.js +262 -0
- package/dist/services/anthropic/index.d.ts +2 -0
- package/dist/services/anthropic/index.js +7 -0
- package/dist/services/assemblyai/index.d.ts +2 -0
- package/dist/services/assemblyai/index.js +7 -0
- package/dist/services/asyncai/index.d.ts +2 -0
- package/dist/services/asyncai/index.js +7 -0
- package/dist/services/aws/index.d.ts +2 -0
- package/dist/services/aws/index.js +7 -0
- package/dist/services/azure/index.d.ts +2 -0
- package/dist/services/azure/index.js +7 -0
- package/dist/services/baseten/index.d.ts +2 -0
- package/dist/services/baseten/index.js +7 -0
- package/dist/services/bland/index.d.ts +2 -0
- package/dist/services/bland/index.js +7 -0
- package/dist/services/camb/index.d.ts +2 -0
- package/dist/services/camb/index.js +7 -0
- package/dist/services/cartesia/index.d.ts +2 -0
- package/dist/services/cartesia/index.js +7 -0
- package/dist/services/cerebras/index.d.ts +2 -0
- package/dist/services/cerebras/index.js +7 -0
- package/dist/services/crusoe/index.d.ts +2 -0
- package/dist/services/crusoe/index.js +7 -0
- package/dist/services/deepgram/index.d.ts +2 -0
- package/dist/services/deepgram/index.js +7 -0
- package/dist/services/deepseek/index.d.ts +2 -0
- package/dist/services/deepseek/index.js +7 -0
- package/dist/services/elevenlabs/index.d.ts +2 -0
- package/dist/services/elevenlabs/index.js +7 -0
- package/dist/services/fal/index.d.ts +2 -0
- package/dist/services/fal/index.js +7 -0
- package/dist/services/fireworks/index.d.ts +2 -0
- package/dist/services/fireworks/index.js +7 -0
- package/dist/services/fish/index.d.ts +2 -0
- package/dist/services/fish/index.js +7 -0
- package/dist/services/funasr/index.d.ts +3 -0
- package/dist/services/funasr/index.js +8 -0
- package/dist/services/gladia/index.d.ts +2 -0
- package/dist/services/gladia/index.js +7 -0
- package/dist/services/google/index.d.ts +2 -0
- package/dist/services/google/index.js +7 -0
- package/dist/services/gradium/index.d.ts +2 -0
- package/dist/services/gradium/index.js +7 -0
- package/dist/services/grok/index.d.ts +2 -0
- package/dist/services/grok/index.js +7 -0
- package/dist/services/groq/index.d.ts +2 -0
- package/dist/services/groq/index.js +7 -0
- package/dist/services/heygen/index.d.ts +2 -0
- package/dist/services/heygen/index.js +7 -0
- package/dist/services/hume/index.d.ts +2 -0
- package/dist/services/hume/index.js +7 -0
- package/dist/services/image_service.d.ts +52 -0
- package/dist/services/image_service.js +60 -0
- package/dist/services/inception/index.d.ts +2 -0
- package/dist/services/inception/index.js +7 -0
- package/dist/services/index.d.ts +2 -0
- package/dist/services/index.js +8 -0
- package/dist/services/inworld/index.d.ts +2 -0
- package/dist/services/inworld/index.js +7 -0
- package/dist/services/keenable/index.d.ts +2 -0
- package/dist/services/keenable/index.js +7 -0
- package/dist/services/kokoro/index.d.ts +2 -0
- package/dist/services/kokoro/index.js +7 -0
- package/dist/services/llm_service.d.ts +1106 -0
- package/dist/services/llm_service.js +2231 -0
- package/dist/services/mem0/index.d.ts +2 -0
- package/dist/services/mem0/index.js +7 -0
- package/dist/services/meta/index.d.ts +2 -0
- package/dist/services/meta/index.js +7 -0
- package/dist/services/minimax/index.d.ts +2 -0
- package/dist/services/minimax/index.js +7 -0
- package/dist/services/mistral/index.d.ts +2 -0
- package/dist/services/mistral/index.js +7 -0
- package/dist/services/moondream/index.d.ts +2 -0
- package/dist/services/moondream/index.js +7 -0
- package/dist/services/moonshine/index.d.ts +2 -0
- package/dist/services/moonshine/index.js +7 -0
- package/dist/services/nebius/index.d.ts +2 -0
- package/dist/services/nebius/index.js +7 -0
- package/dist/services/neuphonic/index.d.ts +2 -0
- package/dist/services/neuphonic/index.js +7 -0
- package/dist/services/novita/index.d.ts +2 -0
- package/dist/services/novita/index.js +7 -0
- package/dist/services/nvidia/index.d.ts +2 -0
- package/dist/services/nvidia/index.js +11 -0
- package/dist/services/ollama/index.d.ts +2 -0
- package/dist/services/ollama/index.js +7 -0
- package/dist/services/openai/index.d.ts +2 -0
- package/dist/services/openai/index.js +7 -0
- package/dist/services/openclaw/index.d.ts +14 -0
- package/dist/services/openclaw/index.js +19 -0
- package/dist/services/openrouter/index.d.ts +2 -0
- package/dist/services/openrouter/index.js +7 -0
- package/dist/services/perplexity/index.d.ts +2 -0
- package/dist/services/perplexity/index.js +7 -0
- package/dist/services/piper/index.d.ts +2 -0
- package/dist/services/piper/index.js +7 -0
- package/dist/services/pocket_tts/index.d.ts +2 -0
- package/dist/services/pocket_tts/index.js +7 -0
- package/dist/services/qwen/index.d.ts +2 -0
- package/dist/services/qwen/index.js +7 -0
- package/dist/services/resembleai/index.d.ts +2 -0
- package/dist/services/resembleai/index.js +7 -0
- package/dist/services/rime/index.d.ts +2 -0
- package/dist/services/rime/index.js +7 -0
- package/dist/services/sambanova/index.d.ts +2 -0
- package/dist/services/sambanova/index.js +7 -0
- package/dist/services/sarvam/index.d.ts +2 -0
- package/dist/services/sarvam/index.js +7 -0
- package/dist/services/settings.d.ts +356 -0
- package/dist/services/settings.js +437 -0
- package/dist/services/simli/index.d.ts +2 -0
- package/dist/services/simli/index.js +7 -0
- package/dist/services/smallest/index.d.ts +2 -0
- package/dist/services/smallest/index.js +7 -0
- package/dist/services/soniox/index.d.ts +2 -0
- package/dist/services/soniox/index.js +7 -0
- package/dist/services/speechify/index.d.ts +2 -0
- package/dist/services/speechify/index.js +7 -0
- package/dist/services/speechmatics/index.d.ts +2 -0
- package/dist/services/speechmatics/index.js +7 -0
- package/dist/services/stt_latency.d.ts +62 -0
- package/dist/services/stt_latency.js +71 -0
- package/dist/services/stt_service.d.ts +601 -0
- package/dist/services/stt_service.js +1044 -0
- package/dist/services/tavus/index.d.ts +2 -0
- package/dist/services/tavus/index.js +7 -0
- package/dist/services/together/index.d.ts +2 -0
- package/dist/services/together/index.js +7 -0
- package/dist/services/tts_service.d.ts +909 -0
- package/dist/services/tts_service.js +2018 -0
- package/dist/services/ultravox/index.d.ts +2 -0
- package/dist/services/ultravox/index.js +9 -0
- package/dist/services/vision_service.d.ts +56 -0
- package/dist/services/vision_service.js +62 -0
- package/dist/services/websocket_service.d.ts +213 -0
- package/dist/services/websocket_service.js +654 -0
- package/dist/services/whisper/index.d.ts +2 -0
- package/dist/services/whisper/index.js +9 -0
- package/dist/services/xai/index.d.ts +2 -0
- package/dist/services/xai/index.js +9 -0
- package/dist/services/xtts/index.d.ts +2 -0
- package/dist/services/xtts/index.js +9 -0
- package/dist/tests/index.d.ts +2 -0
- package/dist/tests/index.js +8 -0
- package/dist/tests/utils.d.ts +123 -0
- package/dist/tests/utils.js +212 -0
- package/dist/transcriptions/index.d.ts +2 -0
- package/dist/transcriptions/index.js +8 -0
- package/dist/transcriptions/language.d.ts +376 -0
- package/dist/transcriptions/language.js +543 -0
- package/dist/transports/base_input.d.ts +149 -0
- package/dist/transports/base_input.js +338 -0
- package/dist/transports/base_output.d.ts +432 -0
- package/dist/transports/base_output.js +1231 -0
- package/dist/transports/base_transport.d.ts +223 -0
- package/dist/transports/base_transport.js +167 -0
- package/dist/transports/daily/index.d.ts +2 -0
- package/dist/transports/daily/index.js +9 -0
- package/dist/transports/heygen/index.d.ts +2 -0
- package/dist/transports/heygen/index.js +9 -0
- package/dist/transports/index.d.ts +2 -0
- package/dist/transports/index.js +8 -0
- package/dist/transports/lemonslice/index.d.ts +2 -0
- package/dist/transports/lemonslice/index.js +9 -0
- package/dist/transports/livekit/index.d.ts +2 -0
- package/dist/transports/livekit/index.js +9 -0
- package/dist/transports/local/index.d.ts +2 -0
- package/dist/transports/local/index.js +9 -0
- package/dist/transports/moq/index.d.ts +2 -0
- package/dist/transports/moq/index.js +9 -0
- package/dist/transports/smallwebrtc/index.d.ts +2 -0
- package/dist/transports/smallwebrtc/index.js +9 -0
- package/dist/transports/tavus/index.d.ts +2 -0
- package/dist/transports/tavus/index.js +9 -0
- package/dist/transports/video_in_sampler.d.ts +157 -0
- package/dist/transports/video_in_sampler.js +258 -0
- package/dist/transports/vonage/index.d.ts +2 -0
- package/dist/transports/vonage/index.js +9 -0
- package/dist/transports/websocket/index.d.ts +2 -0
- package/dist/transports/websocket/index.js +9 -0
- package/dist/transports/whatsapp/index.d.ts +2 -0
- package/dist/transports/whatsapp/index.js +9 -0
- package/dist/turns/eager_end_of_turn_mixin.d.ts +70 -0
- package/dist/turns/eager_end_of_turn_mixin.js +122 -0
- package/dist/turns/empty_user_turn.d.ts +55 -0
- package/dist/turns/empty_user_turn.js +53 -0
- package/dist/turns/index.d.ts +2 -0
- package/dist/turns/index.js +8 -0
- package/dist/turns/speculation_gate.d.ts +169 -0
- package/dist/turns/speculation_gate.js +294 -0
- package/dist/turns/types.d.ts +33 -0
- package/dist/turns/types.js +35 -0
- package/dist/turns/user_idle_controller.d.ts +94 -0
- package/dist/turns/user_idle_controller.js +187 -0
- package/dist/turns/user_mute/always_user_mute_strategy.d.ts +15 -0
- package/dist/turns/user_mute/always_user_mute_strategy.js +29 -0
- package/dist/turns/user_mute/base_user_mute_strategy.d.ts +39 -0
- package/dist/turns/user_mute/base_user_mute_strategy.js +45 -0
- package/dist/turns/user_mute/first_speech_user_mute_strategy.d.ts +28 -0
- package/dist/turns/user_mute/first_speech_user_mute_strategy.js +52 -0
- package/dist/turns/user_mute/function_call_user_mute_strategy.d.ts +25 -0
- package/dist/turns/user_mute/function_call_user_mute_strategy.js +50 -0
- package/dist/turns/user_mute/index.d.ts +6 -0
- package/dist/turns/user_mute/index.js +11 -0
- package/dist/turns/user_mute/mute_until_first_bot_complete_user_mute_strategy.d.ts +43 -0
- package/dist/turns/user_mute/mute_until_first_bot_complete_user_mute_strategy.js +76 -0
- package/dist/turns/user_start/base_user_turn_start_strategy.d.ts +135 -0
- package/dist/turns/user_start/base_user_turn_start_strategy.js +134 -0
- package/dist/turns/user_start/external_user_turn_start_strategy.d.ts +45 -0
- package/dist/turns/user_start/external_user_turn_start_strategy.js +64 -0
- package/dist/turns/user_start/index.d.ts +8 -0
- package/dist/turns/user_start/index.js +16 -0
- package/dist/turns/user_start/krisp_viva_ip_user_turn_start_strategy.d.ts +133 -0
- package/dist/turns/user_start/krisp_viva_ip_user_turn_start_strategy.js +243 -0
- package/dist/turns/user_start/min_words_user_turn_start_strategy.d.ts +60 -0
- package/dist/turns/user_start/min_words_user_turn_start_strategy.js +102 -0
- package/dist/turns/user_start/transcription_user_turn_start_strategy.d.ts +29 -0
- package/dist/turns/user_start/transcription_user_turn_start_strategy.js +43 -0
- package/dist/turns/user_start/vad_user_turn_start_strategy.d.ts +20 -0
- package/dist/turns/user_start/vad_user_turn_start_strategy.js +31 -0
- package/dist/turns/user_start/wake_phrase_user_turn_start_strategy.d.ts +141 -0
- package/dist/turns/user_start/wake_phrase_user_turn_start_strategy.js +238 -0
- package/dist/turns/user_stop/base_user_turn_stop_strategy.d.ts +174 -0
- package/dist/turns/user_stop/base_user_turn_stop_strategy.js +170 -0
- package/dist/turns/user_stop/deferred_user_turn_stop_strategy.d.ts +81 -0
- package/dist/turns/user_stop/deferred_user_turn_stop_strategy.js +107 -0
- package/dist/turns/user_stop/eager_match_policy.d.ts +36 -0
- package/dist/turns/user_stop/eager_match_policy.js +50 -0
- package/dist/turns/user_stop/eager_user_turn_stop_strategy.d.ts +84 -0
- package/dist/turns/user_stop/eager_user_turn_stop_strategy.js +154 -0
- package/dist/turns/user_stop/external_user_turn_completion_stop_strategy.d.ts +45 -0
- package/dist/turns/user_stop/external_user_turn_completion_stop_strategy.js +55 -0
- package/dist/turns/user_stop/external_user_turn_stop_strategy.d.ts +115 -0
- package/dist/turns/user_stop/external_user_turn_stop_strategy.js +205 -0
- package/dist/turns/user_stop/index.d.ts +10 -0
- package/dist/turns/user_stop/index.js +15 -0
- package/dist/turns/user_stop/llm_turn_completion_user_turn_stop_strategy.d.ts +44 -0
- package/dist/turns/user_stop/llm_turn_completion_user_turn_stop_strategy.js +59 -0
- package/dist/turns/user_stop/speech_timeout_user_turn_stop_strategy.d.ts +140 -0
- package/dist/turns/user_stop/speech_timeout_user_turn_stop_strategy.js +303 -0
- package/dist/turns/user_stop/turn_analyzer_user_turn_stop_strategy.d.ts +135 -0
- package/dist/turns/user_stop/turn_analyzer_user_turn_stop_strategy.js +332 -0
- package/dist/turns/user_turn_completion_mixin.d.ts +147 -0
- package/dist/turns/user_turn_completion_mixin.js +591 -0
- package/dist/turns/user_turn_controller.d.ts +134 -0
- package/dist/turns/user_turn_controller.js +335 -0
- package/dist/turns/user_turn_processor.d.ts +87 -0
- package/dist/turns/user_turn_processor.js +180 -0
- package/dist/turns/user_turn_strategies.d.ts +187 -0
- package/dist/turns/user_turn_strategies.js +222 -0
- package/dist/utils/async_tool_cancellation.d.ts +35 -0
- package/dist/utils/async_tool_cancellation.js +78 -0
- package/dist/utils/asyncio/index.d.ts +2 -0
- package/dist/utils/asyncio/index.js +7 -0
- package/dist/utils/asyncio/task_manager.d.ts +49 -0
- package/dist/utils/asyncio/task_manager.js +105 -0
- package/dist/utils/base_object.d.ts +76 -0
- package/dist/utils/base_object.js +154 -0
- package/dist/utils/context/aggregated_frame_sequencer.d.ts +491 -0
- package/dist/utils/context/aggregated_frame_sequencer.js +803 -0
- package/dist/utils/context/audio_context_manager.d.ts +131 -0
- package/dist/utils/context/audio_context_manager.js +233 -0
- package/dist/utils/context/index.d.ts +2 -0
- package/dist/utils/context/index.js +7 -0
- package/dist/utils/context/llm_context_summarization.d.ts +258 -0
- package/dist/utils/context/llm_context_summarization.js +511 -0
- package/dist/utils/context/text_segment_map.d.ts +611 -0
- package/dist/utils/context/text_segment_map.js +1132 -0
- package/dist/utils/context/word_completion_tracker.d.ts +273 -0
- package/dist/utils/context/word_completion_tracker.js +441 -0
- package/dist/utils/context/word_timestamp_tracker.d.ts +72 -0
- package/dist/utils/context/word_timestamp_tracker.js +98 -0
- package/dist/utils/deprecation.d.ts +28 -0
- package/dist/utils/deprecation.js +75 -0
- package/dist/utils/env.d.ts +38 -0
- package/dist/utils/env.js +62 -0
- package/dist/utils/errors.d.ts +75 -0
- package/dist/utils/errors.js +202 -0
- package/dist/utils/file_resolver.d.ts +126 -0
- package/dist/utils/file_resolver.js +314 -0
- package/dist/utils/file_storage.d.ts +99 -0
- package/dist/utils/file_storage.js +100 -0
- package/dist/utils/frame_queue.d.ts +36 -0
- package/dist/utils/frame_queue.js +66 -0
- package/dist/utils/http.d.ts +39 -0
- package/dist/utils/http.js +53 -0
- package/dist/utils/image.d.ts +15 -0
- package/dist/utils/image.js +30 -0
- package/dist/utils/index.d.ts +2 -0
- package/dist/utils/index.js +8 -0
- package/dist/utils/log_config.d.ts +34 -0
- package/dist/utils/log_config.js +76 -0
- package/dist/utils/logger.d.ts +40 -0
- package/dist/utils/logger.js +91 -0
- package/dist/utils/mixin.d.ts +14 -0
- package/dist/utils/mixin.js +63 -0
- package/dist/utils/network.d.ts +73 -0
- package/dist/utils/network.js +90 -0
- package/dist/utils/security/allowed_origins.d.ts +16 -0
- package/dist/utils/security/allowed_origins.js +34 -0
- package/dist/utils/security/index.d.ts +2 -0
- package/dist/utils/security/index.js +7 -0
- package/dist/utils/security/redaction.d.ts +27 -0
- package/dist/utils/security/redaction.js +123 -0
- package/dist/utils/security/ssrf.d.ts +79 -0
- package/dist/utils/security/ssrf.js +327 -0
- package/dist/utils/shared.d.ts +35 -0
- package/dist/utils/shared.js +136 -0
- package/dist/utils/startup.d.ts +17 -0
- package/dist/utils/startup.js +130 -0
- package/dist/utils/string.d.ts +116 -0
- package/dist/utils/string.js +460 -0
- package/dist/utils/sync/base_notifier.d.ts +25 -0
- package/dist/utils/sync/base_notifier.js +16 -0
- package/dist/utils/sync/event_notifier.d.ts +38 -0
- package/dist/utils/sync/event_notifier.js +51 -0
- package/dist/utils/sync/index.d.ts +2 -0
- package/dist/utils/sync/index.js +7 -0
- package/dist/utils/text/alnum_utils.d.ts +109 -0
- package/dist/utils/text/alnum_utils.js +262 -0
- package/dist/utils/text/base_text_aggregator.d.ts +146 -0
- package/dist/utils/text/base_text_aggregator.js +102 -0
- package/dist/utils/text/base_text_filter.d.ts +54 -0
- package/dist/utils/text/base_text_filter.js +50 -0
- package/dist/utils/text/index.d.ts +2 -0
- package/dist/utils/text/index.js +7 -0
- package/dist/utils/text/markdown_text_filter.d.ts +147 -0
- package/dist/utils/text/markdown_text_filter.js +522 -0
- package/dist/utils/text/markup_utils.d.ts +54 -0
- package/dist/utils/text/markup_utils.js +156 -0
- package/dist/utils/text/pattern_pair_aggregator.d.ts +251 -0
- package/dist/utils/text/pattern_pair_aggregator.js +478 -0
- package/dist/utils/text/phonemes.d.ts +60 -0
- package/dist/utils/text/phonemes.js +172 -0
- package/dist/utils/text/simple_text_aggregator.d.ts +135 -0
- package/dist/utils/text/simple_text_aggregator.js +254 -0
- package/dist/utils/text/skip_tags_aggregator.d.ts +85 -0
- package/dist/utils/text/skip_tags_aggregator.js +147 -0
- package/dist/utils/text/transforms/pronunciations.d.ts +65 -0
- package/dist/utils/text/transforms/pronunciations.js +111 -0
- package/dist/utils/text/word_timestamp_utils.d.ts +26 -0
- package/dist/utils/text/word_timestamp_utils.js +52 -0
- package/dist/utils/time.d.ts +13 -0
- package/dist/utils/time.js +31 -0
- package/dist/utils/tracing/index.d.ts +3 -0
- package/dist/utils/tracing/index.js +8 -0
- package/dist/utils/tracing/service_attributes.d.ts +198 -0
- package/dist/utils/tracing/service_attributes.js +434 -0
- package/dist/utils/tracing/service_decorators.d.ts +214 -0
- package/dist/utils/tracing/service_decorators.js +1662 -0
- package/dist/utils/tracing/setup.d.ts +95 -0
- package/dist/utils/tracing/setup.js +110 -0
- package/dist/utils/tracing/tracing_context.d.ts +55 -0
- package/dist/utils/tracing/tracing_context.js +97 -0
- package/dist/utils/tracing/turn_trace_observer.d.ts +90 -0
- package/dist/utils/tracing/turn_trace_observer.js +219 -0
- package/dist/utils/types.d.ts +80 -0
- package/dist/utils/types.js +108 -0
- package/dist/utils/utils.d.ts +5 -0
- package/dist/utils/utils.js +22 -0
- package/dist/utils/yaml.d.ts +64 -0
- package/dist/utils/yaml.js +41 -0
- package/dist/version.d.ts +3 -0
- package/dist/version.js +7 -0
- package/dist/workers/base_worker.d.ts +772 -0
- package/dist/workers/base_worker.js +1298 -0
- package/dist/workers/index.d.ts +2 -0
- package/dist/workers/index.js +8 -0
- package/dist/workers/llm/backend_llm_worker.d.ts +552 -0
- package/dist/workers/llm/backend_llm_worker.js +1055 -0
- package/dist/workers/llm/index.d.ts +6 -0
- package/dist/workers/llm/index.js +11 -0
- package/dist/workers/llm/llm_context_worker.d.ts +66 -0
- package/dist/workers/llm/llm_context_worker.js +72 -0
- package/dist/workers/llm/llm_worker.d.ts +180 -0
- package/dist/workers/llm/llm_worker.js +323 -0
- package/dist/workers/llm/tool_decorator.d.ts +68 -0
- package/dist/workers/llm/tool_decorator.js +60 -0
- package/dist/workers/runner.d.ts +268 -0
- package/dist/workers/runner.js +489 -0
- package/dist/workers/ui/index.d.ts +22 -0
- package/dist/workers/ui/index.js +27 -0
- package/dist/workers/ui/ui_event_decorator.d.ts +43 -0
- package/dist/workers/ui/ui_event_decorator.js +75 -0
- package/dist/workers/ui/ui_job_context.d.ts +12 -0
- package/dist/workers/ui/ui_job_context.js +17 -0
- package/dist/workers/ui/ui_prompts.d.ts +21 -0
- package/dist/workers/ui/ui_prompts.js +41 -0
- package/dist/workers/ui/ui_tools.d.ts +32 -0
- package/dist/workers/ui/ui_tools.js +112 -0
- package/dist/workers/ui/ui_worker.d.ts +571 -0
- package/dist/workers/ui/ui_worker.js +1156 -0
- package/package.json +46 -3
- package/README.md +0 -5
|
@@ -0,0 +1,228 @@
|
|
|
1
|
+
// Copyright (c) 2024-2026, Daily
|
|
2
|
+
// Copyright (c) 2026, pipedog contributors
|
|
3
|
+
// SPDX-License-Identifier: BSD-2-Clause
|
|
4
|
+
//
|
|
5
|
+
// Ported from pipecat: src/pipecat/audio/vad/vad_controller.py
|
|
6
|
+
/**
|
|
7
|
+
* Voice Activity Detection controller for managing speech state transitions.
|
|
8
|
+
*
|
|
9
|
+
* Provides a controller that wraps a VADAnalyzer to track speech state and
|
|
10
|
+
* emit events when speech starts, stops, or is actively detected.
|
|
11
|
+
*/
|
|
12
|
+
import { monotonic, sleep, time } from "@pipedog/asyncio";
|
|
13
|
+
import { InputAudioRawFrame, SpeechControlParamsFrame, VADParamsUpdateFrame, } from "../../frames/frames.js";
|
|
14
|
+
import { FrameDirection } from "../../processors/frame_processor.js";
|
|
15
|
+
import { BaseObject } from "../../utils/base_object.js";
|
|
16
|
+
import { logger } from "../../utils/logger.js";
|
|
17
|
+
import { VADState } from "./vad_analyzer.js";
|
|
18
|
+
/**
|
|
19
|
+
* Manages voice activity detection state and emits speech events.
|
|
20
|
+
*
|
|
21
|
+
* Wraps a `VADAnalyzer` to process audio and trigger events based on speech
|
|
22
|
+
* state transitions. Tracks whether the user is speaking, quiet, or
|
|
23
|
+
* transitioning between states.
|
|
24
|
+
*
|
|
25
|
+
* Event handlers available:
|
|
26
|
+
*
|
|
27
|
+
* - on_speech_started: Called when speech begins.
|
|
28
|
+
* - on_speech_stopped: Called when speech ends, including forced stop when
|
|
29
|
+
* the audio stream goes idle (no frames received while speaking).
|
|
30
|
+
* - on_speech_activity: Called periodically while speech is detected.
|
|
31
|
+
* - on_push_frame: Called when the controller wants to push a frame.
|
|
32
|
+
* - on_broadcast_frame: Called when the controller wants to broadcast a frame.
|
|
33
|
+
*
|
|
34
|
+
* @example
|
|
35
|
+
* vadController.eventHandler("on_speech_started", async (controller) => { ... });
|
|
36
|
+
* vadController.eventHandler("on_speech_stopped", async (controller) => { ... });
|
|
37
|
+
* vadController.eventHandler("on_speech_activity", async (controller) => { ... });
|
|
38
|
+
* vadController.eventHandler("on_push_frame", async (controller, frame, direction) => { ... });
|
|
39
|
+
* vadController.eventHandler("on_broadcast_frame", async (controller, frameCls, options) => { ... });
|
|
40
|
+
*/
|
|
41
|
+
export class VADController extends BaseObject {
|
|
42
|
+
_vadAnalyzer;
|
|
43
|
+
_vadState = VADState.QUIET;
|
|
44
|
+
// Last time a on_speech_activity was triggered.
|
|
45
|
+
_speechActivityTime = 0;
|
|
46
|
+
// How often a on_speech_activity event should be triggered (value should
|
|
47
|
+
// be greater than the audio chunks to have any effect).
|
|
48
|
+
_speechActivityPeriod;
|
|
49
|
+
// Audio idle detection: force speech stop when no audio arrives while in
|
|
50
|
+
// SPEAKING state (e.g. user mutes mic mid-speech).
|
|
51
|
+
_lastAudioTime = 0.0;
|
|
52
|
+
_audioIdleTimeout;
|
|
53
|
+
_audioIdleTask = null;
|
|
54
|
+
/**
|
|
55
|
+
* Initialize the VAD controller.
|
|
56
|
+
*
|
|
57
|
+
* @param vadAnalyzer The `VADAnalyzer` instance for processing audio.
|
|
58
|
+
* @param options Speech activity period and audio idle timeout.
|
|
59
|
+
*/
|
|
60
|
+
constructor(vadAnalyzer, options = {}) {
|
|
61
|
+
super();
|
|
62
|
+
this._vadAnalyzer = vadAnalyzer;
|
|
63
|
+
this._taskManager = null;
|
|
64
|
+
this._speechActivityPeriod = options.speechActivityPeriod ?? 0.2;
|
|
65
|
+
this._audioIdleTimeout = options.audioIdleTimeout ?? 1.0;
|
|
66
|
+
this._registerEventHandler("on_speech_started", true);
|
|
67
|
+
this._registerEventHandler("on_speech_stopped", true);
|
|
68
|
+
this._registerEventHandler("on_speech_activity", true);
|
|
69
|
+
this._registerEventHandler("on_push_frame", true);
|
|
70
|
+
this._registerEventHandler("on_broadcast_frame", true);
|
|
71
|
+
}
|
|
72
|
+
/**
|
|
73
|
+
* Set up the controller.
|
|
74
|
+
*
|
|
75
|
+
* @param setup Configuration object containing setup parameters.
|
|
76
|
+
*/
|
|
77
|
+
async setup(setup) {
|
|
78
|
+
await super.setup(setup.taskManager);
|
|
79
|
+
this._lastAudioTime = monotonic();
|
|
80
|
+
this._vadAnalyzer.setSampleRate(setup.audioInSampleRate);
|
|
81
|
+
}
|
|
82
|
+
/**
|
|
83
|
+
* Process a frame and handle VAD-related events.
|
|
84
|
+
*
|
|
85
|
+
* Handles `InputAudioRawFrame` to analyze audio for voice activity, and
|
|
86
|
+
* `VADParamsUpdateFrame` to reconfigure the analyzer.
|
|
87
|
+
*
|
|
88
|
+
* @param frame The frame to process.
|
|
89
|
+
*/
|
|
90
|
+
async processFrame(frame) {
|
|
91
|
+
if (frame instanceof InputAudioRawFrame) {
|
|
92
|
+
await this._handleAudio(frame);
|
|
93
|
+
}
|
|
94
|
+
else if (frame instanceof VADParamsUpdateFrame) {
|
|
95
|
+
this._vadAnalyzer.setParams(frame.params);
|
|
96
|
+
await this.broadcastFrame(SpeechControlParamsFrame, { vadParams: frame.params });
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
/**
|
|
100
|
+
* Announce the analyzer's params and start the idle timer.
|
|
101
|
+
*
|
|
102
|
+
* Paired with `stop()`.
|
|
103
|
+
*/
|
|
104
|
+
async start() {
|
|
105
|
+
// Broadcast initial VAD params so other services (e.g. STT) can use them
|
|
106
|
+
await this.broadcastFrame(SpeechControlParamsFrame, { vadParams: this._vadAnalyzer.params });
|
|
107
|
+
if (this._audioIdleTimeout > 0 && !this._audioIdleTask) {
|
|
108
|
+
this._audioIdleTask = this.createTask(() => this._audioIdleHandler(), "_audioIdleHandler");
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
/**
|
|
112
|
+
* Stop the idle timer, leaving the analyzer alone.
|
|
113
|
+
*
|
|
114
|
+
* Called at session end so the timer can't report silence that only means
|
|
115
|
+
* the session is over. The analyzer may be shared, so releasing it waits
|
|
116
|
+
* for `cleanup()`.
|
|
117
|
+
*/
|
|
118
|
+
async stop() {
|
|
119
|
+
if (this._audioIdleTask && this._taskManager) {
|
|
120
|
+
await this._taskManager.cancelTask(this._audioIdleTask);
|
|
121
|
+
this._audioIdleTask = null;
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
/**
|
|
125
|
+
* Clean up resources.
|
|
126
|
+
*
|
|
127
|
+
* This method should be called when the object is no longer needed. It
|
|
128
|
+
* waits for all currently executing event handler tasks to finish before
|
|
129
|
+
* returning.
|
|
130
|
+
*/
|
|
131
|
+
async cleanup() {
|
|
132
|
+
await super.cleanup();
|
|
133
|
+
await this.stop();
|
|
134
|
+
if (this._vadAnalyzer) {
|
|
135
|
+
await this._vadAnalyzer.cleanup();
|
|
136
|
+
}
|
|
137
|
+
}
|
|
138
|
+
/**
|
|
139
|
+
* Process an audio chunk and emit speech events as needed.
|
|
140
|
+
*
|
|
141
|
+
* Analyzes the audio for voice activity and triggers `on_speech_started`,
|
|
142
|
+
* `on_speech_stopped`, or `on_speech_activity` events based on state
|
|
143
|
+
* changes.
|
|
144
|
+
*
|
|
145
|
+
* @param frame Audio frame to process.
|
|
146
|
+
*/
|
|
147
|
+
async _handleAudio(frame) {
|
|
148
|
+
this._lastAudioTime = monotonic();
|
|
149
|
+
this._vadState = await this._handleVad(frame.audio, this._vadState);
|
|
150
|
+
if (this._vadState === VADState.SPEAKING) {
|
|
151
|
+
await this._maybeSpeechActivity();
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
/** Handle Voice Activity Detection results and trigger appropriate events. */
|
|
155
|
+
async _handleVad(audio, vadState) {
|
|
156
|
+
const newVadState = await this._vadAnalyzer.analyzeAudio(audio);
|
|
157
|
+
if (newVadState !== vadState &&
|
|
158
|
+
newVadState !== VADState.STARTING &&
|
|
159
|
+
newVadState !== VADState.STOPPING) {
|
|
160
|
+
if (newVadState === VADState.SPEAKING) {
|
|
161
|
+
await this._callEventHandler("on_speech_started");
|
|
162
|
+
}
|
|
163
|
+
else if (newVadState === VADState.QUIET) {
|
|
164
|
+
await this._callEventHandler("on_speech_stopped");
|
|
165
|
+
}
|
|
166
|
+
vadState = newVadState;
|
|
167
|
+
}
|
|
168
|
+
return vadState;
|
|
169
|
+
}
|
|
170
|
+
/**
|
|
171
|
+
* Monitor for an idle audio stream while in SPEAKING state.
|
|
172
|
+
*
|
|
173
|
+
* When no audio frames arrive for `audioIdleTimeout` seconds (e.g. user
|
|
174
|
+
* mutes mic mid-speech), forces a transition to QUIET and emits
|
|
175
|
+
* `on_speech_stopped`.
|
|
176
|
+
*/
|
|
177
|
+
async _audioIdleHandler() {
|
|
178
|
+
while (true) {
|
|
179
|
+
const deadline = this._lastAudioTime + this._audioIdleTimeout;
|
|
180
|
+
const remaining = deadline - monotonic();
|
|
181
|
+
if (remaining > 0) {
|
|
182
|
+
// Audio is still recent; sleep only for the remaining window.
|
|
183
|
+
await sleep(remaining);
|
|
184
|
+
continue;
|
|
185
|
+
}
|
|
186
|
+
if (this._vadState === VADState.SPEAKING) {
|
|
187
|
+
logger.warning(`${this}: no audio received while speaking, forcing speech stop`);
|
|
188
|
+
this._vadState = VADState.QUIET;
|
|
189
|
+
await this._callEventHandler("on_speech_stopped");
|
|
190
|
+
}
|
|
191
|
+
// Wait for the next potential idle window.
|
|
192
|
+
await sleep(this._audioIdleTimeout);
|
|
193
|
+
}
|
|
194
|
+
}
|
|
195
|
+
/** Handle user speaking frame. */
|
|
196
|
+
async _maybeSpeechActivity() {
|
|
197
|
+
const diffTime = time() - this._speechActivityTime;
|
|
198
|
+
if (diffTime >= this._speechActivityPeriod) {
|
|
199
|
+
this._speechActivityTime = time();
|
|
200
|
+
await this._callEventHandler("on_speech_activity");
|
|
201
|
+
}
|
|
202
|
+
}
|
|
203
|
+
/**
|
|
204
|
+
* Request a frame to be pushed through the pipeline.
|
|
205
|
+
*
|
|
206
|
+
* This emits an on_push_frame event that must be handled by a processor to
|
|
207
|
+
* actually push the frame into the pipeline.
|
|
208
|
+
*
|
|
209
|
+
* @param frame The frame to push.
|
|
210
|
+
* @param direction The direction to push the frame.
|
|
211
|
+
*/
|
|
212
|
+
async pushFrame(frame, direction = FrameDirection.DOWNSTREAM) {
|
|
213
|
+
await this._callEventHandler("on_push_frame", frame, direction);
|
|
214
|
+
}
|
|
215
|
+
/**
|
|
216
|
+
* Request a frame to be broadcast upstream and downstream.
|
|
217
|
+
*
|
|
218
|
+
* This emits an on_broadcast_frame event that must be handled by a
|
|
219
|
+
* processor to actually broadcast the frame in the pipeline.
|
|
220
|
+
*
|
|
221
|
+
* @param frameCls The class of the frame to broadcast.
|
|
222
|
+
* @param options Arguments to pass to the frame constructor.
|
|
223
|
+
*/
|
|
224
|
+
async broadcastFrame(frameCls, options) {
|
|
225
|
+
await this._callEventHandler("on_broadcast_frame", frameCls, options);
|
|
226
|
+
}
|
|
227
|
+
}
|
|
228
|
+
//# sourceMappingURL=vad_controller.js.map
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
export declare const VOLUME_WINDOW_SECS = 0.4;
|
|
2
|
+
/**
|
|
3
|
+
* Tracks the volume of an audio stream over a rolling window.
|
|
4
|
+
*
|
|
5
|
+
* Audio is fed in chunks of any size with `update()`, which retains the most
|
|
6
|
+
* recent `VOLUME_WINDOW_SECS` of it, and `volume` measures that window. Volume
|
|
7
|
+
* reads 0 until the window holds enough audio for loudness to be measurable,
|
|
8
|
+
* and the window is discarded if the sample rate changes.
|
|
9
|
+
*
|
|
10
|
+
* Measuring costs a few hundred microseconds and grows with the sample rate,
|
|
11
|
+
* so it happens on read and the result is cached until more audio arrives.
|
|
12
|
+
* Callers that report volume less often than they receive audio pay only for
|
|
13
|
+
* the reads.
|
|
14
|
+
*
|
|
15
|
+
* Audio is expected to be mono; interleaved channels would be measured as if
|
|
16
|
+
* they were consecutive samples.
|
|
17
|
+
*/
|
|
18
|
+
export declare class AudioVolumeTracker {
|
|
19
|
+
protected _sampleRate: number;
|
|
20
|
+
protected _windowNumBytes: number;
|
|
21
|
+
protected _buffer: Uint8Array;
|
|
22
|
+
protected _volume: number | null;
|
|
23
|
+
/**
|
|
24
|
+
* The volume of the audio in the rolling window, between 0 (quiet) and 1
|
|
25
|
+
* (loud). Reads 0 until the window holds a measurable amount of audio.
|
|
26
|
+
*/
|
|
27
|
+
get volume(): number;
|
|
28
|
+
/**
|
|
29
|
+
* Add audio to the rolling window.
|
|
30
|
+
*
|
|
31
|
+
* @param audio Audio data as raw bytes (16-bit signed integers, mono).
|
|
32
|
+
* @param sampleRate Sample rate of the audio in Hz.
|
|
33
|
+
*/
|
|
34
|
+
update(audio: Uint8Array, sampleRate: number): void;
|
|
35
|
+
/** Clear the rolling window and the tracked volume. */
|
|
36
|
+
reset(): void;
|
|
37
|
+
}
|
|
38
|
+
//# sourceMappingURL=volume.d.ts.map
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
// Copyright (c) 2024-2026, Daily
|
|
2
|
+
// Copyright (c) 2026, pipedog contributors
|
|
3
|
+
// SPDX-License-Identifier: BSD-2-Clause
|
|
4
|
+
//
|
|
5
|
+
// Ported from pipecat: src/pipecat/audio/volume.py
|
|
6
|
+
/** Audio volume tracking over a rolling window of recent audio. */
|
|
7
|
+
import { calculateAudioVolume } from "./utils.js";
|
|
8
|
+
// Loudness is measured over a BS.1770 gating block, which is 400ms. The audio
|
|
9
|
+
// chunks flowing through a pipeline are shorter than that, so they accumulate
|
|
10
|
+
// into a rolling window of this size.
|
|
11
|
+
export const VOLUME_WINDOW_SECS = 0.4;
|
|
12
|
+
/**
|
|
13
|
+
* Tracks the volume of an audio stream over a rolling window.
|
|
14
|
+
*
|
|
15
|
+
* Audio is fed in chunks of any size with `update()`, which retains the most
|
|
16
|
+
* recent `VOLUME_WINDOW_SECS` of it, and `volume` measures that window. Volume
|
|
17
|
+
* reads 0 until the window holds enough audio for loudness to be measurable,
|
|
18
|
+
* and the window is discarded if the sample rate changes.
|
|
19
|
+
*
|
|
20
|
+
* Measuring costs a few hundred microseconds and grows with the sample rate,
|
|
21
|
+
* so it happens on read and the result is cached until more audio arrives.
|
|
22
|
+
* Callers that report volume less often than they receive audio pay only for
|
|
23
|
+
* the reads.
|
|
24
|
+
*
|
|
25
|
+
* Audio is expected to be mono; interleaved channels would be measured as if
|
|
26
|
+
* they were consecutive samples.
|
|
27
|
+
*/
|
|
28
|
+
export class AudioVolumeTracker {
|
|
29
|
+
_sampleRate = 0;
|
|
30
|
+
_windowNumBytes = 0;
|
|
31
|
+
_buffer = new Uint8Array(0);
|
|
32
|
+
// null once the window holds audio that hasn't been measured yet.
|
|
33
|
+
_volume = 0.0;
|
|
34
|
+
/**
|
|
35
|
+
* The volume of the audio in the rolling window, between 0 (quiet) and 1
|
|
36
|
+
* (loud). Reads 0 until the window holds a measurable amount of audio.
|
|
37
|
+
*/
|
|
38
|
+
get volume() {
|
|
39
|
+
if (this._volume === null) {
|
|
40
|
+
this._volume = calculateAudioVolume(this._buffer, this._sampleRate);
|
|
41
|
+
}
|
|
42
|
+
return this._volume;
|
|
43
|
+
}
|
|
44
|
+
/**
|
|
45
|
+
* Add audio to the rolling window.
|
|
46
|
+
*
|
|
47
|
+
* @param audio Audio data as raw bytes (16-bit signed integers, mono).
|
|
48
|
+
* @param sampleRate Sample rate of the audio in Hz.
|
|
49
|
+
*/
|
|
50
|
+
update(audio, sampleRate) {
|
|
51
|
+
if (sampleRate !== this._sampleRate) {
|
|
52
|
+
this._sampleRate = sampleRate;
|
|
53
|
+
// Rounded up so the window is never a sample short of a gating block,
|
|
54
|
+
// which loudness rejects.
|
|
55
|
+
this._windowNumBytes = Math.ceil(VOLUME_WINDOW_SECS * sampleRate) * 2;
|
|
56
|
+
this.reset();
|
|
57
|
+
}
|
|
58
|
+
const combined = new Uint8Array(this._buffer.length + audio.length);
|
|
59
|
+
combined.set(this._buffer, 0);
|
|
60
|
+
combined.set(audio, this._buffer.length);
|
|
61
|
+
this._buffer =
|
|
62
|
+
combined.length > this._windowNumBytes
|
|
63
|
+
? combined.slice(combined.length - this._windowNumBytes)
|
|
64
|
+
: combined;
|
|
65
|
+
if (this._buffer.length === this._windowNumBytes) {
|
|
66
|
+
this._volume = null;
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
/** Clear the rolling window and the tracked volume. */
|
|
70
|
+
reset() {
|
|
71
|
+
this._buffer = new Uint8Array(0);
|
|
72
|
+
this._volume = 0.0;
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
//# sourceMappingURL=volume.js.map
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
/** Abstract base class for type adapters used by message serializers. */
|
|
2
|
+
import type { Ctor } from "../../utils/mixin.ts";
|
|
3
|
+
export type SerializeFunc = (value: unknown) => unknown;
|
|
4
|
+
export type DeserializeFunc = (value: unknown) => unknown;
|
|
5
|
+
/**
|
|
6
|
+
* Serialize and deserialize instances of a specific type for network transport.
|
|
7
|
+
*
|
|
8
|
+
* Each adapter handles one or more types, converting them to/from a
|
|
9
|
+
* JSON-compatible dict. Register adapters on a `JSONMessageSerializer` to
|
|
10
|
+
* handle non-JSON-native field values (e.g. `LLMContext`, `ToolsSchema`).
|
|
11
|
+
*
|
|
12
|
+
* Adapters receive `serializeValue` and `deserializeValue` callbacks from the
|
|
13
|
+
* serializer so they can recursively serialize nested fields without
|
|
14
|
+
* importing the serializer itself.
|
|
15
|
+
*/
|
|
16
|
+
export declare abstract class TypeAdapter {
|
|
17
|
+
/**
|
|
18
|
+
* Convert an object to a JSON-compatible dict.
|
|
19
|
+
*
|
|
20
|
+
* @param obj The object to serialize.
|
|
21
|
+
* @param serializeValue Callback to recursively serialize nested values.
|
|
22
|
+
* @returns A dict representation of the object.
|
|
23
|
+
*/
|
|
24
|
+
abstract serialize(obj: unknown, serializeValue: SerializeFunc): Record<string, unknown>;
|
|
25
|
+
/**
|
|
26
|
+
* Reconstruct an object from a dict.
|
|
27
|
+
*
|
|
28
|
+
* @param data The dict representation produced by `serialize()`.
|
|
29
|
+
* @param deserializeValue Callback to recursively deserialize nested values.
|
|
30
|
+
* @param targetType The resolved target class. Defaults to null.
|
|
31
|
+
* @returns The reconstructed object.
|
|
32
|
+
*/
|
|
33
|
+
abstract deserialize(data: Record<string, unknown>, deserializeValue: DeserializeFunc, targetType?: Ctor | null): unknown;
|
|
34
|
+
}
|
|
35
|
+
//# sourceMappingURL=base.d.ts.map
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
// Copyright (c) 2024-2026, Daily
|
|
2
|
+
// Copyright (c) 2026, pipedog contributors
|
|
3
|
+
// SPDX-License-Identifier: BSD-2-Clause
|
|
4
|
+
//
|
|
5
|
+
// Ported from pipecat: src/pipecat/bus/adapters/base.py
|
|
6
|
+
/**
|
|
7
|
+
* Serialize and deserialize instances of a specific type for network transport.
|
|
8
|
+
*
|
|
9
|
+
* Each adapter handles one or more types, converting them to/from a
|
|
10
|
+
* JSON-compatible dict. Register adapters on a `JSONMessageSerializer` to
|
|
11
|
+
* handle non-JSON-native field values (e.g. `LLMContext`, `ToolsSchema`).
|
|
12
|
+
*
|
|
13
|
+
* Adapters receive `serializeValue` and `deserializeValue` callbacks from the
|
|
14
|
+
* serializer so they can recursively serialize nested fields without
|
|
15
|
+
* importing the serializer itself.
|
|
16
|
+
*/
|
|
17
|
+
export class TypeAdapter {
|
|
18
|
+
}
|
|
19
|
+
//# sourceMappingURL=base.js.map
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Type adapters for bus message serialization.
|
|
3
|
+
*
|
|
4
|
+
* Provides ready-made `TypeAdapter` implementations for common Pipecat types
|
|
5
|
+
* (`LLMContext`, `ToolsSchema`) used in bus messages.
|
|
6
|
+
*/
|
|
7
|
+
export { TypeAdapter } from "./base.ts";
|
|
8
|
+
export { LLMContextAdapter } from "./llm_context_adapter.ts";
|
|
9
|
+
export { ToolsSchemaAdapter } from "./tools_schema_adapter.ts";
|
|
10
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
// Copyright (c) 2024-2026, Daily
|
|
2
|
+
// Copyright (c) 2026, pipedog contributors
|
|
3
|
+
// SPDX-License-Identifier: BSD-2-Clause
|
|
4
|
+
//
|
|
5
|
+
// Ported from pipecat: src/pipecat/bus/adapters/__init__.py
|
|
6
|
+
/**
|
|
7
|
+
* Type adapters for bus message serialization.
|
|
8
|
+
*
|
|
9
|
+
* Provides ready-made `TypeAdapter` implementations for common Pipecat types
|
|
10
|
+
* (`LLMContext`, `ToolsSchema`) used in bus messages.
|
|
11
|
+
*/
|
|
12
|
+
export { TypeAdapter } from "./base.js";
|
|
13
|
+
export { LLMContextAdapter } from "./llm_context_adapter.js";
|
|
14
|
+
export { ToolsSchemaAdapter } from "./tools_schema_adapter.js";
|
|
15
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
import { type LLMContextMessage } from "../../processors/aggregators/llm_context.ts";
|
|
2
|
+
import type { Ctor } from "../../utils/mixin.ts";
|
|
3
|
+
import { type DeserializeFunc, type SerializeFunc, TypeAdapter } from "./base.ts";
|
|
4
|
+
import { ToolsSchemaAdapter } from "./tools_schema_adapter.ts";
|
|
5
|
+
/**
|
|
6
|
+
* Serialize and deserialize `LLMContext` instances.
|
|
7
|
+
*
|
|
8
|
+
* The `NOT_GIVEN` sentinel is preserved across serialization: missing keys
|
|
9
|
+
* are restored as `NOT_GIVEN` on deserialization.
|
|
10
|
+
*/
|
|
11
|
+
export declare class LLMContextAdapter extends TypeAdapter {
|
|
12
|
+
protected _toolsSchemaAdapter: ToolsSchemaAdapter;
|
|
13
|
+
/**
|
|
14
|
+
* Serialize an `LLMContext` to a JSON-compatible dict.
|
|
15
|
+
*
|
|
16
|
+
* @param obj An `LLMContext` instance.
|
|
17
|
+
* @param serializeValue Callback to recursively serialize nested values.
|
|
18
|
+
* @returns A dict with `messages` and, optionally, `tools` and
|
|
19
|
+
* `tool_choice` keys.
|
|
20
|
+
*/
|
|
21
|
+
serialize(obj: unknown, serializeValue: SerializeFunc): Record<string, unknown>;
|
|
22
|
+
/**
|
|
23
|
+
* Reconstruct an `LLMContext` from a serialized dict.
|
|
24
|
+
*
|
|
25
|
+
* Missing `tools` and `tool_choice` keys are restored as `NOT_GIVEN`.
|
|
26
|
+
*
|
|
27
|
+
* @param data A dict produced by `serialize()`.
|
|
28
|
+
* @param deserializeValue Callback to recursively deserialize nested values.
|
|
29
|
+
* @param _targetType Unused. `LLMContext` is always the target.
|
|
30
|
+
* @returns A new `LLMContext` instance.
|
|
31
|
+
*/
|
|
32
|
+
deserialize(data: Record<string, unknown>, deserializeValue: DeserializeFunc, _targetType?: Ctor | null): unknown;
|
|
33
|
+
protected _serializeMessage(msg: unknown, serializeValue: SerializeFunc): unknown;
|
|
34
|
+
protected _deserializeMessage(data: unknown, deserializeValue: DeserializeFunc): LLMContextMessage;
|
|
35
|
+
}
|
|
36
|
+
//# sourceMappingURL=llm_context_adapter.d.ts.map
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
// Copyright (c) 2024-2026, Daily
|
|
2
|
+
// Copyright (c) 2026, pipedog contributors
|
|
3
|
+
// SPDX-License-Identifier: BSD-2-Clause
|
|
4
|
+
//
|
|
5
|
+
// Ported from pipecat: src/pipecat/bus/adapters/llm_context_adapter.py
|
|
6
|
+
import { LLMContext, LLMSpecificMessage, NOT_GIVEN, } from "../../processors/aggregators/llm_context.js";
|
|
7
|
+
import { isGiven } from "../../utils/types.js";
|
|
8
|
+
import { TypeAdapter } from "./base.js";
|
|
9
|
+
import { ToolsSchemaAdapter } from "./tools_schema_adapter.js";
|
|
10
|
+
function isDict(value) {
|
|
11
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
12
|
+
}
|
|
13
|
+
/**
|
|
14
|
+
* Serialize and deserialize `LLMContext` instances.
|
|
15
|
+
*
|
|
16
|
+
* The `NOT_GIVEN` sentinel is preserved across serialization: missing keys
|
|
17
|
+
* are restored as `NOT_GIVEN` on deserialization.
|
|
18
|
+
*/
|
|
19
|
+
export class LLMContextAdapter extends TypeAdapter {
|
|
20
|
+
_toolsSchemaAdapter = new ToolsSchemaAdapter();
|
|
21
|
+
/**
|
|
22
|
+
* Serialize an `LLMContext` to a JSON-compatible dict.
|
|
23
|
+
*
|
|
24
|
+
* @param obj An `LLMContext` instance.
|
|
25
|
+
* @param serializeValue Callback to recursively serialize nested values.
|
|
26
|
+
* @returns A dict with `messages` and, optionally, `tools` and
|
|
27
|
+
* `tool_choice` keys.
|
|
28
|
+
*/
|
|
29
|
+
serialize(obj, serializeValue) {
|
|
30
|
+
const context = obj;
|
|
31
|
+
const result = {
|
|
32
|
+
messages: context.messages.map((m) => this._serializeMessage(m, serializeValue)),
|
|
33
|
+
};
|
|
34
|
+
if (isGiven(context.tools)) {
|
|
35
|
+
result.tools = this._toolsSchemaAdapter.serialize(context.tools, serializeValue);
|
|
36
|
+
}
|
|
37
|
+
if (isGiven(context.toolChoice)) {
|
|
38
|
+
result.tool_choice = serializeValue(context.toolChoice);
|
|
39
|
+
}
|
|
40
|
+
return result;
|
|
41
|
+
}
|
|
42
|
+
/**
|
|
43
|
+
* Reconstruct an `LLMContext` from a serialized dict.
|
|
44
|
+
*
|
|
45
|
+
* Missing `tools` and `tool_choice` keys are restored as `NOT_GIVEN`.
|
|
46
|
+
*
|
|
47
|
+
* @param data A dict produced by `serialize()`.
|
|
48
|
+
* @param deserializeValue Callback to recursively deserialize nested values.
|
|
49
|
+
* @param _targetType Unused. `LLMContext` is always the target.
|
|
50
|
+
* @returns A new `LLMContext` instance.
|
|
51
|
+
*/
|
|
52
|
+
deserialize(data, deserializeValue, _targetType = null) {
|
|
53
|
+
const messages = data.messages.map((m) => this._deserializeMessage(m, deserializeValue));
|
|
54
|
+
const tools = "tools" in data
|
|
55
|
+
? this._toolsSchemaAdapter.deserialize(data.tools, deserializeValue)
|
|
56
|
+
: NOT_GIVEN;
|
|
57
|
+
const toolChoice = "tool_choice" in data
|
|
58
|
+
? deserializeValue(data.tool_choice)
|
|
59
|
+
: NOT_GIVEN;
|
|
60
|
+
return new LLMContext({ messages, tools, toolChoice });
|
|
61
|
+
}
|
|
62
|
+
_serializeMessage(msg, serializeValue) {
|
|
63
|
+
if (msg instanceof LLMSpecificMessage) {
|
|
64
|
+
return {
|
|
65
|
+
__specific__: true,
|
|
66
|
+
llm: msg.llm,
|
|
67
|
+
message: serializeValue(msg.message),
|
|
68
|
+
};
|
|
69
|
+
}
|
|
70
|
+
return serializeValue(msg);
|
|
71
|
+
}
|
|
72
|
+
_deserializeMessage(data, deserializeValue) {
|
|
73
|
+
if (isDict(data) && data.__specific__) {
|
|
74
|
+
return new LLMSpecificMessage({
|
|
75
|
+
llm: data.llm,
|
|
76
|
+
message: deserializeValue(data.message),
|
|
77
|
+
});
|
|
78
|
+
}
|
|
79
|
+
return deserializeValue(data);
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
//# sourceMappingURL=llm_context_adapter.js.map
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
import type { Ctor } from "../../utils/mixin.ts";
|
|
2
|
+
import { type DeserializeFunc, type SerializeFunc, TypeAdapter } from "./base.ts";
|
|
3
|
+
/** Serialize and deserialize `ToolsSchema` instances for network transport. */
|
|
4
|
+
export declare class ToolsSchemaAdapter extends TypeAdapter {
|
|
5
|
+
/**
|
|
6
|
+
* Serialize a `ToolsSchema` to a JSON-compatible dict.
|
|
7
|
+
*
|
|
8
|
+
* @param obj A `ToolsSchema` instance.
|
|
9
|
+
* @param _serializeValue Callback to recursively serialize nested values.
|
|
10
|
+
* @returns A dict with a `standard_tools` list and, if present, a
|
|
11
|
+
* `custom_tools` mapping.
|
|
12
|
+
*/
|
|
13
|
+
serialize(obj: unknown, _serializeValue: SerializeFunc): Record<string, unknown>;
|
|
14
|
+
/**
|
|
15
|
+
* Reconstruct a `ToolsSchema` from a serialized dict.
|
|
16
|
+
*
|
|
17
|
+
* @param data A dict produced by `serialize()`.
|
|
18
|
+
* @param _deserializeValue Callback to recursively deserialize nested values.
|
|
19
|
+
* @param _targetType Unused. `ToolsSchema` is always the target.
|
|
20
|
+
* @returns A new `ToolsSchema` instance, including `customTools` if present
|
|
21
|
+
* in `data`.
|
|
22
|
+
*/
|
|
23
|
+
deserialize(data: Record<string, unknown>, _deserializeValue: DeserializeFunc, _targetType?: Ctor | null): unknown;
|
|
24
|
+
}
|
|
25
|
+
//# sourceMappingURL=tools_schema_adapter.d.ts.map
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
// Copyright (c) 2024-2026, Daily
|
|
2
|
+
// Copyright (c) 2026, pipedog contributors
|
|
3
|
+
// SPDX-License-Identifier: BSD-2-Clause
|
|
4
|
+
//
|
|
5
|
+
// Ported from pipecat: src/pipecat/bus/adapters/tools_schema_adapter.py
|
|
6
|
+
/** Type adapter for ToolsSchema serialization. */
|
|
7
|
+
import { FunctionSchema } from "../../adapters/schemas/function_schema.js";
|
|
8
|
+
import { AdapterType, ToolsSchema } from "../../adapters/schemas/tools_schema.js";
|
|
9
|
+
import { TypeAdapter } from "./base.js";
|
|
10
|
+
const ADAPTER_TYPES = Object.values(AdapterType);
|
|
11
|
+
/** Python's `AdapterType(value)`: the adapter type with this value, or a ValueError. */
|
|
12
|
+
function toAdapterType(value) {
|
|
13
|
+
if (!ADAPTER_TYPES.includes(value)) {
|
|
14
|
+
throw new Error(`'${value}' is not a valid AdapterType`);
|
|
15
|
+
}
|
|
16
|
+
return value;
|
|
17
|
+
}
|
|
18
|
+
/** Serialize and deserialize `ToolsSchema` instances for network transport. */
|
|
19
|
+
export class ToolsSchemaAdapter extends TypeAdapter {
|
|
20
|
+
/**
|
|
21
|
+
* Serialize a `ToolsSchema` to a JSON-compatible dict.
|
|
22
|
+
*
|
|
23
|
+
* @param obj A `ToolsSchema` instance.
|
|
24
|
+
* @param _serializeValue Callback to recursively serialize nested values.
|
|
25
|
+
* @returns A dict with a `standard_tools` list and, if present, a
|
|
26
|
+
* `custom_tools` mapping.
|
|
27
|
+
*/
|
|
28
|
+
serialize(obj, _serializeValue) {
|
|
29
|
+
const schema = obj;
|
|
30
|
+
const result = {
|
|
31
|
+
standard_tools: schema.standardTools.map((tool) => tool.toDefaultDict()),
|
|
32
|
+
};
|
|
33
|
+
const customTools = schema.customTools;
|
|
34
|
+
if (customTools && Object.keys(customTools).length > 0) {
|
|
35
|
+
// AdapterType values are the wire keys already.
|
|
36
|
+
result.custom_tools = { ...customTools };
|
|
37
|
+
}
|
|
38
|
+
return result;
|
|
39
|
+
}
|
|
40
|
+
/**
|
|
41
|
+
* Reconstruct a `ToolsSchema` from a serialized dict.
|
|
42
|
+
*
|
|
43
|
+
* @param data A dict produced by `serialize()`.
|
|
44
|
+
* @param _deserializeValue Callback to recursively deserialize nested values.
|
|
45
|
+
* @param _targetType Unused. `ToolsSchema` is always the target.
|
|
46
|
+
* @returns A new `ToolsSchema` instance, including `customTools` if present
|
|
47
|
+
* in `data`.
|
|
48
|
+
*/
|
|
49
|
+
deserialize(data, _deserializeValue, _targetType = null) {
|
|
50
|
+
const tools = [];
|
|
51
|
+
for (const item of data.standard_tools) {
|
|
52
|
+
const params = (item.parameters ?? {});
|
|
53
|
+
tools.push(new FunctionSchema({
|
|
54
|
+
name: item.name,
|
|
55
|
+
description: (item.description ?? ""),
|
|
56
|
+
properties: (params.properties ?? {}),
|
|
57
|
+
required: (params.required ?? []),
|
|
58
|
+
}));
|
|
59
|
+
}
|
|
60
|
+
let customTools = null;
|
|
61
|
+
if ("custom_tools" in data) {
|
|
62
|
+
customTools = {};
|
|
63
|
+
for (const [key, value] of Object.entries(data.custom_tools)) {
|
|
64
|
+
customTools[toAdapterType(key)] = value;
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
return new ToolsSchema({ standardTools: tools, customTools });
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
//# sourceMappingURL=tools_schema_adapter.js.map
|