@pipedog/core 0.0.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +25 -0
- package/NOTICE +9 -0
- package/dist/adapters/base_llm_adapter.d.ts +266 -0
- package/dist/adapters/base_llm_adapter.js +489 -0
- package/dist/adapters/index.d.ts +2 -0
- package/dist/adapters/index.js +8 -0
- package/dist/adapters/schemas/direct_function.d.ts +202 -0
- package/dist/adapters/schemas/direct_function.js +302 -0
- package/dist/adapters/schemas/function_schema.d.ts +67 -0
- package/dist/adapters/schemas/function_schema.js +72 -0
- package/dist/adapters/schemas/index.d.ts +2 -0
- package/dist/adapters/schemas/index.js +8 -0
- package/dist/adapters/schemas/tools_schema.d.ts +57 -0
- package/dist/adapters/schemas/tools_schema.js +79 -0
- package/dist/adapters/services/index.d.ts +2 -0
- package/dist/adapters/services/index.js +8 -0
- package/dist/adapters/services/open_ai_adapter.d.ts +147 -0
- package/dist/adapters/services/open_ai_adapter.js +250 -0
- package/dist/audio/dtmf/dtmf_data.d.ts +3 -0
- package/dist/audio/dtmf/dtmf_data.js +22 -0
- package/dist/audio/dtmf/index.d.ts +2 -0
- package/dist/audio/dtmf/index.js +8 -0
- package/dist/audio/dtmf/types.d.ts +31 -0
- package/dist/audio/dtmf/types.js +35 -0
- package/dist/audio/dtmf/utils.d.ts +13 -0
- package/dist/audio/dtmf/utils.js +62 -0
- package/dist/audio/filters/base_audio_filter.d.ts +51 -0
- package/dist/audio/filters/base_audio_filter.js +17 -0
- package/dist/audio/filters/index.d.ts +2 -0
- package/dist/audio/filters/index.js +8 -0
- package/dist/audio/index.d.ts +2 -0
- package/dist/audio/index.js +8 -0
- package/dist/audio/krisp_instance.d.ts +95 -0
- package/dist/audio/krisp_instance.js +229 -0
- package/dist/audio/mixers/base_audio_mixer.d.ts +56 -0
- package/dist/audio/mixers/base_audio_mixer.js +16 -0
- package/dist/audio/mixers/index.d.ts +2 -0
- package/dist/audio/mixers/index.js +8 -0
- package/dist/audio/resamplers/base_audio_resampler.d.ts +46 -0
- package/dist/audio/resamplers/base_audio_resampler.js +35 -0
- package/dist/audio/resamplers/engine.d.ts +60 -0
- package/dist/audio/resamplers/engine.js +89 -0
- package/dist/audio/resamplers/index.d.ts +2 -0
- package/dist/audio/resamplers/index.js +8 -0
- package/dist/audio/resamplers/resampy_resampler.d.ts +10 -0
- package/dist/audio/resamplers/resampy_resampler.js +15 -0
- package/dist/audio/resamplers/sinc_engine.d.ts +10 -0
- package/dist/audio/resamplers/sinc_engine.js +235 -0
- package/dist/audio/resamplers/soxr_resampler.d.ts +58 -0
- package/dist/audio/resamplers/soxr_resampler.js +79 -0
- package/dist/audio/resamplers/soxr_stream_resampler.d.ts +78 -0
- package/dist/audio/resamplers/soxr_stream_resampler.js +159 -0
- package/dist/audio/turn/base_turn_analyzer.d.ts +84 -0
- package/dist/audio/turn/base_turn_analyzer.js +61 -0
- package/dist/audio/turn/index.d.ts +2 -0
- package/dist/audio/turn/index.js +8 -0
- package/dist/audio/turn/smart_turn/_whisper_features.d.ts +30 -0
- package/dist/audio/turn/smart_turn/_whisper_features.js +225 -0
- package/dist/audio/turn/smart_turn/base_smart_turn.d.ts +94 -0
- package/dist/audio/turn/smart_turn/base_smart_turn.js +239 -0
- package/dist/audio/turn/smart_turn/local_smart_turn_v3.d.ts +76 -0
- package/dist/audio/turn/smart_turn/local_smart_turn_v3.js +151 -0
- package/dist/audio/utils.d.ts +224 -0
- package/dist/audio/utils.js +634 -0
- package/dist/audio/vad/data/index.d.ts +2 -0
- package/dist/audio/vad/data/index.js +8 -0
- package/dist/audio/vad/index.d.ts +2 -0
- package/dist/audio/vad/index.js +8 -0
- package/dist/audio/vad/vad_analyzer.d.ts +134 -0
- package/dist/audio/vad/vad_analyzer.js +249 -0
- package/dist/audio/vad/vad_controller.d.ts +143 -0
- package/dist/audio/vad/vad_controller.js +228 -0
- package/dist/audio/volume.d.ts +38 -0
- package/dist/audio/volume.js +75 -0
- package/dist/bus/adapters/base.d.ts +35 -0
- package/dist/bus/adapters/base.js +19 -0
- package/dist/bus/adapters/index.d.ts +10 -0
- package/dist/bus/adapters/index.js +15 -0
- package/dist/bus/adapters/llm_context_adapter.d.ts +36 -0
- package/dist/bus/adapters/llm_context_adapter.js +82 -0
- package/dist/bus/adapters/tools_schema_adapter.d.ts +25 -0
- package/dist/bus/adapters/tools_schema_adapter.js +70 -0
- package/dist/bus/bridge_processor.d.ts +111 -0
- package/dist/bus/bridge_processor.js +166 -0
- package/dist/bus/bus.d.ts +110 -0
- package/dist/bus/bus.js +200 -0
- package/dist/bus/index.d.ts +22 -0
- package/dist/bus/index.js +27 -0
- package/dist/bus/local/async_queue.d.ts +12 -0
- package/dist/bus/local/async_queue.js +21 -0
- package/dist/bus/local/index.d.ts +3 -0
- package/dist/bus/local/index.js +8 -0
- package/dist/bus/messages.d.ts +378 -0
- package/dist/bus/messages.js +357 -0
- package/dist/bus/network/index.d.ts +16 -0
- package/dist/bus/network/index.js +21 -0
- package/dist/bus/queue.d.ts +27 -0
- package/dist/bus/queue.js +51 -0
- package/dist/bus/serializers/base.d.ts +25 -0
- package/dist/bus/serializers/base.js +14 -0
- package/dist/bus/serializers/index.d.ts +9 -0
- package/dist/bus/serializers/index.js +14 -0
- package/dist/bus/serializers/json.d.ts +77 -0
- package/dist/bus/serializers/json.js +367 -0
- package/dist/bus/subscriber.d.ts +46 -0
- package/dist/bus/subscriber.js +27 -0
- package/dist/bus/ui/index.d.ts +3 -0
- package/dist/bus/ui/index.js +8 -0
- package/dist/bus/ui/messages.d.ts +186 -0
- package/dist/bus/ui/messages.js +171 -0
- package/dist/classifiers/base_classifier.d.ts +263 -0
- package/dist/classifiers/base_classifier.js +346 -0
- package/dist/classifiers/cloudflare/clef/classifier.d.ts +70 -0
- package/dist/classifiers/cloudflare/clef/classifier.js +204 -0
- package/dist/classifiers/cloudflare/clef/client.d.ts +104 -0
- package/dist/classifiers/cloudflare/clef/client.js +234 -0
- package/dist/classifiers/cloudflare/clef/index.d.ts +2 -0
- package/dist/classifiers/cloudflare/clef/index.js +8 -0
- package/dist/classifiers/cloudflare/index.d.ts +2 -0
- package/dist/classifiers/cloudflare/index.js +8 -0
- package/dist/classifiers/http_client.d.ts +70 -0
- package/dist/classifiers/http_client.js +161 -0
- package/dist/classifiers/index.d.ts +2 -0
- package/dist/classifiers/index.js +8 -0
- package/dist/classifiers/jev/classifier.d.ts +8 -0
- package/dist/classifiers/jev/classifier.js +13 -0
- package/dist/classifiers/jev/client.d.ts +8 -0
- package/dist/classifiers/jev/client.js +13 -0
- package/dist/classifiers/jev/index.d.ts +2 -0
- package/dist/classifiers/jev/index.js +8 -0
- package/dist/classifiers/llm/classifier.d.ts +72 -0
- package/dist/classifiers/llm/classifier.js +304 -0
- package/dist/classifiers/llm/index.d.ts +2 -0
- package/dist/classifiers/llm/index.js +8 -0
- package/dist/classifiers/openai/decisions/classifier.d.ts +77 -0
- package/dist/classifiers/openai/decisions/classifier.js +241 -0
- package/dist/classifiers/openai/decisions/client.d.ts +108 -0
- package/dist/classifiers/openai/decisions/client.js +234 -0
- package/dist/classifiers/openai/decisions/index.d.ts +2 -0
- package/dist/classifiers/openai/decisions/index.js +8 -0
- package/dist/classifiers/openai/index.d.ts +2 -0
- package/dist/classifiers/openai/index.js +8 -0
- package/dist/classifiers/typesafe/index.d.ts +2 -0
- package/dist/classifiers/typesafe/index.js +8 -0
- package/dist/classifiers/typesafe/jev/classifier.d.ts +70 -0
- package/dist/classifiers/typesafe/jev/classifier.js +220 -0
- package/dist/classifiers/typesafe/jev/client.d.ts +105 -0
- package/dist/classifiers/typesafe/jev/client.js +233 -0
- package/dist/classifiers/typesafe/jev/index.d.ts +2 -0
- package/dist/classifiers/typesafe/jev/index.js +8 -0
- package/dist/clocks/base_clock.d.ts +13 -0
- package/dist/clocks/base_clock.js +14 -0
- package/dist/clocks/index.d.ts +2 -0
- package/dist/clocks/index.js +8 -0
- package/dist/clocks/system_clock.d.ts +15 -0
- package/dist/clocks/system_clock.js +25 -0
- package/dist/frames/frames.d.ts +2533 -0
- package/dist/frames/frames.js +2485 -0
- package/dist/frames/index.d.ts +2 -0
- package/dist/frames/index.js +8 -0
- package/dist/frames/protobufs/frames_pb2.d.ts +229 -0
- package/dist/frames/protobufs/frames_pb2.js +466 -0
- package/dist/index.d.ts +11 -0
- package/dist/index.js +56 -0
- package/dist/metrics/index.d.ts +2 -0
- package/dist/metrics/index.js +8 -0
- package/dist/metrics/metrics.d.ts +276 -0
- package/dist/metrics/metrics.js +254 -0
- package/dist/observers/base_observer.d.ts +173 -0
- package/dist/observers/base_observer.js +151 -0
- package/dist/observers/error_observer.d.ts +75 -0
- package/dist/observers/error_observer.js +87 -0
- package/dist/observers/function_call_observer.d.ts +157 -0
- package/dist/observers/function_call_observer.js +222 -0
- package/dist/observers/index.d.ts +2 -0
- package/dist/observers/index.js +8 -0
- package/dist/observers/loggers/debug_log_observer.d.ts +91 -0
- package/dist/observers/loggers/debug_log_observer.js +228 -0
- package/dist/observers/loggers/index.d.ts +2 -0
- package/dist/observers/loggers/index.js +8 -0
- package/dist/observers/loggers/llm_log_observer.d.ts +24 -0
- package/dist/observers/loggers/llm_log_observer.js +110 -0
- package/dist/observers/loggers/metrics_log_observer.d.ts +85 -0
- package/dist/observers/loggers/metrics_log_observer.js +186 -0
- package/dist/observers/loggers/transcription_log_observer.d.ts +23 -0
- package/dist/observers/loggers/transcription_log_observer.js +74 -0
- package/dist/observers/service_metrics_observer.d.ts +168 -0
- package/dist/observers/service_metrics_observer.js +236 -0
- package/dist/observers/speaking_observer.d.ts +104 -0
- package/dist/observers/speaking_observer.js +150 -0
- package/dist/observers/startup_timing_observer.d.ts +221 -0
- package/dist/observers/startup_timing_observer.js +365 -0
- package/dist/observers/turn_tracking_observer.d.ts +78 -0
- package/dist/observers/turn_tracking_observer.js +189 -0
- package/dist/observers/user_bot_latency_observer.d.ts +411 -0
- package/dist/observers/user_bot_latency_observer.js +974 -0
- package/dist/pipeline/base_pipeline.d.ts +49 -0
- package/dist/pipeline/base_pipeline.js +101 -0
- package/dist/pipeline/capabilities.d.ts +76 -0
- package/dist/pipeline/capabilities.js +97 -0
- package/dist/pipeline/index.d.ts +2 -0
- package/dist/pipeline/index.js +8 -0
- package/dist/pipeline/job_context.d.ts +317 -0
- package/dist/pipeline/job_context.js +450 -0
- package/dist/pipeline/job_decorator.d.ts +54 -0
- package/dist/pipeline/job_decorator.js +92 -0
- package/dist/pipeline/llm_switcher.d.ts +84 -0
- package/dist/pipeline/llm_switcher.js +107 -0
- package/dist/pipeline/llm_with_backend.d.ts +435 -0
- package/dist/pipeline/llm_with_backend.js +806 -0
- package/dist/pipeline/parallel_pipeline.d.ts +83 -0
- package/dist/pipeline/parallel_pipeline.js +200 -0
- package/dist/pipeline/pipeline.d.ts +126 -0
- package/dist/pipeline/pipeline.js +196 -0
- package/dist/pipeline/service_switcher.d.ts +287 -0
- package/dist/pipeline/service_switcher.js +509 -0
- package/dist/pipeline/sync_parallel_pipeline.d.ts +192 -0
- package/dist/pipeline/sync_parallel_pipeline.js +387 -0
- package/dist/pipeline/worker.d.ts +821 -0
- package/dist/pipeline/worker.js +1656 -0
- package/dist/pipeline/worker_observer.d.ts +131 -0
- package/dist/pipeline/worker_observer.js +274 -0
- package/dist/pipeline/worker_ready_decorator.d.ts +45 -0
- package/dist/pipeline/worker_ready_decorator.js +76 -0
- package/dist/processors/aggregators/async_tool_messages.d.ts +146 -0
- package/dist/processors/aggregators/async_tool_messages.js +291 -0
- package/dist/processors/aggregators/dtmf_aggregator.d.ts +61 -0
- package/dist/processors/aggregators/dtmf_aggregator.js +144 -0
- package/dist/processors/aggregators/gated.d.ts +43 -0
- package/dist/processors/aggregators/gated.js +86 -0
- package/dist/processors/aggregators/gated_llm_context.d.ts +42 -0
- package/dist/processors/aggregators/gated_llm_context.js +86 -0
- package/dist/processors/aggregators/index.d.ts +2 -0
- package/dist/processors/aggregators/index.js +8 -0
- package/dist/processors/aggregators/llm_context.d.ts +370 -0
- package/dist/processors/aggregators/llm_context.js +580 -0
- package/dist/processors/aggregators/llm_context_summarizer.d.ts +219 -0
- package/dist/processors/aggregators/llm_context_summarizer.js +447 -0
- package/dist/processors/aggregators/llm_response.d.ts +37 -0
- package/dist/processors/aggregators/llm_response.js +71 -0
- package/dist/processors/aggregators/llm_response_universal.d.ts +966 -0
- package/dist/processors/aggregators/llm_response_universal.js +2394 -0
- package/dist/processors/aggregators/llm_text_processor.d.ts +53 -0
- package/dist/processors/aggregators/llm_text_processor.js +102 -0
- package/dist/processors/aggregators/sentence.d.ts +39 -0
- package/dist/processors/aggregators/sentence.js +69 -0
- package/dist/processors/async_generator.d.ts +42 -0
- package/dist/processors/async_generator.js +76 -0
- package/dist/processors/audio/audio_buffer_processor.d.ts +241 -0
- package/dist/processors/audio/audio_buffer_processor.js +565 -0
- package/dist/processors/audio/index.d.ts +2 -0
- package/dist/processors/audio/index.js +8 -0
- package/dist/processors/audio/vad_processor.d.ts +67 -0
- package/dist/processors/audio/vad_processor.js +103 -0
- package/dist/processors/consumer_processor.d.ts +49 -0
- package/dist/processors/consumer_processor.js +85 -0
- package/dist/processors/filters/frame_filter.d.ts +39 -0
- package/dist/processors/filters/frame_filter.js +52 -0
- package/dist/processors/filters/function_filter.d.ts +59 -0
- package/dist/processors/filters/function_filter.js +78 -0
- package/dist/processors/filters/identity_filter.d.ts +33 -0
- package/dist/processors/filters/identity_filter.js +37 -0
- package/dist/processors/filters/index.d.ts +2 -0
- package/dist/processors/filters/index.js +8 -0
- package/dist/processors/filters/null_filter.d.ts +34 -0
- package/dist/processors/filters/null_filter.js +47 -0
- package/dist/processors/filters/wake_notifier_filter.d.ts +48 -0
- package/dist/processors/filters/wake_notifier_filter.js +49 -0
- package/dist/processors/frame_processor.d.ts +360 -0
- package/dist/processors/frame_processor.js +932 -0
- package/dist/processors/frameworks/index.d.ts +2 -0
- package/dist/processors/frameworks/index.js +8 -0
- package/dist/processors/frameworks/rtvi/frames.d.ts +241 -0
- package/dist/processors/frameworks/rtvi/frames.js +259 -0
- package/dist/processors/frameworks/rtvi/index.d.ts +6 -0
- package/dist/processors/frameworks/rtvi/index.js +11 -0
- package/dist/processors/frameworks/rtvi/models.d.ts +1741 -0
- package/dist/processors/frameworks/rtvi/models.js +1807 -0
- package/dist/processors/frameworks/rtvi/observer.d.ts +340 -0
- package/dist/processors/frameworks/rtvi/observer.js +1113 -0
- package/dist/processors/frameworks/rtvi/processor.d.ts +182 -0
- package/dist/processors/frameworks/rtvi/processor.js +676 -0
- package/dist/processors/idle_frame_processor.d.ts +44 -0
- package/dist/processors/idle_frame_processor.js +86 -0
- package/dist/processors/index.d.ts +2 -0
- package/dist/processors/index.js +8 -0
- package/dist/processors/logger.d.ts +42 -0
- package/dist/processors/logger.js +87 -0
- package/dist/processors/metrics/frame_processor_metrics.d.ts +172 -0
- package/dist/processors/metrics/frame_processor_metrics.js +372 -0
- package/dist/processors/metrics/index.d.ts +2 -0
- package/dist/processors/metrics/index.js +8 -0
- package/dist/processors/producer_processor.d.ts +68 -0
- package/dist/processors/producer_processor.js +78 -0
- package/dist/processors/text_transformer.d.ts +30 -0
- package/dist/processors/text_transformer.js +48 -0
- package/dist/registry/index.d.ts +3 -0
- package/dist/registry/index.js +8 -0
- package/dist/registry/registry.d.ts +69 -0
- package/dist/registry/registry.js +149 -0
- package/dist/registry/types.d.ts +47 -0
- package/dist/registry/types.js +39 -0
- package/dist/serializers/base_serializer.d.ts +94 -0
- package/dist/serializers/base_serializer.js +82 -0
- package/dist/serializers/index.d.ts +2 -0
- package/dist/serializers/index.js +8 -0
- package/dist/services/ai_service.d.ts +145 -0
- package/dist/services/ai_service.js +262 -0
- package/dist/services/anthropic/index.d.ts +2 -0
- package/dist/services/anthropic/index.js +7 -0
- package/dist/services/assemblyai/index.d.ts +2 -0
- package/dist/services/assemblyai/index.js +7 -0
- package/dist/services/asyncai/index.d.ts +2 -0
- package/dist/services/asyncai/index.js +7 -0
- package/dist/services/aws/index.d.ts +2 -0
- package/dist/services/aws/index.js +7 -0
- package/dist/services/azure/index.d.ts +2 -0
- package/dist/services/azure/index.js +7 -0
- package/dist/services/baseten/index.d.ts +2 -0
- package/dist/services/baseten/index.js +7 -0
- package/dist/services/bland/index.d.ts +2 -0
- package/dist/services/bland/index.js +7 -0
- package/dist/services/camb/index.d.ts +2 -0
- package/dist/services/camb/index.js +7 -0
- package/dist/services/cartesia/index.d.ts +2 -0
- package/dist/services/cartesia/index.js +7 -0
- package/dist/services/cerebras/index.d.ts +2 -0
- package/dist/services/cerebras/index.js +7 -0
- package/dist/services/crusoe/index.d.ts +2 -0
- package/dist/services/crusoe/index.js +7 -0
- package/dist/services/deepgram/index.d.ts +2 -0
- package/dist/services/deepgram/index.js +7 -0
- package/dist/services/deepseek/index.d.ts +2 -0
- package/dist/services/deepseek/index.js +7 -0
- package/dist/services/elevenlabs/index.d.ts +2 -0
- package/dist/services/elevenlabs/index.js +7 -0
- package/dist/services/fal/index.d.ts +2 -0
- package/dist/services/fal/index.js +7 -0
- package/dist/services/fireworks/index.d.ts +2 -0
- package/dist/services/fireworks/index.js +7 -0
- package/dist/services/fish/index.d.ts +2 -0
- package/dist/services/fish/index.js +7 -0
- package/dist/services/funasr/index.d.ts +3 -0
- package/dist/services/funasr/index.js +8 -0
- package/dist/services/gladia/index.d.ts +2 -0
- package/dist/services/gladia/index.js +7 -0
- package/dist/services/google/index.d.ts +2 -0
- package/dist/services/google/index.js +7 -0
- package/dist/services/gradium/index.d.ts +2 -0
- package/dist/services/gradium/index.js +7 -0
- package/dist/services/grok/index.d.ts +2 -0
- package/dist/services/grok/index.js +7 -0
- package/dist/services/groq/index.d.ts +2 -0
- package/dist/services/groq/index.js +7 -0
- package/dist/services/heygen/index.d.ts +2 -0
- package/dist/services/heygen/index.js +7 -0
- package/dist/services/hume/index.d.ts +2 -0
- package/dist/services/hume/index.js +7 -0
- package/dist/services/image_service.d.ts +52 -0
- package/dist/services/image_service.js +60 -0
- package/dist/services/inception/index.d.ts +2 -0
- package/dist/services/inception/index.js +7 -0
- package/dist/services/index.d.ts +2 -0
- package/dist/services/index.js +8 -0
- package/dist/services/inworld/index.d.ts +2 -0
- package/dist/services/inworld/index.js +7 -0
- package/dist/services/keenable/index.d.ts +2 -0
- package/dist/services/keenable/index.js +7 -0
- package/dist/services/kokoro/index.d.ts +2 -0
- package/dist/services/kokoro/index.js +7 -0
- package/dist/services/llm_service.d.ts +1106 -0
- package/dist/services/llm_service.js +2231 -0
- package/dist/services/mem0/index.d.ts +2 -0
- package/dist/services/mem0/index.js +7 -0
- package/dist/services/meta/index.d.ts +2 -0
- package/dist/services/meta/index.js +7 -0
- package/dist/services/minimax/index.d.ts +2 -0
- package/dist/services/minimax/index.js +7 -0
- package/dist/services/mistral/index.d.ts +2 -0
- package/dist/services/mistral/index.js +7 -0
- package/dist/services/moondream/index.d.ts +2 -0
- package/dist/services/moondream/index.js +7 -0
- package/dist/services/moonshine/index.d.ts +2 -0
- package/dist/services/moonshine/index.js +7 -0
- package/dist/services/nebius/index.d.ts +2 -0
- package/dist/services/nebius/index.js +7 -0
- package/dist/services/neuphonic/index.d.ts +2 -0
- package/dist/services/neuphonic/index.js +7 -0
- package/dist/services/novita/index.d.ts +2 -0
- package/dist/services/novita/index.js +7 -0
- package/dist/services/nvidia/index.d.ts +2 -0
- package/dist/services/nvidia/index.js +11 -0
- package/dist/services/ollama/index.d.ts +2 -0
- package/dist/services/ollama/index.js +7 -0
- package/dist/services/openai/index.d.ts +2 -0
- package/dist/services/openai/index.js +7 -0
- package/dist/services/openclaw/index.d.ts +14 -0
- package/dist/services/openclaw/index.js +19 -0
- package/dist/services/openrouter/index.d.ts +2 -0
- package/dist/services/openrouter/index.js +7 -0
- package/dist/services/perplexity/index.d.ts +2 -0
- package/dist/services/perplexity/index.js +7 -0
- package/dist/services/piper/index.d.ts +2 -0
- package/dist/services/piper/index.js +7 -0
- package/dist/services/pocket_tts/index.d.ts +2 -0
- package/dist/services/pocket_tts/index.js +7 -0
- package/dist/services/qwen/index.d.ts +2 -0
- package/dist/services/qwen/index.js +7 -0
- package/dist/services/resembleai/index.d.ts +2 -0
- package/dist/services/resembleai/index.js +7 -0
- package/dist/services/rime/index.d.ts +2 -0
- package/dist/services/rime/index.js +7 -0
- package/dist/services/sambanova/index.d.ts +2 -0
- package/dist/services/sambanova/index.js +7 -0
- package/dist/services/sarvam/index.d.ts +2 -0
- package/dist/services/sarvam/index.js +7 -0
- package/dist/services/settings.d.ts +356 -0
- package/dist/services/settings.js +437 -0
- package/dist/services/simli/index.d.ts +2 -0
- package/dist/services/simli/index.js +7 -0
- package/dist/services/smallest/index.d.ts +2 -0
- package/dist/services/smallest/index.js +7 -0
- package/dist/services/soniox/index.d.ts +2 -0
- package/dist/services/soniox/index.js +7 -0
- package/dist/services/speechify/index.d.ts +2 -0
- package/dist/services/speechify/index.js +7 -0
- package/dist/services/speechmatics/index.d.ts +2 -0
- package/dist/services/speechmatics/index.js +7 -0
- package/dist/services/stt_latency.d.ts +62 -0
- package/dist/services/stt_latency.js +71 -0
- package/dist/services/stt_service.d.ts +601 -0
- package/dist/services/stt_service.js +1044 -0
- package/dist/services/tavus/index.d.ts +2 -0
- package/dist/services/tavus/index.js +7 -0
- package/dist/services/together/index.d.ts +2 -0
- package/dist/services/together/index.js +7 -0
- package/dist/services/tts_service.d.ts +909 -0
- package/dist/services/tts_service.js +2018 -0
- package/dist/services/ultravox/index.d.ts +2 -0
- package/dist/services/ultravox/index.js +9 -0
- package/dist/services/vision_service.d.ts +56 -0
- package/dist/services/vision_service.js +62 -0
- package/dist/services/websocket_service.d.ts +213 -0
- package/dist/services/websocket_service.js +654 -0
- package/dist/services/whisper/index.d.ts +2 -0
- package/dist/services/whisper/index.js +9 -0
- package/dist/services/xai/index.d.ts +2 -0
- package/dist/services/xai/index.js +9 -0
- package/dist/services/xtts/index.d.ts +2 -0
- package/dist/services/xtts/index.js +9 -0
- package/dist/tests/index.d.ts +2 -0
- package/dist/tests/index.js +8 -0
- package/dist/tests/utils.d.ts +123 -0
- package/dist/tests/utils.js +212 -0
- package/dist/transcriptions/index.d.ts +2 -0
- package/dist/transcriptions/index.js +8 -0
- package/dist/transcriptions/language.d.ts +376 -0
- package/dist/transcriptions/language.js +543 -0
- package/dist/transports/base_input.d.ts +149 -0
- package/dist/transports/base_input.js +338 -0
- package/dist/transports/base_output.d.ts +432 -0
- package/dist/transports/base_output.js +1231 -0
- package/dist/transports/base_transport.d.ts +223 -0
- package/dist/transports/base_transport.js +167 -0
- package/dist/transports/daily/index.d.ts +2 -0
- package/dist/transports/daily/index.js +9 -0
- package/dist/transports/heygen/index.d.ts +2 -0
- package/dist/transports/heygen/index.js +9 -0
- package/dist/transports/index.d.ts +2 -0
- package/dist/transports/index.js +8 -0
- package/dist/transports/lemonslice/index.d.ts +2 -0
- package/dist/transports/lemonslice/index.js +9 -0
- package/dist/transports/livekit/index.d.ts +2 -0
- package/dist/transports/livekit/index.js +9 -0
- package/dist/transports/local/index.d.ts +2 -0
- package/dist/transports/local/index.js +9 -0
- package/dist/transports/moq/index.d.ts +2 -0
- package/dist/transports/moq/index.js +9 -0
- package/dist/transports/smallwebrtc/index.d.ts +2 -0
- package/dist/transports/smallwebrtc/index.js +9 -0
- package/dist/transports/tavus/index.d.ts +2 -0
- package/dist/transports/tavus/index.js +9 -0
- package/dist/transports/video_in_sampler.d.ts +157 -0
- package/dist/transports/video_in_sampler.js +258 -0
- package/dist/transports/vonage/index.d.ts +2 -0
- package/dist/transports/vonage/index.js +9 -0
- package/dist/transports/websocket/index.d.ts +2 -0
- package/dist/transports/websocket/index.js +9 -0
- package/dist/transports/whatsapp/index.d.ts +2 -0
- package/dist/transports/whatsapp/index.js +9 -0
- package/dist/turns/eager_end_of_turn_mixin.d.ts +70 -0
- package/dist/turns/eager_end_of_turn_mixin.js +122 -0
- package/dist/turns/empty_user_turn.d.ts +55 -0
- package/dist/turns/empty_user_turn.js +53 -0
- package/dist/turns/index.d.ts +2 -0
- package/dist/turns/index.js +8 -0
- package/dist/turns/speculation_gate.d.ts +169 -0
- package/dist/turns/speculation_gate.js +294 -0
- package/dist/turns/types.d.ts +33 -0
- package/dist/turns/types.js +35 -0
- package/dist/turns/user_idle_controller.d.ts +94 -0
- package/dist/turns/user_idle_controller.js +187 -0
- package/dist/turns/user_mute/always_user_mute_strategy.d.ts +15 -0
- package/dist/turns/user_mute/always_user_mute_strategy.js +29 -0
- package/dist/turns/user_mute/base_user_mute_strategy.d.ts +39 -0
- package/dist/turns/user_mute/base_user_mute_strategy.js +45 -0
- package/dist/turns/user_mute/first_speech_user_mute_strategy.d.ts +28 -0
- package/dist/turns/user_mute/first_speech_user_mute_strategy.js +52 -0
- package/dist/turns/user_mute/function_call_user_mute_strategy.d.ts +25 -0
- package/dist/turns/user_mute/function_call_user_mute_strategy.js +50 -0
- package/dist/turns/user_mute/index.d.ts +6 -0
- package/dist/turns/user_mute/index.js +11 -0
- package/dist/turns/user_mute/mute_until_first_bot_complete_user_mute_strategy.d.ts +43 -0
- package/dist/turns/user_mute/mute_until_first_bot_complete_user_mute_strategy.js +76 -0
- package/dist/turns/user_start/base_user_turn_start_strategy.d.ts +135 -0
- package/dist/turns/user_start/base_user_turn_start_strategy.js +134 -0
- package/dist/turns/user_start/external_user_turn_start_strategy.d.ts +45 -0
- package/dist/turns/user_start/external_user_turn_start_strategy.js +64 -0
- package/dist/turns/user_start/index.d.ts +8 -0
- package/dist/turns/user_start/index.js +16 -0
- package/dist/turns/user_start/krisp_viva_ip_user_turn_start_strategy.d.ts +133 -0
- package/dist/turns/user_start/krisp_viva_ip_user_turn_start_strategy.js +243 -0
- package/dist/turns/user_start/min_words_user_turn_start_strategy.d.ts +60 -0
- package/dist/turns/user_start/min_words_user_turn_start_strategy.js +102 -0
- package/dist/turns/user_start/transcription_user_turn_start_strategy.d.ts +29 -0
- package/dist/turns/user_start/transcription_user_turn_start_strategy.js +43 -0
- package/dist/turns/user_start/vad_user_turn_start_strategy.d.ts +20 -0
- package/dist/turns/user_start/vad_user_turn_start_strategy.js +31 -0
- package/dist/turns/user_start/wake_phrase_user_turn_start_strategy.d.ts +141 -0
- package/dist/turns/user_start/wake_phrase_user_turn_start_strategy.js +238 -0
- package/dist/turns/user_stop/base_user_turn_stop_strategy.d.ts +174 -0
- package/dist/turns/user_stop/base_user_turn_stop_strategy.js +170 -0
- package/dist/turns/user_stop/deferred_user_turn_stop_strategy.d.ts +81 -0
- package/dist/turns/user_stop/deferred_user_turn_stop_strategy.js +107 -0
- package/dist/turns/user_stop/eager_match_policy.d.ts +36 -0
- package/dist/turns/user_stop/eager_match_policy.js +50 -0
- package/dist/turns/user_stop/eager_user_turn_stop_strategy.d.ts +84 -0
- package/dist/turns/user_stop/eager_user_turn_stop_strategy.js +154 -0
- package/dist/turns/user_stop/external_user_turn_completion_stop_strategy.d.ts +45 -0
- package/dist/turns/user_stop/external_user_turn_completion_stop_strategy.js +55 -0
- package/dist/turns/user_stop/external_user_turn_stop_strategy.d.ts +115 -0
- package/dist/turns/user_stop/external_user_turn_stop_strategy.js +205 -0
- package/dist/turns/user_stop/index.d.ts +10 -0
- package/dist/turns/user_stop/index.js +15 -0
- package/dist/turns/user_stop/llm_turn_completion_user_turn_stop_strategy.d.ts +44 -0
- package/dist/turns/user_stop/llm_turn_completion_user_turn_stop_strategy.js +59 -0
- package/dist/turns/user_stop/speech_timeout_user_turn_stop_strategy.d.ts +140 -0
- package/dist/turns/user_stop/speech_timeout_user_turn_stop_strategy.js +303 -0
- package/dist/turns/user_stop/turn_analyzer_user_turn_stop_strategy.d.ts +135 -0
- package/dist/turns/user_stop/turn_analyzer_user_turn_stop_strategy.js +332 -0
- package/dist/turns/user_turn_completion_mixin.d.ts +147 -0
- package/dist/turns/user_turn_completion_mixin.js +591 -0
- package/dist/turns/user_turn_controller.d.ts +134 -0
- package/dist/turns/user_turn_controller.js +335 -0
- package/dist/turns/user_turn_processor.d.ts +87 -0
- package/dist/turns/user_turn_processor.js +180 -0
- package/dist/turns/user_turn_strategies.d.ts +187 -0
- package/dist/turns/user_turn_strategies.js +222 -0
- package/dist/utils/async_tool_cancellation.d.ts +35 -0
- package/dist/utils/async_tool_cancellation.js +78 -0
- package/dist/utils/asyncio/index.d.ts +2 -0
- package/dist/utils/asyncio/index.js +7 -0
- package/dist/utils/asyncio/task_manager.d.ts +49 -0
- package/dist/utils/asyncio/task_manager.js +105 -0
- package/dist/utils/base_object.d.ts +76 -0
- package/dist/utils/base_object.js +154 -0
- package/dist/utils/context/aggregated_frame_sequencer.d.ts +491 -0
- package/dist/utils/context/aggregated_frame_sequencer.js +803 -0
- package/dist/utils/context/audio_context_manager.d.ts +131 -0
- package/dist/utils/context/audio_context_manager.js +233 -0
- package/dist/utils/context/index.d.ts +2 -0
- package/dist/utils/context/index.js +7 -0
- package/dist/utils/context/llm_context_summarization.d.ts +258 -0
- package/dist/utils/context/llm_context_summarization.js +511 -0
- package/dist/utils/context/text_segment_map.d.ts +611 -0
- package/dist/utils/context/text_segment_map.js +1132 -0
- package/dist/utils/context/word_completion_tracker.d.ts +273 -0
- package/dist/utils/context/word_completion_tracker.js +441 -0
- package/dist/utils/context/word_timestamp_tracker.d.ts +72 -0
- package/dist/utils/context/word_timestamp_tracker.js +98 -0
- package/dist/utils/deprecation.d.ts +28 -0
- package/dist/utils/deprecation.js +75 -0
- package/dist/utils/env.d.ts +38 -0
- package/dist/utils/env.js +62 -0
- package/dist/utils/errors.d.ts +75 -0
- package/dist/utils/errors.js +202 -0
- package/dist/utils/file_resolver.d.ts +126 -0
- package/dist/utils/file_resolver.js +314 -0
- package/dist/utils/file_storage.d.ts +115 -0
- package/dist/utils/file_storage.js +178 -0
- package/dist/utils/frame_queue.d.ts +36 -0
- package/dist/utils/frame_queue.js +66 -0
- package/dist/utils/http.d.ts +39 -0
- package/dist/utils/http.js +53 -0
- package/dist/utils/image.d.ts +14 -0
- package/dist/utils/image.js +63 -0
- package/dist/utils/index.d.ts +2 -0
- package/dist/utils/index.js +8 -0
- package/dist/utils/jpeg_encoder.d.ts +14 -0
- package/dist/utils/jpeg_encoder.js +315 -0
- package/dist/utils/log_config.d.ts +34 -0
- package/dist/utils/log_config.js +76 -0
- package/dist/utils/logger.d.ts +40 -0
- package/dist/utils/logger.js +91 -0
- package/dist/utils/mixin.d.ts +14 -0
- package/dist/utils/mixin.js +63 -0
- package/dist/utils/network.d.ts +73 -0
- package/dist/utils/network.js +90 -0
- package/dist/utils/security/allowed_origins.d.ts +16 -0
- package/dist/utils/security/allowed_origins.js +34 -0
- package/dist/utils/security/index.d.ts +2 -0
- package/dist/utils/security/index.js +7 -0
- package/dist/utils/security/redaction.d.ts +27 -0
- package/dist/utils/security/redaction.js +123 -0
- package/dist/utils/security/ssrf.d.ts +79 -0
- package/dist/utils/security/ssrf.js +327 -0
- package/dist/utils/shared.d.ts +35 -0
- package/dist/utils/shared.js +136 -0
- package/dist/utils/startup.d.ts +17 -0
- package/dist/utils/startup.js +130 -0
- package/dist/utils/string.d.ts +116 -0
- package/dist/utils/string.js +460 -0
- package/dist/utils/sync/base_notifier.d.ts +25 -0
- package/dist/utils/sync/base_notifier.js +16 -0
- package/dist/utils/sync/event_notifier.d.ts +38 -0
- package/dist/utils/sync/event_notifier.js +51 -0
- package/dist/utils/sync/index.d.ts +2 -0
- package/dist/utils/sync/index.js +7 -0
- package/dist/utils/text/alnum_utils.d.ts +109 -0
- package/dist/utils/text/alnum_utils.js +262 -0
- package/dist/utils/text/base_text_aggregator.d.ts +146 -0
- package/dist/utils/text/base_text_aggregator.js +102 -0
- package/dist/utils/text/base_text_filter.d.ts +54 -0
- package/dist/utils/text/base_text_filter.js +50 -0
- package/dist/utils/text/index.d.ts +2 -0
- package/dist/utils/text/index.js +7 -0
- package/dist/utils/text/markdown_text_filter.d.ts +147 -0
- package/dist/utils/text/markdown_text_filter.js +522 -0
- package/dist/utils/text/markup_utils.d.ts +54 -0
- package/dist/utils/text/markup_utils.js +156 -0
- package/dist/utils/text/pattern_pair_aggregator.d.ts +251 -0
- package/dist/utils/text/pattern_pair_aggregator.js +478 -0
- package/dist/utils/text/phonemes.d.ts +60 -0
- package/dist/utils/text/phonemes.js +172 -0
- package/dist/utils/text/simple_text_aggregator.d.ts +135 -0
- package/dist/utils/text/simple_text_aggregator.js +254 -0
- package/dist/utils/text/skip_tags_aggregator.d.ts +85 -0
- package/dist/utils/text/skip_tags_aggregator.js +147 -0
- package/dist/utils/text/transforms/acronyms.d.ts +21 -0
- package/dist/utils/text/transforms/acronyms.js +31 -0
- package/dist/utils/text/transforms/currency.d.ts +16 -0
- package/dist/utils/text/transforms/currency.js +56 -0
- package/dist/utils/text/transforms/dates.d.ts +18 -0
- package/dist/utils/text/transforms/dates.js +102 -0
- package/dist/utils/text/transforms/email.d.ts +17 -0
- package/dist/utils/text/transforms/email.js +35 -0
- package/dist/utils/text/transforms/index.d.ts +37 -0
- package/dist/utils/text/transforms/index.js +42 -0
- package/dist/utils/text/transforms/numbers.d.ts +25 -0
- package/dist/utils/text/transforms/numbers.js +59 -0
- package/dist/utils/text/transforms/percentages.d.ts +16 -0
- package/dist/utils/text/transforms/percentages.js +32 -0
- package/dist/utils/text/transforms/phone.d.ts +21 -0
- package/dist/utils/text/transforms/phone.js +39 -0
- package/dist/utils/text/transforms/pronunciations.d.ts +65 -0
- package/dist/utils/text/transforms/pronunciations.js +112 -0
- package/dist/utils/text/transforms/replacements.d.ts +39 -0
- package/dist/utils/text/transforms/replacements.js +46 -0
- package/dist/utils/text/transforms/strip_markdown.d.ts +25 -0
- package/dist/utils/text/transforms/strip_markdown.js +60 -0
- package/dist/utils/text/transforms/units.d.ts +22 -0
- package/dist/utils/text/transforms/units.js +81 -0
- package/dist/utils/text/transforms/voice_formatter.d.ts +86 -0
- package/dist/utils/text/transforms/voice_formatter.js +98 -0
- package/dist/utils/text/unicode_digits.d.ts +3 -0
- package/dist/utils/text/unicode_digits.js +34 -0
- package/dist/utils/text/word_timestamp_utils.d.ts +26 -0
- package/dist/utils/text/word_timestamp_utils.js +52 -0
- package/dist/utils/time.d.ts +13 -0
- package/dist/utils/time.js +31 -0
- package/dist/utils/tracing/index.d.ts +3 -0
- package/dist/utils/tracing/index.js +8 -0
- package/dist/utils/tracing/service_attributes.d.ts +198 -0
- package/dist/utils/tracing/service_attributes.js +434 -0
- package/dist/utils/tracing/service_decorators.d.ts +214 -0
- package/dist/utils/tracing/service_decorators.js +1662 -0
- package/dist/utils/tracing/setup.d.ts +107 -0
- package/dist/utils/tracing/setup.js +160 -0
- package/dist/utils/tracing/tracing_context.d.ts +55 -0
- package/dist/utils/tracing/tracing_context.js +97 -0
- package/dist/utils/tracing/turn_trace_observer.d.ts +95 -0
- package/dist/utils/tracing/turn_trace_observer.js +233 -0
- package/dist/utils/types.d.ts +80 -0
- package/dist/utils/types.js +108 -0
- package/dist/utils/utils.d.ts +5 -0
- package/dist/utils/utils.js +22 -0
- package/dist/utils/yaml.d.ts +179 -0
- package/dist/utils/yaml.js +556 -0
- package/dist/version.d.ts +3 -0
- package/dist/version.js +7 -0
- package/dist/workers/base_worker.d.ts +772 -0
- package/dist/workers/base_worker.js +1298 -0
- package/dist/workers/index.d.ts +2 -0
- package/dist/workers/index.js +8 -0
- package/dist/workers/llm/backend_llm_worker.d.ts +552 -0
- package/dist/workers/llm/backend_llm_worker.js +1055 -0
- package/dist/workers/llm/index.d.ts +6 -0
- package/dist/workers/llm/index.js +11 -0
- package/dist/workers/llm/llm_context_worker.d.ts +66 -0
- package/dist/workers/llm/llm_context_worker.js +72 -0
- package/dist/workers/llm/llm_worker.d.ts +180 -0
- package/dist/workers/llm/llm_worker.js +323 -0
- package/dist/workers/llm/tool_decorator.d.ts +68 -0
- package/dist/workers/llm/tool_decorator.js +60 -0
- package/dist/workers/proxy/index.d.ts +3 -0
- package/dist/workers/proxy/index.js +9 -0
- package/dist/workers/proxy/websocket/client.d.ts +110 -0
- package/dist/workers/proxy/websocket/client.js +196 -0
- package/dist/workers/proxy/websocket/index.d.ts +3 -0
- package/dist/workers/proxy/websocket/index.js +10 -0
- package/dist/workers/runner.d.ts +268 -0
- package/dist/workers/runner.js +489 -0
- package/dist/workers/ui/index.d.ts +22 -0
- package/dist/workers/ui/index.js +27 -0
- package/dist/workers/ui/ui_event_decorator.d.ts +43 -0
- package/dist/workers/ui/ui_event_decorator.js +75 -0
- package/dist/workers/ui/ui_job_context.d.ts +12 -0
- package/dist/workers/ui/ui_job_context.js +17 -0
- package/dist/workers/ui/ui_prompts.d.ts +21 -0
- package/dist/workers/ui/ui_prompts.js +41 -0
- package/dist/workers/ui/ui_tools.d.ts +32 -0
- package/dist/workers/ui/ui_tools.js +112 -0
- package/dist/workers/ui/ui_worker.d.ts +571 -0
- package/dist/workers/ui/ui_worker.js +1156 -0
- package/package.json +61 -3
- package/README.md +0 -5
|
@@ -0,0 +1,591 @@
|
|
|
1
|
+
// Copyright (c) 2024-2026, Daily
|
|
2
|
+
// Copyright (c) 2026, pipedog contributors
|
|
3
|
+
// SPDX-License-Identifier: BSD-2-Clause
|
|
4
|
+
//
|
|
5
|
+
// Ported from pipecat: src/pipecat/turns/user_turn_completion_mixin.py
|
|
6
|
+
/**
|
|
7
|
+
* Mixin for adding turn completion detection to LLM services.
|
|
8
|
+
*
|
|
9
|
+
* This mixin enables LLM services to detect and process turn completion
|
|
10
|
+
* markers (COMPLETE/INCOMPLETE) in LLM responses, allowing for smarter
|
|
11
|
+
* conversation flow where the LLM can indicate whether the user's input was
|
|
12
|
+
* complete or if they were interrupted mid-thought.
|
|
13
|
+
*/
|
|
14
|
+
import { CancelledError, sleep } from "@pipedog/asyncio";
|
|
15
|
+
import { FunctionCallsStartedFrame, InterruptionFrame, LLMFullResponseEndFrame, LLMFullResponseStartFrame, LLMMarkerFrame, LLMMarkerResponseFrame, LLMMessagesAppendFrame, LLMRunFrame, LLMTextFrame, UserStartedSpeakingFrame, UserTurnInferenceCompletedFrame, VADUserStartedSpeakingFrame, VADUserStoppedSpeakingFrame, } from "../frames/frames.js";
|
|
16
|
+
import { FrameDirection } from "../processors/frame_processor.js";
|
|
17
|
+
import { logger } from "../utils/logger.js";
|
|
18
|
+
import { defineMixin } from "../utils/mixin.js";
|
|
19
|
+
// Turn completion markers. Fill level tracks how much of the user's turn has
|
|
20
|
+
// arrived: full is a finished turn, half is a turn cut off mid-thought, empty
|
|
21
|
+
// is a user who hasn't started answering. Each is a single token in every
|
|
22
|
+
// major tokenizer, which matters because the complete marker is generated
|
|
23
|
+
// before any speakable text.
|
|
24
|
+
export const USER_TURN_COMPLETE_MARKER = "●";
|
|
25
|
+
/** Short wait - user likely continues soon. */
|
|
26
|
+
export const USER_TURN_INCOMPLETE_SHORT_MARKER = "◐";
|
|
27
|
+
/** Long wait - user needs more time. */
|
|
28
|
+
export const USER_TURN_INCOMPLETE_LONG_MARKER = "○";
|
|
29
|
+
/**
|
|
30
|
+
* Completion verdict detected in the current LLM response.
|
|
31
|
+
*
|
|
32
|
+
* - `COMPLETE`: ● detected, response flows through as speech.
|
|
33
|
+
* - `INCOMPLETE`: ◐/○ detected, response suppressed, timeout armed.
|
|
34
|
+
*/
|
|
35
|
+
export const TurnMarker = {
|
|
36
|
+
COMPLETE: "complete",
|
|
37
|
+
INCOMPLETE: "incomplete",
|
|
38
|
+
};
|
|
39
|
+
/**
|
|
40
|
+
* How long to wait before re-prompting after an incomplete turn.
|
|
41
|
+
*
|
|
42
|
+
* - `SHORT`: ◐ detected, the user was cut off and likely continues soon.
|
|
43
|
+
* - `LONG`: ○ detected, the user needs more time to think.
|
|
44
|
+
*/
|
|
45
|
+
export const IncompleteType = {
|
|
46
|
+
SHORT: "short",
|
|
47
|
+
LONG: "long",
|
|
48
|
+
};
|
|
49
|
+
// Default prompts for incomplete timeouts
|
|
50
|
+
/**
|
|
51
|
+
* Build the re-prompt sent when a short incomplete-turn timeout expires.
|
|
52
|
+
*
|
|
53
|
+
* @param complete Marker the LLM emits when the user turn is complete.
|
|
54
|
+
* @param short Marker for a turn cut off mid-thought.
|
|
55
|
+
* @param long Marker for a user who needs more time.
|
|
56
|
+
* @returns The prompt text with the markers substituted in.
|
|
57
|
+
*/
|
|
58
|
+
function renderIncompleteShortPrompt(complete, short, long) {
|
|
59
|
+
return `The user paused briefly. Generate a brief, natural prompt to encourage them to continue.
|
|
60
|
+
|
|
61
|
+
IMPORTANT: You MUST respond with ${complete} followed by your message. Do NOT output ${short} or ${long} - the user has already been given time to continue.
|
|
62
|
+
|
|
63
|
+
Your response should:
|
|
64
|
+
- Be contextually relevant to what was just discussed
|
|
65
|
+
- Sound natural and conversational
|
|
66
|
+
- Be very concise (1 sentence max)
|
|
67
|
+
- Gently prompt them to continue
|
|
68
|
+
|
|
69
|
+
Example format: ${complete} Go ahead, I'm listening.
|
|
70
|
+
|
|
71
|
+
Generate your ${complete} response now.`;
|
|
72
|
+
}
|
|
73
|
+
export const DEFAULT_INCOMPLETE_SHORT_PROMPT = renderIncompleteShortPrompt(USER_TURN_COMPLETE_MARKER, USER_TURN_INCOMPLETE_SHORT_MARKER, USER_TURN_INCOMPLETE_LONG_MARKER);
|
|
74
|
+
/**
|
|
75
|
+
* Build the re-prompt sent when a long incomplete-turn timeout expires.
|
|
76
|
+
*
|
|
77
|
+
* @param complete Marker the LLM emits when the user turn is complete.
|
|
78
|
+
* @param short Marker for a turn cut off mid-thought.
|
|
79
|
+
* @param long Marker for a user who needs more time.
|
|
80
|
+
* @returns The prompt text with the markers substituted in.
|
|
81
|
+
*/
|
|
82
|
+
function renderIncompleteLongPrompt(complete, short, long) {
|
|
83
|
+
return `The user has been quiet for a while. Generate a friendly check-in message.
|
|
84
|
+
|
|
85
|
+
IMPORTANT: You MUST respond with ${complete} followed by your message. Do NOT output ${short} or ${long} - the user has already been given plenty of time.
|
|
86
|
+
|
|
87
|
+
Your response should:
|
|
88
|
+
- Acknowledge they might be thinking or busy
|
|
89
|
+
- Offer to help or continue when ready
|
|
90
|
+
- Be warm and understanding
|
|
91
|
+
- Be brief (1 sentence)
|
|
92
|
+
|
|
93
|
+
Example format: ${complete} No rush! Let me know when you're ready to continue.
|
|
94
|
+
|
|
95
|
+
Generate your ${complete} response now.`;
|
|
96
|
+
}
|
|
97
|
+
export const DEFAULT_INCOMPLETE_LONG_PROMPT = renderIncompleteLongPrompt(USER_TURN_COMPLETE_MARKER, USER_TURN_INCOMPLETE_SHORT_MARKER, USER_TURN_INCOMPLETE_LONG_MARKER);
|
|
98
|
+
// System prompt instructions for turn completion that can be appended to any
|
|
99
|
+
// base prompt.
|
|
100
|
+
/**
|
|
101
|
+
* Build the turn completion instructions appended to the system prompt.
|
|
102
|
+
*
|
|
103
|
+
* @param complete Marker the LLM emits when the user turn is complete.
|
|
104
|
+
* @param short Marker for a turn cut off mid-thought.
|
|
105
|
+
* @param long Marker for a user who needs more time.
|
|
106
|
+
* @returns The prompt text with the markers substituted in.
|
|
107
|
+
*/
|
|
108
|
+
function renderCompletionInstructions(complete, short, long) {
|
|
109
|
+
return `
|
|
110
|
+
TURN COMPLETION PROTOCOL (mandatory):
|
|
111
|
+
The user's words reach you from speech recognition, usually without punctuation, and sometimes before they have finished talking. Before you reply, decide whether their turn is complete, and start every response with exactly one of these markers as its very first character:
|
|
112
|
+
|
|
113
|
+
${complete} the user's turn is complete: answer them. Write ${complete}, a space, then your full reply. Never write ${complete} on its own.
|
|
114
|
+
${short} the user stopped mid-sentence and will continue in a few seconds. Write ${short} and nothing else.
|
|
115
|
+
${long} the user needs time to think or asked you to wait. Write ${long} and nothing else.
|
|
116
|
+
|
|
117
|
+
Deciding:
|
|
118
|
+
- Complete means conversationally complete, not long. One word can be a complete answer to your question: "yes", "no thanks", "Tuesday", "Japan", "four". A question to you is complete. A correction, or a request to repeat yourself, is complete.
|
|
119
|
+
- Grammatically complete is not the same as conversationally complete. If the user has only acknowledged your question or reacted to what you said, without answering it ("that's a good question", "oh wow, okay", "that's interesting", "hmm", "well"), they have not taken their turn yet: ${long}.
|
|
120
|
+
- Cut off (${short}): the last words leave a phrase open, in whatever language the user speaks: a sentence that ends on a conjunction, a preposition, an article, or the word for "because"; a list that is still going; a number that is only partly said. A fragment like this is not a request for help. Do not answer it, do not ask what they need, and do not guess the rest: wait with ${short}.
|
|
121
|
+
- Needs time (${long}): "hold on", "let me think", "give me a second", "one moment". Filler followed by a real answer, such as "hmm, I'd go to Japan for the food", is complete.
|
|
122
|
+
- When the user's latest words continue an earlier fragment (your previous response was ${short} or ${long}), judge the fragments together as one turn. If the combined turn still ends open, ${short}; if it now answers your question, ${complete}.
|
|
123
|
+
- If a tool call is the right response, make the tool call; the turn is complete.
|
|
124
|
+
|
|
125
|
+
Format rules:
|
|
126
|
+
- The marker is the first character. No text, quotes, backticks or explanation before it.
|
|
127
|
+
- After ${short} or ${long}, output nothing: no words, no explanation, no nudge. The system waits and prompts you again later.
|
|
128
|
+
- Exactly one marker per response.
|
|
129
|
+
|
|
130
|
+
Examples:
|
|
131
|
+
- You asked where they would go and the user says "i'd go to japan because i love". Respond with only ${short}.
|
|
132
|
+
- The user says "i need help with". Respond with only ${short}.
|
|
133
|
+
- You asked for their phone number and the user says "it's five five five". Respond with only ${short}.
|
|
134
|
+
- You asked what they want to order and the user says "a large pepperoni pizza a garden salad and". Respond with only ${short}.
|
|
135
|
+
- You asked where they would go and the user says "that's a good question let me think". Respond with only ${long}.
|
|
136
|
+
- You asked where they would go and the user says "that's interesting". Respond with only ${long}.
|
|
137
|
+
- The user says "hold on a second". Respond with only ${long}.
|
|
138
|
+
- You asked where they would go and the user says "japan". Respond with ${complete} followed by your reply, for example "${complete} Japan is a wonderful choice. What draws you there?"
|
|
139
|
+
- You asked whether to book it and the user says "yes". Respond with ${complete} followed by your reply, for example "${complete} Done, I'll book it now."
|
|
140
|
+
- The user says "can you help me book a flight to new york next week". Respond with ${complete} followed by your reply, for example "${complete} Of course. What day would you like to leave, and from which city?"
|
|
141
|
+
- Your previous response was ${short} after the user said "i'd go to", and now the user says "japan because". Together that is still open: respond with only ${short}.`;
|
|
142
|
+
}
|
|
143
|
+
export const USER_TURN_COMPLETION_INSTRUCTIONS = renderCompletionInstructions(USER_TURN_COMPLETE_MARKER, USER_TURN_INCOMPLETE_SHORT_MARKER, USER_TURN_INCOMPLETE_LONG_MARKER);
|
|
144
|
+
/** Configuration for turn completion behavior. See UserTurnCompletionConfigOptions. */
|
|
145
|
+
export class UserTurnCompletionConfig {
|
|
146
|
+
instructions;
|
|
147
|
+
completeMarker;
|
|
148
|
+
incompleteShortMarker;
|
|
149
|
+
incompleteLongMarker;
|
|
150
|
+
incompleteShortTimeout;
|
|
151
|
+
incompleteLongTimeout;
|
|
152
|
+
incompleteShortPrompt;
|
|
153
|
+
incompleteLongPrompt;
|
|
154
|
+
constructor(options = {}) {
|
|
155
|
+
this.instructions = options.instructions ?? null;
|
|
156
|
+
this.completeMarker = options.completeMarker ?? USER_TURN_COMPLETE_MARKER;
|
|
157
|
+
this.incompleteShortMarker = options.incompleteShortMarker ?? USER_TURN_INCOMPLETE_SHORT_MARKER;
|
|
158
|
+
this.incompleteLongMarker = options.incompleteLongMarker ?? USER_TURN_INCOMPLETE_LONG_MARKER;
|
|
159
|
+
this.incompleteShortTimeout = options.incompleteShortTimeout ?? 5.0;
|
|
160
|
+
this.incompleteLongTimeout = options.incompleteLongTimeout ?? 10.0;
|
|
161
|
+
this.incompleteShortPrompt = options.incompleteShortPrompt ?? null;
|
|
162
|
+
this.incompleteLongPrompt = options.incompleteLongPrompt ?? null;
|
|
163
|
+
}
|
|
164
|
+
/** The complete, short and long markers, in that order. */
|
|
165
|
+
get markers() {
|
|
166
|
+
return [this.completeMarker, this.incompleteShortMarker, this.incompleteLongMarker];
|
|
167
|
+
}
|
|
168
|
+
/** Turn completion instructions, using default if not set. */
|
|
169
|
+
get completionInstructions() {
|
|
170
|
+
return this.instructions || renderCompletionInstructions(...this.markers);
|
|
171
|
+
}
|
|
172
|
+
/** Short incomplete prompt, using default if not set. */
|
|
173
|
+
get shortPrompt() {
|
|
174
|
+
return this.incompleteShortPrompt || renderIncompleteShortPrompt(...this.markers);
|
|
175
|
+
}
|
|
176
|
+
/** Long incomplete prompt, using default if not set. */
|
|
177
|
+
get longPrompt() {
|
|
178
|
+
return this.incompleteLongPrompt || renderIncompleteLongPrompt(...this.markers);
|
|
179
|
+
}
|
|
180
|
+
}
|
|
181
|
+
/**
|
|
182
|
+
* Mixin that adds turn completion detection to LLM services.
|
|
183
|
+
*
|
|
184
|
+
* This mixin provides methods to push LLM text with turn completion
|
|
185
|
+
* detection. It processes turn completion markers to enable smarter
|
|
186
|
+
* conversation flow:
|
|
187
|
+
*
|
|
188
|
+
* - ● (COMPLETE): Push response normally
|
|
189
|
+
* - ◐ (INCOMPLETE SHORT): Suppress response, wait 5s, then prompt
|
|
190
|
+
* - ○ (INCOMPLETE LONG): Suppress response, wait 10s, then prompt
|
|
191
|
+
*
|
|
192
|
+
* A ● that arrives while VAD still hears the user is stale (the user resumed
|
|
193
|
+
* speaking after the inference was triggered) and is handled as ◐.
|
|
194
|
+
*
|
|
195
|
+
* When incomplete timeouts expire, the mixin automatically prompts the LLM
|
|
196
|
+
* with a contextual follow-up message to re-engage the user.
|
|
197
|
+
*
|
|
198
|
+
* The LLM service controls when to use turn completion by calling
|
|
199
|
+
* `_pushTurnText` instead of `pushFrame`.
|
|
200
|
+
*
|
|
201
|
+
* The mixin requires that the base class is a FrameProcessor (it relies on
|
|
202
|
+
* `pushFrame` with FrameProcessor's signature).
|
|
203
|
+
*
|
|
204
|
+
* Upstream: `class LLMService(UserTurnCompletionLLMServiceMixin, AIService)`
|
|
205
|
+
* → `class LLMService extends UserTurnCompletionLLMServiceMixin(AIService)`.
|
|
206
|
+
*
|
|
207
|
+
* @example
|
|
208
|
+
* // With turn completion:
|
|
209
|
+
* if (this._filterIncompleteUserTurns) {
|
|
210
|
+
* await this._pushTurnText(chunk.text);
|
|
211
|
+
* } else {
|
|
212
|
+
* await this.pushFrame(new LLMTextFrame({ text: chunk.text }));
|
|
213
|
+
* }
|
|
214
|
+
*/
|
|
215
|
+
export const UserTurnCompletionLLMServiceMixin = defineMixin("UserTurnCompletionLLMServiceMixin", (Base) => {
|
|
216
|
+
// The base must be a FrameProcessor (upstream's mixin subclasses it).
|
|
217
|
+
const FrameProcessorBase = Base;
|
|
218
|
+
class UserTurnCompletionLLMServiceMixinImpl extends FrameProcessorBase {
|
|
219
|
+
_turnTextBuffer = "";
|
|
220
|
+
// The current response as the LLM produced it, and the marker read from
|
|
221
|
+
// it, reported together when the response ends.
|
|
222
|
+
_responseRaw = "";
|
|
223
|
+
_responseMarker = null;
|
|
224
|
+
// Completion verdict for the current LLM response, set when a marker is
|
|
225
|
+
// detected in the text stream. `null` means no marker yet, so keep
|
|
226
|
+
// buffering text until one appears. `INCOMPLETE` also doubles as a safety
|
|
227
|
+
// latch: the prompt tells the LLM to emit only the marker, but if it
|
|
228
|
+
// disobeys and streams more text we keep suppressing it.
|
|
229
|
+
_turnMarker = null;
|
|
230
|
+
// True once `UserTurnInferenceCompletedFrame` has been broadcast for this
|
|
231
|
+
// turn. Prevents double-broadcast when ● and a tool call both occur in the
|
|
232
|
+
// same turn.
|
|
233
|
+
_turnCompletionBroadcasted = false;
|
|
234
|
+
// True once a ● has been voiced since the user last started speaking.
|
|
235
|
+
// Reset on UserStartedSpeakingFrame / InterruptionFrame (new turn) and on
|
|
236
|
+
// VADUserStartedSpeakingFrame (resumed mid-turn), unlike the per-response
|
|
237
|
+
// flags above (reset every LLMFullResponseEnd). The acoustic detector can
|
|
238
|
+
// trigger several inferences within one user turn, each independently
|
|
239
|
+
// producing a ●; this latch voices at most one per speaking segment, so
|
|
240
|
+
// the bot does not immediately repeat itself. It is not a per-turn
|
|
241
|
+
// guarantee: resetting on a mid-turn resume is required so a completion
|
|
242
|
+
// stale-dropped by the controller (see
|
|
243
|
+
// UserTurnController._triggerUserTurnStop) doesn't permanently silence the
|
|
244
|
+
// turn.
|
|
245
|
+
_userTurnCompletionVoiced = false;
|
|
246
|
+
_userSpeaking = false;
|
|
247
|
+
// Timeout handling
|
|
248
|
+
_userTurnCompletionConfig = new UserTurnCompletionConfig();
|
|
249
|
+
_incompleteTimeoutTask = null;
|
|
250
|
+
/**
|
|
251
|
+
* Set the turn completion configuration.
|
|
252
|
+
*
|
|
253
|
+
* @param config The turn completion configuration, or null to restore the
|
|
254
|
+
* default configuration.
|
|
255
|
+
*/
|
|
256
|
+
setUserTurnCompletionConfig(config) {
|
|
257
|
+
this._userTurnCompletionConfig = config ?? new UserTurnCompletionConfig();
|
|
258
|
+
}
|
|
259
|
+
/**
|
|
260
|
+
* Broadcast `UserTurnInferenceCompletedFrame` at most once per turn.
|
|
261
|
+
*
|
|
262
|
+
* Called from the two places we know the LLM has committed to a response
|
|
263
|
+
* for the current user turn:
|
|
264
|
+
*
|
|
265
|
+
* - the `●` marker is detected in the text stream
|
|
266
|
+
* - a `FunctionCallsStartedFrame` is emitted — the LLM committed to a tool
|
|
267
|
+
* call before producing (or instead of) a marker.
|
|
268
|
+
*
|
|
269
|
+
* Broadcasting on the tool-call path matters for races: the downstream
|
|
270
|
+
* `UserStoppedSpeakingFrame` needs to propagate before the function
|
|
271
|
+
* actually executes and a `FunctionCallResultFrame` flows back to the
|
|
272
|
+
* assistant aggregator.
|
|
273
|
+
*/
|
|
274
|
+
async _broadcastTurnCompletion() {
|
|
275
|
+
if (this._turnCompletionBroadcasted)
|
|
276
|
+
return;
|
|
277
|
+
this._turnCompletionBroadcasted = true;
|
|
278
|
+
await this.broadcastFrame(UserTurnInferenceCompletedFrame);
|
|
279
|
+
}
|
|
280
|
+
/**
|
|
281
|
+
* Start a timeout task for incomplete turn handling.
|
|
282
|
+
*
|
|
283
|
+
* @param incompleteType Whether to use the short or long timeout duration.
|
|
284
|
+
*/
|
|
285
|
+
async _startIncompleteTimeout(incompleteType) {
|
|
286
|
+
// Cancel any existing timeout
|
|
287
|
+
await this._cancelIncompleteTimeout();
|
|
288
|
+
const timeout = incompleteType === IncompleteType.SHORT
|
|
289
|
+
? this._userTurnCompletionConfig.incompleteShortTimeout
|
|
290
|
+
: this._userTurnCompletionConfig.incompleteLongTimeout;
|
|
291
|
+
logger.debug(`Starting ${incompleteType} incomplete timeout (${timeout}s)`);
|
|
292
|
+
this._incompleteTimeoutTask = this.createTask(() => this._incompleteTimeoutHandler(incompleteType, timeout), `_incomplete_timeout_${incompleteType}`);
|
|
293
|
+
}
|
|
294
|
+
/** Cancel any pending incomplete timeout task. */
|
|
295
|
+
async _cancelIncompleteTimeout() {
|
|
296
|
+
if (this._incompleteTimeoutTask && !this._incompleteTimeoutTask.done()) {
|
|
297
|
+
logger.debug("Cancelling incomplete timeout");
|
|
298
|
+
await this.cancelTask(this._incompleteTimeoutTask);
|
|
299
|
+
}
|
|
300
|
+
this._incompleteTimeoutTask = null;
|
|
301
|
+
}
|
|
302
|
+
/**
|
|
303
|
+
* Handle incomplete timeout expiration.
|
|
304
|
+
*
|
|
305
|
+
* @param incompleteType Whether this is the short or long timeout.
|
|
306
|
+
* @param timeout The timeout duration in seconds.
|
|
307
|
+
*/
|
|
308
|
+
async _incompleteTimeoutHandler(incompleteType, timeout) {
|
|
309
|
+
try {
|
|
310
|
+
await sleep(timeout);
|
|
311
|
+
// Timeout expired - reset state before prompting LLM
|
|
312
|
+
logger.debug(`Incomplete ${incompleteType} timeout expired, prompting LLM`);
|
|
313
|
+
await this._turnReset();
|
|
314
|
+
this._incompleteTimeoutTask = null;
|
|
315
|
+
// Get the appropriate prompt
|
|
316
|
+
const prompt = incompleteType === IncompleteType.SHORT
|
|
317
|
+
? this._userTurnCompletionConfig.shortPrompt
|
|
318
|
+
: this._userTurnCompletionConfig.longPrompt;
|
|
319
|
+
// Push through pipeline to trigger LLM response
|
|
320
|
+
await this.pushFrame(new LLMMessagesAppendFrame({ messages: [{ role: "developer", content: prompt }] }));
|
|
321
|
+
await this.pushFrame(new LLMRunFrame());
|
|
322
|
+
}
|
|
323
|
+
catch (e) {
|
|
324
|
+
// Timeout was cancelled (user spoke or interruption). Upstream
|
|
325
|
+
// swallows the CancelledError here; the task still ends as done.
|
|
326
|
+
if (!(e instanceof CancelledError))
|
|
327
|
+
throw e;
|
|
328
|
+
}
|
|
329
|
+
}
|
|
330
|
+
/**
|
|
331
|
+
* Reset turn completion state between responses.
|
|
332
|
+
*
|
|
333
|
+
* Call this at the end of each LLM response to clear buffered text and
|
|
334
|
+
* reset state. If no marker was found, pushes the buffered text to avoid
|
|
335
|
+
* losing content.
|
|
336
|
+
*
|
|
337
|
+
* Note: This does NOT cancel pending incomplete timeouts. Timeouts are
|
|
338
|
+
* cancelled on InterruptionFrame (when the user speaks) and on
|
|
339
|
+
* LLMFullResponseStartFrame (when a new inference begins).
|
|
340
|
+
*/
|
|
341
|
+
async _turnReset() {
|
|
342
|
+
// If no marker was found in this response, push the buffered text so
|
|
343
|
+
// it's not lost.
|
|
344
|
+
if (this._turnMarker === null && this._turnTextBuffer) {
|
|
345
|
+
// Graceful degradation: push the buffered text so it's not lost
|
|
346
|
+
logger.warning(`${this}: filter_incomplete_user_turns is enabled but LLM response did not ` +
|
|
347
|
+
"contain turn completion markers (●/◐/○). Pushing text anyway. " +
|
|
348
|
+
"The system prompt may be missing turn completion instructions.");
|
|
349
|
+
await this.pushFrame(new LLMTextFrame({ text: this._turnTextBuffer }));
|
|
350
|
+
}
|
|
351
|
+
this._turnTextBuffer = "";
|
|
352
|
+
this._turnMarker = null;
|
|
353
|
+
this._turnCompletionBroadcasted = false;
|
|
354
|
+
}
|
|
355
|
+
/**
|
|
356
|
+
* Process frames, handling turn completion state resets.
|
|
357
|
+
*
|
|
358
|
+
* @param frame The frame to process.
|
|
359
|
+
* @param direction The direction of frame processing.
|
|
360
|
+
*/
|
|
361
|
+
async processFrame(frame, direction) {
|
|
362
|
+
// Handle interruptions by cancelling timeout and resetting state
|
|
363
|
+
if (frame instanceof InterruptionFrame) {
|
|
364
|
+
await this._cancelIncompleteTimeout();
|
|
365
|
+
await this._turnReset();
|
|
366
|
+
this._userTurnCompletionVoiced = false;
|
|
367
|
+
}
|
|
368
|
+
else if (frame instanceof UserStartedSpeakingFrame) {
|
|
369
|
+
// A new user turn begins, so allow one fresh spoken completion.
|
|
370
|
+
this._userTurnCompletionVoiced = false;
|
|
371
|
+
}
|
|
372
|
+
else if (frame instanceof LLMMessagesAppendFrame && frame.runLlm) {
|
|
373
|
+
// An externally appended message that asks for a run (e.g. a
|
|
374
|
+
// user-idle check-in) is an explicit request for fresh speech, and it
|
|
375
|
+
// arrives precisely while the user is silent. Clear the voiced latch
|
|
376
|
+
// so the ● guard in `_pushTurnText` does not drop its text.
|
|
377
|
+
this._userTurnCompletionVoiced = false;
|
|
378
|
+
}
|
|
379
|
+
else if (frame instanceof VADUserStartedSpeakingFrame) {
|
|
380
|
+
this._userSpeaking = true;
|
|
381
|
+
// The user resumed speaking within the same open turn. A new turn's
|
|
382
|
+
// InterruptionFrame does not fire for a resume inside an already-open
|
|
383
|
+
// turn, so two things that normally reset on a fresh turn need
|
|
384
|
+
// handling here instead:
|
|
385
|
+
//
|
|
386
|
+
// 1. An armed ◐/○ re-prompt timeout would otherwise expire and nudge
|
|
387
|
+
// (talking over) a user who is speaking again. Cancel it. The ◐/○
|
|
388
|
+
// response already ended and reset the per-response state; the
|
|
389
|
+
// next inference re-arms a fresh timer if needed.
|
|
390
|
+
await this._cancelIncompleteTimeout();
|
|
391
|
+
// 2. Allow one fresh spoken completion: resetting on a mid-turn
|
|
392
|
+
// resume is required so a completion stale-dropped by the
|
|
393
|
+
// controller (see UserTurnController._triggerUserTurnStop) doesn't
|
|
394
|
+
// permanently silence the turn.
|
|
395
|
+
this._userTurnCompletionVoiced = false;
|
|
396
|
+
}
|
|
397
|
+
else if (frame instanceof VADUserStoppedSpeakingFrame) {
|
|
398
|
+
this._userSpeaking = false;
|
|
399
|
+
}
|
|
400
|
+
// Pass frame to parent
|
|
401
|
+
await super.processFrame(frame, direction);
|
|
402
|
+
}
|
|
403
|
+
/**
|
|
404
|
+
* Push a frame downstream, resetting turn state at end of each LLM
|
|
405
|
+
* response.
|
|
406
|
+
*
|
|
407
|
+
* `LLMFullResponseEndFrame` is generated by the LLM service itself
|
|
408
|
+
* (pushed, not received), so it must be handled here rather than in
|
|
409
|
+
* `processFrame`.
|
|
410
|
+
*
|
|
411
|
+
* @param frame The frame to push downstream.
|
|
412
|
+
* @param direction The direction of frame flow. Defaults to downstream.
|
|
413
|
+
*/
|
|
414
|
+
async pushFrame(frame, direction = FrameDirection.DOWNSTREAM) {
|
|
415
|
+
if (frame instanceof FunctionCallsStartedFrame) {
|
|
416
|
+
// Broadcast turn completion now, before the function dispatches —
|
|
417
|
+
// gives `UserStoppedSpeakingFrame` maximum time to propagate so the
|
|
418
|
+
// assistant aggregator's `_userSpeaking` is false by the time a
|
|
419
|
+
// `FunctionCallResultFrame` arrives.
|
|
420
|
+
await this._broadcastTurnCompletion();
|
|
421
|
+
// A function call means a fresh post-tool inference is coming, and
|
|
422
|
+
// that response is expected to speak. Clear the voiced latch so the ●
|
|
423
|
+
// guard in `_pushTurnText` does not drop its text. The response that
|
|
424
|
+
// voiced the ● keeps streaming: its `_turnMarker` is COMPLETE, so its
|
|
425
|
+
// text takes the COMPLETE branch, not the latch guard.
|
|
426
|
+
this._userTurnCompletionVoiced = false;
|
|
427
|
+
}
|
|
428
|
+
else if (frame instanceof LLMFullResponseStartFrame) {
|
|
429
|
+
// A new LLM response is starting. If an incomplete timeout is still
|
|
430
|
+
// pending from a prior ◐/○, the LLM is already re-engaging: either the
|
|
431
|
+
// user's turn completed and this response carries the ●, or the
|
|
432
|
+
// timeout already fired its own re-prompt. Either way the pending
|
|
433
|
+
// re-prompt is now redundant, so cancel it to avoid running a second
|
|
434
|
+
// inference. This is the single point that resolves the race between
|
|
435
|
+
// the timeout firing and a ● arriving: whichever inference starts
|
|
436
|
+
// first cancels the timeout before its text is parsed.
|
|
437
|
+
await this._cancelIncompleteTimeout();
|
|
438
|
+
this._responseRaw = "";
|
|
439
|
+
this._responseMarker = null;
|
|
440
|
+
}
|
|
441
|
+
else if (frame instanceof LLMFullResponseEndFrame) {
|
|
442
|
+
await this._reportResponse();
|
|
443
|
+
await this._turnReset();
|
|
444
|
+
}
|
|
445
|
+
await super.pushFrame(frame, direction);
|
|
446
|
+
}
|
|
447
|
+
/** Push the marker read from the response, and remember it for the response's report. */
|
|
448
|
+
async _pushMarker(marker, kind, appendToContextImmediately = true) {
|
|
449
|
+
this._responseMarker = [marker, kind];
|
|
450
|
+
await this.pushFrame(new LLMMarkerFrame({ marker, appendToContextImmediately }));
|
|
451
|
+
}
|
|
452
|
+
/** Push the response's raw text and marker, once there was a response to read. */
|
|
453
|
+
async _reportResponse() {
|
|
454
|
+
if (!this._responseRaw && this._responseMarker === null)
|
|
455
|
+
return;
|
|
456
|
+
const config = this._userTurnCompletionConfig;
|
|
457
|
+
const [marker, kind] = this._responseMarker ?? [null, null];
|
|
458
|
+
await this.pushFrame(new LLMMarkerResponseFrame({
|
|
459
|
+
raw: this._responseRaw,
|
|
460
|
+
marker,
|
|
461
|
+
kind,
|
|
462
|
+
markers: [
|
|
463
|
+
config.completeMarker,
|
|
464
|
+
config.incompleteShortMarker,
|
|
465
|
+
config.incompleteLongMarker,
|
|
466
|
+
],
|
|
467
|
+
}));
|
|
468
|
+
this._responseRaw = "";
|
|
469
|
+
this._responseMarker = null;
|
|
470
|
+
}
|
|
471
|
+
/**
|
|
472
|
+
* Push LLM text with turn completion detection.
|
|
473
|
+
*
|
|
474
|
+
* This method should be used instead of
|
|
475
|
+
* `pushFrame(new LLMTextFrame({ text }))` when turn completion is enabled.
|
|
476
|
+
* It will:
|
|
477
|
+
* 1. Detect turn markers (●, ◐, or ○)
|
|
478
|
+
* 2. When ◐ (SHORT) is found: suppress text, start short timeout
|
|
479
|
+
* 3. When ○ (LONG) is found: suppress text, start long timeout
|
|
480
|
+
* 4. When ● (COMPLETE) is found: push all text with marker marked as skip_tts
|
|
481
|
+
* 5. After marker detected: all subsequent text flows through immediately
|
|
482
|
+
*
|
|
483
|
+
* @param text The text content from the LLM to push.
|
|
484
|
+
*/
|
|
485
|
+
async _pushTurnText(text) {
|
|
486
|
+
this._responseRaw += text;
|
|
487
|
+
// One spoken completion per user turn: once a ● has been voiced this
|
|
488
|
+
// user turn, drop text from any later inference (the acoustic detector
|
|
489
|
+
// can trigger several within one turn). `_turnMarker === null` scopes
|
|
490
|
+
// this to *fresh* responses — the response that produced the voiced ●
|
|
491
|
+
// keeps streaming, since by then its marker is COMPLETE.
|
|
492
|
+
if (this._userTurnCompletionVoiced && this._turnMarker === null)
|
|
493
|
+
return;
|
|
494
|
+
// If we've already detected incomplete, suppress all remaining text.
|
|
495
|
+
// This is a safety mechanism in case the LLM disobeys the prompt and
|
|
496
|
+
// outputs additional text after the marker (e.g., "◐ Please continue...").
|
|
497
|
+
if (this._turnMarker === TurnMarker.INCOMPLETE)
|
|
498
|
+
return;
|
|
499
|
+
// If ● (COMPLETE) was already found, push text immediately without buffering
|
|
500
|
+
if (this._turnMarker === TurnMarker.COMPLETE) {
|
|
501
|
+
await this.pushFrame(new LLMTextFrame({ text }));
|
|
502
|
+
return;
|
|
503
|
+
}
|
|
504
|
+
const config = this._userTurnCompletionConfig;
|
|
505
|
+
// Add text to buffer
|
|
506
|
+
this._turnTextBuffer += text;
|
|
507
|
+
// Check for incomplete markers (◐ short, ○ long). These indicate the
|
|
508
|
+
// user was cut off or needs time - we suppress the bot's response and
|
|
509
|
+
// start a timeout to re-prompt later.
|
|
510
|
+
let incompleteType = null;
|
|
511
|
+
if (this._turnTextBuffer.includes(config.incompleteShortMarker)) {
|
|
512
|
+
incompleteType = IncompleteType.SHORT;
|
|
513
|
+
}
|
|
514
|
+
else if (this._turnTextBuffer.includes(config.incompleteLongMarker)) {
|
|
515
|
+
incompleteType = IncompleteType.LONG;
|
|
516
|
+
}
|
|
517
|
+
if (incompleteType) {
|
|
518
|
+
const marker = incompleteType === IncompleteType.SHORT
|
|
519
|
+
? config.incompleteShortMarker
|
|
520
|
+
: config.incompleteLongMarker;
|
|
521
|
+
logger.debug(`INCOMPLETE ${incompleteType.toUpperCase()} (${marker}) detected, suppressing text`);
|
|
522
|
+
this._turnMarker = TurnMarker.INCOMPLETE;
|
|
523
|
+
// No UserTurnInferenceCompletedFrame is broadcast here: the turn is
|
|
524
|
+
// explicitly not complete. The re-prompt path is driven by this
|
|
525
|
+
// mixin's own timeout.
|
|
526
|
+
// Persist the marker to context as a stand-alone assistant message
|
|
527
|
+
// via LLMMarkerFrame: the bot produces no spoken output for
|
|
528
|
+
// incomplete turns, so the marker is the entire context entry.
|
|
529
|
+
await this._pushMarker(marker, incompleteType);
|
|
530
|
+
this._turnTextBuffer = "";
|
|
531
|
+
await this._startIncompleteTimeout(incompleteType);
|
|
532
|
+
return;
|
|
533
|
+
}
|
|
534
|
+
// Check for ● (COMPLETE) marker - user's turn was complete, respond normally
|
|
535
|
+
if (this._turnTextBuffer.includes(config.completeMarker)) {
|
|
536
|
+
if (this._userSpeaking) {
|
|
537
|
+
// Stale: the user resumed speaking after this inference was
|
|
538
|
+
// triggered, so the turn isn't over after all. Record it as ◐ and
|
|
539
|
+
// re-arm the short timeout, exactly as if the LLM had said so; the
|
|
540
|
+
// next inference re-evaluates the fuller turn.
|
|
541
|
+
logger.debug(`COMPLETE (${config.completeMarker}) detected while user is speaking, ` +
|
|
542
|
+
"treating as stale: suppressing text");
|
|
543
|
+
this._turnMarker = TurnMarker.INCOMPLETE;
|
|
544
|
+
await this._pushMarker(config.incompleteShortMarker, IncompleteType.SHORT);
|
|
545
|
+
this._turnTextBuffer = "";
|
|
546
|
+
await this._startIncompleteTimeout(IncompleteType.SHORT);
|
|
547
|
+
return;
|
|
548
|
+
}
|
|
549
|
+
logger.debug(`COMPLETE (${config.completeMarker}) detected, pushing buffered text`);
|
|
550
|
+
// Latch: this user turn now has its one spoken completion. Later
|
|
551
|
+
// duplicate inferences within the same turn are dropped by the guard
|
|
552
|
+
// at the top of this method.
|
|
553
|
+
this._userTurnCompletionVoiced = true;
|
|
554
|
+
// Any pending incomplete timeout was already cancelled when this
|
|
555
|
+
// response's LLMFullResponseStartFrame arrived (see `pushFrame`).
|
|
556
|
+
// Broadcast that the user turn is complete so a stop strategy gating
|
|
557
|
+
// finalization on this signal (e.g.
|
|
558
|
+
// LLMTurnCompletionUserTurnStopStrategy) can fire
|
|
559
|
+
// `on_user_turn_stopped`. Must fire before the marker so downstream
|
|
560
|
+
// consumers see the signal before the response. Idempotent: a tool
|
|
561
|
+
// call earlier in the turn may have already broadcast.
|
|
562
|
+
await this._broadcastTurnCompletion();
|
|
563
|
+
// Push the marker as a sideband signal that the assistant aggregator
|
|
564
|
+
// will prepend to the upcoming aggregated text, so the context message
|
|
565
|
+
// ends up as "● <response>".
|
|
566
|
+
await this._pushMarker(config.completeMarker, TurnMarker.COMPLETE, false);
|
|
567
|
+
// Split buffer at the marker to handle cases where marker and text
|
|
568
|
+
// arrive in the same chunk (e.g., "● Hello!" from some LLMs)
|
|
569
|
+
const markerPos = this._turnTextBuffer.indexOf(config.completeMarker);
|
|
570
|
+
const markerEnd = markerPos + config.completeMarker.length;
|
|
571
|
+
// Push remaining text after marker as normal speech
|
|
572
|
+
let remainingText = this._turnTextBuffer.slice(markerEnd);
|
|
573
|
+
if (remainingText) {
|
|
574
|
+
// Strip leading space after marker if present (● Hello -> Hello)
|
|
575
|
+
if (remainingText.startsWith(" ")) {
|
|
576
|
+
remainingText = remainingText.slice(1);
|
|
577
|
+
}
|
|
578
|
+
if (remainingText) {
|
|
579
|
+
await this.pushFrame(new LLMTextFrame({ text: remainingText }));
|
|
580
|
+
}
|
|
581
|
+
}
|
|
582
|
+
// Mark complete - all subsequent text flows through immediately
|
|
583
|
+
this._turnTextBuffer = "";
|
|
584
|
+
this._turnMarker = TurnMarker.COMPLETE;
|
|
585
|
+
return;
|
|
586
|
+
}
|
|
587
|
+
}
|
|
588
|
+
}
|
|
589
|
+
return UserTurnCompletionLLMServiceMixinImpl;
|
|
590
|
+
});
|
|
591
|
+
//# sourceMappingURL=user_turn_completion_mixin.js.map
|