@livekit/agents 0.7.9 → 1.0.0-next.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/_exceptions.cjs +109 -0
- package/dist/_exceptions.cjs.map +1 -0
- package/dist/_exceptions.d.cts +64 -0
- package/dist/_exceptions.d.ts +64 -0
- package/dist/_exceptions.d.ts.map +1 -0
- package/dist/_exceptions.js +80 -0
- package/dist/_exceptions.js.map +1 -0
- package/dist/audio.cjs +10 -3
- package/dist/audio.cjs.map +1 -1
- package/dist/audio.d.cts +2 -0
- package/dist/audio.d.ts +2 -0
- package/dist/audio.d.ts.map +1 -1
- package/dist/audio.js +8 -2
- package/dist/audio.js.map +1 -1
- package/dist/cli.cjs +25 -0
- package/dist/cli.cjs.map +1 -1
- package/dist/cli.d.ts.map +1 -1
- package/dist/cli.js +25 -0
- package/dist/cli.js.map +1 -1
- package/dist/constants.cjs +6 -3
- package/dist/constants.cjs.map +1 -1
- package/dist/constants.d.cts +2 -1
- package/dist/constants.d.ts +2 -1
- package/dist/constants.d.ts.map +1 -1
- package/dist/constants.js +4 -2
- package/dist/constants.js.map +1 -1
- package/dist/http_server.cjs.map +1 -1
- package/dist/http_server.d.cts +1 -0
- package/dist/http_server.d.ts +1 -0
- package/dist/http_server.d.ts.map +1 -1
- package/dist/http_server.js.map +1 -1
- package/dist/index.cjs +27 -20
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +13 -10
- package/dist/index.d.ts +13 -10
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +15 -11
- package/dist/index.js.map +1 -1
- package/dist/inference_runner.cjs +0 -1
- package/dist/inference_runner.cjs.map +1 -1
- package/dist/inference_runner.d.cts +2 -3
- package/dist/inference_runner.d.ts +2 -3
- package/dist/inference_runner.d.ts.map +1 -1
- package/dist/inference_runner.js +0 -1
- package/dist/inference_runner.js.map +1 -1
- package/dist/ipc/inference_proc_executor.cjs +2 -2
- package/dist/ipc/inference_proc_executor.cjs.map +1 -1
- package/dist/ipc/inference_proc_executor.js +2 -2
- package/dist/ipc/inference_proc_executor.js.map +1 -1
- package/dist/ipc/job_executor.cjs.map +1 -1
- package/dist/ipc/job_executor.js.map +1 -1
- package/dist/ipc/job_proc_executor.cjs +1 -0
- package/dist/ipc/job_proc_executor.cjs.map +1 -1
- package/dist/ipc/job_proc_executor.js +1 -0
- package/dist/ipc/job_proc_executor.js.map +1 -1
- package/dist/ipc/job_proc_lazy_main.cjs +1 -1
- package/dist/ipc/job_proc_lazy_main.cjs.map +1 -1
- package/dist/ipc/job_proc_lazy_main.js +1 -1
- package/dist/ipc/job_proc_lazy_main.js.map +1 -1
- package/dist/ipc/supervised_proc.d.cts +1 -1
- package/dist/ipc/supervised_proc.d.ts +1 -1
- package/dist/ipc/supervised_proc.d.ts.map +1 -1
- package/dist/job.cjs +14 -2
- package/dist/job.cjs.map +1 -1
- package/dist/job.d.cts +8 -0
- package/dist/job.d.ts +8 -0
- package/dist/job.d.ts.map +1 -1
- package/dist/job.js +12 -1
- package/dist/job.js.map +1 -1
- package/dist/llm/chat_context.cjs +332 -82
- package/dist/llm/chat_context.cjs.map +1 -1
- package/dist/llm/chat_context.d.cts +152 -48
- package/dist/llm/chat_context.d.ts +152 -48
- package/dist/llm/chat_context.d.ts.map +1 -1
- package/dist/llm/chat_context.js +327 -81
- package/dist/llm/chat_context.js.map +1 -1
- package/dist/llm/chat_context.test.cjs +380 -0
- package/dist/llm/chat_context.test.cjs.map +1 -0
- package/dist/llm/chat_context.test.js +385 -0
- package/dist/llm/chat_context.test.js.map +1 -0
- package/dist/llm/index.cjs +37 -8
- package/dist/llm/index.cjs.map +1 -1
- package/dist/llm/index.d.cts +7 -3
- package/dist/llm/index.d.ts +7 -3
- package/dist/llm/index.d.ts.map +1 -1
- package/dist/llm/index.js +39 -9
- package/dist/llm/index.js.map +1 -1
- package/dist/llm/llm.cjs +98 -33
- package/dist/llm/llm.cjs.map +1 -1
- package/dist/llm/llm.d.cts +50 -24
- package/dist/llm/llm.d.ts +50 -24
- package/dist/llm/llm.d.ts.map +1 -1
- package/dist/llm/llm.js +99 -33
- package/dist/llm/llm.js.map +1 -1
- package/dist/llm/provider_format/google.cjs +128 -0
- package/dist/llm/provider_format/google.cjs.map +1 -0
- package/dist/llm/provider_format/google.d.cts +6 -0
- package/dist/llm/provider_format/google.d.ts +6 -0
- package/dist/llm/provider_format/google.d.ts.map +1 -0
- package/dist/llm/provider_format/google.js +104 -0
- package/dist/llm/provider_format/google.js.map +1 -0
- package/dist/llm/provider_format/google.test.cjs +676 -0
- package/dist/llm/provider_format/google.test.cjs.map +1 -0
- package/dist/llm/provider_format/google.test.js +675 -0
- package/dist/llm/provider_format/google.test.js.map +1 -0
- package/dist/llm/provider_format/index.cjs +40 -0
- package/dist/llm/provider_format/index.cjs.map +1 -0
- package/dist/llm/provider_format/index.d.cts +4 -0
- package/dist/llm/provider_format/index.d.ts +4 -0
- package/dist/llm/provider_format/index.d.ts.map +1 -0
- package/dist/llm/provider_format/index.js +16 -0
- package/dist/llm/provider_format/index.js.map +1 -0
- package/dist/llm/provider_format/openai.cjs +116 -0
- package/dist/llm/provider_format/openai.cjs.map +1 -0
- package/dist/llm/provider_format/openai.d.cts +3 -0
- package/dist/llm/provider_format/openai.d.ts +3 -0
- package/dist/llm/provider_format/openai.d.ts.map +1 -0
- package/dist/llm/provider_format/openai.js +92 -0
- package/dist/llm/provider_format/openai.js.map +1 -0
- package/dist/llm/provider_format/openai.test.cjs +490 -0
- package/dist/llm/provider_format/openai.test.cjs.map +1 -0
- package/dist/llm/provider_format/openai.test.js +489 -0
- package/dist/llm/provider_format/openai.test.js.map +1 -0
- package/dist/llm/provider_format/utils.cjs +146 -0
- package/dist/llm/provider_format/utils.cjs.map +1 -0
- package/dist/llm/provider_format/utils.d.cts +38 -0
- package/dist/llm/provider_format/utils.d.ts +38 -0
- package/dist/llm/provider_format/utils.d.ts.map +1 -0
- package/dist/llm/provider_format/utils.js +122 -0
- package/dist/llm/provider_format/utils.js.map +1 -0
- package/dist/llm/realtime.cjs +77 -0
- package/dist/llm/realtime.cjs.map +1 -0
- package/dist/llm/realtime.d.cts +98 -0
- package/dist/llm/realtime.d.ts +98 -0
- package/dist/llm/realtime.d.ts.map +1 -0
- package/dist/llm/realtime.js +52 -0
- package/dist/llm/realtime.js.map +1 -0
- package/dist/llm/remote_chat_context.cjs +112 -0
- package/dist/llm/remote_chat_context.cjs.map +1 -0
- package/dist/llm/remote_chat_context.d.cts +23 -0
- package/dist/llm/remote_chat_context.d.ts +23 -0
- package/dist/llm/remote_chat_context.d.ts.map +1 -0
- package/dist/llm/remote_chat_context.js +88 -0
- package/dist/llm/remote_chat_context.js.map +1 -0
- package/dist/llm/remote_chat_context.test.cjs +225 -0
- package/dist/llm/remote_chat_context.test.cjs.map +1 -0
- package/dist/llm/remote_chat_context.test.js +224 -0
- package/dist/llm/remote_chat_context.test.js.map +1 -0
- package/dist/llm/tool_context.cjs +111 -0
- package/dist/llm/tool_context.cjs.map +1 -0
- package/dist/llm/tool_context.d.cts +125 -0
- package/dist/llm/tool_context.d.ts +125 -0
- package/dist/llm/tool_context.d.ts.map +1 -0
- package/dist/llm/tool_context.js +80 -0
- package/dist/llm/tool_context.js.map +1 -0
- package/dist/llm/tool_context.test.cjs +162 -0
- package/dist/llm/tool_context.test.cjs.map +1 -0
- package/dist/llm/tool_context.test.js +161 -0
- package/dist/llm/tool_context.test.js.map +1 -0
- package/dist/llm/tool_context.type.test.cjs +92 -0
- package/dist/llm/tool_context.type.test.cjs.map +1 -0
- package/dist/llm/tool_context.type.test.js +91 -0
- package/dist/llm/tool_context.type.test.js.map +1 -0
- package/dist/llm/utils.cjs +260 -0
- package/dist/llm/utils.cjs.map +1 -0
- package/dist/llm/utils.d.cts +42 -0
- package/dist/llm/utils.d.ts +42 -0
- package/dist/llm/utils.d.ts.map +1 -0
- package/dist/llm/utils.js +223 -0
- package/dist/llm/utils.js.map +1 -0
- package/dist/llm/utils.test.cjs +513 -0
- package/dist/llm/utils.test.cjs.map +1 -0
- package/dist/llm/utils.test.js +490 -0
- package/dist/llm/utils.test.js.map +1 -0
- package/dist/metrics/base.cjs +0 -27
- package/dist/metrics/base.cjs.map +1 -1
- package/dist/metrics/base.d.cts +105 -63
- package/dist/metrics/base.d.ts +105 -63
- package/dist/metrics/base.d.ts.map +1 -1
- package/dist/metrics/base.js +0 -19
- package/dist/metrics/base.js.map +1 -1
- package/dist/metrics/index.cjs +0 -3
- package/dist/metrics/index.cjs.map +1 -1
- package/dist/metrics/index.d.cts +2 -3
- package/dist/metrics/index.d.ts +2 -3
- package/dist/metrics/index.d.ts.map +1 -1
- package/dist/metrics/index.js +0 -2
- package/dist/metrics/index.js.map +1 -1
- package/dist/metrics/usage_collector.cjs +17 -12
- package/dist/metrics/usage_collector.cjs.map +1 -1
- package/dist/metrics/usage_collector.d.cts +3 -2
- package/dist/metrics/usage_collector.d.ts +3 -2
- package/dist/metrics/usage_collector.d.ts.map +1 -1
- package/dist/metrics/usage_collector.js +17 -12
- package/dist/metrics/usage_collector.js.map +1 -1
- package/dist/metrics/utils.cjs +22 -59
- package/dist/metrics/utils.cjs.map +1 -1
- package/dist/metrics/utils.d.cts +1 -8
- package/dist/metrics/utils.d.ts +1 -8
- package/dist/metrics/utils.d.ts.map +1 -1
- package/dist/metrics/utils.js +22 -52
- package/dist/metrics/utils.js.map +1 -1
- package/dist/multimodal/index.cjs +0 -2
- package/dist/multimodal/index.cjs.map +1 -1
- package/dist/multimodal/index.d.cts +0 -1
- package/dist/multimodal/index.d.ts +0 -1
- package/dist/multimodal/index.d.ts.map +1 -1
- package/dist/multimodal/index.js +0 -1
- package/dist/multimodal/index.js.map +1 -1
- package/dist/plugin.cjs +24 -8
- package/dist/plugin.cjs.map +1 -1
- package/dist/plugin.d.cts +18 -4
- package/dist/plugin.d.ts +18 -4
- package/dist/plugin.d.ts.map +1 -1
- package/dist/plugin.js +22 -7
- package/dist/plugin.js.map +1 -1
- package/dist/stream/deferred_stream.cjs +98 -0
- package/dist/stream/deferred_stream.cjs.map +1 -0
- package/dist/stream/deferred_stream.d.cts +27 -0
- package/dist/stream/deferred_stream.d.ts +27 -0
- package/dist/stream/deferred_stream.d.ts.map +1 -0
- package/dist/stream/deferred_stream.js +73 -0
- package/dist/stream/deferred_stream.js.map +1 -0
- package/dist/stream/deferred_stream.test.cjs +527 -0
- package/dist/stream/deferred_stream.test.cjs.map +1 -0
- package/dist/stream/deferred_stream.test.js +526 -0
- package/dist/stream/deferred_stream.test.js.map +1 -0
- package/dist/stream/identity_transform.cjs +42 -0
- package/dist/stream/identity_transform.cjs.map +1 -0
- package/dist/stream/identity_transform.d.cts +6 -0
- package/dist/stream/identity_transform.d.ts +6 -0
- package/dist/stream/identity_transform.d.ts.map +1 -0
- package/dist/stream/identity_transform.js +18 -0
- package/dist/stream/identity_transform.js.map +1 -0
- package/dist/stream/identity_transform.test.cjs +125 -0
- package/dist/stream/identity_transform.test.cjs.map +1 -0
- package/dist/stream/identity_transform.test.js +124 -0
- package/dist/stream/identity_transform.test.js.map +1 -0
- package/dist/stream/index.cjs +38 -0
- package/dist/stream/index.cjs.map +1 -0
- package/dist/stream/index.d.cts +5 -0
- package/dist/stream/index.d.ts +5 -0
- package/dist/stream/index.d.ts.map +1 -0
- package/dist/stream/index.js +11 -0
- package/dist/stream/index.js.map +1 -0
- package/dist/stream/merge_readable_streams.cjs +59 -0
- package/dist/stream/merge_readable_streams.cjs.map +1 -0
- package/dist/stream/merge_readable_streams.d.cts +4 -0
- package/dist/stream/merge_readable_streams.d.ts +4 -0
- package/dist/stream/merge_readable_streams.d.ts.map +1 -0
- package/dist/stream/merge_readable_streams.js +35 -0
- package/dist/stream/merge_readable_streams.js.map +1 -0
- package/dist/stream/stream_channel.cjs +47 -0
- package/dist/stream/stream_channel.cjs.map +1 -0
- package/dist/stream/stream_channel.d.cts +9 -0
- package/dist/stream/stream_channel.d.ts +9 -0
- package/dist/stream/stream_channel.d.ts.map +1 -0
- package/dist/stream/stream_channel.js +23 -0
- package/dist/stream/stream_channel.js.map +1 -0
- package/dist/stream/stream_channel.test.cjs +97 -0
- package/dist/stream/stream_channel.test.cjs.map +1 -0
- package/dist/stream/stream_channel.test.js +96 -0
- package/dist/stream/stream_channel.test.js.map +1 -0
- package/dist/stt/stream_adapter.cjs +3 -4
- package/dist/stt/stream_adapter.cjs.map +1 -1
- package/dist/stt/stream_adapter.d.cts +1 -0
- package/dist/stt/stream_adapter.d.ts +1 -0
- package/dist/stt/stream_adapter.d.ts.map +1 -1
- package/dist/stt/stream_adapter.js +3 -4
- package/dist/stt/stream_adapter.js.map +1 -1
- package/dist/stt/stt.cjs +101 -10
- package/dist/stt/stt.cjs.map +1 -1
- package/dist/stt/stt.d.cts +26 -5
- package/dist/stt/stt.d.ts +26 -5
- package/dist/stt/stt.d.ts.map +1 -1
- package/dist/stt/stt.js +102 -11
- package/dist/stt/stt.js.map +1 -1
- package/dist/tokenize/basic/basic.cjs +10 -5
- package/dist/tokenize/basic/basic.cjs.map +1 -1
- package/dist/tokenize/basic/basic.d.cts +7 -1
- package/dist/tokenize/basic/basic.d.ts +7 -1
- package/dist/tokenize/basic/basic.d.ts.map +1 -1
- package/dist/tokenize/basic/basic.js +10 -5
- package/dist/tokenize/basic/basic.js.map +1 -1
- package/dist/tokenize/basic/sentence.cjs +14 -6
- package/dist/tokenize/basic/sentence.cjs.map +1 -1
- package/dist/tokenize/basic/sentence.d.cts +1 -1
- package/dist/tokenize/basic/sentence.d.ts +1 -1
- package/dist/tokenize/basic/sentence.d.ts.map +1 -1
- package/dist/tokenize/basic/sentence.js +14 -6
- package/dist/tokenize/basic/sentence.js.map +1 -1
- package/dist/tokenize/token_stream.cjs +5 -3
- package/dist/tokenize/token_stream.cjs.map +1 -1
- package/dist/tokenize/token_stream.d.cts +1 -0
- package/dist/tokenize/token_stream.d.ts +1 -0
- package/dist/tokenize/token_stream.d.ts.map +1 -1
- package/dist/tokenize/token_stream.js +6 -4
- package/dist/tokenize/token_stream.js.map +1 -1
- package/dist/transcription.cjs +1 -2
- package/dist/transcription.cjs.map +1 -1
- package/dist/transcription.d.ts.map +1 -1
- package/dist/transcription.js +2 -3
- package/dist/transcription.js.map +1 -1
- package/dist/tts/index.cjs +2 -4
- package/dist/tts/index.cjs.map +1 -1
- package/dist/tts/index.d.cts +1 -1
- package/dist/tts/index.d.ts +1 -1
- package/dist/tts/index.d.ts.map +1 -1
- package/dist/tts/index.js +1 -3
- package/dist/tts/index.js.map +1 -1
- package/dist/tts/stream_adapter.cjs +26 -13
- package/dist/tts/stream_adapter.cjs.map +1 -1
- package/dist/tts/stream_adapter.d.cts +1 -1
- package/dist/tts/stream_adapter.d.ts +1 -1
- package/dist/tts/stream_adapter.d.ts.map +1 -1
- package/dist/tts/stream_adapter.js +27 -14
- package/dist/tts/stream_adapter.js.map +1 -1
- package/dist/tts/tts.cjs +157 -25
- package/dist/tts/tts.cjs.map +1 -1
- package/dist/tts/tts.d.cts +29 -5
- package/dist/tts/tts.d.ts +29 -5
- package/dist/tts/tts.d.ts.map +1 -1
- package/dist/tts/tts.js +157 -24
- package/dist/tts/tts.js.map +1 -1
- package/dist/types.cjs +60 -0
- package/dist/types.cjs.map +1 -0
- package/dist/types.d.cts +13 -0
- package/dist/types.d.ts +13 -0
- package/dist/types.d.ts.map +1 -0
- package/dist/types.js +35 -0
- package/dist/types.js.map +1 -0
- package/dist/utils.cjs +281 -27
- package/dist/utils.cjs.map +1 -1
- package/dist/utils.d.cts +134 -9
- package/dist/utils.d.ts +134 -9
- package/dist/utils.d.ts.map +1 -1
- package/dist/utils.js +265 -26
- package/dist/utils.js.map +1 -1
- package/dist/utils.test.cjs +492 -0
- package/dist/utils.test.cjs.map +1 -0
- package/dist/utils.test.js +498 -0
- package/dist/utils.test.js.map +1 -0
- package/dist/vad.cjs +76 -20
- package/dist/vad.cjs.map +1 -1
- package/dist/vad.d.cts +25 -5
- package/dist/vad.d.ts +25 -5
- package/dist/vad.d.ts.map +1 -1
- package/dist/vad.js +76 -20
- package/dist/vad.js.map +1 -1
- package/dist/voice/agent.cjs +245 -0
- package/dist/voice/agent.cjs.map +1 -0
- package/dist/voice/agent.d.cts +78 -0
- package/dist/voice/agent.d.ts +78 -0
- package/dist/voice/agent.d.ts.map +1 -0
- package/dist/voice/agent.js +220 -0
- package/dist/voice/agent.js.map +1 -0
- package/dist/voice/agent.test.cjs +61 -0
- package/dist/voice/agent.test.cjs.map +1 -0
- package/dist/voice/agent.test.js +60 -0
- package/dist/voice/agent.test.js.map +1 -0
- package/dist/voice/agent_activity.cjs +1453 -0
- package/dist/voice/agent_activity.cjs.map +1 -0
- package/dist/voice/agent_activity.d.cts +94 -0
- package/dist/voice/agent_activity.d.ts +94 -0
- package/dist/voice/agent_activity.d.ts.map +1 -0
- package/dist/voice/agent_activity.js +1449 -0
- package/dist/voice/agent_activity.js.map +1 -0
- package/dist/voice/agent_session.cjs +312 -0
- package/dist/voice/agent_session.cjs.map +1 -0
- package/dist/voice/agent_session.d.cts +121 -0
- package/dist/voice/agent_session.d.ts +121 -0
- package/dist/voice/agent_session.d.ts.map +1 -0
- package/dist/voice/agent_session.js +295 -0
- package/dist/voice/agent_session.js.map +1 -0
- package/dist/voice/audio_recognition.cjs +375 -0
- package/dist/voice/audio_recognition.cjs.map +1 -0
- package/dist/voice/audio_recognition.d.cts +80 -0
- package/dist/voice/audio_recognition.d.ts +80 -0
- package/dist/voice/audio_recognition.d.ts.map +1 -0
- package/dist/voice/audio_recognition.js +351 -0
- package/dist/voice/audio_recognition.js.map +1 -0
- package/dist/voice/events.cjs +145 -0
- package/dist/voice/events.cjs.map +1 -0
- package/dist/voice/events.d.cts +124 -0
- package/dist/voice/events.d.ts +124 -0
- package/dist/voice/events.d.ts.map +1 -0
- package/dist/voice/events.js +110 -0
- package/dist/voice/events.js.map +1 -0
- package/dist/voice/generation.cjs +700 -0
- package/dist/voice/generation.cjs.map +1 -0
- package/dist/voice/generation.d.cts +115 -0
- package/dist/voice/generation.d.ts +115 -0
- package/dist/voice/generation.d.ts.map +1 -0
- package/dist/voice/generation.js +672 -0
- package/dist/voice/generation.js.map +1 -0
- package/dist/voice/index.cjs +40 -0
- package/dist/voice/index.cjs.map +1 -0
- package/dist/voice/index.d.cts +5 -0
- package/dist/voice/index.d.ts +5 -0
- package/dist/voice/index.d.ts.map +1 -0
- package/dist/voice/index.js +11 -0
- package/dist/voice/index.js.map +1 -0
- package/dist/voice/io.cjs +245 -0
- package/dist/voice/io.cjs.map +1 -0
- package/dist/voice/io.d.cts +101 -0
- package/dist/voice/io.d.ts +101 -0
- package/dist/voice/io.d.ts.map +1 -0
- package/dist/voice/io.js +217 -0
- package/dist/voice/io.js.map +1 -0
- package/dist/voice/room_io/_input.cjs +121 -0
- package/dist/voice/room_io/_input.cjs.map +1 -0
- package/dist/voice/room_io/_input.d.cts +24 -0
- package/dist/voice/room_io/_input.d.ts +24 -0
- package/dist/voice/room_io/_input.d.ts.map +1 -0
- package/dist/voice/room_io/_input.js +102 -0
- package/dist/voice/room_io/_input.js.map +1 -0
- package/dist/voice/room_io/_output.cjs +358 -0
- package/dist/voice/room_io/_output.cjs.map +1 -0
- package/dist/voice/room_io/_output.d.cts +75 -0
- package/dist/voice/room_io/_output.d.ts +75 -0
- package/dist/voice/room_io/_output.d.ts.map +1 -0
- package/dist/voice/room_io/_output.js +342 -0
- package/dist/voice/room_io/_output.js.map +1 -0
- package/dist/voice/room_io/index.cjs +25 -0
- package/dist/voice/room_io/index.cjs.map +1 -0
- package/dist/voice/room_io/index.d.cts +3 -0
- package/dist/voice/room_io/index.d.ts +3 -0
- package/dist/voice/room_io/index.d.ts.map +1 -0
- package/dist/voice/room_io/index.js +3 -0
- package/dist/voice/room_io/index.js.map +1 -0
- package/dist/voice/room_io/room_io.cjs +370 -0
- package/dist/voice/room_io/room_io.cjs.map +1 -0
- package/dist/voice/room_io/room_io.d.cts +73 -0
- package/dist/voice/room_io/room_io.d.ts +73 -0
- package/dist/voice/room_io/room_io.d.ts.map +1 -0
- package/dist/voice/room_io/room_io.js +361 -0
- package/dist/voice/room_io/room_io.js.map +1 -0
- package/dist/{pipeline/index.cjs → voice/run_context.cjs} +16 -11
- package/dist/voice/run_context.cjs.map +1 -0
- package/dist/voice/run_context.d.cts +12 -0
- package/dist/voice/run_context.d.ts +12 -0
- package/dist/voice/run_context.d.ts.map +1 -0
- package/dist/voice/run_context.js +14 -0
- package/dist/voice/run_context.js.map +1 -0
- package/dist/voice/speech_handle.cjs +105 -0
- package/dist/voice/speech_handle.cjs.map +1 -0
- package/dist/voice/speech_handle.d.cts +46 -0
- package/dist/voice/speech_handle.d.ts +46 -0
- package/dist/voice/speech_handle.d.ts.map +1 -0
- package/dist/voice/speech_handle.js +81 -0
- package/dist/voice/speech_handle.js.map +1 -0
- package/dist/voice/transcription/_utils.cjs +45 -0
- package/dist/voice/transcription/_utils.cjs.map +1 -0
- package/dist/voice/transcription/_utils.d.cts +3 -0
- package/dist/voice/transcription/_utils.d.ts +3 -0
- package/dist/voice/transcription/_utils.d.ts.map +1 -0
- package/dist/voice/transcription/_utils.js +21 -0
- package/dist/voice/transcription/_utils.js.map +1 -0
- package/dist/voice/transcription/index.cjs +23 -0
- package/dist/voice/transcription/index.cjs.map +1 -0
- package/dist/voice/transcription/index.d.cts +2 -0
- package/dist/voice/transcription/index.d.ts +2 -0
- package/dist/voice/transcription/index.d.ts.map +1 -0
- package/dist/voice/transcription/index.js +2 -0
- package/dist/voice/transcription/index.js.map +1 -0
- package/dist/voice/transcription/synchronizer.cjs +380 -0
- package/dist/voice/transcription/synchronizer.cjs.map +1 -0
- package/dist/voice/transcription/synchronizer.d.cts +86 -0
- package/dist/voice/transcription/synchronizer.d.ts +86 -0
- package/dist/voice/transcription/synchronizer.d.ts.map +1 -0
- package/dist/voice/transcription/synchronizer.js +355 -0
- package/dist/voice/transcription/synchronizer.js.map +1 -0
- package/dist/worker.cjs +22 -4
- package/dist/worker.cjs.map +1 -1
- package/dist/worker.d.cts +1 -1
- package/dist/worker.d.ts +1 -1
- package/dist/worker.d.ts.map +1 -1
- package/dist/worker.js +22 -4
- package/dist/worker.js.map +1 -1
- package/package.json +9 -2
- package/src/_exceptions.ts +137 -0
- package/src/audio.ts +12 -1
- package/src/cli.ts +37 -0
- package/src/constants.ts +2 -1
- package/src/http_server.ts +1 -0
- package/src/index.ts +13 -10
- package/src/inference_runner.ts +2 -3
- package/src/ipc/inference_proc_executor.ts +2 -2
- package/src/ipc/job_executor.ts +1 -1
- package/src/ipc/job_proc_executor.ts +1 -1
- package/src/ipc/job_proc_lazy_main.ts +1 -1
- package/src/job.ts +18 -0
- package/src/llm/__snapshots__/chat_context.test.ts.snap +527 -0
- package/src/llm/__snapshots__/tool_context.test.ts.snap +177 -0
- package/src/llm/__snapshots__/utils.test.ts.snap +65 -0
- package/src/llm/chat_context.test.ts +450 -0
- package/src/llm/chat_context.ts +501 -103
- package/src/llm/index.ts +53 -18
- package/src/llm/llm.ts +149 -50
- package/src/llm/provider_format/google.test.ts +772 -0
- package/src/llm/provider_format/google.ts +130 -0
- package/src/llm/provider_format/index.ts +23 -0
- package/src/llm/provider_format/openai.test.ts +581 -0
- package/src/llm/provider_format/openai.ts +118 -0
- package/src/llm/provider_format/utils.ts +183 -0
- package/src/llm/realtime.ts +151 -0
- package/src/llm/remote_chat_context.test.ts +290 -0
- package/src/llm/remote_chat_context.ts +114 -0
- package/src/llm/tool_context.test.ts +198 -0
- package/src/llm/tool_context.ts +259 -0
- package/src/llm/tool_context.type.test.ts +115 -0
- package/src/llm/utils.test.ts +670 -0
- package/src/llm/utils.ts +324 -0
- package/src/metrics/base.ts +110 -78
- package/src/metrics/index.ts +3 -9
- package/src/metrics/usage_collector.ts +19 -13
- package/src/metrics/utils.ts +24 -69
- package/src/multimodal/index.ts +0 -1
- package/src/plugin.ts +26 -8
- package/src/stream/deferred_stream.test.ts +755 -0
- package/src/stream/deferred_stream.ts +110 -0
- package/src/stream/identity_transform.test.ts +179 -0
- package/src/stream/identity_transform.ts +18 -0
- package/src/stream/index.ts +7 -0
- package/src/stream/merge_readable_streams.ts +40 -0
- package/src/stream/stream_channel.test.ts +129 -0
- package/src/stream/stream_channel.ts +32 -0
- package/src/stt/stream_adapter.ts +3 -5
- package/src/stt/stt.ts +135 -17
- package/src/tokenize/basic/basic.ts +13 -5
- package/src/tokenize/basic/sentence.ts +20 -6
- package/src/tokenize/token_stream.ts +7 -4
- package/src/transcription.ts +2 -3
- package/src/tts/index.ts +0 -1
- package/src/tts/stream_adapter.ts +42 -16
- package/src/tts/tts.ts +203 -21
- package/src/types.ts +42 -0
- package/src/utils.test.ts +658 -0
- package/src/utils.ts +375 -44
- package/src/vad.ts +90 -22
- package/src/voice/agent.test.ts +80 -0
- package/src/voice/agent.ts +332 -0
- package/src/voice/agent_activity.ts +1913 -0
- package/src/voice/agent_session.ts +460 -0
- package/src/voice/audio_recognition.ts +474 -0
- package/src/voice/events.ts +252 -0
- package/src/voice/generation.ts +881 -0
- package/src/voice/index.ts +7 -0
- package/src/voice/io.ts +304 -0
- package/src/voice/room_io/_input.ts +144 -0
- package/src/voice/room_io/_output.ts +436 -0
- package/src/voice/room_io/index.ts +5 -0
- package/src/voice/room_io/room_io.ts +495 -0
- package/src/voice/run_context.ts +20 -0
- package/src/voice/speech_handle.ts +104 -0
- package/src/voice/transcription/_utils.ts +25 -0
- package/src/voice/transcription/index.ts +4 -0
- package/src/voice/transcription/synchronizer.ts +478 -0
- package/src/worker.ts +22 -2
- package/dist/llm/function_context.cjs +0 -103
- package/dist/llm/function_context.cjs.map +0 -1
- package/dist/llm/function_context.d.cts +0 -47
- package/dist/llm/function_context.d.ts +0 -47
- package/dist/llm/function_context.d.ts.map +0 -1
- package/dist/llm/function_context.js +0 -78
- package/dist/llm/function_context.js.map +0 -1
- package/dist/llm/function_context.test.cjs +0 -218
- package/dist/llm/function_context.test.cjs.map +0 -1
- package/dist/llm/function_context.test.js +0 -217
- package/dist/llm/function_context.test.js.map +0 -1
- package/dist/multimodal/multimodal_agent.cjs +0 -486
- package/dist/multimodal/multimodal_agent.cjs.map +0 -1
- package/dist/multimodal/multimodal_agent.d.cts +0 -48
- package/dist/multimodal/multimodal_agent.d.ts +0 -48
- package/dist/multimodal/multimodal_agent.d.ts.map +0 -1
- package/dist/multimodal/multimodal_agent.js +0 -461
- package/dist/multimodal/multimodal_agent.js.map +0 -1
- package/dist/pipeline/agent_output.cjs +0 -197
- package/dist/pipeline/agent_output.cjs.map +0 -1
- package/dist/pipeline/agent_output.d.cts +0 -33
- package/dist/pipeline/agent_output.d.ts +0 -33
- package/dist/pipeline/agent_output.d.ts.map +0 -1
- package/dist/pipeline/agent_output.js +0 -172
- package/dist/pipeline/agent_output.js.map +0 -1
- package/dist/pipeline/agent_playout.cjs +0 -175
- package/dist/pipeline/agent_playout.cjs.map +0 -1
- package/dist/pipeline/agent_playout.d.cts +0 -40
- package/dist/pipeline/agent_playout.d.ts +0 -40
- package/dist/pipeline/agent_playout.d.ts.map +0 -1
- package/dist/pipeline/agent_playout.js +0 -139
- package/dist/pipeline/agent_playout.js.map +0 -1
- package/dist/pipeline/human_input.cjs +0 -171
- package/dist/pipeline/human_input.cjs.map +0 -1
- package/dist/pipeline/human_input.d.cts +0 -30
- package/dist/pipeline/human_input.d.ts +0 -30
- package/dist/pipeline/human_input.d.ts.map +0 -1
- package/dist/pipeline/human_input.js +0 -146
- package/dist/pipeline/human_input.js.map +0 -1
- package/dist/pipeline/index.cjs.map +0 -1
- package/dist/pipeline/index.d.cts +0 -2
- package/dist/pipeline/index.d.ts +0 -2
- package/dist/pipeline/index.d.ts.map +0 -1
- package/dist/pipeline/index.js +0 -11
- package/dist/pipeline/index.js.map +0 -1
- package/dist/pipeline/pipeline_agent.cjs +0 -859
- package/dist/pipeline/pipeline_agent.cjs.map +0 -1
- package/dist/pipeline/pipeline_agent.d.cts +0 -150
- package/dist/pipeline/pipeline_agent.d.ts +0 -150
- package/dist/pipeline/pipeline_agent.d.ts.map +0 -1
- package/dist/pipeline/pipeline_agent.js +0 -837
- package/dist/pipeline/pipeline_agent.js.map +0 -1
- package/dist/pipeline/speech_handle.cjs +0 -176
- package/dist/pipeline/speech_handle.cjs.map +0 -1
- package/dist/pipeline/speech_handle.d.cts +0 -37
- package/dist/pipeline/speech_handle.d.ts +0 -37
- package/dist/pipeline/speech_handle.d.ts.map +0 -1
- package/dist/pipeline/speech_handle.js +0 -152
- package/dist/pipeline/speech_handle.js.map +0 -1
- package/src/llm/function_context.test.ts +0 -248
- package/src/llm/function_context.ts +0 -142
- package/src/multimodal/multimodal_agent.ts +0 -592
- package/src/pipeline/agent_output.ts +0 -219
- package/src/pipeline/agent_playout.ts +0 -192
- package/src/pipeline/human_input.ts +0 -188
- package/src/pipeline/index.ts +0 -15
- package/src/pipeline/pipeline_agent.ts +0 -1197
- package/src/pipeline/speech_handle.ts +0 -201
|
@@ -0,0 +1,223 @@
|
|
|
1
|
+
import { VideoBufferType, VideoFrame } from "@livekit/rtc-node";
|
|
2
|
+
import sharp from "sharp";
|
|
3
|
+
import { ZodObject } from "zod";
|
|
4
|
+
import { zodToJsonSchema } from "zod-to-json-schema";
|
|
5
|
+
import {
|
|
6
|
+
FunctionCall,
|
|
7
|
+
FunctionCallOutput
|
|
8
|
+
} from "./chat_context.js";
|
|
9
|
+
function getChannelsFromVideoBufferType(type) {
|
|
10
|
+
switch (type) {
|
|
11
|
+
case VideoBufferType.RGBA:
|
|
12
|
+
case VideoBufferType.ABGR:
|
|
13
|
+
case VideoBufferType.ARGB:
|
|
14
|
+
case VideoBufferType.BGRA:
|
|
15
|
+
return 4;
|
|
16
|
+
case VideoBufferType.RGB24:
|
|
17
|
+
return 3;
|
|
18
|
+
default:
|
|
19
|
+
throw new Error(`Unsupported VideoBufferType: ${type}. Only RGB/RGBA formats are supported.`);
|
|
20
|
+
}
|
|
21
|
+
}
|
|
22
|
+
function ensureRGBCompatible(frame) {
|
|
23
|
+
if (frame.type === VideoBufferType.RGBA || frame.type === VideoBufferType.BGRA || frame.type === VideoBufferType.ARGB || frame.type === VideoBufferType.ABGR || frame.type === VideoBufferType.RGB24) {
|
|
24
|
+
return frame;
|
|
25
|
+
}
|
|
26
|
+
try {
|
|
27
|
+
return frame.convert(VideoBufferType.RGBA);
|
|
28
|
+
} catch (error) {
|
|
29
|
+
throw new Error(
|
|
30
|
+
`Failed to convert format ${frame.type} to RGB: ${error}. Consider using RGB/RGBA formats or converting on the client side.`
|
|
31
|
+
);
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
async function serializeImage(image) {
|
|
35
|
+
var _a;
|
|
36
|
+
if (typeof image.image === "string") {
|
|
37
|
+
if (image.image.startsWith("data:")) {
|
|
38
|
+
const [header, base64Data] = image.image.split(",", 2);
|
|
39
|
+
const headerParts = header.split(";");
|
|
40
|
+
const mimeParts = (_a = headerParts[0]) == null ? void 0 : _a.split(":");
|
|
41
|
+
const headerMime = mimeParts == null ? void 0 : mimeParts[1];
|
|
42
|
+
if (!headerMime) {
|
|
43
|
+
throw new Error("Invalid data URL format");
|
|
44
|
+
}
|
|
45
|
+
let mimeType;
|
|
46
|
+
if (image.mimeType && image.mimeType !== headerMime) {
|
|
47
|
+
console.warn(
|
|
48
|
+
`Provided mimeType '${image.mimeType}' does not match data URL mime type '${headerMime}'. Using provided mimeType.`
|
|
49
|
+
);
|
|
50
|
+
mimeType = image.mimeType;
|
|
51
|
+
} else {
|
|
52
|
+
mimeType = headerMime;
|
|
53
|
+
}
|
|
54
|
+
const supportedTypes = /* @__PURE__ */ new Set(["image/jpeg", "image/png", "image/webp", "image/gif"]);
|
|
55
|
+
if (!supportedTypes.has(mimeType)) {
|
|
56
|
+
throw new Error(`Unsupported mimeType ${mimeType}. Must be jpeg, png, webp, or gif`);
|
|
57
|
+
}
|
|
58
|
+
return {
|
|
59
|
+
base64Data,
|
|
60
|
+
mimeType,
|
|
61
|
+
inferenceDetail: image.inferenceDetail
|
|
62
|
+
};
|
|
63
|
+
}
|
|
64
|
+
return {
|
|
65
|
+
mimeType: image.mimeType,
|
|
66
|
+
inferenceDetail: image.inferenceDetail,
|
|
67
|
+
externalUrl: image.image
|
|
68
|
+
};
|
|
69
|
+
} else if (image.image instanceof VideoFrame) {
|
|
70
|
+
const frame = ensureRGBCompatible(image.image);
|
|
71
|
+
const channels = getChannelsFromVideoBufferType(frame.type);
|
|
72
|
+
let encoded = sharp(frame.data, {
|
|
73
|
+
raw: {
|
|
74
|
+
width: frame.width,
|
|
75
|
+
height: frame.height,
|
|
76
|
+
channels
|
|
77
|
+
}
|
|
78
|
+
});
|
|
79
|
+
if (image.inferenceWidth && image.inferenceHeight) {
|
|
80
|
+
encoded = encoded.resize(image.inferenceWidth, image.inferenceHeight);
|
|
81
|
+
}
|
|
82
|
+
const base64Data = await encoded.png().toBuffer().then((buffer) => buffer.toString("base64"));
|
|
83
|
+
return {
|
|
84
|
+
base64Data,
|
|
85
|
+
mimeType: "image/png",
|
|
86
|
+
inferenceDetail: image.inferenceDetail
|
|
87
|
+
};
|
|
88
|
+
} else {
|
|
89
|
+
throw new Error("Unsupported image type");
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
const createToolOptions = (toolCallId, userData = {}) => {
|
|
93
|
+
return { ctx: { userData }, toolCallId };
|
|
94
|
+
};
|
|
95
|
+
const oaiParams = (p) => {
|
|
96
|
+
const { properties, required, additionalProperties } = zodToJsonSchema(p, {
|
|
97
|
+
target: "openAi"
|
|
98
|
+
});
|
|
99
|
+
return {
|
|
100
|
+
type: "object",
|
|
101
|
+
properties,
|
|
102
|
+
required,
|
|
103
|
+
additionalProperties
|
|
104
|
+
};
|
|
105
|
+
};
|
|
106
|
+
const oaiBuildFunctionInfo = (toolCtx, toolCallId, toolName, rawArgs) => {
|
|
107
|
+
const tool = toolCtx[toolName];
|
|
108
|
+
if (!tool) {
|
|
109
|
+
throw new Error(`AI tool ${toolName} not found`);
|
|
110
|
+
}
|
|
111
|
+
return FunctionCall.create({
|
|
112
|
+
callId: toolCallId,
|
|
113
|
+
name: toolName,
|
|
114
|
+
args: rawArgs
|
|
115
|
+
});
|
|
116
|
+
};
|
|
117
|
+
async function executeToolCall(toolCall, toolCtx) {
|
|
118
|
+
const tool = toolCtx[toolCall.name];
|
|
119
|
+
let args;
|
|
120
|
+
let params;
|
|
121
|
+
try {
|
|
122
|
+
args = JSON.parse(toolCall.args);
|
|
123
|
+
} catch (error) {
|
|
124
|
+
return FunctionCallOutput.create({
|
|
125
|
+
callId: toolCall.callId,
|
|
126
|
+
output: `Invalid JSON: ${error}`,
|
|
127
|
+
isError: true
|
|
128
|
+
});
|
|
129
|
+
}
|
|
130
|
+
try {
|
|
131
|
+
if (tool.parameters instanceof ZodObject) {
|
|
132
|
+
params = tool.parameters.parse(args);
|
|
133
|
+
} else {
|
|
134
|
+
params = args;
|
|
135
|
+
}
|
|
136
|
+
} catch (error) {
|
|
137
|
+
return FunctionCallOutput.create({
|
|
138
|
+
callId: toolCall.callId,
|
|
139
|
+
output: `Arguments parsing failed: ${error}`,
|
|
140
|
+
isError: true
|
|
141
|
+
});
|
|
142
|
+
}
|
|
143
|
+
try {
|
|
144
|
+
const result = await tool.execute(params, createToolOptions(toolCall.callId));
|
|
145
|
+
return FunctionCallOutput.create({
|
|
146
|
+
callId: toolCall.callId,
|
|
147
|
+
output: JSON.stringify(result),
|
|
148
|
+
isError: false
|
|
149
|
+
});
|
|
150
|
+
} catch (error) {
|
|
151
|
+
return FunctionCallOutput.create({
|
|
152
|
+
callId: toolCall.callId,
|
|
153
|
+
output: `Tool execution failed: ${error}`,
|
|
154
|
+
isError: true
|
|
155
|
+
});
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
function computeLCS(oldIds, newIds) {
|
|
159
|
+
const n = oldIds.length;
|
|
160
|
+
const m = newIds.length;
|
|
161
|
+
const dp = Array(n + 1).fill(null).map(() => Array(m + 1).fill(0));
|
|
162
|
+
for (let i2 = 1; i2 <= n; i2++) {
|
|
163
|
+
for (let j2 = 1; j2 <= m; j2++) {
|
|
164
|
+
if (oldIds[i2 - 1] === newIds[j2 - 1]) {
|
|
165
|
+
dp[i2][j2] = dp[i2 - 1][j2 - 1] + 1;
|
|
166
|
+
} else {
|
|
167
|
+
dp[i2][j2] = Math.max(dp[i2 - 1][j2], dp[i2][j2 - 1]);
|
|
168
|
+
}
|
|
169
|
+
}
|
|
170
|
+
}
|
|
171
|
+
const lcsIds = [];
|
|
172
|
+
let i = n;
|
|
173
|
+
let j = m;
|
|
174
|
+
while (i > 0 && j > 0) {
|
|
175
|
+
if (oldIds[i - 1] === newIds[j - 1]) {
|
|
176
|
+
lcsIds.push(oldIds[i - 1]);
|
|
177
|
+
i--;
|
|
178
|
+
j--;
|
|
179
|
+
} else if (dp[i - 1][j] > dp[i][j - 1]) {
|
|
180
|
+
i--;
|
|
181
|
+
} else {
|
|
182
|
+
j--;
|
|
183
|
+
}
|
|
184
|
+
}
|
|
185
|
+
return lcsIds.reverse();
|
|
186
|
+
}
|
|
187
|
+
function computeChatCtxDiff(oldCtx, newCtx) {
|
|
188
|
+
const oldIds = oldCtx.items.map((item) => item.id);
|
|
189
|
+
const newIds = newCtx.items.map((item) => item.id);
|
|
190
|
+
const lcsIds = new Set(computeLCS(oldIds, newIds));
|
|
191
|
+
const toRemove = oldCtx.items.filter((msg) => !lcsIds.has(msg.id)).map((msg) => msg.id);
|
|
192
|
+
const toCreate = [];
|
|
193
|
+
let lastIdInSequence = null;
|
|
194
|
+
for (const newItem of newCtx.items) {
|
|
195
|
+
if (lcsIds.has(newItem.id)) {
|
|
196
|
+
lastIdInSequence = newItem.id;
|
|
197
|
+
} else {
|
|
198
|
+
const prevId = lastIdInSequence;
|
|
199
|
+
toCreate.push([prevId, newItem.id]);
|
|
200
|
+
lastIdInSequence = newItem.id;
|
|
201
|
+
}
|
|
202
|
+
}
|
|
203
|
+
return {
|
|
204
|
+
toRemove,
|
|
205
|
+
toCreate
|
|
206
|
+
};
|
|
207
|
+
}
|
|
208
|
+
function toJsonSchema(schema) {
|
|
209
|
+
if (schema instanceof ZodObject) {
|
|
210
|
+
return oaiParams(schema);
|
|
211
|
+
}
|
|
212
|
+
return schema;
|
|
213
|
+
}
|
|
214
|
+
export {
|
|
215
|
+
computeChatCtxDiff,
|
|
216
|
+
createToolOptions,
|
|
217
|
+
executeToolCall,
|
|
218
|
+
oaiBuildFunctionInfo,
|
|
219
|
+
oaiParams,
|
|
220
|
+
serializeImage,
|
|
221
|
+
toJsonSchema
|
|
222
|
+
};
|
|
223
|
+
//# sourceMappingURL=utils.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"sources":["../../src/llm/utils.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2025 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport { VideoBufferType, VideoFrame } from '@livekit/rtc-node';\nimport type { JSONSchema7 } from 'json-schema';\nimport sharp from 'sharp';\nimport { ZodObject } from 'zod';\nimport { zodToJsonSchema } from 'zod-to-json-schema';\nimport type { UnknownUserData } from '../voice/run_context.js';\nimport type { ChatContext } from './chat_context.js';\nimport {\n type ChatItem,\n FunctionCall,\n FunctionCallOutput,\n type ImageContent,\n} from './chat_context.js';\nimport type { ToolContext, ToolInputSchema, ToolOptions } from './tool_context.js';\n\nexport interface SerializedImage {\n inferenceDetail: 'auto' | 'high' | 'low';\n mimeType?: string;\n base64Data?: string;\n externalUrl?: string;\n}\n\nfunction getChannelsFromVideoBufferType(type: VideoBufferType): 3 | 4 {\n switch (type) {\n case VideoBufferType.RGBA:\n case VideoBufferType.ABGR:\n case VideoBufferType.ARGB:\n case VideoBufferType.BGRA:\n return 4;\n case VideoBufferType.RGB24:\n return 3;\n default:\n // YUV formats (I420, I420A, I422, I444, I010, NV12) need conversion\n throw new Error(`Unsupported VideoBufferType: ${type}. Only RGB/RGBA formats are supported.`);\n }\n}\n\nfunction ensureRGBCompatible(frame: VideoFrame): VideoFrame {\n // If the frame is already in an RGB/RGBA-compatible format, return it directly\n if (\n frame.type === VideoBufferType.RGBA ||\n frame.type === VideoBufferType.BGRA ||\n frame.type === VideoBufferType.ARGB ||\n frame.type === VideoBufferType.ABGR ||\n frame.type === VideoBufferType.RGB24\n ) {\n return frame;\n }\n\n // Otherwise, attempt conversion for other formats (like YUV)\n try {\n return frame.convert(VideoBufferType.RGBA);\n } catch (error) {\n throw new Error(\n `Failed to convert format ${frame.type} to RGB: ${error}. ` +\n `Consider using RGB/RGBA formats or converting on the client side.`,\n );\n }\n}\n\nexport async function serializeImage(image: ImageContent): Promise<SerializedImage> {\n if (typeof image.image === 'string') {\n if (image.image.startsWith('data:')) {\n const [header, base64Data] = image.image.split(',', 2) as [string, string];\n const headerParts = header.split(';');\n const mimeParts = headerParts[0]?.split(':');\n const headerMime = mimeParts?.[1];\n\n if (!headerMime) {\n throw new Error('Invalid data URL format');\n }\n\n let mimeType: string;\n if (image.mimeType && image.mimeType !== headerMime) {\n console.warn(\n `Provided mimeType '${image.mimeType}' does not match data URL mime type '${headerMime}'. Using provided mimeType.`,\n );\n mimeType = image.mimeType;\n } else {\n mimeType = headerMime;\n }\n\n const supportedTypes = new Set(['image/jpeg', 'image/png', 'image/webp', 'image/gif']);\n if (!supportedTypes.has(mimeType)) {\n throw new Error(`Unsupported mimeType ${mimeType}. Must be jpeg, png, webp, or gif`);\n }\n\n return {\n base64Data,\n mimeType: mimeType,\n inferenceDetail: image.inferenceDetail,\n };\n }\n\n // External URL\n return {\n mimeType: image.mimeType,\n inferenceDetail: image.inferenceDetail,\n externalUrl: image.image,\n };\n } else if (image.image instanceof VideoFrame) {\n const frame = ensureRGBCompatible(image.image);\n const channels = getChannelsFromVideoBufferType(frame.type);\n\n // Sharp needs to know the format of raw pixel data\n let encoded = sharp(frame.data, {\n raw: {\n width: frame.width,\n height: frame.height,\n channels,\n },\n });\n\n if (image.inferenceWidth && image.inferenceHeight) {\n encoded = encoded.resize(image.inferenceWidth, image.inferenceHeight);\n }\n\n const base64Data = await encoded\n .png()\n .toBuffer()\n .then((buffer) => buffer.toString('base64'));\n\n return {\n base64Data,\n mimeType: 'image/png',\n inferenceDetail: image.inferenceDetail,\n };\n } else {\n throw new Error('Unsupported image type');\n }\n}\n\n/** Raw OpenAI-adherent function parameters. */\nexport type OpenAIFunctionParameters = {\n type: 'object';\n properties: { [id: string]: any }; // eslint-disable-line @typescript-eslint/no-explicit-any\n required: string[];\n additionalProperties?: boolean;\n};\n\n// TODO(brian): remove this helper once we have the real RunContext user data\nexport const createToolOptions = <UserData extends UnknownUserData>(\n toolCallId: string,\n userData: UserData = {} as UserData,\n): ToolOptions<UserData> => {\n return { ctx: { userData }, toolCallId } as unknown as ToolOptions<UserData>;\n};\n\n/** @internal */\n// eslint-disable-next-line @typescript-eslint/no-explicit-any\nexport const oaiParams = (p: ZodObject<any>): OpenAIFunctionParameters => {\n // TODO(AJS-162): make zod to JSON parsing able to handle z.optional(v.field())\n const { properties, required, additionalProperties } = zodToJsonSchema(p, {\n target: 'openAi',\n }) as OpenAIFunctionParameters;\n\n return {\n type: 'object',\n properties,\n required,\n additionalProperties,\n };\n};\n\n/** @internal */\nexport const oaiBuildFunctionInfo = (\n toolCtx: ToolContext,\n toolCallId: string,\n toolName: string,\n rawArgs: string,\n): FunctionCall => {\n const tool = toolCtx[toolName];\n if (!tool) {\n throw new Error(`AI tool ${toolName} not found`);\n }\n\n return FunctionCall.create({\n callId: toolCallId,\n name: toolName,\n args: rawArgs,\n });\n};\n\nexport async function executeToolCall(\n toolCall: FunctionCall,\n toolCtx: ToolContext,\n): Promise<FunctionCallOutput> {\n const tool = toolCtx[toolCall.name]!;\n let args: object | undefined;\n let params: object | undefined;\n\n // Ensure valid JSON\n try {\n args = JSON.parse(toolCall.args);\n } catch (error) {\n return FunctionCallOutput.create({\n callId: toolCall.callId,\n output: `Invalid JSON: ${error}`,\n isError: true,\n });\n }\n\n // Ensure valid arguments schema\n try {\n if (tool.parameters instanceof ZodObject) {\n params = tool.parameters.parse(args);\n } else {\n params = args;\n }\n } catch (error) {\n return FunctionCallOutput.create({\n callId: toolCall.callId,\n output: `Arguments parsing failed: ${error}`,\n isError: true,\n });\n }\n\n try {\n const result = await tool.execute(params, createToolOptions(toolCall.callId));\n return FunctionCallOutput.create({\n callId: toolCall.callId,\n output: JSON.stringify(result),\n isError: false,\n });\n } catch (error) {\n return FunctionCallOutput.create({\n callId: toolCall.callId,\n output: `Tool execution failed: ${error}`,\n isError: true,\n });\n }\n}\n\n/**\n * Standard dynamic-programming LCS to get the common subsequence\n * of IDs (in order) that appear in both old_ids and new_ids.\n *\n * @param oldIds - The old list of IDs.\n * @param newIds - The new list of IDs.\n * @returns The longest common subsequence of the two lists of IDs.\n */\nfunction computeLCS(oldIds: string[], newIds: string[]): string[] {\n const n = oldIds.length;\n const m = newIds.length;\n const dp: number[][] = Array(n + 1)\n .fill(null)\n .map(() => Array(m + 1).fill(0));\n\n // Fill DP table\n for (let i = 1; i <= n; i++) {\n for (let j = 1; j <= m; j++) {\n if (oldIds[i - 1] === newIds[j - 1]) {\n dp[i]![j] = dp[i - 1]![j - 1]! + 1;\n } else {\n dp[i]![j] = Math.max(dp[i - 1]![j]!, dp[i]![j - 1]!);\n }\n }\n }\n\n // Backtrack to find the actual LCS sequence\n const lcsIds: string[] = [];\n let i = n;\n let j = m;\n while (i > 0 && j > 0) {\n if (oldIds[i - 1] === newIds[j - 1]) {\n lcsIds.push(oldIds[i - 1]!);\n i--;\n j--;\n } else if (dp[i - 1]![j]! > dp[i]![j - 1]!) {\n i--;\n } else {\n j--;\n }\n }\n\n return lcsIds.reverse();\n}\n\ninterface DiffOps {\n toRemove: string[];\n toCreate: Array<[string | null, string]>; // (previous_item_id, id), if previous_item_id is null, add to the root\n}\n\n/**\n * Compute the minimal list of create/remove operations to transform oldCtx into newCtx.\n *\n * @param oldCtx - The old chat context.\n * @param newCtx - The new chat context.\n * @returns The minimal list of create/remove operations to transform oldCtx into newCtx.\n */\nexport function computeChatCtxDiff(oldCtx: ChatContext, newCtx: ChatContext): DiffOps {\n const oldIds = oldCtx.items.map((item: ChatItem) => item.id);\n const newIds = newCtx.items.map((item: ChatItem) => item.id);\n const lcsIds = new Set(computeLCS(oldIds, newIds));\n\n const toRemove = oldCtx.items.filter((msg) => !lcsIds.has(msg.id)).map((msg) => msg.id);\n const toCreate: Array<[string | null, string]> = [];\n\n let lastIdInSequence: string | null = null;\n for (const newItem of newCtx.items) {\n if (lcsIds.has(newItem.id)) {\n lastIdInSequence = newItem.id;\n } else {\n const prevId = lastIdInSequence; // null if root\n toCreate.push([prevId, newItem.id]);\n lastIdInSequence = newItem.id;\n }\n }\n\n return {\n toRemove,\n toCreate,\n };\n}\n\nexport function toJsonSchema(schema: ToolInputSchema<any>): JSONSchema7 {\n if (schema instanceof ZodObject) {\n return oaiParams(schema);\n }\n return schema;\n}\n"],"mappings":"AAGA,SAAS,iBAAiB,kBAAkB;AAE5C,OAAO,WAAW;AAClB,SAAS,iBAAiB;AAC1B,SAAS,uBAAuB;AAGhC;AAAA,EAEE;AAAA,EACA;AAAA,OAEK;AAUP,SAAS,+BAA+B,MAA8B;AACpE,UAAQ,MAAM;AAAA,IACZ,KAAK,gBAAgB;AAAA,IACrB,KAAK,gBAAgB;AAAA,IACrB,KAAK,gBAAgB;AAAA,IACrB,KAAK,gBAAgB;AACnB,aAAO;AAAA,IACT,KAAK,gBAAgB;AACnB,aAAO;AAAA,IACT;AAEE,YAAM,IAAI,MAAM,gCAAgC,IAAI,wCAAwC;AAAA,EAChG;AACF;AAEA,SAAS,oBAAoB,OAA+B;AAE1D,MACE,MAAM,SAAS,gBAAgB,QAC/B,MAAM,SAAS,gBAAgB,QAC/B,MAAM,SAAS,gBAAgB,QAC/B,MAAM,SAAS,gBAAgB,QAC/B,MAAM,SAAS,gBAAgB,OAC/B;AACA,WAAO;AAAA,EACT;AAGA,MAAI;AACF,WAAO,MAAM,QAAQ,gBAAgB,IAAI;AAAA,EAC3C,SAAS,OAAO;AACd,UAAM,IAAI;AAAA,MACR,4BAA4B,MAAM,IAAI,YAAY,KAAK;AAAA,IAEzD;AAAA,EACF;AACF;AAEA,eAAsB,eAAe,OAA+C;AA/DpF;AAgEE,MAAI,OAAO,MAAM,UAAU,UAAU;AACnC,QAAI,MAAM,MAAM,WAAW,OAAO,GAAG;AACnC,YAAM,CAAC,QAAQ,UAAU,IAAI,MAAM,MAAM,MAAM,KAAK,CAAC;AACrD,YAAM,cAAc,OAAO,MAAM,GAAG;AACpC,YAAM,aAAY,iBAAY,CAAC,MAAb,mBAAgB,MAAM;AACxC,YAAM,aAAa,uCAAY;AAE/B,UAAI,CAAC,YAAY;AACf,cAAM,IAAI,MAAM,yBAAyB;AAAA,MAC3C;AAEA,UAAI;AACJ,UAAI,MAAM,YAAY,MAAM,aAAa,YAAY;AACnD,gBAAQ;AAAA,UACN,sBAAsB,MAAM,QAAQ,wCAAwC,UAAU;AAAA,QACxF;AACA,mBAAW,MAAM;AAAA,MACnB,OAAO;AACL,mBAAW;AAAA,MACb;AAEA,YAAM,iBAAiB,oBAAI,IAAI,CAAC,cAAc,aAAa,cAAc,WAAW,CAAC;AACrF,UAAI,CAAC,eAAe,IAAI,QAAQ,GAAG;AACjC,cAAM,IAAI,MAAM,wBAAwB,QAAQ,mCAAmC;AAAA,MACrF;AAEA,aAAO;AAAA,QACL;AAAA,QACA;AAAA,QACA,iBAAiB,MAAM;AAAA,MACzB;AAAA,IACF;AAGA,WAAO;AAAA,MACL,UAAU,MAAM;AAAA,MAChB,iBAAiB,MAAM;AAAA,MACvB,aAAa,MAAM;AAAA,IACrB;AAAA,EACF,WAAW,MAAM,iBAAiB,YAAY;AAC5C,UAAM,QAAQ,oBAAoB,MAAM,KAAK;AAC7C,UAAM,WAAW,+BAA+B,MAAM,IAAI;AAG1D,QAAI,UAAU,MAAM,MAAM,MAAM;AAAA,MAC9B,KAAK;AAAA,QACH,OAAO,MAAM;AAAA,QACb,QAAQ,MAAM;AAAA,QACd;AAAA,MACF;AAAA,IACF,CAAC;AAED,QAAI,MAAM,kBAAkB,MAAM,iBAAiB;AACjD,gBAAU,QAAQ,OAAO,MAAM,gBAAgB,MAAM,eAAe;AAAA,IACtE;AAEA,UAAM,aAAa,MAAM,QACtB,IAAI,EACJ,SAAS,EACT,KAAK,CAAC,WAAW,OAAO,SAAS,QAAQ,CAAC;AAE7C,WAAO;AAAA,MACL;AAAA,MACA,UAAU;AAAA,MACV,iBAAiB,MAAM;AAAA,IACzB;AAAA,EACF,OAAO;AACL,UAAM,IAAI,MAAM,wBAAwB;AAAA,EAC1C;AACF;AAWO,MAAM,oBAAoB,CAC/B,YACA,WAAqB,CAAC,MACI;AAC1B,SAAO,EAAE,KAAK,EAAE,SAAS,GAAG,WAAW;AACzC;AAIO,MAAM,YAAY,CAAC,MAAgD;AAExE,QAAM,EAAE,YAAY,UAAU,qBAAqB,IAAI,gBAAgB,GAAG;AAAA,IACxE,QAAQ;AAAA,EACV,CAAC;AAED,SAAO;AAAA,IACL,MAAM;AAAA,IACN;AAAA,IACA;AAAA,IACA;AAAA,EACF;AACF;AAGO,MAAM,uBAAuB,CAClC,SACA,YACA,UACA,YACiB;AACjB,QAAM,OAAO,QAAQ,QAAQ;AAC7B,MAAI,CAAC,MAAM;AACT,UAAM,IAAI,MAAM,WAAW,QAAQ,YAAY;AAAA,EACjD;AAEA,SAAO,aAAa,OAAO;AAAA,IACzB,QAAQ;AAAA,IACR,MAAM;AAAA,IACN,MAAM;AAAA,EACR,CAAC;AACH;AAEA,eAAsB,gBACpB,UACA,SAC6B;AAC7B,QAAM,OAAO,QAAQ,SAAS,IAAI;AAClC,MAAI;AACJ,MAAI;AAGJ,MAAI;AACF,WAAO,KAAK,MAAM,SAAS,IAAI;AAAA,EACjC,SAAS,OAAO;AACd,WAAO,mBAAmB,OAAO;AAAA,MAC/B,QAAQ,SAAS;AAAA,MACjB,QAAQ,iBAAiB,KAAK;AAAA,MAC9B,SAAS;AAAA,IACX,CAAC;AAAA,EACH;AAGA,MAAI;AACF,QAAI,KAAK,sBAAsB,WAAW;AACxC,eAAS,KAAK,WAAW,MAAM,IAAI;AAAA,IACrC,OAAO;AACL,eAAS;AAAA,IACX;AAAA,EACF,SAAS,OAAO;AACd,WAAO,mBAAmB,OAAO;AAAA,MAC/B,QAAQ,SAAS;AAAA,MACjB,QAAQ,6BAA6B,KAAK;AAAA,MAC1C,SAAS;AAAA,IACX,CAAC;AAAA,EACH;AAEA,MAAI;AACF,UAAM,SAAS,MAAM,KAAK,QAAQ,QAAQ,kBAAkB,SAAS,MAAM,CAAC;AAC5E,WAAO,mBAAmB,OAAO;AAAA,MAC/B,QAAQ,SAAS;AAAA,MACjB,QAAQ,KAAK,UAAU,MAAM;AAAA,MAC7B,SAAS;AAAA,IACX,CAAC;AAAA,EACH,SAAS,OAAO;AACd,WAAO,mBAAmB,OAAO;AAAA,MAC/B,QAAQ,SAAS;AAAA,MACjB,QAAQ,0BAA0B,KAAK;AAAA,MACvC,SAAS;AAAA,IACX,CAAC;AAAA,EACH;AACF;AAUA,SAAS,WAAW,QAAkB,QAA4B;AAChE,QAAM,IAAI,OAAO;AACjB,QAAM,IAAI,OAAO;AACjB,QAAM,KAAiB,MAAM,IAAI,CAAC,EAC/B,KAAK,IAAI,EACT,IAAI,MAAM,MAAM,IAAI,CAAC,EAAE,KAAK,CAAC,CAAC;AAGjC,WAASA,KAAI,GAAGA,MAAK,GAAGA,MAAK;AAC3B,aAASC,KAAI,GAAGA,MAAK,GAAGA,MAAK;AAC3B,UAAI,OAAOD,KAAI,CAAC,MAAM,OAAOC,KAAI,CAAC,GAAG;AACnC,WAAGD,EAAC,EAAGC,EAAC,IAAI,GAAGD,KAAI,CAAC,EAAGC,KAAI,CAAC,IAAK;AAAA,MACnC,OAAO;AACL,WAAGD,EAAC,EAAGC,EAAC,IAAI,KAAK,IAAI,GAAGD,KAAI,CAAC,EAAGC,EAAC,GAAI,GAAGD,EAAC,EAAGC,KAAI,CAAC,CAAE;AAAA,MACrD;AAAA,IACF;AAAA,EACF;AAGA,QAAM,SAAmB,CAAC;AAC1B,MAAI,IAAI;AACR,MAAI,IAAI;AACR,SAAO,IAAI,KAAK,IAAI,GAAG;AACrB,QAAI,OAAO,IAAI,CAAC,MAAM,OAAO,IAAI,CAAC,GAAG;AACnC,aAAO,KAAK,OAAO,IAAI,CAAC,CAAE;AAC1B;AACA;AAAA,IACF,WAAW,GAAG,IAAI,CAAC,EAAG,CAAC,IAAK,GAAG,CAAC,EAAG,IAAI,CAAC,GAAI;AAC1C;AAAA,IACF,OAAO;AACL;AAAA,IACF;AAAA,EACF;AAEA,SAAO,OAAO,QAAQ;AACxB;AAcO,SAAS,mBAAmB,QAAqB,QAA8B;AACpF,QAAM,SAAS,OAAO,MAAM,IAAI,CAAC,SAAmB,KAAK,EAAE;AAC3D,QAAM,SAAS,OAAO,MAAM,IAAI,CAAC,SAAmB,KAAK,EAAE;AAC3D,QAAM,SAAS,IAAI,IAAI,WAAW,QAAQ,MAAM,CAAC;AAEjD,QAAM,WAAW,OAAO,MAAM,OAAO,CAAC,QAAQ,CAAC,OAAO,IAAI,IAAI,EAAE,CAAC,EAAE,IAAI,CAAC,QAAQ,IAAI,EAAE;AACtF,QAAM,WAA2C,CAAC;AAElD,MAAI,mBAAkC;AACtC,aAAW,WAAW,OAAO,OAAO;AAClC,QAAI,OAAO,IAAI,QAAQ,EAAE,GAAG;AAC1B,yBAAmB,QAAQ;AAAA,IAC7B,OAAO;AACL,YAAM,SAAS;AACf,eAAS,KAAK,CAAC,QAAQ,QAAQ,EAAE,CAAC;AAClC,yBAAmB,QAAQ;AAAA,IAC7B;AAAA,EACF;AAEA,SAAO;AAAA,IACL;AAAA,IACA;AAAA,EACF;AACF;AAEO,SAAS,aAAa,QAA2C;AACtE,MAAI,kBAAkB,WAAW;AAC/B,WAAO,UAAU,MAAM;AAAA,EACzB;AACA,SAAO;AACT;","names":["i","j"]}
|