codicent-app-sdk 0.4.19 → 0.4.20
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"useRealtimeVoiceAI.d.ts","sourceRoot":"","sources":["../../../src/hooks/useRealtimeVoiceAI.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,kBAAkB,EAAE,MAAM,2CAA2C,CAAC;AAK/E,OAAO,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAE9C;;GAEG;AACH,MAAM,WAAW,QAAQ;IACvB,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,OAAO,CAAC,EAAE,GAAG,EAAE,CAAC;IAChB,SAAS,CAAC,EAAE;QACV,KAAK,CAAC,EAAE,UAAU,CAAC;QACnB,IAAI,CAAC,EAAE,MAAM,CAAC;QACd,UAAU,CAAC,EAAE,MAAM,CAAC;QACpB,IAAI,CAAC,EAAE,GAAG,CAAC;KACZ,CAAC;CACH;AAED;;GAEG;AACH,UAAU,aAAa;IACrB,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,QAAQ,GAAG,QAAQ,CAAC;IAC5B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,KAAK,EAAE;QAAE,CAAC,GAAG,EAAE,MAAM,GAAG,GAAG,CAAA;KAAE,CAAC;CAC/B;AAED,MAAM,WAAW,aAAa;IAC5B,KAAK,EAAE,QAAQ,EAAE,CAAC;IAClB,cAAc,EAAE,aAAa,EAAE,CAAC;IAChC,WAAW,EAAE,OAAO,CAAC;IACrB,cAAc,EAAE,OAAO,CAAC;IACxB,aAAa,EAAE,OAAO,CAAC;IACvB,WAAW,EAAE,OAAO,CAAC;IACrB,eAAe,EAAE,KAAK,CAAC,SAAS,CAAC,iBAAiB,CAAC,CAAC;IACpD,eAAe,EAAE,KAAK,CAAC,SAAS,CAAC,iBAAiB,CAAC,CAAC;IACpD,eAAe,EAAE,KAAK,CAAC,SAAS,CAAC,cAAc,CAAC,CAAC;IACjD,UAAU,EAAE,CAAC,SAAS,EAAE,MAAM,KAAK,MAAM,CAAC;IAC1C,mBAAmB,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IACzC,sBAAsB,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IAC5C,sBAAsB,EAAE,CAAC,EAAE,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IACtD,cAAc,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IACpC,aAAa,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IACnC,iBAAiB,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IACpD,gBAAgB,EAAE,MAAM,MAAM,CAAC;IAC/B,cAAc,EAAE,MAAM,MAAM,CAAC;IAC7B,WAAW,EAAE,CAAC,QAAQ,EAAE,MAAM,KAAK,IAAI,CAAC;IACxC,kBAAkB,EAAE,CAAC,YAAY,EAAE,MAAM,KAAK,IAAI,CAAC;IACnD,WAAW,EAAE,CAAC,QAAQ,EAAE,MAAM,KAAK,IAAI,CAAC;CACzC;AAED,QAAA,MAAM,kBAAkB,oBACL,eAAe,UACxB,MAAM,SACP;IAAE,UAAU,EAAE,kBAAkB,CAAC;IAAC,OAAO,EAAE,QAAQ,CAAA;CAAE,EAAE,UACtD,MAAM,aACH,MAAM,KAChB,aAAa,GAAG,
|
|
1
|
+
{"version":3,"file":"useRealtimeVoiceAI.d.ts","sourceRoot":"","sources":["../../../src/hooks/useRealtimeVoiceAI.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,kBAAkB,EAAE,MAAM,2CAA2C,CAAC;AAK/E,OAAO,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAE9C;;GAEG;AACH,MAAM,WAAW,QAAQ;IACvB,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,OAAO,CAAC,EAAE,GAAG,EAAE,CAAC;IAChB,SAAS,CAAC,EAAE;QACV,KAAK,CAAC,EAAE,UAAU,CAAC;QACnB,IAAI,CAAC,EAAE,MAAM,CAAC;QACd,UAAU,CAAC,EAAE,MAAM,CAAC;QACpB,IAAI,CAAC,EAAE,GAAG,CAAC;KACZ,CAAC;CACH;AAED;;GAEG;AACH,UAAU,aAAa;IACrB,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,QAAQ,GAAG,QAAQ,CAAC;IAC5B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,KAAK,EAAE;QAAE,CAAC,GAAG,EAAE,MAAM,GAAG,GAAG,CAAA;KAAE,CAAC;CAC/B;AAED,MAAM,WAAW,aAAa;IAC5B,KAAK,EAAE,QAAQ,EAAE,CAAC;IAClB,cAAc,EAAE,aAAa,EAAE,CAAC;IAChC,WAAW,EAAE,OAAO,CAAC;IACrB,cAAc,EAAE,OAAO,CAAC;IACxB,aAAa,EAAE,OAAO,CAAC;IACvB,WAAW,EAAE,OAAO,CAAC;IACrB,eAAe,EAAE,KAAK,CAAC,SAAS,CAAC,iBAAiB,CAAC,CAAC;IACpD,eAAe,EAAE,KAAK,CAAC,SAAS,CAAC,iBAAiB,CAAC,CAAC;IACpD,eAAe,EAAE,KAAK,CAAC,SAAS,CAAC,cAAc,CAAC,CAAC;IACjD,UAAU,EAAE,CAAC,SAAS,EAAE,MAAM,KAAK,MAAM,CAAC;IAC1C,mBAAmB,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IACzC,sBAAsB,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IAC5C,sBAAsB,EAAE,CAAC,EAAE,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IACtD,cAAc,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IACpC,aAAa,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IACnC,iBAAiB,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IACpD,gBAAgB,EAAE,MAAM,MAAM,CAAC;IAC/B,cAAc,EAAE,MAAM,MAAM,CAAC;IAC7B,WAAW,EAAE,CAAC,QAAQ,EAAE,MAAM,KAAK,IAAI,CAAC;IACxC,kBAAkB,EAAE,CAAC,YAAY,EAAE,MAAM,KAAK,IAAI,CAAC;IACnD,WAAW,EAAE,CAAC,QAAQ,EAAE,MAAM,KAAK,IAAI,CAAC;CACzC;AAED,QAAA,MAAM,kBAAkB,oBACL,eAAe,UACxB,MAAM,SACP;IAAE,UAAU,EAAE,kBAAkB,CAAC;IAAC,OAAO,EAAE,QAAQ,CAAA;CAAE,EAAE,UACtD,MAAM,aACH,MAAM,KAChB,aAAa,GAAG,SAgsBlB,CAAC;AAEF,eAAe,kBAAkB,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
"use strict";Object.defineProperty(exports,"__esModule",{value:!0});var e=require("react"),t=require("../utils/wav_renderer.js");require("../lib/wavtools/lib/wav_packer.js"),require("../lib/wavtools/lib/analysis/audio_analysis.js");var n=require("../lib/wavtools/lib/wav_stream_player.js"),r=require("../lib/wavtools/lib/wav_recorder.js"),a=require("../config/index.js");exports.default=(o,s,i,c,l)=>{const u=!!a.getConfigValue("APP_CONFIG"),d=!!a.getConfigValue("APP_BUTTONS");u||console.warn("APP_CONFIG is not set. Voice AI will not be available."),d||console.warn("APP_BUTTONS is not set. Voice AI will not be available.");const p=a.getConfigValue("APP_CONFIG"),f=a.getConfigValue("APP_BUTTONS");a.getConfigValue("API_BASE_URL").replace(/\/$/,""),a.getConfigValue("USE_REALTIME_SESSION_ENDPOINT"),a.getConfigValue("REALTIME_SESSION_ENDPOINT");const g=c||a.getConfigValue("REALTIME_VOICE_MODEL")||"alloy",m=l||a.getConfigValue("REALTIME_VOICE_PROVIDER")||"openai",y=["alloy","shimmer","echo","verse","nova","fable","onyx"],_=y.includes(g)?g:"alloy";g!==_&&console.warn(`[codicent-app-sdk] Voice "${g}" is not supported in the current SDK version. Supported voices: ${y.join(", ")}. Falling back to "${_}".`);const S=e.useRef(new r.WavRecorder({sampleRate:24e3})),w=e.useRef(new n.WavStreamPlayer({sampleRate:24e3})),v=e.useRef(null),h=e.useRef(null),
|
|
1
|
+
"use strict";Object.defineProperty(exports,"__esModule",{value:!0});var e=require("react"),t=require("../utils/wav_renderer.js");require("../lib/wavtools/lib/wav_packer.js"),require("../lib/wavtools/lib/analysis/audio_analysis.js");var n=require("../lib/wavtools/lib/wav_stream_player.js"),r=require("../lib/wavtools/lib/wav_recorder.js"),a=require("../config/index.js");exports.default=(o,s,i,c,l)=>{const u=!!a.getConfigValue("APP_CONFIG"),d=!!a.getConfigValue("APP_BUTTONS");u||console.warn("APP_CONFIG is not set. Voice AI will not be available."),d||console.warn("APP_BUTTONS is not set. Voice AI will not be available.");const p=a.getConfigValue("APP_CONFIG"),f=a.getConfigValue("APP_BUTTONS");a.getConfigValue("API_BASE_URL").replace(/\/$/,""),a.getConfigValue("USE_REALTIME_SESSION_ENDPOINT"),a.getConfigValue("REALTIME_SESSION_ENDPOINT");const g=c||a.getConfigValue("REALTIME_VOICE_MODEL")||"alloy",m=l||a.getConfigValue("REALTIME_VOICE_PROVIDER")||"openai",y=["alloy","shimmer","echo","verse","nova","fable","onyx"],_=y.includes(g)?g:"alloy";g!==_&&console.warn(`[codicent-app-sdk] Voice "${g}" is not supported in the current SDK version. Supported voices: ${y.join(", ")}. Falling back to "${_}".`);const S=e.useRef(new r.WavRecorder({sampleRate:24e3})),w=e.useRef(new n.WavStreamPlayer({sampleRate:24e3})),v=e.useRef(null),h=e.useRef(null),O=e.useRef(null),R=e.useRef(null),C=e.useRef(!1),I=e.useRef(null),T=e.useRef(null),E=e.useRef(null),N=e.useRef((new Date).toISOString()),[b,A]=e.useState([]),[P,V]=e.useState([]),[k,D]=e.useState(!1),[M,x]=e.useState(!1),[L,F]=e.useState(!1),[J,$]=e.useState(!1),j=e.useRef(0),q=e.useRef(0),[U,B]=e.useState(""),[W,G]=e.useState("en-US"),z=e.useRef(new Map),H=e.useRef(null),[K,Q]=e.useState((()=>u&&d&&f&&p&&p.apps&&p.apps[f]?p.apps[f].voiceInstructions||p.REALTIME_VOICE_INSTRUCTIONS||"":p&&p.REALTIME_VOICE_INSTRUCTIONS||"")),X=e.useCallback((e=>{const t=N.current,n=new Date(t).valueOf(),r=new Date(e).valueOf()-n,a=Math.floor(r/10)%100,o=Math.floor(r/1e3)%60,s=e=>{let t=e+"";for(;t.length<2;)t="0"+t;return t};return`${s(Math.floor(r/6e4)%60)}:${s(o)}.${s(a)}`}),[]),Y=e.useCallback((async()=>{try{N.current=(new Date).toISOString(),D(!0),V([]),A([]),z.current.clear();const e=await o.getRealtimeSessionToken(_,m);if(!e)throw new Error("No ephemeral key returned from session endpoint");const t=new RTCPeerConnection;v.current=t,O.current||(O.current=new Audio,O.current.autoplay=!0),t.ontrack=e=>{O.current&&e.streams[0]&&(O.current.srcObject=e.streams[0])};const n=await navigator.mediaDevices.getUserMedia({audio:!0});R.current=n;const r=n.getTracks()[0];t.addTrack(r,n);const s=t.createDataChannel("oai-events");h.current=s,s.addEventListener("message",(e=>{try{const t=JSON.parse(e.data);if("session.created"===t.type);else if("conversation.item.created"===t.type)A((e=>[...e,t.item]));else if("conversation.item.input_audio_transcription.completed"===t.type)A((e=>{const n=[...e],r=n.findIndex((e=>e.id===t.item_id));return-1!==r&&n[r].formatted&&(n[r].formatted.transcript=t.transcript),n}));else if("response.audio_transcript.delta"===t.type);else if("response.audio_transcript.done"===t.type);else if("response.output_item.added"===t.type){const e=t.item;"function_call"===e?.type&&(console.log("[Voice AI] Function call initiated:",e.name),z.current.set(e.id,{name:e.name||"",arguments:"",call_id:e.call_id||e.id}))}else if("response.function_call_arguments.delta"===t.type){const e=t.item_id,n=t.delta,r=z.current.get(e);r&&n&&(r.arguments+=n,z.current.set(e,r))}else if("response.function_call_arguments.done"===t.type){const e=t.item_id,n=z.current.get(e);if(n){console.log(`[Voice AI] Executing tool: ${n.name}`);const t=i.find((e=>e.definition.name===n.name));if(t){let e={};try{e=n.arguments?JSON.parse(n.arguments):{}}catch(t){console.error("[Voice AI] Failed to parse tool arguments:",t),e={}}Promise.resolve(t.handler(e)).then((e=>{console.log(`[Voice AI] Tool ${n.name} completed:`,e),s&&"open"===s.readyState&&(s.send(JSON.stringify({type:"conversation.item.create",item:{type:"function_call_output",call_id:n.call_id,output:JSON.stringify(e)}})),s.send(JSON.stringify({type:"response.create"})))})).catch((e=>{console.error(`[Voice AI] Tool ${n.name} failed:`,e),s&&"open"===s.readyState&&(s.send(JSON.stringify({type:"conversation.item.create",item:{type:"function_call_output",call_id:n.call_id,output:JSON.stringify({error:e.message||"Tool execution failed"})}})),s.send(JSON.stringify({type:"response.create"})))}))}else console.error(`[Voice AI] Tool not found: ${n.name}`);z.current.delete(e)}}else"error"===t.type&&console.error("Server error:",t.error)}catch(t){console.warn("Invalid message:",e.data)}})),s.onopen=()=>{console.log("Data channel opened, configuring session");const e=H.current||K;H.current=null,s.send(JSON.stringify({type:"session.update",session:{instructions:e.replace("{{name}}",U).replace("{{language}}",W).replace("{{time}}",(new Date).toISOString()),modalities:["text","audio"],input_audio_transcription:{model:"whisper-1"},turn_detection:{type:"server_vad",threshold:.5,prefix_padding_ms:300,silence_duration_ms:500},voice:_,temperature:.8,max_response_output_tokens:4096,input_audio_format:"pcm16",output_audio_format:"pcm16",tools:i.map((e=>({type:"function",...e.definition})))}})),x(!0)},s.onclose=()=>{console.log("Data channel closed"),x(!1)};const c=await t.createOffer();await t.setLocalDescription(c);let l="gpt-4o-realtime-preview",u=null;try{const e=a.getConfigValue("REALTIME_CONFIG_ENDPOINT"),t="azure"===m.toLowerCase();if(e||t){const e=await o.getRealtimeConfig(m);e&&e.model&&(l=e.model),e&&e.url&&(u=e.url)}}catch(e){console.warn("Failed to fetch realtime config, using default model:",e)}m.toLowerCase();const d=u??`https://api.openai.com/v1/realtime?model=${l}`,p={Authorization:`Bearer ${e}`,"Content-Type":"application/sdp"},f=await fetch(d,{method:"POST",body:c.sdp,headers:p});if(!f.ok)throw new Error(`Failed to get SDP answer: ${f.statusText}`);const g={type:"answer",sdp:await f.text()};await t.setRemoteDescription(g),F(!1);const y=S.current,C=w.current;await y.begin(),await C.connect()}catch(e){throw console.error("[codicent-app-sdk] Failed to establish WebRTC connection:",e),D(!1),e}}),[o,_,K,U,W,i]),Z=e.useCallback((async()=>{D(!1),x(!1),V([]),A([]),z.current.clear(),H.current=null,h.current&&(h.current.close(),h.current=null),v.current&&(v.current.close(),v.current=null),R.current&&(R.current.getTracks().forEach((e=>e.stop())),R.current=null),O.current&&(O.current.pause(),O.current.srcObject=null);const e=S.current;await e.end();const t=w.current;await t.interrupt()}),[]),ee=e.useCallback((async e=>{const t=h.current;t&&"open"===t.readyState&&t.send(JSON.stringify({type:"conversation.item.delete",item_id:e}))}),[]),te=e.useCallback((async()=>{$(!0);const e=h.current;e&&"open"===e.readyState&&e.send(JSON.stringify({type:"input_audio_buffer.commit"}))}),[]),ne=e.useCallback((async()=>{$(!1);const e=h.current;e&&"open"===e.readyState&&e.send(JSON.stringify({type:"response.create"}))}),[]),re=e.useCallback((async e=>{const t=h.current;t&&"open"===t.readyState&&t.send(JSON.stringify({type:"session.update",session:{turn_detection:"none"===e?null:{type:"server_vad"}}})),F("none"===e)}),[]);e.useEffect((()=>{if(U&&M){const e=h.current;e&&"open"===e.readyState&&e.send(JSON.stringify({type:"session.update",session:{instructions:K.replace("{{name}}",U).replace("{{language}}",W).replace("{{time}}",(new Date).toISOString())}}))}}),[K,U,W,M]),e.useEffect((()=>{let e=!0;const n=S.current,r=I.current;let a=null;const o=w.current,s=T.current;let i=null;const c=()=>{if(e){if(r&&(r.width&&r.height||(r.width=r.offsetWidth,r.height=r.offsetHeight),a=a||r.getContext("2d"),a)){a.clearRect(0,0,r.width,r.height);const e=n.recording?n.getFrequencies("voice"):{values:new Float32Array([0])},o=1-Math.max(...e.values);j.current=o,t.WavRenderer.drawCircularBars(r,a,e.values,"#0099ff",20,0,8)}if(s&&(s.width&&s.height||(s.width=s.offsetWidth,s.height=s.offsetHeight),i=i||s.getContext("2d"),i)){i.clearRect(0,0,s.width,s.height);const e=o.analyser?o.getFrequencies("voice"):{values:new Float32Array([0])},n=1-Math.max(...e.values);q.current=n,t.WavRenderer.drawCircularBars(s,i,e.values,"#009900",20,0,8)}window.requestAnimationFrame(c)}};return c(),()=>{e=!1}}),[]),e.useEffect((()=>{!C.current&&i&&K&&(C.current=!0)}),[i,K]);const ae=e.useCallback((e=>{Q(e);const t=h.current;if(t&&"open"===t.readyState){const n=e.replace("{{name}}",U).replace("{{language}}",W).replace("{{time}}",(new Date).toISOString());t.send(JSON.stringify({type:"session.update",session:{instructions:n}}))}else H.current=e}),[U,W]);return e.useMemo((()=>{if(u&&d)return{items:b,realtimeEvents:P,isConnected:k,isSessionReady:M,canPushToTalk:L,isRecording:J,clientCanvasRef:I,serverCanvasRef:T,eventsScrollRef:E,formatTime:X,connectConversation:Y,disconnectConversation:Z,deleteConversationItem:ee,startRecording:te,stopRecording:ne,changeTurnEndType:re,getRecorderLevel:()=>j.current,getStreamLevel:()=>q.current,setUsername:B,updateInstructions:ae,setLanguage:G}}),[u,d,b,P,k,M,L,J,I,T,E])};
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"useRealtimeVoiceAI.d.ts","sourceRoot":"","sources":["../../../src/hooks/useRealtimeVoiceAI.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,kBAAkB,EAAE,MAAM,2CAA2C,CAAC;AAK/E,OAAO,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAE9C;;GAEG;AACH,MAAM,WAAW,QAAQ;IACvB,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,OAAO,CAAC,EAAE,GAAG,EAAE,CAAC;IAChB,SAAS,CAAC,EAAE;QACV,KAAK,CAAC,EAAE,UAAU,CAAC;QACnB,IAAI,CAAC,EAAE,MAAM,CAAC;QACd,UAAU,CAAC,EAAE,MAAM,CAAC;QACpB,IAAI,CAAC,EAAE,GAAG,CAAC;KACZ,CAAC;CACH;AAED;;GAEG;AACH,UAAU,aAAa;IACrB,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,QAAQ,GAAG,QAAQ,CAAC;IAC5B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,KAAK,EAAE;QAAE,CAAC,GAAG,EAAE,MAAM,GAAG,GAAG,CAAA;KAAE,CAAC;CAC/B;AAED,MAAM,WAAW,aAAa;IAC5B,KAAK,EAAE,QAAQ,EAAE,CAAC;IAClB,cAAc,EAAE,aAAa,EAAE,CAAC;IAChC,WAAW,EAAE,OAAO,CAAC;IACrB,cAAc,EAAE,OAAO,CAAC;IACxB,aAAa,EAAE,OAAO,CAAC;IACvB,WAAW,EAAE,OAAO,CAAC;IACrB,eAAe,EAAE,KAAK,CAAC,SAAS,CAAC,iBAAiB,CAAC,CAAC;IACpD,eAAe,EAAE,KAAK,CAAC,SAAS,CAAC,iBAAiB,CAAC,CAAC;IACpD,eAAe,EAAE,KAAK,CAAC,SAAS,CAAC,cAAc,CAAC,CAAC;IACjD,UAAU,EAAE,CAAC,SAAS,EAAE,MAAM,KAAK,MAAM,CAAC;IAC1C,mBAAmB,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IACzC,sBAAsB,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IAC5C,sBAAsB,EAAE,CAAC,EAAE,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IACtD,cAAc,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IACpC,aAAa,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IACnC,iBAAiB,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IACpD,gBAAgB,EAAE,MAAM,MAAM,CAAC;IAC/B,cAAc,EAAE,MAAM,MAAM,CAAC;IAC7B,WAAW,EAAE,CAAC,QAAQ,EAAE,MAAM,KAAK,IAAI,CAAC;IACxC,kBAAkB,EAAE,CAAC,YAAY,EAAE,MAAM,KAAK,IAAI,CAAC;IACnD,WAAW,EAAE,CAAC,QAAQ,EAAE,MAAM,KAAK,IAAI,CAAC;CACzC;AAED,QAAA,MAAM,kBAAkB,oBACL,eAAe,UACxB,MAAM,SACP;IAAE,UAAU,EAAE,kBAAkB,CAAC;IAAC,OAAO,EAAE,QAAQ,CAAA;CAAE,EAAE,UACtD,MAAM,aACH,MAAM,KAChB,aAAa,GAAG,
|
|
1
|
+
{"version":3,"file":"useRealtimeVoiceAI.d.ts","sourceRoot":"","sources":["../../../src/hooks/useRealtimeVoiceAI.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,kBAAkB,EAAE,MAAM,2CAA2C,CAAC;AAK/E,OAAO,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAE9C;;GAEG;AACH,MAAM,WAAW,QAAQ;IACvB,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,OAAO,CAAC,EAAE,GAAG,EAAE,CAAC;IAChB,SAAS,CAAC,EAAE;QACV,KAAK,CAAC,EAAE,UAAU,CAAC;QACnB,IAAI,CAAC,EAAE,MAAM,CAAC;QACd,UAAU,CAAC,EAAE,MAAM,CAAC;QACpB,IAAI,CAAC,EAAE,GAAG,CAAC;KACZ,CAAC;CACH;AAED;;GAEG;AACH,UAAU,aAAa;IACrB,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,QAAQ,GAAG,QAAQ,CAAC;IAC5B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,KAAK,EAAE;QAAE,CAAC,GAAG,EAAE,MAAM,GAAG,GAAG,CAAA;KAAE,CAAC;CAC/B;AAED,MAAM,WAAW,aAAa;IAC5B,KAAK,EAAE,QAAQ,EAAE,CAAC;IAClB,cAAc,EAAE,aAAa,EAAE,CAAC;IAChC,WAAW,EAAE,OAAO,CAAC;IACrB,cAAc,EAAE,OAAO,CAAC;IACxB,aAAa,EAAE,OAAO,CAAC;IACvB,WAAW,EAAE,OAAO,CAAC;IACrB,eAAe,EAAE,KAAK,CAAC,SAAS,CAAC,iBAAiB,CAAC,CAAC;IACpD,eAAe,EAAE,KAAK,CAAC,SAAS,CAAC,iBAAiB,CAAC,CAAC;IACpD,eAAe,EAAE,KAAK,CAAC,SAAS,CAAC,cAAc,CAAC,CAAC;IACjD,UAAU,EAAE,CAAC,SAAS,EAAE,MAAM,KAAK,MAAM,CAAC;IAC1C,mBAAmB,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IACzC,sBAAsB,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IAC5C,sBAAsB,EAAE,CAAC,EAAE,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IACtD,cAAc,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IACpC,aAAa,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IACnC,iBAAiB,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IACpD,gBAAgB,EAAE,MAAM,MAAM,CAAC;IAC/B,cAAc,EAAE,MAAM,MAAM,CAAC;IAC7B,WAAW,EAAE,CAAC,QAAQ,EAAE,MAAM,KAAK,IAAI,CAAC;IACxC,kBAAkB,EAAE,CAAC,YAAY,EAAE,MAAM,KAAK,IAAI,CAAC;IACnD,WAAW,EAAE,CAAC,QAAQ,EAAE,MAAM,KAAK,IAAI,CAAC;CACzC;AAED,QAAA,MAAM,kBAAkB,oBACL,eAAe,UACxB,MAAM,SACP;IAAE,UAAU,EAAE,kBAAkB,CAAC;IAAC,OAAO,EAAE,QAAQ,CAAA;CAAE,EAAE,UACtD,MAAM,aACH,MAAM,KAChB,aAAa,GAAG,SAgsBlB,CAAC;AAEF,eAAe,kBAAkB,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
import{useRef as e,useState as t,useCallback as n,useEffect as r,useMemo as o}from"react";import{WavRenderer as a}from"../utils/wav_renderer.js";import"../lib/wavtools/lib/wav_packer.js";import"../lib/wavtools/lib/analysis/audio_analysis.js";import{WavStreamPlayer as s}from"../lib/wavtools/lib/wav_stream_player.js";import{WavRecorder as i}from"../lib/wavtools/lib/wav_recorder.js";import{getConfigValue as c}from"../config/index.js";const l=(l,u,d,p,m)=>{const f=!!c("APP_CONFIG"),g=!!c("APP_BUTTONS");f||console.warn("APP_CONFIG is not set. Voice AI will not be available."),g||console.warn("APP_BUTTONS is not set. Voice AI will not be available.");const y=c("APP_CONFIG"),_=c("APP_BUTTONS");c("API_BASE_URL").replace(/\/$/,""),c("USE_REALTIME_SESSION_ENDPOINT"),c("REALTIME_SESSION_ENDPOINT");const w=p||c("REALTIME_VOICE_MODEL")||"alloy",h=m||c("REALTIME_VOICE_PROVIDER")||"openai",S=["alloy","shimmer","echo","verse","nova","fable","onyx"],v=S.includes(w)?w:"alloy";w!==v&&console.warn(`[codicent-app-sdk] Voice "${w}" is not supported in the current SDK version. Supported voices: ${S.join(", ")}. Falling back to "${v}".`);const O=e(new i({sampleRate:24e3})),I=e(new s({sampleRate:24e3})),T=e(null),N=e(null),E=e(null),
|
|
1
|
+
import{useRef as e,useState as t,useCallback as n,useEffect as r,useMemo as o}from"react";import{WavRenderer as a}from"../utils/wav_renderer.js";import"../lib/wavtools/lib/wav_packer.js";import"../lib/wavtools/lib/analysis/audio_analysis.js";import{WavStreamPlayer as s}from"../lib/wavtools/lib/wav_stream_player.js";import{WavRecorder as i}from"../lib/wavtools/lib/wav_recorder.js";import{getConfigValue as c}from"../config/index.js";const l=(l,u,d,p,m)=>{const f=!!c("APP_CONFIG"),g=!!c("APP_BUTTONS");f||console.warn("APP_CONFIG is not set. Voice AI will not be available."),g||console.warn("APP_BUTTONS is not set. Voice AI will not be available.");const y=c("APP_CONFIG"),_=c("APP_BUTTONS");c("API_BASE_URL").replace(/\/$/,""),c("USE_REALTIME_SESSION_ENDPOINT"),c("REALTIME_SESSION_ENDPOINT");const w=p||c("REALTIME_VOICE_MODEL")||"alloy",h=m||c("REALTIME_VOICE_PROVIDER")||"openai",S=["alloy","shimmer","echo","verse","nova","fable","onyx"],v=S.includes(w)?w:"alloy";w!==v&&console.warn(`[codicent-app-sdk] Voice "${w}" is not supported in the current SDK version. Supported voices: ${S.join(", ")}. Falling back to "${v}".`);const O=e(new i({sampleRate:24e3})),I=e(new s({sampleRate:24e3})),T=e(null),N=e(null),E=e(null),R=e(null),A=e(!1),C=e(null),P=e(null),b=e(null),D=e((new Date).toISOString()),[L,x]=t([]),[F,J]=t([]),[M,V]=t(!1),[$,k]=t(!1),[U,j]=t(!1),[B,G]=t(!1),q=e(0),W=e(0),[z,H]=t(""),[K,Q]=t("en-US"),X=e(new Map),Y=e(null),[Z,ee]=t((()=>f&&g&&_&&y&&y.apps&&y.apps[_]?y.apps[_].voiceInstructions||y.REALTIME_VOICE_INSTRUCTIONS||"":y&&y.REALTIME_VOICE_INSTRUCTIONS||"")),te=n((e=>{const t=D.current,n=new Date(t).valueOf(),r=new Date(e).valueOf()-n,o=Math.floor(r/10)%100,a=Math.floor(r/1e3)%60,s=e=>{let t=e+"";for(;t.length<2;)t="0"+t;return t};return`${s(Math.floor(r/6e4)%60)}:${s(a)}.${s(o)}`}),[]),ne=n((async()=>{try{D.current=(new Date).toISOString(),V(!0),J([]),x([]),X.current.clear();const e=await l.getRealtimeSessionToken(v,h);if(!e)throw new Error("No ephemeral key returned from session endpoint");const t=new RTCPeerConnection;T.current=t,E.current||(E.current=new Audio,E.current.autoplay=!0),t.ontrack=e=>{E.current&&e.streams[0]&&(E.current.srcObject=e.streams[0])};const n=await navigator.mediaDevices.getUserMedia({audio:!0});R.current=n;const r=n.getTracks()[0];t.addTrack(r,n);const o=t.createDataChannel("oai-events");N.current=o,o.addEventListener("message",(e=>{try{const t=JSON.parse(e.data);if("session.created"===t.type);else if("conversation.item.created"===t.type)x((e=>[...e,t.item]));else if("conversation.item.input_audio_transcription.completed"===t.type)x((e=>{const n=[...e],r=n.findIndex((e=>e.id===t.item_id));return-1!==r&&n[r].formatted&&(n[r].formatted.transcript=t.transcript),n}));else if("response.audio_transcript.delta"===t.type);else if("response.audio_transcript.done"===t.type);else if("response.output_item.added"===t.type){const e=t.item;"function_call"===e?.type&&(console.log("[Voice AI] Function call initiated:",e.name),X.current.set(e.id,{name:e.name||"",arguments:"",call_id:e.call_id||e.id}))}else if("response.function_call_arguments.delta"===t.type){const e=t.item_id,n=t.delta,r=X.current.get(e);r&&n&&(r.arguments+=n,X.current.set(e,r))}else if("response.function_call_arguments.done"===t.type){const e=t.item_id,n=X.current.get(e);if(n){console.log(`[Voice AI] Executing tool: ${n.name}`);const t=d.find((e=>e.definition.name===n.name));if(t){let e={};try{e=n.arguments?JSON.parse(n.arguments):{}}catch(t){console.error("[Voice AI] Failed to parse tool arguments:",t),e={}}Promise.resolve(t.handler(e)).then((e=>{console.log(`[Voice AI] Tool ${n.name} completed:`,e),o&&"open"===o.readyState&&(o.send(JSON.stringify({type:"conversation.item.create",item:{type:"function_call_output",call_id:n.call_id,output:JSON.stringify(e)}})),o.send(JSON.stringify({type:"response.create"})))})).catch((e=>{console.error(`[Voice AI] Tool ${n.name} failed:`,e),o&&"open"===o.readyState&&(o.send(JSON.stringify({type:"conversation.item.create",item:{type:"function_call_output",call_id:n.call_id,output:JSON.stringify({error:e.message||"Tool execution failed"})}})),o.send(JSON.stringify({type:"response.create"})))}))}else console.error(`[Voice AI] Tool not found: ${n.name}`);X.current.delete(e)}}else"error"===t.type&&console.error("Server error:",t.error)}catch(t){console.warn("Invalid message:",e.data)}})),o.onopen=()=>{console.log("Data channel opened, configuring session");const e=Y.current||Z;Y.current=null,o.send(JSON.stringify({type:"session.update",session:{instructions:e.replace("{{name}}",z).replace("{{language}}",K).replace("{{time}}",(new Date).toISOString()),modalities:["text","audio"],input_audio_transcription:{model:"whisper-1"},turn_detection:{type:"server_vad",threshold:.5,prefix_padding_ms:300,silence_duration_ms:500},voice:v,temperature:.8,max_response_output_tokens:4096,input_audio_format:"pcm16",output_audio_format:"pcm16",tools:d.map((e=>({type:"function",...e.definition})))}})),k(!0)},o.onclose=()=>{console.log("Data channel closed"),k(!1)};const a=await t.createOffer();await t.setLocalDescription(a);let s="gpt-4o-realtime-preview",i=null;try{const e=c("REALTIME_CONFIG_ENDPOINT"),t="azure"===h.toLowerCase();if(e||t){const e=await l.getRealtimeConfig(h);e&&e.model&&(s=e.model),e&&e.url&&(i=e.url)}}catch(e){console.warn("Failed to fetch realtime config, using default model:",e)}h.toLowerCase();const u=i??`https://api.openai.com/v1/realtime?model=${s}`,p={Authorization:`Bearer ${e}`,"Content-Type":"application/sdp"},m=await fetch(u,{method:"POST",body:a.sdp,headers:p});if(!m.ok)throw new Error(`Failed to get SDP answer: ${m.statusText}`);const f={type:"answer",sdp:await m.text()};await t.setRemoteDescription(f),j(!1);const g=O.current,y=I.current;await g.begin(),await y.connect()}catch(e){throw console.error("[codicent-app-sdk] Failed to establish WebRTC connection:",e),V(!1),e}}),[l,v,Z,z,K,d]),re=n((async()=>{V(!1),k(!1),J([]),x([]),X.current.clear(),Y.current=null,N.current&&(N.current.close(),N.current=null),T.current&&(T.current.close(),T.current=null),R.current&&(R.current.getTracks().forEach((e=>e.stop())),R.current=null),E.current&&(E.current.pause(),E.current.srcObject=null);const e=O.current;await e.end();const t=I.current;await t.interrupt()}),[]),oe=n((async e=>{const t=N.current;t&&"open"===t.readyState&&t.send(JSON.stringify({type:"conversation.item.delete",item_id:e}))}),[]),ae=n((async()=>{G(!0);const e=N.current;e&&"open"===e.readyState&&e.send(JSON.stringify({type:"input_audio_buffer.commit"}))}),[]),se=n((async()=>{G(!1);const e=N.current;e&&"open"===e.readyState&&e.send(JSON.stringify({type:"response.create"}))}),[]),ie=n((async e=>{const t=N.current;t&&"open"===t.readyState&&t.send(JSON.stringify({type:"session.update",session:{turn_detection:"none"===e?null:{type:"server_vad"}}})),j("none"===e)}),[]);r((()=>{if(z&&$){const e=N.current;e&&"open"===e.readyState&&e.send(JSON.stringify({type:"session.update",session:{instructions:Z.replace("{{name}}",z).replace("{{language}}",K).replace("{{time}}",(new Date).toISOString())}}))}}),[Z,z,K,$]),r((()=>{let e=!0;const t=O.current,n=C.current;let r=null;const o=I.current,s=P.current;let i=null;const c=()=>{if(e){if(n&&(n.width&&n.height||(n.width=n.offsetWidth,n.height=n.offsetHeight),r=r||n.getContext("2d"),r)){r.clearRect(0,0,n.width,n.height);const e=t.recording?t.getFrequencies("voice"):{values:new Float32Array([0])},o=1-Math.max(...e.values);q.current=o,a.drawCircularBars(n,r,e.values,"#0099ff",20,0,8)}if(s&&(s.width&&s.height||(s.width=s.offsetWidth,s.height=s.offsetHeight),i=i||s.getContext("2d"),i)){i.clearRect(0,0,s.width,s.height);const e=o.analyser?o.getFrequencies("voice"):{values:new Float32Array([0])},t=1-Math.max(...e.values);W.current=t,a.drawCircularBars(s,i,e.values,"#009900",20,0,8)}window.requestAnimationFrame(c)}};return c(),()=>{e=!1}}),[]),r((()=>{!A.current&&d&&Z&&(A.current=!0)}),[d,Z]);const ce=n((e=>{ee(e);const t=N.current;if(t&&"open"===t.readyState){const n=e.replace("{{name}}",z).replace("{{language}}",K).replace("{{time}}",(new Date).toISOString());t.send(JSON.stringify({type:"session.update",session:{instructions:n}}))}else Y.current=e}),[z,K]);return o((()=>{if(f&&g)return{items:L,realtimeEvents:F,isConnected:M,isSessionReady:$,canPushToTalk:U,isRecording:B,clientCanvasRef:C,serverCanvasRef:P,eventsScrollRef:b,formatTime:te,connectConversation:ne,disconnectConversation:re,deleteConversationItem:oe,startRecording:ae,stopRecording:se,changeTurnEndType:ie,getRecorderLevel:()=>q.current,getStreamLevel:()=>W.current,setUsername:H,updateInstructions:ce,setLanguage:Q}}),[f,g,L,F,M,$,U,B,C,P,b])};export{l as default};
|