codicent-app-sdk 0.3.79 → 0.3.80

package/dist/cjs/hooks/useCodicentApp.d.ts

@@ -19,7 +19,7 @@ export interface CodicentAppState {
     setAnonymous: (value: boolean) => void;
     allStates: AppStateDefinition;
     isBusy: () => boolean;
-    voice: RealtimeVoice;
+    voice: RealtimeVoice | undefined;
     html: string;
     setHtml: (html: string) => void;
     script: string;

package/dist/cjs/hooks/useCodicentApp.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"useCodicentApp.d.ts","sourceRoot":"","sources":["../../../src/hooks/useCodicentApp.tsx"],"names":[],"mappings":"AACA,OAAO,EAAE,qBAAqB,EAAE,MAAM,oBAAoB,CAAC;AAC3D,OAAO,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAC9C,OAAO,EAAE,eAAe,EAAE,YAAY,EAAE,kBAAkB,EAAyB,MAAM,UAAU,CAAC;AACpG,OAAyB,EAAE,kBAAkB,EAAE,MAAM,oBAAoB,CAAC;AAC1E,OAAqB,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC5D,OAA2B,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AAIzE,MAAM,WAAW,gBAAgB;IAC/B,KAAK,EAAE,kBAAkB,CAAC;IAC1B,IAAI,EAAE,YAAY,CAAC;IACnB,YAAY,EAAE,eAAe,CAAC;IAC9B,OAAO,EAAE,eAAe,CAAC;IACzB,OAAO,EAAE,YAAY,CAAC;IACtB,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,MAAM,CAAC;IACnC,YAAY,EAAE,CAAC,KAAK,EAAE,OAAO,KAAK,IAAI,CAAC;IACvC,SAAS,EAAE,kBAAkB,CAAC;IAC9B,MAAM,EAAE,MAAM,OAAO,CAAC;IACtB,KAAK,EAAE,aAAa,CAAC;IACrB,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,CAAC;IAChC,MAAM,EAAE,MAAM,CAAC;IACf,gBAAgB,EAAE,MAAM,CAAC;CAC1B;AAED,eAAO,MAAM,cAAc,cAAe;IAAE,KAAK,EAAE,qBAAqB,CAAA;CAAE,KAoKnE,gBACN,CAAC"}
+{"version":3,"file":"useCodicentApp.d.ts","sourceRoot":"","sources":["../../../src/hooks/useCodicentApp.tsx"],"names":[],"mappings":"AACA,OAAO,EAAE,qBAAqB,EAAE,MAAM,oBAAoB,CAAC;AAC3D,OAAO,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAC9C,OAAO,EAAE,eAAe,EAAE,YAAY,EAAE,kBAAkB,EAAyB,MAAM,UAAU,CAAC;AACpG,OAAyB,EAAE,kBAAkB,EAAE,MAAM,oBAAoB,CAAC;AAC1E,OAAqB,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC5D,OAA2B,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AAIzE,MAAM,WAAW,gBAAgB;IAC/B,KAAK,EAAE,kBAAkB,CAAC;IAC1B,IAAI,EAAE,YAAY,CAAC;IACnB,YAAY,EAAE,eAAe,CAAC;IAC9B,OAAO,EAAE,eAAe,CAAC;IACzB,OAAO,EAAE,YAAY,CAAC;IACtB,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,MAAM,CAAC;IACnC,YAAY,EAAE,CAAC,KAAK,EAAE,OAAO,KAAK,IAAI,CAAC;IACvC,SAAS,EAAE,kBAAkB,CAAC;IAC9B,MAAM,EAAE,MAAM,OAAO,CAAC;IACtB,KAAK,EAAE,aAAa,GAAG,SAAS,CAAC;IACjC,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,CAAC;IAChC,MAAM,EAAE,MAAM,CAAC;IACf,gBAAgB,EAAE,MAAM,CAAC;CAC1B;AAED,eAAO,MAAM,cAAc,cAAe;IAAE,KAAK,EAAE,qBAAqB,CAAA;CAAE,KAqKnE,gBACN,CAAC"}

package/dist/cjs/hooks/useCodicentApp.js

@@ -1 +1 @@
-"use strict";var e=require("react");require("../utils/MessageContent.js"),require("../node_modules/tinycolor2/esm/tinycolor.js"),require("react/jsx-runtime");var t=require("../utils/appState.js");require("../_virtual/index.js"),require("../lib/wavtools/lib/wav_packer.js"),require("../lib/wavtools/lib/analysis/audio_analysis.js"),require("../lib/wavtools/lib/wav_stream_player.js"),require("../lib/wavtools/lib/wav_recorder.js");var r=require("./useAudioRecorder.js"),s=require("./useAuthState.js"),n=require("./useRealtimeVoiceAI.js"),a=require("./useTools.js"),c=require("../config/index.js");exports.useCodicentApp=({auth0:u})=>{const i=s.default(u),o=e.useRef(t.createAppStateMachine(i.codicentService,(()=>{}))),[l,d]=e.useState(),{service:m,context:p}=o.current,{nickname:f,name:k,error:v,errorType:g,selectedApp:h}=p,A=e.useCallback(((e,t)=>{const r=e.get("file").name;m.uploadFile(r,e).then((e=>{m.sendMessage(`#transcribe ${""!==t?(t.startsWith("#")?"":"#")+t+" ":""}#file:${e}`).then((()=>{})).catch(console.warn)}))}),[m]),C=r.useAudioRecorder(A),[b,S]=e.useState(),[q,w]=e.useState(),[j,y]=e.useState(!1),x=e.useMemo((()=>new Audio("./notification.mp3")),[]),T=e.useCallback((()=>y(!0)),[]),_=e.useMemo((()=>a.default(m,S,w,x,b,T)),[m,S,w,x,b,T]),I=e.useMemo((()=>c.getConfigValue("REALTIME_VOICE_API_KEY")||""),[]),M=n.default(I,_);e.useEffect((()=>{j&&(M?.disconnectConversation(),y(!1))}),[j,M]),e.useEffect((()=>{i.isAuthenticated&&i.user&&i.accessToken&&o.current.updateContext({isAuthenticated:i.isAuthenticated,user:{sub:i.user.sub,email:i.user.email},accessToken:i.accessToken})}),[i.isAuthenticated,i.user,i.accessToken]),e.useEffect((()=>o.current.updateContext({nickname:i.nickname})),[i.nickname]),e.useEffect((()=>{o.current.update();let e,t=o.current.getCurrentStateName();d(o.current.getCurrentStateName()),JSON.stringify(o.current.context);let r=!1;const s=n=>setInterval((()=>{const n=o.current.getCurrentStateName();if("hasAccess"===n&&!r)return console.log("🎯 Reached stable state - switching to slower polling (1000ms)"),r=!0,clearInterval(e),void(e=s(1e3));if("hasAccess"!==n&&r)return console.log("🚨 Left stable state - switching to fast polling (200ms)"),r=!1,clearInterval(e),void(e=s(200));const a=JSON.stringify(o.current.context);o.current.update();const c=JSON.stringify(o.current.context),u=o.current.getCurrentStateName();u===t&&a===c||(d(u),t=u)}),n);return e=s(200),()=>clearInterval(e)}),[]);const E=e.useCallback((e=>e.replace("{nickname}",f||"unknown").replace("{token}",encodeURIComponent(i.accessToken||"unknown")).replace("{app}",h||"unknown")),[f,i.accessToken,h]),N=e.useCallback((e=>{o.current.updateContext({isAnonymous:e})}),[]),R=e.useCallback((()=>o.current.isBusy()),[]);return e.useMemo((()=>({audio:C,auth:i,currentStateName:l,stateMachine:o.current,service:m,context:p,state:l,nickname:f,name:k,error:v,errorType:g,fixAppUrl:E,setAnonymous:N,allStates:o.current.allStates,isBusy:R,voice:M,html:b,setHtml:S,script:q})),[l,f,k,v,g,b,q,i.isAuthenticated,i.isLoading,C.isRecording,M?.isConnected])};
+"use strict";var e=require("react");require("../utils/MessageContent.js"),require("../node_modules/tinycolor2/esm/tinycolor.js"),require("react/jsx-runtime");var t=require("../utils/appState.js");require("../_virtual/index.js"),require("../lib/wavtools/lib/wav_packer.js"),require("../lib/wavtools/lib/analysis/audio_analysis.js"),require("../lib/wavtools/lib/wav_stream_player.js"),require("../lib/wavtools/lib/wav_recorder.js");var r=require("./useAudioRecorder.js"),s=require("./useAuthState.js"),n=require("./useRealtimeVoiceAI.js"),a=require("./useTools.js"),c=require("../config/index.js");exports.useCodicentApp=({auth0:u})=>{const i=s.default(u),o=e.useRef(t.createAppStateMachine(i.codicentService,(()=>{}))),[l,d]=e.useState(),{service:m,context:p}=o.current,{nickname:f,name:k,error:v,errorType:g,selectedApp:h}=p,A=e.useCallback(((e,t)=>{const r=e.get("file").name;m.uploadFile(r,e).then((e=>{m.sendMessage(`#transcribe ${""!==t?(t.startsWith("#")?"":"#")+t+" ":""}#file:${e}`).then((()=>{})).catch(console.warn)}))}),[m]),C=r.useAudioRecorder(A),[b,S]=e.useState(),[q,w]=e.useState(),[j,y]=e.useState(!1),x=e.useMemo((()=>new Audio("./notification.mp3")),[]),T=e.useCallback((()=>y(!0)),[]),_=a.default(m,S,w,x,b,T),I=e.useMemo((()=>c.getConfigValue("REALTIME_VOICE_API_KEY")||""),[]),E=n.default(I,_);e.useEffect((()=>{j&&(E?.disconnectConversation(),y(!1))}),[j,E]),e.useEffect((()=>{i.isAuthenticated&&i.user&&i.accessToken&&o.current.updateContext({isAuthenticated:i.isAuthenticated,user:{sub:i.user.sub,email:i.user.email},accessToken:i.accessToken})}),[i.isAuthenticated,i.user,i.accessToken]),e.useEffect((()=>o.current.updateContext({nickname:i.nickname})),[i.nickname]),e.useEffect((()=>{o.current.update();let e,t=o.current.getCurrentStateName();d(o.current.getCurrentStateName()),JSON.stringify(o.current.context);let r=!1;const s=n=>setInterval((()=>{const n=o.current.getCurrentStateName();if("hasAccess"===n&&!r)return console.log("🎯 Reached stable state - switching to slower polling (1000ms)"),r=!0,clearInterval(e),void(e=s(1e3));if("hasAccess"!==n&&r)return console.log("🚨 Left stable state - switching to fast polling (200ms)"),r=!1,clearInterval(e),void(e=s(200));const a=JSON.stringify(o.current.context);o.current.update();const c=JSON.stringify(o.current.context),u=o.current.getCurrentStateName();u===t&&a===c||(d(u),t=u)}),n);return e=s(200),()=>clearInterval(e)}),[]);const M=e.useCallback((e=>e.replace("{nickname}",f||"unknown").replace("{token}",encodeURIComponent(i.accessToken||"unknown")).replace("{app}",h||"unknown")),[f,i.accessToken,h]),N=e.useCallback((e=>{o.current.updateContext({isAnonymous:e})}),[]),R=e.useCallback((()=>o.current.isBusy()),[]);return e.useMemo((()=>({audio:C,auth:i,currentStateName:l,stateMachine:o.current,service:m,context:p,state:l,nickname:f,name:k,error:v,errorType:g,fixAppUrl:M,setAnonymous:N,allStates:o.current.allStates,isBusy:R,voice:E,html:b,setHtml:S,script:q})),[l,f,k,v,g,b,q,i.isAuthenticated,i.isLoading,C.isRecording,E?.isConnected,M,N,R])};

package/dist/cjs/hooks/useRealtimeVoiceAI.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"useRealtimeVoiceAI.d.ts","sourceRoot":"","sources":["../../../src/hooks/useRealtimeVoiceAI.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,QAAQ,EAAE,kBAAkB,EAAE,MAAM,2CAA2C,CAAC;AAMzF;;GAEG;AACH,UAAU,aAAa;IACrB,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,QAAQ,GAAG,QAAQ,CAAC;IAC5B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,KAAK,EAAE;QAAE,CAAC,GAAG,EAAE,MAAM,GAAG,GAAG,CAAA;KAAE,CAAC;CAC/B;AAED,MAAM,WAAW,aAAa;IAC5B,KAAK,EAAE,QAAQ,EAAE,CAAC;IAClB,cAAc,EAAE,aAAa,EAAE,CAAC;IAChC,WAAW,EAAE,OAAO,CAAC;IACrB,aAAa,EAAE,OAAO,CAAC;IACvB,WAAW,EAAE,OAAO,CAAC;IACrB,eAAe,EAAE,KAAK,CAAC,SAAS,CAAC,iBAAiB,CAAC,CAAC;IACpD,eAAe,EAAE,KAAK,CAAC,SAAS,CAAC,iBAAiB,CAAC,CAAC;IACpD,eAAe,EAAE,KAAK,CAAC,SAAS,CAAC,cAAc,CAAC,CAAC;IACjD,UAAU,EAAE,CAAC,SAAS,EAAE,MAAM,KAAK,MAAM,CAAC;IAC1C,mBAAmB,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IACzC,sBAAsB,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IAC5C,sBAAsB,EAAE,CAAC,EAAE,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IACtD,cAAc,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IACpC,aAAa,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IACnC,iBAAiB,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IACpD,gBAAgB,EAAE,MAAM,MAAM,CAAC;IAC/B,cAAc,EAAE,MAAM,MAAM,CAAC;IAC7B,WAAW,EAAE,CAAC,QAAQ,EAAE,MAAM,KAAK,IAAI,CAAC;IACxC,kBAAkB,EAAE,CAAC,YAAY,EAAE,MAAM,KAAK,IAAI,CAAC;IACnD,WAAW,EAAE,CAAC,QAAQ,EAAE,MAAM,KAAK,IAAI,CAAC;CACzC;AAED,QAAA,MAAM,kBAAkB,WAAY,MAAM,SAAS;IAAE,UAAU,EAAE,kBAAkB,CAAC;IAAC,OAAO,EAAE,QAAQ,CAAA;CAAE,EAAE,8BAqYzG,CAAC;AAEF,eAAe,kBAAkB,CAAC"}
+{"version":3,"file":"useRealtimeVoiceAI.d.ts","sourceRoot":"","sources":["../../../src/hooks/useRealtimeVoiceAI.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,QAAQ,EAAE,kBAAkB,EAAE,MAAM,2CAA2C,CAAC;AAMzF;;GAEG;AACH,UAAU,aAAa;IACrB,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,QAAQ,GAAG,QAAQ,CAAC;IAC5B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,KAAK,EAAE;QAAE,CAAC,GAAG,EAAE,MAAM,GAAG,GAAG,CAAA;KAAE,CAAC;CAC/B;AAED,MAAM,WAAW,aAAa;IAC5B,KAAK,EAAE,QAAQ,EAAE,CAAC;IAClB,cAAc,EAAE,aAAa,EAAE,CAAC;IAChC,WAAW,EAAE,OAAO,CAAC;IACrB,aAAa,EAAE,OAAO,CAAC;IACvB,WAAW,EAAE,OAAO,CAAC;IACrB,eAAe,EAAE,KAAK,CAAC,SAAS,CAAC,iBAAiB,CAAC,CAAC;IACpD,eAAe,EAAE,KAAK,CAAC,SAAS,CAAC,iBAAiB,CAAC,CAAC;IACpD,eAAe,EAAE,KAAK,CAAC,SAAS,CAAC,cAAc,CAAC,CAAC;IACjD,UAAU,EAAE,CAAC,SAAS,EAAE,MAAM,KAAK,MAAM,CAAC;IAC1C,mBAAmB,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IACzC,sBAAsB,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IAC5C,sBAAsB,EAAE,CAAC,EAAE,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IACtD,cAAc,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IACpC,aAAa,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IACnC,iBAAiB,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IACpD,gBAAgB,EAAE,MAAM,MAAM,CAAC;IAC/B,cAAc,EAAE,MAAM,MAAM,CAAC;IAC7B,WAAW,EAAE,CAAC,QAAQ,EAAE,MAAM,KAAK,IAAI,CAAC;IACxC,kBAAkB,EAAE,CAAC,YAAY,EAAE,MAAM,KAAK,IAAI,CAAC;IACnD,WAAW,EAAE,CAAC,QAAQ,EAAE,MAAM,KAAK,IAAI,CAAC;CACzC;AAED,QAAA,MAAM,kBAAkB,WAAY,MAAM,SAAS;IAAE,UAAU,EAAE,kBAAkB,CAAC;IAAC,OAAO,EAAE,QAAQ,CAAA;CAAE,EAAE,KAAG,aAAa,GAAG,SAgZ5H,CAAC;AAEF,eAAe,kBAAkB,CAAC"}

package/dist/cjs/hooks/useRealtimeVoiceAI.js

@@ -1 +1 @@
-"use strict";Object.defineProperty(exports,"__esModule",{value:!0});var e=require("../node_modules/@openai/realtime-api-beta/lib/client.js"),t=require("react"),n=require("../utils/wav_renderer.js");require("../lib/wavtools/lib/wav_packer.js"),require("../lib/wavtools/lib/analysis/audio_analysis.js");var r=require("../lib/wavtools/lib/wav_stream_player.js"),a=require("../lib/wavtools/lib/wav_recorder.js"),s=require("../config/index.js");exports.default=(o,i)=>{if(!s.getConfigValue("APP_CONFIG"))return void console.warn("APP_CONFIG is not set. Voice AI will not be available.");if(!s.getConfigValue("APP_BUTTONS"))return void console.warn("APP_BUTTONS is not set. Voice AI will not be available.");const c=s.getConfigValue("APP_CONFIG"),u=s.getConfigValue("APP_BUTTONS"),l=t.useRef(new a.WavRecorder({sampleRate:24e3})),d=t.useRef(new r.WavStreamPlayer({sampleRate:24e3})),f=t.useRef(new e.RealtimeClient({apiKey:o,dangerouslyAllowAPIKeyInBrowser:!0,url:"wss://codicent-ai-sweden.openai.azure.com/openai/realtime?api-version=2024-10-01-preview&deployment=gpt-4o-realtime-preview-2&api-key=9013365c13ee4f3dad0016fb0f0c450a"})),p=t.useRef(!1),v=t.useRef(null),w=t.useRef(null),g=t.useRef(null),m=t.useRef((new Date).toISOString()),[h,C]=t.useState([]),[I,R]=t.useState([]),[S,_]=t.useState(!1),[b,y]=t.useState(!1),[T,A]=t.useState(!1),O=t.useRef(0),P=t.useRef(0),[k,q]=t.useState(""),[E,M]=t.useState("en-US"),[N,x]=t.useState((()=>u&&c.apps[u]?c.apps[u].voiceInstructions||c.REALTIME_VOICE_INSTRUCTIONS||"":c.REALTIME_VOICE_INSTRUCTIONS||"")),V=t.useCallback((e=>{const t=m.current,n=new Date(t).valueOf(),r=new Date(e).valueOf()-n,a=Math.floor(r/10)%100,s=Math.floor(r/1e3)%60,o=e=>{let t=e+"";for(;t.length<2;)t="0"+t;return t};return`${o(Math.floor(r/6e4)%60)}:${o(s)}.${o(a)}`}),[]),j=t.useCallback((async()=>{const e=f.current,t=l.current,n=d.current;m.current=(new Date).toISOString(),_(!0),R([]),C(e.conversation.getItems()),W("server_vad"),await t.begin(),await n.connect(),await e.connect(),e.sendUserMessageContent([{type:"input_text",text:"Hello!"}]),"server_vad"===e.getTurnDetectionType()&&await t.record((t=>e.appendInputAudio(t.mono)))}),[]),F=t.useCallback((async()=>{_(!1),R([]),C([]);const e=f.current,t=l.current;await t.end(),e.disconnect();const n=d.current;await n.interrupt()}),[]),U=t.useCallback((async e=>{f.current.deleteItem(e)}),[]),B=t.useCallback((async()=>{A(!0);const e=f.current,t=l.current,n=d.current,r=await n.interrupt();if(r?.trackId){const{trackId:t,offset:n}=r;await e.cancelResponse(t,n)}await t.record((t=>e.appendInputAudio(t.mono)))}),[]),D=t.useCallback((async()=>{A(!1);const e=f.current,t=l.current;await t.pause(),e.createResponse()}),[]),W=t.useCallback((async e=>{const t=f.current,n=l.current;"none"===e&&"recording"===n.getStatus()&&await n.pause(),t.updateSession({turn_detection:"none"===e?null:{type:"server_vad"}}),"server_vad"===e&&t.isConnected()&&await n.record((e=>t.appendInputAudio(e.mono))),y("none"===e)}),[]);t.useEffect((()=>{if(k){f.current.updateSession({instructions:N.replace("{{name}}",k).replace("{{language}}",E).replace("{{time}}",(new Date).toISOString())})}}),[N,k]),t.useEffect((()=>{let e=!0;const t=l.current,r=v.current;let a=null;const s=d.current,o=w.current;let i=null;const c=()=>{if(e){if(r&&(r.width&&r.height||(r.width=r.offsetWidth,r.height=r.offsetHeight),a=a||r.getContext("2d"),a)){a.clearRect(0,0,r.width,r.height);const e=t.recording?t.getFrequencies("voice"):{values:new Float32Array([0])},s=1-Math.max(...e.values);O.current=s,n.WavRenderer.drawCircularBars(r,a,e.values,"#0099ff",20,0,8)}if(o&&(o.width&&o.height||(o.width=o.offsetWidth,o.height=o.offsetHeight),i=i||o.getContext("2d"),i)){i.clearRect(0,0,o.width,o.height);const e=s.analyser?s.getFrequencies("voice"):{values:new Float32Array([0])},t=1-Math.max(...e.values);P.current=t,n.WavRenderer.drawCircularBars(o,i,e.values,"#009900",20,0,8)}window.requestAnimationFrame(c)}};return c(),()=>{e=!1}}),[]),t.useEffect((()=>{i&&N&&(()=>{if(!p.current){p.current=!0;const e=d.current,t=f.current;t.updateSession({instructions:N}),t.updateSession({input_audio_transcription:{model:"whisper-1"}});for(const e of i)t.addTool(e.definition,e.handler);t.on("error",(e=>console.error(e))),t.on("conversation.interrupted",(async()=>{const n=await e.interrupt();if(n?.trackId){const{trackId:e,offset:r}=n;await t.cancelResponse(e,r)}})),t.on("conversation.updated",(async({item:n,delta:r})=>{const s=t.conversation.getItems();if(r?.audio&&e.add16BitPCM(r.audio,n.id),"completed"===n.status&&n.formatted.audio?.length){const e=await a.WavRecorder.decode(n.formatted.audio,24e3,24e3);n.formatted.file=e}C(s)})),C(t.conversation.getItems())}})()}),[i,N]);const L=t.useCallback((e=>{const t=f.current,n=e.replace("{{name}}",k).replace("{{language}}",E).replace("{{time}}",(new Date).toISOString());t.updateSession({instructions:n}),x(e),console.log(`Voice instructions updated: ${n.substring(0,50)}...`)}),[k,E]);return t.useMemo((()=>({items:h,realtimeEvents:I,isConnected:S,canPushToTalk:b,isRecording:T,clientCanvasRef:v,serverCanvasRef:w,eventsScrollRef:g,formatTime:V,connectConversation:j,disconnectConversation:F,deleteConversationItem:U,startRecording:B,stopRecording:D,changeTurnEndType:W,getRecorderLevel:()=>O.current,getStreamLevel:()=>P.current,setUsername:q,updateInstructions:L,setLanguage:M})),[h,I,S,b,T,v,w,g])};
+"use strict";Object.defineProperty(exports,"__esModule",{value:!0});var e=require("../node_modules/@openai/realtime-api-beta/lib/client.js"),t=require("react"),n=require("../utils/wav_renderer.js");require("../lib/wavtools/lib/wav_packer.js"),require("../lib/wavtools/lib/analysis/audio_analysis.js");var a=require("../lib/wavtools/lib/wav_stream_player.js"),r=require("../lib/wavtools/lib/wav_recorder.js"),s=require("../config/index.js");exports.default=(o,i)=>{const c=!!s.getConfigValue("APP_CONFIG"),u=!!s.getConfigValue("APP_BUTTONS");c||console.warn("APP_CONFIG is not set. Voice AI will not be available."),u||console.warn("APP_BUTTONS is not set. Voice AI will not be available.");const l=s.getConfigValue("APP_CONFIG"),d=s.getConfigValue("APP_BUTTONS"),f=t.useRef(new r.WavRecorder({sampleRate:24e3})),p=t.useRef(new a.WavStreamPlayer({sampleRate:24e3})),v=t.useRef(new e.RealtimeClient({apiKey:o,dangerouslyAllowAPIKeyInBrowser:!0,url:"wss://codicent-ai-sweden.openai.azure.com/openai/realtime?api-version=2024-10-01-preview&deployment=gpt-4o-realtime-preview-2&api-key=9013365c13ee4f3dad0016fb0f0c450a"})),w=t.useRef(!1),g=t.useRef(null),m=t.useRef(null),h=t.useRef(null),C=t.useRef((new Date).toISOString()),[I,R]=t.useState([]),[S,_]=t.useState([]),[b,y]=t.useState(!1),[T,A]=t.useState(!1),[O,P]=t.useState(!1),k=t.useRef(0),q=t.useRef(0),[E,M]=t.useState(""),[N,x]=t.useState("en-US"),[V,j]=t.useState((()=>c&&u&&d&&l&&l.apps&&l.apps[d]?l.apps[d].voiceInstructions||l.REALTIME_VOICE_INSTRUCTIONS||"":l&&l.REALTIME_VOICE_INSTRUCTIONS||"")),F=t.useCallback((e=>{const t=C.current,n=new Date(t).valueOf(),a=new Date(e).valueOf()-n,r=Math.floor(a/10)%100,s=Math.floor(a/1e3)%60,o=e=>{let t=e+"";for(;t.length<2;)t="0"+t;return t};return`${o(Math.floor(a/6e4)%60)}:${o(s)}.${o(r)}`}),[]),U=t.useCallback((async()=>{const e=v.current,t=f.current,n=p.current;C.current=(new Date).toISOString(),y(!0),_([]),R(e.conversation.getItems()),$("server_vad"),await t.begin(),await n.connect(),await e.connect(),e.sendUserMessageContent([{type:"input_text",text:"Hello!"}]),"server_vad"===e.getTurnDetectionType()&&await t.record((t=>e.appendInputAudio(t.mono)))}),[]),B=t.useCallback((async()=>{y(!1),_([]),R([]);const e=v.current,t=f.current;await t.end(),e.disconnect();const n=p.current;await n.interrupt()}),[]),D=t.useCallback((async e=>{v.current.deleteItem(e)}),[]),W=t.useCallback((async()=>{P(!0);const e=v.current,t=f.current,n=p.current,a=await n.interrupt();if(a?.trackId){const{trackId:t,offset:n}=a;await e.cancelResponse(t,n)}await t.record((t=>e.appendInputAudio(t.mono)))}),[]),L=t.useCallback((async()=>{P(!1);const e=v.current,t=f.current;await t.pause(),e.createResponse()}),[]),$=t.useCallback((async e=>{const t=v.current,n=f.current;"none"===e&&"recording"===n.getStatus()&&await n.pause(),t.updateSession({turn_detection:"none"===e?null:{type:"server_vad"}}),"server_vad"===e&&t.isConnected()&&await n.record((e=>t.appendInputAudio(e.mono))),A("none"===e)}),[]);t.useEffect((()=>{if(E){v.current.updateSession({instructions:V.replace("{{name}}",E).replace("{{language}}",N).replace("{{time}}",(new Date).toISOString())})}}),[V,E]),t.useEffect((()=>{let e=!0;const t=f.current,a=g.current;let r=null;const s=p.current,o=m.current;let i=null;const c=()=>{if(e){if(a&&(a.width&&a.height||(a.width=a.offsetWidth,a.height=a.offsetHeight),r=r||a.getContext("2d"),r)){r.clearRect(0,0,a.width,a.height);const e=t.recording?t.getFrequencies("voice"):{values:new Float32Array([0])},s=1-Math.max(...e.values);k.current=s,n.WavRenderer.drawCircularBars(a,r,e.values,"#0099ff",20,0,8)}if(o&&(o.width&&o.height||(o.width=o.offsetWidth,o.height=o.offsetHeight),i=i||o.getContext("2d"),i)){i.clearRect(0,0,o.width,o.height);const e=s.analyser?s.getFrequencies("voice"):{values:new Float32Array([0])},t=1-Math.max(...e.values);q.current=t,n.WavRenderer.drawCircularBars(o,i,e.values,"#009900",20,0,8)}window.requestAnimationFrame(c)}};return c(),()=>{e=!1}}),[]),t.useEffect((()=>{i&&V&&(()=>{if(!w.current){w.current=!0;const e=p.current,t=v.current;t.updateSession({instructions:V}),t.updateSession({input_audio_transcription:{model:"whisper-1"}});for(const e of i)t.addTool(e.definition,e.handler);t.on("error",(e=>console.error(e))),t.on("conversation.interrupted",(async()=>{const n=await e.interrupt();if(n?.trackId){const{trackId:e,offset:a}=n;await t.cancelResponse(e,a)}})),t.on("conversation.updated",(async({item:n,delta:a})=>{const s=t.conversation.getItems();if(a?.audio&&e.add16BitPCM(a.audio,n.id),"completed"===n.status&&n.formatted.audio?.length){const e=await r.WavRecorder.decode(n.formatted.audio,24e3,24e3);n.formatted.file=e}R(s)})),R(t.conversation.getItems())}})()}),[i,V]);const G=t.useCallback((e=>{const t=v.current,n=e.replace("{{name}}",E).replace("{{language}}",N).replace("{{time}}",(new Date).toISOString());t.updateSession({instructions:n}),j(e),console.log(`Voice instructions updated: ${n.substring(0,50)}...`)}),[E,N]);return t.useMemo((()=>{if(c&&u)return{items:I,realtimeEvents:S,isConnected:b,canPushToTalk:T,isRecording:O,clientCanvasRef:g,serverCanvasRef:m,eventsScrollRef:h,formatTime:F,connectConversation:U,disconnectConversation:B,deleteConversationItem:D,startRecording:W,stopRecording:L,changeTurnEndType:$,getRecorderLevel:()=>k.current,getStreamLevel:()=>q.current,setUsername:M,updateInstructions:G,setLanguage:x}}),[c,u,I,S,b,T,O,g,m,h])};

package/dist/esm/hooks/useCodicentApp.d.ts

@@ -19,7 +19,7 @@ export interface CodicentAppState {
     setAnonymous: (value: boolean) => void;
     allStates: AppStateDefinition;
     isBusy: () => boolean;
-    voice: RealtimeVoice;
+    voice: RealtimeVoice | undefined;
     html: string;
     setHtml: (html: string) => void;
     script: string;

package/dist/esm/hooks/useCodicentApp.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"useCodicentApp.d.ts","sourceRoot":"","sources":["../../../src/hooks/useCodicentApp.tsx"],"names":[],"mappings":"AACA,OAAO,EAAE,qBAAqB,EAAE,MAAM,oBAAoB,CAAC;AAC3D,OAAO,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAC9C,OAAO,EAAE,eAAe,EAAE,YAAY,EAAE,kBAAkB,EAAyB,MAAM,UAAU,CAAC;AACpG,OAAyB,EAAE,kBAAkB,EAAE,MAAM,oBAAoB,CAAC;AAC1E,OAAqB,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC5D,OAA2B,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AAIzE,MAAM,WAAW,gBAAgB;IAC/B,KAAK,EAAE,kBAAkB,CAAC;IAC1B,IAAI,EAAE,YAAY,CAAC;IACnB,YAAY,EAAE,eAAe,CAAC;IAC9B,OAAO,EAAE,eAAe,CAAC;IACzB,OAAO,EAAE,YAAY,CAAC;IACtB,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,MAAM,CAAC;IACnC,YAAY,EAAE,CAAC,KAAK,EAAE,OAAO,KAAK,IAAI,CAAC;IACvC,SAAS,EAAE,kBAAkB,CAAC;IAC9B,MAAM,EAAE,MAAM,OAAO,CAAC;IACtB,KAAK,EAAE,aAAa,CAAC;IACrB,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,CAAC;IAChC,MAAM,EAAE,MAAM,CAAC;IACf,gBAAgB,EAAE,MAAM,CAAC;CAC1B;AAED,eAAO,MAAM,cAAc,cAAe;IAAE,KAAK,EAAE,qBAAqB,CAAA;CAAE,KAoKnE,gBACN,CAAC"}
+{"version":3,"file":"useCodicentApp.d.ts","sourceRoot":"","sources":["../../../src/hooks/useCodicentApp.tsx"],"names":[],"mappings":"AACA,OAAO,EAAE,qBAAqB,EAAE,MAAM,oBAAoB,CAAC;AAC3D,OAAO,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAC9C,OAAO,EAAE,eAAe,EAAE,YAAY,EAAE,kBAAkB,EAAyB,MAAM,UAAU,CAAC;AACpG,OAAyB,EAAE,kBAAkB,EAAE,MAAM,oBAAoB,CAAC;AAC1E,OAAqB,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC5D,OAA2B,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AAIzE,MAAM,WAAW,gBAAgB;IAC/B,KAAK,EAAE,kBAAkB,CAAC;IAC1B,IAAI,EAAE,YAAY,CAAC;IACnB,YAAY,EAAE,eAAe,CAAC;IAC9B,OAAO,EAAE,eAAe,CAAC;IACzB,OAAO,EAAE,YAAY,CAAC;IACtB,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,MAAM,CAAC;IACnC,YAAY,EAAE,CAAC,KAAK,EAAE,OAAO,KAAK,IAAI,CAAC;IACvC,SAAS,EAAE,kBAAkB,CAAC;IAC9B,MAAM,EAAE,MAAM,OAAO,CAAC;IACtB,KAAK,EAAE,aAAa,GAAG,SAAS,CAAC;IACjC,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,CAAC;IAChC,MAAM,EAAE,MAAM,CAAC;IACf,gBAAgB,EAAE,MAAM,CAAC;CAC1B;AAED,eAAO,MAAM,cAAc,cAAe;IAAE,KAAK,EAAE,qBAAqB,CAAA;CAAE,KAqKnE,gBACN,CAAC"}

package/dist/esm/hooks/useCodicentApp.js

@@ -1 +1 @@
-import{useRef as e,useState as t,useCallback as r,useMemo as n,useEffect as s}from"react";import"../utils/MessageContent.js";import"../node_modules/tinycolor2/esm/tinycolor.js";import"react/jsx-runtime";import{createAppStateMachine as o}from"../utils/appState.js";import"../_virtual/index.js";import"../lib/wavtools/lib/wav_packer.js";import"../lib/wavtools/lib/analysis/audio_analysis.js";import"../lib/wavtools/lib/wav_stream_player.js";import"../lib/wavtools/lib/wav_recorder.js";import{useAudioRecorder as i}from"./useAudioRecorder.js";import a from"./useAuthState.js";import c from"./useRealtimeVoiceAI.js";import u from"./useTools.js";import{getConfigValue as l}from"../config/index.js";const m=({auth0:m})=>{const p=a(m),d=e(o(p.codicentService,(()=>{}))),[f,g]=t(),{service:h,context:v}=d.current,{nickname:k,name:A,error:w,errorType:j,selectedApp:y}=v,b=r(((e,t)=>{const r=e.get("file").name;h.uploadFile(r,e).then((e=>{h.sendMessage(`#transcribe ${""!==t?(t.startsWith("#")?"":"#")+t+" ":""}#file:${e}`).then((()=>{})).catch(console.warn)}))}),[h]),x=i(b),[S,C]=t(),[T,_]=t(),[I,N]=t(!1),R=n((()=>new Audio("./notification.mp3")),[]),E=r((()=>N(!0)),[]),M=n((()=>u(h,C,_,R,S,E)),[h,C,_,R,S,E]),O=n((()=>l("REALTIME_VOICE_API_KEY")||""),[]),J=c(O,M);s((()=>{I&&(J?.disconnectConversation(),N(!1))}),[I,J]),s((()=>{p.isAuthenticated&&p.user&&p.accessToken&&d.current.updateContext({isAuthenticated:p.isAuthenticated,user:{sub:p.user.sub,email:p.user.email},accessToken:p.accessToken})}),[p.isAuthenticated,p.user,p.accessToken]),s((()=>d.current.updateContext({nickname:p.nickname})),[p.nickname]),s((()=>{d.current.update();let e,t=d.current.getCurrentStateName();g(d.current.getCurrentStateName()),JSON.stringify(d.current.context);let r=!1;const n=s=>setInterval((()=>{const s=d.current.getCurrentStateName();if("hasAccess"===s&&!r)return console.log("🎯 Reached stable state - switching to slower polling (1000ms)"),r=!0,clearInterval(e),void(e=n(1e3));if("hasAccess"!==s&&r)return console.log("🚨 Left stable state - switching to fast polling (200ms)"),r=!1,clearInterval(e),void(e=n(200));const o=JSON.stringify(d.current.context);d.current.update();const i=JSON.stringify(d.current.context),a=d.current.getCurrentStateName();a===t&&o===i||(g(a),t=a)}),s);return e=n(200),()=>clearInterval(e)}),[]);const L=r((e=>e.replace("{nickname}",k||"unknown").replace("{token}",encodeURIComponent(p.accessToken||"unknown")).replace("{app}",y||"unknown")),[k,p.accessToken,y]),B=r((e=>{d.current.updateContext({isAnonymous:e})}),[]),U=r((()=>d.current.isBusy()),[]);return n((()=>({audio:x,auth:p,currentStateName:f,stateMachine:d.current,service:h,context:v,state:f,nickname:k,name:A,error:w,errorType:j,fixAppUrl:L,setAnonymous:B,allStates:d.current.allStates,isBusy:U,voice:J,html:S,setHtml:C,script:T})),[f,k,A,w,j,S,T,p.isAuthenticated,p.isLoading,x.isRecording,J?.isConnected])};export{m as useCodicentApp};
+import{useRef as e,useState as t,useCallback as r,useMemo as n,useEffect as s}from"react";import"../utils/MessageContent.js";import"../node_modules/tinycolor2/esm/tinycolor.js";import"react/jsx-runtime";import{createAppStateMachine as o}from"../utils/appState.js";import"../_virtual/index.js";import"../lib/wavtools/lib/wav_packer.js";import"../lib/wavtools/lib/analysis/audio_analysis.js";import"../lib/wavtools/lib/wav_stream_player.js";import"../lib/wavtools/lib/wav_recorder.js";import{useAudioRecorder as i}from"./useAudioRecorder.js";import a from"./useAuthState.js";import c from"./useRealtimeVoiceAI.js";import u from"./useTools.js";import{getConfigValue as l}from"../config/index.js";const m=({auth0:m})=>{const p=a(m),d=e(o(p.codicentService,(()=>{}))),[f,g]=t(),{service:h,context:v}=d.current,{nickname:k,name:A,error:w,errorType:j,selectedApp:y}=v,b=r(((e,t)=>{const r=e.get("file").name;h.uploadFile(r,e).then((e=>{h.sendMessage(`#transcribe ${""!==t?(t.startsWith("#")?"":"#")+t+" ":""}#file:${e}`).then((()=>{})).catch(console.warn)}))}),[h]),x=i(b),[S,C]=t(),[T,_]=t(),[I,N]=t(!1),R=n((()=>new Audio("./notification.mp3")),[]),E=r((()=>N(!0)),[]),M=u(h,C,_,R,S,E),O=n((()=>l("REALTIME_VOICE_API_KEY")||""),[]),J=c(O,M);s((()=>{I&&(J?.disconnectConversation(),N(!1))}),[I,J]),s((()=>{p.isAuthenticated&&p.user&&p.accessToken&&d.current.updateContext({isAuthenticated:p.isAuthenticated,user:{sub:p.user.sub,email:p.user.email},accessToken:p.accessToken})}),[p.isAuthenticated,p.user,p.accessToken]),s((()=>d.current.updateContext({nickname:p.nickname})),[p.nickname]),s((()=>{d.current.update();let e,t=d.current.getCurrentStateName();g(d.current.getCurrentStateName()),JSON.stringify(d.current.context);let r=!1;const n=s=>setInterval((()=>{const s=d.current.getCurrentStateName();if("hasAccess"===s&&!r)return console.log("🎯 Reached stable state - switching to slower polling (1000ms)"),r=!0,clearInterval(e),void(e=n(1e3));if("hasAccess"!==s&&r)return console.log("🚨 Left stable state - switching to fast polling (200ms)"),r=!1,clearInterval(e),void(e=n(200));const o=JSON.stringify(d.current.context);d.current.update();const i=JSON.stringify(d.current.context),a=d.current.getCurrentStateName();a===t&&o===i||(g(a),t=a)}),s);return e=n(200),()=>clearInterval(e)}),[]);const L=r((e=>e.replace("{nickname}",k||"unknown").replace("{token}",encodeURIComponent(p.accessToken||"unknown")).replace("{app}",y||"unknown")),[k,p.accessToken,y]),B=r((e=>{d.current.updateContext({isAnonymous:e})}),[]),U=r((()=>d.current.isBusy()),[]);return n((()=>({audio:x,auth:p,currentStateName:f,stateMachine:d.current,service:h,context:v,state:f,nickname:k,name:A,error:w,errorType:j,fixAppUrl:L,setAnonymous:B,allStates:d.current.allStates,isBusy:U,voice:J,html:S,setHtml:C,script:T})),[f,k,A,w,j,S,T,p.isAuthenticated,p.isLoading,x.isRecording,J?.isConnected,L,B,U])};export{m as useCodicentApp};

package/dist/esm/hooks/useRealtimeVoiceAI.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"useRealtimeVoiceAI.d.ts","sourceRoot":"","sources":["../../../src/hooks/useRealtimeVoiceAI.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,QAAQ,EAAE,kBAAkB,EAAE,MAAM,2CAA2C,CAAC;AAMzF;;GAEG;AACH,UAAU,aAAa;IACrB,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,QAAQ,GAAG,QAAQ,CAAC;IAC5B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,KAAK,EAAE;QAAE,CAAC,GAAG,EAAE,MAAM,GAAG,GAAG,CAAA;KAAE,CAAC;CAC/B;AAED,MAAM,WAAW,aAAa;IAC5B,KAAK,EAAE,QAAQ,EAAE,CAAC;IAClB,cAAc,EAAE,aAAa,EAAE,CAAC;IAChC,WAAW,EAAE,OAAO,CAAC;IACrB,aAAa,EAAE,OAAO,CAAC;IACvB,WAAW,EAAE,OAAO,CAAC;IACrB,eAAe,EAAE,KAAK,CAAC,SAAS,CAAC,iBAAiB,CAAC,CAAC;IACpD,eAAe,EAAE,KAAK,CAAC,SAAS,CAAC,iBAAiB,CAAC,CAAC;IACpD,eAAe,EAAE,KAAK,CAAC,SAAS,CAAC,cAAc,CAAC,CAAC;IACjD,UAAU,EAAE,CAAC,SAAS,EAAE,MAAM,KAAK,MAAM,CAAC;IAC1C,mBAAmB,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IACzC,sBAAsB,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IAC5C,sBAAsB,EAAE,CAAC,EAAE,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IACtD,cAAc,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IACpC,aAAa,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IACnC,iBAAiB,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IACpD,gBAAgB,EAAE,MAAM,MAAM,CAAC;IAC/B,cAAc,EAAE,MAAM,MAAM,CAAC;IAC7B,WAAW,EAAE,CAAC,QAAQ,EAAE,MAAM,KAAK,IAAI,CAAC;IACxC,kBAAkB,EAAE,CAAC,YAAY,EAAE,MAAM,KAAK,IAAI,CAAC;IACnD,WAAW,EAAE,CAAC,QAAQ,EAAE,MAAM,KAAK,IAAI,CAAC;CACzC;AAED,QAAA,MAAM,kBAAkB,WAAY,MAAM,SAAS;IAAE,UAAU,EAAE,kBAAkB,CAAC;IAAC,OAAO,EAAE,QAAQ,CAAA;CAAE,EAAE,8BAqYzG,CAAC;AAEF,eAAe,kBAAkB,CAAC"}
+{"version":3,"file":"useRealtimeVoiceAI.d.ts","sourceRoot":"","sources":["../../../src/hooks/useRealtimeVoiceAI.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,QAAQ,EAAE,kBAAkB,EAAE,MAAM,2CAA2C,CAAC;AAMzF;;GAEG;AACH,UAAU,aAAa;IACrB,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,QAAQ,GAAG,QAAQ,CAAC;IAC5B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,KAAK,EAAE;QAAE,CAAC,GAAG,EAAE,MAAM,GAAG,GAAG,CAAA;KAAE,CAAC;CAC/B;AAED,MAAM,WAAW,aAAa;IAC5B,KAAK,EAAE,QAAQ,EAAE,CAAC;IAClB,cAAc,EAAE,aAAa,EAAE,CAAC;IAChC,WAAW,EAAE,OAAO,CAAC;IACrB,aAAa,EAAE,OAAO,CAAC;IACvB,WAAW,EAAE,OAAO,CAAC;IACrB,eAAe,EAAE,KAAK,CAAC,SAAS,CAAC,iBAAiB,CAAC,CAAC;IACpD,eAAe,EAAE,KAAK,CAAC,SAAS,CAAC,iBAAiB,CAAC,CAAC;IACpD,eAAe,EAAE,KAAK,CAAC,SAAS,CAAC,cAAc,CAAC,CAAC;IACjD,UAAU,EAAE,CAAC,SAAS,EAAE,MAAM,KAAK,MAAM,CAAC;IAC1C,mBAAmB,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IACzC,sBAAsB,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IAC5C,sBAAsB,EAAE,CAAC,EAAE,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IACtD,cAAc,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IACpC,aAAa,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IACnC,iBAAiB,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IACpD,gBAAgB,EAAE,MAAM,MAAM,CAAC;IAC/B,cAAc,EAAE,MAAM,MAAM,CAAC;IAC7B,WAAW,EAAE,CAAC,QAAQ,EAAE,MAAM,KAAK,IAAI,CAAC;IACxC,kBAAkB,EAAE,CAAC,YAAY,EAAE,MAAM,KAAK,IAAI,CAAC;IACnD,WAAW,EAAE,CAAC,QAAQ,EAAE,MAAM,KAAK,IAAI,CAAC;CACzC;AAED,QAAA,MAAM,kBAAkB,WAAY,MAAM,SAAS;IAAE,UAAU,EAAE,kBAAkB,CAAC;IAAC,OAAO,EAAE,QAAQ,CAAA;CAAE,EAAE,KAAG,aAAa,GAAG,SAgZ5H,CAAC;AAEF,eAAe,kBAAkB,CAAC"}

package/dist/esm/hooks/useRealtimeVoiceAI.js

@@ -1 +1 @@
-import{RealtimeClient as e}from"../node_modules/@openai/realtime-api-beta/lib/client.js";import{useRef as t,useState as n,useCallback as r,useEffect as a,useMemo as o}from"react";import{WavRenderer as i}from"../utils/wav_renderer.js";import"../lib/wavtools/lib/wav_packer.js";import"../lib/wavtools/lib/analysis/audio_analysis.js";import{WavStreamPlayer as s}from"../lib/wavtools/lib/wav_stream_player.js";import{WavRecorder as c}from"../lib/wavtools/lib/wav_recorder.js";import{getConfigValue as l}from"../config/index.js";const u=(u,d)=>{if(!l("APP_CONFIG"))return void console.warn("APP_CONFIG is not set. Voice AI will not be available.");if(!l("APP_BUTTONS"))return void console.warn("APP_BUTTONS is not set. Voice AI will not be available.");const p=l("APP_CONFIG"),w=l("APP_BUTTONS"),f=t(new c({sampleRate:24e3})),m=t(new s({sampleRate:24e3})),v=t(new e({apiKey:u,dangerouslyAllowAPIKeyInBrowser:!0,url:"wss://codicent-ai-sweden.openai.azure.com/openai/realtime?api-version=2024-10-01-preview&deployment=gpt-4o-realtime-preview-2&api-key=9013365c13ee4f3dad0016fb0f0c450a"})),g=t(!1),h=t(null),I=t(null),_=t(null),S=t((new Date).toISOString()),[y,C]=n([]),[T,A]=n([]),[R,b]=n(!1),[O,P]=n(!1),[N,M]=n(!1),x=t(0),E=t(0),[F,U]=n(""),[j,k]=n("en-US"),[B,D]=n((()=>w&&p.apps[w]?p.apps[w].voiceInstructions||p.REALTIME_VOICE_INSTRUCTIONS||"":p.REALTIME_VOICE_INSTRUCTIONS||"")),L=r((e=>{const t=S.current,n=new Date(t).valueOf(),r=new Date(e).valueOf()-n,a=Math.floor(r/10)%100,o=Math.floor(r/1e3)%60,i=e=>{let t=e+"";for(;t.length<2;)t="0"+t;return t};return`${i(Math.floor(r/6e4)%60)}:${i(o)}.${i(a)}`}),[]),V=r((async()=>{const e=v.current,t=f.current,n=m.current;S.current=(new Date).toISOString(),b(!0),A([]),C(e.conversation.getItems()),K("server_vad"),await t.begin(),await n.connect(),await e.connect(),e.sendUserMessageContent([{type:"input_text",text:"Hello!"}]),"server_vad"===e.getTurnDetectionType()&&await t.record((t=>e.appendInputAudio(t.mono)))}),[]),$=r((async()=>{b(!1),A([]),C([]);const e=v.current,t=f.current;await t.end(),e.disconnect();const n=m.current;await n.interrupt()}),[]),q=r((async e=>{v.current.deleteItem(e)}),[]),G=r((async()=>{M(!0);const e=v.current,t=f.current,n=m.current,r=await n.interrupt();if(r?.trackId){const{trackId:t,offset:n}=r;await e.cancelResponse(t,n)}await t.record((t=>e.appendInputAudio(t.mono)))}),[]),H=r((async()=>{M(!1);const e=v.current,t=f.current;await t.pause(),e.createResponse()}),[]),K=r((async e=>{const t=v.current,n=f.current;"none"===e&&"recording"===n.getStatus()&&await n.pause(),t.updateSession({turn_detection:"none"===e?null:{type:"server_vad"}}),"server_vad"===e&&t.isConnected()&&await n.record((e=>t.appendInputAudio(e.mono))),P("none"===e)}),[]);a((()=>{if(F){v.current.updateSession({instructions:B.replace("{{name}}",F).replace("{{language}}",j).replace("{{time}}",(new Date).toISOString())})}}),[B,F]),a((()=>{let e=!0;const t=f.current,n=h.current;let r=null;const a=m.current,o=I.current;let s=null;const c=()=>{if(e){if(n&&(n.width&&n.height||(n.width=n.offsetWidth,n.height=n.offsetHeight),r=r||n.getContext("2d"),r)){r.clearRect(0,0,n.width,n.height);const e=t.recording?t.getFrequencies("voice"):{values:new Float32Array([0])},a=1-Math.max(...e.values);x.current=a,i.drawCircularBars(n,r,e.values,"#0099ff",20,0,8)}if(o&&(o.width&&o.height||(o.width=o.offsetWidth,o.height=o.offsetHeight),s=s||o.getContext("2d"),s)){s.clearRect(0,0,o.width,o.height);const e=a.analyser?a.getFrequencies("voice"):{values:new Float32Array([0])},t=1-Math.max(...e.values);E.current=t,i.drawCircularBars(o,s,e.values,"#009900",20,0,8)}window.requestAnimationFrame(c)}};return c(),()=>{e=!1}}),[]),a((()=>{d&&B&&(()=>{if(!g.current){g.current=!0;const e=m.current,t=v.current;t.updateSession({instructions:B}),t.updateSession({input_audio_transcription:{model:"whisper-1"}});for(const e of d)t.addTool(e.definition,e.handler);t.on("error",(e=>console.error(e))),t.on("conversation.interrupted",(async()=>{const n=await e.interrupt();if(n?.trackId){const{trackId:e,offset:r}=n;await t.cancelResponse(e,r)}})),t.on("conversation.updated",(async({item:n,delta:r})=>{const a=t.conversation.getItems();if(r?.audio&&e.add16BitPCM(r.audio,n.id),"completed"===n.status&&n.formatted.audio?.length){const e=await c.decode(n.formatted.audio,24e3,24e3);n.formatted.file=e}C(a)})),C(t.conversation.getItems())}})()}),[d,B]);const W=r((e=>{const t=v.current,n=e.replace("{{name}}",F).replace("{{language}}",j).replace("{{time}}",(new Date).toISOString());t.updateSession({instructions:n}),D(e),console.log(`Voice instructions updated: ${n.substring(0,50)}...`)}),[F,j]);return o((()=>({items:y,realtimeEvents:T,isConnected:R,canPushToTalk:O,isRecording:N,clientCanvasRef:h,serverCanvasRef:I,eventsScrollRef:_,formatTime:L,connectConversation:V,disconnectConversation:$,deleteConversationItem:q,startRecording:G,stopRecording:H,changeTurnEndType:K,getRecorderLevel:()=>x.current,getStreamLevel:()=>E.current,setUsername:U,updateInstructions:W,setLanguage:k})),[y,T,R,O,N,h,I,_])};export{u as default};
+import{RealtimeClient as e}from"../node_modules/@openai/realtime-api-beta/lib/client.js";import{useRef as t,useState as n,useCallback as r,useEffect as a,useMemo as o}from"react";import{WavRenderer as i}from"../utils/wav_renderer.js";import"../lib/wavtools/lib/wav_packer.js";import"../lib/wavtools/lib/analysis/audio_analysis.js";import{WavStreamPlayer as s}from"../lib/wavtools/lib/wav_stream_player.js";import{WavRecorder as c}from"../lib/wavtools/lib/wav_recorder.js";import{getConfigValue as l}from"../config/index.js";const u=(u,d)=>{const p=!!l("APP_CONFIG"),w=!!l("APP_BUTTONS");p||console.warn("APP_CONFIG is not set. Voice AI will not be available."),w||console.warn("APP_BUTTONS is not set. Voice AI will not be available.");const f=l("APP_CONFIG"),m=l("APP_BUTTONS"),v=t(new c({sampleRate:24e3})),g=t(new s({sampleRate:24e3})),h=t(new e({apiKey:u,dangerouslyAllowAPIKeyInBrowser:!0,url:"wss://codicent-ai-sweden.openai.azure.com/openai/realtime?api-version=2024-10-01-preview&deployment=gpt-4o-realtime-preview-2&api-key=9013365c13ee4f3dad0016fb0f0c450a"})),I=t(!1),_=t(null),S=t(null),y=t(null),C=t((new Date).toISOString()),[T,A]=n([]),[R,b]=n([]),[O,P]=n(!1),[N,M]=n(!1),[x,E]=n(!1),F=t(0),U=t(0),[j,k]=n(""),[B,D]=n("en-US"),[L,V]=n((()=>p&&w&&m&&f&&f.apps&&f.apps[m]?f.apps[m].voiceInstructions||f.REALTIME_VOICE_INSTRUCTIONS||"":f&&f.REALTIME_VOICE_INSTRUCTIONS||"")),$=r((e=>{const t=C.current,n=new Date(t).valueOf(),r=new Date(e).valueOf()-n,a=Math.floor(r/10)%100,o=Math.floor(r/1e3)%60,i=e=>{let t=e+"";for(;t.length<2;)t="0"+t;return t};return`${i(Math.floor(r/6e4)%60)}:${i(o)}.${i(a)}`}),[]),q=r((async()=>{const e=h.current,t=v.current,n=g.current;C.current=(new Date).toISOString(),P(!0),b([]),A(e.conversation.getItems()),z("server_vad"),await t.begin(),await n.connect(),await e.connect(),e.sendUserMessageContent([{type:"input_text",text:"Hello!"}]),"server_vad"===e.getTurnDetectionType()&&await t.record((t=>e.appendInputAudio(t.mono)))}),[]),G=r((async()=>{P(!1),b([]),A([]);const e=h.current,t=v.current;await t.end(),e.disconnect();const n=g.current;await n.interrupt()}),[]),H=r((async e=>{h.current.deleteItem(e)}),[]),K=r((async()=>{E(!0);const e=h.current,t=v.current,n=g.current,r=await n.interrupt();if(r?.trackId){const{trackId:t,offset:n}=r;await e.cancelResponse(t,n)}await t.record((t=>e.appendInputAudio(t.mono)))}),[]),W=r((async()=>{E(!1);const e=h.current,t=v.current;await t.pause(),e.createResponse()}),[]),z=r((async e=>{const t=h.current,n=v.current;"none"===e&&"recording"===n.getStatus()&&await n.pause(),t.updateSession({turn_detection:"none"===e?null:{type:"server_vad"}}),"server_vad"===e&&t.isConnected()&&await n.record((e=>t.appendInputAudio(e.mono))),M("none"===e)}),[]);a((()=>{if(j){h.current.updateSession({instructions:L.replace("{{name}}",j).replace("{{language}}",B).replace("{{time}}",(new Date).toISOString())})}}),[L,j]),a((()=>{let e=!0;const t=v.current,n=_.current;let r=null;const a=g.current,o=S.current;let s=null;const c=()=>{if(e){if(n&&(n.width&&n.height||(n.width=n.offsetWidth,n.height=n.offsetHeight),r=r||n.getContext("2d"),r)){r.clearRect(0,0,n.width,n.height);const e=t.recording?t.getFrequencies("voice"):{values:new Float32Array([0])},a=1-Math.max(...e.values);F.current=a,i.drawCircularBars(n,r,e.values,"#0099ff",20,0,8)}if(o&&(o.width&&o.height||(o.width=o.offsetWidth,o.height=o.offsetHeight),s=s||o.getContext("2d"),s)){s.clearRect(0,0,o.width,o.height);const e=a.analyser?a.getFrequencies("voice"):{values:new Float32Array([0])},t=1-Math.max(...e.values);U.current=t,i.drawCircularBars(o,s,e.values,"#009900",20,0,8)}window.requestAnimationFrame(c)}};return c(),()=>{e=!1}}),[]),a((()=>{d&&L&&(()=>{if(!I.current){I.current=!0;const e=g.current,t=h.current;t.updateSession({instructions:L}),t.updateSession({input_audio_transcription:{model:"whisper-1"}});for(const e of d)t.addTool(e.definition,e.handler);t.on("error",(e=>console.error(e))),t.on("conversation.interrupted",(async()=>{const n=await e.interrupt();if(n?.trackId){const{trackId:e,offset:r}=n;await t.cancelResponse(e,r)}})),t.on("conversation.updated",(async({item:n,delta:r})=>{const a=t.conversation.getItems();if(r?.audio&&e.add16BitPCM(r.audio,n.id),"completed"===n.status&&n.formatted.audio?.length){const e=await c.decode(n.formatted.audio,24e3,24e3);n.formatted.file=e}A(a)})),A(t.conversation.getItems())}})()}),[d,L]);const J=r((e=>{const t=h.current,n=e.replace("{{name}}",j).replace("{{language}}",B).replace("{{time}}",(new Date).toISOString());t.updateSession({instructions:n}),V(e),console.log(`Voice instructions updated: ${n.substring(0,50)}...`)}),[j,B]);return o((()=>{if(p&&w)return{items:T,realtimeEvents:R,isConnected:O,canPushToTalk:N,isRecording:x,clientCanvasRef:_,serverCanvasRef:S,eventsScrollRef:y,formatTime:$,connectConversation:q,disconnectConversation:G,deleteConversationItem:H,startRecording:K,stopRecording:W,changeTurnEndType:z,getRecorderLevel:()=>F.current,getStreamLevel:()=>U.current,setUsername:k,updateInstructions:J,setLanguage:D}}),[p,w,T,R,O,N,x,_,S,y])};export{u as default};

package/dist/index.d.ts

@@ -511,7 +511,7 @@ interface CodicentAppState {
     setAnonymous: (value: boolean) => void;
     allStates: AppStateDefinition;
     isBusy: () => boolean;
-    voice: RealtimeVoice;
+    voice: RealtimeVoice | undefined;
     html: string;
     setHtml: (html: string) => void;
     script: string;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "codicent-app-sdk",
-  "version": "0.3.79",
+  "version": "0.3.80",
   "description": "SDK for building AI-powered applications with Codicent",
   "type": "module",
   "main": "dist/cjs/index.js",