codicent-app-sdk 0.3.41 → 0.3.42

package/dist/cjs/hooks/useCodicentApp.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"useCodicentApp.d.ts","sourceRoot":"","sources":["../../../src/hooks/useCodicentApp.tsx"],"names":[],"mappings":"AACA,OAAO,EAAE,qBAAqB,EAAE,MAAM,oBAAoB,CAAC;AAC3D,OAAO,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAC9C,OAAO,EAAE,eAAe,EAAE,YAAY,EAAE,kBAAkB,EAAyB,MAAM,UAAU,CAAC;AACpG,OAAyB,EAAE,kBAAkB,EAAE,MAAM,oBAAoB,CAAC;AAC1E,OAAqB,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC5D,OAA2B,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AAIzE,MAAM,WAAW,gBAAgB;IAC/B,KAAK,EAAE,kBAAkB,CAAC;IAC1B,IAAI,EAAE,YAAY,CAAC;IACnB,YAAY,EAAE,eAAe,CAAC;IAC9B,OAAO,EAAE,eAAe,CAAC;IACzB,OAAO,EAAE,YAAY,CAAC;IACtB,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,MAAM,CAAC;IACnC,YAAY,EAAE,CAAC,KAAK,EAAE,OAAO,KAAK,IAAI,CAAC;IACvC,SAAS,EAAE,kBAAkB,CAAC;IAC9B,MAAM,EAAE,MAAM,OAAO,CAAC;IACtB,KAAK,EAAE,aAAa,CAAC;IACrB,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,CAAC;IAChC,MAAM,EAAE,MAAM,CAAC;IACf,gBAAgB,EAAE,MAAM,CAAC;CAC1B;AAED,eAAO,MAAM,cAAc,cAAa;IAAC,KAAK,EAAE,qBAAqB,CAAA;CAAC,KA0F/D,gBACN,CAAC"}
+{"version":3,"file":"useCodicentApp.d.ts","sourceRoot":"","sources":["../../../src/hooks/useCodicentApp.tsx"],"names":[],"mappings":"AACA,OAAO,EAAE,qBAAqB,EAAE,MAAM,oBAAoB,CAAC;AAC3D,OAAO,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAC9C,OAAO,EAAE,eAAe,EAAE,YAAY,EAAE,kBAAkB,EAAyB,MAAM,UAAU,CAAC;AACpG,OAAyB,EAAE,kBAAkB,EAAE,MAAM,oBAAoB,CAAC;AAC1E,OAAqB,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC5D,OAA2B,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AAIzE,MAAM,WAAW,gBAAgB;IAC/B,KAAK,EAAE,kBAAkB,CAAC;IAC1B,IAAI,EAAE,YAAY,CAAC;IACnB,YAAY,EAAE,eAAe,CAAC;IAC9B,OAAO,EAAE,eAAe,CAAC;IACzB,OAAO,EAAE,YAAY,CAAC;IACtB,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,MAAM,CAAC;IACnC,YAAY,EAAE,CAAC,KAAK,EAAE,OAAO,KAAK,IAAI,CAAC;IACvC,SAAS,EAAE,kBAAkB,CAAC;IAC9B,MAAM,EAAE,MAAM,OAAO,CAAC;IACtB,KAAK,EAAE,aAAa,CAAC;IACrB,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,CAAC;IAChC,MAAM,EAAE,MAAM,CAAC;IACf,gBAAgB,EAAE,MAAM,CAAC;CAC1B;AAED,eAAO,MAAM,cAAc,cAAe;IAAE,KAAK,EAAE,qBAAqB,CAAA;CAAE,KAkGnE,gBACN,CAAC"}

package/dist/cjs/hooks/useCodicentApp.js

@@ -1 +1 @@
-"use strict";var e=require("react");require("../utils/MessageContent.js"),require("../node_modules/tinycolor2/esm/tinycolor.js"),require("react/jsx-runtime");var t=require("../utils/appState.js");require("../_virtual/index.js"),require("../lib/wavtools/lib/wav_packer.js"),require("../lib/wavtools/lib/analysis/audio_analysis.js"),require("../lib/wavtools/lib/wav_stream_player.js"),require("../lib/wavtools/lib/wav_recorder.js");var r=require("./useAudioRecorder.js"),s=require("./useAuthState.js"),a=require("./useRealtimeVoiceAI.js"),n=require("./useTools.js"),u=require("../config/index.js");exports.useCodicentApp=({auth0:c})=>{const i=s.default(c),o=e.useRef(t.createAppStateMachine(i.codicentService,(()=>{}))),[l,d]=e.useState(),{service:p,context:m}=o.current,{nickname:A,name:k,error:v,errorType:f,selectedApp:h}=m,q=e.useCallback(((e,t)=>{const r=e.get("file").name;p.uploadFile(r,e).then((e=>{p.sendMessage(`#transcribe ${""!==t?(t.startsWith("#")?"":"#")+t+" ":""}#file:${e}`).then((()=>{})).catch(console.warn)}))}),[p]),j=r.useAudioRecorder(q),[S,b]=e.useState(),[w,y]=e.useState(),x=e.useMemo((()=>new Audio("./notification.mp3")),[]),C=n.default(p,b,y,x,S),_=a.default(u.getConfigValue("REALTIME_VOICE_API_KEY")||"",C);e.useEffect((()=>{i.isAuthenticated&&i.user&&i.accessToken&&o.current.updateContext({isAuthenticated:i.isAuthenticated,user:{sub:i.user.sub,email:i.user.email},accessToken:i.accessToken})}),[i.isAuthenticated,i.user,i.accessToken]),e.useEffect((()=>o.current.updateContext({nickname:i.nickname})),[i.nickname]),e.useEffect((()=>{o.current.update();let e=o.current;const t=setInterval((()=>{"hasAccess"===o.current.currentStateName&&e.currentStateName===o.current.currentStateName||(o.current.update(),d(o.current.getCurrentStateName()),e=o.current)}),50);return()=>clearInterval(t)}),[]);return{audio:j,auth:i,currentStateName:l,stateMachine:o.current,service:p,context:m,state:l,nickname:A,name:k,error:v,errorType:f,fixAppUrl:e=>e.replace("{nickname}",A||"unknown").replace("{token}",encodeURIComponent(i.accessToken||"unknown")).replace("{app}",h||"unknown"),setAnonymous:e=>{o.current.updateContext({isAnonymous:e})},allStates:o.current.allStates,isBusy:()=>o.current.isBusy(),voice:_,html:S,setHtml:b,script:w}};
+"use strict";var e=require("react");require("../utils/MessageContent.js"),require("../node_modules/tinycolor2/esm/tinycolor.js"),require("react/jsx-runtime");var t=require("../utils/appState.js");require("../_virtual/index.js"),require("../lib/wavtools/lib/wav_packer.js"),require("../lib/wavtools/lib/analysis/audio_analysis.js"),require("../lib/wavtools/lib/wav_stream_player.js"),require("../lib/wavtools/lib/wav_recorder.js");var r=require("./useAudioRecorder.js"),s=require("./useAuthState.js"),n=require("./useRealtimeVoiceAI.js"),a=require("./useTools.js"),u=require("../config/index.js");exports.useCodicentApp=({auth0:c})=>{const i=s.default(c),o=e.useRef(t.createAppStateMachine(i.codicentService,(()=>{}))),[l,d]=e.useState(),{service:p,context:m}=o.current,{nickname:f,name:v,error:A,errorType:k,selectedApp:h}=m,q=e.useCallback(((e,t)=>{const r=e.get("file").name;p.uploadFile(r,e).then((e=>{p.sendMessage(`#transcribe ${""!==t?(t.startsWith("#")?"":"#")+t+" ":""}#file:${e}`).then((()=>{})).catch(console.warn)}))}),[p]),S=r.useAudioRecorder(q),[j,b]=e.useState(),[w,y]=e.useState(),[C,x]=e.useState(!1),_=e.useMemo((()=>new Audio("./notification.mp3")),[]),T=a.default(p,b,y,_,j,(()=>x(!0))),E=n.default(u.getConfigValue("REALTIME_VOICE_API_KEY")||"",T);e.useEffect((()=>{C&&(E?.disconnectConversation(),x(!1))}),[C,E]),e.useEffect((()=>{i.isAuthenticated&&i.user&&i.accessToken&&o.current.updateContext({isAuthenticated:i.isAuthenticated,user:{sub:i.user.sub,email:i.user.email},accessToken:i.accessToken})}),[i.isAuthenticated,i.user,i.accessToken]),e.useEffect((()=>o.current.updateContext({nickname:i.nickname})),[i.nickname]),e.useEffect((()=>{o.current.update();let e=o.current;const t=setInterval((()=>{"hasAccess"===o.current.currentStateName&&e.currentStateName===o.current.currentStateName||(o.current.update(),d(o.current.getCurrentStateName()),e=o.current)}),50);return()=>clearInterval(t)}),[]);return{audio:S,auth:i,currentStateName:l,stateMachine:o.current,service:p,context:m,state:l,nickname:f,name:v,error:A,errorType:k,fixAppUrl:e=>e.replace("{nickname}",f||"unknown").replace("{token}",encodeURIComponent(i.accessToken||"unknown")).replace("{app}",h||"unknown"),setAnonymous:e=>{o.current.updateContext({isAnonymous:e})},allStates:o.current.allStates,isBusy:()=>o.current.isBusy(),voice:E,html:j,setHtml:b,script:w}};

package/dist/cjs/hooks/useRealtimeVoiceAI.d.ts

@@ -30,6 +30,7 @@ export interface RealtimeVoice {
     streamLevel: number;
     setUsername: (username: string) => void;
     updateInstructions: (instructions: string) => void;
+    setLanguage: (language: string) => void;
 }
 declare const useRealtimeVoiceAI: (apiKey: string, tools: {
     definition: ToolDefinitionType;

package/dist/cjs/hooks/useRealtimeVoiceAI.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"useRealtimeVoiceAI.d.ts","sourceRoot":"","sources":["../../../src/hooks/useRealtimeVoiceAI.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,QAAQ,EAAE,kBAAkB,EAAE,MAAM,2CAA2C,CAAC;AAMzF;;GAEG;AACH,UAAU,aAAa;IACrB,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,QAAQ,GAAG,QAAQ,CAAC;IAC5B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,KAAK,EAAE;QAAE,CAAC,GAAG,EAAE,MAAM,GAAG,GAAG,CAAA;KAAE,CAAC;CAC/B;AAED,MAAM,WAAW,aAAa;IAC5B,KAAK,EAAE,QAAQ,EAAE,CAAC;IAClB,cAAc,EAAE,aAAa,EAAE,CAAC;IAChC,WAAW,EAAE,OAAO,CAAC;IACrB,aAAa,EAAE,OAAO,CAAC;IACvB,WAAW,EAAE,OAAO,CAAC;IACrB,eAAe,EAAE,KAAK,CAAC,SAAS,CAAC,iBAAiB,CAAC,CAAC;IACpD,eAAe,EAAE,KAAK,CAAC,SAAS,CAAC,iBAAiB,CAAC,CAAC;IACpD,eAAe,EAAE,KAAK,CAAC,SAAS,CAAC,cAAc,CAAC,CAAC;IACjD,UAAU,EAAE,CAAC,SAAS,EAAE,MAAM,KAAK,MAAM,CAAC;IAC1C,mBAAmB,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IACzC,sBAAsB,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IAC5C,sBAAsB,EAAE,CAAC,EAAE,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IACtD,cAAc,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IACpC,aAAa,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IACnC,iBAAiB,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IACpD,aAAa,EAAE,MAAM,CAAC;IACtB,WAAW,EAAE,MAAM,CAAC;IACpB,WAAW,EAAE,CAAC,QAAQ,EAAE,MAAM,KAAK,IAAI,CAAC;IACxC,kBAAkB,EAAE,CAAC,YAAY,EAAE,MAAM,KAAK,IAAI,CAAC;CACpD;AAED,QAAA,MAAM,kBAAkB,WACd,MAAM,SACP;IAAE,UAAU,EAAE,kBAAkB,CAAC;IAAC,OAAO,EAAE,QAAQ,CAAA;CAAE,EAAE,8BA4W/D,CAAC;AAEF,eAAe,kBAAkB,CAAC"}
+{"version":3,"file":"useRealtimeVoiceAI.d.ts","sourceRoot":"","sources":["../../../src/hooks/useRealtimeVoiceAI.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,QAAQ,EAAE,kBAAkB,EAAE,MAAM,2CAA2C,CAAC;AAMzF;;GAEG;AACH,UAAU,aAAa;IACrB,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,QAAQ,GAAG,QAAQ,CAAC;IAC5B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,KAAK,EAAE;QAAE,CAAC,GAAG,EAAE,MAAM,GAAG,GAAG,CAAA;KAAE,CAAC;CAC/B;AAED,MAAM,WAAW,aAAa;IAC5B,KAAK,EAAE,QAAQ,EAAE,CAAC;IAClB,cAAc,EAAE,aAAa,EAAE,CAAC;IAChC,WAAW,EAAE,OAAO,CAAC;IACrB,aAAa,EAAE,OAAO,CAAC;IACvB,WAAW,EAAE,OAAO,CAAC;IACrB,eAAe,EAAE,KAAK,CAAC,SAAS,CAAC,iBAAiB,CAAC,CAAC;IACpD,eAAe,EAAE,KAAK,CAAC,SAAS,CAAC,iBAAiB,CAAC,CAAC;IACpD,eAAe,EAAE,KAAK,CAAC,SAAS,CAAC,cAAc,CAAC,CAAC;IACjD,UAAU,EAAE,CAAC,SAAS,EAAE,MAAM,KAAK,MAAM,CAAC;IAC1C,mBAAmB,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IACzC,sBAAsB,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IAC5C,sBAAsB,EAAE,CAAC,EAAE,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IACtD,cAAc,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IACpC,aAAa,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IACnC,iBAAiB,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IACpD,aAAa,EAAE,MAAM,CAAC;IACtB,WAAW,EAAE,MAAM,CAAC;IACpB,WAAW,EAAE,CAAC,QAAQ,EAAE,MAAM,KAAK,IAAI,CAAC;IACxC,kBAAkB,EAAE,CAAC,YAAY,EAAE,MAAM,KAAK,IAAI,CAAC;IACnD,WAAW,EAAE,CAAC,QAAQ,EAAE,MAAM,KAAK,IAAI,CAAC;CACzC;AAED,QAAA,MAAM,kBAAkB,WAAY,MAAM,SAAS;IAAE,UAAU,EAAE,kBAAkB,CAAC;IAAC,OAAO,EAAE,QAAQ,CAAA;CAAE,EAAE,8BAmXzG,CAAC;AAEF,eAAe,kBAAkB,CAAC"}

package/dist/cjs/hooks/useRealtimeVoiceAI.js

@@ -1 +1 @@
-"use strict";Object.defineProperty(exports,"__esModule",{value:!0});var e=require("../node_modules/@openai/realtime-api-beta/lib/client.js"),t=require("react"),n=require("../utils/wav_renderer.js");require("../lib/wavtools/lib/wav_packer.js"),require("../lib/wavtools/lib/analysis/audio_analysis.js");var r=require("../lib/wavtools/lib/wav_stream_player.js"),a=require("../lib/wavtools/lib/wav_recorder.js"),o=require("../config/index.js");exports.default=(s,i)=>{if(!o.getConfigValue("APP_CONFIG"))return void console.warn("APP_CONFIG is not set. Voice AI will not be available.");if(!o.getConfigValue("APP_BUTTONS"))return void console.warn("APP_BUTTONS is not set. Voice AI will not be available.");const c=o.getConfigValue("APP_CONFIG"),u=o.getConfigValue("APP_BUTTONS"),l=t.useRef(new a.WavRecorder({sampleRate:24e3})),d=t.useRef(new r.WavStreamPlayer({sampleRate:24e3})),f=t.useRef(new e.RealtimeClient({apiKey:s,dangerouslyAllowAPIKeyInBrowser:!0,url:"wss://codicent-ai-sweden.openai.azure.com/openai/realtime?api-version=2024-10-01-preview&deployment=gpt-4o-realtime-preview-2&api-key=9013365c13ee4f3dad0016fb0f0c450a"})),p=t.useRef(!1),v=t.useRef(null),w=t.useRef(null),g=t.useRef(null),m=t.useRef((new Date).toISOString()),[h,I]=t.useState([]),[R,C]=t.useState([]),[S,_]=t.useState(!1),[y,b]=t.useState(!1),[T,A]=t.useState(!1),[P,O]=t.useState(0),[k,q]=t.useState(0),[E,M]=t.useState(""),[N,x]=t.useState((()=>u&&c.apps[u]?c.apps[u].voiceInstructions||c.REALTIME_VOICE_INSTRUCTIONS||"":c.REALTIME_VOICE_INSTRUCTIONS||"")),V=t.useCallback((e=>{const t=m.current,n=new Date(t).valueOf(),r=new Date(e).valueOf()-n,a=Math.floor(r/10)%100,o=Math.floor(r/1e3)%60,s=e=>{let t=e+"";for(;t.length<2;)t="0"+t;return t};return`${s(Math.floor(r/6e4)%60)}:${s(o)}.${s(a)}`}),[]),j=t.useCallback((async()=>{const e=f.current,t=l.current,n=d.current;m.current=(new Date).toISOString(),_(!0),C([]),I(e.conversation.getItems()),U("server_vad"),await t.begin(),await n.connect(),await e.connect(),e.sendUserMessageContent([{type:"input_text",text:"Hello!"}]),"server_vad"===e.getTurnDetectionType()&&await t.record((t=>e.appendInputAudio(t.mono)))}),[]),F=t.useCallback((async()=>{_(!1),C([]),I([]);const e=f.current,t=l.current;await t.end(),e.disconnect();const n=d.current;await n.interrupt()}),[]),B=t.useCallback((async e=>{f.current.deleteItem(e)}),[]),U=async e=>{const t=f.current,n=l.current;"none"===e&&"recording"===n.getStatus()&&await n.pause(),t.updateSession({turn_detection:"none"===e?null:{type:"server_vad"}}),"server_vad"===e&&t.isConnected()&&await n.record((e=>t.appendInputAudio(e.mono))),b("none"===e)};t.useEffect((()=>{if(E){f.current.updateSession({instructions:N.replace("{{name}}",E)})}}),[N,E]),t.useEffect((()=>{let e=!0;const t=l.current,r=v.current;let a=null;const o=d.current,s=w.current;let i=null;const c=()=>{if(e){if(r&&(r.width&&r.height||(r.width=r.offsetWidth,r.height=r.offsetHeight),a=a||r.getContext("2d"),a)){a.clearRect(0,0,r.width,r.height);const e=t.recording?t.getFrequencies("voice"):{values:new Float32Array([0])},o=1-Math.max(...e.values);O(o),n.WavRenderer.drawCircularBars(r,a,e.values,"#0099ff",20,0,8)}if(s&&(s.width&&s.height||(s.width=s.offsetWidth,s.height=s.offsetHeight),i=i||s.getContext("2d"),i)){i.clearRect(0,0,s.width,s.height);const e=o.analyser?o.getFrequencies("voice"):{values:new Float32Array([0])},t=1-Math.max(...e.values);q(t),n.WavRenderer.drawCircularBars(s,i,e.values,"#009900",20,0,8)}window.requestAnimationFrame(c)}};return c(),()=>{e=!1}}),[]),t.useEffect((()=>{i&&N&&(()=>{if(!p.current){p.current=!0;const e=d.current,t=f.current;t.updateSession({instructions:N}),t.updateSession({input_audio_transcription:{model:"whisper-1"}});for(const e of i)t.addTool(e.definition,e.handler);t.on("error",(e=>console.error(e))),t.on("conversation.interrupted",(async()=>{const n=await e.interrupt();if(n?.trackId){const{trackId:e,offset:r}=n;await t.cancelResponse(e,r)}})),t.on("conversation.updated",(async({item:n,delta:r})=>{const o=t.conversation.getItems();if(r?.audio&&e.add16BitPCM(r.audio,n.id),"completed"===n.status&&n.formatted.audio?.length){const e=await a.WavRecorder.decode(n.formatted.audio,24e3,24e3);n.formatted.file=e}I(o)})),I(t.conversation.getItems())}})()}),[i,N]);return{items:h,realtimeEvents:R,isConnected:S,canPushToTalk:y,isRecording:T,clientCanvasRef:v,serverCanvasRef:w,eventsScrollRef:g,formatTime:V,connectConversation:j,disconnectConversation:F,deleteConversationItem:B,startRecording:async()=>{A(!0);const e=f.current,t=l.current,n=d.current,r=await n.interrupt();if(r?.trackId){const{trackId:t,offset:n}=r;await e.cancelResponse(t,n)}await t.record((t=>e.appendInputAudio(t.mono)))},stopRecording:async()=>{A(!1);const e=f.current,t=l.current;await t.pause(),e.createResponse()},changeTurnEndType:U,recorderLevel:P,streamLevel:k,setUsername:M,updateInstructions:e=>{const t=f.current,n=e.replace("{{name}}",E);t.updateSession({instructions:n}),x(e),console.log(`Voice instructions updated: ${n.substring(0,50)}...`)}}};
+"use strict";Object.defineProperty(exports,"__esModule",{value:!0});var e=require("../node_modules/@openai/realtime-api-beta/lib/client.js"),t=require("react"),n=require("../utils/wav_renderer.js");require("../lib/wavtools/lib/wav_packer.js"),require("../lib/wavtools/lib/analysis/audio_analysis.js");var r=require("../lib/wavtools/lib/wav_stream_player.js"),a=require("../lib/wavtools/lib/wav_recorder.js"),o=require("../config/index.js");exports.default=(s,i)=>{if(!o.getConfigValue("APP_CONFIG"))return void console.warn("APP_CONFIG is not set. Voice AI will not be available.");if(!o.getConfigValue("APP_BUTTONS"))return void console.warn("APP_BUTTONS is not set. Voice AI will not be available.");const c=o.getConfigValue("APP_CONFIG"),u=o.getConfigValue("APP_BUTTONS"),l=t.useRef(new a.WavRecorder({sampleRate:24e3})),d=t.useRef(new r.WavStreamPlayer({sampleRate:24e3})),f=t.useRef(new e.RealtimeClient({apiKey:s,dangerouslyAllowAPIKeyInBrowser:!0,url:"wss://codicent-ai-sweden.openai.azure.com/openai/realtime?api-version=2024-10-01-preview&deployment=gpt-4o-realtime-preview-2&api-key=9013365c13ee4f3dad0016fb0f0c450a"})),p=t.useRef(!1),v=t.useRef(null),w=t.useRef(null),g=t.useRef(null),m=t.useRef((new Date).toISOString()),[S,h]=t.useState([]),[I,R]=t.useState([]),[C,_]=t.useState(!1),[y,b]=t.useState(!1),[T,A]=t.useState(!1),[O,P]=t.useState(0),[k,q]=t.useState(0),[E,M]=t.useState(""),[N,x]=t.useState("en-US"),[V,j]=t.useState((()=>u&&c.apps[u]?c.apps[u].voiceInstructions||c.REALTIME_VOICE_INSTRUCTIONS||"":c.REALTIME_VOICE_INSTRUCTIONS||"")),F=t.useCallback((e=>{const t=m.current,n=new Date(t).valueOf(),r=new Date(e).valueOf()-n,a=Math.floor(r/10)%100,o=Math.floor(r/1e3)%60,s=e=>{let t=e+"";for(;t.length<2;)t="0"+t;return t};return`${s(Math.floor(r/6e4)%60)}:${s(o)}.${s(a)}`}),[]),U=t.useCallback((async()=>{const e=f.current,t=l.current,n=d.current;m.current=(new Date).toISOString(),_(!0),R([]),h(e.conversation.getItems()),W("server_vad"),await t.begin(),await n.connect(),await e.connect(),e.sendUserMessageContent([{type:"input_text",text:"Hello!"}]),"server_vad"===e.getTurnDetectionType()&&await t.record((t=>e.appendInputAudio(t.mono)))}),[]),B=t.useCallback((async()=>{_(!1),R([]),h([]);const e=f.current,t=l.current;await t.end(),e.disconnect();const n=d.current;await n.interrupt()}),[]),D=t.useCallback((async e=>{f.current.deleteItem(e)}),[]),W=async e=>{const t=f.current,n=l.current;"none"===e&&"recording"===n.getStatus()&&await n.pause(),t.updateSession({turn_detection:"none"===e?null:{type:"server_vad"}}),"server_vad"===e&&t.isConnected()&&await n.record((e=>t.appendInputAudio(e.mono))),b("none"===e)};t.useEffect((()=>{if(E){f.current.updateSession({instructions:V.replace("{{name}}",E).replace("{{language}}",N).replace("{{time}}",(new Date).toISOString())})}}),[V,E]),t.useEffect((()=>{let e=!0;const t=l.current,r=v.current;let a=null;const o=d.current,s=w.current;let i=null;const c=()=>{if(e){if(r&&(r.width&&r.height||(r.width=r.offsetWidth,r.height=r.offsetHeight),a=a||r.getContext("2d"),a)){a.clearRect(0,0,r.width,r.height);const e=t.recording?t.getFrequencies("voice"):{values:new Float32Array([0])},o=1-Math.max(...e.values);P(o),n.WavRenderer.drawCircularBars(r,a,e.values,"#0099ff",20,0,8)}if(s&&(s.width&&s.height||(s.width=s.offsetWidth,s.height=s.offsetHeight),i=i||s.getContext("2d"),i)){i.clearRect(0,0,s.width,s.height);const e=o.analyser?o.getFrequencies("voice"):{values:new Float32Array([0])},t=1-Math.max(...e.values);q(t),n.WavRenderer.drawCircularBars(s,i,e.values,"#009900",20,0,8)}window.requestAnimationFrame(c)}};return c(),()=>{e=!1}}),[]),t.useEffect((()=>{i&&V&&(()=>{if(!p.current){p.current=!0;const e=d.current,t=f.current;t.updateSession({instructions:V}),t.updateSession({input_audio_transcription:{model:"whisper-1"}});for(const e of i)t.addTool(e.definition,e.handler);t.on("error",(e=>console.error(e))),t.on("conversation.interrupted",(async()=>{const n=await e.interrupt();if(n?.trackId){const{trackId:e,offset:r}=n;await t.cancelResponse(e,r)}})),t.on("conversation.updated",(async({item:n,delta:r})=>{const o=t.conversation.getItems();if(r?.audio&&e.add16BitPCM(r.audio,n.id),"completed"===n.status&&n.formatted.audio?.length){const e=await a.WavRecorder.decode(n.formatted.audio,24e3,24e3);n.formatted.file=e}h(o)})),h(t.conversation.getItems())}})()}),[i,V]);return{items:S,realtimeEvents:I,isConnected:C,canPushToTalk:y,isRecording:T,clientCanvasRef:v,serverCanvasRef:w,eventsScrollRef:g,formatTime:F,connectConversation:U,disconnectConversation:B,deleteConversationItem:D,startRecording:async()=>{A(!0);const e=f.current,t=l.current,n=d.current,r=await n.interrupt();if(r?.trackId){const{trackId:t,offset:n}=r;await e.cancelResponse(t,n)}await t.record((t=>e.appendInputAudio(t.mono)))},stopRecording:async()=>{A(!1);const e=f.current,t=l.current;await t.pause(),e.createResponse()},changeTurnEndType:W,recorderLevel:O,streamLevel:k,setUsername:M,updateInstructions:e=>{const t=f.current,n=e.replace("{{name}}",E).replace("{{language}}",N).replace("{{time}}",(new Date).toISOString());t.updateSession({instructions:n}),j(e),console.log(`Voice instructions updated: ${n.substring(0,50)}...`)},setLanguage:x}};

package/dist/cjs/hooks/useTools.d.ts

@@ -1,5 +1,5 @@
 import { CodicentService } from "../services/codicent";
-declare const useTools: (api: CodicentService, setHtml: ((html: string | undefined) => void) | undefined, injectScript: ((script: string) => void) | undefined, beep: HTMLAudioElement, html?: string) => {
+declare const useTools: (api: CodicentService, setHtml: ((html: string | undefined) => void) | undefined, injectScript: ((script: string) => void) | undefined, beep: HTMLAudioElement, html?: string, stopVoiceSession?: () => void) => {
     definition: any;
     handler: any;
 }[];

package/dist/cjs/hooks/useTools.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"useTools.d.ts","sourceRoot":"","sources":["../../../src/hooks/useTools.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAC;AAIvD,QAAA,MAAM,QAAQ,QACP,eAAe,YACX,CAAC,IAAI,EAAE,MAAM,GAAG,SAAS,KAAK,IAAI,8BAG7B,CAAC,MAAM,EAAE,MAAM,KAAK,IAAI,qBAGhC,gBAAgB,SACf,MAAM;gBAGgB,GAAG;aAAW,GAAG;GAgjB/C,CAAC;AAEF,eAAe,QAAQ,CAAC"}
+{"version":3,"file":"useTools.d.ts","sourceRoot":"","sources":["../../../src/hooks/useTools.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAC;AAIvD,QAAA,MAAM,QAAQ,QACP,eAAe,YACX,CAAC,IAAI,EAAE,MAAM,GAAG,SAAS,KAAK,IAAI,8BAG7B,CAAC,MAAM,EAAE,MAAM,KAAK,IAAI,qBAGhC,gBAAgB,SACf,MAAM,qBACK,MAAM,IAAI;gBAKC,GAAG;aAAW,GAAG;GA+jB/C,CAAC;AAEF,eAAe,QAAQ,CAAC"}

package/dist/cjs/hooks/useTools.js

@@ -1 +1 @@
-"use strict";Object.defineProperty(exports,"__esModule",{value:!0});var e=require("react"),t=require("../config/index.js");exports.default=(s,a=e=>{console.log(e)},r=e=>{console.log(e)},n,i)=>e.useMemo((()=>{const e=[],o=(t,s)=>{e.push({definition:t,handler:s})},p=async()=>{const e=t.getConfigValue("APP_CONFIG"),a=t.getConfigValue("APP_BUTTONS");return[...a&&e.apps[a]?e.apps[a].tasks:[],...await s.getAppTasks()]};return o({name:"inject_script",description:"Injects a script into the html/canvas view.",parameters:{type:"object",properties:{script:{type:"string",description:"The script to inject"}},required:["script"]}},(async({script:e})=>(console.log("TOOL: inject_script",e),r(e),{ok:!0}))),o({name:"post_codicent_message",description:"Posts a message to Codicent, only when requested by the user.",parameters:{type:"object",properties:{content:{type:"string",description:'The codicent message text, e.g. "remember to call Linda"'},parent_id:{type:"string",description:'The parent message id of a previous message that you will add a new revision to, by setting parent_id to the origin message being "edited"'}},required:["content"]}},(async({content:e,parent_id:t})=>await s.sendMessage(e,t))),o({name:"update_codicent_message",description:"Posts an edited message, which replaces the old message.",parameters:{type:"object",properties:{content:{type:"string",description:'The updated codicent message text, e.g. "remember to call Johan", usually including the tags from the original message.'},parent_id:{type:"string",description:"The message id of a previous/old message to be updated."}},required:["content","parent_id"]}},(async({content:e,parent_id:t})=>await s.sendMessage(e,t))),o({name:"get_messages_list",description:"Gets messages tagged with given tag name, optionally limited by num_messages. The messages are listed in chronological order, latest first.",parameters:{type:"object",properties:{num_messages:{type:"integer",description:"The number of matching messages to return"},search_text:{type:"string",description:"Search text to filter messages by"},tag_name:{type:"string",description:"Name of tag to find messages by"},after_timestamp:{type:"string",description:"Only return messages after this timestamp"},before_timestamp:{type:"string",description:"Only return messages before this timestamp"}},required:["num_messages"]}},(async({num_messages:e,search_text:t,tag_name:a,after_timestamp:r,before_timestamp:n})=>{let i=await s.getMessages(a?[a]:[]);return t&&(i=i.filter((e=>e.content.includes(t)))),r&&(i=i.filter((e=>e.createdAt>r))),n&&(i=i.filter((e=>e.createdAt<n))),i.slice(0,e)})),o({name:"chat_with_codicent_ai",description:"Sends a message to Codicent AI and gets a response message back. Remember that you can always ask codicent via this tool to find out more about what you can achieve from chatting with the codicent AI.",parameters:{type:"object",properties:{message:{type:"string",description:"Text of message to the codicent"}},required:["message"]}},(async({message:e})=>await s.chat(e))),o({name:"create_todo",description:"Saves a todo message in Codicent.",parameters:{type:"object",properties:{text:{type:"string",description:"Text of todo"}},required:["text"]}},(async({text:e})=>await s.sendMessage("#todo "+e))),o({name:"place_order",description:"Places and order with the Codicent AI.",parameters:{type:"object",properties:{message:{type:"string",description:"The text message of the order as spoken by the user."}},required:["message"]}},(async({message:e})=>await s.sendMessage(`#do Jag kommer ge dig en text (nedan) på vad jag ska beställa. Använd bara den text du får av mig, inga tools.\n    Du ska skapa en tabell med detaljerad information inklusive lagerstatus \n    för varje artikel i beställningen. Svara först när du har all info på plats. Ställ inga frågor på vägen utan använd det du får av mig.\n    \n    ----\n    EXEMPEL PÅ SVAR:\n    ----\n    Här är en tabell med detaljerad information inklusive lagerstatus för varje artikel i beställningen:\n    \n    Artikel\tHyresobjekt\tNamn\tLagertyp\tGrupp\tNamn.1\tSRA/SBEF\tLeverantör\tLevnr\tDepå\tLager\tReserverat\tUthyrt nu\tTillgängligt nu\tMärke/typ\tTillverkarnr\tTillv.år\tInköpsdato\tInköpspris\tVikt\n    Elverk\t12345\tElverk 1\tHyresobjekt\tMaskiner\tElverk\tSRA\tLeverantör A\t001\tDepå 1\t5\t0\t1\t4\tHonda EU22i\t123456\t2020\t2020-01-01\t10000 SEK\t21 kg\n    Elverk\t12345\tElverk 2\tHyresobjekt\tMaskiner\tElverk\tSRA\tLeverantör B\t002\tDepå 2\t3\t0\t1\t2\tYamaha EF2000iS\t654321\t2019\t2019-05-01\t9000 SEK\t20 kg\n    Observera att denna tabell är baserad på den information som finns tillgänglig i lagerstatusen. Om du behöver ytterligare detaljer eller om något specifikt saknas, vänligen meddela mig.\n    ----\n    \n    --- här följer texten på vad jag vill beställa ---\n    \n    \n    ${e}`))),o({name:"remember_info",description:"Saves info in your internal memory for future use. Use this tool to save insights on how to perform our conversations and tasks related to them.",parameters:{type:"object",properties:{message:{type:"string",description:"Your insights to improve our future conversations."}},required:["message"]}},(async({message:e})=>await s.sendMessage(`#index #insights\n\n${e}`))),o({name:"save_to_crm",description:"Saves CRM related info to your memory (logbook).",parameters:{type:"object",properties:{message:{type:"string",description:"CRM log entry description/contents."},company:{type:"string",description:"Company name for CRM log entry."},contact:{type:"string",description:"Contact name for CRM log entry."}},required:["message","company"]}},(async({message:e,company:t,contact:a})=>await s.sendMessage(`#crm\n\nFöretag: ${t}\n\nInfo: ${e}${a?"\n\nKontakt: "+a:""}`))),o({name:"set_page_html",description:"Sets the innerHtml of the div covering the full app page. You can display any html/javascript/style/mermaid content here for the user to see ;-). This view is usually referred to as the canvas, or tavlan in Swedish.",parameters:{type:"object",properties:{html:{type:"string",description:"The inner html to set the page to."}},required:["html"]}},(async({html:e})=>(a(e),n.play(),{ok:!0}))),o({name:"hide_page_html",description:"Hides the inner html of the page.",parameters:{type:"object",properties:{}}},(async()=>(a(void 0),{ok:!0}))),o({name:"get_page_html",description:"Gets the inner html of the page.",parameters:{type:"object",properties:{}}},(async()=>({html:i}))),o({name:"get_tasks",description:"Gets the list of possible tasks that can be performed. This lists task names and their id:s. Use get_task to get the task instructions given a task id.",parameters:{type:"object",properties:{},required:[]}},(async e=>(await p()).map((e=>({id:e.id,name:e.title}))))),o({name:"get_task",description:"Gets the task instructions for a given task id. Use get_tasks to get a list of possible tasks (with their id:s).",parameters:{type:"object",properties:{task_id:{type:"string",description:"The task id to get instructions for."}},required:["task_id"]}},(async({task_id:e})=>{const t=(await p()).find((t=>t.id===e));return t?t.content:"Task not found"})),e}),[s,a,r,i,n]);
+"use strict";Object.defineProperty(exports,"__esModule",{value:!0});var e=require("react"),t=require("../config/index.js");exports.default=(s,a=e=>{console.log(e)},n=e=>{console.log(e)},r,i,o=()=>{console.log("stopVoiceSession")})=>e.useMemo((()=>{const e=[],p=(t,s)=>{e.push({definition:t,handler:s})},c=async()=>{const e=t.getConfigValue("APP_CONFIG"),a=t.getConfigValue("APP_BUTTONS");return[...a&&e.apps[a]?e.apps[a].tasks:[],...await s.getAppTasks()]};return p({name:"stop_voice_session",description:"Stops the voice session.",parameters:{type:"object",properties:{}}},(async()=>(console.log("TOOL: stop_voice_session"),a(void 0),o(),{ok:!0}))),p({name:"inject_script",description:"Injects a script into the html/canvas view.",parameters:{type:"object",properties:{script:{type:"string",description:"The script to inject"}},required:["script"]}},(async({script:e})=>(console.log("TOOL: inject_script",e),n(e),{ok:!0}))),p({name:"post_codicent_message",description:"Posts a message to Codicent, only when requested by the user.",parameters:{type:"object",properties:{content:{type:"string",description:'The codicent message text, e.g. "remember to call Linda"'},parent_id:{type:"string",description:'The parent message id of a previous message that you will add a new revision to, by setting parent_id to the origin message being "edited"'}},required:["content"]}},(async({content:e,parent_id:t})=>await s.sendMessage(e,t))),p({name:"update_codicent_message",description:"Posts an edited message, which replaces the old message.",parameters:{type:"object",properties:{content:{type:"string",description:'The updated codicent message text, e.g. "remember to call Johan", usually including the tags from the original message.'},parent_id:{type:"string",description:"The message id of a previous/old message to be updated."}},required:["content","parent_id"]}},(async({content:e,parent_id:t})=>await s.sendMessage(e,t))),p({name:"get_messages_list",description:"Gets messages tagged with given tag name, optionally limited by num_messages. The messages are listed in chronological order, latest first.",parameters:{type:"object",properties:{num_messages:{type:"integer",description:"The number of matching messages to return"},search_text:{type:"string",description:"Search text to filter messages by"},tag_name:{type:"string",description:"Name of tag to find messages by"},after_timestamp:{type:"string",description:"Only return messages after this timestamp"},before_timestamp:{type:"string",description:"Only return messages before this timestamp"}},required:["num_messages"]}},(async({num_messages:e,search_text:t,tag_name:a,after_timestamp:n,before_timestamp:r})=>{let i=await s.getMessages(a?[a]:[]);return t&&(i=i.filter((e=>e.content.includes(t)))),n&&(i=i.filter((e=>e.createdAt>n))),r&&(i=i.filter((e=>e.createdAt<r))),i.slice(0,e)})),p({name:"chat_with_codicent_ai",description:"Sends a message to Codicent AI and gets a response message back. Remember that you can always ask codicent via this tool to find out more about what you can achieve from chatting with the codicent AI.",parameters:{type:"object",properties:{message:{type:"string",description:"Text of message to the codicent"}},required:["message"]}},(async({message:e})=>await s.chat(e))),p({name:"create_todo",description:"Saves a todo message in Codicent.",parameters:{type:"object",properties:{text:{type:"string",description:"Text of todo"}},required:["text"]}},(async({text:e})=>await s.sendMessage("#todo "+e))),p({name:"place_order",description:"Places and order with the Codicent AI.",parameters:{type:"object",properties:{message:{type:"string",description:"The text message of the order as spoken by the user."}},required:["message"]}},(async({message:e})=>await s.sendMessage(`#do Jag kommer ge dig en text (nedan) på vad jag ska beställa. Använd bara den text du får av mig, inga tools.\n    Du ska skapa en tabell med detaljerad information inklusive lagerstatus \n    för varje artikel i beställningen. Svara först när du har all info på plats. Ställ inga frågor på vägen utan använd det du får av mig.\n    \n    ----\n    EXEMPEL PÅ SVAR:\n    ----\n    Här är en tabell med detaljerad information inklusive lagerstatus för varje artikel i beställningen:\n    \n    Artikel\tHyresobjekt\tNamn\tLagertyp\tGrupp\tNamn.1\tSRA/SBEF\tLeverantör\tLevnr\tDepå\tLager\tReserverat\tUthyrt nu\tTillgängligt nu\tMärke/typ\tTillverkarnr\tTillv.år\tInköpsdato\tInköpspris\tVikt\n    Elverk\t12345\tElverk 1\tHyresobjekt\tMaskiner\tElverk\tSRA\tLeverantör A\t001\tDepå 1\t5\t0\t1\t4\tHonda EU22i\t123456\t2020\t2020-01-01\t10000 SEK\t21 kg\n    Elverk\t12345\tElverk 2\tHyresobjekt\tMaskiner\tElverk\tSRA\tLeverantör B\t002\tDepå 2\t3\t0\t1\t2\tYamaha EF2000iS\t654321\t2019\t2019-05-01\t9000 SEK\t20 kg\n    Observera att denna tabell är baserad på den information som finns tillgänglig i lagerstatusen. Om du behöver ytterligare detaljer eller om något specifikt saknas, vänligen meddela mig.\n    ----\n    \n    --- här följer texten på vad jag vill beställa ---\n    \n    \n    ${e}`))),p({name:"remember_info",description:"Saves info in your internal memory for future use. Use this tool to save insights on how to perform our conversations and tasks related to them.",parameters:{type:"object",properties:{message:{type:"string",description:"Your insights to improve our future conversations."}},required:["message"]}},(async({message:e})=>await s.sendMessage(`#index #insights\n\n${e}`))),p({name:"save_to_crm",description:"Saves CRM related info to your memory (logbook).",parameters:{type:"object",properties:{message:{type:"string",description:"CRM log entry description/contents."},company:{type:"string",description:"Company name for CRM log entry."},contact:{type:"string",description:"Contact name for CRM log entry."}},required:["message","company"]}},(async({message:e,company:t,contact:a})=>await s.sendMessage(`#crm\n\nFöretag: ${t}\n\nInfo: ${e}${a?"\n\nKontakt: "+a:""}`))),p({name:"set_page_html",description:"Sets the innerHtml of the div covering the full app page. You can display any html/javascript/style/mermaid content here for the user to see ;-). This view is usually referred to as the canvas, or tavlan in Swedish.",parameters:{type:"object",properties:{html:{type:"string",description:"The inner html to set the page to."}},required:["html"]}},(async({html:e})=>(a(e),r.play(),{ok:!0}))),p({name:"hide_page_html",description:"Hides the inner html of the page.",parameters:{type:"object",properties:{}}},(async()=>(a(void 0),{ok:!0}))),p({name:"get_page_html",description:"Gets the inner html of the page.",parameters:{type:"object",properties:{}}},(async()=>({html:i}))),p({name:"get_tasks",description:"Gets the list of possible tasks that can be performed. This lists task names and their id:s. Use get_task to get the task instructions given a task id.",parameters:{type:"object",properties:{},required:[]}},(async e=>(await c()).map((e=>({id:e.id,name:e.title}))))),p({name:"get_task",description:"Gets the task instructions for a given task id. Use get_tasks to get a list of possible tasks (with their id:s).",parameters:{type:"object",properties:{task_id:{type:"string",description:"The task id to get instructions for."}},required:["task_id"]}},(async({task_id:e})=>{const t=(await c()).find((t=>t.id===e));return t?t.content:"Task not found"})),e}),[s,a,n,i,r]);

package/dist/esm/hooks/useCodicentApp.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"useCodicentApp.d.ts","sourceRoot":"","sources":["../../../src/hooks/useCodicentApp.tsx"],"names":[],"mappings":"AACA,OAAO,EAAE,qBAAqB,EAAE,MAAM,oBAAoB,CAAC;AAC3D,OAAO,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAC9C,OAAO,EAAE,eAAe,EAAE,YAAY,EAAE,kBAAkB,EAAyB,MAAM,UAAU,CAAC;AACpG,OAAyB,EAAE,kBAAkB,EAAE,MAAM,oBAAoB,CAAC;AAC1E,OAAqB,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC5D,OAA2B,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AAIzE,MAAM,WAAW,gBAAgB;IAC/B,KAAK,EAAE,kBAAkB,CAAC;IAC1B,IAAI,EAAE,YAAY,CAAC;IACnB,YAAY,EAAE,eAAe,CAAC;IAC9B,OAAO,EAAE,eAAe,CAAC;IACzB,OAAO,EAAE,YAAY,CAAC;IACtB,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,MAAM,CAAC;IACnC,YAAY,EAAE,CAAC,KAAK,EAAE,OAAO,KAAK,IAAI,CAAC;IACvC,SAAS,EAAE,kBAAkB,CAAC;IAC9B,MAAM,EAAE,MAAM,OAAO,CAAC;IACtB,KAAK,EAAE,aAAa,CAAC;IACrB,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,CAAC;IAChC,MAAM,EAAE,MAAM,CAAC;IACf,gBAAgB,EAAE,MAAM,CAAC;CAC1B;AAED,eAAO,MAAM,cAAc,cAAa;IAAC,KAAK,EAAE,qBAAqB,CAAA;CAAC,KA0F/D,gBACN,CAAC"}
+{"version":3,"file":"useCodicentApp.d.ts","sourceRoot":"","sources":["../../../src/hooks/useCodicentApp.tsx"],"names":[],"mappings":"AACA,OAAO,EAAE,qBAAqB,EAAE,MAAM,oBAAoB,CAAC;AAC3D,OAAO,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAC9C,OAAO,EAAE,eAAe,EAAE,YAAY,EAAE,kBAAkB,EAAyB,MAAM,UAAU,CAAC;AACpG,OAAyB,EAAE,kBAAkB,EAAE,MAAM,oBAAoB,CAAC;AAC1E,OAAqB,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC5D,OAA2B,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AAIzE,MAAM,WAAW,gBAAgB;IAC/B,KAAK,EAAE,kBAAkB,CAAC;IAC1B,IAAI,EAAE,YAAY,CAAC;IACnB,YAAY,EAAE,eAAe,CAAC;IAC9B,OAAO,EAAE,eAAe,CAAC;IACzB,OAAO,EAAE,YAAY,CAAC;IACtB,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,MAAM,CAAC;IACnC,YAAY,EAAE,CAAC,KAAK,EAAE,OAAO,KAAK,IAAI,CAAC;IACvC,SAAS,EAAE,kBAAkB,CAAC;IAC9B,MAAM,EAAE,MAAM,OAAO,CAAC;IACtB,KAAK,EAAE,aAAa,CAAC;IACrB,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,CAAC;IAChC,MAAM,EAAE,MAAM,CAAC;IACf,gBAAgB,EAAE,MAAM,CAAC;CAC1B;AAED,eAAO,MAAM,cAAc,cAAe;IAAE,KAAK,EAAE,qBAAqB,CAAA;CAAE,KAkGnE,gBACN,CAAC"}

package/dist/esm/hooks/useCodicentApp.js

@@ -1 +1 @@
-import{useRef as e,useState as t,useCallback as r,useMemo as n,useEffect as s}from"react";import"../utils/MessageContent.js";import"../node_modules/tinycolor2/esm/tinycolor.js";import"react/jsx-runtime";import{createAppStateMachine as o}from"../utils/appState.js";import"../_virtual/index.js";import"../lib/wavtools/lib/wav_packer.js";import"../lib/wavtools/lib/analysis/audio_analysis.js";import"../lib/wavtools/lib/wav_stream_player.js";import"../lib/wavtools/lib/wav_recorder.js";import{useAudioRecorder as a}from"./useAudioRecorder.js";import i from"./useAuthState.js";import c from"./useRealtimeVoiceAI.js";import u from"./useTools.js";import{getConfigValue as m}from"../config/index.js";const l=({auth0:l})=>{const p=i(l),d=e(o(p.codicentService,(()=>{}))),[k,A]=t(),{service:h,context:j}=d.current,{nickname:v,name:f,error:w,errorType:b,selectedApp:y}=j,x=r(((e,t)=>{const r=e.get("file").name;h.uploadFile(r,e).then((e=>{h.sendMessage(`#transcribe ${""!==t?(t.startsWith("#")?"":"#")+t+" ":""}#file:${e}`).then((()=>{})).catch(console.warn)}))}),[h]),S=a(x),[_,T]=t(),[C,I]=t(),g=n((()=>new Audio("./notification.mp3")),[]),N=u(h,T,I,g,_),E=c(m("REALTIME_VOICE_API_KEY")||"",N);s((()=>{p.isAuthenticated&&p.user&&p.accessToken&&d.current.updateContext({isAuthenticated:p.isAuthenticated,user:{sub:p.user.sub,email:p.user.email},accessToken:p.accessToken})}),[p.isAuthenticated,p.user,p.accessToken]),s((()=>d.current.updateContext({nickname:p.nickname})),[p.nickname]),s((()=>{d.current.update();let e=d.current;const t=setInterval((()=>{"hasAccess"===d.current.currentStateName&&e.currentStateName===d.current.currentStateName||(d.current.update(),A(d.current.getCurrentStateName()),e=d.current)}),50);return()=>clearInterval(t)}),[]);return{audio:S,auth:p,currentStateName:k,stateMachine:d.current,service:h,context:j,state:k,nickname:v,name:f,error:w,errorType:b,fixAppUrl:e=>e.replace("{nickname}",v||"unknown").replace("{token}",encodeURIComponent(p.accessToken||"unknown")).replace("{app}",y||"unknown"),setAnonymous:e=>{d.current.updateContext({isAnonymous:e})},allStates:d.current.allStates,isBusy:()=>d.current.isBusy(),voice:E,html:_,setHtml:T,script:C}};export{l as useCodicentApp};
+import{useRef as e,useState as t,useCallback as r,useMemo as n,useEffect as s}from"react";import"../utils/MessageContent.js";import"../node_modules/tinycolor2/esm/tinycolor.js";import"react/jsx-runtime";import{createAppStateMachine as o}from"../utils/appState.js";import"../_virtual/index.js";import"../lib/wavtools/lib/wav_packer.js";import"../lib/wavtools/lib/analysis/audio_analysis.js";import"../lib/wavtools/lib/wav_stream_player.js";import"../lib/wavtools/lib/wav_recorder.js";import{useAudioRecorder as a}from"./useAudioRecorder.js";import i from"./useAuthState.js";import c from"./useRealtimeVoiceAI.js";import u from"./useTools.js";import{getConfigValue as m}from"../config/index.js";const l=({auth0:l})=>{const p=i(l),d=e(o(p.codicentService,(()=>{}))),[k,v]=t(),{service:A,context:h}=d.current,{nickname:j,name:f,error:w,errorType:b,selectedApp:y}=h,x=r(((e,t)=>{const r=e.get("file").name;A.uploadFile(r,e).then((e=>{A.sendMessage(`#transcribe ${""!==t?(t.startsWith("#")?"":"#")+t+" ":""}#file:${e}`).then((()=>{})).catch(console.warn)}))}),[A]),S=a(x),[_,T]=t(),[C,I]=t(),[g,N]=t(!1),E=n((()=>new Audio("./notification.mp3")),[]),M=u(A,T,I,E,_,(()=>N(!0))),R=c(m("REALTIME_VOICE_API_KEY")||"",M);s((()=>{g&&(R?.disconnectConversation(),N(!1))}),[g,R]),s((()=>{p.isAuthenticated&&p.user&&p.accessToken&&d.current.updateContext({isAuthenticated:p.isAuthenticated,user:{sub:p.user.sub,email:p.user.email},accessToken:p.accessToken})}),[p.isAuthenticated,p.user,p.accessToken]),s((()=>d.current.updateContext({nickname:p.nickname})),[p.nickname]),s((()=>{d.current.update();let e=d.current;const t=setInterval((()=>{"hasAccess"===d.current.currentStateName&&e.currentStateName===d.current.currentStateName||(d.current.update(),v(d.current.getCurrentStateName()),e=d.current)}),50);return()=>clearInterval(t)}),[]);return{audio:S,auth:p,currentStateName:k,stateMachine:d.current,service:A,context:h,state:k,nickname:j,name:f,error:w,errorType:b,fixAppUrl:e=>e.replace("{nickname}",j||"unknown").replace("{token}",encodeURIComponent(p.accessToken||"unknown")).replace("{app}",y||"unknown"),setAnonymous:e=>{d.current.updateContext({isAnonymous:e})},allStates:d.current.allStates,isBusy:()=>d.current.isBusy(),voice:R,html:_,setHtml:T,script:C}};export{l as useCodicentApp};

package/dist/esm/hooks/useRealtimeVoiceAI.d.ts

@@ -30,6 +30,7 @@ export interface RealtimeVoice {
     streamLevel: number;
     setUsername: (username: string) => void;
     updateInstructions: (instructions: string) => void;
+    setLanguage: (language: string) => void;
 }
 declare const useRealtimeVoiceAI: (apiKey: string, tools: {
     definition: ToolDefinitionType;

package/dist/esm/hooks/useRealtimeVoiceAI.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"useRealtimeVoiceAI.d.ts","sourceRoot":"","sources":["../../../src/hooks/useRealtimeVoiceAI.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,QAAQ,EAAE,kBAAkB,EAAE,MAAM,2CAA2C,CAAC;AAMzF;;GAEG;AACH,UAAU,aAAa;IACrB,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,QAAQ,GAAG,QAAQ,CAAC;IAC5B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,KAAK,EAAE;QAAE,CAAC,GAAG,EAAE,MAAM,GAAG,GAAG,CAAA;KAAE,CAAC;CAC/B;AAED,MAAM,WAAW,aAAa;IAC5B,KAAK,EAAE,QAAQ,EAAE,CAAC;IAClB,cAAc,EAAE,aAAa,EAAE,CAAC;IAChC,WAAW,EAAE,OAAO,CAAC;IACrB,aAAa,EAAE,OAAO,CAAC;IACvB,WAAW,EAAE,OAAO,CAAC;IACrB,eAAe,EAAE,KAAK,CAAC,SAAS,CAAC,iBAAiB,CAAC,CAAC;IACpD,eAAe,EAAE,KAAK,CAAC,SAAS,CAAC,iBAAiB,CAAC,CAAC;IACpD,eAAe,EAAE,KAAK,CAAC,SAAS,CAAC,cAAc,CAAC,CAAC;IACjD,UAAU,EAAE,CAAC,SAAS,EAAE,MAAM,KAAK,MAAM,CAAC;IAC1C,mBAAmB,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IACzC,sBAAsB,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IAC5C,sBAAsB,EAAE,CAAC,EAAE,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IACtD,cAAc,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IACpC,aAAa,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IACnC,iBAAiB,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IACpD,aAAa,EAAE,MAAM,CAAC;IACtB,WAAW,EAAE,MAAM,CAAC;IACpB,WAAW,EAAE,CAAC,QAAQ,EAAE,MAAM,KAAK,IAAI,CAAC;IACxC,kBAAkB,EAAE,CAAC,YAAY,EAAE,MAAM,KAAK,IAAI,CAAC;CACpD;AAED,QAAA,MAAM,kBAAkB,WACd,MAAM,SACP;IAAE,UAAU,EAAE,kBAAkB,CAAC;IAAC,OAAO,EAAE,QAAQ,CAAA;CAAE,EAAE,8BA4W/D,CAAC;AAEF,eAAe,kBAAkB,CAAC"}
+{"version":3,"file":"useRealtimeVoiceAI.d.ts","sourceRoot":"","sources":["../../../src/hooks/useRealtimeVoiceAI.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,QAAQ,EAAE,kBAAkB,EAAE,MAAM,2CAA2C,CAAC;AAMzF;;GAEG;AACH,UAAU,aAAa;IACrB,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,QAAQ,GAAG,QAAQ,CAAC;IAC5B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,KAAK,EAAE;QAAE,CAAC,GAAG,EAAE,MAAM,GAAG,GAAG,CAAA;KAAE,CAAC;CAC/B;AAED,MAAM,WAAW,aAAa;IAC5B,KAAK,EAAE,QAAQ,EAAE,CAAC;IAClB,cAAc,EAAE,aAAa,EAAE,CAAC;IAChC,WAAW,EAAE,OAAO,CAAC;IACrB,aAAa,EAAE,OAAO,CAAC;IACvB,WAAW,EAAE,OAAO,CAAC;IACrB,eAAe,EAAE,KAAK,CAAC,SAAS,CAAC,iBAAiB,CAAC,CAAC;IACpD,eAAe,EAAE,KAAK,CAAC,SAAS,CAAC,iBAAiB,CAAC,CAAC;IACpD,eAAe,EAAE,KAAK,CAAC,SAAS,CAAC,cAAc,CAAC,CAAC;IACjD,UAAU,EAAE,CAAC,SAAS,EAAE,MAAM,KAAK,MAAM,CAAC;IAC1C,mBAAmB,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IACzC,sBAAsB,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IAC5C,sBAAsB,EAAE,CAAC,EAAE,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IACtD,cAAc,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IACpC,aAAa,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IACnC,iBAAiB,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IACpD,aAAa,EAAE,MAAM,CAAC;IACtB,WAAW,EAAE,MAAM,CAAC;IACpB,WAAW,EAAE,CAAC,QAAQ,EAAE,MAAM,KAAK,IAAI,CAAC;IACxC,kBAAkB,EAAE,CAAC,YAAY,EAAE,MAAM,KAAK,IAAI,CAAC;IACnD,WAAW,EAAE,CAAC,QAAQ,EAAE,MAAM,KAAK,IAAI,CAAC;CACzC;AAED,QAAA,MAAM,kBAAkB,WAAY,MAAM,SAAS;IAAE,UAAU,EAAE,kBAAkB,CAAC;IAAC,OAAO,EAAE,QAAQ,CAAA;CAAE,EAAE,8BAmXzG,CAAC;AAEF,eAAe,kBAAkB,CAAC"}

package/dist/esm/hooks/useRealtimeVoiceAI.js

@@ -1 +1 @@
-import{RealtimeClient as e}from"../node_modules/@openai/realtime-api-beta/lib/client.js";import{useRef as t,useState as n,useCallback as r,useEffect as o}from"react";import{WavRenderer as a}from"../utils/wav_renderer.js";import"../lib/wavtools/lib/wav_packer.js";import"../lib/wavtools/lib/analysis/audio_analysis.js";import{WavStreamPlayer as i}from"../lib/wavtools/lib/wav_stream_player.js";import{WavRecorder as s}from"../lib/wavtools/lib/wav_recorder.js";import{getConfigValue as c}from"../config/index.js";const l=(l,u)=>{if(!c("APP_CONFIG"))return void console.warn("APP_CONFIG is not set. Voice AI will not be available.");if(!c("APP_BUTTONS"))return void console.warn("APP_BUTTONS is not set. Voice AI will not be available.");const d=c("APP_CONFIG"),p=c("APP_BUTTONS"),w=t(new s({sampleRate:24e3})),f=t(new i({sampleRate:24e3})),m=t(new e({apiKey:l,dangerouslyAllowAPIKeyInBrowser:!0,url:"wss://codicent-ai-sweden.openai.azure.com/openai/realtime?api-version=2024-10-01-preview&deployment=gpt-4o-realtime-preview-2&api-key=9013365c13ee4f3dad0016fb0f0c450a"})),v=t(!1),g=t(null),h=t(null),I=t(null),_=t((new Date).toISOString()),[y,C]=n([]),[T,A]=n([]),[S,b]=n(!1),[R,P]=n(!1),[O,N]=n(!1),[M,x]=n(0),[E,F]=n(0),[j,k]=n(""),[B,U]=n((()=>p&&d.apps[p]?d.apps[p].voiceInstructions||d.REALTIME_VOICE_INSTRUCTIONS||"":d.REALTIME_VOICE_INSTRUCTIONS||"")),D=r((e=>{const t=_.current,n=new Date(t).valueOf(),r=new Date(e).valueOf()-n,o=Math.floor(r/10)%100,a=Math.floor(r/1e3)%60,i=e=>{let t=e+"";for(;t.length<2;)t="0"+t;return t};return`${i(Math.floor(r/6e4)%60)}:${i(a)}.${i(o)}`}),[]),V=r((async()=>{const e=m.current,t=w.current,n=f.current;_.current=(new Date).toISOString(),b(!0),A([]),C(e.conversation.getItems()),q("server_vad"),await t.begin(),await n.connect(),await e.connect(),e.sendUserMessageContent([{type:"input_text",text:"Hello!"}]),"server_vad"===e.getTurnDetectionType()&&await t.record((t=>e.appendInputAudio(t.mono)))}),[]),L=r((async()=>{b(!1),A([]),C([]);const e=m.current,t=w.current;await t.end(),e.disconnect();const n=f.current;await n.interrupt()}),[]),$=r((async e=>{m.current.deleteItem(e)}),[]),q=async e=>{const t=m.current,n=w.current;"none"===e&&"recording"===n.getStatus()&&await n.pause(),t.updateSession({turn_detection:"none"===e?null:{type:"server_vad"}}),"server_vad"===e&&t.isConnected()&&await n.record((e=>t.appendInputAudio(e.mono))),P("none"===e)};o((()=>{if(j){m.current.updateSession({instructions:B.replace("{{name}}",j)})}}),[B,j]),o((()=>{let e=!0;const t=w.current,n=g.current;let r=null;const o=f.current,i=h.current;let s=null;const c=()=>{if(e){if(n&&(n.width&&n.height||(n.width=n.offsetWidth,n.height=n.offsetHeight),r=r||n.getContext("2d"),r)){r.clearRect(0,0,n.width,n.height);const e=t.recording?t.getFrequencies("voice"):{values:new Float32Array([0])},o=1-Math.max(...e.values);x(o),a.drawCircularBars(n,r,e.values,"#0099ff",20,0,8)}if(i&&(i.width&&i.height||(i.width=i.offsetWidth,i.height=i.offsetHeight),s=s||i.getContext("2d"),s)){s.clearRect(0,0,i.width,i.height);const e=o.analyser?o.getFrequencies("voice"):{values:new Float32Array([0])},t=1-Math.max(...e.values);F(t),a.drawCircularBars(i,s,e.values,"#009900",20,0,8)}window.requestAnimationFrame(c)}};return c(),()=>{e=!1}}),[]),o((()=>{u&&B&&(()=>{if(!v.current){v.current=!0;const e=f.current,t=m.current;t.updateSession({instructions:B}),t.updateSession({input_audio_transcription:{model:"whisper-1"}});for(const e of u)t.addTool(e.definition,e.handler);t.on("error",(e=>console.error(e))),t.on("conversation.interrupted",(async()=>{const n=await e.interrupt();if(n?.trackId){const{trackId:e,offset:r}=n;await t.cancelResponse(e,r)}})),t.on("conversation.updated",(async({item:n,delta:r})=>{const o=t.conversation.getItems();if(r?.audio&&e.add16BitPCM(r.audio,n.id),"completed"===n.status&&n.formatted.audio?.length){const e=await s.decode(n.formatted.audio,24e3,24e3);n.formatted.file=e}C(o)})),C(t.conversation.getItems())}})()}),[u,B]);return{items:y,realtimeEvents:T,isConnected:S,canPushToTalk:R,isRecording:O,clientCanvasRef:g,serverCanvasRef:h,eventsScrollRef:I,formatTime:D,connectConversation:V,disconnectConversation:L,deleteConversationItem:$,startRecording:async()=>{N(!0);const e=m.current,t=w.current,n=f.current,r=await n.interrupt();if(r?.trackId){const{trackId:t,offset:n}=r;await e.cancelResponse(t,n)}await t.record((t=>e.appendInputAudio(t.mono)))},stopRecording:async()=>{N(!1);const e=m.current,t=w.current;await t.pause(),e.createResponse()},changeTurnEndType:q,recorderLevel:M,streamLevel:E,setUsername:k,updateInstructions:e=>{const t=m.current,n=e.replace("{{name}}",j);t.updateSession({instructions:n}),U(e),console.log(`Voice instructions updated: ${n.substring(0,50)}...`)}}};export{l as default};
+import{RealtimeClient as e}from"../node_modules/@openai/realtime-api-beta/lib/client.js";import{useRef as t,useState as n,useCallback as r,useEffect as a}from"react";import{WavRenderer as o}from"../utils/wav_renderer.js";import"../lib/wavtools/lib/wav_packer.js";import"../lib/wavtools/lib/analysis/audio_analysis.js";import{WavStreamPlayer as i}from"../lib/wavtools/lib/wav_stream_player.js";import{WavRecorder as s}from"../lib/wavtools/lib/wav_recorder.js";import{getConfigValue as c}from"../config/index.js";const l=(l,u)=>{if(!c("APP_CONFIG"))return void console.warn("APP_CONFIG is not set. Voice AI will not be available.");if(!c("APP_BUTTONS"))return void console.warn("APP_BUTTONS is not set. Voice AI will not be available.");const d=c("APP_CONFIG"),p=c("APP_BUTTONS"),w=t(new s({sampleRate:24e3})),f=t(new i({sampleRate:24e3})),m=t(new e({apiKey:l,dangerouslyAllowAPIKeyInBrowser:!0,url:"wss://codicent-ai-sweden.openai.azure.com/openai/realtime?api-version=2024-10-01-preview&deployment=gpt-4o-realtime-preview-2&api-key=9013365c13ee4f3dad0016fb0f0c450a"})),v=t(!1),g=t(null),h=t(null),I=t(null),_=t((new Date).toISOString()),[y,S]=n([]),[C,T]=n([]),[A,b]=n(!1),[R,O]=n(!1),[P,N]=n(!1),[M,x]=n(0),[E,F]=n(0),[U,j]=n(""),[k,B]=n("en-US"),[D,L]=n((()=>p&&d.apps[p]?d.apps[p].voiceInstructions||d.REALTIME_VOICE_INSTRUCTIONS||"":d.REALTIME_VOICE_INSTRUCTIONS||"")),V=r((e=>{const t=_.current,n=new Date(t).valueOf(),r=new Date(e).valueOf()-n,a=Math.floor(r/10)%100,o=Math.floor(r/1e3)%60,i=e=>{let t=e+"";for(;t.length<2;)t="0"+t;return t};return`${i(Math.floor(r/6e4)%60)}:${i(o)}.${i(a)}`}),[]),$=r((async()=>{const e=m.current,t=w.current,n=f.current;_.current=(new Date).toISOString(),b(!0),T([]),S(e.conversation.getItems()),H("server_vad"),await t.begin(),await n.connect(),await e.connect(),e.sendUserMessageContent([{type:"input_text",text:"Hello!"}]),"server_vad"===e.getTurnDetectionType()&&await t.record((t=>e.appendInputAudio(t.mono)))}),[]),q=r((async()=>{b(!1),T([]),S([]);const e=m.current,t=w.current;await t.end(),e.disconnect();const n=f.current;await n.interrupt()}),[]),G=r((async e=>{m.current.deleteItem(e)}),[]),H=async e=>{const t=m.current,n=w.current;"none"===e&&"recording"===n.getStatus()&&await n.pause(),t.updateSession({turn_detection:"none"===e?null:{type:"server_vad"}}),"server_vad"===e&&t.isConnected()&&await n.record((e=>t.appendInputAudio(e.mono))),O("none"===e)};a((()=>{if(U){m.current.updateSession({instructions:D.replace("{{name}}",U).replace("{{language}}",k).replace("{{time}}",(new Date).toISOString())})}}),[D,U]),a((()=>{let e=!0;const t=w.current,n=g.current;let r=null;const a=f.current,i=h.current;let s=null;const c=()=>{if(e){if(n&&(n.width&&n.height||(n.width=n.offsetWidth,n.height=n.offsetHeight),r=r||n.getContext("2d"),r)){r.clearRect(0,0,n.width,n.height);const e=t.recording?t.getFrequencies("voice"):{values:new Float32Array([0])},a=1-Math.max(...e.values);x(a),o.drawCircularBars(n,r,e.values,"#0099ff",20,0,8)}if(i&&(i.width&&i.height||(i.width=i.offsetWidth,i.height=i.offsetHeight),s=s||i.getContext("2d"),s)){s.clearRect(0,0,i.width,i.height);const e=a.analyser?a.getFrequencies("voice"):{values:new Float32Array([0])},t=1-Math.max(...e.values);F(t),o.drawCircularBars(i,s,e.values,"#009900",20,0,8)}window.requestAnimationFrame(c)}};return c(),()=>{e=!1}}),[]),a((()=>{u&&D&&(()=>{if(!v.current){v.current=!0;const e=f.current,t=m.current;t.updateSession({instructions:D}),t.updateSession({input_audio_transcription:{model:"whisper-1"}});for(const e of u)t.addTool(e.definition,e.handler);t.on("error",(e=>console.error(e))),t.on("conversation.interrupted",(async()=>{const n=await e.interrupt();if(n?.trackId){const{trackId:e,offset:r}=n;await t.cancelResponse(e,r)}})),t.on("conversation.updated",(async({item:n,delta:r})=>{const a=t.conversation.getItems();if(r?.audio&&e.add16BitPCM(r.audio,n.id),"completed"===n.status&&n.formatted.audio?.length){const e=await s.decode(n.formatted.audio,24e3,24e3);n.formatted.file=e}S(a)})),S(t.conversation.getItems())}})()}),[u,D]);return{items:y,realtimeEvents:C,isConnected:A,canPushToTalk:R,isRecording:P,clientCanvasRef:g,serverCanvasRef:h,eventsScrollRef:I,formatTime:V,connectConversation:$,disconnectConversation:q,deleteConversationItem:G,startRecording:async()=>{N(!0);const e=m.current,t=w.current,n=f.current,r=await n.interrupt();if(r?.trackId){const{trackId:t,offset:n}=r;await e.cancelResponse(t,n)}await t.record((t=>e.appendInputAudio(t.mono)))},stopRecording:async()=>{N(!1);const e=m.current,t=w.current;await t.pause(),e.createResponse()},changeTurnEndType:H,recorderLevel:M,streamLevel:E,setUsername:j,updateInstructions:e=>{const t=m.current,n=e.replace("{{name}}",U).replace("{{language}}",k).replace("{{time}}",(new Date).toISOString());t.updateSession({instructions:n}),L(e),console.log(`Voice instructions updated: ${n.substring(0,50)}...`)},setLanguage:B}};export{l as default};

package/dist/esm/hooks/useTools.d.ts

@@ -1,5 +1,5 @@
 import { CodicentService } from "../services/codicent";
-declare const useTools: (api: CodicentService, setHtml: ((html: string | undefined) => void) | undefined, injectScript: ((script: string) => void) | undefined, beep: HTMLAudioElement, html?: string) => {
+declare const useTools: (api: CodicentService, setHtml: ((html: string | undefined) => void) | undefined, injectScript: ((script: string) => void) | undefined, beep: HTMLAudioElement, html?: string, stopVoiceSession?: () => void) => {
     definition: any;
     handler: any;
 }[];

package/dist/esm/hooks/useTools.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"useTools.d.ts","sourceRoot":"","sources":["../../../src/hooks/useTools.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAC;AAIvD,QAAA,MAAM,QAAQ,QACP,eAAe,YACX,CAAC,IAAI,EAAE,MAAM,GAAG,SAAS,KAAK,IAAI,8BAG7B,CAAC,MAAM,EAAE,MAAM,KAAK,IAAI,qBAGhC,gBAAgB,SACf,MAAM;gBAGgB,GAAG;aAAW,GAAG;GAgjB/C,CAAC;AAEF,eAAe,QAAQ,CAAC"}
+{"version":3,"file":"useTools.d.ts","sourceRoot":"","sources":["../../../src/hooks/useTools.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAC;AAIvD,QAAA,MAAM,QAAQ,QACP,eAAe,YACX,CAAC,IAAI,EAAE,MAAM,GAAG,SAAS,KAAK,IAAI,8BAG7B,CAAC,MAAM,EAAE,MAAM,KAAK,IAAI,qBAGhC,gBAAgB,SACf,MAAM,qBACK,MAAM,IAAI;gBAKC,GAAG;aAAW,GAAG;GA+jB/C,CAAC;AAEF,eAAe,QAAQ,CAAC"}

package/dist/esm/hooks/useTools.js

@@ -1 +1 @@
-import{useMemo as e}from"react";import{getConfigValue as t}from"../config/index.js";const s=(s,a=e=>{console.log(e)},n=e=>{console.log(e)},r,i)=>e((()=>{const e=[],o=(t,s)=>{e.push({definition:t,handler:s})},p=async()=>{const e=t("APP_CONFIG"),a=t("APP_BUTTONS");return[...a&&e.apps[a]?e.apps[a].tasks:[],...await s.getAppTasks()]};return o({name:"inject_script",description:"Injects a script into the html/canvas view.",parameters:{type:"object",properties:{script:{type:"string",description:"The script to inject"}},required:["script"]}},(async({script:e})=>(console.log("TOOL: inject_script",e),n(e),{ok:!0}))),o({name:"post_codicent_message",description:"Posts a message to Codicent, only when requested by the user.",parameters:{type:"object",properties:{content:{type:"string",description:'The codicent message text, e.g. "remember to call Linda"'},parent_id:{type:"string",description:'The parent message id of a previous message that you will add a new revision to, by setting parent_id to the origin message being "edited"'}},required:["content"]}},(async({content:e,parent_id:t})=>await s.sendMessage(e,t))),o({name:"update_codicent_message",description:"Posts an edited message, which replaces the old message.",parameters:{type:"object",properties:{content:{type:"string",description:'The updated codicent message text, e.g. "remember to call Johan", usually including the tags from the original message.'},parent_id:{type:"string",description:"The message id of a previous/old message to be updated."}},required:["content","parent_id"]}},(async({content:e,parent_id:t})=>await s.sendMessage(e,t))),o({name:"get_messages_list",description:"Gets messages tagged with given tag name, optionally limited by num_messages. The messages are listed in chronological order, latest first.",parameters:{type:"object",properties:{num_messages:{type:"integer",description:"The number of matching messages to return"},search_text:{type:"string",description:"Search text to filter messages by"},tag_name:{type:"string",description:"Name of tag to find messages by"},after_timestamp:{type:"string",description:"Only return messages after this timestamp"},before_timestamp:{type:"string",description:"Only return messages before this timestamp"}},required:["num_messages"]}},(async({num_messages:e,search_text:t,tag_name:a,after_timestamp:n,before_timestamp:r})=>{let i=await s.getMessages(a?[a]:[]);return t&&(i=i.filter((e=>e.content.includes(t)))),n&&(i=i.filter((e=>e.createdAt>n))),r&&(i=i.filter((e=>e.createdAt<r))),i.slice(0,e)})),o({name:"chat_with_codicent_ai",description:"Sends a message to Codicent AI and gets a response message back. Remember that you can always ask codicent via this tool to find out more about what you can achieve from chatting with the codicent AI.",parameters:{type:"object",properties:{message:{type:"string",description:"Text of message to the codicent"}},required:["message"]}},(async({message:e})=>await s.chat(e))),o({name:"create_todo",description:"Saves a todo message in Codicent.",parameters:{type:"object",properties:{text:{type:"string",description:"Text of todo"}},required:["text"]}},(async({text:e})=>await s.sendMessage("#todo "+e))),o({name:"place_order",description:"Places and order with the Codicent AI.",parameters:{type:"object",properties:{message:{type:"string",description:"The text message of the order as spoken by the user."}},required:["message"]}},(async({message:e})=>await s.sendMessage(`#do Jag kommer ge dig en text (nedan) på vad jag ska beställa. Använd bara den text du får av mig, inga tools.\n    Du ska skapa en tabell med detaljerad information inklusive lagerstatus \n    för varje artikel i beställningen. Svara först när du har all info på plats. Ställ inga frågor på vägen utan använd det du får av mig.\n    \n    ----\n    EXEMPEL PÅ SVAR:\n    ----\n    Här är en tabell med detaljerad information inklusive lagerstatus för varje artikel i beställningen:\n    \n    Artikel\tHyresobjekt\tNamn\tLagertyp\tGrupp\tNamn.1\tSRA/SBEF\tLeverantör\tLevnr\tDepå\tLager\tReserverat\tUthyrt nu\tTillgängligt nu\tMärke/typ\tTillverkarnr\tTillv.år\tInköpsdato\tInköpspris\tVikt\n    Elverk\t12345\tElverk 1\tHyresobjekt\tMaskiner\tElverk\tSRA\tLeverantör A\t001\tDepå 1\t5\t0\t1\t4\tHonda EU22i\t123456\t2020\t2020-01-01\t10000 SEK\t21 kg\n    Elverk\t12345\tElverk 2\tHyresobjekt\tMaskiner\tElverk\tSRA\tLeverantör B\t002\tDepå 2\t3\t0\t1\t2\tYamaha EF2000iS\t654321\t2019\t2019-05-01\t9000 SEK\t20 kg\n    Observera att denna tabell är baserad på den information som finns tillgänglig i lagerstatusen. Om du behöver ytterligare detaljer eller om något specifikt saknas, vänligen meddela mig.\n    ----\n    \n    --- här följer texten på vad jag vill beställa ---\n    \n    \n    ${e}`))),o({name:"remember_info",description:"Saves info in your internal memory for future use. Use this tool to save insights on how to perform our conversations and tasks related to them.",parameters:{type:"object",properties:{message:{type:"string",description:"Your insights to improve our future conversations."}},required:["message"]}},(async({message:e})=>await s.sendMessage(`#index #insights\n\n${e}`))),o({name:"save_to_crm",description:"Saves CRM related info to your memory (logbook).",parameters:{type:"object",properties:{message:{type:"string",description:"CRM log entry description/contents."},company:{type:"string",description:"Company name for CRM log entry."},contact:{type:"string",description:"Contact name for CRM log entry."}},required:["message","company"]}},(async({message:e,company:t,contact:a})=>await s.sendMessage(`#crm\n\nFöretag: ${t}\n\nInfo: ${e}${a?"\n\nKontakt: "+a:""}`))),o({name:"set_page_html",description:"Sets the innerHtml of the div covering the full app page. You can display any html/javascript/style/mermaid content here for the user to see ;-). This view is usually referred to as the canvas, or tavlan in Swedish.",parameters:{type:"object",properties:{html:{type:"string",description:"The inner html to set the page to."}},required:["html"]}},(async({html:e})=>(a(e),r.play(),{ok:!0}))),o({name:"hide_page_html",description:"Hides the inner html of the page.",parameters:{type:"object",properties:{}}},(async()=>(a(void 0),{ok:!0}))),o({name:"get_page_html",description:"Gets the inner html of the page.",parameters:{type:"object",properties:{}}},(async()=>({html:i}))),o({name:"get_tasks",description:"Gets the list of possible tasks that can be performed. This lists task names and their id:s. Use get_task to get the task instructions given a task id.",parameters:{type:"object",properties:{},required:[]}},(async e=>(await p()).map((e=>({id:e.id,name:e.title}))))),o({name:"get_task",description:"Gets the task instructions for a given task id. Use get_tasks to get a list of possible tasks (with their id:s).",parameters:{type:"object",properties:{task_id:{type:"string",description:"The task id to get instructions for."}},required:["task_id"]}},(async({task_id:e})=>{const t=(await p()).find((t=>t.id===e));return t?t.content:"Task not found"})),e}),[s,a,n,i,r]);export{s as default};
+import{useMemo as e}from"react";import{getConfigValue as t}from"../config/index.js";const s=(s,a=e=>{console.log(e)},n=e=>{console.log(e)},r,i,o=()=>{console.log("stopVoiceSession")})=>e((()=>{const e=[],p=(t,s)=>{e.push({definition:t,handler:s})},c=async()=>{const e=t("APP_CONFIG"),a=t("APP_BUTTONS");return[...a&&e.apps[a]?e.apps[a].tasks:[],...await s.getAppTasks()]};return p({name:"stop_voice_session",description:"Stops the voice session.",parameters:{type:"object",properties:{}}},(async()=>(console.log("TOOL: stop_voice_session"),a(void 0),o(),{ok:!0}))),p({name:"inject_script",description:"Injects a script into the html/canvas view.",parameters:{type:"object",properties:{script:{type:"string",description:"The script to inject"}},required:["script"]}},(async({script:e})=>(console.log("TOOL: inject_script",e),n(e),{ok:!0}))),p({name:"post_codicent_message",description:"Posts a message to Codicent, only when requested by the user.",parameters:{type:"object",properties:{content:{type:"string",description:'The codicent message text, e.g. "remember to call Linda"'},parent_id:{type:"string",description:'The parent message id of a previous message that you will add a new revision to, by setting parent_id to the origin message being "edited"'}},required:["content"]}},(async({content:e,parent_id:t})=>await s.sendMessage(e,t))),p({name:"update_codicent_message",description:"Posts an edited message, which replaces the old message.",parameters:{type:"object",properties:{content:{type:"string",description:'The updated codicent message text, e.g. "remember to call Johan", usually including the tags from the original message.'},parent_id:{type:"string",description:"The message id of a previous/old message to be updated."}},required:["content","parent_id"]}},(async({content:e,parent_id:t})=>await s.sendMessage(e,t))),p({name:"get_messages_list",description:"Gets messages tagged with given tag name, optionally limited by num_messages. The messages are listed in chronological order, latest first.",parameters:{type:"object",properties:{num_messages:{type:"integer",description:"The number of matching messages to return"},search_text:{type:"string",description:"Search text to filter messages by"},tag_name:{type:"string",description:"Name of tag to find messages by"},after_timestamp:{type:"string",description:"Only return messages after this timestamp"},before_timestamp:{type:"string",description:"Only return messages before this timestamp"}},required:["num_messages"]}},(async({num_messages:e,search_text:t,tag_name:a,after_timestamp:n,before_timestamp:r})=>{let i=await s.getMessages(a?[a]:[]);return t&&(i=i.filter((e=>e.content.includes(t)))),n&&(i=i.filter((e=>e.createdAt>n))),r&&(i=i.filter((e=>e.createdAt<r))),i.slice(0,e)})),p({name:"chat_with_codicent_ai",description:"Sends a message to Codicent AI and gets a response message back. Remember that you can always ask codicent via this tool to find out more about what you can achieve from chatting with the codicent AI.",parameters:{type:"object",properties:{message:{type:"string",description:"Text of message to the codicent"}},required:["message"]}},(async({message:e})=>await s.chat(e))),p({name:"create_todo",description:"Saves a todo message in Codicent.",parameters:{type:"object",properties:{text:{type:"string",description:"Text of todo"}},required:["text"]}},(async({text:e})=>await s.sendMessage("#todo "+e))),p({name:"place_order",description:"Places and order with the Codicent AI.",parameters:{type:"object",properties:{message:{type:"string",description:"The text message of the order as spoken by the user."}},required:["message"]}},(async({message:e})=>await s.sendMessage(`#do Jag kommer ge dig en text (nedan) på vad jag ska beställa. Använd bara den text du får av mig, inga tools.\n    Du ska skapa en tabell med detaljerad information inklusive lagerstatus \n    för varje artikel i beställningen. Svara först när du har all info på plats. Ställ inga frågor på vägen utan använd det du får av mig.\n    \n    ----\n    EXEMPEL PÅ SVAR:\n    ----\n    Här är en tabell med detaljerad information inklusive lagerstatus för varje artikel i beställningen:\n    \n    Artikel\tHyresobjekt\tNamn\tLagertyp\tGrupp\tNamn.1\tSRA/SBEF\tLeverantör\tLevnr\tDepå\tLager\tReserverat\tUthyrt nu\tTillgängligt nu\tMärke/typ\tTillverkarnr\tTillv.år\tInköpsdato\tInköpspris\tVikt\n    Elverk\t12345\tElverk 1\tHyresobjekt\tMaskiner\tElverk\tSRA\tLeverantör A\t001\tDepå 1\t5\t0\t1\t4\tHonda EU22i\t123456\t2020\t2020-01-01\t10000 SEK\t21 kg\n    Elverk\t12345\tElverk 2\tHyresobjekt\tMaskiner\tElverk\tSRA\tLeverantör B\t002\tDepå 2\t3\t0\t1\t2\tYamaha EF2000iS\t654321\t2019\t2019-05-01\t9000 SEK\t20 kg\n    Observera att denna tabell är baserad på den information som finns tillgänglig i lagerstatusen. Om du behöver ytterligare detaljer eller om något specifikt saknas, vänligen meddela mig.\n    ----\n    \n    --- här följer texten på vad jag vill beställa ---\n    \n    \n    ${e}`))),p({name:"remember_info",description:"Saves info in your internal memory for future use. Use this tool to save insights on how to perform our conversations and tasks related to them.",parameters:{type:"object",properties:{message:{type:"string",description:"Your insights to improve our future conversations."}},required:["message"]}},(async({message:e})=>await s.sendMessage(`#index #insights\n\n${e}`))),p({name:"save_to_crm",description:"Saves CRM related info to your memory (logbook).",parameters:{type:"object",properties:{message:{type:"string",description:"CRM log entry description/contents."},company:{type:"string",description:"Company name for CRM log entry."},contact:{type:"string",description:"Contact name for CRM log entry."}},required:["message","company"]}},(async({message:e,company:t,contact:a})=>await s.sendMessage(`#crm\n\nFöretag: ${t}\n\nInfo: ${e}${a?"\n\nKontakt: "+a:""}`))),p({name:"set_page_html",description:"Sets the innerHtml of the div covering the full app page. You can display any html/javascript/style/mermaid content here for the user to see ;-). This view is usually referred to as the canvas, or tavlan in Swedish.",parameters:{type:"object",properties:{html:{type:"string",description:"The inner html to set the page to."}},required:["html"]}},(async({html:e})=>(a(e),r.play(),{ok:!0}))),p({name:"hide_page_html",description:"Hides the inner html of the page.",parameters:{type:"object",properties:{}}},(async()=>(a(void 0),{ok:!0}))),p({name:"get_page_html",description:"Gets the inner html of the page.",parameters:{type:"object",properties:{}}},(async()=>({html:i}))),p({name:"get_tasks",description:"Gets the list of possible tasks that can be performed. This lists task names and their id:s. Use get_task to get the task instructions given a task id.",parameters:{type:"object",properties:{},required:[]}},(async e=>(await c()).map((e=>({id:e.id,name:e.title}))))),p({name:"get_task",description:"Gets the task instructions for a given task id. Use get_tasks to get a list of possible tasks (with their id:s).",parameters:{type:"object",properties:{task_id:{type:"string",description:"The task id to get instructions for."}},required:["task_id"]}},(async({task_id:e})=>{const t=(await c()).find((t=>t.id===e));return t?t.content:"Task not found"})),e}),[s,a,n,i,r]);export{s as default};

package/dist/index.d.ts

@@ -402,7 +402,7 @@ interface AudioRecorder extends RecordingState {
 }
 declare const useAudioRecorder: (onNewRecording: (file: FormData, tag: string) => void) => AudioRecorderState;
 
-declare const useTools: (api: CodicentService, setHtml: ((html: string | undefined) => void) | undefined, injectScript: ((script: string) => void) | undefined, beep: HTMLAudioElement, html?: string) => {
+declare const useTools: (api: CodicentService, setHtml: ((html: string | undefined) => void) | undefined, injectScript: ((script: string) => void) | undefined, beep: HTMLAudioElement, html?: string, stopVoiceSession?: () => void) => {
     definition: any;
     handler: any;
 }[];
@@ -438,6 +438,7 @@ interface RealtimeVoice {
     streamLevel: number;
     setUsername: (username: string) => void;
     updateInstructions: (instructions: string) => void;
+    setLanguage: (language: string) => void;
 }
 declare const useRealtimeVoiceAI: (apiKey: string, tools: {
     definition: ToolDefinitionType;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "codicent-app-sdk",
-  "version": "0.3.41",
+  "version": "0.3.42",
   "description": "SDK for building AI-powered applications with Codicent",
   "type": "module",
   "main": "dist/cjs/index.js",