@copilotkit/react-ui 0.37.0 → 0.38.0-beta.0

package/dist/hooks/use-push-to-talk.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/hooks/use-push-to-talk.tsx"],"sourcesContent":["import { CopilotContextParams, useCopilotContext } from \"@copilotkit/react-core\";\nimport { Message } from \"@copilotkit/shared\";\nimport { MutableRefObject, useEffect, useRef, useState } from \"react\";\n\nexport const checkMicrophonePermission = async () => {\n  try {\n    const permissionStatus = await navigator.permissions.query({\n      name: \"microphone\" as PermissionName,\n    });\n    if (permissionStatus.state === \"granted\") {\n      return true;\n    } else {\n      return false;\n    }\n  } catch (err) {\n    console.error(\"Error checking microphone permission\", err);\n  }\n};\n\nexport const requestMicAndPlaybackPermission = async () => {\n  try {\n    const stream = await navigator.mediaDevices.getUserMedia({ audio: true });\n    const audioContext = new window.AudioContext();\n    await audioContext.resume();\n    return { stream, audioContext };\n  } catch (err) {\n    console.error(\"Error requesting microphone and playback permissions\", err);\n    return null;\n  }\n};\n\nconst startRecording = async (\n  mediaStreamRef: MutableRefObject<MediaStream | null>,\n  mediaRecorderRef: MutableRefObject<MediaRecorder | null>,\n  audioContextRef: MutableRefObject<AudioContext | null>,\n  recordedChunks: Blob[],\n  onStop: () => void,\n) => {\n  if (!mediaStreamRef.current || !audioContextRef.current) {\n    mediaStreamRef.current = await navigator.mediaDevices.getUserMedia({ audio: true });\n    audioContextRef.current = new window.AudioContext();\n    await audioContextRef.current.resume();\n  }\n\n  mediaRecorderRef.current = new MediaRecorder(mediaStreamRef.current!);\n  mediaRecorderRef.current.start(1000);\n  mediaRecorderRef.current.ondataavailable = (event) => {\n    recordedChunks.push(event.data);\n  };\n  mediaRecorderRef.current.onstop = onStop;\n};\n\nconst stopRecording = (mediaRecorderRef: MutableRefObject<MediaRecorder | null>) => {\n  if (mediaRecorderRef.current && mediaRecorderRef.current.state !== \"inactive\") {\n    mediaRecorderRef.current.stop();\n  }\n};\n\nconst transcribeAudio = async (recordedChunks: Blob[], transcribeAudioUrl: string) => {\n  const completeBlob = new Blob(recordedChunks, { type: \"audio/mp4\" });\n  const formData = new FormData();\n  formData.append(\"file\", completeBlob, \"recording.mp4\");\n\n  const response = await fetch(transcribeAudioUrl, {\n    method: \"POST\",\n    body: formData,\n  });\n\n  if (!response.ok) {\n    throw new Error(`Error: ${response.statusText}`);\n  }\n\n  const transcription = await response.json();\n  return transcription.text;\n};\n\nconst playAudioResponse = (text: string, textToSpeechUrl: string, audioContext: AudioContext) => {\n  const encodedText = encodeURIComponent(text);\n  const url = `${textToSpeechUrl}?text=${encodedText}`;\n\n  fetch(url)\n    .then((response) => response.arrayBuffer())\n    .then((arrayBuffer) => audioContext.decodeAudioData(arrayBuffer))\n    .then((audioBuffer) => {\n      const source = audioContext.createBufferSource();\n      source.buffer = audioBuffer;\n      source.connect(audioContext.destination);\n      source.start(0);\n    })\n    .catch((error) => {\n      console.error(\"Error with decoding audio data\", error);\n    });\n};\n\nexport type PushToTalkState = \"idle\" | \"recording\" | \"transcribing\";\n\nexport type SendFunction = (text: string) => Promise<Message>;\n\nexport const usePushToTalk = ({\n  sendFunction,\n  inProgress,\n}: {\n  sendFunction: SendFunction;\n  inProgress: boolean;\n}) => {\n  const [pushToTalkState, setPushToTalkState] = useState<PushToTalkState>(\"idle\");\n  const mediaStreamRef = useRef<MediaStream | null>(null);\n  const audioContextRef = useRef<AudioContext | null>(null);\n  const mediaRecorderRef = useRef<MediaRecorder | null>(null);\n  const recordedChunks = useRef<Blob[]>([]);\n  const context = useCopilotContext();\n  const [startReadingFromMessageId, setStartReadingFromMessageId] = useState<string | null>(null);\n\n  useEffect(() => {\n    if (pushToTalkState === \"recording\") {\n      startRecording(\n        mediaStreamRef,\n        mediaRecorderRef,\n        audioContextRef,\n        recordedChunks.current,\n        () => {\n          setPushToTalkState(\"transcribing\");\n        },\n      );\n    } else {\n      stopRecording(mediaRecorderRef);\n      if (pushToTalkState === \"transcribing\") {\n        transcribeAudio(recordedChunks.current, context.copilotApiConfig.transcribeAudioUrl!).then(\n          async (transcription) => {\n            recordedChunks.current = [];\n            setPushToTalkState(\"idle\");\n            const message = await sendFunction(transcription);\n            setStartReadingFromMessageId(message.id);\n          },\n        );\n      }\n    }\n\n    return () => {\n      stopRecording(mediaRecorderRef);\n    };\n  }, [pushToTalkState]);\n\n  useEffect(() => {\n    if (inProgress === false && startReadingFromMessageId) {\n      const lastMessageIndex = context.messages.findIndex(\n        (message) => message.id === startReadingFromMessageId,\n      );\n\n      const messagesAfterLast = context.messages\n        .slice(lastMessageIndex + 1)\n        .filter((message) => message.role === \"assistant\" && message.content);\n\n      const text = messagesAfterLast.map((message) => message.content).join(\"\\n\");\n      playAudioResponse(text, context.copilotApiConfig.textToSpeechUrl!, audioContextRef.current!);\n\n      setStartReadingFromMessageId(null);\n    }\n  }, [startReadingFromMessageId, inProgress]);\n\n  return { pushToTalkState, setPushToTalkState };\n};\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,wBAAwD;AAExD,mBAA8D;AAEvD,IAAM,4BAA4B,MAAY;AACnD,MAAI;AACF,UAAM,mBAAmB,MAAM,UAAU,YAAY,MAAM;AAAA,MACzD,MAAM;AAAA,IACR,CAAC;AACD,QAAI,iBAAiB,UAAU,WAAW;AACxC,aAAO;AAAA,IACT,OAAO;AACL,aAAO;AAAA,IACT;AAAA,EACF,SAAS,KAAP;AACA,YAAQ,MAAM,wCAAwC,GAAG;AAAA,EAC3D;AACF;AAEO,IAAM,kCAAkC,MAAY;AACzD,MAAI;AACF,UAAM,SAAS,MAAM,UAAU,aAAa,aAAa,EAAE,OAAO,KAAK,CAAC;AACxE,UAAM,eAAe,IAAI,OAAO,aAAa;AAC7C,UAAM,aAAa,OAAO;AAC1B,WAAO,EAAE,QAAQ,aAAa;AAAA,EAChC,SAAS,KAAP;AACA,YAAQ,MAAM,wDAAwD,GAAG;AACzE,WAAO;AAAA,EACT;AACF;AAEA,IAAM,iBAAiB,CACrB,gBACA,kBACA,iBACA,gBACA,WACG;AACH,MAAI,CAAC,eAAe,WAAW,CAAC,gBAAgB,SAAS;AACvD,mBAAe,UAAU,MAAM,UAAU,aAAa,aAAa,EAAE,OAAO,KAAK,CAAC;AAClF,oBAAgB,UAAU,IAAI,OAAO,aAAa;AAClD,UAAM,gBAAgB,QAAQ,OAAO;AAAA,EACvC;AAEA,mBAAiB,UAAU,IAAI,cAAc,eAAe,OAAQ;AACpE,mBAAiB,QAAQ,MAAM,GAAI;AACnC,mBAAiB,QAAQ,kBAAkB,CAAC,UAAU;AACpD,mBAAe,KAAK,MAAM,IAAI;AAAA,EAChC;AACA,mBAAiB,QAAQ,SAAS;AACpC;AAEA,IAAM,gBAAgB,CAAC,qBAA6D;AAClF,MAAI,iBAAiB,WAAW,iBAAiB,QAAQ,UAAU,YAAY;AAC7E,qBAAiB,QAAQ,KAAK;AAAA,EAChC;AACF;AAEA,IAAM,kBAAkB,CAAO,gBAAwB,uBAA+B;AACpF,QAAM,eAAe,IAAI,KAAK,gBAAgB,EAAE,MAAM,YAAY,CAAC;AACnE,QAAM,WAAW,IAAI,SAAS;AAC9B,WAAS,OAAO,QAAQ,cAAc,eAAe;AAErD,QAAM,WAAW,MAAM,MAAM,oBAAoB;AAAA,IAC/C,QAAQ;AAAA,IACR,MAAM;AAAA,EACR,CAAC;AAED,MAAI,CAAC,SAAS,IAAI;AAChB,UAAM,IAAI,MAAM,UAAU,SAAS,YAAY;AAAA,EACjD;AAEA,QAAM,gBAAgB,MAAM,SAAS,KAAK;AAC1C,SAAO,cAAc;AACvB;AAEA,IAAM,oBAAoB,CAAC,MAAc,iBAAyB,iBAA+B;AAC/F,QAAM,cAAc,mBAAmB,IAAI;AAC3C,QAAM,MAAM,GAAG,wBAAwB;AAEvC,QAAM,GAAG,EACN,KAAK,CAAC,aAAa,SAAS,YAAY,CAAC,EACzC,KAAK,CAAC,gBAAgB,aAAa,gBAAgB,WAAW,CAAC,EAC/D,KAAK,CAAC,gBAAgB;AACrB,UAAM,SAAS,aAAa,mBAAmB;AAC/C,WAAO,SAAS;AAChB,WAAO,QAAQ,aAAa,WAAW;AACvC,WAAO,MAAM,CAAC;AAAA,EAChB,CAAC,EACA,MAAM,CAAC,UAAU;AAChB,YAAQ,MAAM,kCAAkC,KAAK;AAAA,EACvD,CAAC;AACL;AAMO,IAAM,gBAAgB,CAAC;AAAA,EAC5B;AAAA,EACA;AACF,MAGM;AACJ,QAAM,CAAC,iBAAiB,kBAAkB,QAAI,uBAA0B,MAAM;AAC9E,QAAM,qBAAiB,qBAA2B,IAAI;AACtD,QAAM,sBAAkB,qBAA4B,IAAI;AACxD,QAAM,uBAAmB,qBAA6B,IAAI;AAC1D,QAAM,qBAAiB,qBAAe,CAAC,CAAC;AACxC,QAAM,cAAU,qCAAkB;AAClC,QAAM,CAAC,2BAA2B,4BAA4B,QAAI,uBAAwB,IAAI;AAE9F,8BAAU,MAAM;AACd,QAAI,oBAAoB,aAAa;AACnC;AAAA,QACE;AAAA,QACA;AAAA,QACA;AAAA,QACA,eAAe;AAAA,QACf,MAAM;AACJ,6BAAmB,cAAc;AAAA,QACnC;AAAA,MACF;AAAA,IACF,OAAO;AACL,oBAAc,gBAAgB;AAC9B,UAAI,oBAAoB,gBAAgB;AACtC,wBAAgB,eAAe,SAAS,QAAQ,iBAAiB,kBAAmB,EAAE;AAAA,UACpF,CAAO,kBAAkB;AACvB,2BAAe,UAAU,CAAC;AAC1B,+BAAmB,MAAM;AACzB,kBAAM,UAAU,MAAM,aAAa,aAAa;AAChD,yCAA6B,QAAQ,EAAE;AAAA,UACzC;AAAA,QACF;AAAA,MACF;AAAA,IACF;AAEA,WAAO,MAAM;AACX,oBAAc,gBAAgB;AAAA,IAChC;AAAA,EACF,GAAG,CAAC,eAAe,CAAC;AAEpB,8BAAU,MAAM;AACd,QAAI,eAAe,SAAS,2BAA2B;AACrD,YAAM,mBAAmB,QAAQ,SAAS;AAAA,QACxC,CAAC,YAAY,QAAQ,OAAO;AAAA,MAC9B;AAEA,YAAM,oBAAoB,QAAQ,SAC/B,MAAM,mBAAmB,CAAC,EAC1B,OAAO,CAAC,YAAY,QAAQ,SAAS,eAAe,QAAQ,OAAO;AAEtE,YAAM,OAAO,kBAAkB,IAAI,CAAC,YAAY,QAAQ,OAAO,EAAE,KAAK,IAAI;AAC1E,wBAAkB,MAAM,QAAQ,iBAAiB,iBAAkB,gBAAgB,OAAQ;AAE3F,mCAA6B,IAAI;AAAA,IACnC;AAAA,EACF,GAAG,CAAC,2BAA2B,UAAU,CAAC;AAE1C,SAAO,EAAE,iBAAiB,mBAAmB;AAC/C;","names":[]}
+{"version":3,"sources":["../../src/hooks/use-push-to-talk.tsx"],"sourcesContent":["import { useCopilotContext } from \"@copilotkit/react-core\";\nimport { Message, TextMessage } from \"@copilotkit/runtime-client-gql\";\nimport { MutableRefObject, useEffect, useRef, useState } from \"react\";\n\nexport const checkMicrophonePermission = async () => {\n  try {\n    const permissionStatus = await navigator.permissions.query({\n      name: \"microphone\" as PermissionName,\n    });\n    if (permissionStatus.state === \"granted\") {\n      return true;\n    } else {\n      return false;\n    }\n  } catch (err) {\n    console.error(\"Error checking microphone permission\", err);\n  }\n};\n\nexport const requestMicAndPlaybackPermission = async () => {\n  try {\n    const stream = await navigator.mediaDevices.getUserMedia({ audio: true });\n    const audioContext = new window.AudioContext();\n    await audioContext.resume();\n    return { stream, audioContext };\n  } catch (err) {\n    console.error(\"Error requesting microphone and playback permissions\", err);\n    return null;\n  }\n};\n\nconst startRecording = async (\n  mediaStreamRef: MutableRefObject<MediaStream | null>,\n  mediaRecorderRef: MutableRefObject<MediaRecorder | null>,\n  audioContextRef: MutableRefObject<AudioContext | null>,\n  recordedChunks: Blob[],\n  onStop: () => void,\n) => {\n  if (!mediaStreamRef.current || !audioContextRef.current) {\n    mediaStreamRef.current = await navigator.mediaDevices.getUserMedia({ audio: true });\n    audioContextRef.current = new window.AudioContext();\n    await audioContextRef.current.resume();\n  }\n\n  mediaRecorderRef.current = new MediaRecorder(mediaStreamRef.current!);\n  mediaRecorderRef.current.start(1000);\n  mediaRecorderRef.current.ondataavailable = (event) => {\n    recordedChunks.push(event.data);\n  };\n  mediaRecorderRef.current.onstop = onStop;\n};\n\nconst stopRecording = (mediaRecorderRef: MutableRefObject<MediaRecorder | null>) => {\n  if (mediaRecorderRef.current && mediaRecorderRef.current.state !== \"inactive\") {\n    mediaRecorderRef.current.stop();\n  }\n};\n\nconst transcribeAudio = async (recordedChunks: Blob[], transcribeAudioUrl: string) => {\n  const completeBlob = new Blob(recordedChunks, { type: \"audio/mp4\" });\n  const formData = new FormData();\n  formData.append(\"file\", completeBlob, \"recording.mp4\");\n\n  const response = await fetch(transcribeAudioUrl, {\n    method: \"POST\",\n    body: formData,\n  });\n\n  if (!response.ok) {\n    throw new Error(`Error: ${response.statusText}`);\n  }\n\n  const transcription = await response.json();\n  return transcription.text;\n};\n\nconst playAudioResponse = (text: string, textToSpeechUrl: string, audioContext: AudioContext) => {\n  const encodedText = encodeURIComponent(text);\n  const url = `${textToSpeechUrl}?text=${encodedText}`;\n\n  fetch(url)\n    .then((response) => response.arrayBuffer())\n    .then((arrayBuffer) => audioContext.decodeAudioData(arrayBuffer))\n    .then((audioBuffer) => {\n      const source = audioContext.createBufferSource();\n      source.buffer = audioBuffer;\n      source.connect(audioContext.destination);\n      source.start(0);\n    })\n    .catch((error) => {\n      console.error(\"Error with decoding audio data\", error);\n    });\n};\n\nexport type PushToTalkState = \"idle\" | \"recording\" | \"transcribing\";\n\nexport type SendFunction = (text: string) => Promise<Message>;\n\nexport const usePushToTalk = ({\n  sendFunction,\n  inProgress,\n}: {\n  sendFunction: SendFunction;\n  inProgress: boolean;\n}) => {\n  const [pushToTalkState, setPushToTalkState] = useState<PushToTalkState>(\"idle\");\n  const mediaStreamRef = useRef<MediaStream | null>(null);\n  const audioContextRef = useRef<AudioContext | null>(null);\n  const mediaRecorderRef = useRef<MediaRecorder | null>(null);\n  const recordedChunks = useRef<Blob[]>([]);\n  const context = useCopilotContext();\n  const [startReadingFromMessageId, setStartReadingFromMessageId] = useState<string | null>(null);\n\n  useEffect(() => {\n    if (pushToTalkState === \"recording\") {\n      startRecording(\n        mediaStreamRef,\n        mediaRecorderRef,\n        audioContextRef,\n        recordedChunks.current,\n        () => {\n          setPushToTalkState(\"transcribing\");\n        },\n      );\n    } else {\n      stopRecording(mediaRecorderRef);\n      if (pushToTalkState === \"transcribing\") {\n        transcribeAudio(recordedChunks.current, context.copilotApiConfig.transcribeAudioUrl!).then(\n          async (transcription) => {\n            recordedChunks.current = [];\n            setPushToTalkState(\"idle\");\n            const message = await sendFunction(transcription);\n            setStartReadingFromMessageId(message.id);\n          },\n        );\n      }\n    }\n\n    return () => {\n      stopRecording(mediaRecorderRef);\n    };\n  }, [pushToTalkState]);\n\n  useEffect(() => {\n    if (inProgress === false && startReadingFromMessageId) {\n      const lastMessageIndex = context.messages.findIndex(\n        (message) => message.id === startReadingFromMessageId,\n      );\n\n      const messagesAfterLast = context.messages\n        .slice(lastMessageIndex + 1)\n        .filter(\n          (message) => message instanceof TextMessage && message.role === \"assistant\",\n        ) as TextMessage[];\n\n      const text = messagesAfterLast.map((message) => message.content).join(\"\\n\");\n      playAudioResponse(text, context.copilotApiConfig.textToSpeechUrl!, audioContextRef.current!);\n\n      setStartReadingFromMessageId(null);\n    }\n  }, [startReadingFromMessageId, inProgress]);\n\n  return { pushToTalkState, setPushToTalkState };\n};\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,wBAAkC;AAClC,gCAAqC;AACrC,mBAA8D;AAEvD,IAAM,4BAA4B,MAAY;AACnD,MAAI;AACF,UAAM,mBAAmB,MAAM,UAAU,YAAY,MAAM;AAAA,MACzD,MAAM;AAAA,IACR,CAAC;AACD,QAAI,iBAAiB,UAAU,WAAW;AACxC,aAAO;AAAA,IACT,OAAO;AACL,aAAO;AAAA,IACT;AAAA,EACF,SAAS,KAAP;AACA,YAAQ,MAAM,wCAAwC,GAAG;AAAA,EAC3D;AACF;AAEO,IAAM,kCAAkC,MAAY;AACzD,MAAI;AACF,UAAM,SAAS,MAAM,UAAU,aAAa,aAAa,EAAE,OAAO,KAAK,CAAC;AACxE,UAAM,eAAe,IAAI,OAAO,aAAa;AAC7C,UAAM,aAAa,OAAO;AAC1B,WAAO,EAAE,QAAQ,aAAa;AAAA,EAChC,SAAS,KAAP;AACA,YAAQ,MAAM,wDAAwD,GAAG;AACzE,WAAO;AAAA,EACT;AACF;AAEA,IAAM,iBAAiB,CACrB,gBACA,kBACA,iBACA,gBACA,WACG;AACH,MAAI,CAAC,eAAe,WAAW,CAAC,gBAAgB,SAAS;AACvD,mBAAe,UAAU,MAAM,UAAU,aAAa,aAAa,EAAE,OAAO,KAAK,CAAC;AAClF,oBAAgB,UAAU,IAAI,OAAO,aAAa;AAClD,UAAM,gBAAgB,QAAQ,OAAO;AAAA,EACvC;AAEA,mBAAiB,UAAU,IAAI,cAAc,eAAe,OAAQ;AACpE,mBAAiB,QAAQ,MAAM,GAAI;AACnC,mBAAiB,QAAQ,kBAAkB,CAAC,UAAU;AACpD,mBAAe,KAAK,MAAM,IAAI;AAAA,EAChC;AACA,mBAAiB,QAAQ,SAAS;AACpC;AAEA,IAAM,gBAAgB,CAAC,qBAA6D;AAClF,MAAI,iBAAiB,WAAW,iBAAiB,QAAQ,UAAU,YAAY;AAC7E,qBAAiB,QAAQ,KAAK;AAAA,EAChC;AACF;AAEA,IAAM,kBAAkB,CAAO,gBAAwB,uBAA+B;AACpF,QAAM,eAAe,IAAI,KAAK,gBAAgB,EAAE,MAAM,YAAY,CAAC;AACnE,QAAM,WAAW,IAAI,SAAS;AAC9B,WAAS,OAAO,QAAQ,cAAc,eAAe;AAErD,QAAM,WAAW,MAAM,MAAM,oBAAoB;AAAA,IAC/C,QAAQ;AAAA,IACR,MAAM;AAAA,EACR,CAAC;AAED,MAAI,CAAC,SAAS,IAAI;AAChB,UAAM,IAAI,MAAM,UAAU,SAAS,YAAY;AAAA,EACjD;AAEA,QAAM,gBAAgB,MAAM,SAAS,KAAK;AAC1C,SAAO,cAAc;AACvB;AAEA,IAAM,oBAAoB,CAAC,MAAc,iBAAyB,iBAA+B;AAC/F,QAAM,cAAc,mBAAmB,IAAI;AAC3C,QAAM,MAAM,GAAG,wBAAwB;AAEvC,QAAM,GAAG,EACN,KAAK,CAAC,aAAa,SAAS,YAAY,CAAC,EACzC,KAAK,CAAC,gBAAgB,aAAa,gBAAgB,WAAW,CAAC,EAC/D,KAAK,CAAC,gBAAgB;AACrB,UAAM,SAAS,aAAa,mBAAmB;AAC/C,WAAO,SAAS;AAChB,WAAO,QAAQ,aAAa,WAAW;AACvC,WAAO,MAAM,CAAC;AAAA,EAChB,CAAC,EACA,MAAM,CAAC,UAAU;AAChB,YAAQ,MAAM,kCAAkC,KAAK;AAAA,EACvD,CAAC;AACL;AAMO,IAAM,gBAAgB,CAAC;AAAA,EAC5B;AAAA,EACA;AACF,MAGM;AACJ,QAAM,CAAC,iBAAiB,kBAAkB,QAAI,uBAA0B,MAAM;AAC9E,QAAM,qBAAiB,qBAA2B,IAAI;AACtD,QAAM,sBAAkB,qBAA4B,IAAI;AACxD,QAAM,uBAAmB,qBAA6B,IAAI;AAC1D,QAAM,qBAAiB,qBAAe,CAAC,CAAC;AACxC,QAAM,cAAU,qCAAkB;AAClC,QAAM,CAAC,2BAA2B,4BAA4B,QAAI,uBAAwB,IAAI;AAE9F,8BAAU,MAAM;AACd,QAAI,oBAAoB,aAAa;AACnC;AAAA,QACE;AAAA,QACA;AAAA,QACA;AAAA,QACA,eAAe;AAAA,QACf,MAAM;AACJ,6BAAmB,cAAc;AAAA,QACnC;AAAA,MACF;AAAA,IACF,OAAO;AACL,oBAAc,gBAAgB;AAC9B,UAAI,oBAAoB,gBAAgB;AACtC,wBAAgB,eAAe,SAAS,QAAQ,iBAAiB,kBAAmB,EAAE;AAAA,UACpF,CAAO,kBAAkB;AACvB,2BAAe,UAAU,CAAC;AAC1B,+BAAmB,MAAM;AACzB,kBAAM,UAAU,MAAM,aAAa,aAAa;AAChD,yCAA6B,QAAQ,EAAE;AAAA,UACzC;AAAA,QACF;AAAA,MACF;AAAA,IACF;AAEA,WAAO,MAAM;AACX,oBAAc,gBAAgB;AAAA,IAChC;AAAA,EACF,GAAG,CAAC,eAAe,CAAC;AAEpB,8BAAU,MAAM;AACd,QAAI,eAAe,SAAS,2BAA2B;AACrD,YAAM,mBAAmB,QAAQ,SAAS;AAAA,QACxC,CAAC,YAAY,QAAQ,OAAO;AAAA,MAC9B;AAEA,YAAM,oBAAoB,QAAQ,SAC/B,MAAM,mBAAmB,CAAC,EAC1B;AAAA,QACC,CAAC,YAAY,mBAAmB,yCAAe,QAAQ,SAAS;AAAA,MAClE;AAEF,YAAM,OAAO,kBAAkB,IAAI,CAAC,YAAY,QAAQ,OAAO,EAAE,KAAK,IAAI;AAC1E,wBAAkB,MAAM,QAAQ,iBAAiB,iBAAkB,gBAAgB,OAAQ;AAE3F,mCAA6B,IAAI;AAAA,IACnC;AAAA,EACF,GAAG,CAAC,2BAA2B,UAAU,CAAC;AAE1C,SAAO,EAAE,iBAAiB,mBAAmB;AAC/C;","names":[]}

package/dist/hooks/use-push-to-talk.mjs

@@ -2,7 +2,7 @@ import {
   checkMicrophonePermission,
   requestMicAndPlaybackPermission,
   usePushToTalk
-} from "../chunk-ANO23V2M.mjs";
+} from "../chunk-VOBX4JOA.mjs";
 import "../chunk-MRXNTQOX.mjs";
 export {
   checkMicrophonePermission,

package/dist/index.d.ts

@@ -5,7 +5,7 @@ export { CopilotChat } from './components/chat/Chat.js';
 export { useChatContext } from './components/chat/ChatContext.js';
 export { useCopilotChatSuggestions } from './hooks/use-copilot-chat-suggestions.js';
 export { CopilotKitCSSProperties } from './types/css.js';
-import '@copilotkit/shared';
+import '@copilotkit/runtime-client-gql';
 import './components/chat/Modal.js';
 import 'react';
 import '@copilotkit/react-core';

package/dist/index.js

@@ -517,8 +517,6 @@ var Header = ({ setOpen }) => {
 
 // src/components/chat/Messages.tsx
 var import_react5 = __toESM(require("react"));
-var import_nanoid = require("nanoid");
-var import_shared = require("@copilotkit/shared");
 
 // src/components/chat/Markdown.tsx
 var import_react4 = require("react");
@@ -990,6 +988,7 @@ var components = {
 
 // src/components/chat/Messages.tsx
 var import_react_core = require("@copilotkit/react-core");
+var import_runtime_client_gql = require("@copilotkit/runtime-client-gql");
 var import_jsx_runtime8 = require("react/jsx-runtime");
 var Messages = ({ messages, inProgress, children }) => {
   const { chatComponentsCache } = (0, import_react_core.useCopilotContext)();
@@ -1001,10 +1000,13 @@ var Messages = ({ messages, inProgress, children }) => {
   messages = [...initialMessages, ...messages];
   const functionResults = {};
   for (let i = 0; i < messages.length; i++) {
-    if (messages[i].role === "assistant" && messages[i].function_call) {
+    if (messages[i] instanceof import_runtime_client_gql.ActionExecutionMessage) {
       const id = messages[i].id;
-      if (i + 1 < messages.length && messages[i + 1].role === "function") {
-        functionResults[id] = (0, import_shared.decodeResult)(messages[i + 1].content || "");
+      const resultMessage = messages.find(
+        (message) => message instanceof import_runtime_client_gql.ResultMessage && message.actionExecutionId === id
+      );
+      if (resultMessage) {
+        functionResults[id] = import_runtime_client_gql.ResultMessage.decodeResult(resultMessage.result || "");
       }
     }
   }
@@ -1021,60 +1023,55 @@ var Messages = ({ messages, inProgress, children }) => {
   }, [messages]);
   return /* @__PURE__ */ (0, import_jsx_runtime8.jsxs)("div", { className: "copilotKitMessages", children: [
     messages.map((message, index) => {
-      var _a, _b, _c;
       const isCurrentMessage = index === messages.length - 1;
-      if (message.role === "user") {
+      if (message instanceof import_runtime_client_gql.TextMessage && message.role === "user") {
         return /* @__PURE__ */ (0, import_jsx_runtime8.jsx)("div", { className: "copilotKitMessage copilotKitUserMessage", children: message.content }, index);
-      } else if (message.role == "assistant") {
-        if (isCurrentMessage && inProgress && !message.content && !message.partialFunctionCall) {
-          return /* @__PURE__ */ (0, import_jsx_runtime8.jsx)("div", { className: `copilotKitMessage copilotKitAssistantMessage`, children: context.icons.spinnerIcon }, index);
-        } else if (message.function_call || message.partialFunctionCall) {
-          const functionCallName = ((_a = message.function_call) == null ? void 0 : _a.name) || ((_b = message.partialFunctionCall) == null ? void 0 : _b.name);
-          if (chatComponentsCache.current !== null && chatComponentsCache.current[functionCallName]) {
-            const render = chatComponentsCache.current[functionCallName];
-            if (typeof render === "string") {
-              if (isCurrentMessage && inProgress) {
-                return /* @__PURE__ */ (0, import_jsx_runtime8.jsxs)("div", { className: `copilotKitMessage copilotKitAssistantMessage`, children: [
-                  context.icons.spinnerIcon,
-                  " ",
-                  /* @__PURE__ */ (0, import_jsx_runtime8.jsx)("span", { className: "inProgressLabel", children: render })
-                ] }, index);
-              } else {
-                return null;
-              }
+      } else if (message instanceof import_runtime_client_gql.TextMessage && message.role == "assistant") {
+        return /* @__PURE__ */ (0, import_jsx_runtime8.jsx)("div", { className: `copilotKitMessage copilotKitAssistantMessage`, children: isCurrentMessage && inProgress && !message.content ? context.icons.spinnerIcon : /* @__PURE__ */ (0, import_jsx_runtime8.jsx)(Markdown, { content: message.content }) }, index);
+      } else if (message instanceof import_runtime_client_gql.ActionExecutionMessage) {
+        if (chatComponentsCache.current !== null && chatComponentsCache.current[message.name]) {
+          const render = chatComponentsCache.current[message.name];
+          if (typeof render === "string") {
+            if (isCurrentMessage && inProgress) {
+              return /* @__PURE__ */ (0, import_jsx_runtime8.jsxs)("div", { className: `copilotKitMessage copilotKitAssistantMessage`, children: [
+                context.icons.spinnerIcon,
+                " ",
+                /* @__PURE__ */ (0, import_jsx_runtime8.jsx)("span", { className: "inProgressLabel", children: render })
+              ] }, index);
             } else {
-              const args = message.function_call ? JSON.parse(message.function_call.arguments || "{}") : (_c = message.partialFunctionCall) == null ? void 0 : _c.arguments;
-              let status = "inProgress";
-              if (functionResults[message.id] !== void 0) {
-                status = "complete";
-              } else if (message.function_call) {
-                status = "executing";
-              }
-              const toRender = render({
-                status,
-                args,
-                result: functionResults[message.id]
-              });
-              if (!toRender && status === "complete") {
-                return null;
-              }
-              if (typeof toRender === "string") {
-                return /* @__PURE__ */ (0, import_jsx_runtime8.jsxs)("div", { className: `copilotKitMessage copilotKitAssistantMessage`, children: [
-                  isCurrentMessage && inProgress && context.icons.spinnerIcon,
-                  " ",
-                  toRender
-                ] }, index);
-              } else {
-                return /* @__PURE__ */ (0, import_jsx_runtime8.jsx)("div", { className: "copilotKitCustomAssistantMessage", children: toRender }, index);
-              }
+              return null;
             }
-          } else if ((!inProgress || !isCurrentMessage) && message.function_call) {
-            return null;
           } else {
-            return /* @__PURE__ */ (0, import_jsx_runtime8.jsx)("div", { className: `copilotKitMessage copilotKitAssistantMessage`, children: context.icons.spinnerIcon }, index);
+            const args = message.arguments;
+            let status = "inProgress";
+            if (functionResults[message.id] !== void 0) {
+              status = "complete";
+            } else if (message.status.code !== import_runtime_client_gql.MessageStatusCode.Pending) {
+              status = "executing";
+            }
+            const toRender = render({
+              status,
+              args,
+              result: functionResults[message.id]
+            });
+            if (!toRender && status === "complete") {
+              return null;
+            }
+            if (typeof toRender === "string") {
+              return /* @__PURE__ */ (0, import_jsx_runtime8.jsxs)("div", { className: `copilotKitMessage copilotKitAssistantMessage`, children: [
+                isCurrentMessage && inProgress && context.icons.spinnerIcon,
+                " ",
+                toRender
+              ] }, index);
+            } else {
+              return /* @__PURE__ */ (0, import_jsx_runtime8.jsx)("div", { className: "copilotKitCustomAssistantMessage", children: toRender }, index);
+            }
           }
+        } else if (!inProgress || !isCurrentMessage) {
+          return null;
+        } else {
+          return /* @__PURE__ */ (0, import_jsx_runtime8.jsx)("div", { className: `copilotKitMessage copilotKitAssistantMessage`, children: context.icons.spinnerIcon }, index);
         }
-        return /* @__PURE__ */ (0, import_jsx_runtime8.jsx)("div", { className: `copilotKitMessage copilotKitAssistantMessage`, children: /* @__PURE__ */ (0, import_jsx_runtime8.jsx)(Markdown, { content: message.content }) }, index);
       }
     }),
     /* @__PURE__ */ (0, import_jsx_runtime8.jsx)("footer", { ref: messagesEndRef, children })
@@ -1089,11 +1086,12 @@ function makeInitialMessages(initial) {
       initialArray.push(initial);
     }
   }
-  return initialArray.map((message) => ({
-    id: (0, import_nanoid.nanoid)(),
-    role: "assistant",
-    content: message
-  }));
+  return initialArray.map(
+    (message) => new import_runtime_client_gql.TextMessage({
+      role: import_runtime_client_gql.Role.Assistant,
+      content: message
+    })
+  );
 }
 
 // src/components/chat/Input.tsx
@@ -1150,6 +1148,7 @@ var Textarea_default = AutoResizingTextarea;
 
 // src/hooks/use-push-to-talk.tsx
 var import_react_core2 = require("@copilotkit/react-core");
+var import_runtime_client_gql2 = require("@copilotkit/runtime-client-gql");
 var import_react7 = require("react");
 var startRecording = (mediaStreamRef, mediaRecorderRef, audioContextRef, recordedChunks, onStop) => __async(void 0, null, function* () {
   if (!mediaStreamRef.current || !audioContextRef.current) {
@@ -1239,7 +1238,9 @@ var usePushToTalk = ({
       const lastMessageIndex = context.messages.findIndex(
         (message) => message.id === startReadingFromMessageId
       );
-      const messagesAfterLast = context.messages.slice(lastMessageIndex + 1).filter((message) => message.role === "assistant" && message.content);
+      const messagesAfterLast = context.messages.slice(lastMessageIndex + 1).filter(
+        (message) => message instanceof import_runtime_client_gql2.TextMessage && message.role === "assistant"
+      );
       const text = messagesAfterLast.map((message) => message.content).join("\n");
       playAudioResponse(text, context.copilotApiConfig.textToSpeechUrl, audioContextRef.current);
       setStartReadingFromMessageId(null);
@@ -1328,6 +1329,7 @@ var ResponseButton = ({ onClick, inProgress }) => {
 
 // src/components/chat/Suggestion.tsx
 var import_react_core4 = require("@copilotkit/react-core");
+var import_shared = require("@copilotkit/shared");
 var import_jsx_runtime12 = require("react/jsx-runtime");
 function Suggestion({ title, message, onClick, partial, className }) {
   return /* @__PURE__ */ (0, import_jsx_runtime12.jsxs)(
@@ -1348,7 +1350,13 @@ function Suggestion({ title, message, onClick, partial, className }) {
 }
 var reloadSuggestions = (context, chatSuggestionConfiguration, setCurrentSuggestions, abortControllerRef) => __async(void 0, null, function* () {
   const abortController = abortControllerRef.current;
-  const tools = JSON.stringify(context.getChatCompletionFunctionDescriptions(context.entryPoints));
+  const tools = JSON.stringify(
+    Object.values(context.actions).map((action) => ({
+      name: action.name,
+      description: action.description,
+      jsonSchema: JSON.stringify((0, import_shared.actionParametersToJsonSchema)(action.parameters))
+    }))
+  );
   const allSuggestions = [];
   for (const config of Object.values(chatSuggestionConfiguration)) {
     try {
@@ -1412,7 +1420,8 @@ var reloadSuggestions = (context, chatSuggestionConfiguration, setCurrentSuggest
 // src/components/chat/Chat.tsx
 var import_react9 = __toESM(require("react"));
 var import_react_core5 = require("@copilotkit/react-core");
-var import_nanoid2 = require("nanoid");
+var import_nanoid = require("nanoid");
+var import_runtime_client_gql3 = require("@copilotkit/runtime-client-gql");
 var import_jsx_runtime13 = require("react/jsx-runtime");
 function CopilotChat({
   instructions,
@@ -1484,7 +1493,7 @@ function WrappedCopilotChat({
 var SUGGESTIONS_DEBOUNCE_TIMEOUT = 1e3;
 var useCopilotChatLogic = (makeSystemMessage, onInProgress, onSubmitMessage) => {
   const { visibleMessages, appendMessage, reloadMessages, stopGeneration, isLoading } = (0, import_react_core5.useCopilotChat)({
-    id: (0, import_nanoid2.nanoid)(),
+    id: (0, import_nanoid.nanoid)(),
     makeSystemMessage
   });
   const [currentSuggestions, setCurrentSuggestions] = (0, import_react9.useState)([]);
@@ -1521,11 +1530,10 @@ var useCopilotChatLogic = (makeSystemMessage, onInProgress, onSubmitMessage) =>
     abortSuggestions();
     setCurrentSuggestions([]);
     onSubmitMessage == null ? void 0 : onSubmitMessage(messageContent);
-    const message = {
-      id: (0, import_nanoid2.nanoid)(),
+    const message = new import_runtime_client_gql3.TextMessage({
       content: messageContent,
-      role: "user"
-    };
+      role: import_runtime_client_gql3.Role.User
+    });
     appendMessage(message);
     return message;
   });
@@ -1631,7 +1639,7 @@ function CopilotSidebar(props) {
 
 // src/hooks/use-copilot-chat-suggestions.tsx
 var import_react12 = require("react");
-var import_nanoid3 = require("nanoid");
+var import_nanoid2 = require("nanoid");
 var import_react_core6 = require("@copilotkit/react-core");
 function useCopilotChatSuggestions({
   instructions,
@@ -1641,7 +1649,7 @@ function useCopilotChatSuggestions({
 }, dependencies = []) {
   const context = (0, import_react_core6.useCopilotContext)();
   (0, import_react12.useEffect)(() => {
-    const id = (0, import_nanoid3.nanoid)();
+    const id = (0, import_nanoid2.nanoid)();
     context.addChatSuggestionConfiguration(id, {
       instructions,
       minSuggestions,