@copilotkit/react-ui 0.36.0-mme-push-to-talk.0 → 0.36.0-mme-pre.1

package/src/components/chat/Input.tsx

@@ -2,9 +2,17 @@ import React, { useEffect, useRef, useState } from "react";
 import { InputProps } from "./props";
 import { useChatContext } from "./ChatContext";
 import AutoResizingTextarea from "./Textarea";
+import { usePushToTalk } from "../../hooks/use-push-to-talk";
+import { useCopilotContext } from "@copilotkit/react-core";
 
 export const Input = ({ inProgress, onSend, isVisible = false }: InputProps) => {
   const context = useChatContext();
+  const copilotContext = useCopilotContext();
+
+  const pushToTalkConfigured =
+    copilotContext.copilotApiConfig.textToSpeechUrl !== undefined &&
+    copilotContext.copilotApiConfig.transcribeAudioUrl !== undefined;
+
   const textareaRef = useRef<HTMLTextAreaElement>(null);
 
   const handleDivClick = (event: React.MouseEvent<HTMLDivElement>) => {
@@ -29,14 +37,23 @@ export const Input = ({ inProgress, onSend, isVisible = false }: InputProps) =>
     }
   }, [isVisible]);
 
-  const icon = inProgress ? context.icons.activityIcon : context.icons.sendIcon;
-  const disabled = inProgress || text.length === 0;
+  const { pushToTalkState, setPushToTalkState } = usePushToTalk({
+    sendFunction: onSend,
+    inProgress,
+  });
+
+  const sendIcon =
+    inProgress || pushToTalkState === "transcribing"
+      ? context.icons.activityIcon
+      : context.icons.sendIcon;
+  const showPushToTalk =
+    pushToTalkConfigured &&
+    (pushToTalkState === "idle" || pushToTalkState === "recording") &&
+    !inProgress;
+  const sendDisabled = inProgress || text.length === 0 || pushToTalkState !== "idle";
 
   return (
     <div className="copilotKitInput" onClick={handleDivClick}>
-      <button className="copilotKitSendButton" disabled={disabled} onClick={send}>
-        {icon}
-      </button>
       <AutoResizingTextarea
         ref={textareaRef}
         placeholder={context.labels.placeholder}
@@ -51,6 +68,21 @@ export const Input = ({ inProgress, onSend, isVisible = false }: InputProps) =>
           }
         }}
       />
+      <div className="copilotKitInputControls">
+        {showPushToTalk && (
+          <button
+            onClick={() =>
+              setPushToTalkState(pushToTalkState === "idle" ? "recording" : "transcribing")
+            }
+            className={pushToTalkState === "recording" ? "copilotKitPushToTalkRecording" : ""}
+          >
+            {context.icons.pushToTalkIcon}
+          </button>
+        )}
+        <button disabled={sendDisabled} onClick={send}>
+          {sendIcon}
+        </button>
+      </div>
     </div>
   );
 };

package/src/components/chat/Modal.tsx

@@ -0,0 +1,115 @@
+import React from "react";
+import { ChatContextProvider } from "./ChatContext";
+import { ButtonProps, HeaderProps, WindowProps } from "./props";
+import { Window as DefaultWindow } from "./Window";
+import { Button as DefaultButton } from "./Button";
+import { Header as DefaultHeader } from "./Header";
+import { Messages as DefaultMessages } from "./Messages";
+import { Input as DefaultInput } from "./Input";
+import { ResponseButton as DefaultResponseButton } from "./Response";
+import { CopilotChat, CopilotChatProps } from "./Chat";
+
+export interface CopilotModalProps extends CopilotChatProps {
+  /**
+   * Whether the chat window should be open by default.
+   * @default false
+   */
+  defaultOpen?: boolean;
+
+  /**
+   * If the chat window should close when the user clicks outside of it.
+   * @default true
+   */
+  clickOutsideToClose?: boolean;
+
+  /**
+   * If the chat window should close when the user hits the Escape key.
+   * @default true
+   */
+  hitEscapeToClose?: boolean;
+
+  /**
+   * The shortcut key to open the chat window.
+   * Uses Command-[shortcut] on a Mac and Ctrl-[shortcut] on Windows.
+   * @default "/"
+   */
+  shortcut?: string;
+
+  /**
+   * A callback that gets called when the chat window opens or closes.
+   */
+  onSetOpen?: (open: boolean) => void;
+
+  /**
+   * A custom Window component to use instead of the default.
+   */
+  Window?: React.ComponentType<WindowProps>;
+
+  /**
+   * A custom Button component to use instead of the default.
+   */
+  Button?: React.ComponentType<ButtonProps>;
+
+  /**
+   * A custom Header component to use instead of the default.
+   */
+  Header?: React.ComponentType<HeaderProps>;
+}
+
+export const CopilotModal = ({
+  instructions,
+  defaultOpen = false,
+  clickOutsideToClose = true,
+  hitEscapeToClose = true,
+  onSetOpen,
+  onSubmitMessage,
+  shortcut = "/",
+  icons,
+  labels,
+  makeSystemMessage,
+  showResponseButton = true,
+  onInProgress,
+  Window = DefaultWindow,
+  Button = DefaultButton,
+  Header = DefaultHeader,
+  Messages = DefaultMessages,
+  Input = DefaultInput,
+  ResponseButton = DefaultResponseButton,
+  className,
+  children,
+}: CopilotModalProps) => {
+  const [openState, setOpenState] = React.useState(defaultOpen);
+
+  const setOpen = (open: boolean) => {
+    onSetOpen?.(open);
+    setOpenState(open);
+  };
+
+  return (
+    <ChatContextProvider icons={icons} labels={labels} open={openState} setOpen={setOpenState}>
+      {children}
+      <div className={className}>
+        <Button open={openState} setOpen={setOpen}></Button>
+        <Window
+          open={openState}
+          setOpen={setOpen}
+          clickOutsideToClose={clickOutsideToClose}
+          shortcut={shortcut}
+          hitEscapeToClose={hitEscapeToClose}
+        >
+          <Header open={openState} setOpen={setOpen} />
+          <CopilotChat
+            instructions={instructions}
+            makeSystemMessage={makeSystemMessage}
+            onInProgress={onInProgress}
+            onSubmitMessage={onSubmitMessage}
+            showResponseButton={showResponseButton}
+            Messages={Messages}
+            Input={Input}
+            ResponseButton={ResponseButton}
+          />
+        </Window>
+      </div>
+    </ChatContextProvider>
+  );
+};

package/src/components/chat/Popup.tsx

@@ -48,12 +48,12 @@
  * ```
  */
 
-import { CopilotChat, CopilotChatProps } from "./Chat";
+import { CopilotModal, CopilotModalProps } from "./Modal";
 
-export function CopilotPopup(props: CopilotChatProps) {
+export function CopilotPopup(props: CopilotModalProps) {
   props = {
     ...props,
     className: props.className ? props.className + " copilotKitPopup" : "copilotKitPopup",
   };
-  return <CopilotChat {...props}>{props.children}</CopilotChat>;
+  return <CopilotModal {...props}>{props.children}</CopilotModal>;
 }

package/src/components/chat/Sidebar.tsx

@@ -56,9 +56,9 @@
  * ```
  */
 import React, { useState } from "react";
-import { CopilotChat, CopilotChatProps } from "./Chat";
+import { CopilotModal, CopilotModalProps } from "./Modal";
 
-export function CopilotSidebar(props: CopilotChatProps) {
+export function CopilotSidebar(props: CopilotModalProps) {
   props = {
     ...props,
     className: props.className ? props.className + " copilotKitSidebar" : "copilotKitSidebar",
@@ -74,9 +74,9 @@ export function CopilotSidebar(props: CopilotChatProps) {
 
   return (
     <div className={`copilotKitSidebarContentWrapper ${expandedClassName}`}>
-      <CopilotChat {...props} {...{ onSetOpen }}>
+      <CopilotModal {...props} {...{ onSetOpen }}>
         {props.children}
-      </CopilotChat>
+      </CopilotModal>
     </div>
   );
 }

package/src/components/chat/Suggestion.tsx

@@ -1,7 +1,11 @@
-import { CopilotContextParams, extract } from "@copilotkit/react-core";
+import {
+  CopilotContextParams,
+  extract,
+  CopilotChatSuggestionConfiguration,
+} from "@copilotkit/react-core";
 import { SuggestionsProps } from "./props";
 import { SmallSpinnerIcon } from "./Icons";
-import { CopilotChatSuggestion, CopilotChatSuggestionConfiguration } from "../../types/suggestions";
+import { CopilotChatSuggestion } from "../../types/suggestions";
 
 export function Suggestion({ title, message, onClick, partial, className }: SuggestionsProps) {
   return (

package/src/components/chat/Textarea.tsx

@@ -48,7 +48,7 @@ const AutoResizingTextarea = forwardRef<HTMLTextAreaElement, AutoResizingTextare
         onKeyDown={onKeyDown}
         placeholder={placeholder}
         style={{
-          overflow: "hidden",
+          overflow: "auto",
           resize: "none",
           maxHeight: `${maxHeight}px`,
         }}

package/src/components/chat/index.tsx

@@ -1,4 +1,5 @@
 export * from "./props";
 export { CopilotPopup } from "./Popup";
 export { CopilotSidebar } from "./Sidebar";
+export { CopilotChat } from "./Chat";
 export { useChatContext } from "./ChatContext";

package/src/components/chat/props.ts

@@ -3,8 +3,6 @@ import { Message } from "@copilotkit/shared";
 export interface ButtonProps {
   open: boolean;
   setOpen: (open: boolean) => void;
-  pushToTalk: boolean;
-  setPushToTalk: (pushToTalk: boolean) => void;
 }
 
 export interface WindowProps {
@@ -37,7 +35,7 @@ export interface MessagesProps {
 
 export interface InputProps {
   inProgress: boolean;
-  onSend: (text: string) => void;
+  onSend: (text: string) => Promise<Message>;
   isVisible?: boolean;
 }
 

package/src/css/input.css

@@ -1,7 +1,7 @@
 .copilotKitInput {
   border-top: 1px solid var(--copilot-kit-input-separator-color);
   padding-left: 2rem;
-  padding-right: 2.5rem;
+  padding-right: 1rem;
   padding-top: 1rem;
   padding-bottom: 1rem;
   display: flex;
@@ -13,9 +13,7 @@
   background-color: var(--copilot-kit-input-background-color);
 }
 
-.copilotKitInput > button {
-  position: absolute;
-  right: 0.5rem;
+.copilotKitInput > .copilotKitInputControls > button {
   padding: 0.25rem;
   cursor: pointer;
   transition-property: transform;
@@ -38,22 +36,27 @@
   text-shadow: none;
   display: inline-block;
   text-align: center;
+  margin-left: 0.5rem; /* Add margin to separate button from textarea */
 }
 
-.copilotKitInput > button:not([disabled]) {
+.copilotKitInput > .copilotKitInputControls > button:not([disabled]) {
   color: var(--copilot-kit-input-send-button-color);
 }
 
-.copilotKitInput > button:not([disabled]):hover {
+.copilotKitInput > .copilotKitInputControls > button:not([disabled]):hover {
   transform: scale(1.1);
 }
 
-.copilotKitInput > button[disabled] {
+.copilotKitInput > .copilotKitInputControls > button[disabled] {
   color: var(--copilot-kit-input-send-button-disabled-color);
 }
 
+.copilotKitInputControls {
+  display: flex;
+}
+
 .copilotKitInput > textarea {
-  width: 100%;
+  flex: 1; /* Allow textarea to take up remaining space */
   outline: 2px solid transparent;
   outline-offset: 2px;
   resize: none;
@@ -77,3 +80,10 @@
   color: var(--copilot-kit-input-placeholder-color);
   opacity: 1;
 }
+
+.copilotKitInput > .copilotKitInputControls > button.copilotKitPushToTalkRecording {
+  background-color: red;
+  color: white;
+  border-radius: 50%;
+  animation: copilotKitPulseAnimation 2s cubic-bezier(0.4, 0, 0.6, 1) infinite;
+}

package/src/css/panel.css

@@ -0,0 +1,38 @@
+.copilotKitPanel {
+  z-index: 30;
+  line-height: 1.5;
+  -webkit-text-size-adjust: 100%;
+  -moz-tab-size: 4;
+  -o-tab-size: 4;
+  tab-size: 4;
+  font-family:
+    ui-sans-serif,
+    system-ui,
+    -apple-system,
+    BlinkMacSystemFont,
+    "Segoe UI",
+    Roboto,
+    "Helvetica Neue",
+    Arial,
+    "Noto Sans",
+    sans-serif,
+    "Apple Color Emoji",
+    "Segoe UI Emoji",
+    "Segoe UI Symbol",
+    "Noto Color Emoji";
+  font-feature-settings: normal;
+  font-variation-settings: normal;
+  touch-action: manipulation;
+  display: flex;
+  flex-direction: column;
+  height: 100%;
+}
+
+.copilotKitPanel svg {
+  display: inline-block;
+  vertical-align: middle;
+}
+
+.copilotKitPanel .copilotKitMessages {
+  flex-grow: 1;
+}

package/src/css/window.css

@@ -7,7 +7,9 @@
   border-radius: 0.75rem;
   box-shadow: rgba(0, 0, 0, 0.16) 0px 5px 40px;
   flex-direction: column;
-  transition: opacity 100ms ease-out, transform 200ms ease-out;
+  transition:
+    opacity 100ms ease-out,
+    transform 200ms ease-out;
   opacity: 0;
   transform: scale(0.95) translateY(20px);
   display: flex;

package/src/hooks/use-copilot-chat-suggestions.tsx

@@ -1,6 +1,11 @@
 /**
  * A hook for providing suggestions to the user in the Copilot chat.
  *
+ * <Warning>
+ *   useCopilotChatSuggestions is experimental. The interface is not final and
+ *   can change without further notice.
+ * </Warning>
+ *
  * <img src="/images/useCopilotChatSuggestions/use-copilot-chat-suggestions.gif" width="500" />
  *
  * <img referrerPolicy="no-referrer-when-downgrade" src="https://static.scarf.sh/a.png?x-pxid=a9b290bb-38f9-4518-ac3b-8f54fdbf43be" />
@@ -61,7 +66,28 @@
 import { useEffect } from "react";
 import { useChatContext } from "../components";
 import { nanoid } from "nanoid";
-import { CopilotChatSuggestionConfiguration } from "../types/suggestions";
+import { CopilotChatSuggestionConfiguration, useCopilotContext } from "@copilotkit/react-core";
+
+interface UseCopilotChatSuggestionsConfiguration {
+  /**
+   * A prompt or instructions for the GPT to generate suggestions.
+   */
+  instructions: string;
+  /**
+   * The minimum number of suggestions to generate. Defaults to `1`.
+   * @default 1
+   */
+  minSuggestions?: number;
+  /**
+   * The maximum number of suggestions to generate. Defaults to `3`.
+   * @default 1
+   */
+  maxSuggestions?: number;
+  /**
+   * An optional class name to apply to the suggestions.
+   */
+  className?: string;
+}
 
 export function useCopilotChatSuggestions(
   {
@@ -69,15 +95,15 @@ export function useCopilotChatSuggestions(
     className,
     minSuggestions = 1,
     maxSuggestions = 3,
-  }: CopilotChatSuggestionConfiguration,
+  }: UseCopilotChatSuggestionsConfiguration,
   dependencies: any[] = [],
 ) {
-  const chatContext = useChatContext();
+  const context = useCopilotContext();
 
   useEffect(() => {
     const id = nanoid();
 
-    chatContext.addChatSuggestionConfiguration(id, {
+    context.addChatSuggestionConfiguration(id, {
       instructions,
       minSuggestions,
       maxSuggestions,
@@ -85,7 +111,7 @@ export function useCopilotChatSuggestions(
     });
 
     return () => {
-      chatContext.removeChatSuggestionConfiguration(id);
+      context.removeChatSuggestionConfiguration(id);
     };
   }, dependencies);
 }

package/src/hooks/use-push-to-talk.tsx

@@ -0,0 +1,162 @@
+import { CopilotContextParams, useCopilotContext } from "@copilotkit/react-core";
+import { Message } from "@copilotkit/shared";
+import { MutableRefObject, useEffect, useRef, useState } from "react";
+
+export const checkMicrophonePermission = async () => {
+  try {
+    const permissionStatus = await navigator.permissions.query({
+      name: "microphone" as PermissionName,
+    });
+    if (permissionStatus.state === "granted") {
+      return true;
+    } else {
+      return false;
+    }
+  } catch (err) {
+    console.error("Error checking microphone permission", err);
+  }
+};
+
+export const requestMicAndPlaybackPermission = async () => {
+  try {
+    const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
+    const audioContext = new window.AudioContext();
+    await audioContext.resume();
+    return { stream, audioContext };
+  } catch (err) {
+    console.error("Error requesting microphone and playback permissions", err);
+    return null;
+  }
+};
+
+const startRecording = async (
+  mediaStreamRef: MutableRefObject<MediaStream | null>,
+  mediaRecorderRef: MutableRefObject<MediaRecorder | null>,
+  audioContextRef: MutableRefObject<AudioContext | null>,
+  recordedChunks: Blob[],
+  onStop: () => void,
+) => {
+  if (!mediaStreamRef.current || !audioContextRef.current) {
+    mediaStreamRef.current = await navigator.mediaDevices.getUserMedia({ audio: true });
+    audioContextRef.current = new window.AudioContext();
+    await audioContextRef.current.resume();
+  }
+
+  mediaRecorderRef.current = new MediaRecorder(mediaStreamRef.current!);
+  mediaRecorderRef.current.start(1000);
+  mediaRecorderRef.current.ondataavailable = (event) => {
+    recordedChunks.push(event.data);
+  };
+  mediaRecorderRef.current.onstop = onStop;
+};
+
+const stopRecording = (mediaRecorderRef: MutableRefObject<MediaRecorder | null>) => {
+  if (mediaRecorderRef.current && mediaRecorderRef.current.state !== "inactive") {
+    mediaRecorderRef.current.stop();
+  }
+};
+
+const transcribeAudio = async (recordedChunks: Blob[], transcribeAudioUrl: string) => {
+  const completeBlob = new Blob(recordedChunks, { type: "audio/mp4" });
+  const formData = new FormData();
+  formData.append("file", completeBlob, "recording.mp4");
+
+  const response = await fetch(transcribeAudioUrl, {
+    method: "POST",
+    body: formData,
+  });
+
+  if (!response.ok) {
+    throw new Error(`Error: ${response.statusText}`);
+  }
+
+  const transcription = await response.json();
+  return transcription.text;
+};
+
+const playAudioResponse = (text: string, textToSpeechUrl: string, audioContext: AudioContext) => {
+  const encodedText = encodeURIComponent(text);
+  const url = `${textToSpeechUrl}?text=${encodedText}`;
+
+  fetch(url)
+    .then((response) => response.arrayBuffer())
+    .then((arrayBuffer) => audioContext.decodeAudioData(arrayBuffer))
+    .then((audioBuffer) => {
+      const source = audioContext.createBufferSource();
+      source.buffer = audioBuffer;
+      source.connect(audioContext.destination);
+      source.start(0);
+    })
+    .catch((error) => {
+      console.error("Error with decoding audio data", error);
+    });
+};
+
+export type PushToTalkState = "idle" | "recording" | "transcribing";
+
+export type SendFunction = (text: string) => Promise<Message>;
+
+export const usePushToTalk = ({
+  sendFunction,
+  inProgress,
+}: {
+  sendFunction: SendFunction;
+  inProgress: boolean;
+}) => {
+  const [pushToTalkState, setPushToTalkState] = useState<PushToTalkState>("idle");
+  const mediaStreamRef = useRef<MediaStream | null>(null);
+  const audioContextRef = useRef<AudioContext | null>(null);
+  const mediaRecorderRef = useRef<MediaRecorder | null>(null);
+  const recordedChunks = useRef<Blob[]>([]);
+  const context = useCopilotContext();
+  const [startReadingFromMessageId, setStartReadingFromMessageId] = useState<string | null>(null);
+
+  useEffect(() => {
+    if (pushToTalkState === "recording") {
+      startRecording(
+        mediaStreamRef,
+        mediaRecorderRef,
+        audioContextRef,
+        recordedChunks.current,
+        () => {
+          setPushToTalkState("transcribing");
+        },
+      );
+    } else {
+      stopRecording(mediaRecorderRef);
+      if (pushToTalkState === "transcribing") {
+        transcribeAudio(recordedChunks.current, context.copilotApiConfig.transcribeAudioUrl!).then(
+          async (transcription) => {
+            recordedChunks.current = [];
+            setPushToTalkState("idle");
+            const message = await sendFunction(transcription);
+            setStartReadingFromMessageId(message.id);
+          },
+        );
+      }
+    }
+
+    return () => {
+      stopRecording(mediaRecorderRef);
+    };
+  }, [pushToTalkState]);
+
+  useEffect(() => {
+    if (inProgress === false && startReadingFromMessageId) {
+      const lastMessageIndex = context.messages.findIndex(
+        (message) => message.id === startReadingFromMessageId,
+      );
+
+      const messagesAfterLast = context.messages
+        .slice(lastMessageIndex + 1)
+        .filter((message) => message.role === "assistant" && message.content);
+
+      const text = messagesAfterLast.map((message) => message.content).join("\n");
+      playAudioResponse(text, context.copilotApiConfig.textToSpeechUrl!, audioContextRef.current!);
+
+      setStartReadingFromMessageId(null);
+    }
+  }, [startReadingFromMessageId, inProgress]);
+
+  return { pushToTalkState, setPushToTalkState };
+};

package/src/styles.css

@@ -10,6 +10,7 @@
 @import "./css/response.css";
 @import "./css/markdown.css";
 @import "./css/suggestions.css";
+@import "./css/panel.css";
 
 @tailwind base;
 @tailwind components;

package/src/types/suggestions.ts

@@ -1,27 +1,3 @@
-export interface CopilotChatSuggestionConfiguration {
-  /**
-   * A prompt or instructions for the GPT to generate suggestions.
-   */
-  instructions: string;
-
-  /**
-   * The minimum number of suggestions to generate. Defaults to `1`.
-   * @default 1
-   */
-  minSuggestions?: number;
-
-  /**
-   * The maximum number of suggestions to generate. Defaults to `3`.
-   * @default 1
-   */
-  maxSuggestions?: number;
-
-  /**
-   * An optional class name to apply to the suggestions.
-   */
-  className?: string;
-}
-
 export interface CopilotChatSuggestion {
   title: string;
   message: string;