@copilotkit/react-ui 0.22.0 → 0.36.0-mme-push-to-talk.0

package/src/components/chat/Button.tsx

@@ -1,12 +1,44 @@
+import { useEffect, useRef, useState } from "react";
 import { ButtonProps } from "./props";
 import { useChatContext } from "./ChatContext";
 
-export const Button = ({ open, setOpen }: ButtonProps) => {
+export const Button = ({ open, setOpen, pushToTalk, setPushToTalk }: ButtonProps) => {
   const context = useChatContext();
-  // To ensure that the mouse handler fires even when the button is scaled down
-  // we wrap the button in a div and attach the handler to the div
+  const timerRef = useRef<NodeJS.Timeout | null>(null);
+  const [isLongPress, setIsLongPress] = useState(false);
+
+  const handleMouseDown = () => {
+    timerRef.current = setTimeout(() => {
+      setPushToTalk(true);
+      setIsLongPress(true);
+    }, 500); // 500ms for long press
+  };
+
+  const handleMouseUp = () => {
+    if (timerRef.current) {
+      clearTimeout(timerRef.current);
+      setPushToTalk(false);
+    }
+  };
+
+  const handleClick = () => {
+    if (!isLongPress) {
+      setOpen(!open);
+    } else {
+      setIsLongPress(false);
+    }
+  };
+
+  // we want to handle the mouse up event event outside of the button component
+  useEffect(() => {
+    document.addEventListener("mouseup", handleMouseUp);
+    return () => {
+      document.removeEventListener("mouseup", handleMouseUp);
+    };
+  }, []);
+
   return (
-    <div onClick={() => setOpen(!open)}>
+    <div onClick={handleClick} onMouseDown={handleMouseDown}>
       <button
         className={`copilotKitButton ${open ? "open" : ""}`}
         aria-label={open ? "Close Chat" : "Open Chat"}

package/src/components/chat/Chat.tsx

@@ -1,6 +1,11 @@
-import React, { useEffect } from "react";
+import React, { useEffect, useRef, useState } from "react";
 import { CopilotChatIcons, ChatContextProvider, CopilotChatLabels } from "./ChatContext";
-import { SystemMessageFunction, useCopilotChat } from "@copilotkit/react-core";
+import {
+  SystemMessageFunction,
+  extract,
+  useCopilotChat,
+  useCopilotContext,
+} from "@copilotkit/react-core";
 import {
   ButtonProps,
   HeaderProps,
@@ -8,6 +13,7 @@ import {
   MessagesProps,
   InputProps,
   ResponseButtonProps,
+  SuggestionsProps,
 } from "./props";
 import { Window as DefaultWindow } from "./Window";
 import { Button as DefaultButton } from "./Button";
@@ -16,6 +22,10 @@ import { Messages as DefaultMessages } from "./Messages";
 import { Input as DefaultInput } from "./Input";
 import { nanoid } from "nanoid";
 import { ResponseButton as DefaultResponseButton } from "./Response";
+import { Suggestion, reloadSuggestions } from "./Suggestion";
+import { CopilotChatSuggestion, CopilotChatSuggestionConfiguration } from "../../types/suggestions";
+import { requestMicAndPlaybackPermission } from "./audio";
+import { Message } from "@copilotkit/shared";
 
 /**
  * Props for CopilotChat component.
@@ -66,8 +76,8 @@ export interface CopilotChatProps {
 
   /**
    * The shortcut key to open the chat window.
-   * Uses Command-<shortcut> on a Mac and Ctrl-<shortcut> on Windows.
-   * @default "e"
+   * Uses Command-[shortcut] on a Mac and Ctrl-[shortcut] on Windows.
+   * @default "/"
    */
   shortcut?: string;
 
@@ -136,6 +146,8 @@ export interface CopilotChatProps {
   children?: React.ReactNode;
 }
 
+const SUGGESTIONS_DEBOUNCE_TIMEOUT = 1000;
+
 export const CopilotChat = ({
   instructions,
   defaultOpen = false,
@@ -164,31 +176,190 @@ export const CopilotChat = ({
     additionalInstructions: instructions,
   });
 
+  const [currentSuggestions, setCurrentSuggestions] = React.useState<CopilotChatSuggestion[]>([]);
+  const suggestionsAbortControllerRef = useRef<AbortController | null>(null);
+  const debounceTimerRef = useRef<any>();
+
+  const abortSuggestions = () => {
+    suggestionsAbortControllerRef.current?.abort();
+    suggestionsAbortControllerRef.current = null;
+  };
+
+  const context = useCopilotContext();
+
+  const [chatSuggestionConfiguration, setChatSuggestionConfiguration] = useState<{
+    [key: string]: CopilotChatSuggestionConfiguration;
+  }>({});
+
+  const addChatSuggestionConfiguration = (
+    id: string,
+    suggestion: CopilotChatSuggestionConfiguration,
+  ) => {
+    setChatSuggestionConfiguration((prev) => ({ ...prev, [id]: suggestion }));
+  };
+
+  const removeChatSuggestion = (id: string) => {
+    setChatSuggestionConfiguration((prev) => {
+      const { [id]: _, ...rest } = prev;
+      return rest;
+    });
+  };
+
   useEffect(() => {
     onInProgress?.(isLoading);
-  }, [isLoading]);
 
-  const [openState, setOpenState] = React.useState(defaultOpen);
+    abortSuggestions();
+
+    debounceTimerRef.current = setTimeout(
+      () => {
+        if (!isLoading && Object.keys(chatSuggestionConfiguration).length !== 0) {
+          suggestionsAbortControllerRef.current = new AbortController();
+          reloadSuggestions(
+            context,
+            chatSuggestionConfiguration,
+            setCurrentSuggestions,
+            suggestionsAbortControllerRef,
+          );
+        }
+      },
+      currentSuggestions.length == 0 ? 0 : SUGGESTIONS_DEBOUNCE_TIMEOUT,
+    );
+
+    return () => {
+      clearTimeout(debounceTimerRef.current);
+    };
+  }, [isLoading, chatSuggestionConfiguration]);
 
   const setOpen = (open: boolean) => {
     onSetOpen?.(open);
     setOpenState(open);
   };
 
-  const sendMessage = async (message: string) => {
-    onSubmitMessage?.(message);
-    append({
+  const sendMessage = async (messageContent: string) => {
+    abortSuggestions();
+    setCurrentSuggestions([]);
+    onSubmitMessage?.(messageContent);
+    const message: Message = {
       id: nanoid(),
-      content: message,
+      content: messageContent,
       role: "user",
-    });
+    };
+    append(message);
+    return message;
   };
 
+  const [openState, setOpenState] = React.useState(defaultOpen);
+  const [pushToTalkState, setPushToTalkState] = React.useState(false);
+  const mediaStreamRef = useRef<MediaStream | null>(null);
+  const audioContextRef = useRef<AudioContext | null>(null);
+  const mediaRecorderRef = useRef<MediaRecorder | null>(null);
+  const [lastMessageIdBeforeAudio, setLastMessageIdBeforeAudio] = useState<string | null>(null);
+
+  useEffect(() => {
+    if (pushToTalkState) {
+      console.log("HERE");
+      if (!mediaStreamRef.current || !audioContextRef.current) {
+        setPushToTalkState(false);
+        requestMicAndPlaybackPermission().then((res) => {
+          if (res) {
+            mediaStreamRef.current = res.stream;
+            audioContextRef.current = res.audioContext;
+          }
+        });
+      } else {
+        console.log("Recording started");
+        const recordedChunks: Blob[] = [];
+
+        mediaRecorderRef.current = new MediaRecorder(mediaStreamRef.current);
+        mediaRecorderRef.current.start(1000);
+        mediaRecorderRef.current.ondataavailable = async (event) => {
+          console.log("Recorded audio: ", event.data);
+          recordedChunks.push(event.data);
+        };
+        mediaRecorderRef.current.onstop = async () => {
+          console.log("Recording stopped");
+          const completeBlob = new Blob(recordedChunks, { type: "audio/mp4" });
+
+          const formData = new FormData();
+          formData.append("file", completeBlob, "recording.mp4");
+
+          const response = await fetch(context.copilotApiConfig.transcribeAudioUrl!, {
+            method: "POST",
+            body: formData,
+          });
+
+          if (!response.ok) {
+            throw new Error(`Error: ${response.statusText}`);
+          }
+
+          const transcription = await response.json();
+          const message = await sendMessage(transcription.text);
+          setLastMessageIdBeforeAudio(message.id);
+        };
+      }
+    } else {
+      if (mediaRecorderRef.current && mediaRecorderRef.current.state !== "inactive") {
+        mediaRecorderRef.current.stop();
+      }
+    }
+
+    return () => {
+      if (mediaRecorderRef.current && mediaRecorderRef.current.state !== "inactive") {
+        mediaRecorderRef.current.stop();
+      }
+    };
+  }, [pushToTalkState]);
+
+  useEffect(() => {
+    if (lastMessageIdBeforeAudio && !isLoading) {
+      if (audioContextRef.current) {
+        const lastMessageIndex = context.messages.findIndex(
+          (message) => message.id === lastMessageIdBeforeAudio,
+        );
+
+        const messagesAfterLast = context.messages
+          .slice(lastMessageIndex + 1)
+          .filter((message) => message.role === "assistant" && message.content);
+
+        const text = messagesAfterLast.map((message) => message.content).join("\n");
+        const encodedText = encodeURIComponent(text);
+        const url = `${context.copilotApiConfig.textToSpeechUrl}?text=${encodedText}`;
+
+        fetch(url)
+          .then((response) => response.arrayBuffer())
+          .then((arrayBuffer) => audioContextRef.current!.decodeAudioData(arrayBuffer))
+          .then((audioBuffer) => {
+            const source = audioContextRef.current!.createBufferSource();
+            source.buffer = audioBuffer;
+            source.connect(audioContextRef.current!.destination);
+            source.start(0);
+          })
+          .catch((error) => {
+            console.error("Error with decoding audio data", error);
+          });
+
+        setLastMessageIdBeforeAudio(null);
+      }
+    }
+  }, [isLoading]);
+
   return (
-    <ChatContextProvider icons={icons} labels={labels} open={openState} setOpen={setOpenState}>
+    <ChatContextProvider
+      icons={icons}
+      labels={labels}
+      open={openState}
+      setOpen={setOpenState}
+      addChatSuggestionConfiguration={addChatSuggestionConfiguration}
+      removeChatSuggestionConfiguration={removeChatSuggestion}
+    >
       {children}
       <div className={className}>
-        <Button open={openState} setOpen={setOpen}></Button>
+        <Button
+          open={openState}
+          setOpen={setOpen}
+          pushToTalk={pushToTalkState}
+          setPushToTalk={setPushToTalkState}
+        ></Button>
         <Window
           open={openState}
           setOpen={setOpen}
@@ -197,12 +368,29 @@ export const CopilotChat = ({
           hitEscapeToClose={hitEscapeToClose}
         >
           <Header open={openState} setOpen={setOpen} />
-          <Messages messages={visibleMessages} inProgress={isLoading} />
-          <Input inProgress={isLoading} onSend={sendMessage} isVisible={openState}>
+          <Messages messages={visibleMessages} inProgress={isLoading}>
+            {currentSuggestions.length > 0 && (
+              <div>
+                <h6>Suggested:</h6>
+                <div className="suggestions">
+                  {currentSuggestions.map((suggestion, index) => (
+                    <Suggestion
+                      key={index}
+                      title={suggestion.title}
+                      message={suggestion.message}
+                      partial={suggestion.partial}
+                      className={suggestion.className}
+                      onClick={(message) => sendMessage(message)}
+                    />
+                  ))}
+                </div>
+              </div>
+            )}
             {showResponseButton && visibleMessages.length > 0 && (
               <ResponseButton onClick={isLoading ? stop : reload} inProgress={isLoading} />
             )}
-          </Input>
+          </Messages>
+          <Input inProgress={isLoading} onSend={sendMessage} isVisible={openState} />
         </Window>
       </div>
     </ChatContextProvider>

package/src/components/chat/ChatContext.tsx

@@ -1,5 +1,6 @@
-import React, { useMemo } from "react";
+import React, { useMemo, useState } from "react";
 import * as DefaultIcons from "./Icons";
+import { CopilotChatSuggestion, CopilotChatSuggestionConfiguration } from "../../types/suggestions";
 
 /**
  * Icons for CopilotChat component.
@@ -105,6 +106,11 @@ interface ChatContext {
   icons: Required<CopilotChatIcons>;
   open: boolean;
   setOpen: (open: boolean) => void;
+  addChatSuggestionConfiguration: (
+    id: string,
+    suggestion: CopilotChatSuggestionConfiguration,
+  ) => void;
+  removeChatSuggestionConfiguration: (id: string) => void;
 }
 
 export const ChatContext = React.createContext<ChatContext | undefined>(undefined);
@@ -128,6 +134,11 @@ interface ChatContextProps {
   children?: React.ReactNode;
   open: boolean;
   setOpen: (open: boolean) => void;
+  addChatSuggestionConfiguration: (
+    id: string,
+    suggestion: CopilotChatSuggestionConfiguration,
+  ) => void;
+  removeChatSuggestionConfiguration: (id: string) => void;
 }
 
 export const ChatContextProvider = ({
@@ -139,39 +150,40 @@ export const ChatContextProvider = ({
   children,
   open,
   setOpen,
+  addChatSuggestionConfiguration,
+  removeChatSuggestionConfiguration,
 }: ChatContextProps) => {
-  const context = useMemo(
-    () => ({
-      labels: {
-        ...{
-          initial: "",
-          title: "CopilotKit",
-          placeholder: "Type a message...",
-          thinking: "Thinking...",
-          error: "❌ An error occurred. Please try again.",
-          stopGenerating: "Stop generating",
-          regenerateResponse: "Regenerate response",
-        },
-        ...labels,
+  const context = {
+    labels: {
+      ...{
+        initial: "",
+        title: "CopilotKit",
+        placeholder: "Type a message...",
+        thinking: "Thinking...",
+        error: "❌ An error occurred. Please try again.",
+        stopGenerating: "Stop generating",
+        regenerateResponse: "Regenerate response",
       },
-
-      icons: {
-        ...{
-          openIcon: DefaultIcons.OpenIcon,
-          closeIcon: DefaultIcons.CloseIcon,
-          headerCloseIcon: DefaultIcons.HeaderCloseIcon,
-          sendIcon: DefaultIcons.SendIcon,
-          activityIcon: DefaultIcons.ActivityIcon,
-          spinnerIcon: DefaultIcons.SpinnerIcon,
-          stopIcon: DefaultIcons.StopIcon,
-          regenerateIcon: DefaultIcons.RegenerateIcon,
-        },
-        icons,
+      ...labels,
+    },
+
+    icons: {
+      ...{
+        openIcon: DefaultIcons.OpenIcon,
+        closeIcon: DefaultIcons.CloseIcon,
+        headerCloseIcon: DefaultIcons.HeaderCloseIcon,
+        sendIcon: DefaultIcons.SendIcon,
+        activityIcon: DefaultIcons.ActivityIcon,
+        spinnerIcon: DefaultIcons.SpinnerIcon,
+        stopIcon: DefaultIcons.StopIcon,
+        regenerateIcon: DefaultIcons.RegenerateIcon,
       },
-      open,
-      setOpen,
-    }),
-    [labels, icons, open, setOpen],
-  );
+      icons,
+    },
+    open,
+    setOpen,
+    addChatSuggestionConfiguration,
+    removeChatSuggestionConfiguration,
+  };
   return <ChatContext.Provider value={context}>{children}</ChatContext.Provider>;
 };

package/src/components/chat/Icons.tsx

@@ -92,6 +92,33 @@ export const SpinnerIcon = (
   </svg>
 );
 
+export const SmallSpinnerIcon = (
+  <svg
+    style={{
+      animation: "copilotKitSpinAnimation 1s linear infinite",
+    }}
+    width="13"
+    height="13"
+    xmlns="http://www.w3.org/2000/svg"
+    fill="none"
+    viewBox="0 0 24 24"
+  >
+    <circle
+      style={{ opacity: 0.25 }}
+      cx="12"
+      cy="12"
+      r="10"
+      stroke="currentColor"
+      strokeWidth="4"
+    ></circle>
+    <path
+      style={{ opacity: 0.75 }}
+      fill="currentColor"
+      d="M4 12a8 8 0 018-8V0C5.373 0 0 5.373 0 12h4zm2 5.291A7.962 7.962 0 014 12H0c0 3.042 1.135 5.824 3 7.938l3-2.647z"
+    ></path>
+  </svg>
+);
+
 export const ActivityIcon = (
   <svg
     style={{

package/src/components/chat/Input.tsx

@@ -3,7 +3,7 @@ import { InputProps } from "./props";
 import { useChatContext } from "./ChatContext";
 import AutoResizingTextarea from "./Textarea";
 
-export const Input = ({ inProgress, onSend, children, isVisible = false }: InputProps) => {
+export const Input = ({ inProgress, onSend, isVisible = false }: InputProps) => {
   const context = useChatContext();
   const textareaRef = useRef<HTMLTextAreaElement>(null);
 
@@ -34,7 +34,6 @@ export const Input = ({ inProgress, onSend, children, isVisible = false }: Input
 
   return (
     <div className="copilotKitInput" onClick={handleDivClick}>
-      <span>{children}</span>
       <button className="copilotKitSendButton" disabled={disabled} onClick={send}>
         {icon}
       </button>

package/src/components/chat/Messages.tsx

@@ -1,12 +1,12 @@
 import React, { useEffect, useMemo } from "react";
-import { MessagesProps } from "./props";
+import { MessagesProps, SuggestionsProps } from "./props";
 import { useChatContext } from "./ChatContext";
 import { nanoid } from "nanoid";
 import { Message, decodeResult } from "@copilotkit/shared";
 import { Markdown } from "./Markdown";
 import { ActionRenderProps, RenderFunctionStatus, useCopilotContext } from "@copilotkit/react-core";
 
-export const Messages = ({ messages, inProgress }: MessagesProps) => {
+export const Messages = ({ messages, inProgress, children }: MessagesProps) => {
   const { chatComponentsCache } = useCopilotContext();
   const context = useChatContext();
   const initialMessages = useMemo(
@@ -145,7 +145,7 @@ export const Messages = ({ messages, inProgress }: MessagesProps) => {
           );
         }
       })}
-      <div ref={messagesEndRef} />
+      <footer ref={messagesEndRef}>{children}</footer>
     </div>
   );
 };

package/src/components/chat/Popup.tsx

@@ -1,13 +1,59 @@
-import { CopilotChat, CopilotChatProps } from "./Chat";
+/**
+ * A chatbot popup component for CopilotKit.
+ *
+ * <img src="/images/CopilotPopup/CopilotPopup.gif" width="500" />
+ *
+ * <img referrerPolicy="no-referrer-when-downgrade" src="https://static.scarf.sh/a.png?x-pxid=a9b290bb-38f9-4518-ac3b-8f54fdbf43be" />
+ *
+ * A chatbot popup component for the CopilotKit framework. The component allows for a high degree
+ * of customization through various props and custom CSS.
+ *
+ * See [CopilotSidebar](./CopilotSidebar) for a sidebar version of this component.
+ *
+ * <RequestExample>
+ *   ```jsx CopilotPopup Example
+ *   import { CopilotPopup } from "@copilotkit/react-ui";
+ *
+ *   <CopilotPopup
+ *     labels={{
+ *       title: "Your Assistant",
+ *       initial: "Hi! 👋 How can I assist you today?",
+ *     }}
+ *   />
+ *   ```
+ * </RequestExample>
+ *
+ * ## Custom CSS
+ *
+ * You can customize the colors of the chat window by overriding the CSS variables
+ * defined in the [default styles](https://github.com/CopilotKit/CopilotKit/blob/main/CopilotKit/packages/react-ui/src/css/colors.css).
+ *
+ * For example, to set the primary color to purple:
+ *
+ * ```jsx
+ * <div style={{ "--copilot-kit-primary-color": "#7D5BA6" }}>
+ *   <CopilotPopup />
+ * </div>
+ * ```
+ *
+ * To further customize the chat window, you can override the CSS classes defined
+ * [here](https://github.com/CopilotKit/CopilotKit/blob/main/CopilotKit/packages/react-ui/src/css/).
+ *
+ * For example:
+ *
+ * ```css
+ * .copilotKitButton {
+ *   border-radius: 0;
+ * }
+ * ```
+ */
 
-interface CopilotPopupProps extends CopilotChatProps {
-  children?: React.ReactNode;
-}
+import { CopilotChat, CopilotChatProps } from "./Chat";
 
-export const CopilotPopup = (props: CopilotChatProps) => {
+export function CopilotPopup(props: CopilotChatProps) {
   props = {
     ...props,
     className: props.className ? props.className + " copilotKitPopup" : "copilotKitPopup",
   };
   return <CopilotChat {...props}>{props.children}</CopilotChat>;
-};
+}

package/src/components/chat/Sidebar.tsx

@@ -1,11 +1,64 @@
+/**
+ * A chatbot sidebar component for CopilotKit.
+ *
+ * <img src="/images/CopilotSidebar/CopilotSidebar.gif" width="500" />
+ *
+ * <img referrerPolicy="no-referrer-when-downgrade" src="https://static.scarf.sh/a.png?x-pxid=a9b290bb-38f9-4518-ac3b-8f54fdbf43be" />
+ *
+ * A chatbot sidebar component for the CopilotKit framework. Highly customizable through various props and custom CSS.
+ *
+ * <RequestExample>
+ *   ```jsx CopilotSidebar Example
+ *   import { CopilotSidebar } from "@copilotkit/react-ui";
+ *
+ *   <CopilotSidebar
+ *     labels={{
+ *       title: "Your Assistant",
+ *       initial: "Hi! 👋 How can I assist you today?",
+ *     }}
+ *   >
+ *     <YourApp/>
+ *   </CopilotSidebar>
+ *   ```
+ * </RequestExample>
+ *
+ *
+ * See [CopilotPopup](./CopilotPopup) for a popup version of this component.
+ *
+ * <Note>
+ *   To make the sidebar push your content to the side, wrap your content in the
+ *   sidebar component. If you want the sidebar to overlay your content, place the
+ *   sidebar component outside of your content.
+ * </Note>
+ *
+ * ## Custom CSS
+ *
+ * You can customize the colors of the chat window by overriding the CSS variables
+ * defined in the [default styles](https://github.com/CopilotKit/CopilotKit/blob/main/CopilotKit/packages/react-ui/src/css/colors.css).
+ *
+ * For example, to set the primary color to purple:
+ *
+ * ```jsx
+ * <div style={{ "--copilot-kit-primary-color": "#7D5BA6" }}>
+ *   <CopilotSidebar />
+ * </div>
+ * ```
+ *
+ * To further customize the chat window, you can override the CSS classes defined
+ * [here](https://github.com/CopilotKit/CopilotKit/blob/main/CopilotKit/packages/react-ui/src/css/).
+ *
+ * For example:
+ *
+ * ```css
+ * .copilotKitButton {
+ *   border-radius: 0;
+ * }
+ * ```
+ */
 import React, { useState } from "react";
 import { CopilotChat, CopilotChatProps } from "./Chat";
 
-interface CopilotSidebarProps extends CopilotChatProps {
-  children?: React.ReactNode;
-}
-
-export const CopilotSidebar = (props: CopilotSidebarProps) => {
+export function CopilotSidebar(props: CopilotChatProps) {
   props = {
     ...props,
     className: props.className ? props.className + " copilotKitSidebar" : "copilotKitSidebar",
@@ -26,4 +79,4 @@ export const CopilotSidebar = (props: CopilotSidebarProps) => {
       </CopilotChat>
     </div>
   );
-};
+}