@clikvn/agent-widget-embedded 0.0.3-dev → 0.0.4-dev

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@clikvn/agent-widget-embedded",
   "description": "This is agent widget",
-  "version": "0.0.3-dev",
+  "version": "0.0.4-dev",
   "author": "Clik JSC",
   "license": "ISC",
   "type": "module",
@@ -32,6 +32,7 @@
     "@types/react-dom": "^18.3.1",
     "class-variance-authority": "^0.7.1",
     "clsx": "^2.1.1",
+    "device-detector-js": "^3.0.3",
     "framer-motion": "^11.18.0",
     "react": "^18.3.1",
     "react-dom": "^18.3.1",

package/src/components/Chat/Icons.tsx

@@ -477,14 +477,14 @@ export const MoreIcon = ({ size = 16 }: { size?: number }) => {
   );
 };
 
-export const TrashIcon = ({ size = 16 }: { size?: number }) => {
+export const TrashIcon = ({ size = 16, color }: { size?: number, color?: string }) => {
   return (
     <svg
       height={size}
       strokeLinejoin="round"
       viewBox="0 0 16 16"
       width={size}
-      style={{ color: 'currentcolor' }}
+      style={{ color: color || 'currentcolor' }}
     >
       <path
         fillRule="evenodd"
@@ -859,9 +859,9 @@ export const CheckCirclFillIcon = ({ size = 16 }: { size?: number }) => {
 export const MicrophoneIcon = ({ size = 16 }: { size?: number }) => {
   return (
     <svg
-      height={size}
       strokeLinejoin="round"
       viewBox="0 0 490.9 490.9"
+      height={size}
       width={size}
       style={{ color: 'currentcolor' }}
     >
@@ -881,3 +881,20 @@ export const MicrophoneIcon = ({ size = 16 }: { size?: number }) => {
     </svg>
   );
 };
+
+export const CircleDotIcon = ({ size = 16, color = 'red' }: { size?: number, color?: string }) => (
+  <svg
+    xmlns="http://www.w3.org/2000/svg"
+    height={size}
+    width={size}
+    viewBox="0 0 24 24"
+    fill="none"
+    stroke={color}
+    stroke-width="2"
+    stroke-linecap="round"
+    stroke-linejoin="round"
+  >
+    <circle cx="12" cy="12" r="10" />
+    <circle cx="12" cy="12" r="1" />
+  </svg>
+);

package/src/components/Chat/Markdown.tsx

@@ -243,7 +243,18 @@ const NonMemoizedMarkdown: FC<{
             content.indexOf('.jpeg') >= 0 ||
             content.indexOf('.png') >= 0 ||
             content.indexOf('.gif') >= 0);
-        if (isImageUrl) {
+        const isAudioUrl =
+          content &&
+          (content.indexOf('.mp3') >= 0 ||
+            content.indexOf('.wav') >= 0 ||
+            content.indexOf('.ogg') >= 0);
+        if (isAudioUrl) {
+          return (
+            <audio controls>
+              <source src={content} />
+            </audio>
+          );
+        } else if (isImageUrl) {
           return (
             <a href={content} target="_blank" rel="noopener noreferrer">
               <img

package/src/components/Chat/MultimodalInput.tsx

@@ -12,15 +12,25 @@ import {
   cn,
   generateExtendedFileName,
   generateUUID,
+  sleep,
 } from '../../utils/commonUtils';
 import { PreviewAttachment } from './PreviewAttachment';
-import { ArrowUpIcon, PaperclipIcon, StopIcon } from './Icons';
+import {
+  ArrowUpIcon,
+  CircleDotIcon,
+  MicrophoneIcon,
+  PaperclipIcon,
+  PlusIcon,
+  StopIcon,
+  TrashIcon,
+} from './Icons';
 import { ChatMessageType, IFileUpload } from '../../types/flowise.type';
 import { BotType } from '../../types/bot.type';
 
 import { createAttachments } from '../../services/chat.service';
 import { Button } from './ui/Button';
 import { Textarea } from './ui/Textarea';
+import { useAudioRecording } from '../../hooks/useAudioRecording';
 
 const suggestedActions = [
   {
@@ -81,6 +91,24 @@ export const MultimodalInput: FC<PropsType> = ({
   bot,
   apiHost,
 }) => {
+  const {
+    isRecording,
+    setIsRecording,
+    onRecordingCancelled,
+    onRecordingStopped,
+    elapsedTime,
+    isLoadingRecording,
+  } = useAudioRecording({
+    addRecordingToPreviews: async (blob: Blob) => {
+      try {
+        const audioFile = await toAudioBase64(blob);
+        handleSubmit(undefined, [audioFile]);
+        setIsRecording(false);
+      } catch (error) {
+        console.error('Error uploading files!', error);
+      }
+    },
+  });
   const textareaRef = useRef<HTMLTextAreaElement | null>(null);
   const { width } = useWindowSize();
   useEffect(() => {
@@ -130,7 +158,7 @@ export const MultimodalInput: FC<PropsType> = ({
     handleSubmit(undefined, attachments);
     setLocalStorageInput('');
     if (setAttachments) {
-      setAttachments((currentAttachments: IFileUpload[]) => []);
+      setAttachments((_) => []);
       if (fileInputRef.current) {
         (fileInputRef.current as HTMLInputElement).value = '';
       }
@@ -157,6 +185,33 @@ export const MultimodalInput: FC<PropsType> = ({
     [chatId]
   );
 
+  const toAudioBase64 = (blob: Blob) => {
+    return new Promise<IFileUpload>((resolve) => {
+      let mimeType = '';
+      const pos = blob.type.indexOf(';');
+      if (pos === -1) {
+        mimeType = blob.type;
+      } else {
+        mimeType = blob.type.substring(0, pos);
+      }
+
+      // read blob and add to previews
+      const reader = new FileReader();
+      reader.readAsDataURL(blob);
+      reader.onloadend = () => {
+        const base64data = reader.result as string;
+        const upload: IFileUpload = {
+          tempId: generateUUID(),
+          data: base64data,
+          type: 'audio',
+          name: `audio_${Date.now()}.wav`,
+          mime: mimeType,
+        };
+        resolve(upload);
+      };
+    });
+  };
+
   const toBase64 = async (
     file: File,
     type = 'file'
@@ -213,7 +268,6 @@ export const MultimodalInput: FC<PropsType> = ({
   const handleFileChange = useCallback(
     async (event: ChangeEvent<HTMLInputElement>) => {
       const files = Array.from(event.target.files || []);
-
       try {
         const uploadPromises = files.map((file) => checkUploadFile(file));
         const uploadedAttachments = await Promise.all(uploadPromises);
@@ -347,24 +401,67 @@ export const MultimodalInput: FC<PropsType> = ({
           className="rounded-full p-1.5 h-fit absolute bottom-2 right-2 m-0.5 border dark:border-zinc-600"
           onClick={(event) => {
             event.preventDefault();
-            submitForm();
+            if (isRecording) {
+              onRecordingStopped();
+            } else {
+              submitForm();
+            }
           }}
-          disabled={input.length === 0 || !!uploadQueue.length}
+          disabled={
+            !isRecording && (input.length === 0 || !!uploadQueue.length)
+          } // zero input or uploading
         >
           <ArrowUpIcon size={14} />
         </Button>
       )}
-
+      {isRecording ? (
+        <>
+          <div
+            className="rounded-full bg-background flex absolute p-1 bottom-2 right-[80px]"
+            data-testid="input"
+          >
+            <div className="flex items-center gap-3">
+              <span>
+                <CircleDotIcon color="red" />
+              </span>
+              <span>{elapsedTime || '00:00'}</span>
+              {isLoadingRecording && <span className="ml-1.5">Sending...</span>}
+            </div>
+          </div>
+          <Button
+            className="rounded-full p-1.5 h-fit absolute bottom-2 right-11 m-0.5 dark:border-zinc-700"
+            variant="outline"
+            onClick={(event) => {
+              event.preventDefault();
+              onRecordingCancelled();
+            }}
+          >
+            <TrashIcon size={14} color="red" />
+          </Button>
+        </>
+      ) : (
+        <Button
+          className="rounded-full p-1.5 h-fit absolute bottom-2 right-11 m-0.5 dark:border-zinc-700"
+          onClick={(event) => {
+            event.preventDefault();
+            setIsRecording(true);
+          }}
+          variant="outline"
+          disabled={isLoading}
+        >
+          <MicrophoneIcon size={14} />
+        </Button>
+      )}
       <Button
-        className="rounded-full p-1.5 h-fit absolute bottom-2 right-11 m-0.5 dark:border-zinc-700"
+        className="rounded-full p-1.5 h-fit absolute bottom-2 left-2 m-0.5 dark:border-zinc-700"
         onClick={(event) => {
           event.preventDefault();
           fileInputRef.current?.click();
         }}
         variant="outline"
-        disabled={isLoading}
+        disabled={isLoading || isRecording}
       >
-        <PaperclipIcon size={14} />
+        <PlusIcon size={14} />
       </Button>
     </div>
   );

package/src/components/Chat/PreviewAttachment.tsx

@@ -1,5 +1,6 @@
 import { IFileUpload } from '../../types/flowise.type';
 import { LoaderIcon } from './Icons';
+import React from 'react';
 
 export const PreviewAttachment = ({
   attachment,
@@ -11,9 +12,13 @@ export const PreviewAttachment = ({
   const { name, data, mime, tempId } = attachment;
   return (
     <div className="flex flex-col gap-2">
-      <div className="w-30 p-2 max-w-[200px] bg-muted rounded-md relative flex flex-col items-center justify-center">
+      <div className="w-30 p-0 max-w-[400px] bg-muted rounded-md relative flex flex-col items-center justify-center">
         {data ? (
-          mime.startsWith('image') ? (
+          mime.startsWith('audio') ? (
+            <audio controls>
+              <source src={data} type={mime} />
+            </audio>
+          ) : mime.startsWith('image') ? (
             <img
               key={tempId}
               src={data}
@@ -33,9 +38,9 @@ export const PreviewAttachment = ({
           </div>
         )}
       </div>
-      <div className="text-xs text-zinc-500 max-w-[200px] truncate ">
-        {name}
-      </div>
+      {/*<div className="text-xs text-zinc-500 max-w-[200px] truncate ">*/}
+      {/*  {name}*/}
+      {/*</div>*/}
     </div>
   );
 };

package/src/hooks/useAudioRecording.ts

@@ -0,0 +1,54 @@
+import { useEffect, useState } from 'react';
+import {
+  cancelAudioRecording,
+  startAudioRecording,
+  stopAudioRecording,
+} from '../utils/audioRecording';
+
+type Props = {
+  addRecordingToPreviews: (blob: Blob) => void;
+};
+
+export const useAudioRecording = (props: Props) => {
+  const { addRecordingToPreviews } = props;
+  const [elapsedTime, setElapsedTime] = useState('00:00');
+  const [recordingNotSupported, setRecordingNotSupported] = useState(false);
+  const [isLoadingRecording, setIsLoadingRecording] = useState(false);
+  const [isRecording, setIsRecording] = useState(false);
+
+  useEffect(() => {
+    if (isRecording) {
+      onRecordingStarted();
+    }
+  }, [isRecording]);
+  const onRecordingStarted = () => {
+    setIsRecording(true);
+    startAudioRecording(
+      setIsRecording,
+      setRecordingNotSupported,
+      setElapsedTime
+    );
+  };
+
+  const onRecordingCancelled = () => {
+    if (!recordingNotSupported) cancelAudioRecording();
+    setIsRecording(false);
+    setRecordingNotSupported(false);
+  };
+
+  const onRecordingStopped = () => {
+    setIsLoadingRecording(true);
+    stopAudioRecording(addRecordingToPreviews);
+  };
+
+  return {
+    elapsedTime,
+    recordingNotSupported,
+    isLoadingRecording,
+    isRecording,
+    setIsRecording,
+    onRecordingCancelled,
+    onRecordingStopped,
+    onRecordingStarted,
+  };
+};

package/src/hooks/useChat.ts

@@ -3,6 +3,7 @@ import { useCallback, useEffect, useRef, useState } from 'react';
 import { StreamResponse } from '../models/FlowiseClient';
 import { predict } from '../services/chat.service';
 import {
+  ChatMessageMetadataType,
   ChatMessageType,
   IFileUpload,
   PredictionData,
@@ -83,6 +84,13 @@ export const useChat = (props: PropsType): ReturnType => {
       } else if (chunk.event == 'sourceDocuments') {
         newMessage.sourceDocuments = chunk.data as SourceDocument[];
         mutateMessages([...msgs, { ...newMessage }]);
+      } else if (chunk.event == 'metadata') {
+        newMessage.metaData = chunk.data as ChatMessageMetadataType;
+        const lastMsg = msgs[msgs.length - 1];
+        if (lastMsg) {
+          lastMsg.content = newMessage.metaData.question;
+        }
+        mutateMessages([...msgs, { ...newMessage }]);
       }
     };
 
@@ -189,20 +197,20 @@ export const useChat = (props: PropsType): ReturnType => {
     async (event?: { preventDefault?: () => void }, files?: IFileUpload[]) => {
       event?.preventDefault?.();
 
-      if (!input) return;
+      if (!input && !files) return;
 
       const msgs = messagesRef.current.concat({
         id: generateUUID(),
         chatId: chatIdRef.current,
         role: 'userMessage',
-        content: input,
+        content: input || '',
         createdDate: new Date().toISOString(),
         fileUploads: files || [],
       });
 
       triggerRequest(msgs, {
         chatId: chatIdRef.current,
-        question: input,
+        question: input || '',
         uploads: files || [],
         chatflowId: bot?.id,
         overrideConfig: overrideConfig?.overrideConfig,

package/src/types/flowise.type.ts

@@ -31,6 +31,13 @@ export interface ChatMessageType {
   fileUploads?: IFileUpload[];
   fileAnnotations?: FileAnnotation[];
   agentReasoning?: AgentReasoning[];
+  metaData?: ChatMessageMetadataType;
+}
+
+export interface ChatMessageMetadataType {
+  chatId: string;
+  chatMessageId?: string;
+  question?: string;
 }
 
 export interface IAction {