@aj-archipelago/cortex 1.3.5 → 1.3.7

package/helper-apps/cortex-realtime-voice-server/client/src/chat/ChatMessageInput.tsx

@@ -0,0 +1,74 @@
+import React, { useCallback, useRef, useState } from "react";
+
+type ChatMessageInputProps = {
+  placeholder: string;
+  onSend?: (message: string) => void;
+  onStartStop?: () => void;
+};
+
+export const ChatMessageInput = ({
+  placeholder,
+  onSend,
+  onStartStop
+}: ChatMessageInputProps) => {
+  const [message, setMessage] = useState("");
+  const [isRecording, setIsRecording] = useState(false);
+  const inputRef = useRef<HTMLInputElement>(null);
+
+  const handleSend = useCallback(() => {
+    if (!onSend) {
+      return;
+    }
+    if (message === "") {
+      return;
+    }
+
+    onSend(message);
+    setMessage("");
+  }, [onSend, message]);
+
+  const handleStartStop = useCallback(() => {
+    if (!onStartStop) {
+      return;
+    }
+    setIsRecording(!isRecording);
+    onStartStop();
+  }, [isRecording, onStartStop]);
+
+  return (
+    <div className="flex flex-col gap-2 px-2 py-2 border-t border-t-gray-800">
+      <button
+        className="bg-transparent p-1.5 rounded-lg border-2 border-gray-600"
+        onClick={handleStartStop}>
+        <p className="text-s uppercase text-gray-200">{isRecording ? 'Stop' : 'Start'}</p>
+      </button>
+      <div className="flex flex-row pt-3 items-center">
+        <input
+          ref={inputRef}
+          className={`grow text-gray-200 bg-transparent border-2 border-gray-600 p-1.5 rounded-lg`}
+          style={{
+            paddingLeft: message.length > 0 ? 12 : 24,
+            caretShape: "block",
+          }}
+          placeholder={placeholder}
+          value={message}
+          onChange={(e: React.ChangeEvent<HTMLInputElement>) => {
+            e.target?.value && setMessage(e.target.value);
+          }}
+          onKeyDown={(e: React.KeyboardEvent<HTMLInputElement>) => {
+            if (e.key === "Enter") {
+              handleSend();
+            }
+          }}
+        />
+        <button
+          disabled={message.length === 0 || !onSend || !isRecording}
+          onClick={handleSend}
+          className={'bg-transparent p-1.5 ms-2 rounded-lg border-2 border-gray-600'}
+        >
+          <p className="text-s uppercase text-gray-200">Send</p>
+        </button>
+      </div>
+    </div>
+  );
+};

package/helper-apps/cortex-realtime-voice-server/client/src/chat/ChatTile.tsx

@@ -0,0 +1,211 @@
+import React, { useEffect, useRef, useState } from 'react';
+import ReactMarkdown, { Components } from 'react-markdown';
+import { Prism as SyntaxHighlighter } from 'react-syntax-highlighter';
+import { nightOwl } from 'react-syntax-highlighter/dist/esm/styles/prism';
+import remarkGfm from 'remark-gfm';
+import remarkMath from 'remark-math';
+import rehypeKatex from 'rehype-katex';
+import rehypeRaw from 'rehype-raw';
+import SendIcon from '@mui/icons-material/Send';
+import 'katex/dist/katex.min.css';
+import { CopyButton } from './components/CopyButton';
+
+// Define the code component props interface
+interface CodeComponentProps {
+  node?: any;
+  inline?: boolean;
+  className?: string;
+  children?: React.ReactNode;
+}
+
+// Add types for math components
+interface MathComponentProps {
+  value: string;
+}
+
+// Extend Components type to include math components
+interface MarkdownComponents extends Components {
+  math?: (props: MathComponentProps) => JSX.Element;
+  inlineMath?: (props: MathComponentProps) => JSX.Element;
+}
+
+export type ChatMessage = {
+  id: string;
+  isSelf: boolean;
+  name: string;
+  message: string;
+  timestamp: number;
+};
+
+export type ChatTileProps = {
+  messages: ChatMessage[];
+  onSend: (message: string) => void;
+  isConnected?: boolean;
+};
+
+const MessageContent = ({ message }: { message: string }) => {
+  if (message.match(/^https?:\/\/.*\.(jpg|jpeg|png|gif|webp)$/i)) {
+    return (
+      <img 
+        src={message} 
+        alt="Shared" 
+        className="max-w-full rounded-lg max-h-64 object-contain"
+      />
+    );
+  }
+
+  const components: MarkdownComponents = {
+    code({ inline, className, children }: CodeComponentProps) {
+      const match = /language-(\w+)|^(\w+)/.exec(className || '');
+      const language = match ? match[1] || match[2] : '';
+      
+      if (!inline && language) {
+        return (
+          <div className="rounded-lg overflow-hidden my-1.5 relative group">
+            <div className="flex justify-between items-center px-2 py-1 bg-gray-800/50">
+              <div className="text-xs text-gray-400 select-none">
+                {language}
+              </div>
+              <div className="opacity-0 group-hover:opacity-100 transition-opacity duration-200">
+                <CopyButton text={String(children)} />
+              </div>
+            </div>
+            <SyntaxHighlighter
+              style={nightOwl}
+              language={language}
+              customStyle={{
+                margin: 0,
+                background: 'transparent',
+                fontSize: '0.75rem',
+              }}
+            >
+              {String(children).replace(/\n$/, '')}
+            </SyntaxHighlighter>
+          </div>
+        );
+      }
+      
+      return (
+        <code className="bg-gray-800/50 px-1 py-0.5 rounded text-cyan-300 text-xs inline-block">
+          {String(children)}
+        </code>
+      );
+    },
+    // Style links
+    a: ({ node, children, ...props }) => (
+      <a 
+        {...props} 
+        className="text-cyan-400 hover:text-cyan-300 underline text-sm"
+        target="_blank" 
+        rel="noopener noreferrer" 
+      >
+        {children}
+      </a>
+    ),
+    // Style tables
+    table: ({ node, ...props }) => (
+      <div className="overflow-x-auto my-2">
+        <table {...props} className="border-collapse table-auto w-full text-sm" />
+      </div>
+    ),
+    th: ({ node, ...props }) => (
+      <th {...props} className="border border-gray-600 px-3 py-1.5 bg-gray-800 text-sm" />
+    ),
+    td: ({ node, ...props }) => (
+      <td {...props} className="border border-gray-600 px-3 py-1.5 text-sm" />
+    ),
+    // Add special styling for math blocks
+    math: ({ value }) => (
+      <div className="py-1.5 overflow-x-auto text-sm">
+        <span>{value}</span>
+      </div>
+    ),
+    inlineMath: ({ value }) => (
+      <span className="px-1 text-sm">{value}</span>
+    ),
+  };
+
+  return (
+    <ReactMarkdown
+      className="prose prose-invert prose-sm max-w-none text-sm [&>p]:text-sm [&>ul]:text-sm [&>ol]:text-sm"
+      remarkPlugins={[remarkGfm, remarkMath]}
+      rehypePlugins={[rehypeRaw, rehypeKatex]}
+      components={components}
+    >
+      {message}
+    </ReactMarkdown>
+  );
+};
+
+export function ChatTile({ messages, onSend, isConnected = false }: ChatTileProps) {
+  const [message, setMessage] = useState('');
+  const messagesEndRef = useRef<HTMLDivElement>(null);
+
+  // Sort messages by timestamp before rendering
+  const sortedMessages = [...messages].sort((a, b) => a.timestamp - b.timestamp);
+
+  const handleSubmit = (e: React.FormEvent) => {
+    e.preventDefault();
+    if (message.trim()) {
+      onSend(message.trim());
+      setMessage('');
+    }
+  };
+
+  useEffect(() => {
+    messagesEndRef.current?.scrollIntoView({ behavior: 'smooth' });
+  }, [messages]);
+
+  return (
+    <div className="h-full flex flex-col">
+      {/* Messages container - scrollable */}
+      <div className="flex-1 h-0 overflow-y-auto">
+        <div className="p-4 space-y-4">
+          {sortedMessages.map((msg) => (
+            <div key={msg.id} className="flex flex-col">
+              <div className={`w-full rounded-lg p-2.5 relative group ${
+                msg.isSelf 
+                  ? 'bg-blue-500/30 border border-blue-500/20' 
+                  : 'bg-gray-700/30 border border-gray-600/20'
+              }`}>
+                <div className="flex justify-between items-start">
+                  <div className="text-2xs text-gray-400 mb-1">{msg.name}</div>
+                  <div className="opacity-0 group-hover:opacity-100 transition-opacity duration-200">
+                    <CopyButton text={msg.message} />
+                  </div>
+                </div>
+                <MessageContent message={msg.message} />
+              </div>
+            </div>
+          ))}
+          <div ref={messagesEndRef} aria-hidden="true" />
+        </div>
+      </div>
+
+      {/* Input area - fixed height */}
+      <div className="flex-none h-[68px] border-t border-gray-700/50 p-4 bg-gray-900/30 backdrop-blur-sm">
+        <form onSubmit={handleSubmit} className="flex space-x-2">
+          <input
+            type="text"
+            value={message}
+            onChange={(e) => setMessage(e.target.value)}
+            className={`flex-grow bg-gray-800/50 border border-gray-700/50 rounded-lg px-3 py-1.5 
+                     text-sm text-gray-100 focus:outline-none focus:ring-2 focus:ring-cyan-500/50
+                     ${!isConnected && 'opacity-50 cursor-not-allowed'}`}
+            placeholder={isConnected ? "Type a message..." : "Connect to send messages..."}
+            disabled={!isConnected}
+          />
+          <button
+            type="submit"
+            disabled={!isConnected}
+            className={`p-1.5 rounded-lg bg-gradient-to-r from-blue-500 to-cyan-500 
+                     ${isConnected ? 'hover:from-blue-600 hover:to-cyan-600' : 'opacity-50 cursor-not-allowed'}
+                     shadow-lg shadow-cyan-500/20`}
+          >
+            <SendIcon sx={{ fontSize: 16 }} />
+          </button>
+        </form>
+      </div>
+    </div>
+  );
+}

package/helper-apps/cortex-realtime-voice-server/client/src/chat/audio/SoundEffects.ts

@@ -0,0 +1,56 @@
+export class SoundEffects {
+  private static audioContext: AudioContext | null = null;
+  private static connectBuffer: AudioBuffer | null = null;
+  private static disconnectBuffer: AudioBuffer | null = null;
+
+  private static async getAudioContext() {
+    if (!this.audioContext) {
+      this.audioContext = new AudioContext();
+    }
+    return this.audioContext;
+  }
+
+  private static async loadSound(url: string): Promise<AudioBuffer> {
+    const context = await this.getAudioContext();
+    const response = await fetch(url);
+    const arrayBuffer = await response.arrayBuffer();
+    return await context.decodeAudioData(arrayBuffer);
+  }
+
+  static async init() {
+    try {
+      this.connectBuffer = await this.loadSound('/sounds/connect.mp3');
+      this.disconnectBuffer = await this.loadSound('/sounds/disconnect.mp3');
+    } catch (error) {
+      console.error('Failed to load sound effects:', error);
+    }
+  }
+
+  static async playConnect() {
+    if (!this.connectBuffer) return;
+    
+    try {
+      const context = await this.getAudioContext();
+      const source = context.createBufferSource();
+      source.buffer = this.connectBuffer;
+      source.connect(context.destination);
+      source.start(0);
+    } catch (error) {
+      console.error('Failed to play connect sound:', error);
+    }
+  }
+
+  static async playDisconnect() {
+    if (!this.disconnectBuffer) return;
+    
+    try {
+      const context = await this.getAudioContext();
+      const source = context.createBufferSource();
+      source.buffer = this.disconnectBuffer;
+      source.connect(context.destination);
+      source.start(0);
+    } catch (error) {
+      console.error('Failed to play disconnect sound:', error);
+    }
+  }
+} 

package/helper-apps/cortex-realtime-voice-server/client/src/chat/audio/WavPacker.ts

@@ -0,0 +1,112 @@
+/**
+ * Raw wav audio file contents
+ */
+export interface WavPackerAudioType {
+  blob: Blob;
+  url: string;
+  channelCount: number;
+  sampleRate: number;
+  duration: number;
+}
+
+/**
+ * Utility class for assembling PCM16 "audio/wav" data
+ * @class
+ */
+export class WavPacker {
+  /**
+   * Converts Float32Array of amplitude data to ArrayBuffer in Int16Array format
+   * @param {Float32Array} float32Array
+   * @returns {ArrayBuffer}
+   */
+  static floatTo16BitPCM(float32Array: Float32Array): ArrayBuffer {
+    const buffer = new ArrayBuffer(float32Array.length * 2);
+    const view = new DataView(buffer);
+    let offset = 0;
+    for (let i = 0; i < float32Array.length; i++, offset += 2) {
+      let s = Math.max(-1, Math.min(1, float32Array[i] || 0));
+      view.setInt16(offset, s < 0 ? s * 0x8000 : s * 0x7fff, true);
+    }
+    return buffer;
+  }
+
+  /**
+   * Concatenates two ArrayBuffers
+   * @param {ArrayBuffer} leftBuffer
+   * @param {ArrayBuffer} rightBuffer
+   * @returns {ArrayBuffer}
+   */
+  static mergeBuffers(leftBuffer: ArrayBuffer, rightBuffer: ArrayBuffer): ArrayBuffer {
+    const tmpArray = new Uint8Array(
+      leftBuffer.byteLength + rightBuffer.byteLength
+    );
+    tmpArray.set(new Uint8Array(leftBuffer), 0);
+    tmpArray.set(new Uint8Array(rightBuffer), leftBuffer.byteLength);
+    return tmpArray.buffer as ArrayBuffer;
+  }
+
+  /**
+   * Packs data into an Int16 format
+   * @private
+   * @param {number} size 0 = 1x Int16, 1 = 2x Int16
+   * @param {number} arg value to pack
+   * @returns
+   */
+  _packData(size: number, arg: number): Uint8Array {
+    return [
+      new Uint8Array([arg, arg >> 8]),
+      new Uint8Array([arg, arg >> 8, arg >> 16, arg >> 24]),
+    ][size] || new Uint8Array();
+  }
+
+  /**
+   * Packs audio into "audio/wav" Blob
+   * @param {number} sampleRate
+   * @param {{bitsPerSample: number, channels: Array<Float32Array>, data: Int16Array}} audio
+   * @returns {WavPackerAudioType}
+   */
+  pack(sampleRate: number, audio: { bitsPerSample: number; channels: Array<Float32Array>; data: Int16Array; }): WavPackerAudioType {
+    if (!audio?.bitsPerSample) {
+      throw new Error(`Missing "bitsPerSample"`);
+    } else if (!audio?.channels) {
+      throw new Error(`Missing "channels"`);
+    } else if (!audio?.data) {
+      throw new Error(`Missing "data"`);
+    }
+    const { bitsPerSample, channels, data } = audio;
+    const output: (string | Uint8Array | Int16Array)[] = [
+      // Header
+      'RIFF',
+      this._packData(
+        1,
+        4 + (8 + 24) /* chunk 1 length */ + (8 + 8) /* chunk 2 length */
+      ), // Length
+      'WAVE',
+      // chunk 1
+      'fmt ', // Sub-chunk identifier
+      this._packData(1, 16), // Chunk length
+      this._packData(0, 1), // Audio format (1 is linear quantization)
+      this._packData(0, channels.length),
+      this._packData(1, sampleRate),
+      this._packData(1, (sampleRate * channels.length * bitsPerSample) / 8), // Byte rate
+      this._packData(0, (channels.length * bitsPerSample) / 8),
+      this._packData(0, bitsPerSample),
+      // chunk 2
+      'data', // Sub-chunk identifier
+      this._packData(
+        1,
+        ((channels[0] ? channels[0].length : 0) * channels.length * bitsPerSample) / 8
+      ), // Chunk length
+      data,
+    ];
+    const blob = new Blob(output, { type: 'audio/mpeg' });
+    const url = URL.createObjectURL(blob);
+    return {
+      blob,
+      url,
+      channelCount: channels.length,
+      sampleRate,
+      duration: data.byteLength / (channels.length * sampleRate * 2),
+    };
+  }
+}