@aj-archipelago/cortex 1.3.5 → 1.3.7

package/helper-apps/cortex-realtime-voice-server/client/src/chat/audio/worklets/AudioProcessor.ts

@@ -0,0 +1,214 @@
+const AudioProcessorWorklet = `
+class AudioProcessor extends AudioWorkletProcessor {
+
+  constructor() {
+    super();
+    this.port.onmessage = this.receive.bind(this);
+    this.initialize();
+  }
+
+  initialize() {
+    this.foundAudio = false;
+    this.recording = false;
+    this.chunks = [];
+  }
+
+  /**
+   * Concatenates sampled chunks into channels
+   * Format is chunk[Left[], Right[]]
+   */
+  readChannelData(chunks, channel = -1, maxChannels = 9) {
+    let channelLimit;
+    if (channel !== -1) {
+      if (chunks[0] && chunks[0].length - 1 < channel) {
+        throw new Error(
+          \`Channel \${channel} out of range: max \${chunks[0].length}\`
+        );
+      }
+      channelLimit = channel + 1;
+    } else {
+      channel = 0;
+      channelLimit = Math.min(chunks[0] ? chunks[0].length : 1, maxChannels);
+    }
+    const channels = [];
+    for (let n = channel; n < channelLimit; n++) {
+      const length = chunks.reduce((sum, chunk) => {
+        return sum + chunk[n].length;
+      }, 0);
+      const buffers = chunks.map((chunk) => chunk[n]);
+      const result = new Float32Array(length);
+      let offset = 0;
+      for (let i = 0; i < buffers.length; i++) {
+        result.set(buffers[i], offset);
+        offset += buffers[i].length;
+      }
+      channels[n] = result;
+    }
+    return channels;
+  }
+
+  /**
+   * Combines parallel audio data into correct format,
+   * channels[Left[], Right[]] to float32Array[LRLRLRLR...]
+   */
+  formatAudioData(channels) {
+    if (channels.length === 1) {
+      // Simple case is only one channel
+      const float32Array = channels[0].slice();
+      const meanValues = channels[0].slice();
+      return { float32Array, meanValues };
+    } else {
+      const float32Array = new Float32Array(
+        channels[0].length * channels.length
+      );
+      const meanValues = new Float32Array(channels[0].length);
+      for (let i = 0; i < channels[0].length; i++) {
+        const offset = i * channels.length;
+        let meanValue = 0;
+        for (let n = 0; n < channels.length; n++) {
+          float32Array[offset + n] = channels[n][i];
+          meanValue += channels[n][i];
+        }
+        meanValues[i] = meanValue / channels.length;
+      }
+      return { float32Array, meanValues };
+    }
+  }
+
+  /**
+   * Converts 32-bit float data to 16-bit integers
+   */
+  floatTo16BitPCM(float32Array) {
+    const buffer = new ArrayBuffer(float32Array.length * 2);
+    const view = new DataView(buffer);
+    let offset = 0;
+    for (let i = 0; i < float32Array.length; i++, offset += 2) {
+      let s = Math.max(-1, Math.min(1, float32Array[i]));
+      view.setInt16(offset, s < 0 ? s * 0x8000 : s * 0x7fff, true);
+    }
+    return buffer;
+  }
+
+  /**
+   * Retrieves the most recent amplitude values from the audio stream
+   * @param {number} channel
+   */
+  getValues(channel = -1) {
+    const channels = this.readChannelData(this.chunks, channel);
+    const { meanValues } = this.formatAudioData(channels);
+    return { meanValues, channels };
+  }
+
+  /**
+   * Exports chunks as an audio/wav file
+   */
+  export() {
+    const channels = this.readChannelData(this.chunks);
+    const { float32Array, meanValues } = this.formatAudioData(channels);
+    const audioData = this.floatTo16BitPCM(float32Array);
+    return {
+      meanValues: meanValues,
+      audio: {
+        bitsPerSample: 16,
+        channels: channels,
+        data: audioData,
+      },
+    };
+  }
+
+  receive(e) {
+    const { event, id } = e.data;
+    let receiptData = {};
+    switch (event) {
+      case 'start':
+        this.recording = true;
+        break;
+      case 'stop':
+        this.recording = false;
+        break;
+      case 'clear':
+        this.initialize();
+        break;
+      case 'export':
+        receiptData = this.export();
+        break;
+      case 'read':
+        receiptData = this.getValues();
+        break;
+      default:
+        break;
+    }
+    // Always send back receipt
+    this.port.postMessage({ event: 'receipt', id, data: receiptData });
+  }
+
+  sendChunk(chunk) {
+    const channels = this.readChannelData([chunk]);
+    const { float32Array, meanValues } = this.formatAudioData(channels);
+    const rawAudioData = this.floatTo16BitPCM(float32Array);
+    const monoAudioData = this.floatTo16BitPCM(meanValues);
+    this.port.postMessage({
+      event: 'chunk',
+      data: {
+        mono: monoAudioData,
+        raw: rawAudioData,
+      },
+    });
+  }
+
+  process(inputList, outputList, parameters) {
+    // Copy input to output (e.g. speakers)
+    // Note that this creates choppy sounds with Mac products
+    const sourceLimit = Math.min(inputList.length, outputList.length);
+    for (let inputNum = 0; inputNum < sourceLimit; inputNum++) {
+      const input = inputList[inputNum];
+      const output = outputList[inputNum];
+      const channelCount = Math.min(input.length, output.length);
+      for (let channelNum = 0; channelNum < channelCount; channelNum++) {
+        input[channelNum].forEach((sample, i) => {
+          output[channelNum][i] = sample;
+        });
+      }
+    }
+    const inputs = inputList[0];
+    // There's latency at the beginning of a stream before recording starts
+    // Make sure we actually receive audio data before we start storing chunks
+    let sliceIndex = 0;
+    if (!this.foundAudio) {
+      for (const channel of inputs) {
+        sliceIndex = 0; // reset for each channel
+        if (this.foundAudio) {
+          break;
+        }
+        if (channel) {
+          for (const value of channel) {
+            if (value !== 0) {
+              // find only one non-zero entry in any channel
+              this.foundAudio = true;
+              break;
+            } else {
+              sliceIndex++;
+            }
+          }
+        }
+      }
+    }
+    if (inputs && inputs[0] && this.foundAudio && this.recording) {
+      // We need to copy the TypedArray, because the \`process\`
+      // internals will reuse the same buffer to hold each input
+      const chunk = inputs.map((input) => input.slice(sliceIndex));
+      this.chunks.push(chunk);
+      this.sendChunk(chunk);
+    }
+    return true;
+  }
+}
+
+registerProcessor('audio_processor', AudioProcessor);
+`;
+
+const script = new Blob([AudioProcessorWorklet], {
+  type: 'application/javascript',
+});
+const src = URL.createObjectURL(script);
+export const AudioProcessorSrc: string = src;

package/helper-apps/cortex-realtime-voice-server/client/src/chat/audio/worklets/StreamProcessor.ts

@@ -0,0 +1,183 @@
+export const StreamProcessorWorklet = `
+class StreamProcessor extends AudioWorkletProcessor {
+  constructor() {
+    super();
+    this.hasStarted = false;
+    this.hasInterrupted = false;
+    this.outputBuffers = [];
+    this.bufferLength = 128;
+    this.minBufferSize = 3;
+    this.write = { buffer: new Float32Array(this.bufferLength), trackId: null };
+    this.writeOffset = 0;
+    this.trackSampleOffsets = {};
+    this.lastErrorTime = 0;
+    this.errorCount = 0;
+    this.noBufferCount = 0;
+    this.maxNoBufferFrames = 100;
+    this.lastUnderrunLog = 0;
+
+    this.port.onmessage = (event) => {
+      try {
+        if (event.data) {
+          const payload = event.data;
+          if (payload.event === 'write') {
+            const int16Array = payload.buffer;
+            const float32Array = new Float32Array(int16Array.length);
+            for (let i = 0; i < int16Array.length; i++) {
+              float32Array[i] = int16Array[i] / 0x8000;
+            }
+            this.writeData(float32Array, payload.trackId);
+          } else if (payload.event === 'config') {
+            this.minBufferSize = payload.minBufferSize;
+          } else if (
+            payload.event === 'offset' ||
+            payload.event === 'interrupt'
+          ) {
+            const requestId = payload.requestId;
+            const trackId = this.write.trackId;
+            const offset = this.trackSampleOffsets[trackId] || 0;
+            this.port.postMessage({
+              event: 'offset',
+              requestId,
+              trackId,
+              offset,
+            });
+            if (payload.event === 'interrupt') {
+              this.hasInterrupted = true;
+            }
+          } else {
+            throw new Error(\`Unhandled event "\${payload.event}"\`);
+          }
+        }
+      } catch (error) {
+        this.handleError(error);
+      }
+    };
+  }
+
+  handleError(error) {
+    const now = currentTime;
+    if (now - this.lastErrorTime > 5) {
+      // Reset error count if more than 5 seconds have passed
+      this.errorCount = 0;
+    }
+    this.lastErrorTime = now;
+    this.errorCount++;
+
+    if (this.errorCount <= 3) {
+      this.port.postMessage({
+        event: 'error',
+        error: error.message || 'Unknown error in stream processor'
+      });
+    }
+  }
+
+  writeData(float32Array, trackId = null) {
+    try {
+      let { buffer } = this.write;
+      let offset = this.writeOffset;
+      
+      for (let i = 0; i < float32Array.length; i++) {
+        buffer[offset++] = float32Array[i];
+        if (offset >= buffer.length) {
+          this.outputBuffers.push(this.write);
+          this.write = { buffer: new Float32Array(this.bufferLength), trackId };
+          buffer = this.write.buffer;
+          offset = 0;
+        }
+      }
+      
+      // If we have a partial buffer at the end, push it too
+      if (offset > 0) {
+        const finalBuffer = new Float32Array(this.bufferLength);
+        finalBuffer.set(buffer.subarray(0, offset));
+        this.outputBuffers.push({ buffer: finalBuffer, trackId });
+        this.write = { buffer: new Float32Array(this.bufferLength), trackId };
+        this.writeOffset = 0;
+      } else {
+        this.writeOffset = offset;
+      }
+      
+      this.noBufferCount = 0;
+      return true;
+    } catch (error) {
+      this.handleError(error);
+      return false;
+    }
+  }
+
+  process(inputs, outputs, parameters) {
+    try {
+      const output = outputs[0];
+      const outputChannelData = output[0];
+      const outputBuffers = this.outputBuffers;
+
+      if (this.hasInterrupted) {
+        this.port.postMessage({ event: 'stop' });
+        return false;
+      } else if (!this.hasStarted && outputBuffers.length < this.minBufferSize) {
+        // Wait for more buffers before starting
+        outputChannelData.fill(0);
+        return true;
+      } else if (outputBuffers.length > 0) {
+        this.hasStarted = true;
+        this.noBufferCount = 0;
+        this.lastUnderrunLog = 0;
+        
+        const { buffer, trackId } = outputBuffers.shift();
+        outputChannelData.set(buffer);
+        
+        if (trackId) {
+          this.trackSampleOffsets[trackId] =
+            this.trackSampleOffsets[trackId] || 0;
+          this.trackSampleOffsets[trackId] += buffer.length;
+          
+          // If this was the last buffer for this track, notify completion
+          if (outputBuffers.length === 0 || outputBuffers[0].trackId !== trackId) {
+            this.port.postMessage({ 
+              event: 'track_complete',
+              trackId,
+              finalOffset: this.trackSampleOffsets[trackId]
+            });
+          }
+        }
+        return true;
+      } else if (this.hasStarted) {
+        this.noBufferCount++;
+        
+        if (this.noBufferCount >= 5 && currentTime - this.lastUnderrunLog > 1) {
+          this.port.postMessage({ 
+            event: 'underrun', 
+            count: this.noBufferCount,
+            bufferSize: this.outputBuffers.length,
+            maxBuffers: this.maxNoBufferFrames
+          });
+          this.lastUnderrunLog = currentTime;
+        }
+        
+        if (this.noBufferCount >= this.maxNoBufferFrames) {
+          this.port.postMessage({ 
+            event: 'stop',
+            reason: 'max_underruns_reached',
+            finalCount: this.noBufferCount
+          });
+          return false;
+        }
+        outputChannelData.fill(0);
+      }
+      return true;
+    } catch (error) {
+      this.handleError(error);
+      return true;
+    }
+  }
+}
+
+registerProcessor('stream_processor', StreamProcessor);
+`;
+
+const script = new Blob([StreamProcessorWorklet], {
+  type: 'application/javascript',
+});
+const src = URL.createObjectURL(script);
+export const StreamProcessorSrc = src;

package/helper-apps/cortex-realtime-voice-server/client/src/chat/components/AudioVisualizer.tsx

@@ -0,0 +1,151 @@
+import { useEffect, useRef } from 'react';
+
+type AudioVisualizerProps = {
+  audioContext: AudioContext | null;
+  analyserNode: AnalyserNode | null;
+  width?: number;
+  height?: number;
+};
+
+export function AudioVisualizer({ 
+  audioContext, 
+  analyserNode,
+  width = 300,
+  height = 300 
+}: AudioVisualizerProps) {
+  const canvasRef = useRef<HTMLCanvasElement>(null);
+  const rotationRef = useRef(0);
+  const colorShiftRef = useRef(0);
+  const animationFrameRef = useRef<number>();
+
+  useEffect(() => {
+    if (!audioContext || !analyserNode || !canvasRef.current) return;
+
+    // Update canvas size when width/height props change
+    if (canvasRef.current) {
+      canvasRef.current.width = width;
+      canvasRef.current.height = height;
+    }
+
+    const draw = () => {
+      const canvas = canvasRef.current;
+      if (!canvas || !analyserNode) return;
+
+      const ctx = canvas.getContext('2d');
+      if (!ctx) return;
+
+      const bufferLength = analyserNode.frequencyBinCount;
+      const dataArray = new Uint8Array(bufferLength);
+      analyserNode.getByteFrequencyData(dataArray);
+
+      // Clear with fade effect
+      ctx.fillStyle = 'rgba(17, 24, 39, 0.3)';
+      ctx.fillRect(0, 0, canvas.width, canvas.height);
+
+      const centerX = canvas.width / 2;
+      const centerY = canvas.height / 2;
+      const maxRadius = Math.min(centerX, centerY) - 10;
+
+      // Draw outer circle
+      ctx.beginPath();
+      ctx.strokeStyle = `hsl(${210 + Math.sin(colorShiftRef.current) * 20}, 80%, 60%)`;
+      ctx.lineWidth = 2;
+      ctx.arc(centerX, centerY, maxRadius, 0, Math.PI * 2);
+      ctx.stroke();
+
+      // Define base colors with shifting hues
+      const baseHue = 210 + Math.sin(colorShiftRef.current) * 20;
+      const waveforms = [
+        { 
+          baseRadius: maxRadius * 0.4, 
+          color: `hsl(${baseHue}, 90%, 70%)`,
+          gradientColors: [`hsla(${baseHue}, 90%, 70%, 0.3)`, `hsla(${baseHue}, 90%, 50%, 0)`],
+          rotation: rotationRef.current 
+        },
+        { 
+          baseRadius: maxRadius * 0.6, 
+          color: `hsl(${baseHue + 10}, 85%, 60%)`,
+          gradientColors: [`hsla(${baseHue + 10}, 85%, 60%, 0.3)`, `hsla(${baseHue + 10}, 85%, 40%, 0)`],
+          rotation: rotationRef.current + (Math.PI * 2 / 3) 
+        },
+        { 
+          baseRadius: maxRadius * 0.8, 
+          color: `hsl(${baseHue + 20}, 80%, 50%)`,
+          gradientColors: [`hsla(${baseHue + 20}, 80%, 50%, 0.3)`, `hsla(${baseHue + 20}, 80%, 30%, 0)`],
+          rotation: rotationRef.current + (Math.PI * 4 / 3) 
+        }
+      ];
+
+      waveforms.forEach(({ baseRadius, color, gradientColors, rotation }) => {
+        const points: [number, number][] = [];
+        
+        for (let i = 0; i <= bufferLength; i++) {
+          const amplitude = dataArray[i % bufferLength] / 255.0;
+          const angle = (i * 2 * Math.PI) / bufferLength + rotation;
+          
+          const radius = baseRadius + (maxRadius * 0.4 * amplitude);
+          const x = centerX + Math.cos(angle) * radius;
+          const y = centerY + Math.sin(angle) * radius;
+          
+          points.push([x, y]);
+        }
+
+        // Create gradient for fill
+        const gradient = ctx.createRadialGradient(
+          centerX, centerY, baseRadius * 0.8,
+          centerX, centerY, baseRadius * 1.2
+        );
+        gradient.addColorStop(0, gradientColors[0]);
+        gradient.addColorStop(1, gradientColors[1]);
+
+        ctx.beginPath();
+        ctx.moveTo(centerX, centerY);
+        points.forEach(([x, y]) => {
+          ctx.lineTo(x, y);
+        });
+        ctx.closePath();
+        ctx.fillStyle = gradient;
+        ctx.fill();
+
+        ctx.beginPath();
+        points.forEach(([x, y], i) => {
+          if (i === 0) ctx.moveTo(x, y);
+          else ctx.lineTo(x, y);
+        });
+        ctx.strokeStyle = color;
+        ctx.lineWidth = 2;
+        ctx.closePath();
+        ctx.stroke();
+
+        ctx.shadowBlur = 15;
+        ctx.shadowColor = color;
+      });
+
+      rotationRef.current += 0.002;
+      colorShiftRef.current += 0.005;
+      
+      animationFrameRef.current = requestAnimationFrame(draw);
+    };
+
+    draw();
+
+    return () => {
+      if (animationFrameRef.current) {
+        cancelAnimationFrame(animationFrameRef.current);
+      }
+    };
+  }, [audioContext, analyserNode, width, height]);
+
+  return (
+    <div className="w-full h-full flex items-center justify-center pointer-events-none">
+      <div className="aspect-square w-full max-h-full">
+        <canvas 
+          ref={canvasRef} 
+          width={width}
+          height={height}
+          className="bg-gray-900 rounded-lg w-full h-full object-contain pointer-events-none"
+        />
+      </div>
+    </div>
+  );
+} 

package/helper-apps/cortex-realtime-voice-server/client/src/chat/components/CopyButton.tsx

@@ -0,0 +1,32 @@
+import React, { useState } from 'react';
+import ContentCopyIcon from '@mui/icons-material/ContentCopy';
+import CheckIcon from '@mui/icons-material/Check';
+
+interface CopyButtonProps {
+  text: string;
+  className?: string;
+}
+
+export const CopyButton: React.FC<CopyButtonProps> = ({ text, className = '' }) => {
+  const [copied, setCopied] = useState(false);
+
+  const handleCopy = async () => {
+    await navigator.clipboard.writeText(text);
+    setCopied(true);
+    setTimeout(() => setCopied(false), 2000);
+  };
+
+  return (
+    <button
+      onClick={handleCopy}
+      className={`p-1 text-gray-400 hover:text-cyan-400 transition-colors duration-200 ${className}`}
+      title="Copy to clipboard"
+    >
+      {copied ? (
+        <CheckIcon sx={{ fontSize: 16 }} />
+      ) : (
+        <ContentCopyIcon sx={{ fontSize: 16 }} />
+      )}
+    </button>
+  );
+};