@readymadeit/mcp 0.0.0-stage → 0.1.0

package/bundle/hkp-frontend/src/runtime/browser/services/AudioInput.ts

@@ -0,0 +1,298 @@
+import { AppInstance, ServiceClass } from "hkp-frontend/src/types";
+import { FloatRingBufferSymbol } from "hkp-frontend/src/runtime/rest/Data";
+import ServiceBase from "./ServiceBase";
+import AudioInputUI from "./AudioInputUI";
+
+const serviceId = "hookup.to/service/audio-input";
+const serviceName = "Audio Input";
+
+// "blob" emits compressed MediaRecorder blobs; "pcm" emits raw PCM samples
+// wrapped in a FloatRingBuffer, resampled to the configured pcmSampleRate.
+export type AudioInputFormat = "blob" | "pcm";
+
+type State = {
+  timeslice: number;
+  isRecording: boolean;
+  availableDevices: MediaDeviceInfo[];
+  format: AudioInputFormat;
+  // Rate pcm output is resampled to before emitting. The default suits
+  // Whisper-family speech models; board creators set it to match consumers.
+  pcmSampleRate: number;
+};
+
+type PcmCapture = {
+  context: AudioContext;
+  source: MediaStreamAudioSourceNode;
+  node: AudioWorkletNode;
+  chunks: Float32Array[];
+};
+
+// Captures channel 0 and posts each render quantum to the main thread. The
+// processor writes no output, so connecting it to the destination is silent.
+const captureWorkletSource = `
+class HkpCaptureProcessor extends AudioWorkletProcessor {
+  process(inputs) {
+    const channel = inputs[0] && inputs[0][0];
+    if (channel) {
+      this.port.postMessage(channel.slice(0));
+    }
+    return true;
+  }
+}
+registerProcessor("hkp-capture", HkpCaptureProcessor);
+`;
+
+function downsampleLinear(
+  input: Float32Array,
+  fromRate: number,
+  toRate: number,
+): Float32Array {
+  if (fromRate === toRate) {
+    return input;
+  }
+  const ratio = fromRate / toRate;
+  const outLength = Math.floor(input.length / ratio);
+  const out = new Float32Array(outLength);
+  for (let i = 0; i < outLength; i++) {
+    const pos = i * ratio;
+    const i0 = Math.floor(pos);
+    const i1 = Math.min(i0 + 1, input.length - 1);
+    const frac = pos - i0;
+    out[i] = input[i0] * (1 - frac) + input[i1] * frac;
+  }
+  return out;
+}
+
+class AudioInput extends ServiceBase<State> {
+  recorder: MediaRecorder | undefined;
+  _stream: MediaStream | undefined;
+  _pcm: PcmCapture | undefined;
+  _pcmBufferId: number = 0;
+
+  constructor(
+    app: AppInstance,
+    board: string,
+    descriptor: ServiceClass,
+    id: string,
+  ) {
+    super(app, board, descriptor, id, {
+      timeslice: 1000,
+      isRecording: false,
+      availableDevices: [],
+      format: "blob",
+      pcmSampleRate: 16000,
+    });
+    this.recorder = undefined;
+    this._stream = undefined;
+    // Boards may contain Audio Input even when microphone access is unavailable
+    // (permission denied, insecure context, or a headless browser). Construction
+    // must remain best-effort: a rejected startup request must not become an
+    // unhandled promise rejection that destabilizes the whole board.
+    void this.prepareStream().catch((err) => {
+      const detail =
+        typeof err === "object" &&
+        err !== null &&
+        "message" in err &&
+        typeof err.message === "string"
+          ? `: ${err.message}`
+          : "";
+      this.pushErrorNotification(`Could not access audio input${detail}`);
+    });
+  }
+
+  async configure(config: any) {
+    const { timeslice, device, format } = config;
+
+    if (timeslice !== undefined) {
+      this.state.timeslice = timeslice;
+      this.app.notify(this, { timeslice });
+      if (this.state.isRecording) {
+        this.startRecorder(timeslice);
+      }
+    }
+
+    if (format === "blob" || format === "pcm") {
+      this.state.format = format;
+      this.app.notify(this, { format });
+    }
+
+    if (typeof config.pcmSampleRate === "number" && config.pcmSampleRate > 0) {
+      this.state.pcmSampleRate = config.pcmSampleRate;
+      this.app.notify(this, { pcmSampleRate: this.state.pcmSampleRate });
+    }
+
+    if (device !== undefined) {
+      await this.prepareStream(device);
+    }
+
+    if (config.command) {
+      if (config.command.action === "start-recording") {
+        this.startRecorder(config.command.params?.timeslice || this.state.timeslice);
+      } else if (config.command.action === "stop-recording") {
+        this.stopRecorder();
+      }
+    }
+  }
+
+  destroy() {
+    this.stopRecorder();
+    if (this._stream) {
+      this._stream.getTracks().forEach((track) => track.stop());
+    }
+  }
+
+  async prepareStream(device?: MediaDeviceInfo) {
+    if (navigator.mediaDevices?.ondevicechange !== undefined) {
+      const updateDeviceList = async () => {
+        this.state.availableDevices = await navigator.mediaDevices.enumerateDevices();
+        this.app.notify(this, { availableDevices: this.state.availableDevices });
+      };
+      navigator.mediaDevices.ondevicechange = updateDeviceList;
+      await updateDeviceList();
+    }
+
+    if (navigator.mediaDevices) {
+      const constraints = device
+        ? { audio: { deviceId: { exact: device.deviceId } } }
+        : { audio: true };
+      this._stream = await navigator.mediaDevices.getUserMedia(constraints);
+      if (this._stream) {
+        this.app.notify(this, { stream: this._stream });
+      }
+    }
+  }
+
+  startRecorder(timeslice: number) {
+    if (this.state.isRecording) {
+      this.stopRecorder();
+    }
+
+    if (this.state.format === "pcm") {
+      void this.startPcmCapture();
+      return;
+    }
+
+    try {
+      if (!this.recorder) {
+        this.recorder = new MediaRecorder(this._stream!);
+        this.recorder.addEventListener("dataavailable", (e: BlobEvent) => {
+          if (e.data && e.data.size > 100) {
+            this.app.next(this, e.data);
+          }
+        });
+      }
+    } catch (err) {
+      console.error("Unexpected error in AudioInput service:", err);
+      return;
+    }
+    this.recorder.start(timeslice);
+    this.state.isRecording = true;
+    this.app.notify(this, { isRecording: true });
+  }
+
+  stopRecorder() {
+    if (this._pcm) {
+      void this.stopPcmCapture();
+      return;
+    }
+    if (this.recorder) {
+      this.recorder.stop();
+      this.recorder = undefined;
+      this.state.isRecording = false;
+      this.app.notify(this, { isRecording: false });
+    }
+  }
+
+  async startPcmCapture() {
+    if (!this._stream) {
+      await this.prepareStream();
+    }
+    if (!this._stream || this._pcm) {
+      return;
+    }
+
+    try {
+      const context = new AudioContext();
+      const moduleUrl = URL.createObjectURL(
+        new Blob([captureWorkletSource], { type: "application/javascript" }),
+      );
+      let node: AudioWorkletNode;
+      try {
+        await context.audioWorklet.addModule(moduleUrl);
+        node = new AudioWorkletNode(context, "hkp-capture");
+      } finally {
+        URL.revokeObjectURL(moduleUrl);
+      }
+
+      const chunks: Float32Array[] = [];
+      node.port.onmessage = (event: MessageEvent<Float32Array>) => {
+        chunks.push(event.data);
+      };
+
+      const source = context.createMediaStreamSource(this._stream);
+      source.connect(node);
+      node.connect(context.destination); // required to drive processing; output is silent
+
+      this._pcm = { context, source, node, chunks };
+      this.state.isRecording = true;
+      this.app.notify(this, { isRecording: true });
+    } catch (err) {
+      console.error("AudioInput: failed to start pcm capture:", err);
+      this.pushErrorNotification("Could not start PCM audio capture");
+    }
+  }
+
+  async stopPcmCapture() {
+    const pcm = this._pcm;
+    if (!pcm) {
+      return;
+    }
+    this._pcm = undefined;
+
+    pcm.node.port.onmessage = null;
+    pcm.source.disconnect();
+    pcm.node.disconnect();
+    const captureRate = pcm.context.sampleRate;
+    await pcm.context.close();
+
+    this.state.isRecording = false;
+    this.app.notify(this, { isRecording: false });
+
+    const totalLength = pcm.chunks.reduce((n, chunk) => n + chunk.length, 0);
+    if (totalLength === 0) {
+      return;
+    }
+    const samples = new Float32Array(totalLength);
+    let offset = 0;
+    for (const chunk of pcm.chunks) {
+      samples.set(chunk, offset);
+      offset += chunk.length;
+    }
+
+    const resampled = downsampleLinear(samples, captureRate, this.state.pcmSampleRate);
+    this.app.next(this, {
+      type: FloatRingBufferSymbol,
+      array: new Uint8Array(resampled.buffer),
+      id: this._pcmBufferId++,
+      ts: Date.now(),
+    });
+  }
+
+  async process(params: any) {
+    return params;
+  }
+}
+
+const descriptor = {
+  serviceName,
+  serviceId,
+  create: (
+    app: AppInstance,
+    board: string,
+    descriptor: ServiceClass,
+    id: string,
+  ) => new AudioInput(app, board, descriptor, id),
+  createUI: AudioInputUI,
+};
+
+export default descriptor;

package/bundle/hkp-frontend/src/runtime/browser/services/AudioInputUI.tsx

@@ -0,0 +1,257 @@
+import { Component } from "react";
+import { Mic, AudioLines } from "lucide-react";
+
+import { ServiceInstance, ServiceUIProps } from "hkp-frontend/src/types";
+
+import AudioDeviceSettings, { AudioInputDevice } from "./AudioDeviceSettings";
+import ServiceUI from "hkp-frontend/src/ui-components/service/ServiceUI";
+import MenuIcon from "hkp-frontend/src/ui-components/MenuIcon";
+import PillRadioGroup from "hkp-frontend/src/ui-components/PillRadioGroup";
+import OneOfVisible from "hkp-frontend/src/ui-components/OneOfVisible";
+import NumberInput from "hkp-frontend/src/ui-components/NumberInput";
+import CustomDialog from "hkp-frontend/src/ui-components/CustomDialog";
+import { Button } from "hkp-frontend/src/ui-components/primitives/button";
+import HoldableButton from "hkp-frontend/src/components/shared/HoldableButton";
+import WaveformDisplay from "./WaveformDisplay";
+
+type State = {
+  showAudioSettings: boolean;
+  recording: boolean;
+  visualize: boolean;
+  devices: Array<AudioInputDevice>;
+  timeslice: number;
+  pushRecording: boolean;
+  recordingMode: string;
+  stream: MediaStream | undefined;
+  format: string;
+  pcmSampleRate: number;
+};
+
+const recordingModeOptions = ["On Push", "Time Slices"];
+
+const formatOptions = ["Compressed", "PCM"];
+const formatValues = ["blob", "pcm"];
+
+export default class AudioInputUI extends Component<ServiceUIProps, State> {
+  state: State = {
+    showAudioSettings: false,
+    recording: false,
+    visualize: true,
+    devices: [],
+    timeslice: 1000,
+    pushRecording: false,
+    recordingMode: recordingModeOptions[0],
+    stream: undefined,
+    format: formatValues[0],
+    pcmSampleRate: 16000,
+  };
+
+  onInit = (initialState: any) => {
+    const { timeslice, availableDevices, format, pcmSampleRate } = initialState;
+    this.setState({
+      devices: availableDevices,
+      timeslice,
+      format: formatValues.includes(format) ? format : formatValues[0],
+      pcmSampleRate:
+        typeof pcmSampleRate === "number"
+          ? pcmSampleRate
+          : this.state.pcmSampleRate,
+    });
+  };
+
+  onNotification = (notification: any) => {
+    const {
+      isRecording,
+      timeslice,
+      availableDevices,
+      stream,
+      format,
+      pcmSampleRate,
+    } = notification;
+
+    if (format !== undefined && formatValues.includes(format)) {
+      this.setState({ format });
+    }
+
+    if (pcmSampleRate !== undefined) {
+      this.setState({ pcmSampleRate });
+    }
+
+    if (isRecording !== undefined) {
+      this.setState({ recording: isRecording });
+    }
+
+    if (timeslice !== undefined) {
+      this.setState({ timeslice });
+    }
+
+    if (availableDevices !== undefined) {
+      this.setState({ devices: availableDevices });
+    }
+
+    if (stream !== undefined) {
+      this.setState({ stream });
+    }
+  };
+
+  onRecord = () => {
+    const { service } = this.props;
+    const { recording } = this.state;
+    if (recording) {
+      service.configure({ command: { action: "stop-recording" } });
+    } else {
+      service.configure({
+        command: {
+          action: "start-recording",
+          params: { timeslice: service.timeslice },
+        },
+      });
+    }
+  };
+
+  onStartPushRecording = () => {
+    this.props.service.configure({
+      command: { action: "start-recording" },
+    });
+    this.setState({ pushRecording: true });
+  };
+
+  onStopPushRecording = () => {
+    this.props.service.configure({ command: { action: "stop-recording" } });
+    this.setState({ pushRecording: false });
+  };
+
+  onChangeFormat = (newFormat: string) => {
+    const index = formatOptions.indexOf(newFormat);
+    if (index >= 0) {
+      this.props.service.configure({ format: formatValues[index] });
+    }
+  };
+
+  renderMain = (service: ServiceInstance) => {
+    const { recording, timeslice, recordingMode, format, pcmSampleRate } =
+      this.state;
+    return (
+      <div className="flex flex-col text-left mx-5 h-full">
+        <WaveformDisplay
+          active={this.state.visualize}
+          stream={this.state.stream}
+        />
+
+        <PillRadioGroup
+          title="Recording Mode"
+          options={recordingModeOptions}
+          value={recordingMode}
+          onChange={(newMode) => this.setState({ recordingMode: newMode })}
+        />
+
+        <PillRadioGroup
+          title="Output Format"
+          options={formatOptions}
+          value={formatOptions[formatValues.indexOf(format)]}
+          onChange={this.onChangeFormat}
+        />
+
+        {format === "pcm" && (
+          <div className="flex">
+            <NumberInput
+              title="Sample Rate"
+              value={pcmSampleRate}
+              onChange={(newRate) =>
+                service.configure({ pcmSampleRate: Number(newRate) })
+              }
+            >
+              Hz
+            </NumberInput>
+          </div>
+        )}
+
+        <OneOfVisible current={recordingModeOptions.indexOf(recordingMode)}>
+          <div className="py-4 h-full">
+            <HoldableButton
+              style={{ width: "100%", height: "100%" }}
+              onDown={this.onStartPushRecording}
+              onUp={this.onStopPushRecording}
+            >
+              {!this.state.pushRecording ? "Push to Record" : "Recording ..."}
+            </HoldableButton>
+          </div>
+
+          <div className="mt-1 flex flex-col gap-2">
+            <div className="flex">
+              <NumberInput
+                title="Timeslice"
+                value={timeslice}
+                onChange={(newSlice) =>
+                  service.configure({ timeslice: Number(newSlice) })
+                }
+              >
+                milliseconds
+              </NumberInput>
+            </div>
+            <Button
+              className="hkp-svc-btn"
+              variant="outline"
+              onClick={this.onRecord}
+            >
+              {recording ? "Stop Recording" : "Start Sliced Recording"}
+            </Button>
+          </div>
+        </OneOfVisible>
+      </div>
+    );
+  };
+
+  onAudioSettingsDialog = (isOpen: boolean) => {
+    this.setState({ showAudioSettings: isOpen });
+  };
+
+  onToggleWaveform = () => this.setState({ visualize: !this.state.visualize });
+
+  onActivateDevice = async (device: AudioInputDevice) => {
+    const { service } = this.props;
+    await service.configure({ device });
+    this.setState({ stream: service._stream, showAudioSettings: false });
+  };
+
+  render() {
+    const { service } = this.props;
+    const { showAudioSettings, visualize, devices } = this.state;
+    const customMenuEntries = [
+      {
+        name: `${showAudioSettings ? "Hide" : "Show"} Audio Settings`,
+        icon: <MenuIcon icon={Mic} />,
+        onClick: () => this.setState({ showAudioSettings: !showAudioSettings }),
+      },
+      {
+        name: `${visualize ? "Disable" : "Enable"} Waveform`,
+        icon: <MenuIcon icon={AudioLines} />,
+        onClick: this.onToggleWaveform,
+      },
+    ];
+
+    return (
+      <ServiceUI
+        {...this.props}
+        customMenuEntries={customMenuEntries}
+        onInit={this.onInit}
+        onNotification={this.onNotification}
+        initialSize={{ width: 440, height: 270 }}
+      >
+        <>
+          {this.renderMain(service)}
+          <CustomDialog
+            title="Audio Input Settings"
+            isOpen={showAudioSettings}
+            onOpenChange={this.onAudioSettingsDialog}
+          >
+            <AudioDeviceSettings
+              devices={devices}
+              onActivate={this.onActivateDevice}
+            />
+          </CustomDialog>
+        </>
+      </ServiceUI>
+    );
+  }
+}