@dxos/plugin-transcription 0.7.5-labs.a8b535d

package/dist/lib/browser/index.mjs

@@ -0,0 +1,652 @@
+import {
+  mergeFloat64Arrays,
+  monitorAudioLevel
+} from "./chunk-ZMES3U2D.mjs";
+import {
+  TranscriptContainer
+} from "./chunk-AS364WZZ.mjs";
+import {
+  Transcript,
+  TranscriptionContainer
+} from "./chunk-R4HHS6FA.mjs";
+import {
+  TRANSCRIPTION_PLUGIN,
+  TRANSCRIPTION_URL,
+  TranscriptType,
+  meta
+} from "./chunk-QIHUPVQD.mjs";
+
+// packages/plugins/experimental/plugin-transcription/src/capabilities/index.ts
+import { lazy } from "@dxos/app-framework";
+
+// packages/plugins/experimental/plugin-transcription/src/capabilities/capabilities.ts
+import { defineCapability } from "@dxos/app-framework";
+var TranscriptionCapabilities;
+(function(TranscriptionCapabilities2) {
+  TranscriptionCapabilities2.Transcription = defineCapability(`${TRANSCRIPTION_PLUGIN}/capability/transcription`);
+})(TranscriptionCapabilities || (TranscriptionCapabilities = {}));
+
+// packages/plugins/experimental/plugin-transcription/src/capabilities/index.ts
+var AppGraphBuilder = lazy(() => import("./app-graph-builder-ZSSJIFY7.mjs"));
+var IntentResolver = lazy(() => import("./intent-resolver-EAR2TFK2.mjs"));
+var ReactSurface = lazy(() => import("./react-surface-FOEWLLTZ.mjs"));
+
+// packages/plugins/experimental/plugin-transcription/src/hooks/useAudioFile.ts
+import { useEffect, useState } from "react";
+import { log } from "@dxos/log";
+var __dxlog_file = "/home/runner/work/dxos/dxos/packages/plugins/experimental/plugin-transcription/src/hooks/useAudioFile.ts";
+var useAudioFile = (audioUrl) => {
+  const [{ audio, stream, track }, setStream] = useState({});
+  useEffect(() => {
+    const t = setTimeout(async () => {
+      const response = await fetch(audioUrl);
+      const blob = await response.blob();
+      const audio2 = new Audio();
+      audio2.src = URL.createObjectURL(blob);
+      await new Promise((resolve, reject) => {
+        audio2.addEventListener("error", (err) => {
+          log.error("error", {
+            err
+          }, {
+            F: __dxlog_file,
+            L: 27,
+            S: void 0,
+            C: (f, a) => f(...a)
+          });
+          reject(err);
+        }, {
+          once: true
+        });
+        audio2.addEventListener("canplay", () => {
+          log.info("starting...", void 0, {
+            F: __dxlog_file,
+            L: 35,
+            S: void 0,
+            C: (f, a) => f(...a)
+          });
+          resolve();
+        }, {
+          once: true
+        });
+        audio2.load();
+      });
+      const ctx = new AudioContext();
+      const destination = ctx.createMediaStreamDestination();
+      destination.channelCount = 1;
+      const source = ctx.createMediaElementSource(audio2);
+      source.connect(destination);
+      setStream({
+        audio: audio2,
+        stream: destination.stream,
+        track: destination.stream.getAudioTracks()[0]
+      });
+    });
+    return () => clearTimeout(t);
+  }, [
+    audioUrl
+  ]);
+  return {
+    audio,
+    stream,
+    track
+  };
+};
+
+// packages/plugins/experimental/plugin-transcription/src/hooks/useAudioTrack.ts
+import { useEffect as useEffect2, useState as useState2 } from "react";
+var useAudioTrack = (active) => {
+  const [track, setTrack] = useState2();
+  useEffect2(() => {
+    if (!active) {
+      track?.stop();
+      setTrack(void 0);
+      return;
+    }
+    const initAudio = async () => {
+      const audio = new Audio();
+      audio.srcObject = await navigator.mediaDevices.getUserMedia({
+        audio: true
+      });
+      const [track2] = audio.srcObject.getAudioTracks();
+      if (track2) {
+        setTrack(track2);
+      }
+    };
+    if (active) {
+      void initAudio();
+    }
+  }, [
+    active
+  ]);
+  return track;
+};
+
+// packages/plugins/experimental/plugin-transcription/src/hooks/useIsSpeaking.ts
+import { useEffect as useEffect3, useRef, useState as useState3 } from "react";
+var useIsSpeaking = (mediaStreamTrack) => {
+  const [isSpeaking, setIsSpeaking] = useState3(false);
+  const isSpeakingRef = useRef(isSpeaking);
+  useEffect3(() => {
+    if (!mediaStreamTrack) {
+      setIsSpeaking(false);
+      return;
+    }
+    isSpeakingRef.current = isSpeaking;
+    const interval = window.setInterval(() => {
+      if (isSpeaking === isSpeakingRef.current) {
+        return;
+      }
+      setIsSpeaking(isSpeakingRef.current);
+    }, 50);
+    return () => {
+      clearInterval(interval);
+    };
+  }, [
+    isSpeaking,
+    isSpeakingRef,
+    mediaStreamTrack
+  ]);
+  useEffect3(() => {
+    if (!mediaStreamTrack) {
+      return;
+    }
+    let timeout = -1;
+    const cleanup = monitorAudioLevel({
+      mediaStreamTrack,
+      onMeasure: (vol) => {
+        const audioLevelAboveThreshold = vol > (isSpeakingRef.current ? 0.02 : 0.05);
+        if (audioLevelAboveThreshold) {
+          clearTimeout(timeout);
+          timeout = -1;
+          isSpeakingRef.current = true;
+        } else if (timeout === -1) {
+          timeout = window.setTimeout(() => {
+            isSpeakingRef.current = false;
+            timeout = -1;
+          }, 1e3);
+        }
+      }
+    });
+    return () => {
+      cleanup();
+    };
+  }, [
+    isSpeaking,
+    mediaStreamTrack
+  ]);
+  return isSpeaking;
+};
+
+// packages/plugins/experimental/plugin-transcription/src/hooks/useTranscriber.ts
+import { useEffect as useEffect4, useMemo } from "react";
+
+// packages/plugins/experimental/plugin-transcription/src/transcriber/media-stream-recorder.ts
+import { MediaRecorder, register } from "extendable-media-recorder";
+import { WaveFile } from "wavefile";
+import { synchronized } from "@dxos/async";
+import { invariant } from "@dxos/invariant";
+import { log as log2 } from "@dxos/log";
+import { trace } from "@dxos/tracing";
+function _ts_decorate(decorators, target, key, desc) {
+  var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+  if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+  else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+  return c > 3 && r && Object.defineProperty(target, key, r), r;
+}
+var __dxlog_file2 = "/home/runner/work/dxos/dxos/packages/plugins/experimental/plugin-transcription/src/transcriber/media-stream-recorder.ts";
+var initializingPromise;
+var initializeExtendableMediaRecorder = async () => {
+  const { connect } = await import("extendable-media-recorder-wav-encoder");
+  await register(await connect());
+};
+var MediaStreamRecorder = class {
+  constructor({ mediaStreamTrack, interval }) {
+    this._mediaRecorder = void 0;
+    this._header = void 0;
+    this._mediaStreamTrack = mediaStreamTrack;
+    this._interval = interval;
+  }
+  get wavConfig() {
+    const settings = this._mediaStreamTrack.getSettings();
+    return {
+      channels: settings.channelCount ?? 1,
+      sampleRate: settings.sampleRate ?? 16e3,
+      bitDepthCode: settings.sampleSize ? String(settings.sampleSize) : "16"
+    };
+  }
+  setOnChunk(onChunk) {
+    this._onChunk = onChunk;
+  }
+  async start() {
+    await (initializingPromise ??= initializeExtendableMediaRecorder()).catch((err) => log2.info("initializeExtendableMediaRecorder", {
+      err
+    }, {
+      F: __dxlog_file2,
+      L: 58,
+      S: this,
+      C: (f, a) => f(...a)
+    }));
+    if (!this._mediaRecorder) {
+      const stream = new MediaStream([
+        this._mediaStreamTrack
+      ]);
+      this._mediaRecorder = new MediaRecorder(stream, {
+        mimeType: "audio/wav"
+      });
+    }
+    if (this._mediaRecorder.state === "recording") {
+      return;
+    }
+    invariant(this._onChunk, "MediaStreamRecorder: onChunk is not set", {
+      F: __dxlog_file2,
+      L: 69,
+      S: this,
+      A: [
+        "this._onChunk",
+        "'MediaStreamRecorder: onChunk is not set'"
+      ]
+    });
+    this._mediaRecorder.ondataavailable = (event) => this._ondataavailable(event);
+    this._mediaRecorder.start(this._interval);
+  }
+  async stop() {
+    if (this._mediaRecorder?.state !== "recording") {
+      return;
+    }
+    this._mediaRecorder.stop();
+  }
+  async _ondataavailable(event) {
+    const blob = event.data;
+    const uint8Array = new Uint8Array(await blob.arrayBuffer());
+    const isHeader = uint8Array[0] === 82 && // R
+    uint8Array[1] === 73 && // I
+    uint8Array[2] === 70 && // F
+    uint8Array[3] === 70;
+    let wav;
+    if (isHeader) {
+      wav = new WaveFile(uint8Array);
+      this._header = uint8Array.slice(0, 44);
+    } else if (this._header) {
+      wav = new WaveFile(new Uint8Array([
+        ...this._header,
+        ...uint8Array
+      ]));
+    } else {
+      log2.warn("MediaStreamRecorder: no header", void 0, {
+        F: __dxlog_file2,
+        L: 100,
+        S: this,
+        C: (f, a) => f(...a)
+      });
+      return;
+    }
+    this._onChunk({
+      timestamp: Date.now(),
+      data: wav.getSamples()
+    });
+  }
+};
+_ts_decorate([
+  synchronized
+], MediaStreamRecorder.prototype, "_ondataavailable", null);
+MediaStreamRecorder = _ts_decorate([
+  trace.resource()
+], MediaStreamRecorder);
+
+// packages/plugins/experimental/plugin-transcription/src/transcriber/transcriber.ts
+import { WaveFile as WaveFile2 } from "wavefile";
+import { DeferredTask, Trigger } from "@dxos/async";
+import { LifecycleState, Resource } from "@dxos/context";
+import { log as log3 } from "@dxos/log";
+import { trace as trace2 } from "@dxos/tracing";
+function _ts_decorate2(decorators, target, key, desc) {
+  var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+  if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+  else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+  return c > 3 && r && Object.defineProperty(target, key, r), r;
+}
+var __dxlog_file3 = "/home/runner/work/dxos/dxos/packages/plugins/experimental/plugin-transcription/src/transcriber/transcriber.ts";
+var Transcriber = class extends Resource {
+  constructor({ config, recorder, onSegments }) {
+    super();
+    this._audioChunks = [];
+    this._lastUsedTimestamp = 0;
+    this._openTrigger = new Trigger({
+      autoReset: true
+    });
+    this._recording = false;
+    this._transcribeTask = void 0;
+    this._config = config;
+    this._recorder = recorder;
+    this._onSegments = onSegments;
+  }
+  async _open(ctx) {
+    this._recorder.setOnChunk((chunk) => this._saveAudioChunk(chunk));
+    await this._recorder.start();
+    this._transcribeTask = new DeferredTask(ctx, async () => this._transcribe());
+    this._openTrigger.wake();
+  }
+  async _close() {
+    this._recording = false;
+    this._transcribeTask = void 0;
+    await this._recorder.stop();
+  }
+  startChunksRecording() {
+    if (this._lifecycleState !== LifecycleState.OPEN) {
+      return;
+    }
+    this._recording = true;
+  }
+  stopChunksRecording() {
+    if (this._lifecycleState !== LifecycleState.OPEN || !this._recording) {
+      return;
+    }
+    this._recording = false;
+    this._transcribeTask?.schedule();
+  }
+  _saveAudioChunk(chunk) {
+    log3("saving audio chunk", {
+      chunk
+    }, {
+      F: __dxlog_file3,
+      L: 124,
+      S: this,
+      C: (f, a) => f(...a)
+    });
+    this._audioChunks.push(chunk);
+    if (!this._recording && (!this._transcribeTask || !this._transcribeTask.scheduled) && this._audioChunks.length >= this._config.prefixBufferChunksAmount) {
+      this._audioChunks = this._dropOldChunks();
+    } else if (this._audioChunks.length >= this._config.transcribeAfterChunksAmount && this._recording) {
+      this._transcribeTask?.schedule();
+    }
+  }
+  async _transcribe() {
+    log3("transcribing", {
+      chunks: this._audioChunks.length
+    }, {
+      F: __dxlog_file3,
+      L: 140,
+      S: this,
+      C: (f, a) => f(...a)
+    });
+    const chunks = this._audioChunks;
+    this._audioChunks = this._dropOldChunks();
+    if (chunks.length === 0) {
+      return;
+    }
+    const audio = await this._mergeAudioChunks(chunks);
+    const segments = await this._fetchTranscription(audio);
+    if (!Array.isArray(segments) || segments.length === 0) {
+      return;
+    }
+    const alignedSegments = this._alignSegments(segments, chunks);
+    if (alignedSegments.length > 0) {
+      await this._onSegments(alignedSegments);
+    }
+  }
+  _dropOldChunks() {
+    return this._config.prefixBufferChunksAmount > 0 ? this._audioChunks.slice(-this._config.prefixBufferChunksAmount) : [];
+  }
+  async _mergeAudioChunks(chunks) {
+    const file = new WaveFile2();
+    const wavConfig = this._recorder.wavConfig;
+    file.fromScratch(wavConfig.channels, wavConfig.sampleRate, wavConfig.bitDepthCode, mergeFloat64Arrays(chunks.map(({ data }) => data)));
+    return file.toBase64();
+  }
+  async _fetchTranscription(audio) {
+    if (audio.length === 0) {
+      this._audioChunks = [];
+      throw new Error("No audio to send for transcribing");
+    }
+    const response = await fetch(`${TRANSCRIPTION_URL}/transcribe`, {
+      method: "POST",
+      body: JSON.stringify({
+        audio
+      }),
+      headers: {
+        "Content-Type": "application/json"
+      }
+    });
+    if (!response.ok) {
+      this._audioChunks = [];
+      throw new Error("Failed to transcribe");
+    }
+    const { segments } = await response.json();
+    log3.info("transcription response", {
+      segments,
+      string: segments.map((segments2) => segments2.text).join(" ")
+    }, {
+      F: __dxlog_file3,
+      L: 206,
+      S: this,
+      C: (f, a) => f(...a)
+    });
+    return segments;
+  }
+  _alignSegments(segments, originalChunks) {
+    const zeroTimestamp = originalChunks.at(0).timestamp;
+    const filteredSegments = segments.filter((segment) => zeroTimestamp + segment.end * 1e3 >= this._lastUsedTimestamp).map((segment) => {
+      const words = segment.words.filter((word) => zeroTimestamp + word.start * 1e3 >= this._lastUsedTimestamp);
+      return {
+        ...segment,
+        words,
+        text: words.map((word) => word.word).join(""),
+        start: words.at(0)?.start ?? segment.end
+      };
+    }).filter((segment) => segment.words.length > 0);
+    if (filteredSegments.length === 0) {
+      return [];
+    }
+    this._lastUsedTimestamp = zeroTimestamp + filteredSegments.at(-1).end * 1e3;
+    return filteredSegments.map((segment) => ({
+      started: new Date(zeroTimestamp + segment.start * 1e3),
+      text: segment.text
+    }));
+  }
+};
+_ts_decorate2([
+  trace2.span({
+    showInBrowserTimeline: true
+  })
+], Transcriber.prototype, "_mergeAudioChunks", null);
+_ts_decorate2([
+  trace2.span({
+    showInBrowserTimeline: true
+  })
+], Transcriber.prototype, "_fetchTranscription", null);
+
+// packages/plugins/experimental/plugin-transcription/src/hooks/useTranscriber.ts
+var RECORD_INTERVAL = 200;
+var PREFIXED_CHUNKS_AMOUNT = 10;
+var TRANSCRIBE_AFTER_CHUNKS_AMOUNT = 50;
+var useTranscriber = ({ audioStreamTrack, onSegments }) => {
+  const transcriber = useMemo(() => {
+    if (!onSegments || !audioStreamTrack) {
+      return void 0;
+    }
+    return new Transcriber({
+      config: {
+        transcribeAfterChunksAmount: TRANSCRIBE_AFTER_CHUNKS_AMOUNT,
+        prefixBufferChunksAmount: PREFIXED_CHUNKS_AMOUNT
+      },
+      recorder: new MediaStreamRecorder({
+        mediaStreamTrack: audioStreamTrack,
+        interval: RECORD_INTERVAL
+      }),
+      onSegments
+    });
+  }, [
+    audioStreamTrack,
+    onSegments
+  ]);
+  useEffect4(() => {
+    return () => {
+      void transcriber?.close();
+    };
+  }, [
+    transcriber
+  ]);
+  return transcriber;
+};
+
+// packages/plugins/experimental/plugin-transcription/src/hooks/useVoiceInput.ts
+import { useState as useState4, useEffect as useEffect5, useCallback } from "react";
+import { scheduleMicroTask } from "@dxos/async";
+import { Context } from "@dxos/context";
+import { log as log4 } from "@dxos/log";
+import { useSoundEffect } from "@dxos/react-ui-sfx";
+var __dxlog_file4 = "/home/runner/work/dxos/dxos/packages/plugins/experimental/plugin-transcription/src/hooks/useVoiceInput.ts";
+var useVoiceInput = ({ active, onUpdate }) => {
+  const soundStart = useSoundEffect("StartRecording");
+  const soundStop = useSoundEffect("StopRecording");
+  const handleSegments = useCallback(async (segments) => {
+    const text = segments.map((str) => str.text.trim().replace(/[^\w\s]/g, "")).join(" ");
+    onUpdate(text);
+  }, []);
+  const [recording, setRecording] = useState4(false);
+  const track = useAudioTrack(active);
+  const transcriber = useTranscriber({
+    audioStreamTrack: track,
+    onSegments: handleSegments
+  });
+  useEffect5(() => {
+    const ctx = new Context(void 0, {
+      F: __dxlog_file4,
+      L: 43
+    });
+    scheduleMicroTask(ctx, async () => {
+      if (active && transcriber) {
+        await transcriber.open();
+        log4.info("starting...", void 0, {
+          F: __dxlog_file4,
+          L: 47,
+          S: void 0,
+          C: (f, a) => f(...a)
+        });
+        setRecording(true);
+        void soundStart.play();
+        transcriber.startChunksRecording();
+      } else if (!active && transcriber?.isOpen) {
+        transcriber?.stopChunksRecording();
+        await transcriber?.close();
+        log4.info("stopped", void 0, {
+          F: __dxlog_file4,
+          L: 54,
+          S: void 0,
+          C: (f, a) => f(...a)
+        });
+        setRecording(false);
+        void soundStop.play();
+      }
+    });
+    return () => {
+      void ctx.dispose();
+    };
+  }, [
+    active,
+    transcriber
+  ]);
+  return {
+    recording
+  };
+};
+
+// packages/plugins/experimental/plugin-transcription/src/TranscriptionPlugin.tsx
+import { Capabilities, Events, contributes, defineModule, definePlugin } from "@dxos/app-framework";
+import { ClientCapabilities, ClientEvents } from "@dxos/plugin-client";
+
+// packages/plugins/experimental/plugin-transcription/src/translations.ts
+var translations_default = [
+  {
+    "en-US": {
+      [TranscriptType.typename]: {
+        "typename label": "Transcript"
+      },
+      [TRANSCRIPTION_PLUGIN]: {
+        "plugin name": "Transcription",
+        "calls label": "Transcript",
+        "calls panel label": "Transcription",
+        "create calls label": "Create transcription",
+        "delete calls label": "Delete",
+        "delete button": "Delete",
+        "bookmark button": "Bookmark",
+        "summary button": "Create summary",
+        "summarizing label": "Summarizing..."
+      }
+    }
+  }
+];
+
+// packages/plugins/experimental/plugin-transcription/src/TranscriptionPlugin.tsx
+var TranscriptionPlugin = () => definePlugin(meta, [
+  defineModule({
+    id: `${meta.id}/module/translations`,
+    activatesOn: Events.SetupTranslations,
+    activate: () => contributes(Capabilities.Translations, translations_default)
+  }),
+  defineModule({
+    id: `${meta.id}/module/transcription`,
+    activatesOn: Events.Startup,
+    activate: () => contributes(TranscriptionCapabilities.Transcription, null)
+  }),
+  defineModule({
+    id: `${meta.id}/module/metadata`,
+    activatesOn: Events.SetupMetadata,
+    activate: () => contributes(Capabilities.Metadata, {
+      id: TranscriptType.typename,
+      metadata: {
+        placeholder: [
+          "transcript title placeholder",
+          {
+            ns: TRANSCRIPTION_PLUGIN
+          }
+        ],
+        icon: "ph--subtitles--regular"
+      }
+    })
+  }),
+  defineModule({
+    id: `${meta.id}/module/schema`,
+    activatesOn: ClientEvents.SetupSchema,
+    activate: () => [
+      contributes(ClientCapabilities.Schema, [
+        TranscriptType
+      ])
+    ]
+  }),
+  defineModule({
+    id: `${meta.id}/module/react-surface`,
+    activatesOn: Events.SetupReactSurface,
+    activate: ReactSurface
+  }),
+  defineModule({
+    id: `${meta.id}/module/intent-resolver`,
+    activatesOn: Events.SetupIntentResolver,
+    activate: IntentResolver
+  }),
+  defineModule({
+    id: `${meta.id}/module/app-graph-builder`,
+    activatesOn: Events.SetupAppGraph,
+    activate: AppGraphBuilder
+  })
+]);
+export {
+  AppGraphBuilder,
+  IntentResolver,
+  MediaStreamRecorder,
+  ReactSurface,
+  TRANSCRIPTION_PLUGIN,
+  Transcriber,
+  Transcript,
+  TranscriptContainer,
+  TranscriptionCapabilities,
+  TranscriptionContainer,
+  TranscriptionPlugin,
+  meta,
+  useAudioFile,
+  useAudioTrack,
+  useIsSpeaking,
+  useTranscriber,
+  useVoiceInput
+};
+//# sourceMappingURL=index.mjs.map