@dxos/plugin-transcription 0.8.1 → 0.8.2-main.12df754

package/dist/lib/browser/chunk-RHHJZQPM.mjs

@@ -0,0 +1,702 @@
+import {
+  TranscriptionCapabilities
+} from "./chunk-5BE6RTNC.mjs";
+
+// packages/plugins/plugin-transcription/src/components/TranscriptContainer.tsx
+import React2 from "react";
+import { fullyQualifiedId, getSpace } from "@dxos/client/echo";
+import { useQueue } from "@dxos/react-client/echo";
+import { StackItem } from "@dxos/react-ui-stack";
+
+// packages/plugins/plugin-transcription/src/components/Transcript/Transcript.tsx
+import React from "react";
+import { DXN } from "@dxos/keys";
+import { log as log2 } from "@dxos/log";
+import { resolveRef, useClient } from "@dxos/react-client";
+import { IconButton, useThemeContext } from "@dxos/react-ui";
+import { createBasicExtensions, createMarkdownExtensions, createRenderer, createThemeExtensions, decorateMarkdown, editorWidth, preview, useTextEditor } from "@dxos/react-ui-editor";
+import { mx } from "@dxos/react-ui-theme";
+import { isNotFalsy } from "@dxos/util";
+
+// packages/plugins/plugin-transcription/src/components/Transcript/transcript-extension.ts
+import { RangeSetBuilder } from "@codemirror/state";
+import { EditorView, GutterMarker, ViewPlugin, gutter } from "@codemirror/view";
+import { format } from "date-fns/format";
+import { intervalToDuration } from "date-fns/intervalToDuration";
+import { addEventListener, combine } from "@dxos/async";
+
+// packages/plugins/plugin-transcription/src/model/model.ts
+import { Text } from "@codemirror/state";
+import { Event } from "@dxos/async";
+import { invariant } from "@dxos/invariant";
+import { log } from "@dxos/log";
+var __dxlog_file = "/home/runner/work/dxos/dxos/packages/plugins/plugin-transcription/src/model/model.ts";
+var DocumentAdapter = class {
+  constructor(_view) {
+    this._view = _view;
+  }
+  /** The document must always have at least one line. */
+  lineCount() {
+    return this._view.state.doc.lines - 1;
+  }
+  replaceLines(from, remove, insert) {
+    log("replaceLines", {
+      count: this._view.state.doc.lines,
+      from,
+      remove,
+      insert
+    }, {
+      F: __dxlog_file,
+      L: 39,
+      S: this,
+      C: (f, a) => f(...a)
+    });
+    const numLines = this._view.state.doc.lines;
+    const posFrom = this._view.state.doc.line(from).from;
+    const posTo = remove === 0 ? posFrom : this._view.state.doc.line(Math.min(numLines, from + remove)).from;
+    let text = insert.join("\n");
+    if (posTo === this._view.state.doc.length) {
+      text += "\n";
+    }
+    this._view.dispatch({
+      changes: {
+        from: posFrom,
+        to: posTo,
+        insert: text
+      }
+    });
+  }
+};
+var BlockModel = class {
+  constructor(_renderer, initialBlocks = []) {
+    this._renderer = _renderer;
+    this._blocks = [];
+    this._blockLineCounts = /* @__PURE__ */ new Map();
+    this._lineToBlock = /* @__PURE__ */ new Map();
+    this._changes = [];
+    this.update = new Event();
+    initialBlocks.forEach((block) => {
+      this.appendBlock(block);
+    });
+  }
+  toJSON() {
+    return {
+      blocks: this._blocks.length,
+      changes: this._changes.length
+    };
+  }
+  get doc() {
+    return Text.of([
+      ...this._blocks.flatMap((block) => this._renderer(block)),
+      ""
+    ]);
+  }
+  get blocks() {
+    return this._blocks;
+  }
+  /**
+  * Get the block ID at the given line number.
+  */
+  getBlockAtLine(line) {
+    return this._lineToBlock.get(line);
+  }
+  reset() {
+    this._blocks.length = 0;
+    this._blockLineCounts.clear();
+    this._lineToBlock.clear();
+    this.update.emit();
+    return this;
+  }
+  appendBlock(block) {
+    const index = this._blocks.length;
+    this._blocks.push(block);
+    this._changes.push({
+      type: "append",
+      index,
+      block
+    });
+    this.update.emit();
+    return this;
+  }
+  updateBlock(block) {
+    const index = this._blocks.findIndex((b) => b.id === block.id);
+    invariant(index !== -1, void 0, {
+      F: __dxlog_file,
+      L: 147,
+      S: this,
+      A: [
+        "index !== -1",
+        ""
+      ]
+    });
+    this._blocks[index] = block;
+    this._changes.push({
+      type: "update",
+      index,
+      block
+    });
+    this.update.emit();
+    return this;
+  }
+  deleteBlock(id) {
+    const index = this._blocks.findIndex((b) => b.id === id);
+    invariant(index !== -1, void 0, {
+      F: __dxlog_file,
+      L: 156,
+      S: this,
+      A: [
+        "index !== -1",
+        ""
+      ]
+    });
+    this._blocks.splice(index, 1);
+    this._changes.push({
+      type: "delete",
+      index,
+      id
+    });
+    this.update.emit();
+    return this;
+  }
+  /**
+  * Syncs the tracked changes with the BlockDocument.
+  */
+  sync(document2) {
+    log("sync", {
+      changes: this._changes
+    }, {
+      F: __dxlog_file,
+      L: 167,
+      S: this,
+      C: (f, a) => f(...a)
+    });
+    for (const change of this._changes) {
+      switch (change.type) {
+        case "append": {
+          const lines = this._renderer(change.block);
+          const lastLine = document2.lineCount();
+          document2.replaceLines(lastLine + 1, 0, lines);
+          break;
+        }
+        case "update": {
+          let lineStart = 1;
+          for (let i = 0; i < change.index; i++) {
+            lineStart += this._blockLineCounts.get(this._blocks[i].id);
+          }
+          const previousLineCount = this._blockLineCounts.get(change.block.id);
+          const newLines = this._renderer(change.block);
+          document2.replaceLines(lineStart, previousLineCount, newLines);
+          break;
+        }
+        case "delete": {
+          let lineStart = 1;
+          for (let i = 0; i < change.index; i++) {
+            lineStart += this._blockLineCounts.get(this._blocks[i].id);
+          }
+          const previousLineCount = this._blockLineCounts.get(change.id);
+          document2.replaceLines(lineStart, previousLineCount, []);
+          break;
+        }
+      }
+    }
+    this._lineToBlock.clear();
+    let currentLine = 1;
+    for (const block of this._blocks) {
+      const lineCount = this._renderer(block).length;
+      this._blockLineCounts.set(block.id, lineCount);
+      this._lineToBlock.set(currentLine, block);
+      currentLine += lineCount;
+    }
+    const blockIds = new Set(this._blocks.map((block) => block.id));
+    for (const id of this._blockLineCounts.keys()) {
+      if (!blockIds.has(id)) {
+        this._blockLineCounts.delete(id);
+      }
+    }
+    this._changes.length = 0;
+    return this;
+  }
+};
+
+// packages/plugins/plugin-transcription/src/components/Transcript/transcript-extension.ts
+var transcript = ({ model, started, renderButton }) => {
+  return [
+    // Show timestamps in the gutter.
+    gutter({
+      class: "cm-timestamp-gutter",
+      lineMarkerChange: (update) => update.docChanged || update.viewportChanged,
+      markers: (view) => {
+        const start = getStartTime(started, model.blocks[0]);
+        const builder = new RangeSetBuilder();
+        for (const { from, to } of view.visibleRanges) {
+          let line = view.state.doc.lineAt(from);
+          while (line.from <= to) {
+            const timestamp = model.getBlockAtLine(line.number)?.segments[0]?.started;
+            if (timestamp) {
+              builder.add(line.from, line.from, new TimestampMarker(line.number, new Date(timestamp), start));
+            }
+            if (line.to + 1 > view.state.doc.length) {
+              break;
+            }
+            line = view.state.doc.lineAt(line.to + 1);
+          }
+        }
+        return builder.finish();
+      }
+    }),
+    // Listen for model updates.
+    ViewPlugin.fromClass(class {
+      constructor(view) {
+        this._initialized = false;
+        this._adapter = new DocumentAdapter(view);
+        const scroller = view.scrollDOM;
+        let isAutoScrolling = false;
+        let hasScrolled = false;
+        const scrollToBottom = () => {
+          scroller.style.scrollBehavior = "smooth";
+          scroller.classList.add("cm-hide-scrollbar");
+          isAutoScrolling = true;
+          setTimeout(() => {
+            this._controls?.classList.add("opacity-0");
+            scroller.classList.remove("cm-hide-scrollbar");
+            isAutoScrolling = false;
+          }, 500);
+          view.dispatch({
+            effects: EditorView.scrollIntoView(view.state.doc.length, {
+              y: "end"
+            })
+          });
+        };
+        if (renderButton) {
+          this._controls = document.createElement("div");
+          this._controls.classList.add("cm-controls", "transition-opacity", "duration-300", "opacity-0");
+          view.dom.appendChild(this._controls);
+          renderButton(this._controls, {
+            onClick: () => {
+              scrollToBottom();
+            }
+          }, view);
+        }
+        this._cleanup = combine(addEventListener(view.scrollDOM, "scroll", () => {
+          if (!isAutoScrolling) {
+            hasScrolled = true;
+            this._controls?.classList.remove("opacity-0");
+          }
+        }), model.update.on(() => {
+          const autoScroll = scroller.scrollHeight - scroller.scrollTop - scroller.clientHeight === 0 || !hasScrolled;
+          model.sync(this._adapter);
+          if (autoScroll) {
+            scrollToBottom();
+          }
+        }));
+      }
+      update(update) {
+        if (!this._initialized) {
+          this._initialized = true;
+          setTimeout(() => {
+            model.sync(this._adapter);
+          });
+        }
+      }
+      destroy() {
+        this._controls?.remove();
+        this._cleanup();
+      }
+    }),
+    EditorView.theme({
+      ".cm-scroller": {
+        overflowY: "scroll"
+      },
+      ".cm-hide-scrollbar": {
+        scrollbarWidth: "none",
+        "-ms-overflow-style": "none"
+      },
+      ".cm-hide-scrollbar::-webkit-scrollbar": {
+        display: "none"
+      },
+      ".cm-controls": {
+        position: "absolute",
+        bottom: "0.5rem",
+        right: "0.5rem",
+        zIndex: 1e3
+      },
+      ".cm-line": {
+        paddingRight: "1rem"
+      },
+      ".cm-timestamp-gutter": {
+        width: "6rem",
+        paddingRight: "1rem"
+      },
+      ".cm-timestamp-gutter > .cm-gutterElement > div": {
+        display: "inline-flex",
+        textAlign: "right",
+        padding: "3px"
+      }
+    })
+  ];
+};
+var TimestampMarker = class extends GutterMarker {
+  constructor(_line, _timestamp, _started) {
+    super();
+    this._line = _line;
+    this._timestamp = _timestamp;
+    this._started = _started;
+  }
+  eq(other) {
+    return other._timestamp === this._timestamp;
+  }
+  toDOM(view) {
+    const el = document.createElement("div");
+    el.className = "text-sm text-subdued hover:bg-hoverSurface cursor-pointer";
+    el.textContent = formatTimestamp(this._timestamp, this._started);
+    el.onclick = () => {
+      const pos = view.state.doc.line(this._line).from;
+      view.dispatch({
+        effects: EditorView.scrollIntoView(pos, {
+          y: "start"
+        })
+      });
+    };
+    return el;
+  }
+};
+var getStartTime = (started, block) => {
+  if (started) {
+    return started;
+  }
+  if (block?.segments[0]?.started) {
+    return new Date(block.segments[0].started);
+  }
+  return void 0;
+};
+var formatTimestamp = (timestamp, relative) => {
+  if (!relative) {
+    return format(timestamp, "HH:mm:ss");
+  } else {
+    const pad = (n = 0) => String(n).padStart(2, "0");
+    const { hours, minutes, seconds } = intervalToDuration({
+      start: relative,
+      end: timestamp
+    });
+    return `${pad(hours)}:${pad(minutes)}:${pad(seconds)}`;
+  }
+};
+
+// packages/plugins/plugin-transcription/src/components/Transcript/Transcript.tsx
+var __dxlog_file2 = "/home/runner/work/dxos/dxos/packages/plugins/plugin-transcription/src/components/Transcript/Transcript.tsx";
+var renderMarkdown = (block, debug = false) => {
+  return [
+    `###### ${block.authorName}` + (debug ? ` (${block.id})` : ""),
+    block.segments.map((segment) => segment.text.trim()).join(" "),
+    ""
+  ];
+};
+var Transcript = ({ classNames, transcript: object, space, model, attendableId, ignoreAttention }) => {
+  const client = useClient();
+  const { themeMode } = useThemeContext();
+  const { parentRef } = useTextEditor(() => {
+    return {
+      extensions: [
+        createBasicExtensions({
+          readOnly: true,
+          lineWrapping: true
+        }),
+        createMarkdownExtensions({
+          themeMode
+        }),
+        createThemeExtensions({
+          themeMode
+        }),
+        decorateMarkdown(),
+        space && preview({
+          onLookup: async ({ label, ref }) => {
+            log2.info("onLookup", {
+              label,
+              ref
+            }, {
+              F: __dxlog_file2,
+              L: 71,
+              S: void 0,
+              C: (f, a) => f(...a)
+            });
+            const dxn = DXN.parse(ref);
+            if (!dxn) {
+              return null;
+            }
+            const object2 = await resolveRef(client, dxn, space);
+            return {
+              label,
+              object: object2
+            };
+          }
+        }),
+        transcript({
+          model,
+          started: object?.started ? new Date(object.started) : void 0,
+          renderButton: createRenderer(({ onClick }) => /* @__PURE__ */ React.createElement(IconButton, {
+            icon: "ph--arrow-line-down--regular",
+            iconOnly: true,
+            label: "Scroll to bottom",
+            onClick
+          }))
+        })
+      ].filter(isNotFalsy)
+    };
+  }, [
+    space,
+    model
+  ]);
+  return /* @__PURE__ */ React.createElement("div", {
+    ref: parentRef,
+    className: mx("flex grow overflow-hidden", editorWidth, classNames)
+  });
+};
+
+// packages/plugins/plugin-transcription/src/hooks/useAudioFile.ts
+import { useEffect, useState } from "react";
+import { log as log3 } from "@dxos/log";
+var __dxlog_file3 = "/home/runner/work/dxos/dxos/packages/plugins/plugin-transcription/src/hooks/useAudioFile.ts";
+var useAudioFile = (audioUrl) => {
+  const [{ audio, stream, track }, setStream] = useState({});
+  useEffect(() => {
+    const t = setTimeout(async () => {
+      const response = await fetch(audioUrl);
+      const blob = await response.blob();
+      const audio2 = new Audio();
+      audio2.src = URL.createObjectURL(blob);
+      await new Promise((resolve, reject) => {
+        audio2.addEventListener("error", (err) => {
+          log3.error("error", {
+            err
+          }, {
+            F: __dxlog_file3,
+            L: 27,
+            S: void 0,
+            C: (f, a) => f(...a)
+          });
+          reject(err);
+        }, {
+          once: true
+        });
+        audio2.addEventListener("canplay", () => {
+          log3.info("starting...", void 0, {
+            F: __dxlog_file3,
+            L: 35,
+            S: void 0,
+            C: (f, a) => f(...a)
+          });
+          resolve();
+        }, {
+          once: true
+        });
+        audio2.load();
+      });
+      const ctx = new AudioContext();
+      const destination = ctx.createMediaStreamDestination();
+      destination.channelCount = 1;
+      const source = ctx.createMediaElementSource(audio2);
+      source.connect(destination);
+      setStream({
+        audio: audio2,
+        stream: destination.stream,
+        track: destination.stream.getAudioTracks()[0]
+      });
+    });
+    return () => clearTimeout(t);
+  }, [
+    audioUrl
+  ]);
+  return {
+    audio,
+    stream,
+    track
+  };
+};
+
+// packages/plugins/plugin-transcription/src/hooks/useAudioTrack.ts
+import { useEffect as useEffect2, useState as useState2 } from "react";
+var useAudioTrack = (active) => {
+  const [track, setTrack] = useState2();
+  useEffect2(() => {
+    if (!active) {
+      track?.stop();
+      setTrack(void 0);
+      return;
+    }
+    const initAudio = async () => {
+      const audio = new Audio();
+      audio.srcObject = await navigator.mediaDevices.getUserMedia({
+        audio: true
+      });
+      const [track2] = audio.srcObject.getAudioTracks();
+      if (track2) {
+        setTrack(track2);
+      }
+    };
+    if (active) {
+      void initAudio();
+    }
+  }, [
+    active
+  ]);
+  return track;
+};
+
+// packages/plugins/plugin-transcription/src/hooks/useQueueModelAdapter.ts
+import { useMemo, useEffect as useEffect3 } from "react";
+var useQueueModelAdapter = (renderer, queue, initialBlocks = []) => {
+  const model = useMemo(() => new BlockModel(renderer, initialBlocks), [
+    queue
+  ]);
+  useEffect3(() => {
+    if (!queue?.items.length) {
+      return;
+    }
+    const block = queue.items[queue.items.length - 1];
+    model.appendBlock(block);
+  }, [
+    model,
+    queue?.items.length
+  ]);
+  return model;
+};
+
+// packages/plugins/plugin-transcription/src/hooks/useTranscriber.ts
+import { useEffect as useEffect4, useMemo as useMemo2 } from "react";
+import { useCapabilities } from "@dxos/app-framework";
+
+// packages/plugins/plugin-transcription/src/capabilities/index.ts
+import { lazy } from "@dxos/app-framework";
+var AppGraphBuilder = lazy(() => import("./app-graph-builder-I4SSIQMW.mjs"));
+var IntentResolver = lazy(() => import("./intent-resolver-ZQQEH27F.mjs"));
+var MeetingTranscriptionState = lazy(() => import("./meeting-transcription-state-5D52VTHU.mjs"));
+var ReactSurface = lazy(() => import("./react-surface-HETJ5XY6.mjs"));
+var Transcriber = lazy(() => import("./transcriber-SMRLU2KY.mjs"));
+
+// packages/plugins/plugin-transcription/src/hooks/useTranscriber.ts
+var useTranscriber = ({ audioStreamTrack, onSegments, config }) => {
+  const [getTranscriber] = useCapabilities(TranscriptionCapabilities.Transcriber);
+  const transcriber = useMemo2(() => {
+    if (!onSegments || !audioStreamTrack || !getTranscriber) {
+      return void 0;
+    }
+    return getTranscriber({
+      audioStreamTrack,
+      onSegments,
+      config
+    });
+  }, [
+    audioStreamTrack,
+    onSegments,
+    getTranscriber,
+    config
+  ]);
+  useEffect4(() => {
+    return () => {
+      void transcriber?.close();
+    };
+  }, [
+    transcriber
+  ]);
+  return transcriber;
+};
+
+// packages/plugins/plugin-transcription/src/hooks/useVoiceInput.ts
+import { useState as useState3, useEffect as useEffect5, useCallback } from "react";
+import { scheduleMicroTask } from "@dxos/async";
+import { Context } from "@dxos/context";
+import { log as log4 } from "@dxos/log";
+import { useSoundEffect } from "@dxos/react-ui-sfx";
+var __dxlog_file4 = "/home/runner/work/dxos/dxos/packages/plugins/plugin-transcription/src/hooks/useVoiceInput.ts";
+var useVoiceInput = ({ active, onUpdate }) => {
+  const soundStart = useSoundEffect("StartRecording");
+  const soundStop = useSoundEffect("StopRecording");
+  const handleSegments = useCallback(async (segments) => {
+    const text = segments.map((str) => str.text.trim().replace(/[^\w\s]/g, "")).join(" ");
+    onUpdate(text);
+  }, []);
+  const [recording, setRecording] = useState3(false);
+  const track = useAudioTrack(active);
+  const transcriber = useTranscriber({
+    audioStreamTrack: track,
+    onSegments: handleSegments
+  });
+  useEffect5(() => {
+    const ctx = new Context(void 0, {
+      F: __dxlog_file4,
+      L: 43
+    });
+    scheduleMicroTask(ctx, async () => {
+      if (active && transcriber) {
+        await transcriber.open();
+        log4.info("starting...", void 0, {
+          F: __dxlog_file4,
+          L: 47,
+          S: void 0,
+          C: (f, a) => f(...a)
+        });
+        setRecording(true);
+        void soundStart.play();
+        transcriber.startChunksRecording();
+      } else if (!active && transcriber?.isOpen) {
+        transcriber?.stopChunksRecording();
+        await transcriber?.close();
+        log4.info("stopped", void 0, {
+          F: __dxlog_file4,
+          L: 54,
+          S: void 0,
+          C: (f, a) => f(...a)
+        });
+        setRecording(false);
+        void soundStop.play();
+      }
+    });
+    return () => {
+      void ctx.dispose();
+    };
+  }, [
+    active,
+    transcriber
+  ]);
+  return {
+    recording
+  };
+};
+
+// packages/plugins/plugin-transcription/src/components/TranscriptContainer.tsx
+var TranscriptionContainer = ({ role, transcript: transcript2 }) => {
+  const attendableId = fullyQualifiedId(transcript2);
+  const space = getSpace(transcript2);
+  const queue = useQueue(transcript2.queue.dxn, {
+    pollInterval: 1e3
+  });
+  const model = useQueueModelAdapter(renderMarkdown, queue);
+  return /* @__PURE__ */ React2.createElement(StackItem.Content, {
+    role,
+    classNames: "container-max-width pli-2"
+  }, /* @__PURE__ */ React2.createElement(Transcript, {
+    attendableId,
+    space,
+    model,
+    transcript: transcript2
+  }));
+};
+var TranscriptContainer_default = TranscriptionContainer;
+
+export {
+  renderMarkdown,
+  Transcript,
+  useAudioFile,
+  useAudioTrack,
+  useQueueModelAdapter,
+  useTranscriber,
+  useVoiceInput,
+  TranscriptionContainer,
+  TranscriptContainer_default,
+  AppGraphBuilder,
+  IntentResolver,
+  MeetingTranscriptionState,
+  ReactSurface,
+  Transcriber
+};
+//# sourceMappingURL=chunk-RHHJZQPM.mjs.map