@dxos/plugin-transcription 0.8.2-staging.7ac8446 → 0.8.2

package/dist/lib/browser/chunk-TJPJBE45.mjs

@@ -0,0 +1,764 @@
+import {
+  TranscriptionCapabilities
+} from "./chunk-W2KNYJD6.mjs";
+
+// packages/plugins/plugin-transcription/src/components/TranscriptContainer.tsx
+import { useSignals as _useSignals2 } from "@preact-signals/safe-react/tracking";
+import React2 from "react";
+import { fullyQualifiedId, getSpace } from "@dxos/client/echo";
+import { useMembers, useQueue } from "@dxos/react-client/echo";
+import { StackItem } from "@dxos/react-ui-stack";
+
+// packages/plugins/plugin-transcription/src/components/Transcript/Transcript.tsx
+import { useSignals as _useSignals } from "@preact-signals/safe-react/tracking";
+import React from "react";
+import { IconButton, useThemeContext } from "@dxos/react-ui";
+import { createBasicExtensions, createMarkdownExtensions, createRenderer, createThemeExtensions, decorateMarkdown, editorWidth, preview, useTextEditor } from "@dxos/react-ui-editor";
+import { mx } from "@dxos/react-ui-theme";
+import { isNotFalsy } from "@dxos/util";
+
+// packages/plugins/plugin-transcription/src/components/Transcript/transcript-extension.ts
+import { RangeSetBuilder } from "@codemirror/state";
+import { EditorView, GutterMarker, ViewPlugin, gutter } from "@codemirror/view";
+import { format } from "date-fns/format";
+import { intervalToDuration } from "date-fns/intervalToDuration";
+import { addEventListener, combine } from "@dxos/async";
+
+// packages/plugins/plugin-transcription/src/model/model.ts
+import { Text } from "@codemirror/state";
+import { Event } from "@dxos/async";
+import { invariant } from "@dxos/invariant";
+import { log } from "@dxos/log";
+var __dxlog_file = "/home/runner/work/dxos/dxos/packages/plugins/plugin-transcription/src/model/model.ts";
+var DocumentAdapter = class {
+  constructor(_view) {
+    this._view = _view;
+  }
+  /** The document must always have at least one line. */
+  lineCount() {
+    return this._view.state.doc.lines - 1;
+  }
+  replaceLines(from, remove, insert) {
+    log("replaceLines", {
+      count: this._view.state.doc.lines,
+      from,
+      remove,
+      insert
+    }, {
+      F: __dxlog_file,
+      L: 39,
+      S: this,
+      C: (f, a) => f(...a)
+    });
+    const numLines = this._view.state.doc.lines;
+    const posFrom = this._view.state.doc.line(from).from;
+    const posTo = remove === 0 ? posFrom : this._view.state.doc.line(Math.min(numLines, from + remove)).from;
+    let text = insert.join("\n");
+    if (posTo === this._view.state.doc.length) {
+      text += "\n";
+    }
+    this._view.dispatch({
+      changes: {
+        from: posFrom,
+        to: posTo,
+        insert: text
+      }
+    });
+  }
+};
+var SerializationModel = class {
+  constructor(_renderer, initialChunks = []) {
+    this._renderer = _renderer;
+    this._chunks = [];
+    this._chunkLineCounts = /* @__PURE__ */ new Map();
+    this._lineToChunk = /* @__PURE__ */ new Map();
+    this._changes = [];
+    this.update = new Event();
+    initialChunks.forEach((chunk) => {
+      this.appendChunk(chunk);
+    });
+  }
+  toJSON() {
+    return {
+      chunks: this._chunks.length,
+      changes: this._changes.length
+    };
+  }
+  get doc() {
+    return Text.of([
+      ...this._chunks.flatMap((chunk, index) => this._renderer(chunk, index)),
+      ""
+    ]);
+  }
+  get chunks() {
+    return this._chunks;
+  }
+  /**
+  * Get the block ID at the given line number.
+  */
+  getChunkAtLine(line) {
+    return this._lineToChunk.get(line);
+  }
+  reset() {
+    this._chunks.length = 0;
+    this._chunkLineCounts.clear();
+    this._lineToChunk.clear();
+    this.update.emit();
+    return this;
+  }
+  appendChunk(chunk) {
+    const index = this._chunks.length;
+    this._chunks.push(chunk);
+    this._changes.push({
+      type: "append",
+      index,
+      chunk
+    });
+    this.update.emit();
+    return this;
+  }
+  updateChunk(chunk) {
+    const index = this._chunks.findIndex((c) => c.id === chunk.id);
+    invariant(index !== -1, void 0, {
+      F: __dxlog_file,
+      L: 147,
+      S: this,
+      A: [
+        "index !== -1",
+        ""
+      ]
+    });
+    this._chunks[index] = chunk;
+    this._changes.push({
+      type: "update",
+      index,
+      chunk
+    });
+    this.update.emit();
+    return this;
+  }
+  deleteBlock(id) {
+    const index = this._chunks.findIndex((c) => c.id === id);
+    invariant(index !== -1, void 0, {
+      F: __dxlog_file,
+      L: 156,
+      S: this,
+      A: [
+        "index !== -1",
+        ""
+      ]
+    });
+    this._chunks.splice(index, 1);
+    this._changes.push({
+      type: "delete",
+      index,
+      id
+    });
+    this.update.emit();
+    return this;
+  }
+  /**
+  * Syncs the tracked changes with the ChunkDocument.
+  */
+  sync(document2) {
+    log("sync", {
+      changes: this._changes
+    }, {
+      F: __dxlog_file,
+      L: 167,
+      S: this,
+      C: (f, a) => f(...a)
+    });
+    for (const change of this._changes) {
+      switch (change.type) {
+        case "append": {
+          const lines = this._renderer(change.chunk, change.index);
+          const lastLine = document2.lineCount();
+          document2.replaceLines(lastLine + 1, 0, lines);
+          break;
+        }
+        case "update": {
+          let lineStart = 1;
+          for (let i = 0; i < change.index; i++) {
+            lineStart += this._chunkLineCounts.get(this._chunks[i].id);
+          }
+          const previousLineCount = this._chunkLineCounts.get(change.chunk.id);
+          const newLines = this._renderer(change.chunk, change.index);
+          document2.replaceLines(lineStart, previousLineCount, newLines);
+          break;
+        }
+        case "delete": {
+          let lineStart = 1;
+          for (let i = 0; i < change.index; i++) {
+            lineStart += this._chunkLineCounts.get(this._chunks[i].id);
+          }
+          const previousLineCount = this._chunkLineCounts.get(change.id);
+          document2.replaceLines(lineStart, previousLineCount, []);
+          break;
+        }
+      }
+    }
+    this._lineToChunk.clear();
+    let currentLine = 1;
+    for (const [index, chunk] of this._chunks.entries()) {
+      const lineCount = this._renderer(chunk, index).length;
+      this._chunkLineCounts.set(chunk.id, lineCount);
+      this._lineToChunk.set(currentLine, chunk);
+      currentLine += lineCount;
+    }
+    const chunkIds = new Set(this._chunks.map((chunk) => chunk.id));
+    for (const id of this._chunkLineCounts.keys()) {
+      if (!chunkIds.has(id)) {
+        this._chunkLineCounts.delete(id);
+      }
+    }
+    this._changes.length = 0;
+    return this;
+  }
+};
+
+// packages/plugins/plugin-transcription/src/components/Transcript/transcript-extension.ts
+var transcript = ({ model, started, renderButton }) => {
+  return [
+    // Show timestamps in the gutter.
+    gutter({
+      class: "cm-timestamp-gutter",
+      lineMarkerChange: (update) => update.docChanged || update.viewportChanged,
+      markers: (view) => {
+        const start = getStartTime(started, model.chunks[0]);
+        const builder = new RangeSetBuilder();
+        for (const { from, to } of view.visibleRanges) {
+          let line = view.state.doc.lineAt(from);
+          while (line.from <= to) {
+            const block = model.getChunkAtLine(line.number)?.blocks[0];
+            const timestamp = block?.type === "transcription" && block.started;
+            if (timestamp) {
+              builder.add(line.from, line.from, new TimestampMarker(line.number, new Date(timestamp), start));
+            }
+            if (line.to + 1 > view.state.doc.length) {
+              break;
+            }
+            line = view.state.doc.lineAt(line.to + 1);
+          }
+        }
+        return builder.finish();
+      }
+    }),
+    // Listen for model updates.
+    ViewPlugin.fromClass(class {
+      constructor(view) {
+        this._initialized = false;
+        this._adapter = new DocumentAdapter(view);
+        const scroller = view.scrollDOM;
+        let isAutoScrolling = false;
+        let hasScrolled = false;
+        let timeout;
+        const scrollToBottom = (smooth = false) => {
+          scroller.style.scrollBehavior = smooth ? "smooth" : "";
+          scroller.classList.add("cm-hide-scrollbar");
+          isAutoScrolling = true;
+          clearTimeout(timeout);
+          timeout = setTimeout(() => {
+            this._controls?.classList.add("opacity-0");
+            scroller.classList.remove("cm-hide-scrollbar");
+            isAutoScrolling = false;
+          }, 500);
+          view.dispatch({
+            effects: EditorView.scrollIntoView(view.state.doc.length, {
+              y: "end"
+            })
+          });
+        };
+        if (renderButton) {
+          this._controls = document.createElement("div");
+          this._controls.classList.add("cm-controls", "transition-opacity", "duration-300", "opacity-0");
+          view.dom.appendChild(this._controls);
+          renderButton(this._controls, {
+            onClick: () => {
+              scrollToBottom(false);
+            }
+          }, view);
+        }
+        this._cleanup = combine(addEventListener(view.scrollDOM, "scroll", () => {
+          if (!isAutoScrolling) {
+            hasScrolled = true;
+            this._controls?.classList.remove("opacity-0");
+          }
+        }), model.update.on(() => {
+          const autoScroll = scroller.scrollHeight - scroller.scrollTop - scroller.clientHeight === 0 || !hasScrolled;
+          model.sync(this._adapter);
+          if (autoScroll) {
+            scrollToBottom(true);
+          }
+        }));
+      }
+      update(update) {
+        if (!this._initialized) {
+          this._initialized = true;
+          setTimeout(() => {
+            model.sync(this._adapter);
+          });
+        }
+      }
+      destroy() {
+        this._controls?.remove();
+        this._cleanup();
+      }
+    }),
+    EditorView.theme({
+      ".cm-scroller": {
+        overflowY: "scroll"
+      },
+      ".cm-hide-scrollbar": {
+        scrollbarWidth: "none",
+        "-ms-overflow-style": "none"
+      },
+      ".cm-hide-scrollbar::-webkit-scrollbar": {
+        display: "none"
+      },
+      ".cm-controls": {
+        position: "absolute",
+        bottom: "0.5rem",
+        right: "0.5rem",
+        zIndex: 1e3
+      },
+      ".cm-line": {
+        paddingRight: "1rem"
+      },
+      ".cm-timestamp-gutter": {
+        width: "6rem",
+        paddingRight: "1rem"
+      },
+      ".cm-timestamp-gutter > .cm-gutterElement > div": {
+        display: "inline-flex",
+        textAlign: "right",
+        padding: "3px"
+      }
+    })
+  ];
+};
+var TimestampMarker = class extends GutterMarker {
+  constructor(_line, _timestamp, _started) {
+    super(), this._line = _line, this._timestamp = _timestamp, this._started = _started;
+  }
+  eq(other) {
+    return other._timestamp === this._timestamp;
+  }
+  toDOM(view) {
+    const el = document.createElement("div");
+    el.className = "text-sm text-subdued hover:bg-hoverSurface cursor-pointer";
+    el.textContent = formatTimestamp(this._timestamp, this._started);
+    el.onclick = () => {
+      const pos = view.state.doc.line(this._line).from;
+      view.dispatch({
+        effects: EditorView.scrollIntoView(pos, {
+          y: "start"
+        })
+      });
+    };
+    return el;
+  }
+};
+var getStartTime = (started, message) => {
+  if (started) {
+    return started;
+  }
+  if (message?.blocks[0]?.type === "transcription" && message.blocks[0].started) {
+    return new Date(message.blocks[0].started);
+  }
+  return void 0;
+};
+var formatTimestamp = (timestamp, relative) => {
+  if (relative) {
+    const pad = (n = 0) => String(n).padStart(2, "0");
+    const { hours, minutes, seconds } = intervalToDuration({
+      start: relative,
+      end: timestamp
+    });
+    return `${pad(hours)}:${pad(minutes)}:${pad(seconds)}`;
+  } else {
+    return format(timestamp, "HH:mm:ss");
+  }
+};
+
+// packages/plugins/plugin-transcription/src/components/Transcript/Transcript.tsx
+var renderMarkdown = (identities) => (message, index, debug = false) => {
+  if (message.sender.role === "assistant") {
+    return [
+      message.blocks.find((block) => block.type === "transcription")?.text ?? "",
+      ""
+    ];
+  }
+  const identity = identities.find((identity2) => identity2.did === message.sender.identityDid);
+  const name = identity?.profile?.displayName ?? message.sender.contact?.target?.fullName ?? message.sender.name ?? message.sender.email ?? message.sender.identityDid;
+  const blocks = message.blocks.filter((block) => block.type === "transcription");
+  return [
+    `###### ${name}` + (debug ? ` [${index}]:${message.id}` : ""),
+    blocks.map((block) => block.text.trim()).join(" "),
+    ""
+  ];
+};
+var Transcript = ({ classNames, transcript: object, space, model, attendableId, ignoreAttention }) => {
+  var _effect = _useSignals();
+  try {
+    const { themeMode } = useThemeContext();
+    const { parentRef } = useTextEditor(() => {
+      return {
+        extensions: [
+          createBasicExtensions({
+            readOnly: true,
+            lineWrapping: true
+          }),
+          createMarkdownExtensions({
+            themeMode
+          }),
+          createThemeExtensions({
+            themeMode
+          }),
+          decorateMarkdown(),
+          space && preview(),
+          transcript({
+            model,
+            started: object?.started ? new Date(object.started) : void 0,
+            renderButton: createRenderer(({ onClick }) => /* @__PURE__ */ React.createElement(IconButton, {
+              icon: "ph--arrow-line-down--regular",
+              iconOnly: true,
+              label: "Scroll to bottom",
+              onClick
+            }))
+          })
+        ].filter(isNotFalsy)
+      };
+    }, [
+      space,
+      model
+    ]);
+    return /* @__PURE__ */ React.createElement("div", {
+      ref: parentRef,
+      className: mx("flex grow overflow-hidden", editorWidth, classNames),
+      "data-popover-collision-boundary": true
+    });
+  } finally {
+    _effect.f();
+  }
+};
+
+// packages/plugins/plugin-transcription/src/hooks/useAudioFile.ts
+import { useEffect, useState } from "react";
+import { scheduleTask } from "@dxos/async";
+import { Context } from "@dxos/context";
+import { log as log2 } from "@dxos/log";
+var __dxlog_file2 = "/home/runner/work/dxos/dxos/packages/plugins/plugin-transcription/src/hooks/useAudioFile.ts";
+var useAudioFile = (audioUrl, constraints) => {
+  const [{ audio, stream, track }, setStream] = useState({});
+  useEffect(() => {
+    const ctx = new Context(void 0, {
+      F: __dxlog_file2,
+      L: 20
+    });
+    scheduleTask(ctx, async () => {
+      const response = await fetch(audioUrl);
+      const blob = await response.blob();
+      const objectUrl = URL.createObjectURL(blob);
+      ctx.onDispose(() => {
+        URL.revokeObjectURL(objectUrl);
+      });
+      const audio2 = new Audio();
+      audio2.src = objectUrl;
+      await new Promise((resolve, reject) => {
+        audio2.addEventListener("error", (err) => {
+          log2.error("error", {
+            err
+          }, {
+            F: __dxlog_file2,
+            L: 37,
+            S: void 0,
+            C: (f, a) => f(...a)
+          });
+          reject(err);
+        }, {
+          once: true
+        });
+        audio2.addEventListener("canplay", async () => {
+          log2.info("starting...", void 0, {
+            F: __dxlog_file2,
+            L: 45,
+            S: void 0,
+            C: (f, a) => f(...a)
+          });
+          try {
+            await audio2.play();
+            resolve();
+          } catch (playError) {
+            log2.error("Play failed", {
+              playError
+            }, {
+              F: __dxlog_file2,
+              L: 51,
+              S: void 0,
+              C: (f, a) => f(...a)
+            });
+            resolve();
+          }
+        }, {
+          once: true
+        });
+        audio2.load();
+      });
+      const audioCtx = new AudioContext();
+      if (audioCtx.state === "suspended") {
+        await audioCtx.resume();
+      }
+      const destination = audioCtx.createMediaStreamDestination();
+      destination.channelCount = 1;
+      const source = audioCtx.createMediaElementSource(audio2);
+      source.connect(destination);
+      const track2 = destination.stream.getAudioTracks()[0];
+      if (constraints) {
+        await track2.applyConstraints(constraints).catch((err) => log2.catch(err, void 0, {
+          F: __dxlog_file2,
+          L: 75,
+          S: void 0,
+          C: (f, a) => f(...a)
+        }));
+      }
+      setStream({
+        audio: audio2,
+        stream: destination.stream,
+        track: destination.stream.getAudioTracks()[0]
+      });
+    });
+    return () => {
+      void ctx.dispose();
+    };
+  }, [
+    audioUrl
+  ]);
+  return {
+    audio,
+    stream,
+    track
+  };
+};
+
+// packages/plugins/plugin-transcription/src/hooks/useAudioTrack.ts
+import { useEffect as useEffect2, useState as useState2 } from "react";
+var useAudioTrack = (active, constraints) => {
+  const [track, setTrack] = useState2();
+  useEffect2(() => {
+    if (!active) {
+      track?.stop();
+      setTrack(void 0);
+      return;
+    }
+    const initAudio = async () => {
+      const audio = new Audio();
+      audio.srcObject = await navigator.mediaDevices.getUserMedia({
+        audio: constraints ?? true
+      });
+      const [track2] = audio.srcObject.getAudioTracks();
+      if (track2) {
+        setTrack(track2);
+      }
+    };
+    if (active) {
+      void initAudio();
+    }
+  }, [
+    active
+  ]);
+  return track;
+};
+
+// packages/plugins/plugin-transcription/src/hooks/useQueueModelAdapter.ts
+import { useMemo, useEffect as useEffect3, useState as useState3 } from "react";
+var useQueueModelAdapter = (renderer, queue, initialChunks = []) => {
+  const model = useMemo(() => new SerializationModel(renderer, initialChunks), [
+    queue
+  ]);
+  const [loaded, setLoaded] = useState3(false);
+  useEffect3(() => {
+    if (!queue) {
+      return;
+    }
+    const update = () => {
+      for (const block of queue?.objects ?? []) {
+        model.appendChunk(block);
+      }
+      setLoaded(true);
+    };
+    let i;
+    if (queue.isLoading) {
+      i = setInterval(() => {
+        if (!queue?.isLoading) {
+          clearInterval(i);
+          update();
+        }
+      }, 1e3);
+    } else {
+      update();
+    }
+    return () => clearTimeout(i);
+  }, [
+    model,
+    queue
+  ]);
+  useEffect3(() => {
+    if (!loaded || !queue || model.chunks.length === queue.objects.length) {
+      return;
+    }
+    const chunk = queue.objects.at(-1);
+    if (chunk) {
+      model.appendChunk(chunk);
+    }
+  }, [
+    model,
+    loaded,
+    queue?.objects.length
+  ]);
+  return model;
+};
+
+// packages/plugins/plugin-transcription/src/hooks/useTranscriber.ts
+import { useEffect as useEffect4, useMemo as useMemo2 } from "react";
+import { useCapabilities } from "@dxos/app-framework";
+
+// packages/plugins/plugin-transcription/src/capabilities/index.ts
+import { lazy } from "@dxos/app-framework";
+var IntentResolver = lazy(() => import("./intent-resolver-I2LJSQDR.mjs"));
+var ReactSurface = lazy(() => import("./react-surface-ID2MLJHZ.mjs"));
+var Transcriber = lazy(() => import("./transcriber-NA7CIFIT.mjs"));
+
+// packages/plugins/plugin-transcription/src/hooks/useTranscriber.ts
+var useTranscriber = ({ audioStreamTrack, onSegments, transcriberConfig, recorderConfig }) => {
+  const [getTranscriber] = useCapabilities(TranscriptionCapabilities.Transcriber);
+  const transcriber = useMemo2(() => {
+    if (!onSegments || !audioStreamTrack || !getTranscriber) {
+      return void 0;
+    }
+    return getTranscriber({
+      audioStreamTrack,
+      onSegments,
+      transcriberConfig,
+      recorderConfig
+    });
+  }, [
+    audioStreamTrack,
+    onSegments,
+    getTranscriber,
+    transcriberConfig,
+    recorderConfig
+  ]);
+  useEffect4(() => {
+    return () => {
+      void transcriber?.close();
+    };
+  }, [
+    transcriber
+  ]);
+  return transcriber;
+};
+
+// packages/plugins/plugin-transcription/src/hooks/useVoiceInput.ts
+import { useState as useState4, useEffect as useEffect5, useCallback } from "react";
+import { scheduleMicroTask } from "@dxos/async";
+import { Context as Context2 } from "@dxos/context";
+import { log as log3 } from "@dxos/log";
+import { useSoundEffect } from "@dxos/react-ui-sfx";
+var __dxlog_file3 = "/home/runner/work/dxos/dxos/packages/plugins/plugin-transcription/src/hooks/useVoiceInput.ts";
+var useVoiceInput = ({ active, onUpdate }) => {
+  const soundStart = useSoundEffect("StartRecording");
+  const soundStop = useSoundEffect("StopRecording");
+  const handleSegments = useCallback(async (segments) => {
+    const text = segments.map((str) => str.text.trim().replace(/[^\w\s]/g, "")).join(" ");
+    onUpdate(text);
+  }, []);
+  const [recording, setRecording] = useState4(false);
+  const track = useAudioTrack(active);
+  const transcriber = useTranscriber({
+    audioStreamTrack: track,
+    onSegments: handleSegments
+  });
+  useEffect5(() => {
+    const ctx = new Context2(void 0, {
+      F: __dxlog_file3,
+      L: 43
+    });
+    scheduleMicroTask(ctx, async () => {
+      if (active && transcriber) {
+        await transcriber.open();
+        log3.info("starting...", void 0, {
+          F: __dxlog_file3,
+          L: 47,
+          S: void 0,
+          C: (f, a) => f(...a)
+        });
+        setRecording(true);
+        void soundStart.play();
+        transcriber.startChunksRecording();
+      } else if (!active && transcriber?.isOpen) {
+        transcriber?.stopChunksRecording();
+        await transcriber?.close();
+        log3.info("stopped", void 0, {
+          F: __dxlog_file3,
+          L: 54,
+          S: void 0,
+          C: (f, a) => f(...a)
+        });
+        setRecording(false);
+        void soundStop.play();
+      }
+    });
+    return () => {
+      void ctx.dispose();
+    };
+  }, [
+    active,
+    transcriber
+  ]);
+  return {
+    recording
+  };
+};
+
+// packages/plugins/plugin-transcription/src/components/TranscriptContainer.tsx
+var TranscriptionContainer = ({ role, transcript: transcript2 }) => {
+  var _effect = _useSignals2();
+  try {
+    const attendableId = fullyQualifiedId(transcript2);
+    const space = getSpace(transcript2);
+    const members = useMembers(space?.key).map((member) => member.identity);
+    const queue = useQueue(transcript2.queue.dxn, {
+      pollInterval: 1e3
+    });
+    const model = useQueueModelAdapter(renderMarkdown(members), queue);
+    return /* @__PURE__ */ React2.createElement(StackItem.Content, {
+      role,
+      classNames: "container-max-width pli-2"
+    }, /* @__PURE__ */ React2.createElement(Transcript, {
+      attendableId,
+      space,
+      model,
+      transcript: transcript2
+    }));
+  } finally {
+    _effect.f();
+  }
+};
+var TranscriptContainer_default = TranscriptionContainer;
+
+export {
+  renderMarkdown,
+  Transcript,
+  useAudioFile,
+  useAudioTrack,
+  useQueueModelAdapter,
+  useTranscriber,
+  useVoiceInput,
+  TranscriptionContainer,
+  TranscriptContainer_default,
+  IntentResolver,
+  ReactSurface,
+  Transcriber
+};
+//# sourceMappingURL=chunk-TJPJBE45.mjs.map