@dxos/plugin-transcription 0.8.1-main.ba2dec9 → 0.8.1-staging.5be625a

package/dist/lib/browser/index.mjs

@@ -1,20 +1,23 @@
 import {
-  mergeFloat64Arrays
-} from "./chunk-GHTRJTEB.mjs";
+  MediaStreamRecorder,
+  Transcriber,
+  summarizeTranscript
+} from "./chunk-N3VKF2J3.mjs";
 import {
   TranscriptContainer
-} from "./chunk-BDVUIRBP.mjs";
+} from "./chunk-3NWYXRZS.mjs";
+import {
+  TranscriptBlock,
+  TranscriptType
+} from "./chunk-YRW4MNRE.mjs";
 import {
   Transcript,
   TranscriptionContainer
-} from "./chunk-YX447BJ4.mjs";
+} from "./chunk-MD4NQO6W.mjs";
 import {
   TRANSCRIPTION_PLUGIN,
-  TRANSCRIPTION_URL,
-  TranscriptBlock,
-  TranscriptType,
   meta
-} from "./chunk-QIHUPVQD.mjs";
+} from "./chunk-YEVVZZTC.mjs";
 
 // packages/plugins/experimental/plugin-transcription/src/capabilities/index.ts
 import { lazy } from "@dxos/app-framework";
@@ -27,9 +30,9 @@ var TranscriptionCapabilities;
 })(TranscriptionCapabilities || (TranscriptionCapabilities = {}));
 
 // packages/plugins/experimental/plugin-transcription/src/capabilities/index.ts
-var AppGraphBuilder = lazy(() => import("./app-graph-builder-ZSSJIFY7.mjs"));
-var IntentResolver = lazy(() => import("./intent-resolver-RW6DEJC2.mjs"));
-var ReactSurface = lazy(() => import("./react-surface-UPTZBHVB.mjs"));
+var AppGraphBuilder = lazy(() => import("./app-graph-builder-ZIIDYMHP.mjs"));
+var IntentResolver = lazy(() => import("./intent-resolver-AC6LFBE4.mjs"));
+var ReactSurface = lazy(() => import("./react-surface-OM2EECBI.mjs"));
 
 // packages/plugins/experimental/plugin-transcription/src/hooks/useAudioFile.ts
 import { useEffect, useState } from "react";
@@ -123,286 +126,6 @@ var useAudioTrack = (active) => {
 
 // packages/plugins/experimental/plugin-transcription/src/hooks/useTranscriber.ts
 import { useEffect as useEffect3, useMemo } from "react";
-
-// packages/plugins/experimental/plugin-transcription/src/transcriber/media-stream-recorder.ts
-import { MediaRecorder, register } from "extendable-media-recorder";
-import { WaveFile } from "wavefile";
-import { synchronized } from "@dxos/async";
-import { invariant } from "@dxos/invariant";
-import { log as log2 } from "@dxos/log";
-import { trace } from "@dxos/tracing";
-function _ts_decorate(decorators, target, key, desc) {
-  var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
-  if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
-  else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
-  return c > 3 && r && Object.defineProperty(target, key, r), r;
-}
-var __dxlog_file2 = "/home/runner/work/dxos/dxos/packages/plugins/experimental/plugin-transcription/src/transcriber/media-stream-recorder.ts";
-var initializingPromise;
-var initializeExtendableMediaRecorder = async () => {
-  const { connect } = await import("extendable-media-recorder-wav-encoder");
-  await register(await connect());
-};
-var MediaStreamRecorder = class {
-  constructor({ mediaStreamTrack, interval }) {
-    this._mediaRecorder = void 0;
-    this._header = void 0;
-    this._mediaStreamTrack = mediaStreamTrack;
-    this._interval = interval;
-  }
-  get wavConfig() {
-    const settings = this._mediaStreamTrack.getSettings();
-    return {
-      channels: settings.channelCount ?? 1,
-      sampleRate: settings.sampleRate ?? 16e3,
-      bitDepthCode: settings.sampleSize ? String(settings.sampleSize) : "16"
-    };
-  }
-  get mediaStreamTrack() {
-    return this._mediaStreamTrack;
-  }
-  setOnChunk(onChunk) {
-    this._onChunk = onChunk;
-  }
-  async start() {
-    await (initializingPromise ??= initializeExtendableMediaRecorder()).catch((err) => log2.info("initializeExtendableMediaRecorder", {
-      err
-    }, {
-      F: __dxlog_file2,
-      L: 62,
-      S: this,
-      C: (f, a) => f(...a)
-    }));
-    if (!this._mediaRecorder) {
-      const stream = new MediaStream([
-        this._mediaStreamTrack
-      ]);
-      this._mediaRecorder = new MediaRecorder(stream, {
-        mimeType: "audio/wav"
-      });
-    }
-    if (this._mediaRecorder.state === "recording") {
-      return;
-    }
-    invariant(this._onChunk, "MediaStreamRecorder: onChunk is not set", {
-      F: __dxlog_file2,
-      L: 73,
-      S: this,
-      A: [
-        "this._onChunk",
-        "'MediaStreamRecorder: onChunk is not set'"
-      ]
-    });
-    this._mediaRecorder.ondataavailable = (event) => this._ondataavailable(event);
-    this._mediaRecorder.start(this._interval);
-  }
-  async stop() {
-    if (this._mediaRecorder?.state !== "recording") {
-      return;
-    }
-    this._mediaRecorder.stop();
-  }
-  async _ondataavailable(event) {
-    const blob = event.data;
-    const uint8Array = new Uint8Array(await blob.arrayBuffer());
-    const isHeader = uint8Array[0] === 82 && // R
-    uint8Array[1] === 73 && // I
-    uint8Array[2] === 70 && // F
-    uint8Array[3] === 70;
-    let wav;
-    if (isHeader) {
-      wav = new WaveFile(uint8Array);
-      this._header = uint8Array.slice(0, 44);
-    } else if (this._header) {
-      wav = new WaveFile(new Uint8Array([
-        ...this._header,
-        ...uint8Array
-      ]));
-    } else {
-      log2.warn("MediaStreamRecorder: no header", void 0, {
-        F: __dxlog_file2,
-        L: 104,
-        S: this,
-        C: (f, a) => f(...a)
-      });
-      return;
-    }
-    this._onChunk({
-      timestamp: Date.now(),
-      data: wav.getSamples()
-    });
-  }
-};
-_ts_decorate([
-  synchronized
-], MediaStreamRecorder.prototype, "_ondataavailable", null);
-MediaStreamRecorder = _ts_decorate([
-  trace.resource()
-], MediaStreamRecorder);
-
-// packages/plugins/experimental/plugin-transcription/src/transcriber/transcriber.ts
-import { WaveFile as WaveFile2 } from "wavefile";
-import { DeferredTask, Trigger } from "@dxos/async";
-import { LifecycleState, Resource } from "@dxos/context";
-import { log as log3 } from "@dxos/log";
-import { trace as trace2 } from "@dxos/tracing";
-function _ts_decorate2(decorators, target, key, desc) {
-  var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
-  if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
-  else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
-  return c > 3 && r && Object.defineProperty(target, key, r), r;
-}
-var __dxlog_file3 = "/home/runner/work/dxos/dxos/packages/plugins/experimental/plugin-transcription/src/transcriber/transcriber.ts";
-var Transcriber = class extends Resource {
-  constructor({ config, recorder, onSegments }) {
-    super();
-    this._audioChunks = [];
-    this._lastUsedTimestamp = 0;
-    this._openTrigger = new Trigger({
-      autoReset: true
-    });
-    this._recording = false;
-    this._transcribeTask = void 0;
-    this._config = config;
-    this._recorder = recorder;
-    this._onSegments = onSegments;
-  }
-  async _open(ctx) {
-    this._recorder.setOnChunk((chunk) => this._saveAudioChunk(chunk));
-    await this._recorder.start();
-    this._transcribeTask = new DeferredTask(ctx, async () => this._transcribe());
-    this._openTrigger.wake();
-  }
-  async _close() {
-    this._recording = false;
-    this._transcribeTask = void 0;
-    await this._recorder.stop();
-  }
-  startChunksRecording() {
-    if (this._lifecycleState !== LifecycleState.OPEN) {
-      return;
-    }
-    this._recording = true;
-  }
-  stopChunksRecording() {
-    if (this._lifecycleState !== LifecycleState.OPEN || !this._recording) {
-      return;
-    }
-    this._recording = false;
-    this._transcribeTask?.schedule();
-  }
-  _saveAudioChunk(chunk) {
-    log3("saving audio chunk", {
-      chunk
-    }, {
-      F: __dxlog_file3,
-      L: 124,
-      S: this,
-      C: (f, a) => f(...a)
-    });
-    this._audioChunks.push(chunk);
-    if (!this._recording && (!this._transcribeTask || !this._transcribeTask.scheduled) && this._audioChunks.length >= this._config.prefixBufferChunksAmount) {
-      this._audioChunks = this._dropOldChunks();
-    } else if (this._audioChunks.length >= this._config.transcribeAfterChunksAmount && this._recording) {
-      this._transcribeTask?.schedule();
-    }
-  }
-  async _transcribe() {
-    log3("transcribing", {
-      chunks: this._audioChunks.length
-    }, {
-      F: __dxlog_file3,
-      L: 140,
-      S: this,
-      C: (f, a) => f(...a)
-    });
-    const chunks = this._audioChunks;
-    this._audioChunks = this._dropOldChunks();
-    if (chunks.length === 0) {
-      return;
-    }
-    const audio = await this._mergeAudioChunks(chunks);
-    const segments = await this._fetchTranscription(audio);
-    if (!Array.isArray(segments) || segments.length === 0) {
-      return;
-    }
-    const alignedSegments = this._alignSegments(segments, chunks);
-    if (alignedSegments.length > 0) {
-      await this._onSegments(alignedSegments);
-    }
-  }
-  _dropOldChunks() {
-    return this._config.prefixBufferChunksAmount > 0 ? this._audioChunks.slice(-this._config.prefixBufferChunksAmount) : [];
-  }
-  async _mergeAudioChunks(chunks) {
-    const file = new WaveFile2();
-    const wavConfig = this._recorder.wavConfig;
-    file.fromScratch(wavConfig.channels, wavConfig.sampleRate, wavConfig.bitDepthCode, mergeFloat64Arrays(chunks.map(({ data }) => data)));
-    return file.toBase64();
-  }
-  async _fetchTranscription(audio) {
-    if (audio.length === 0) {
-      this._audioChunks = [];
-      throw new Error("No audio to send for transcribing");
-    }
-    const response = await fetch(`${TRANSCRIPTION_URL}/transcribe`, {
-      method: "POST",
-      body: JSON.stringify({
-        audio
-      }),
-      headers: {
-        "Content-Type": "application/json"
-      }
-    });
-    if (!response.ok) {
-      this._audioChunks = [];
-      throw new Error("Failed to transcribe");
-    }
-    const { segments } = await response.json();
-    log3.info("transcription response", {
-      segments,
-      string: segments.map((segments2) => segments2.text).join(" ")
-    }, {
-      F: __dxlog_file3,
-      L: 206,
-      S: this,
-      C: (f, a) => f(...a)
-    });
-    return segments;
-  }
-  _alignSegments(segments, originalChunks) {
-    const zeroTimestamp = originalChunks.at(0).timestamp;
-    const filteredSegments = segments.filter((segment) => zeroTimestamp + segment.end * 1e3 >= this._lastUsedTimestamp).map((segment) => {
-      const words = segment.words.filter((word) => zeroTimestamp + word.start * 1e3 >= this._lastUsedTimestamp);
-      return {
-        ...segment,
-        words,
-        text: words.map((word) => word.word).join(""),
-        start: words.at(0)?.start ?? segment.end
-      };
-    }).filter((segment) => segment.words.length > 0);
-    if (filteredSegments.length === 0) {
-      return [];
-    }
-    this._lastUsedTimestamp = zeroTimestamp + filteredSegments.at(-1).end * 1e3;
-    return filteredSegments.map((segment) => ({
-      started: new Date(zeroTimestamp + segment.start * 1e3),
-      text: segment.text
-    }));
-  }
-};
-_ts_decorate2([
-  trace2.span({
-    showInBrowserTimeline: true
-  })
-], Transcriber.prototype, "_mergeAudioChunks", null);
-_ts_decorate2([
-  trace2.span({
-    showInBrowserTimeline: true
-  })
-], Transcriber.prototype, "_fetchTranscription", null);
-
-// packages/plugins/experimental/plugin-transcription/src/hooks/useTranscriber.ts
 var RECORD_INTERVAL = 200;
 var PREFIXED_CHUNKS_AMOUNT = 10;
 var TRANSCRIBE_AFTER_CHUNKS_AMOUNT = 50;
@@ -440,9 +163,9 @@ var useTranscriber = ({ audioStreamTrack, onSegments }) => {
 import { useState as useState3, useEffect as useEffect4, useCallback } from "react";
 import { scheduleMicroTask } from "@dxos/async";
 import { Context } from "@dxos/context";
-import { log as log4 } from "@dxos/log";
+import { log as log2 } from "@dxos/log";
 import { useSoundEffect } from "@dxos/react-ui-sfx";
-var __dxlog_file4 = "/home/runner/work/dxos/dxos/packages/plugins/experimental/plugin-transcription/src/hooks/useVoiceInput.ts";
+var __dxlog_file2 = "/home/runner/work/dxos/dxos/packages/plugins/experimental/plugin-transcription/src/hooks/useVoiceInput.ts";
 var useVoiceInput = ({ active, onUpdate }) => {
   const soundStart = useSoundEffect("StartRecording");
   const soundStop = useSoundEffect("StopRecording");
@@ -458,14 +181,14 @@ var useVoiceInput = ({ active, onUpdate }) => {
   });
   useEffect4(() => {
     const ctx = new Context(void 0, {
-      F: __dxlog_file4,
+      F: __dxlog_file2,
       L: 43
     });
     scheduleMicroTask(ctx, async () => {
       if (active && transcriber) {
         await transcriber.open();
-        log4.info("starting...", void 0, {
-          F: __dxlog_file4,
+        log2.info("starting...", void 0, {
+          F: __dxlog_file2,
           L: 47,
           S: void 0,
           C: (f, a) => f(...a)
@@ -476,8 +199,8 @@ var useVoiceInput = ({ active, onUpdate }) => {
       } else if (!active && transcriber?.isOpen) {
         transcriber?.stopChunksRecording();
         await transcriber?.close();
-        log4.info("stopped", void 0, {
-          F: __dxlog_file4,
+        log2.info("stopped", void 0, {
+          F: __dxlog_file2,
           L: 54,
           S: void 0,
           C: (f, a) => f(...a)
@@ -499,27 +222,28 @@ var useVoiceInput = ({ active, onUpdate }) => {
 };
 
 // packages/plugins/experimental/plugin-transcription/src/transcription-manager.ts
-import { synchronized as synchronized2 } from "@dxos/async";
-import { Resource as Resource2 } from "@dxos/context";
+import { synchronized } from "@dxos/async";
+import { Resource } from "@dxos/context";
 import { QueueImpl } from "@dxos/echo-db";
 import { createStatic } from "@dxos/echo-schema";
-import { invariant as invariant2 } from "@dxos/invariant";
+import { invariant } from "@dxos/invariant";
 import { DXN } from "@dxos/keys";
-function _ts_decorate3(decorators, target, key, desc) {
+function _ts_decorate(decorators, target, key, desc) {
   var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
   if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
   else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
   return c > 3 && r && Object.defineProperty(target, key, r), r;
 }
-var __dxlog_file5 = "/home/runner/work/dxos/dxos/packages/plugins/experimental/plugin-transcription/src/transcription-manager.ts";
+var __dxlog_file3 = "/home/runner/work/dxos/dxos/packages/plugins/experimental/plugin-transcription/src/transcription-manager.ts";
 var RECORD_INTERVAL2 = 200;
 var PREFIXED_CHUNKS_AMOUNT2 = 10;
 var TRANSCRIBE_AFTER_CHUNKS_AMOUNT2 = 50;
-var TranscriptionManager = class extends Resource2 {
+var TranscriptionManager = class extends Resource {
   constructor(edgeClient) {
     super();
     this._audioStreamTrack = void 0;
     this._name = void 0;
+    this._hue = void 0;
     this._mediaRecorder = void 0;
     this._transcriber = void 0;
     this._enabled = false;
@@ -576,9 +300,9 @@ var TranscriptionManager = class extends Resource2 {
         if (this._queue === queue) {
           return;
         }
-        invariant2(queue instanceof QueueImpl, void 0, {
-          F: __dxlog_file5,
-          L: 110,
+        invariant(queue instanceof QueueImpl, void 0, {
+          F: __dxlog_file3,
+          L: 112,
           S: this,
           A: [
             "queue instanceof QueueImpl",
@@ -598,6 +322,12 @@ var TranscriptionManager = class extends Resource2 {
     }
     this._name = name;
   }
+  setHue(hue) {
+    if (this._hue === hue) {
+      return;
+    }
+    this._hue = hue;
+  }
   async _toggleTranscriber() {
     await this._maybeReinitTranscriber();
     if (this._enabled) {
@@ -635,7 +365,8 @@ var TranscriptionManager = class extends Resource2 {
       return;
     }
     const block = createStatic(TranscriptBlock, {
-      author: this._name,
+      authorName: this._name,
+      authorHue: this._hue,
       segments
     });
     this._queue.append([
@@ -643,21 +374,24 @@ var TranscriptionManager = class extends Resource2 {
     ]);
   }
 };
-_ts_decorate3([
-  synchronized2
+_ts_decorate([
+  synchronized
 ], TranscriptionManager.prototype, "setRecording", null);
-_ts_decorate3([
-  synchronized2
+_ts_decorate([
+  synchronized
 ], TranscriptionManager.prototype, "setEnabled", null);
-_ts_decorate3([
-  synchronized2
+_ts_decorate([
+  synchronized
 ], TranscriptionManager.prototype, "setAudioTrack", null);
-_ts_decorate3([
-  synchronized2
+_ts_decorate([
+  synchronized
 ], TranscriptionManager.prototype, "setQueue", null);
-_ts_decorate3([
-  synchronized2
+_ts_decorate([
+  synchronized
 ], TranscriptionManager.prototype, "setName", null);
+_ts_decorate([
+  synchronized
+], TranscriptionManager.prototype, "setHue", null);
 
 // packages/plugins/experimental/plugin-transcription/src/TranscriptionPlugin.tsx
 import { Capabilities, Events, contributes, defineModule, definePlugin } from "@dxos/app-framework";
@@ -668,7 +402,8 @@ var translations_default = [
   {
     "en-US": {
       [TranscriptType.typename]: {
-        "typename label": "Transcript"
+        "typename label": "Transcript",
+        "object name placeholder": "New transcript"
       },
       [TRANSCRIPTION_PLUGIN]: {
         "plugin name": "Transcription",
@@ -678,8 +413,7 @@ var translations_default = [
         "delete calls label": "Delete",
         "delete button": "Delete",
         "bookmark button": "Bookmark",
-        "summary button": "Create summary",
-        "summarizing label": "Summarizing..."
+        "scroll to end label": "Scroll to latest"
       }
     }
   }
@@ -703,12 +437,6 @@ var TranscriptionPlugin = () => definePlugin(meta, [
     activate: () => contributes(Capabilities.Metadata, {
       id: TranscriptType.typename,
       metadata: {
-        placeholder: [
-          "transcript title placeholder",
-          {
-            ns: TRANSCRIPTION_PLUGIN
-          }
-        ],
         icon: "ph--subtitles--regular"
       }
     })
@@ -752,6 +480,7 @@ export {
   TranscriptionManager,
   TranscriptionPlugin,
   meta,
+  summarizeTranscript,
   useAudioFile,
   useAudioTrack,
   useTranscriber,