@dxos/plugin-transcription 0.8.3 → 0.8.4-main.1c7ec43d41

package/dist/lib/node-esm/index.mjs

@@ -1,122 +1,470 @@
 import { createRequire } from 'node:module';const require = createRequire(import.meta.url);
 import {
-  TranscriptContainer
-} from "./chunk-MFJO7EKD.mjs";
-import {
-  IntentResolver,
-  ReactSurface,
-  Transcriber as Transcriber2,
-  Transcript,
-  TranscriptionContainer,
-  renderMarkdown,
   useAudioFile,
   useAudioTrack,
   useQueueModelAdapter,
   useTranscriber,
   useVoiceInput
-} from "./chunk-56QHOX4A.mjs";
-import {
-  MediaStreamRecorder,
-  Transcriber,
-  TranscriptionManager
-} from "./chunk-PRL64SRU.mjs";
-import {
-  TranscriptType
-} from "./chunk-FH75X4JC.mjs";
+} from "./chunk-MQK4R6GT.mjs";
+import "./chunk-C3AE2NYJ.mjs";
 import {
   TranscriptionCapabilities
-} from "./chunk-MFJPEIVB.mjs";
+} from "./chunk-KO3IOD2O.mjs";
 import {
-  TRANSCRIPTION_PLUGIN,
-  meta
-} from "./chunk-TXMDLLBR.mjs";
+  mergeFloat64Arrays
+} from "./chunk-O533NBMK.mjs";
+import "./chunk-HSLMI22Q.mjs";
 
-// packages/plugins/plugin-transcription/src/TranscriptionPlugin.tsx
-import { Capabilities, Events, contributes, defineModule, definePlugin } from "@dxos/app-framework";
-import { Obj } from "@dxos/echo";
-import { ClientCapabilities, ClientEvents } from "@dxos/plugin-client";
-import { getSpace } from "@dxos/react-client/echo";
-import { DataType } from "@dxos/schema";
+// src/index.ts
+import { Plugin } from "@dxos/app-framework";
 
-// packages/plugins/plugin-transcription/src/translations.ts
-var translations_default = [
-  {
-    "en-US": {
-      [TRANSCRIPTION_PLUGIN]: {
-        "plugin name": "Transcription",
-        "transcript companion label": "Transcript",
-        "delete button": "Delete",
-        "bookmark button": "Bookmark",
-        "scroll to end label": "Scroll to latest",
-        "settings entity extraction label": "Entity extraction"
-      }
+// src/meta.ts
+import { trim } from "@dxos/util";
+var meta = {
+  id: "org.dxos.plugin.transcription",
+  name: "Transcription",
+  description: trim`
+    Real-time voice-to-text transcription service for capturing spoken content.
+    Convert audio input into searchable text that integrates seamlessly with notes and documents.
+  `,
+  icon: "ph--microphone--regular",
+  iconHue: "sky",
+  source: "https://github.com/dxos/dxos/tree/main/packages/plugins/plugin-transcription",
+  tags: [
+    "labs"
+  ],
+  screenshots: []
+};
+
+// src/transcriber/media-stream-recorder.ts
+import { MediaRecorder, register } from "extendable-media-recorder";
+import { WaveFile } from "wavefile";
+import { synchronized } from "@dxos/async";
+import { invariant } from "@dxos/invariant";
+import { log } from "@dxos/log";
+import { trace } from "@dxos/tracing";
+var __dxlog_file = "/__w/dxos/dxos/packages/plugins/plugin-transcription/src/transcriber/media-stream-recorder.ts";
+function _ts_decorate(decorators, target, key, desc) {
+  var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+  if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+  else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+  return c > 3 && r && Object.defineProperty(target, key, r), r;
+}
+var initializingPromise;
+var initializeExtendableMediaRecorder = async () => {
+  const { connect } = await import("extendable-media-recorder-wav-encoder");
+  await register(await connect());
+};
+var MediaStreamRecorder = class {
+  _mediaStreamTrack;
+  /**
+  * Default MediaRecorder implementation do not support wav encoding.
+  */
+  _config;
+  _mediaRecorder = void 0;
+  _header = void 0;
+  _onChunk;
+  constructor({ mediaStreamTrack, config }) {
+    this._mediaStreamTrack = mediaStreamTrack;
+    this._config = config;
+  }
+  get wavConfig() {
+    const settings = this._mediaStreamTrack.getSettings();
+    return {
+      channels: settings.channelCount ?? 1,
+      sampleRate: settings.sampleRate ?? 16e3,
+      bitDepthCode: settings.sampleSize ? String(settings.sampleSize) : "16"
+    };
+  }
+  get mediaStreamTrack() {
+    return this._mediaStreamTrack;
+  }
+  setOnChunk(onChunk) {
+    this._onChunk = onChunk;
+  }
+  async start() {
+    await (initializingPromise ??= initializeExtendableMediaRecorder()).catch((err) => log.info("initializeExtendableMediaRecorder", {
+      err
+    }, { "~LogMeta": "~LogMeta", F: __dxlog_file, L: 48, S: this }));
+    if (!this._mediaRecorder) {
+      const stream = new MediaStream([
+        this._mediaStreamTrack
+      ]);
+      this._mediaRecorder = new MediaRecorder(stream, {
+        mimeType: "audio/wav"
+      });
+    }
+    if (this._mediaRecorder.state === "recording") {
+      return;
+    }
+    invariant(this._onChunk, "MediaStreamRecorder: onChunk is not set", { "~LogMeta": "~LogMeta", F: __dxlog_file, L: 62, S: this, A: ["this._onChunk", "'MediaStreamRecorder: onChunk is not set'"] });
+    this._mediaRecorder.ondataavailable = (event) => this._ondataavailable(event);
+    this._mediaRecorder.start(this._config.interval);
+  }
+  async stop() {
+    if (this._mediaRecorder?.state !== "recording") {
+      return;
+    }
+    this._mediaRecorder.stop();
+  }
+  async _ondataavailable(event) {
+    const blob = event.data;
+    const uint8Array = new Uint8Array(await blob.arrayBuffer());
+    const isHeader = uint8Array[0] === 82 && // R
+    uint8Array[1] === 73 && // I
+    uint8Array[2] === 70 && // F
+    uint8Array[3] === 70;
+    let wav;
+    if (isHeader) {
+      wav = new WaveFile(uint8Array);
+      this._header = uint8Array.slice(0, 44);
+    } else if (this._header) {
+      wav = new WaveFile(new Uint8Array([
+        ...this._header,
+        ...uint8Array
+      ]));
+    } else {
+      log.warn("MediaStreamRecorder: no header", void 0, { "~LogMeta": "~LogMeta", F: __dxlog_file, L: 90, S: this });
+      return;
     }
+    this._onChunk({
+      timestamp: Date.now(),
+      data: wav.getSamples()
+    });
   }
-];
+};
+_ts_decorate([
+  synchronized
+], MediaStreamRecorder.prototype, "_ondataavailable", null);
+MediaStreamRecorder = _ts_decorate([
+  trace.resource()
+], MediaStreamRecorder);
 
-// packages/plugins/plugin-transcription/src/TranscriptionPlugin.tsx
-var TranscriptionPlugin = () => definePlugin(meta, [
-  defineModule({
-    id: `${meta.id}/module/translations`,
-    activatesOn: Events.SetupTranslations,
-    activate: () => contributes(Capabilities.Translations, translations_default)
-  }),
-  defineModule({
-    id: `${meta.id}/module/metadata`,
-    activatesOn: Events.SetupMetadata,
-    activate: (context) => contributes(Capabilities.Metadata, {
-      id: TranscriptType.typename,
-      metadata: {
-        icon: "ph--subtitles--regular",
-        // TODO(wittjosiah): Factor out. Artifact? Separate capability?
-        getTextContent: async (transcript) => {
-          const space = getSpace(transcript);
-          const members = space?.members.get().map((member) => member.identity) ?? [];
-          const queue = space?.queues.get(transcript.queue.dxn);
-          await queue?.refresh();
-          const content = queue?.objects.filter((message) => Obj.instanceOf(DataType.Message, message)).flatMap((message, index) => renderMarkdown(members)(message, index)).join("\n\n");
-          return content;
+// src/transcriber/transcriber.ts
+import { WaveFile as WaveFile2 } from "wavefile";
+import { DeferredTask, Trigger } from "@dxos/async";
+import { LifecycleState, Resource } from "@dxos/context";
+import { log as log2 } from "@dxos/log";
+import { trace as trace2 } from "@dxos/tracing";
+import { TRANSCRIPTION_URL } from "#types";
+var __dxlog_file2 = "/__w/dxos/dxos/packages/plugins/plugin-transcription/src/transcriber/transcriber.ts";
+function _ts_decorate2(decorators, target, key, desc) {
+  var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+  if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+  else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+  return c > 3 && r && Object.defineProperty(target, key, r), r;
+}
+var Transcriber = class extends Resource {
+  _audioChunks = [];
+  _lastUsedTimestamp = 0;
+  _openTrigger = new Trigger({
+    autoReset: true
+  });
+  _config;
+  _recorder;
+  _onSegments;
+  _transcribeFn;
+  _recording = false;
+  _transcribeTask = void 0;
+  constructor({ config, recorder, onSegments, transcribe }) {
+    super();
+    this._config = config;
+    this._recorder = recorder;
+    this._onSegments = onSegments;
+    this._transcribeFn = transcribe;
+  }
+  async _open(ctx) {
+    log2.info("opening", void 0, { "~LogMeta": "~LogMeta", F: __dxlog_file2, L: 41, S: this });
+    this._recorder.setOnChunk((chunk) => this._saveAudioChunk(chunk));
+    await this._recorder.start();
+    this._transcribeTask = new DeferredTask(ctx, async () => this._transcribe());
+    this._openTrigger.wake();
+  }
+  async _close() {
+    log2.info("closing", void 0, { "~LogMeta": "~LogMeta", F: __dxlog_file2, L: 48, S: this });
+    this._recording = false;
+    this._transcribeTask = void 0;
+    await this._recorder.stop();
+  }
+  startChunksRecording() {
+    log2.info("starting", void 0, { "~LogMeta": "~LogMeta", F: __dxlog_file2, L: 54, S: this });
+    if (this._lifecycleState !== LifecycleState.OPEN) {
+      return;
+    }
+    this._recording = true;
+  }
+  stopChunksRecording() {
+    if (this._lifecycleState !== LifecycleState.OPEN || !this._recording) {
+      return;
+    }
+    this._recording = false;
+    this._transcribeTask?.schedule();
+    log2.info("stopped", void 0, { "~LogMeta": "~LogMeta", F: __dxlog_file2, L: 66, S: this });
+  }
+  _saveAudioChunk(chunk) {
+    log2("saving audio chunk", {
+      chunk
+    }, { "~LogMeta": "~LogMeta", F: __dxlog_file2, L: 69, S: this });
+    this._audioChunks.push(chunk);
+    if (!this._recording && (!this._transcribeTask || !this._transcribeTask.scheduled) && this._audioChunks.length >= this._config.prefixBufferChunksAmount) {
+      this._audioChunks = this._dropOldChunks();
+    } else if (this._audioChunks.length >= this._config.transcribeAfterChunksAmount && this._recording) {
+      this._transcribeTask?.schedule();
+    }
+  }
+  async _transcribe() {
+    log2("transcribing", {
+      chunks: this._audioChunks.length
+    }, { "~LogMeta": "~LogMeta", F: __dxlog_file2, L: 81, S: this });
+    const chunks = this._audioChunks;
+    this._audioChunks = this._dropOldChunks();
+    if (chunks.length === 0) {
+      return;
+    }
+    const audio = await this._mergeAudioChunks(chunks);
+    const segments = await this._fetchTranscription(audio);
+    if (!Array.isArray(segments) || segments.length === 0) {
+      return;
+    }
+    const alignedSegments = this._alignSegments(segments, chunks);
+    if (alignedSegments.length > 0) {
+      await this._onSegments(alignedSegments);
+    }
+  }
+  _dropOldChunks() {
+    return this._config.prefixBufferChunksAmount > 0 ? this._audioChunks.slice(-this._config.prefixBufferChunksAmount) : [];
+  }
+  async _mergeAudioChunks(chunks) {
+    const file = new WaveFile2();
+    const wavConfig = this._recorder.wavConfig;
+    file.fromScratch(wavConfig.channels, wavConfig.sampleRate, wavConfig.bitDepthCode, mergeFloat64Arrays(chunks.map(({ data }) => data)));
+    return file.toBase64();
+  }
+  async _fetchTranscription(audio) {
+    if (audio.length === 0) {
+      throw new Error("No audio to send for transcribing");
+    }
+    let segments;
+    if (this._transcribeFn) {
+      segments = await this._transcribeFn(audio);
+    } else {
+      const endpoint = this._config.endpoint ?? TRANSCRIPTION_URL;
+      const response = await fetch(`${endpoint}/transcribe`, {
+        method: "POST",
+        body: JSON.stringify({
+          audio
+        }),
+        headers: {
+          "Content-Type": "application/json"
         }
+      });
+      if (!response.ok) {
+        throw new Error(`Transcription failed: ${response.statusText}`);
       }
-    })
-  }),
-  defineModule({
-    id: `${meta.id}/module/schema`,
-    activatesOn: ClientEvents.SetupSchema,
-    activate: () => [
-      contributes(ClientCapabilities.Schema, [
-        TranscriptType
-      ])
-    ]
-  }),
-  defineModule({
-    id: `${meta.id}/module/react-surface`,
-    activatesOn: Events.SetupReactSurface,
-    activate: ReactSurface
-  }),
-  defineModule({
-    id: `${meta.id}/module/intent-resolver`,
-    activatesOn: Events.SetupIntentResolver,
-    activate: IntentResolver
-  }),
-  defineModule({
-    id: `${meta.id}/module/transcription`,
-    activatesOn: Events.SetupAppGraph,
-    activate: Transcriber2
+      ({ segments } = await response.json());
+    }
+    if (!Array.isArray(segments)) {
+      throw new Error("Transcription response payload is invalid");
+    }
+    log2.info("transcription response", {
+      segments: segments.length
+    }, { "~LogMeta": "~LogMeta", F: __dxlog_file2, L: 134, S: this });
+    return segments;
+  }
+  _alignSegments(segments, originalChunks) {
+    const result = alignWhisperSegments(segments, originalChunks, this._lastUsedTimestamp);
+    if (result.lastUsedTimestamp !== void 0) {
+      this._lastUsedTimestamp = result.lastUsedTimestamp;
+    }
+    return result.transcripts;
+  }
+};
+_ts_decorate2([
+  trace2.span({
+    showInBrowserTimeline: true
+  })
+], Transcriber.prototype, "_mergeAudioChunks", null);
+_ts_decorate2([
+  trace2.span({
+    showInBrowserTimeline: true
   })
-]);
+], Transcriber.prototype, "_fetchTranscription", null);
+var alignWhisperSegments = (segments, originalChunks, lastUsedTimestamp) => {
+  if (originalChunks.length === 0) {
+    return {
+      transcripts: []
+    };
+  }
+  const zeroTimestamp = originalChunks[0].timestamp;
+  const filteredSegments = segments.filter((segment) => zeroTimestamp + segment.end * 1e3 >= lastUsedTimestamp).map((segment) => {
+    const words = segment.words.filter((word) => zeroTimestamp + word.start * 1e3 >= lastUsedTimestamp);
+    return {
+      ...segment,
+      words,
+      text: words.map((word) => word.word).join(""),
+      start: words.at(0)?.start ?? segment.end
+    };
+  }).filter((segment) => segment.words.length > 0);
+  if (filteredSegments.length === 0) {
+    return {
+      transcripts: []
+    };
+  }
+  return {
+    transcripts: filteredSegments.map((segment) => ({
+      _tag: "transcript",
+      started: new Date(zeroTimestamp + segment.start * 1e3).toISOString(),
+      text: segment.text.trim()
+    })),
+    lastUsedTimestamp: zeroTimestamp + filteredSegments.at(-1).end * 1e3
+  };
+};
+
+// src/transcriber/transcription-manager.ts
+import { Atom } from "@effect-atom/atom-react";
+import { synchronized as synchronized2 } from "@dxos/async";
+import { Resource as Resource2 } from "@dxos/context";
+import { Obj } from "@dxos/echo";
+import { Message } from "@dxos/types";
+function _ts_decorate3(decorators, target, key, desc) {
+  var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+  if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+  else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+  return c > 3 && r && Object.defineProperty(target, key, r), r;
+}
+var RECORD_INTERVAL = 200;
+var PREFIXED_CHUNKS_AMOUNT = 10;
+var TRANSCRIBE_AFTER_CHUNKS_AMOUNT = 50;
+var TranscriptionManager = class extends Resource2 {
+  _edgeClient;
+  _messageEnricher;
+  _registry;
+  _audioStreamTrack = void 0;
+  _identityDid = void 0;
+  _mediaRecorder = void 0;
+  _transcriber = void 0;
+  _queue = void 0;
+  _enabledAtom = Atom.make(false);
+  constructor(options) {
+    super();
+    this._edgeClient = options.edgeClient;
+    this._messageEnricher = options.messageEnricher;
+    this._registry = options.registry;
+  }
+  get enabled() {
+    return this._enabledAtom;
+  }
+  getEnabled() {
+    return this._registry.get(this._enabledAtom);
+  }
+  setQueue(queue) {
+    this._queue = queue;
+    return this;
+  }
+  setIdentityDid(did) {
+    if (this._identityDid !== did) {
+      this._identityDid = did;
+    }
+    return this;
+  }
+  setRecording(recording) {
+    if (!this.isOpen || !this.getEnabled()) {
+      return this;
+    }
+    if (recording) {
+      this._transcriber?.startChunksRecording();
+    } else {
+      this._transcriber?.stopChunksRecording();
+    }
+    return this;
+  }
+  async setEnabled(enabled) {
+    if (this.getEnabled() === enabled) {
+      return;
+    }
+    this._registry.set(this._enabledAtom, enabled ?? false);
+    if (enabled) {
+      await this._maybeRestartTranscriber();
+    } else {
+      await this._stopTranscriber();
+    }
+  }
+  async setAudioTrack(track) {
+    if (this._audioStreamTrack === track) {
+      return;
+    }
+    this._audioStreamTrack = track;
+    await this._maybeRestartTranscriber();
+  }
+  async _open() {
+    await this._maybeRestartTranscriber();
+  }
+  async _close() {
+    await this._stopTranscriber();
+  }
+  async _maybeRestartTranscriber() {
+    if (!this._audioStreamTrack || !this.getEnabled() || !this.isOpen) {
+      return;
+    }
+    let needReinit = false;
+    if (this._audioStreamTrack !== this._mediaRecorder?.mediaStreamTrack) {
+      this._mediaRecorder = new MediaStreamRecorder({
+        mediaStreamTrack: this._audioStreamTrack,
+        config: {
+          interval: RECORD_INTERVAL
+        }
+      });
+      needReinit = true;
+    }
+    if (needReinit) {
+      await this._transcriber?.close();
+      this._transcriber = new Transcriber({
+        config: {
+          transcribeAfterChunksAmount: TRANSCRIBE_AFTER_CHUNKS_AMOUNT,
+          prefixBufferChunksAmount: PREFIXED_CHUNKS_AMOUNT
+        },
+        recorder: this._mediaRecorder,
+        onSegments: (segments) => this._onSegments(segments)
+      });
+      await this._transcriber?.open();
+    }
+  }
+  async _stopTranscriber() {
+    await this._transcriber?.close();
+  }
+  async _onSegments(segments) {
+    if (!this.isOpen || !this._queue) {
+      return;
+    }
+    let block = Obj.make(Message.Message, {
+      created: (/* @__PURE__ */ new Date()).toISOString(),
+      blocks: segments,
+      sender: {
+        identityDid: this._identityDid
+      }
+    });
+    if (this._messageEnricher) {
+      block = await this._messageEnricher(block);
+    }
+    await this._queue.append([
+      block
+    ]);
+  }
+};
+_ts_decorate3([
+  synchronized2
+], TranscriptionManager.prototype, "setAudioTrack", null);
+
+// src/index.ts
+var TranscriptionPlugin = Plugin.lazy(meta, () => import("#plugin"));
 export {
   MediaStreamRecorder,
-  TRANSCRIPTION_PLUGIN,
   Transcriber,
-  Transcript,
-  TranscriptContainer,
   TranscriptionCapabilities,
-  TranscriptionContainer,
   TranscriptionManager,
   TranscriptionPlugin,
+  alignWhisperSegments,
   meta,
-  renderMarkdown,
   useAudioFile,
   useAudioTrack,
   useQueueModelAdapter,