@dxos/plugin-transcription 0.9.1-main.c7dcc2e112 → 0.10.0

package/src/stories/TranscriptionStory.tsx

@@ -1,101 +0,0 @@
-//
-// Copyright 2025 DXOS.org
-//
-
-import React, {
-  type ChangeEvent,
-  type Dispatch,
-  type RefObject,
-  type SetStateAction,
-  useCallback,
-  useRef,
-} from 'react';
-
-import { IconButton, ScrollContainer, Toolbar } from '@dxos/react-ui';
-import { type Message } from '@dxos/types';
-
-import { Transcription } from '../components';
-import { type SerializationModel } from '../model';
-
-export type TranscriptionStoryProps = {
-  audioRef?: RefObject<HTMLAudioElement | null>;
-  model: SerializationModel<Message.Message>;
-  running: boolean;
-  disabled?: boolean;
-  uploadAccept?: string;
-  onRunningChange: Dispatch<SetStateAction<boolean>>;
-  onUpload?: (file: File) => void;
-};
-
-/**
- * Story wrapper that renders a transcript with playback controls, an optional audio file
- * upload affordance, and a toolbar slot for additional controls.
- *
- * @param onUpload - Callback fired when the user picks a local audio file via the upload button.
- * @param uploadAccept - HTML `accept` filter for the hidden file input. Defaults to `audio/*`.
- */
-export const TranscriptionStory = ({
-  audioRef,
-  model,
-  running,
-  disabled,
-  uploadAccept = 'audio/*',
-  onRunningChange,
-  onUpload,
-}: TranscriptionStoryProps) => {
-  const fileInputRef = useRef<HTMLInputElement>(null);
-
-  const handleFileChange = useCallback(
-    (event: ChangeEvent<HTMLInputElement>) => {
-      const file = event.target.files?.[0];
-      if (file && onUpload) {
-        onUpload(file);
-      }
-
-      // Reset so the same file can be re-selected.
-      event.target.value = '';
-    },
-    [onUpload],
-  );
-
-  return (
-    <>
-      {audioRef && <audio ref={audioRef} autoPlay />}
-      <Toolbar.Root>
-        <IconButton
-          iconOnly
-          disabled={disabled}
-          icon={running ? 'ph--pause--regular' : 'ph--play--regular'}
-          label={running ? 'Pause' : 'Play'}
-          onClick={() => onRunningChange((running) => !running)}
-        />
-        {onUpload && (
-          <>
-            <input
-              ref={fileInputRef}
-              type='file'
-              accept={uploadAccept}
-              disabled={disabled}
-              className='hidden'
-              onChange={handleFileChange}
-            />
-            <IconButton
-              iconOnly
-              disabled={disabled}
-              icon='ph--upload--regular'
-              label='Upload audio'
-              onClick={() => fileInputRef.current?.click()}
-            />
-          </>
-        )}
-      </Toolbar.Root>
-      <ScrollContainer.Root pin>
-        <ScrollContainer.Content>
-          <ScrollContainer.Viewport>
-            <Transcription model={model} />
-          </ScrollContainer.Viewport>
-        </ScrollContainer.Content>
-      </ScrollContainer.Root>
-    </>
-  );
-};

package/src/stories/common.ts

@@ -1,234 +0,0 @@
-//
-// Copyright 2024 DXOS.org
-//
-
-// Helpers shared between LiveTranscription / FileTranscription stories.
-// Each "live" story differs only in its *audio source* (mic vs file); everything else —
-// the local in-memory message buffer, the transcriber lifecycle, and the plugin-manager
-// decorator stack — is identical and lives here.
-
-import { type Decorator } from '@storybook/react-vite';
-import * as Effect from 'effect/Effect';
-import { useCallback, useEffect, useMemo, useRef, useState } from 'react';
-
-import { withPluginManager } from '@dxos/app-framework/testing';
-import { AppActivationEvents } from '@dxos/app-toolkit';
-import { scheduleTask } from '@dxos/async';
-import { SpeakingMonitor } from '@dxos/av';
-import { Context } from '@dxos/context';
-import { Obj } from '@dxos/echo';
-import { ClientPlugin, initializeIdentity } from '@dxos/plugin-client/testing';
-import { PreviewPlugin } from '@dxos/plugin-preview/testing';
-import { StorybookPlugin, corePlugins } from '@dxos/plugin-testing';
-import { IndexKind } from '@dxos/react-client/echo';
-import { withLayout } from '@dxos/react-ui/testing';
-import { Message, Organization, Person } from '@dxos/types';
-import { seedTestData } from '@dxos/types/testing';
-
-import { useFeedModelAdapter, useTranscriber } from '#hooks';
-import { TestItem } from '#testing';
-import { type TranscriptionCapabilities } from '#types';
-
-import { type Transcriber, type TranscriberProps } from '../transcriber';
-import { TranscriptionPlugin } from '../TranscriptionPlugin';
-import { renderByline } from '../util';
-
-export const useIsSpeaking = (track?: MediaStreamTrack) => {
-  const [isSpeaking, setIsSpeaking] = useState(false);
-  const speakingMonitor = useMemo(() => {
-    if (!track) {
-      return;
-    }
-
-    return new SpeakingMonitor(track);
-  }, [track]);
-
-  useEffect(() => {
-    if (!speakingMonitor) {
-      return;
-    }
-
-    const ctx = new Context();
-    scheduleTask(ctx, async () => {
-      speakingMonitor.speakingChanged.on(ctx, () => setIsSpeaking(speakingMonitor.isSpeaking));
-      await speakingMonitor.open();
-      ctx.onDispose(() => speakingMonitor.close());
-    });
-
-    return () => {
-      void ctx.dispose();
-    };
-  }, [speakingMonitor]);
-
-  return isSpeaking;
-};
-
-//
-// Message model.
-//
-
-/**
- * In-memory message buffer + model adapter for storybook use (production wires up a real
- * space-backed `Feed`). Returns the renderable `model` plus an `appendMessage` callback.
- */
-export const useStoryMessageModel = () => {
-  const [messages, setMessages] = useState<Message.Message[]>([]);
-  const appendMessage = useCallback((message: Message.Message) => setMessages((prev) => [...prev, message]), []);
-  const model = useFeedModelAdapter(renderByline([]), messages);
-  return { model, appendMessage };
-};
-
-/**
- * Wraps blocks in a `Message.Message` with the given sender and appends it to the local buffer.
- * Provided as a convenience for stories that don't need entity-extraction or custom enrichment.
- */
-export const useStoryAppendSegments = (
-  appendMessage: (message: Message.Message) => void,
-  sender: Message.Message['sender'] = {},
-): TranscriberProps['onSegments'] =>
-  useCallback<TranscriberProps['onSegments']>(
-    async (blocks) => {
-      appendMessage(
-        Obj.make(Message.Message, {
-          created: new Date().toISOString(),
-          sender,
-          blocks,
-        }),
-      );
-    },
-    [appendMessage, sender],
-  );
-
-//
-// Transcriber lifecycle.
-//
-
-export type UseStoryTranscriberOptions = Pick<
-  TranscriptionCapabilities.TranscriberProviderProps,
-  'transcriberConfig' | 'recorderConfig' | 'onSegments'
-> & {
-  audioStreamTrack?: MediaStreamTrack;
-  /** Whether the user has the toolbar toggle on. */
-  running: boolean;
-  /** Voice-activity flag — chunks recording only fires while `true`. */
-  isSpeaking: boolean;
-};
-
-/**
- * Drives the transcriber's open/close + chunk-recording lifecycle off `running`,
- * the open flag, and an `isSpeaking` voice-activity gate. Returns the transcriber instance
- * (`undefined` until the capability resolves and an audio track is available).
- */
-export const useStoryTranscriber = ({
-  audioStreamTrack,
-  running,
-  isSpeaking,
-  transcriberConfig,
-  recorderConfig,
-  onSegments,
-}: UseStoryTranscriberOptions): Transcriber | undefined => {
-  const transcriber = useTranscriber({
-    audioStreamTrack,
-    onSegments,
-    transcriberConfig,
-    recorderConfig,
-  });
-
-  // Open the transcriber when it becomes available; close on teardown.
-  // `tokenRef` is a per-effect identity so a late `close()` completion from a stale
-  // transcriber instance can't clobber `isOpen` after a newer instance has opened.
-  const [isOpen, setIsOpen] = useState(false);
-  const tokenRef = useRef<symbol | null>(null);
-  useEffect(() => {
-    if (!transcriber) {
-      return;
-    }
-    const token = Symbol();
-    tokenRef.current = token;
-    void transcriber.open().then(
-      () => tokenRef.current === token && setIsOpen(true),
-      () => tokenRef.current === token && setIsOpen(false),
-    );
-    return () => {
-      void transcriber.close().then(() => {
-        if (tokenRef.current === token) {
-          setIsOpen(false);
-        }
-      });
-    };
-  }, [transcriber]);
-
-  // Start/stop chunk recording based on running + open + isSpeaking.
-  useEffect(() => {
-    if (!transcriber || !isOpen) {
-      return;
-    }
-    if (running && isSpeaking) {
-      transcriber.startChunksRecording();
-    } else {
-      transcriber.stopChunksRecording();
-    }
-  }, [transcriber, isOpen, running, isSpeaking]);
-
-  return transcriber;
-};
-
-//
-// Plugin manager decorator.
-//
-
-export type StoryDecoratorsOptions = {
-  /**
-   * Enable schema/graph/vector indexes after the client boots; needed for entity-extraction
-   * queries over seeded `Person` / `Organization` objects.
-   * @default false
-   */
-  enableVectorIndex?: boolean;
-};
-
-/**
- * Standard plugin-manager decorator for transcription stories: core plugins,
- * `ClientPlugin` with `Person`/`Organization`/`TestItem` schemas + seeded test data, plus
- * the `PreviewPlugin` and `TranscriptionPlugin`. Toggle `enableVectorIndex` for entity-extraction.
- */
-export const createStoryDecorators = ({ enableVectorIndex = false }: StoryDecoratorsOptions = {}): Decorator[] => [
-  withLayout({ layout: 'column' }),
-  withPluginManager({
-    plugins: [
-      ...corePlugins(),
-      StorybookPlugin({}),
-      ClientPlugin({
-        types: [TestItem, Person.Person, Organization.Organization],
-        onClientInitialized: ({ client }) =>
-          Effect.gen(function* () {
-            const { personalSpace } = yield* initializeIdentity(client);
-            if (enableVectorIndex) {
-              yield* enableQueryIndexes(client.services.services);
-            }
-            yield* Effect.promise(() => seedTestData(personalSpace));
-          }),
-      }),
-      PreviewPlugin(),
-      TranscriptionPlugin(),
-    ],
-    fireEvents: [AppActivationEvents.SetupAppGraph],
-  }),
-];
-
-// TODO(mykola): Make API easier to use.
-// TODO(mykola): Delete after enabling vector indexing by default.
-const enableQueryIndexes = (services: { QueryService?: any }) =>
-  Effect.gen(function* () {
-    yield* Effect.promise(() =>
-      services.QueryService!.setConfig({
-        enabled: true,
-        indexes: [
-          { kind: IndexKind.Kind.SCHEMA_MATCH },
-          { kind: IndexKind.Kind.GRAPH },
-          { kind: IndexKind.Kind.VECTOR },
-          { kind: IndexKind.Kind.FULL_TEXT },
-        ],
-      }),
-    );
-    yield* Effect.promise(() => services.QueryService!.reindex());
-  });

package/src/transcriber/audio-recorder.ts

@@ -1,56 +0,0 @@
-//
-// Copyright 2025 DXOS.org
-//
-
-export type AudioChunk = {
-  /**
-   * Timestamp of the beginning of the chunk.
-   */
-  timestamp: number;
-
-  /**
-   * 16-bit PCM data.
-   */
-  data: Float64Array;
-};
-
-export type WavConfig = {
-  channels: number;
-  sampleRate: number;
-  bitDepthCode: string;
-};
-
-/**
- * Records MediaStreamTrack to Uint8Array 16-bit PCM data.
- * The data is passed to the onChunk callback with the timestamp of the chunk.
- * All chunks are aligned and sequential.
- * Time ─────────────────────────────────────────────────►
- *
- * Recording:
- * t1    ┌──────┐
- * t2    ┌──────┬──────┐
- * t3    ┌──────┬──────┬──────┐
- * t4    ┌──────┬──────┬──────┬──────┐
- *       └──────┴──────┴──────┴──────┘
- *         1st     2nd     3rd     4th
- *        chunk   chunk   chunk   chunk
- *
- */
-export interface AudioRecorder {
-  wavConfig: WavConfig;
-
-  /**
-   * Sets callback for receiving audio chunks.
-   */
-  setOnChunk(onChunk: (chunk: AudioChunk) => void): void;
-
-  /**
-   * Starts the recorder.
-   */
-  start(): Promise<void>;
-
-  /**
-   * Stops the recorder.
-   */
-  stop(): Promise<void>;
-}

package/src/transcriber/index.ts

@@ -1,8 +0,0 @@
-//
-// Copyright 2025 DXOS.org
-//
-
-export * from './audio-recorder';
-export * from './media-stream-recorder';
-export * from './transcriber';
-export * from './transcription-manager';

package/src/transcriber/media-stream-recorder.ts

@@ -1,120 +0,0 @@
-//
-// Copyright 2025 DXOS.org
-//
-
-import { type IBlobEvent, type IMediaRecorder, MediaRecorder, register } from 'extendable-media-recorder';
-import { WaveFile } from 'wavefile';
-
-import { synchronized } from '@dxos/async';
-import { invariant } from '@dxos/invariant';
-import { log } from '@dxos/log';
-import { trace } from '@dxos/tracing';
-
-import { type AudioChunk, type AudioRecorder, type WavConfig } from './audio-recorder';
-
-let initializingPromise: Promise<void> | undefined;
-
-const initializeExtendableMediaRecorder = async () => {
-  const { connect } = await import('extendable-media-recorder-wav-encoder');
-  await register(await connect());
-};
-
-export type MediaStreamRecorderProps = {
-  mediaStreamTrack: MediaStreamTrack;
-  config: {
-    interval: number;
-  };
-};
-
-/**
- * Recorder that uses the MediaContext API and AudioNode API to record audio.
- *
- * It records MediaStream using https://www.npmjs.com/package/extendable-media-recorder.
- */
-@trace.resource()
-export class MediaStreamRecorder implements AudioRecorder {
-  private readonly _mediaStreamTrack: MediaStreamTrack;
-  /**
-   * Default MediaRecorder implementation do not support wav encoding.
-   */
-  private readonly _config: MediaStreamRecorderProps['config'];
-  private _mediaRecorder?: IMediaRecorder = undefined;
-  private _header?: Uint8Array = undefined;
-  private _onChunk?: (chunk: AudioChunk) => void;
-
-  constructor({ mediaStreamTrack, config }: MediaStreamRecorderProps) {
-    this._mediaStreamTrack = mediaStreamTrack;
-    this._config = config;
-  }
-
-  get wavConfig(): WavConfig {
-    const settings = this._mediaStreamTrack.getSettings();
-    return {
-      channels: settings.channelCount ?? 1,
-      sampleRate: settings.sampleRate ?? 16000,
-      bitDepthCode: settings.sampleSize ? String(settings.sampleSize) : '16',
-    };
-  }
-
-  get mediaStreamTrack() {
-    return this._mediaStreamTrack;
-  }
-
-  setOnChunk(onChunk: (chunk: AudioChunk) => void): void {
-    this._onChunk = onChunk;
-  }
-
-  async start(): Promise<void> {
-    await (initializingPromise ??= initializeExtendableMediaRecorder()).catch((err) =>
-      log.info('initializeExtendableMediaRecorder', { err }),
-    );
-
-    if (!this._mediaRecorder) {
-      const stream = new MediaStream([this._mediaStreamTrack]);
-      this._mediaRecorder = new MediaRecorder(stream, { mimeType: 'audio/wav' });
-    }
-
-    if (this._mediaRecorder.state === 'recording') {
-      return;
-    }
-    invariant(this._onChunk, 'MediaStreamRecorder: onChunk is not set');
-    this._mediaRecorder.ondataavailable = (event) => this._ondataavailable(event);
-    this._mediaRecorder.start(this._config.interval);
-  }
-
-  async stop(): Promise<void> {
-    if (this._mediaRecorder?.state !== 'recording') {
-      return;
-    }
-    this._mediaRecorder.stop();
-  }
-
-  @synchronized
-  private async _ondataavailable(event: IBlobEvent): Promise<void> {
-    const blob = event.data;
-    const uint8Array = new Uint8Array(await blob.arrayBuffer());
-
-    // First chunk from the MediaRecorder has a header.
-    const isHeader =
-      uint8Array[0] === 0x52 && // R
-      uint8Array[1] === 0x49 && // I
-      uint8Array[2] === 0x46 && // F
-      uint8Array[3] === 0x46; // F
-    let wav: WaveFile;
-
-    if (isHeader) {
-      wav = new WaveFile(uint8Array);
-      this._header = uint8Array.slice(0, 44);
-    } else if (this._header) {
-      wav = new WaveFile(new Uint8Array([...this._header, ...uint8Array]));
-    } else {
-      log.warn('MediaStreamRecorder: no header');
-      return;
-    }
-
-    this._onChunk!({
-      timestamp: Date.now(),
-      data: wav.getSamples(),
-    });
-  }
-}