@dxos/plugin-transcription 0.8.4-main.ae835ea → 0.8.4-main.bc2380dfbc

package/src/{components/stories → stories}/KeyWordRecognition.stories.tsx

@@ -6,24 +6,23 @@ import { type Meta, type StoryObj } from '@storybook/react-vite';
 import React, { useEffect, useMemo, useState } from 'react';
 
 import { log } from '@dxos/log';
-import { Toolbar } from '@dxos/react-ui';
+import { Panel, Toolbar } from '@dxos/react-ui';
 import { withLayout, withTheme } from '@dxos/react-ui/testing';
-import { mx } from '@dxos/react-ui-theme';
+import { mx } from '@dxos/ui-theme';
 
 type Word = {
   text: string;
   matched: boolean;
 };
 
-type StoryProps = {
+type DefaultStoryProps = {
   keywords: string[];
 };
 
-const DefaultStory = ({ keywords }: StoryProps) => {
+const DefaultStory = ({ keywords }: DefaultStoryProps) => {
   const [running, setRunning] = useState(false);
   const [matchingWords, setMatchingWords] = useState<Word[]>(keywords.map((word) => ({ text: word, matched: false })));
 
-  const [transcript, setTranscript] = useState('');
   const recognition = useMemo(() => {
     // TODO(mykola): Fix types
     const SpeechRecognition = (window as any).SpeechRecognition || (window as any).webkitSpeechRecognition;
@@ -38,16 +37,16 @@ const DefaultStory = ({ keywords }: StoryProps) => {
     recognition.interimResults = true;
     recognition.lang = 'en-US';
     recognition.maxAlternatives = 1;
-    // Add this line to enable punctuation
+    // Add this line to enable punctuation.
     recognition.grammars = new SpeechGrammarList();
 
-    // TODO(mykola): Fix types
+    // TODO(mykola): Fix types.
     recognition.onresult = (event: any) => {
       log.info('recognition result', { event });
       const current = event.resultIndex;
       const transcript = event.results[current][0].transcript;
 
-      // Remove punctuation and normalize whitespace for comparison
+      // Remove punctuation and normalize whitespace for comparison.
       const normalizeText = (text: string) =>
         text
           .toLowerCase()
@@ -57,7 +56,7 @@ const DefaultStory = ({ keywords }: StoryProps) => {
 
       const normalizedTranscript = normalizeText(transcript);
 
-      // Check each keyword and update matching words
+      // Check each keyword and update matching words.
       const updatedWords = matchingWords.map((word) => {
         const normalizedKeyword = normalizeText(word.text);
         const matched = normalizedTranscript.includes(normalizedKeyword);
@@ -65,10 +64,9 @@ const DefaultStory = ({ keywords }: StoryProps) => {
       });
 
       setMatchingWords(updatedWords);
-      setTranscript(transcript);
     };
 
-    // TODO(mykola): Fix types
+    // TODO(mykola): Fix types.
     recognition.onerror = (event: any) => {
       log.error('Speech recognition error:', { error: event.error });
       setRunning(false);
@@ -86,39 +84,41 @@ const DefaultStory = ({ keywords }: StoryProps) => {
   }, [running, recognition]);
 
   return (
-    <div className={mx('flex flex-col gap-2 w-[30rem]')}>
-      <Toolbar.Root>
-        <Toolbar.IconButton
-          iconOnly
-          icon={running ? 'ph--pause--regular' : 'ph--play--regular'}
-          label={running ? 'Pause' : 'Play'}
-          onClick={() => setRunning((running) => !running)}
-        />
-      </Toolbar.Root>
-
-      <div className='flex flex-wrap'>
-        {matchingWords.map((word, index) => (
-          <span
-            key={index}
-            className={mx(
-              'px-3 py-1 m-2 rounded border',
-              word.matched ? 'bg-red-200 text-red-800 border-red-300' : 'bg-gray-200 text-gray-600 border-gray-300',
-            )}
-          >
-            {word.text}
-          </span>
-        ))}
-      </div>
-
-      <span>{transcript || (running ? 'Listening...' : 'Click play to start speech recognition')}</span>
-    </div>
+    <Panel.Root>
+      <Panel.Toolbar asChild>
+        <Toolbar.Root>
+          <Toolbar.IconButton
+            iconOnly
+            icon={running ? 'ph--pause--regular' : 'ph--play--regular'}
+            label={running ? 'Pause' : 'Play'}
+            onClick={() => setRunning((running) => !running)}
+          />
+        </Toolbar.Root>
+      </Panel.Toolbar>
+
+      <Panel.Content>
+        <div className='p-2 grid grid-cols-[repeat(auto-fill,minmax(6rem,1fr))] gap-2'>
+          {matchingWords.map((word, index) => (
+            <span
+              key={index}
+              className={mx(
+                'grid place-items-center p-2 aspect-square rounded-sm border border-separator',
+                word.matched && 'border-red-300',
+              )}
+            >
+              {word.text}
+            </span>
+          ))}
+        </div>
+      </Panel.Content>
+    </Panel.Root>
   );
 };
 
 const meta = {
-  title: 'plugins/plugin-transcription/KeyWordDetection',
+  title: 'plugins/plugin-transcription/stories/KeyWordDetection',
   render: DefaultStory,
-  decorators: [withTheme, withLayout({ container: 'column' })],
+  decorators: [withTheme(), withLayout({ layout: 'column' })],
 } satisfies Meta<typeof DefaultStory>;
 
 export default meta;
@@ -127,6 +127,6 @@ type Story = StoryObj<typeof meta>;
 
 export const SpeechRecognitionAPI: Story = {
   args: {
-    keywords: ['kai', 'computer', 'hello'],
+    keywords: ['kai', 'computer', 'hello', 'dxos'],
   },
 };

package/src/stories/LiveTranscription.stories.tsx

@@ -0,0 +1,153 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import { type Meta, type StoryObj } from '@storybook/react-vite';
+import React, { useCallback, useMemo, useState } from 'react';
+
+import {
+  type ExtractionFunction,
+  extractionAnthropicFunction,
+  extractionNerFunction,
+  getNer,
+  processTranscriptMessage,
+} from '@dxos/assistant/extraction';
+import { Filter, type Obj } from '@dxos/echo';
+import { invariant } from '@dxos/invariant';
+import { useSpaces } from '@dxos/react-client/echo';
+import { Message, Organization, Person } from '@dxos/types';
+
+import { useAudioTrack } from '#hooks';
+
+import { type MediaStreamRecorderProps, type TranscriberProps } from '../transcriber';
+import { useIsSpeaking, createStoryDecorators, useStoryMessageModel, useStoryTranscriber } from './common';
+import { TranscriptionStory } from './TranscriptionStory';
+
+const DEFAULT_TRANSCRIBER_CONFIG = {
+  transcribeAfterChunksAmount: 50,
+  prefixBufferChunksAmount: 10,
+};
+
+const DEFAULT_RECORDER_CONFIG = {
+  interval: 200,
+};
+
+type DefaultStoryProps = {
+  detectSpeaking?: boolean;
+  transcriberConfig?: TranscriberProps['config'];
+  recorderConfig?: MediaStreamRecorderProps['config'];
+  audioConstraints?: MediaTrackConstraints;
+  entityExtraction?: 'ner' | 'llm';
+};
+
+const DefaultStory = ({
+  detectSpeaking,
+  entityExtraction,
+  transcriberConfig = DEFAULT_TRANSCRIBER_CONFIG,
+  recorderConfig = DEFAULT_RECORDER_CONFIG,
+  audioConstraints,
+}: DefaultStoryProps) => {
+  const [running, setRunning] = useState(false);
+  const track = useAudioTrack(running, audioConstraints);
+  const isSpeaking = detectSpeaking ? useIsSpeaking(track) : true;
+  const { model, appendMessage } = useStoryMessageModel();
+  const [space] = useSpaces();
+
+  // Resolve the entity-extraction function + nearby objects once per `entityExtraction` toggle.
+  const { extractionFunction, objects } = useMemo(() => {
+    if (!space || !entityExtraction) {
+      return {};
+    }
+
+    let extractionFunction: ExtractionFunction | undefined;
+    let objects: Promise<Obj.Unknown[]> | undefined;
+    if (entityExtraction === 'ner') {
+      void getNer(); // Init model loading. Takes time.
+      extractionFunction = extractionNerFunction;
+    } else if (entityExtraction === 'llm') {
+      extractionFunction = extractionAnthropicFunction;
+      objects = space.db.query(Filter.or(Filter.type(Person.Person), Filter.type(Organization.Organization))).run();
+    }
+    return { extractionFunction, objects };
+  }, [entityExtraction, space]);
+
+  // Build a Message from each transcribed batch; optionally enrich via entity extraction.
+  const handleSegments = useCallback<TranscriberProps['onSegments']>(
+    async (blocks) => {
+      const message = Message.make({
+        sender: { name: 'You' },
+        created: new Date().toISOString(),
+        blocks,
+      });
+
+      if (entityExtraction && space) {
+        invariant(extractionFunction, 'extractionFunction is required');
+        const result = await processTranscriptMessage({
+          function: extractionFunction,
+          input: { message, objects: await objects },
+          options: { fallbackToRaw: true, timeout: 30_000 },
+        });
+        appendMessage(result.message);
+      } else {
+        appendMessage(message);
+      }
+    },
+    [appendMessage, space, entityExtraction, extractionFunction, objects],
+  );
+
+  useStoryTranscriber({
+    audioStreamTrack: track,
+    running,
+    isSpeaking,
+    transcriberConfig,
+    recorderConfig,
+    onSegments: handleSegments,
+  });
+
+  return <TranscriptionStory model={model} running={running} onRunningChange={setRunning} />;
+};
+
+const meta = {
+  title: 'plugins/plugin-transcription/stories/LiveTranscription',
+  render: DefaultStory,
+  decorators: createStoryDecorators({ enableVectorIndex: true }),
+} satisfies Meta<typeof DefaultStory>;
+
+export default meta;
+
+type Story = StoryObj<typeof meta>;
+
+export const Default: Story = {
+  args: {
+    detectSpeaking: false,
+    audioConstraints: {
+      echoCancellation: false,
+      noiseSuppression: false,
+      autoGainControl: false,
+    },
+  },
+};
+
+export const SpeechDetection: Story = {
+  args: {
+    detectSpeaking: true,
+    audioConstraints: {
+      echoCancellation: true,
+      noiseSuppression: true,
+      autoGainControl: true,
+    },
+  },
+};
+
+// TODO(mykola): Fix hugging face quota issues.
+// export const EntityExtraction: Story = {
+//   args: {
+//     detectSpeaking: true,
+//     entityExtraction: 'ner',
+//     audioConstraints: {
+//       echoCancellation: true,
+//       noiseSuppression: true,
+//       autoGainControl: true,
+//     },
+//   },
+// };

package/src/stories/TranscriptionStory.tsx

@@ -0,0 +1,101 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import React, {
+  type ChangeEvent,
+  type Dispatch,
+  type RefObject,
+  type SetStateAction,
+  useCallback,
+  useRef,
+} from 'react';
+
+import { IconButton, ScrollContainer, Toolbar } from '@dxos/react-ui';
+import { type Message } from '@dxos/types';
+
+import { Transcription } from '../components';
+import { type SerializationModel } from '../model';
+
+export type TranscriptionStoryProps = {
+  audioRef?: RefObject<HTMLAudioElement | null>;
+  model: SerializationModel<Message.Message>;
+  running: boolean;
+  disabled?: boolean;
+  uploadAccept?: string;
+  onRunningChange: Dispatch<SetStateAction<boolean>>;
+  onUpload?: (file: File) => void;
+};
+
+/**
+ * Story wrapper that renders a transcript with playback controls, an optional audio file
+ * upload affordance, and a toolbar slot for additional controls.
+ *
+ * @param onUpload - Callback fired when the user picks a local audio file via the upload button.
+ * @param uploadAccept - HTML `accept` filter for the hidden file input. Defaults to `audio/*`.
+ */
+export const TranscriptionStory = ({
+  audioRef,
+  model,
+  running,
+  disabled,
+  uploadAccept = 'audio/*',
+  onRunningChange,
+  onUpload,
+}: TranscriptionStoryProps) => {
+  const fileInputRef = useRef<HTMLInputElement>(null);
+
+  const handleFileChange = useCallback(
+    (event: ChangeEvent<HTMLInputElement>) => {
+      const file = event.target.files?.[0];
+      if (file && onUpload) {
+        onUpload(file);
+      }
+
+      // Reset so the same file can be re-selected.
+      event.target.value = '';
+    },
+    [onUpload],
+  );
+
+  return (
+    <>
+      {audioRef && <audio ref={audioRef} autoPlay />}
+      <Toolbar.Root>
+        <IconButton
+          iconOnly
+          disabled={disabled}
+          icon={running ? 'ph--pause--regular' : 'ph--play--regular'}
+          label={running ? 'Pause' : 'Play'}
+          onClick={() => onRunningChange((running) => !running)}
+        />
+        {onUpload && (
+          <>
+            <input
+              ref={fileInputRef}
+              type='file'
+              accept={uploadAccept}
+              disabled={disabled}
+              className='hidden'
+              onChange={handleFileChange}
+            />
+            <IconButton
+              iconOnly
+              disabled={disabled}
+              icon='ph--upload--regular'
+              label='Upload audio'
+              onClick={() => fileInputRef.current?.click()}
+            />
+          </>
+        )}
+      </Toolbar.Root>
+      <ScrollContainer.Root pin>
+        <ScrollContainer.Content>
+          <ScrollContainer.Viewport>
+            <Transcription model={model} />
+          </ScrollContainer.Viewport>
+        </ScrollContainer.Content>
+      </ScrollContainer.Root>
+    </>
+  );
+};

package/src/stories/common.ts

@@ -0,0 +1,233 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+// Helpers shared between LiveTranscription / FileTranscription stories.
+// Each "live" story differs only in its *audio source* (mic vs file); everything else —
+// the local in-memory message buffer, the transcriber lifecycle, and the plugin-manager
+// decorator stack — is identical and lives here.
+
+import { type Decorator } from '@storybook/react-vite';
+import * as Effect from 'effect/Effect';
+import { useCallback, useEffect, useMemo, useRef, useState } from 'react';
+
+import { withPluginManager } from '@dxos/app-framework/testing';
+import { AppActivationEvents } from '@dxos/app-toolkit';
+import { scheduleTask } from '@dxos/async';
+import { Context } from '@dxos/context';
+import { Obj } from '@dxos/echo';
+import { SpeakingMonitor } from '@dxos/plugin-calls';
+import { ClientPlugin, initializeIdentity } from '@dxos/plugin-client/testing';
+import { PreviewPlugin } from '@dxos/plugin-preview/testing';
+import { StorybookPlugin, corePlugins } from '@dxos/plugin-testing';
+import { IndexKind } from '@dxos/react-client/echo';
+import { withLayout } from '@dxos/react-ui/testing';
+import { Message, Organization, Person } from '@dxos/types';
+import { seedTestData } from '@dxos/types/testing';
+
+import { useFeedModelAdapter, useTranscriber } from '#hooks';
+import { TestItem } from '#testing';
+import { type TranscriptionCapabilities } from '#types';
+
+import { type Transcriber, type TranscriberProps } from '../transcriber';
+import { TranscriptionPlugin } from '../TranscriptionPlugin';
+import { renderByline } from '../util';
+
+export const useIsSpeaking = (track?: MediaStreamTrack) => {
+  const [isSpeaking, setIsSpeaking] = useState(false);
+  const speakingMonitor = useMemo(() => {
+    if (!track) {
+      return;
+    }
+
+    return new SpeakingMonitor(track);
+  }, [track]);
+
+  useEffect(() => {
+    if (!speakingMonitor) {
+      return;
+    }
+
+    const ctx = new Context();
+    scheduleTask(ctx, async () => {
+      speakingMonitor.speakingChanged.on(ctx, () => setIsSpeaking(speakingMonitor.isSpeaking));
+      await speakingMonitor.open();
+      ctx.onDispose(() => speakingMonitor.close());
+    });
+
+    return () => {
+      void ctx.dispose();
+    };
+  }, [speakingMonitor]);
+
+  return isSpeaking;
+};
+
+//
+// Message model.
+//
+
+/**
+ * In-memory message buffer + model adapter for storybook use (production wires up a real
+ * space-backed `Feed`). Returns the renderable `model` plus an `appendMessage` callback.
+ */
+export const useStoryMessageModel = () => {
+  const [messages, setMessages] = useState<Message.Message[]>([]);
+  const appendMessage = useCallback((message: Message.Message) => setMessages((prev) => [...prev, message]), []);
+  const model = useFeedModelAdapter(renderByline([]), messages);
+  return { model, appendMessage };
+};
+
+/**
+ * Wraps blocks in a `Message.Message` with the given sender and appends it to the local buffer.
+ * Provided as a convenience for stories that don't need entity-extraction or custom enrichment.
+ */
+export const useStoryAppendSegments = (
+  appendMessage: (message: Message.Message) => void,
+  sender: Message.Message['sender'] = {},
+): TranscriberProps['onSegments'] =>
+  useCallback<TranscriberProps['onSegments']>(
+    async (blocks) => {
+      appendMessage(
+        Obj.make(Message.Message, {
+          created: new Date().toISOString(),
+          sender,
+          blocks,
+        }),
+      );
+    },
+    [appendMessage, sender],
+  );
+
+//
+// Transcriber lifecycle.
+//
+
+export type UseStoryTranscriberOptions = Pick<
+  TranscriptionCapabilities.GetTranscriberProps,
+  'transcriberConfig' | 'recorderConfig' | 'onSegments'
+> & {
+  audioStreamTrack?: MediaStreamTrack;
+  /** Whether the user has the toolbar toggle on. */
+  running: boolean;
+  /** Voice-activity flag — chunks recording only fires while `true`. */
+  isSpeaking: boolean;
+};
+
+/**
+ * Drives the transcriber's open/close + chunk-recording lifecycle off `running`,
+ * the open flag, and an `isSpeaking` voice-activity gate. Returns the transcriber instance
+ * (`undefined` until the capability resolves and an audio track is available).
+ */
+export const useStoryTranscriber = ({
+  audioStreamTrack,
+  running,
+  isSpeaking,
+  transcriberConfig,
+  recorderConfig,
+  onSegments,
+}: UseStoryTranscriberOptions): Transcriber | undefined => {
+  const transcriber = useTranscriber({
+    audioStreamTrack,
+    onSegments,
+    transcriberConfig,
+    recorderConfig,
+  });
+
+  // Open the transcriber when it becomes available; close on teardown.
+  // `tokenRef` is a per-effect identity so a late `close()` completion from a stale
+  // transcriber instance can't clobber `isOpen` after a newer instance has opened.
+  const [isOpen, setIsOpen] = useState(false);
+  const tokenRef = useRef<symbol | null>(null);
+  useEffect(() => {
+    if (!transcriber) {
+      return;
+    }
+    const token = Symbol();
+    tokenRef.current = token;
+    void transcriber.open().then(
+      () => tokenRef.current === token && setIsOpen(true),
+      () => tokenRef.current === token && setIsOpen(false),
+    );
+    return () => {
+      void transcriber.close().then(() => {
+        if (tokenRef.current === token) {
+          setIsOpen(false);
+        }
+      });
+    };
+  }, [transcriber]);
+
+  // Start/stop chunk recording based on running + open + isSpeaking.
+  useEffect(() => {
+    if (!transcriber || !isOpen) {
+      return;
+    }
+    if (running && isSpeaking) {
+      transcriber.startChunksRecording();
+    } else {
+      transcriber.stopChunksRecording();
+    }
+  }, [transcriber, isOpen, running, isSpeaking]);
+
+  return transcriber;
+};
+
+//
+// Plugin manager decorator.
+//
+
+export type StoryDecoratorsOptions = {
+  /**
+   * Enable schema/graph/vector indexes after the client boots; needed for entity-extraction
+   * queries over seeded `Person` / `Organization` objects.
+   * @default false
+   */
+  enableVectorIndex?: boolean;
+};
+
+/**
+ * Standard plugin-manager decorator for transcription stories: core plugins,
+ * `ClientPlugin` with `Person`/`Organization`/`TestItem` schemas + seeded test data, plus
+ * the `PreviewPlugin` and `TranscriptionPlugin`. Toggle `enableVectorIndex` for entity-extraction.
+ */
+export const createStoryDecorators = ({ enableVectorIndex = false }: StoryDecoratorsOptions = {}): Decorator[] => [
+  withLayout({ layout: 'column' }),
+  withPluginManager({
+    plugins: [
+      ...corePlugins(),
+      StorybookPlugin({}),
+      ClientPlugin({
+        types: [TestItem, Person.Person, Organization.Organization],
+        onClientInitialized: ({ client }) =>
+          Effect.gen(function* () {
+            const { personalSpace } = yield* initializeIdentity(client);
+            if (enableVectorIndex) {
+              yield* enableQueryIndexes(client.services.services);
+            }
+            yield* Effect.promise(() => seedTestData(personalSpace));
+          }),
+      }),
+      PreviewPlugin(),
+      TranscriptionPlugin(),
+    ],
+    fireEvents: [AppActivationEvents.SetupAppGraph],
+  }),
+];
+
+// TODO(mykola): Make API easier to use.
+// TODO(mykola): Delete after enabling vector indexing by default.
+const enableQueryIndexes = (services: { QueryService?: any }) =>
+  Effect.gen(function* () {
+    yield* Effect.promise(() =>
+      services.QueryService!.setConfig({
+        enabled: true,
+        indexes: [
+          { kind: IndexKind.Kind.SCHEMA_MATCH },
+          { kind: IndexKind.Kind.GRAPH },
+          { kind: IndexKind.Kind.VECTOR },
+        ],
+      }),
+    );
+    yield* Effect.promise(() => services.QueryService!.reindex());
+  });