@dxos/plugin-transcription 0.8.3 → 0.8.4-main.1c7ec43d41

package/src/{components/stories → stories}/KeyWordRecognition.stories.tsx

@@ -2,22 +2,24 @@
 // Copyright 2025 DXOS.org
 //
 
-import '@dxos-theme';
-
-import { type StoryObj, type Meta } from '@storybook/react';
+import { type Meta, type StoryObj } from '@storybook/react-vite';
 import React, { useEffect, useMemo, useState } from 'react';
 
 import { log } from '@dxos/log';
-import { Toolbar } from '@dxos/react-ui';
-import { mx } from '@dxos/react-ui-theme';
-import { withLayout, withTheme } from '@dxos/storybook-utils';
+import { Panel, Toolbar } from '@dxos/react-ui';
+import { withLayout, withTheme } from '@dxos/react-ui/testing';
+import { mx } from '@dxos/ui-theme';
 
 type Word = {
   text: string;
   matched: boolean;
 };
 
-const SpeechRecognition = ({ keywords }: { keywords: string[] }) => {
+type DefaultStoryProps = {
+  keywords: string[];
+};
+
+const DefaultStory = ({ keywords }: DefaultStoryProps) => {
   const [running, setRunning] = useState(false);
   const [matchingWords, setMatchingWords] = useState<Word[]>(keywords.map((word) => ({ text: word, matched: false })));
 
@@ -84,45 +86,49 @@ const SpeechRecognition = ({ keywords }: { keywords: string[] }) => {
   }, [running, recognition]);
 
   return (
-    <div className={mx('flex flex-col gap-2 w-[30rem]')}>
-      <Toolbar.Root>
-        <Toolbar.IconButton
-          iconOnly
-          icon={running ? 'ph--pause--regular' : 'ph--play--regular'}
-          label={running ? 'Pause' : 'Play'}
-          onClick={() => setRunning((running) => !running)}
-        />
-      </Toolbar.Root>
-
-      <div className='flex flex-wrap'>
-        {matchingWords.map((word, index) => (
-          <span
-            key={index}
-            className={mx(
-              'px-3 py-1 m-2 rounded border',
-              word.matched ? 'bg-red-200 text-red-800 border-red-300' : 'bg-gray-200 text-gray-600 border-gray-300',
-            )}
-          >
-            {word.text}
-          </span>
-        ))}
-      </div>
-
-      <span>{transcript || (running ? 'Listening...' : 'Click play to start speech recognition')}</span>
-    </div>
+    <Panel.Root>
+      <Panel.Toolbar asChild>
+        <Toolbar.Root>
+          <Toolbar.IconButton
+            iconOnly
+            icon={running ? 'ph--pause--regular' : 'ph--play--regular'}
+            label={running ? 'Pause' : 'Play'}
+            onClick={() => setRunning((running) => !running)}
+          />
+        </Toolbar.Root>
+      </Panel.Toolbar>
+
+      <Panel.Content>
+        <div className='p-4 grid grid-cols-[repeat(auto-fill,minmax(6rem,1fr))] gap-2'>
+          {matchingWords.map((word, index) => (
+            <span
+              key={index}
+              className={mx(
+                'grid place-items-center p-2 aspect-square rounded-sm border border-separator',
+                word.matched && 'border-red-300',
+              )}
+            >
+              {word.text}
+            </span>
+          ))}
+        </div>
+      </Panel.Content>
+    </Panel.Root>
   );
 };
 
-const meta: Meta<typeof SpeechRecognition> = {
-  title: 'plugins/plugin-transcription/KeyWordDetection',
-  decorators: [withTheme, withLayout({ fullscreen: true, classNames: 'justify-center' })],
-};
+const meta = {
+  title: 'plugins/plugin-transcription/components/KeyWordDetection',
+  render: DefaultStory,
+  decorators: [withTheme(), withLayout({ layout: 'column' })],
+} satisfies Meta<typeof DefaultStory>;
 
 export default meta;
 
-export const SpeechRecognitionAPI: StoryObj<typeof SpeechRecognition> = {
-  render: SpeechRecognition,
+type Story = StoryObj<typeof meta>;
+
+export const SpeechRecognitionAPI: Story = {
   args: {
-    keywords: ['kai', 'computer', 'hello'],
+    keywords: ['kai', 'computer', 'hello', 'dxos'],
   },
 };

package/src/{components/stories → stories}/MicrophoneTranscription.stories.tsx

@@ -2,46 +2,42 @@
 // Copyright 2025 DXOS.org
 //
 
-import '@dxos-theme';
-
-import { type Meta, type StoryObj } from '@storybook/react';
+import { type Meta, type StoryObj } from '@storybook/react-vite';
+import * as Effect from 'effect/Effect';
 import React, { useCallback, useEffect, useMemo, useState } from 'react';
 
-import { EdgeAiServiceClient } from '@dxos/ai';
-import { AI_SERVICE_ENDPOINT } from '@dxos/ai/testing';
-import { Events, IntentPlugin, SettingsPlugin } from '@dxos/app-framework';
 import { withPluginManager } from '@dxos/app-framework/testing';
+import { AppActivationEvents } from '@dxos/app-toolkit/events';
 import {
-  extractionAnthropicFn,
   type ExtractionFunction,
-  extractionNerFn,
-  processTranscriptMessage,
+  extractionAnthropicFunction,
+  extractionNerFunction,
   getNer,
-} from '@dxos/assistant';
-import { Filter, Obj, type Type } from '@dxos/echo';
+  processTranscriptMessage,
+} from '@dxos/assistant/extraction';
+import { Filter, type Obj } from '@dxos/echo';
 import { MemoryQueue } from '@dxos/echo-db';
-import { createQueueDXN } from '@dxos/echo-schema';
-import { FunctionExecutor, ServiceContainer } from '@dxos/functions';
+import { createQueueDXN } from '@dxos/echo/internal';
 import { invariant } from '@dxos/invariant';
 import { log } from '@dxos/log';
 import { ClientPlugin } from '@dxos/plugin-client';
+import { initializeIdentity } from '@dxos/plugin-client/testing';
 import { PreviewPlugin } from '@dxos/plugin-preview';
-import { SpacePlugin } from '@dxos/plugin-space';
-import { StorybookLayoutPlugin } from '@dxos/plugin-storybook-layout';
-import { ThemePlugin } from '@dxos/plugin-theme';
-import { IndexKind, useSpace } from '@dxos/react-client/echo';
-import { defaultTx } from '@dxos/react-ui-theme';
-import { DataType } from '@dxos/schema';
-import { seedTestData, Testing } from '@dxos/schema/testing';
-import { withLayout, withTheme } from '@dxos/storybook-utils';
+import { StorybookPlugin, corePlugins } from '@dxos/plugin-testing';
+import { IndexKind, useSpaces } from '@dxos/react-client/echo';
+import { withLayout } from '@dxos/react-ui/testing';
+import { TestSchema } from '@dxos/schema/testing';
+import { Message, Organization, Person } from '@dxos/types';
+import { seedTestData } from '@dxos/types/testing';
 
+import { useAudioTrack, useQueueModelAdapter, useTranscriber } from '#hooks';
+import { TestItem } from '#testing';
+
+import { type MediaStreamRecorderProps, type TranscriberProps } from '../transcriber';
+import { TranscriptionPlugin } from '../TranscriptionPlugin';
+import { renderByline } from '../util';
 import { TranscriptionStory } from './TranscriptionStory';
 import { useIsSpeaking } from './useIsSpeaking';
-import { TranscriptionPlugin } from '../../TranscriptionPlugin';
-import { useAudioTrack, useQueueModelAdapter, useTranscriber } from '../../hooks';
-import { TestItem } from '../../testing';
-import { type MediaStreamRecorderParams, type TranscriberParams } from '../../transcriber';
-import { renderMarkdown } from '../Transcript';
 
 const TRANSCRIBER_CONFIG = {
   transcribeAfterChunksAmount: 50,
@@ -52,10 +48,10 @@ const RECORDER_CONFIG = {
   interval: 200,
 };
 
-type StoryProps = {
+type DefaultStoryProps = {
   detectSpeaking?: boolean;
-  transcriberConfig: TranscriberParams['config'];
-  recorderConfig: MediaStreamRecorderParams['config'];
+  transcriberConfig: TranscriberProps['config'];
+  recorderConfig: MediaStreamRecorderProps['config'];
   audioConstraints?: MediaTrackConstraints;
   entityExtraction: 'none' | 'ner' | 'llm';
 };
@@ -66,7 +62,7 @@ const DefaultStory = ({
   transcriberConfig,
   recorderConfig,
   audioConstraints,
-}: StoryProps) => {
+}: DefaultStoryProps) => {
   const [running, setRunning] = useState(false);
 
   // Audio.
@@ -78,9 +74,9 @@ const DefaultStory = ({
   // Queue.
   // TODO(dmaretskyi): Use space.queues.create() instead.
   const queueDxn = useMemo(() => createQueueDXN(), []);
-  const queue = useMemo(() => new MemoryQueue<DataType.Message>(queueDxn), [queueDxn]);
-  const model = useQueueModelAdapter(renderMarkdown([]), queue);
-  const space = useSpace();
+  const queue = useMemo(() => new MemoryQueue<Message.Message>(queueDxn), [queueDxn]);
+  const model = useQueueModelAdapter(renderByline([]), queue);
+  const [space] = useSpaces();
 
   useEffect(() => {
     if (!space) {
@@ -89,49 +85,39 @@ const DefaultStory = ({
   }, [space]);
 
   // Entity extraction.
-  const { extractionFunction, executor, objects } = useMemo(() => {
+  const { extractionFunction, objects } = useMemo(() => {
     if (!space) {
       log.warn('no space');
       return {};
     }
 
-    let executor: FunctionExecutor | undefined;
     let extractionFunction: ExtractionFunction | undefined;
-    let objects: Promise<Type.Expando[]> | undefined;
+    let objects: Promise<Obj.Unknown[]> | undefined;
 
     if (entityExtraction === 'ner') {
       // Init model loading. Takes time.
       void getNer();
-      extractionFunction = extractionNerFn;
+      extractionFunction = extractionNerFunction;
     } else if (entityExtraction === 'llm') {
-      extractionFunction = extractionAnthropicFn;
+      extractionFunction = extractionAnthropicFunction;
       objects = space.db
         .query(
           Filter.or(
-            Filter.type(DataType.Person),
-            Filter.type(DataType.Organization),
-            Filter.type(Testing.DocumentType),
+            Filter.type(Person.Person),
+            Filter.type(Organization.Organization),
+            Filter.type(TestSchema.DocumentType),
           ),
         )
-        .run()
-        .then((result) => result.objects);
-    }
-    if (entityExtraction !== 'none') {
-      const AiService = new EdgeAiServiceClient({
-        endpoint: AI_SERVICE_ENDPOINT.REMOTE,
-      });
-      executor = new FunctionExecutor(
-        new ServiceContainer().setServices({ ai: { client: AiService }, database: { db: space!.db } }),
-      );
+        .run();
     }
 
-    return { extractionFunction, executor, objects };
+    return { extractionFunction, objects };
   }, [entityExtraction, space]);
 
   // Transcriber.
-  const handleSegments = useCallback<TranscriberParams['onSegments']>(
+  const handleSegments = useCallback<TranscriberProps['onSegments']>(
     async (blocks) => {
-      const message = Obj.make(DataType.Message, {
+      const message = Message.make({
         sender: { name: 'You' },
         created: new Date().toISOString(),
         blocks,
@@ -143,9 +129,7 @@ const DefaultStory = ({
 
       if (entityExtraction !== 'none') {
         invariant(extractionFunction, 'extractionFunction is required');
-        invariant(executor, 'executor is required');
         const result = await processTranscriptMessage({
-          executor,
           function: extractionFunction,
           input: {
             message,
@@ -190,52 +174,50 @@ const DefaultStory = ({
   return <TranscriptionStory model={model} running={running} onRunningChange={setRunning} />;
 };
 
-const meta: Meta<typeof DefaultStory> = {
-  title: 'plugins/plugin-transcription/MicrophoneTranscription',
+const meta = {
+  title: 'plugins/plugin-transcription/components/MicrophoneTranscription',
   render: DefaultStory,
   decorators: [
+    withLayout({ layout: 'column' }),
     withPluginManager({
       plugins: [
-        ThemePlugin({ tx: defaultTx }),
-        StorybookLayoutPlugin(),
+        ...corePlugins(),
+        StorybookPlugin({}),
         ClientPlugin({
-          types: [TestItem, DataType.Person, DataType.Organization, Testing.DocumentType],
-          onClientInitialized: async (_, client) => {
-            await client.halo.createIdentity();
-            await client.spaces.waitUntilReady();
-            await client.spaces.default.waitUntilReady();
-            // TODO(mykola): Make API easier to use.
-            // TODO(mykola): Delete after enabling vector indexing by default.
-            // Enable vector indexing.
-            await client.services.services.QueryService!.setConfig({
-              enabled: true,
-              indexes: [
-                //
-                { kind: IndexKind.Kind.SCHEMA_MATCH },
-                { kind: IndexKind.Kind.GRAPH },
-                { kind: IndexKind.Kind.VECTOR },
-              ],
-            });
-            await client.services.services.QueryService!.reindex();
-            await seedTestData(client.spaces.default);
-          },
+          types: [TestItem, Person.Person, Organization.Organization, TestSchema.DocumentType],
+          onClientInitialized: ({ client }) =>
+            Effect.gen(function* () {
+              const { personalSpace } = yield* initializeIdentity(client);
+              // TODO(mykola): Make API easier to use.
+              // TODO(mykola): Delete after enabling vector indexing by default.
+              // Enable vector indexing.
+              yield* Effect.promise(() =>
+                client.services.services.QueryService!.setConfig({
+                  enabled: true,
+                  indexes: [
+                    //
+                    { kind: IndexKind.Kind.SCHEMA_MATCH },
+                    { kind: IndexKind.Kind.GRAPH },
+                    { kind: IndexKind.Kind.VECTOR },
+                  ],
+                }),
+              );
+              yield* Effect.promise(() => client.services.services.QueryService!.reindex());
+              yield* Effect.promise(() => seedTestData(personalSpace));
+            }),
         }),
-        SpacePlugin(),
-        SettingsPlugin(),
+
         PreviewPlugin(),
-        IntentPlugin(),
         TranscriptionPlugin(),
       ],
-      fireEvents: [Events.SetupAppGraph],
+      fireEvents: [AppActivationEvents.SetupAppGraph],
     }),
-    withTheme,
-    withLayout({ fullscreen: true, classNames: 'justify-center' }),
   ],
-};
+} satisfies Meta<typeof DefaultStory>;
 
 export default meta;
 
-type Story = StoryObj<typeof DefaultStory>;
+type Story = StoryObj<typeof meta>;
 
 export const Default: Story = {
   args: {
@@ -251,19 +233,21 @@ export const Default: Story = {
   },
 };
 
-export const EntityExtraction: Story = {
-  args: {
-    detectSpeaking: true,
-    entityExtraction: 'ner',
-    transcriberConfig: TRANSCRIBER_CONFIG,
-    recorderConfig: RECORDER_CONFIG,
-    audioConstraints: {
-      echoCancellation: true,
-      noiseSuppression: true,
-      autoGainControl: true,
-    },
-  },
-};
+// NOTE: We are running out of free quota on hugging face entity extraction.
+// TODO(mykola): Fix hugging face quota issues.
+// export const EntityExtraction: Story = {
+//   args: {
+//     detectSpeaking: true,
+//     entityExtraction: 'ner',
+//     transcriberConfig: TRANSCRIBER_CONFIG,
+//     recorderConfig: RECORDER_CONFIG,
+//     audioConstraints: {
+//       echoCancellation: true,
+//       noiseSuppression: true,
+//       autoGainControl: true,
+//     },
+//   },
+// };
 
 export const SpeechDetection: Story = {
   args: {

package/src/stories/TranscriptionStory.tsx

@@ -0,0 +1,93 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import '@dxos-theme';
+
+import React, {
+  type ChangeEvent,
+  type Dispatch,
+  type FC,
+  type ReactNode,
+  type RefObject,
+  type SetStateAction,
+  useRef,
+} from 'react';
+
+import { IconButton, ScrollContainer, Toolbar } from '@dxos/react-ui';
+import { type Message } from '@dxos/types';
+
+import { Transcription } from '../components';
+import { type SerializationModel } from '../model';
+
+/**
+ * Story wrapper that renders a transcript with playback controls, an optional audio file
+ * upload affordance, and a toolbar slot for additional controls.
+ *
+ * @param onUpload - Callback fired when the user picks a local audio file via the upload button.
+ * @param uploadAccept - HTML `accept` filter for the hidden file input. Defaults to `audio/*`.
+ * @param toolbarSlot - Extra content rendered after the upload button inside the toolbar.
+ */
+export const TranscriptionStory: FC<{
+  model: SerializationModel<Message.Message>;
+  disabled?: boolean;
+  running: boolean;
+  onRunningChange: Dispatch<SetStateAction<boolean>>;
+  audioRef?: RefObject<HTMLAudioElement | null>;
+  onUpload?: (file: File) => void;
+  uploadAccept?: string;
+  toolbarSlot?: ReactNode;
+}> = ({ model, running, onRunningChange, audioRef, disabled, onUpload, uploadAccept = 'audio/*', toolbarSlot }) => {
+  const fileInputRef = useRef<HTMLInputElement>(null);
+
+  const handleFileChange = (event: ChangeEvent<HTMLInputElement>) => {
+    const file = event.target.files?.[0];
+    if (file && onUpload) {
+      onUpload(file);
+    }
+    // Reset so the same file can be re-selected.
+    event.target.value = '';
+  };
+
+  return (
+    <>
+      {audioRef && <audio ref={audioRef} autoPlay />}
+      <Toolbar.Root>
+        <IconButton
+          iconOnly
+          disabled={disabled}
+          icon={running ? 'ph--pause--regular' : 'ph--play--regular'}
+          label={running ? 'Pause' : 'Play'}
+          onClick={() => onRunningChange((running) => !running)}
+        />
+        {onUpload && (
+          <>
+            <input
+              ref={fileInputRef}
+              type='file'
+              accept={uploadAccept}
+              disabled={disabled}
+              className='hidden'
+              onChange={handleFileChange}
+            />
+            <IconButton
+              iconOnly
+              disabled={disabled}
+              icon='ph--upload--regular'
+              label='Upload audio'
+              onClick={() => fileInputRef.current?.click()}
+            />
+          </>
+        )}
+        {toolbarSlot}
+      </Toolbar.Root>
+      <ScrollContainer.Root pin>
+        <ScrollContainer.Content>
+          <ScrollContainer.Viewport>
+            <Transcription model={model} />
+          </ScrollContainer.Viewport>
+        </ScrollContainer.Content>
+      </ScrollContainer.Root>
+    </>
+  );
+};