@dxos/plugin-transcription 0.8.4-main.dedc0f3 → 0.8.4-main.dfabb4ec29

package/src/{components/stories → stories}/FileTranscription.stories.tsx

@@ -2,37 +2,35 @@
 // Copyright 2025 DXOS.org
 //
 
-import '@dxos-theme';
-
 import { type Meta, type StoryObj } from '@storybook/react-vite';
+import * as Effect from 'effect/Effect';
 import React, { useCallback, useEffect, useMemo, useRef, useState } from 'react';
 
-import { Events, IntentPlugin, SettingsPlugin } from '@dxos/app-framework';
 import { withPluginManager } from '@dxos/app-framework/testing';
+import { AppActivationEvents } from '@dxos/app-toolkit';
 import { scheduleTask } from '@dxos/async';
 import { Context } from '@dxos/context';
 import { Obj } from '@dxos/echo';
 import { MemoryQueue } from '@dxos/echo-db';
-import { createQueueDXN } from '@dxos/echo-schema';
-import { FunctionExecutor, ServiceContainer } from '@dxos/functions';
+import { createQueueDXN } from '@dxos/echo/internal';
+import { FunctionExecutor, ServiceContainer } from '@dxos/functions-runtime';
 import { log } from '@dxos/log';
-import { ClientPlugin } from '@dxos/plugin-client';
-import { PreviewPlugin } from '@dxos/plugin-preview';
-import { SpacePlugin } from '@dxos/plugin-space';
-import { StorybookLayoutPlugin } from '@dxos/plugin-storybook-layout';
-import { ThemePlugin } from '@dxos/plugin-theme';
-import { defaultTx } from '@dxos/react-ui-theme';
-import { DataType } from '@dxos/schema';
-import { Testing, seedTestData } from '@dxos/schema/testing';
-import { withLayout, withTheme } from '@dxos/storybook-utils';
+import { ClientPlugin } from '@dxos/plugin-client/plugin';
+import { initializeIdentity } from '@dxos/plugin-client/testing';
+import { PreviewPlugin } from '@dxos/plugin-preview/testing';
+import { StorybookPlugin, corePlugins } from '@dxos/plugin-testing';
+import { withLayout } from '@dxos/react-ui/testing';
+import { TestSchema } from '@dxos/schema/testing';
+import { type Actor, Message, Organization, Person } from '@dxos/types';
+import { seedTestData } from '@dxos/types/testing';
 
-import { useAudioFile, useQueueModelAdapter, useTranscriber } from '../../hooks';
-import { MessageNormalizer, getActorId } from '../../segments-normalization';
-import { TestItem } from '../../testing';
-import { type MediaStreamRecorderParams, type TranscriberParams } from '../../transcriber';
-import { TranscriptionPlugin } from '../../TranscriptionPlugin';
-import { renderByline } from '../Transcript';
+import { useAudioFile, useQueueModelAdapter, useTranscriber } from '#hooks';
+import { TestItem } from '#testing';
 
+import { MessageNormalizer, getActorId } from '../normalization';
+import { type MediaStreamRecorderProps, type TranscriberProps } from '../transcriber';
+import { TranscriptionPlugin } from '../TranscriptionPlugin';
+import { renderByline } from '../util';
 import { TranscriptionStory } from './TranscriptionStory';
 import { useIsSpeaking } from './useIsSpeaking';
 
@@ -47,15 +45,33 @@ const AudioFile = ({
   detectSpeaking?: boolean;
   normalizeSentences?: boolean;
   audioUrl: string;
-  transcriberConfig?: TranscriberParams['config'];
-  recorderConfig?: MediaStreamRecorderParams['config'];
+  transcriberConfig?: TranscriberProps['config'];
+  recorderConfig?: MediaStreamRecorderProps['config'];
   audioConstraints?: MediaTrackConstraints;
 }) => {
   const [running, setRunning] = useState(false);
-  const actor: DataType.Actor = useMemo(() => ({ name: 'You' }), []);
+  const actor: Actor.Actor = useMemo(() => ({ name: 'You' }), []);
+
+  // Optional uploaded file overrides the default URL.
+  // The useEffect below revokes the previous URL whenever it changes (and on unmount).
+  const [uploadedUrl, setUploadedUrl] = useState<string>();
+  useEffect(() => {
+    return () => {
+      if (uploadedUrl) {
+        URL.revokeObjectURL(uploadedUrl);
+      }
+    };
+  }, [uploadedUrl]);
+
+  const handleUpload = useCallback((file: File) => {
+    setRunning(false);
+    setUploadedUrl(URL.createObjectURL(file));
+  }, []);
+
+  const sourceUrl = uploadedUrl ?? audioUrl;
 
   // Audio.
-  const { audio, track, stream } = useAudioFile(audioUrl, audioConstraints);
+  const { audio, track, stream } = useAudioFile(sourceUrl, audioConstraints);
 
   // Speaking.
   const isSpeaking = detectSpeaking ? useIsSpeaking(track) : true;
@@ -81,13 +97,13 @@ const AudioFile = ({
   // Transcriber.
   // TODO(dmaretskyi): Use space.queues.create() instead.
   const queueDxn = useMemo(() => createQueueDXN(), []);
-  const queue = useMemo(() => new MemoryQueue<DataType.Message>(queueDxn), [queueDxn]);
+  const queue = useMemo(() => new MemoryQueue<Message.Message>(queueDxn), [queueDxn]);
 
   const model = useQueueModelAdapter(renderByline([]), queue);
-  const handleSegments = useCallback<TranscriberParams['onSegments']>(
+  const handleSegments = useCallback<TranscriberProps['onSegments']>(
     async (blocks) => {
       void queue?.append([
-        Obj.make(DataType.Message, {
+        Obj.make(Message.Message, {
           created: new Date().toISOString(),
           sender: actor,
           blocks,
@@ -182,36 +198,33 @@ const AudioFile = ({
       running={running}
       onRunningChange={setRunning}
       audioRef={ref}
+      onUpload={handleUpload}
     />
   );
 };
 
 const meta = {
-  title: 'plugins/plugin-transcription/FileTranscription',
+  title: 'plugins/plugin-transcription/components/FileTranscription',
   decorators: [
+    withLayout({ layout: 'column' }),
     withPluginManager({
       plugins: [
-        ThemePlugin({ tx: defaultTx }),
-        StorybookLayoutPlugin(),
+        ...corePlugins(),
+        StorybookPlugin({}),
         ClientPlugin({
-          types: [TestItem, DataType.Person, DataType.Organization, Testing.DocumentType],
-          onClientInitialized: async ({ client }) => {
-            await client.halo.createIdentity();
-            await client.spaces.waitUntilReady();
-            await client.spaces.default.waitUntilReady();
-            await seedTestData(client.spaces.default);
-          },
+          types: [TestItem, Person.Person, Organization.Organization, TestSchema.DocumentType],
+          onClientInitialized: ({ client }) =>
+            Effect.gen(function* () {
+              const { personalSpace } = yield* initializeIdentity(client);
+              yield* Effect.promise(() => seedTestData(personalSpace));
+            }),
         }),
-        SpacePlugin(),
-        SettingsPlugin(),
+
         PreviewPlugin(),
-        IntentPlugin(),
         TranscriptionPlugin(),
       ],
-      fireEvents: [Events.SetupAppGraph],
+      fireEvents: [AppActivationEvents.SetupAppGraph],
     }),
-    withTheme,
-    withLayout({ fullscreen: true, classNames: 'justify-center' }),
   ],
 } satisfies Meta;
 

package/src/{components/stories → stories}/KeyWordRecognition.stories.tsx

@@ -2,26 +2,24 @@
 // Copyright 2025 DXOS.org
 //
 
-import '@dxos-theme';
-
 import { type Meta, type StoryObj } from '@storybook/react-vite';
 import React, { useEffect, useMemo, useState } from 'react';
 
 import { log } from '@dxos/log';
-import { Toolbar } from '@dxos/react-ui';
-import { mx } from '@dxos/react-ui-theme';
-import { withLayout, withTheme } from '@dxos/storybook-utils';
+import { Panel, Toolbar } from '@dxos/react-ui';
+import { withLayout, withTheme } from '@dxos/react-ui/testing';
+import { mx } from '@dxos/ui-theme';
 
 type Word = {
   text: string;
   matched: boolean;
 };
 
-type StoryProps = {
+type DefaultStoryProps = {
   keywords: string[];
 };
 
-const DefaultStory = ({ keywords }: StoryProps) => {
+const DefaultStory = ({ keywords }: DefaultStoryProps) => {
   const [running, setRunning] = useState(false);
   const [matchingWords, setMatchingWords] = useState<Word[]>(keywords.map((word) => ({ text: word, matched: false })));
 
@@ -88,39 +86,41 @@ const DefaultStory = ({ keywords }: StoryProps) => {
   }, [running, recognition]);
 
   return (
-    <div className={mx('flex flex-col gap-2 w-[30rem]')}>
-      <Toolbar.Root>
-        <Toolbar.IconButton
-          iconOnly
-          icon={running ? 'ph--pause--regular' : 'ph--play--regular'}
-          label={running ? 'Pause' : 'Play'}
-          onClick={() => setRunning((running) => !running)}
-        />
-      </Toolbar.Root>
-
-      <div className='flex flex-wrap'>
-        {matchingWords.map((word, index) => (
-          <span
-            key={index}
-            className={mx(
-              'px-3 py-1 m-2 rounded border',
-              word.matched ? 'bg-red-200 text-red-800 border-red-300' : 'bg-gray-200 text-gray-600 border-gray-300',
-            )}
-          >
-            {word.text}
-          </span>
-        ))}
-      </div>
-
-      <span>{transcript || (running ? 'Listening...' : 'Click play to start speech recognition')}</span>
-    </div>
+    <Panel.Root>
+      <Panel.Toolbar asChild>
+        <Toolbar.Root>
+          <Toolbar.IconButton
+            iconOnly
+            icon={running ? 'ph--pause--regular' : 'ph--play--regular'}
+            label={running ? 'Pause' : 'Play'}
+            onClick={() => setRunning((running) => !running)}
+          />
+        </Toolbar.Root>
+      </Panel.Toolbar>
+
+      <Panel.Content>
+        <div className='p-4 grid grid-cols-[repeat(auto-fill,minmax(6rem,1fr))] gap-2'>
+          {matchingWords.map((word, index) => (
+            <span
+              key={index}
+              className={mx(
+                'grid place-items-center p-2 aspect-square rounded-sm border border-separator',
+                word.matched && 'border-red-300',
+              )}
+            >
+              {word.text}
+            </span>
+          ))}
+        </div>
+      </Panel.Content>
+    </Panel.Root>
   );
 };
 
 const meta = {
-  title: 'plugins/plugin-transcription/KeyWordDetection',
+  title: 'plugins/plugin-transcription/components/KeyWordDetection',
   render: DefaultStory,
-  decorators: [withTheme, withLayout({ fullscreen: true, classNames: 'justify-center' })],
+  decorators: [withTheme(), withLayout({ layout: 'column' })],
 } satisfies Meta<typeof DefaultStory>;
 
 export default meta;
@@ -129,6 +129,6 @@ type Story = StoryObj<typeof meta>;
 
 export const SpeechRecognitionAPI: Story = {
   args: {
-    keywords: ['kai', 'computer', 'hello'],
+    keywords: ['kai', 'computer', 'hello', 'dxos'],
   },
 };

package/src/{components/stories → stories}/MicrophoneTranscription.stories.tsx

@@ -2,13 +2,12 @@
 // Copyright 2025 DXOS.org
 //
 
-import '@dxos-theme';
-
 import { type Meta, type StoryObj } from '@storybook/react-vite';
+import * as Effect from 'effect/Effect';
 import React, { useCallback, useEffect, useMemo, useState } from 'react';
 
-import { Events, IntentPlugin, SettingsPlugin } from '@dxos/app-framework';
 import { withPluginManager } from '@dxos/app-framework/testing';
+import { AppActivationEvents } from '@dxos/app-toolkit';
 import {
   type ExtractionFunction,
   extractionAnthropicFunction,
@@ -16,29 +15,27 @@ import {
   getNer,
   processTranscriptMessage,
 } from '@dxos/assistant/extraction';
-import { Filter, Obj, type Type } from '@dxos/echo';
+import { Filter, type Obj } from '@dxos/echo';
 import { MemoryQueue } from '@dxos/echo-db';
-import { createQueueDXN } from '@dxos/echo-schema';
-import { FunctionExecutor, ServiceContainer } from '@dxos/functions';
+import { createQueueDXN } from '@dxos/echo/internal';
 import { invariant } from '@dxos/invariant';
 import { log } from '@dxos/log';
-import { ClientPlugin } from '@dxos/plugin-client';
-import { PreviewPlugin } from '@dxos/plugin-preview';
-import { SpacePlugin } from '@dxos/plugin-space';
-import { StorybookLayoutPlugin } from '@dxos/plugin-storybook-layout';
-import { ThemePlugin } from '@dxos/plugin-theme';
-import { IndexKind, useSpace } from '@dxos/react-client/echo';
-import { defaultTx } from '@dxos/react-ui-theme';
-import { DataType } from '@dxos/schema';
-import { Testing, seedTestData } from '@dxos/schema/testing';
-import { withLayout, withTheme } from '@dxos/storybook-utils';
+import { ClientPlugin } from '@dxos/plugin-client/plugin';
+import { initializeIdentity } from '@dxos/plugin-client/testing';
+import { PreviewPlugin } from '@dxos/plugin-preview/testing';
+import { StorybookPlugin, corePlugins } from '@dxos/plugin-testing';
+import { IndexKind, useSpaces } from '@dxos/react-client/echo';
+import { withLayout } from '@dxos/react-ui/testing';
+import { TestSchema } from '@dxos/schema/testing';
+import { Message, Organization, Person } from '@dxos/types';
+import { seedTestData } from '@dxos/types/testing';
 
-import { useAudioTrack, useQueueModelAdapter, useTranscriber } from '../../hooks';
-import { TestItem } from '../../testing';
-import { type MediaStreamRecorderParams, type TranscriberParams } from '../../transcriber';
-import { TranscriptionPlugin } from '../../TranscriptionPlugin';
-import { renderByline } from '../Transcript';
+import { useAudioTrack, useQueueModelAdapter, useTranscriber } from '#hooks';
+import { TestItem } from '#testing';
 
+import { type MediaStreamRecorderProps, type TranscriberProps } from '../transcriber';
+import { TranscriptionPlugin } from '../TranscriptionPlugin';
+import { renderByline } from '../util';
 import { TranscriptionStory } from './TranscriptionStory';
 import { useIsSpeaking } from './useIsSpeaking';
 
@@ -51,10 +48,10 @@ const RECORDER_CONFIG = {
   interval: 200,
 };
 
-type StoryProps = {
+type DefaultStoryProps = {
   detectSpeaking?: boolean;
-  transcriberConfig: TranscriberParams['config'];
-  recorderConfig: MediaStreamRecorderParams['config'];
+  transcriberConfig: TranscriberProps['config'];
+  recorderConfig: MediaStreamRecorderProps['config'];
   audioConstraints?: MediaTrackConstraints;
   entityExtraction: 'none' | 'ner' | 'llm';
 };
@@ -65,7 +62,7 @@ const DefaultStory = ({
   transcriberConfig,
   recorderConfig,
   audioConstraints,
-}: StoryProps) => {
+}: DefaultStoryProps) => {
   const [running, setRunning] = useState(false);
 
   // Audio.
@@ -77,9 +74,9 @@ const DefaultStory = ({
   // Queue.
   // TODO(dmaretskyi): Use space.queues.create() instead.
   const queueDxn = useMemo(() => createQueueDXN(), []);
-  const queue = useMemo(() => new MemoryQueue<DataType.Message>(queueDxn), [queueDxn]);
+  const queue = useMemo(() => new MemoryQueue<Message.Message>(queueDxn), [queueDxn]);
   const model = useQueueModelAdapter(renderByline([]), queue);
-  const space = useSpace();
+  const [space] = useSpaces();
 
   useEffect(() => {
     if (!space) {
@@ -88,15 +85,14 @@ const DefaultStory = ({
   }, [space]);
 
   // Entity extraction.
-  const { extractionFunction, executor, objects } = useMemo(() => {
+  const { extractionFunction, objects } = useMemo(() => {
     if (!space) {
       log.warn('no space');
       return {};
     }
 
-    let executor: FunctionExecutor | undefined;
     let extractionFunction: ExtractionFunction | undefined;
-    let objects: Promise<Type.Expando[]> | undefined;
+    let objects: Promise<Obj.Unknown[]> | undefined;
 
     if (entityExtraction === 'ner') {
       // Init model loading. Takes time.
@@ -107,34 +103,21 @@ const DefaultStory = ({
       objects = space.db
         .query(
           Filter.or(
-            Filter.type(DataType.Person),
-            Filter.type(DataType.Organization),
-            Filter.type(Testing.DocumentType),
+            Filter.type(Person.Person),
+            Filter.type(Organization.Organization),
+            Filter.type(TestSchema.DocumentType),
           ),
         )
-        .run()
-        .then((result) => result.objects);
-    }
-    if (entityExtraction !== 'none') {
-      executor = new FunctionExecutor(
-        new ServiceContainer().setServices({
-          // ai: {
-          //   client: new Edge AiServiceClient({
-          //     endpoint: AI_SERVICE_ENDPOINT.REMOTE,
-          //   }),
-          // },
-          // database: { db: space!.db },
-        }),
-      );
+        .run();
     }
 
-    return { extractionFunction, executor, objects };
+    return { extractionFunction, objects };
   }, [entityExtraction, space]);
 
   // Transcriber.
-  const handleSegments = useCallback<TranscriberParams['onSegments']>(
+  const handleSegments = useCallback<TranscriberProps['onSegments']>(
     async (blocks) => {
-      const message = Obj.make(DataType.Message, {
+      const message = Message.make({
         sender: { name: 'You' },
         created: new Date().toISOString(),
         blocks,
@@ -146,9 +129,7 @@ const DefaultStory = ({
 
       if (entityExtraction !== 'none') {
         invariant(extractionFunction, 'extractionFunction is required');
-        invariant(executor, 'executor is required');
         const result = await processTranscriptMessage({
-          executor,
           function: extractionFunction,
           input: {
             message,
@@ -194,45 +175,43 @@ const DefaultStory = ({
 };
 
 const meta = {
-  title: 'plugins/plugin-transcription/MicrophoneTranscription',
+  title: 'plugins/plugin-transcription/components/MicrophoneTranscription',
   render: DefaultStory,
   decorators: [
+    withLayout({ layout: 'column' }),
     withPluginManager({
       plugins: [
-        ThemePlugin({ tx: defaultTx }),
-        StorybookLayoutPlugin(),
+        ...corePlugins(),
+        StorybookPlugin({}),
         ClientPlugin({
-          types: [TestItem, DataType.Person, DataType.Organization, Testing.DocumentType],
-          onClientInitialized: async ({ client }) => {
-            await client.halo.createIdentity();
-            await client.spaces.waitUntilReady();
-            await client.spaces.default.waitUntilReady();
-            // TODO(mykola): Make API easier to use.
-            // TODO(mykola): Delete after enabling vector indexing by default.
-            // Enable vector indexing.
-            await client.services.services.QueryService!.setConfig({
-              enabled: true,
-              indexes: [
-                //
-                { kind: IndexKind.Kind.SCHEMA_MATCH },
-                { kind: IndexKind.Kind.GRAPH },
-                { kind: IndexKind.Kind.VECTOR },
-              ],
-            });
-            await client.services.services.QueryService!.reindex();
-            await seedTestData(client.spaces.default);
-          },
+          types: [TestItem, Person.Person, Organization.Organization, TestSchema.DocumentType],
+          onClientInitialized: ({ client }) =>
+            Effect.gen(function* () {
+              const { personalSpace } = yield* initializeIdentity(client);
+              // TODO(mykola): Make API easier to use.
+              // TODO(mykola): Delete after enabling vector indexing by default.
+              // Enable vector indexing.
+              yield* Effect.promise(() =>
+                client.services.services.QueryService!.setConfig({
+                  enabled: true,
+                  indexes: [
+                    //
+                    { kind: IndexKind.Kind.SCHEMA_MATCH },
+                    { kind: IndexKind.Kind.GRAPH },
+                    { kind: IndexKind.Kind.VECTOR },
+                  ],
+                }),
+              );
+              yield* Effect.promise(() => client.services.services.QueryService!.reindex());
+              yield* Effect.promise(() => seedTestData(personalSpace));
+            }),
         }),
-        SpacePlugin(),
-        SettingsPlugin(),
+
         PreviewPlugin(),
-        IntentPlugin(),
         TranscriptionPlugin(),
       ],
-      fireEvents: [Events.SetupAppGraph],
+      fireEvents: [AppActivationEvents.SetupAppGraph],
     }),
-    withTheme,
-    withLayout({ fullscreen: true, classNames: 'justify-center' }),
   ],
 } satisfies Meta<typeof DefaultStory>;
 

package/src/stories/TranscriptionStory.tsx

@@ -0,0 +1,93 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import '@dxos-theme';
+
+import React, {
+  type ChangeEvent,
+  type Dispatch,
+  type FC,
+  type ReactNode,
+  type RefObject,
+  type SetStateAction,
+  useRef,
+} from 'react';
+
+import { IconButton, ScrollContainer, Toolbar } from '@dxos/react-ui';
+import { type Message } from '@dxos/types';
+
+import { Transcription } from '../components';
+import { type SerializationModel } from '../model';
+
+/**
+ * Story wrapper that renders a transcript with playback controls, an optional audio file
+ * upload affordance, and a toolbar slot for additional controls.
+ *
+ * @param onUpload - Callback fired when the user picks a local audio file via the upload button.
+ * @param uploadAccept - HTML `accept` filter for the hidden file input. Defaults to `audio/*`.
+ * @param toolbarSlot - Extra content rendered after the upload button inside the toolbar.
+ */
+export const TranscriptionStory: FC<{
+  model: SerializationModel<Message.Message>;
+  disabled?: boolean;
+  running: boolean;
+  onRunningChange: Dispatch<SetStateAction<boolean>>;
+  audioRef?: RefObject<HTMLAudioElement | null>;
+  onUpload?: (file: File) => void;
+  uploadAccept?: string;
+  toolbarSlot?: ReactNode;
+}> = ({ model, running, onRunningChange, audioRef, disabled, onUpload, uploadAccept = 'audio/*', toolbarSlot }) => {
+  const fileInputRef = useRef<HTMLInputElement>(null);
+
+  const handleFileChange = (event: ChangeEvent<HTMLInputElement>) => {
+    const file = event.target.files?.[0];
+    if (file && onUpload) {
+      onUpload(file);
+    }
+    // Reset so the same file can be re-selected.
+    event.target.value = '';
+  };
+
+  return (
+    <>
+      {audioRef && <audio ref={audioRef} autoPlay />}
+      <Toolbar.Root>
+        <IconButton
+          iconOnly
+          disabled={disabled}
+          icon={running ? 'ph--pause--regular' : 'ph--play--regular'}
+          label={running ? 'Pause' : 'Play'}
+          onClick={() => onRunningChange((running) => !running)}
+        />
+        {onUpload && (
+          <>
+            <input
+              ref={fileInputRef}
+              type='file'
+              accept={uploadAccept}
+              disabled={disabled}
+              className='hidden'
+              onChange={handleFileChange}
+            />
+            <IconButton
+              iconOnly
+              disabled={disabled}
+              icon='ph--upload--regular'
+              label='Upload audio'
+              onClick={() => fileInputRef.current?.click()}
+            />
+          </>
+        )}
+        {toolbarSlot}
+      </Toolbar.Root>
+      <ScrollContainer.Root pin>
+        <ScrollContainer.Content>
+          <ScrollContainer.Viewport>
+            <Transcription model={model} />
+          </ScrollContainer.Viewport>
+        </ScrollContainer.Content>
+      </ScrollContainer.Root>
+    </>
+  );
+};