@dxos/plugin-transcription 0.8.2-main.12df754 → 0.8.2-main.30e4dbb

package/src/components/Transcript/transcript-extension.ts

@@ -9,15 +9,15 @@ import { intervalToDuration } from 'date-fns/intervalToDuration';
 
 import { type CleanupFn, addEventListener, combine } from '@dxos/async';
 import { type RenderCallback } from '@dxos/react-ui-editor';
+import { type DataType } from '@dxos/schema';
 
-import { DocumentAdapter, type BlockModel } from '../../model';
-import { type TranscriptBlock } from '../../types';
+import { DocumentAdapter, type SerializationModel } from '../../model';
 
 /**
- * Data structure that maps Blocks queue to lines with transcript state.
+ * Data structure that maps Chunks queue to lines with transcript state.
  */
 export type TranscriptOptions = {
-  model: BlockModel<TranscriptBlock>;
+  model: SerializationModel<DataType.Message>;
   started?: Date;
   renderButton?: RenderCallback<{ onClick: () => void }>;
 };
@@ -32,12 +32,13 @@ export const transcript = ({ model, started, renderButton }: TranscriptOptions):
       class: 'cm-timestamp-gutter',
       lineMarkerChange: (update) => update.docChanged || update.viewportChanged,
       markers: (view) => {
-        const start = getStartTime(started, model.blocks[0]);
+        const start = getStartTime(started, model.chunks[0]);
         const builder = new RangeSetBuilder<GutterMarker>();
         for (const { from, to } of view.visibleRanges) {
           let line = view.state.doc.lineAt(from);
           while (line.from <= to) {
-            const timestamp = model.getBlockAtLine(line.number)?.segments[0]?.started;
+            const block = model.getChunkAtLine(line.number)?.blocks[0];
+            const timestamp = block?.type === 'transcription' && block.started;
             if (timestamp) {
               builder.add(line.from, line.from, new TimestampMarker(line.number, new Date(timestamp), start));
             }
@@ -69,13 +70,15 @@ export const transcript = ({ model, started, renderButton }: TranscriptOptions):
           let isAutoScrolling = false;
           let hasScrolled = false;
 
-          const scrollToBottom = () => {
-            scroller.style.scrollBehavior = 'smooth';
+          let timeout: NodeJS.Timeout | undefined;
+          const scrollToBottom = (smooth = false) => {
+            scroller.style.scrollBehavior = smooth ? 'smooth' : '';
 
             // Temporarily hide scrollbar to prevent flicker.
             scroller.classList.add('cm-hide-scrollbar');
             isAutoScrolling = true;
-            setTimeout(() => {
+            clearTimeout(timeout);
+            timeout = setTimeout(() => {
               this._controls?.classList.add('opacity-0');
               scroller.classList.remove('cm-hide-scrollbar');
               isAutoScrolling = false;
@@ -96,7 +99,7 @@ export const transcript = ({ model, started, renderButton }: TranscriptOptions):
               this._controls,
               {
                 onClick: () => {
-                  scrollToBottom();
+                  scrollToBottom(false);
                 },
               },
               view,
@@ -122,7 +125,7 @@ export const transcript = ({ model, started, renderButton }: TranscriptOptions):
 
               // Scroll.
               if (autoScroll) {
-                scrollToBottom();
+                scrollToBottom(true);
               }
             }),
           );
@@ -210,24 +213,24 @@ class TimestampMarker extends GutterMarker {
   }
 }
 
-const getStartTime = (started?: Date, block?: TranscriptBlock): Date | undefined => {
+const getStartTime = (started?: Date, message?: DataType.Message): Date | undefined => {
   if (started) {
     return started;
   }
 
-  if (block?.segments[0]?.started) {
-    return new Date(block.segments[0].started);
+  if (message?.blocks[0]?.type === 'transcription' && message.blocks[0].started) {
+    return new Date(message.blocks[0].started);
   }
 
   return undefined;
 };
 
 const formatTimestamp = (timestamp: Date, relative?: Date) => {
-  if (!relative) {
-    return format(timestamp, 'HH:mm:ss');
-  } else {
+  if (relative) {
     const pad = (n = 0) => String(n).padStart(2, '0');
     const { hours, minutes, seconds } = intervalToDuration({ start: relative, end: timestamp });
     return `${pad(hours)}:${pad(minutes)}:${pad(seconds)}`;
+  } else {
+    return format(timestamp, 'HH:mm:ss');
   }
 };

package/src/components/TranscriptContainer.tsx

@@ -5,18 +5,20 @@
 import React, { type FC } from 'react';
 
 import { fullyQualifiedId, getSpace } from '@dxos/client/echo';
-import { useQueue } from '@dxos/react-client/echo';
+import { useMembers, useQueue } from '@dxos/react-client/echo';
 import { StackItem } from '@dxos/react-ui-stack';
+import { type DataType } from '@dxos/schema';
 
 import { Transcript, renderMarkdown } from './Transcript';
 import { useQueueModelAdapter } from '../hooks';
-import { type TranscriptBlock, type TranscriptType } from '../types';
+import { type TranscriptType } from '../types';
 
 export const TranscriptionContainer: FC<{ role: string; transcript: TranscriptType }> = ({ role, transcript }) => {
   const attendableId = fullyQualifiedId(transcript);
   const space = getSpace(transcript);
-  const queue = useQueue<TranscriptBlock>(transcript.queue.dxn, { pollInterval: 1_000 });
-  const model = useQueueModelAdapter(renderMarkdown, queue);
+  const members = useMembers(space?.key).map((member) => member.identity);
+  const queue = useQueue<DataType.Message>(transcript.queue.dxn, { pollInterval: 1_000 });
+  const model = useQueueModelAdapter(renderMarkdown(members), queue);
 
   return (
     <StackItem.Content role={role} classNames='container-max-width pli-2'>

package/src/components/TranscriptionSettings/TranscriptionSettings.tsx

@@ -0,0 +1,26 @@
+//
+// Copyright 2023 DXOS.org
+//
+
+import React from 'react';
+
+import { Input, useTranslation } from '@dxos/react-ui';
+import { DeprecatedFormContainer, DeprecatedFormInput } from '@dxos/react-ui-form';
+
+import { TRANSCRIPTION_PLUGIN } from '../../meta';
+import { type TranscriptionSettingsProps } from '../../types';
+
+export const TranscriptionSettings = ({ settings }: { settings: TranscriptionSettingsProps }) => {
+  const { t } = useTranslation(TRANSCRIPTION_PLUGIN);
+
+  return (
+    <DeprecatedFormContainer>
+      <DeprecatedFormInput label={t('settings entity extraction label')}>
+        <Input.Switch
+          checked={!!settings.entityExtraction}
+          onCheckedChange={(checked) => (settings.entityExtraction = checked)}
+        />
+      </DeprecatedFormInput>
+    </DeprecatedFormContainer>
+  );
+};

package/src/components/TranscriptionSettings/index.ts

@@ -0,0 +1,5 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+export * from './TranscriptionSettings';

package/src/components/index.ts

@@ -6,5 +6,6 @@ import { lazy } from 'react';
 
 export * from './Transcript';
 export * from './TranscriptContainer';
+export * from './TranscriptionSettings';
 
 export const TranscriptContainer = lazy(() => import('./TranscriptContainer'));

package/src/components/transcription.stories.tsx

@@ -18,32 +18,40 @@ import React, {
 
 import { Events, IntentPlugin, SettingsPlugin } from '@dxos/app-framework';
 import { withPluginManager } from '@dxos/app-framework/testing';
-import { MemoryQueue } from '@dxos/echo-db';
+import { AIServiceEdgeClient } from '@dxos/assistant';
+import { AI_SERVICE_ENDPOINT } from '@dxos/assistant/testing';
+import { Filter, MemoryQueue } from '@dxos/echo-db';
 import { create } from '@dxos/echo-schema';
 import { type DXN } from '@dxos/keys';
 import { log } from '@dxos/log';
 import { ClientPlugin } from '@dxos/plugin-client';
+import { PreviewPlugin } from '@dxos/plugin-preview';
 import { SpacePlugin } from '@dxos/plugin-space';
+import { StorybookLayoutPlugin } from '@dxos/plugin-storybook-layout';
 import { ThemePlugin } from '@dxos/plugin-theme';
-import { randomQueueDxn, useQueue } from '@dxos/react-client/echo';
+import { createQueueDxn, useQueue, useSpace } from '@dxos/react-client/echo';
 import { IconButton, Toolbar } from '@dxos/react-ui';
 import { ScrollContainer } from '@dxos/react-ui-components';
 import { defaultTx } from '@dxos/react-ui-theme';
+import { DataType } from '@dxos/schema';
+import { seedTestData, Testing } from '@dxos/schema/testing';
 import { withLayout, withTheme } from '@dxos/storybook-utils';
 
 import { renderMarkdown, Transcript } from './Transcript';
-import { TestItem } from './Transcript/testings';
 import { TranscriptionPlugin } from '../TranscriptionPlugin';
+import { processTranscriptMessage } from '../entity-extraction';
 import { useAudioFile, useAudioTrack, useQueueModelAdapter, useTranscriber } from '../hooks';
-import { type BlockModel } from '../model';
+import { type SerializationModel } from '../model';
+import { TestItem } from '../testing';
 import { type TranscriberParams } from '../transcriber';
-import { TranscriptBlock } from '../types';
 
 const TranscriptionStory: FC<{
-  model: BlockModel<TranscriptBlock>;
+  model: SerializationModel<DataType.Message>;
   running: boolean;
   onRunningChange: Dispatch<SetStateAction<boolean>>;
 }> = ({ model, running, onRunningChange }) => {
+  const space = useSpace();
+
   return (
     <div className='flex flex-col w-[30rem]'>
       <Toolbar.Root>
@@ -55,30 +63,67 @@ const TranscriptionStory: FC<{
         />
       </Toolbar.Root>
       <ScrollContainer>
-        <Transcript model={model} attendableId='story' />
+        <Transcript space={space} model={model} attendableId='story' />
       </ScrollContainer>
     </div>
   );
 };
 
-const Microphone = () => {
+const aiService = new AIServiceEdgeClient({
+  endpoint: AI_SERVICE_ENDPOINT.REMOTE,
+});
+
+const Microphone = ({ entityExtraction }: { entityExtraction?: boolean }) => {
   const [running, setRunning] = useState(false);
 
   // Audio.
   const track = useAudioTrack(running);
 
   // Queue.
-  const queueDxn = useMemo(() => randomQueueDxn(), []);
-  const queue = useMemo(() => new MemoryQueue<TranscriptBlock>(queueDxn), [queueDxn]);
-  const model = useQueueModelAdapter(renderMarkdown, queue);
+  const queueDxn = useMemo(() => createQueueDxn(), []);
+  const queue = useMemo(() => new MemoryQueue<DataType.Message>(queueDxn), [queueDxn]);
+  const model = useQueueModelAdapter(renderMarkdown([]), queue);
+  const space = useSpace();
 
   // Transcriber.
   const handleSegments = useCallback<TranscriberParams['onSegments']>(
-    async (segments) => {
-      const block = create(TranscriptBlock, { segments });
-      void queue.append([block]);
+    async (blocks) => {
+      const message = create(DataType.Message, { sender: { name: 'You' }, created: new Date().toISOString(), blocks });
+
+      if (entityExtraction) {
+        if (!space) {
+          log.warn('no space');
+          return;
+        }
+        // TODO(dmaretskyi): Move to vector search index.
+        const { objects } = await space.db
+          .query(
+            Filter.or(
+              Filter.schema(DataType.Person),
+              Filter.schema(DataType.Organization),
+              Filter.schema(Testing.DocumentType),
+            ),
+          )
+          .run();
+
+        log.info('context', { objects });
+
+        const result = await processTranscriptMessage({
+          message,
+          aiService,
+          context: {
+            objects,
+          },
+          options: {
+            fallbackToRaw: true,
+          },
+        });
+        void queue.append([result.message]);
+      } else {
+        void queue.append([message]);
+      }
     },
-    [queue],
+    [queue, space],
   );
 
   const config = useRef<TranscriberParams['config']>({
@@ -94,10 +139,8 @@ const Microphone = () => {
   const [isOpen, setIsOpen] = useState(false);
   useEffect(() => {
     void transcriber?.open().then(() => setIsOpen(true));
-    log.info('transcriber open');
     return () => {
       void transcriber?.close().then(() => setIsOpen(false));
-      log.info('transcriber close');
     };
   }, [transcriber]);
 
@@ -105,10 +148,8 @@ const Microphone = () => {
   useEffect(() => {
     if (running && transcriber?.isOpen) {
       transcriber?.startChunksRecording();
-      log.info('transcriber startChunksRecording');
     } else if (!running) {
       transcriber?.stopChunksRecording();
-      log.info('transcriber stopChunksRecording');
     }
   }, [transcriber, running, isOpen]);
 
@@ -141,12 +182,12 @@ const AudioFile = ({ queueDxn, audioUrl }: { queueDxn: DXN; audioUrl: string; tr
   }, [audio, running]);
 
   // Transcriber.
-  const queue = useQueue<TranscriptBlock>(queueDxn, { pollInterval: 500 });
-  const model = useQueueModelAdapter(renderMarkdown, queue);
+  const queue = useQueue<DataType.Message>(queueDxn, { pollInterval: 500 });
+  const model = useQueueModelAdapter(renderMarkdown([]), queue);
   const handleSegments = useCallback<TranscriberParams['onSegments']>(
-    async (segments) => {
-      const block = create(TranscriptBlock, { authorName: 'test', authorHue: 'cyan', segments });
-      queue?.append([block]);
+    async (blocks) => {
+      const message = create(DataType.Message, { sender: { name: 'You' }, created: new Date().toISOString(), blocks });
+      void queue?.append([message]);
     },
     [queue],
   );
@@ -187,14 +228,19 @@ const meta: Meta<typeof AudioFile> = {
     withPluginManager({
       plugins: [
         ThemePlugin({ tx: defaultTx }),
+        StorybookLayoutPlugin(),
         ClientPlugin({
-          types: [TestItem],
+          types: [TestItem, DataType.Person, DataType.Organization, Testing.DocumentType],
           onClientInitialized: async (_, client) => {
             await client.halo.createIdentity();
+            await client.spaces.waitUntilReady();
+            await client.spaces.default.waitUntilReady();
+            await seedTestData(client.spaces.default);
           },
         }),
         SpacePlugin(),
         SettingsPlugin(),
+        PreviewPlugin(),
         IntentPlugin(),
         TranscriptionPlugin(),
       ],
@@ -218,26 +264,30 @@ const meta: Meta<typeof AudioFile> = {
       // ],
     }),
     withTheme,
-    withLayout({
-      tooltips: true,
-      fullscreen: true,
-      classNames: 'justify-center',
-    }),
+    withLayout({ fullscreen: true, classNames: 'justify-center' }),
   ],
 };
 
 export default meta;
 
-type Story = StoryObj<typeof AudioFile>;
+export const Default: StoryObj<typeof Microphone> = {
+  render: Microphone,
+  args: {
+    entityExtraction: false,
+  },
+};
 
-export const Default: Story = {
+export const EntityExtraction: StoryObj<typeof Microphone> = {
   render: Microphone,
+  args: {
+    entityExtraction: true,
+  },
 };
 
-export const File: Story = {
+export const File: StoryObj<typeof AudioFile> = {
   render: AudioFile,
   args: {
-    queueDxn: randomQueueDxn(),
+    queueDxn: createQueueDxn(),
     // https://learnenglish.britishcouncil.org/general-english/audio-zone/living-london
     transcriptUrl: 'https://dxos.network/audio-london.txt',
     audioUrl: 'https://dxos.network/audio-london.m4a',

package/src/entity-extraction/entity-extraction.test.ts

@@ -2,31 +2,96 @@
 // Copyright 2025 DXOS.org
 //
 
-import { describe, test } from 'vitest';
+import { describe, test, expect } from 'vitest';
 
 import { AIServiceEdgeClient } from '@dxos/assistant';
 import { AI_SERVICE_ENDPOINT } from '@dxos/assistant/testing';
 import { log } from '@dxos/log';
+import { createTestData } from '@dxos/schema/testing';
+import { range } from '@dxos/util';
 
-import { processTranscriptBlock } from './entity-extraction';
-import * as TestData from '../testing/test-data';
+import { processTranscriptMessage, postprocessText } from './entity-extraction';
 
 const aiService = new AIServiceEdgeClient({
   endpoint: AI_SERVICE_ENDPOINT.REMOTE,
 });
 
 describe.skip('EntityExtraction', { timeout: 180_000 }, () => {
-  test('should process a transcript block', async () => {
-    for (const block of TestData.transcriptBlocks) {
-      log.info('input', block);
-      const { block: enhancedBlock } = await processTranscriptBlock({
-        block,
+  test.skip('should process a transcript block', async () => {
+    const { transcriptMessages, documents, contacts } = await createTestData();
+
+    log.info('context', { documents, contacts });
+
+    for (const message of transcriptMessages) {
+      log.info('input', message);
+      const { message: enhancedMessage } = await processTranscriptMessage({
+        message,
+        aiService,
+        context: {
+          objects: [...documents, ...Object.values(contacts)],
+        },
+      });
+      log.info('output', enhancedMessage);
+    }
+  });
+
+  test.only('computational irreducibility', async () => {
+    const { transcriptWoflram, documents, contacts } = await createTestData();
+
+    log.info('context', { documents, contacts });
+    const message = transcriptWoflram[0];
+    log.info('input', message);
+
+    await Promise.all(
+      range(10).map(async () => {
+        const { message: enhancedMessage, timeElapsed } = await processTranscriptMessage({
+          message,
+          aiService,
+          context: {
+            objects: [...documents, ...Object.values(contacts)],
+          },
+        });
+        log.info('output', { message: enhancedMessage.blocks[0], timeElapsed });
+      }),
+    );
+  });
+
+  test('org and document linking', async () => {
+    const { transcriptJosiah, documents, contacts, organizations } = await createTestData();
+
+    log.info('context', { contacts, organizations, documents });
+
+    for (const message of transcriptJosiah) {
+      log.info('input', message);
+
+      const { message: enhancedMessage } = await processTranscriptMessage({
+        message,
         aiService,
         context: {
-          objects: [...TestData.documents, ...Object.values(TestData.contacts)],
+          objects: [...documents, ...Object.values(contacts), ...Object.values(organizations)],
         },
       });
-      log.info('output', enhancedBlock);
+      log.info('output', enhancedMessage);
     }
   });
 });
+
+describe('postprocessText', () => {
+  test('should replace quotes with DXN references', () => {
+    const quotes = {
+      references: [{ quote: 'computational irreducibility', id: '01JTG9JW11XGWJZ32AW8ET93D1' }],
+    };
+
+    expect(postprocessText('This is a computational irreducibility test.', quotes)).toBe(
+      'This is a [computational irreducibility][dxn:echo:@:01JTG9JW11XGWJZ32AW8ET93D1] test.',
+    );
+    expect(
+      postprocessText(
+        "And what I'd like to talk today about is Steven Wolfram's concept of a computational irreducibility.",
+        quotes,
+      ),
+    ).toBe(
+      "And what I'd like to talk today about is Steven Wolfram's concept of a [computational irreducibility][dxn:echo:@:01JTG9JW11XGWJZ32AW8ET93D1].",
+    );
+  });
+});