@dxos/plugin-transcription 0.8.1 → 0.8.2-main.f081794

package/src/components/transcription.stories.tsx

@@ -16,84 +16,107 @@ import React, {
   useState,
 } from 'react';
 
-import { createStatic } from '@dxos/echo-schema';
+import { Events, IntentPlugin, SettingsPlugin } from '@dxos/app-framework';
+import { withPluginManager } from '@dxos/app-framework/testing';
+import { MemoryQueue } from '@dxos/echo-db';
+import { create } from '@dxos/echo-schema';
 import { type DXN } from '@dxos/keys';
 import { log } from '@dxos/log';
-import { Config } from '@dxos/react-client';
-import { useQueue } from '@dxos/react-client/echo';
-import { withClientProvider } from '@dxos/react-client/testing';
+import { ClientPlugin } from '@dxos/plugin-client';
+import { SpacePlugin } from '@dxos/plugin-space';
+import { ThemePlugin } from '@dxos/plugin-theme';
+import { randomQueueDxn, useQueue } from '@dxos/react-client/echo';
 import { IconButton, Toolbar } from '@dxos/react-ui';
 import { ScrollContainer } from '@dxos/react-ui-components';
+import { defaultTx } from '@dxos/react-ui-theme';
 import { withLayout, withTheme } from '@dxos/storybook-utils';
 
-import { Transcript } from './Transcript';
-import { useAudioFile, useAudioTrack, useTranscriber } from '../hooks';
+import { renderMarkdown, Transcript } from './Transcript';
+import { TestItem } from './Transcript/testings';
+import { TranscriptionPlugin } from '../TranscriptionPlugin';
+import { useAudioFile, useAudioTrack, useQueueModelAdapter, useTranscriber } from '../hooks';
+import { type BlockModel } from '../model';
 import { type TranscriberParams } from '../transcriber';
 import { TranscriptBlock } from '../types';
-import { randomQueueDxn } from '../util';
 
 const TranscriptionStory: FC<{
-  playing: boolean;
-  setPlaying: Dispatch<SetStateAction<boolean>>;
-  blocks?: TranscriptBlock[];
-}> = ({ playing, setPlaying, blocks }) => {
+  model: BlockModel<TranscriptBlock>;
+  running: boolean;
+  onRunningChange: Dispatch<SetStateAction<boolean>>;
+}> = ({ model, running, onRunningChange }) => {
   return (
     <div className='flex flex-col w-[30rem]'>
       <Toolbar.Root>
         <IconButton
           iconOnly
-          icon={playing ? 'ph--pause--regular' : 'ph--play--regular'}
-          label={playing ? 'Pause' : 'Play'}
-          onClick={() => setPlaying((playing) => !playing)}
+          icon={running ? 'ph--pause--regular' : 'ph--play--regular'}
+          label={running ? 'Pause' : 'Play'}
+          onClick={() => onRunningChange((running) => !running)}
         />
       </Toolbar.Root>
       <ScrollContainer>
-        <Transcript blocks={blocks} attendableId='story' />
+        <Transcript model={model} attendableId='story' />
       </ScrollContainer>
     </div>
   );
 };
 
 const Microphone = () => {
-  const [playing, setPlaying] = useState(false);
+  const [running, setRunning] = useState(false);
 
   // Audio.
-  const track = useAudioTrack(playing);
+  const track = useAudioTrack(running);
 
   // Queue.
   const queueDxn = useMemo(() => randomQueueDxn(), []);
-  const queue = useQueue<TranscriptBlock>(queueDxn, { pollInterval: 500 });
+  const queue = useMemo(() => new MemoryQueue<TranscriptBlock>(queueDxn), [queueDxn]);
+  const model = useQueueModelAdapter(renderMarkdown, queue);
 
   // Transcriber.
   const handleSegments = useCallback<TranscriberParams['onSegments']>(
     async (segments) => {
-      const block = createStatic(TranscriptBlock, { segments });
-      queue?.append([block]);
+      const block = create(TranscriptBlock, { segments });
+      void queue.append([block]);
     },
     [queue],
   );
-  const transcriber = useTranscriber({ audioStreamTrack: track, onSegments: handleSegments });
+
+  const config = useRef<TranscriberParams['config']>({
+    transcribeAfterChunksAmount: 20,
+    prefixBufferChunksAmount: 5,
+  });
+
+  const transcriber = useTranscriber({
+    audioStreamTrack: track,
+    onSegments: handleSegments,
+    config: config.current,
+  });
+  const [isOpen, setIsOpen] = useState(false);
   useEffect(() => {
-    void transcriber?.open();
+    void transcriber?.open().then(() => setIsOpen(true));
+    log.info('transcriber open');
     return () => {
-      void transcriber?.close();
+      void transcriber?.close().then(() => setIsOpen(false));
+      log.info('transcriber close');
     };
   }, [transcriber]);
 
   // Manage transcription state.
   useEffect(() => {
-    if (playing && transcriber?.isOpen) {
-      void transcriber?.startChunksRecording();
-    } else if (!playing) {
-      void transcriber?.stopChunksRecording();
+    if (running && transcriber?.isOpen) {
+      transcriber?.startChunksRecording();
+      log.info('transcriber startChunksRecording');
+    } else if (!running) {
+      transcriber?.stopChunksRecording();
+      log.info('transcriber stopChunksRecording');
     }
-  }, [transcriber, playing, transcriber?.isOpen]);
+  }, [transcriber, running, isOpen]);
 
-  return <TranscriptionStory playing={playing} setPlaying={setPlaying} blocks={queue?.items} />;
+  return <TranscriptionStory model={model} running={running} onRunningChange={setRunning} />;
 };
 
 const AudioFile = ({ queueDxn, audioUrl }: { queueDxn: DXN; audioUrl: string; transcriptUrl: string }) => {
-  const [playing, setPlaying] = useState(false);
+  const [running, setRunning] = useState(false);
 
   // Audio.
   const audioElement = useRef<HTMLAudioElement>(null);
@@ -110,18 +133,19 @@ const AudioFile = ({ queueDxn, audioUrl }: { queueDxn: DXN; audioUrl: string; tr
       return;
     }
 
-    if (playing) {
+    if (running) {
       void audio.play();
     } else {
       void audio.pause();
     }
-  }, [audio, playing]);
+  }, [audio, running]);
 
   // Transcriber.
   const queue = useQueue<TranscriptBlock>(queueDxn, { pollInterval: 500 });
+  const model = useQueueModelAdapter(renderMarkdown, queue);
   const handleSegments = useCallback<TranscriberParams['onSegments']>(
     async (segments) => {
-      const block = createStatic(TranscriptBlock, { authorName: 'test', authorHue: 'cyan', segments });
+      const block = create(TranscriptBlock, { authorName: 'test', authorHue: 'cyan', segments });
       queue?.append([block]);
     },
     [queue],
@@ -133,15 +157,15 @@ const AudioFile = ({ queueDxn, audioUrl }: { queueDxn: DXN; audioUrl: string; tr
   });
 
   useEffect(() => {
-    if (transcriber && playing) {
-      void transcriber.open();
-    } else if (!playing) {
+    if (transcriber && running) {
+      void transcriber.open().then(manageChunkRecording);
+    } else if (!running) {
       transcriber?.stopChunksRecording();
       void transcriber?.close();
     }
-  }, [transcriber, playing]);
+  }, [transcriber, running]);
 
-  useEffect(() => {
+  const manageChunkRecording = () => {
     if (track?.readyState === 'live' && transcriber?.isOpen) {
       log.info('starting transcription');
       transcriber.startChunksRecording();
@@ -149,24 +173,49 @@ const AudioFile = ({ queueDxn, audioUrl }: { queueDxn: DXN; audioUrl: string; tr
       log.info('stopping transcription');
       transcriber.stopChunksRecording();
     }
+  };
+  useEffect(() => {
+    manageChunkRecording();
   }, [transcriber, track?.readyState, transcriber?.isOpen]);
 
-  return <TranscriptionStory playing={playing} setPlaying={setPlaying} blocks={queue?.items} />;
+  return <TranscriptionStory model={model} running={running} onRunningChange={setRunning} />;
 };
 
 const meta: Meta<typeof AudioFile> = {
   title: 'plugins/plugin-transcription/transcription',
   decorators: [
-    withClientProvider({
-      config: new Config({
-        runtime: {
-          client: { edgeFeatures: { signaling: true } },
-          services: {
-            edge: { url: 'https://edge.dxos.workers.dev/' },
-            iceProviders: [{ urls: 'https://edge.dxos.workers.dev/ice' }],
+    withPluginManager({
+      plugins: [
+        ThemePlugin({ tx: defaultTx }),
+        ClientPlugin({
+          types: [TestItem],
+          onClientInitialized: async (_, client) => {
+            await client.halo.createIdentity();
           },
-        },
-      }),
+        }),
+        SpacePlugin(),
+        SettingsPlugin(),
+        IntentPlugin(),
+        TranscriptionPlugin(),
+      ],
+      fireEvents: [Events.SetupAppGraph],
+      // capabilities: [
+      //   contributes(
+      //     Capabilities.ReactSurface,
+      //     createSurface({
+      //       id: 'preview-test',
+      //       role: 'preview',
+      //       component: ({ data }) => {
+      //         const schema = getSchema(data);
+      //         if (!schema) {
+      //           return null;
+      //         }
+
+      //         return <Form schema={schema} values={data} />;
+      //       },
+      //     }),
+      //   ),
+      // ],
     }),
     withTheme,
     withLayout({

package/src/entity-extraction/entity-extraction.test.ts

@@ -0,0 +1,32 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import { describe, test } from 'vitest';
+
+import { AIServiceEdgeClient } from '@dxos/assistant';
+import { AI_SERVICE_ENDPOINT } from '@dxos/assistant/testing';
+import { log } from '@dxos/log';
+
+import { processTranscriptBlock } from './entity-extraction';
+import * as TestData from '../testing/test-data';
+
+const aiService = new AIServiceEdgeClient({
+  endpoint: AI_SERVICE_ENDPOINT.REMOTE,
+});
+
+describe.skip('EntityExtraction', { timeout: 180_000 }, () => {
+  test('should process a transcript block', async () => {
+    for (const block of TestData.transcriptBlocks) {
+      log.info('input', block);
+      const { block: enhancedBlock } = await processTranscriptBlock({
+        block,
+        aiService,
+        context: {
+          objects: [...TestData.documents, ...Object.values(TestData.contacts)],
+        },
+      });
+      log.info('output', enhancedBlock);
+    }
+  });
+});

package/src/entity-extraction/entity-extraction.ts

@@ -0,0 +1,94 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import { Schema } from 'effect';
+
+import { createTemplate, Message, structuredOutputParser } from '@dxos/artifact';
+import type { AIServiceClient } from '@dxos/assistant';
+import { MixedStreamParser } from '@dxos/assistant';
+import { raise } from '@dxos/debug';
+import { create } from '@dxos/echo-schema';
+import type { BaseEchoObject } from '@dxos/echo-schema';
+
+import SYSTEM_PROMPT from './system-prompt.tpl?raw';
+import type { TranscriptBlock } from '../types';
+
+type ProcessTranscriptBlockParams = {
+  block: TranscriptBlock;
+  aiService: AIServiceClient;
+  context: {
+    objects?: BaseEchoObject[];
+  };
+};
+
+type ProcessTranscriptBlockResult = {
+  block: TranscriptBlock;
+};
+
+/**
+ * Process Handlebars template.
+ */
+const createSystemPrompt = (): string => {
+  const template = createTemplate(SYSTEM_PROMPT);
+  return template({});
+};
+
+export const processTranscriptBlock = async (
+  params: ProcessTranscriptBlockParams,
+): Promise<ProcessTranscriptBlockResult> => {
+  // TODO(dmaretskyi): Move context to a vector search index.
+  const systemPrompt = `
+    ${createSystemPrompt()}
+    Context:
+    ${JSON.stringify(params.context.objects)}
+  `;
+
+  const outputParser = structuredOutputParser(
+    Schema.Struct({
+      segments: Schema.Array(Schema.String).annotations({
+        description: 'The enhanced text of the transcript segments, keep the order and structure exactly as is',
+      }),
+    }),
+  );
+  const result = outputParser.getResult(
+    await new MixedStreamParser().parse(
+      await params.aiService.execStream({
+        model: '@anthropic/claude-3-5-haiku-20241022',
+        systemPrompt,
+        history: [
+          create(Message, {
+            role: 'user',
+            content: [
+              {
+                type: 'text',
+                text: JSON.stringify(params.block),
+              },
+            ],
+          }),
+        ],
+        tools: [outputParser.tool],
+      }),
+    ),
+  );
+
+  return {
+    block: {
+      ...params.block,
+      segments: params.block.segments.map((segment, i) => ({
+        ...segment,
+        text: postprocessText(result?.segments[i] ?? raise(new Error('failed to process email'))),
+      })),
+    },
+  };
+};
+
+/**
+ * Finds and replaces all inline references with DXNs references.
+ */
+// TODO(dmaretskyi): Lookup and verifiy ids from provided context.
+const postprocessText = (text: string) => {
+  return text.replace(/\n/g, ' ').replace(/\[([^\]]+)\]\[([A-Z0-9]+)\]/g, (match, name, id) => {
+    return `[${name}][dxn:echo:@:${id}]`;
+  });
+};

package/src/entity-extraction/index.ts

@@ -0,0 +1,5 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+export * from './entity-extraction';

package/src/entity-extraction/system-prompt.tpl

@@ -0,0 +1,9 @@
+  You are a helpful assistant that processes transcripts.
+  Your goal is to enhance the transcript with entity references.
+  Insert inline references to the entities in the transcript.
+  Keep the transcript structure and text as is.
+  Call the submit_result tool to submit your result.
+
+  The inline reference syntax is as follows:
+   - [<optional name>][<ID>] or just [<ID>]
+   - Example: [Earnings Report][01JT0JP9AX0XKGZX4MV4B69VT6]

package/src/entity-extraction/types.d.ts

@@ -0,0 +1,8 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+declare module '*.tpl?raw' {
+  const content: string;
+  export default content;
+}

package/src/hooks/index.ts

@@ -4,5 +4,6 @@
 
 export * from './useAudioFile';
 export * from './useAudioTrack';
+export * from './useQueueModelAdapter';
 export * from './useTranscriber';
 export * from './useVoiceInput';

package/src/hooks/useQueueModelAdapter.ts

@@ -0,0 +1,30 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import { useMemo, useEffect } from 'react';
+
+import { type Queue } from '@dxos/react-client/echo';
+
+import { type Block, BlockModel, type BlockRenderer } from '../model';
+
+/**
+ * Model adapter for a queue.
+ */
+export const useQueueModelAdapter = <T extends Block>(
+  renderer: BlockRenderer<T>,
+  queue: Queue<T> | undefined,
+  initialBlocks: T[] = [],
+): BlockModel<T> => {
+  const model = useMemo(() => new BlockModel<T>(renderer, initialBlocks), [queue]);
+  useEffect(() => {
+    if (!queue?.items.length) {
+      return;
+    }
+
+    const block = queue.items[queue.items.length - 1];
+    model.appendBlock(block);
+  }, [model, queue?.items.length]);
+
+  return model;
+};

package/src/hooks/useTranscriber.ts

@@ -15,6 +15,7 @@ import { type Transcriber } from '../transcriber';
 export const useTranscriber = ({
   audioStreamTrack,
   onSegments,
+  config,
 }: Partial<TranscriptionCapabilities.GetTranscriberProps>) => {
   const [getTranscriber] = useCapabilities(TranscriptionCapabilities.Transcriber);
 
@@ -24,8 +25,8 @@ export const useTranscriber = ({
       return undefined;
     }
 
-    return getTranscriber({ audioStreamTrack, onSegments });
-  }, [audioStreamTrack, onSegments, getTranscriber]);
+    return getTranscriber({ audioStreamTrack, onSegments, config });
+  }, [audioStreamTrack, onSegments, getTranscriber, config]);
 
   useEffect(() => {
     return () => {

package/src/model/index.ts

@@ -0,0 +1,5 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+export * from './model';

package/src/model/model.test.ts

@@ -0,0 +1,123 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import { EditorView } from '@codemirror/view';
+import { describe, test } from 'vitest';
+
+import { BlockModel, DocumentAdapter, type BlockRenderer } from './model';
+import { type TranscriptBlock } from '../types';
+
+const blockToMarkdown: BlockRenderer<TranscriptBlock> = (block: TranscriptBlock, debug = true): string[] => {
+  return [`###### ${block.authorName}`, ...block.segments.map((segment) => segment.text), ''];
+};
+
+const createDate = () => new Date().toISOString();
+
+describe('BlockModel', () => {
+  test('basic', ({ expect }) => {
+    const model = new BlockModel<TranscriptBlock>(blockToMarkdown);
+    expect(model.blocks.length).to.eq(0);
+    expect(model.doc.toString()).to.eq('');
+
+    // Create block.
+    const block = {
+      id: '1',
+      authorName: 'Alice',
+      segments: [{ started: createDate(), text: 'Hello world!' }],
+    };
+    model.appendBlock(block);
+    {
+      const text = model.doc.toString();
+      expect(text).to.eq('###### Alice\nHello world!\n\n');
+      expect(model.blocks.length).to.eq(1);
+    }
+
+    // Update block.
+    block.segments.push({ started: new Date().toISOString(), text: 'Hello again!' });
+    model.updateBlock(block);
+    {
+      const text = model.doc.toString();
+      expect(text).to.eq('###### Alice\nHello world!\nHello again!\n\n');
+      expect(model.blocks.length).to.eq(1);
+    }
+  });
+
+  test('sync - append', ({ expect }) => {
+    const view = new EditorView({ extensions: [], doc: '' });
+    const model = new BlockModel<TranscriptBlock>(blockToMarkdown);
+    const adapter = new DocumentAdapter(view);
+    expect(adapter.lineCount()).to.eq(0);
+    expect(view.state.doc.toString()).to.eq('');
+
+    // Append block.
+    {
+      const block = {
+        id: '1',
+        authorName: 'Alice',
+        segments: [{ started: createDate(), text: 'Hello world!' }],
+      };
+      model.appendBlock(block);
+      model.sync(adapter);
+      expect(view.state.doc.toString()).to.eq('###### Alice\nHello world!\n\n');
+    }
+
+    // Append block.
+    {
+      const block = {
+        id: '2',
+        authorName: 'Bob',
+        segments: [{ started: createDate(), text: 'Hello world!' }],
+      };
+      model.appendBlock(block);
+      model.sync(adapter);
+      expect(view.state.doc.toString()).to.eq('###### Alice\nHello world!\n\n###### Bob\nHello world!\n\n');
+    }
+  });
+
+  test('sync - append, update, delete', ({ expect }) => {
+    const view = new EditorView({ extensions: [], doc: '' });
+    const model = new BlockModel<TranscriptBlock>(blockToMarkdown);
+    const adapter = new DocumentAdapter(view);
+    expect(adapter.lineCount()).to.eq(0);
+
+    // Append block.
+    {
+      const block = { id: '1', authorName: 'Alice', segments: [{ started: createDate(), text: 'Hello world!' }] };
+      model.appendBlock(block);
+      model.sync(adapter);
+      expect(view.state.doc.toString()).to.eq('###### Alice\nHello world!\n\n');
+
+      // Update block (add segment).
+      block.segments.push({ started: createDate(), text: 'Hello again!' });
+      model.updateBlock(block);
+      model.sync(adapter);
+      expect(view.state.doc.toString()).to.eq('###### Alice\nHello world!\nHello again!\n\n');
+
+      // Update block (remove segment).
+      block.segments.splice(0, 1);
+      model.updateBlock(block);
+      model.sync(adapter);
+      expect(view.state.doc.toString()).to.eq('###### Alice\nHello again!\n\n');
+
+      expect(model.getBlockAtLine(1)).to.eq(block);
+    }
+
+    // Append block.
+    {
+      const block = { id: '2', authorName: 'Bob', segments: [{ started: createDate(), text: 'Hello again!' }] };
+      model.appendBlock(block);
+      model.sync(adapter);
+      expect(view.state.doc.toString()).to.eq('###### Alice\nHello again!\n\n###### Bob\nHello again!\n\n');
+
+      expect(model.getBlockAtLine(4)).to.eq(block);
+    }
+
+    // Delete block.
+    {
+      model.deleteBlock('1');
+      model.sync(adapter);
+      expect(view.state.doc.toString()).to.eq('###### Bob\nHello again!\n\n');
+    }
+  });
+});