@dxos/plugin-transcription 0.8.3 → 0.8.4-main.1da679c

package/src/components/Transcript/transcript-extension.ts

@@ -8,7 +8,6 @@ import { format } from 'date-fns/format';
 import { intervalToDuration } from 'date-fns/intervalToDuration';
 
 import { type CleanupFn, addEventListener, combine } from '@dxos/async';
-import { type RenderCallback } from '@dxos/react-ui-editor';
 import { type DataType } from '@dxos/schema';
 
 import { DocumentAdapter, type SerializationModel } from '../../model';
@@ -19,13 +18,12 @@ import { DocumentAdapter, type SerializationModel } from '../../model';
 export type TranscriptOptions = {
   model: SerializationModel<DataType.Message>;
   started?: Date;
-  renderButton?: RenderCallback<{ onClick: () => void }>;
 };
 
 /**
  * Scrolling transcript with timestamps.
  */
-export const transcript = ({ model, started, renderButton }: TranscriptOptions): Extension => {
+export const transcript = ({ model, started }: TranscriptOptions): Extension => {
   return [
     // Show timestamps in the gutter.
     gutter({
@@ -38,7 +36,7 @@ export const transcript = ({ model, started, renderButton }: TranscriptOptions):
           let line = view.state.doc.lineAt(from);
           while (line.from <= to) {
             const block = model.getChunkAtLine(line.number)?.blocks[0];
-            const timestamp = block?.type === 'transcription' && block.started;
+            const timestamp = block?._tag === 'transcript' && block.started;
             if (timestamp) {
               builder.add(line.from, line.from, new TimestampMarker(line.number, new Date(timestamp), start));
             }
@@ -90,22 +88,6 @@ export const transcript = ({ model, started, renderButton }: TranscriptOptions):
             });
           };
 
-          // Scroll button.
-          if (renderButton) {
-            this._controls = document.createElement('div');
-            this._controls.classList.add('cm-controls', 'transition-opacity', 'duration-300', 'opacity-0');
-            view.dom.appendChild(this._controls);
-            renderButton(
-              this._controls,
-              {
-                onClick: () => {
-                  scrollToBottom(false);
-                },
-              },
-              view,
-            );
-          }
-
           // Event listeners.
           this._cleanup = combine(
             addEventListener(view.scrollDOM, 'scroll', () => {
@@ -131,7 +113,7 @@ export const transcript = ({ model, started, renderButton }: TranscriptOptions):
           );
         }
 
-        update(update: ViewUpdate) {
+        update(_update: ViewUpdate) {
           // Initial sync.
           if (!this._initialized) {
             this._initialized = true;
@@ -149,25 +131,6 @@ export const transcript = ({ model, started, renderButton }: TranscriptOptions):
     ),
 
     EditorView.theme({
-      '.cm-scroller': {
-        overflowY: 'scroll',
-      },
-      '.cm-hide-scrollbar': {
-        scrollbarWidth: 'none',
-        '-ms-overflow-style': 'none',
-      },
-      '.cm-hide-scrollbar::-webkit-scrollbar': {
-        display: 'none',
-      },
-      '.cm-controls': {
-        position: 'absolute',
-        bottom: '0.5rem',
-        right: '0.5rem',
-        zIndex: 1000,
-      },
-      '.cm-line': {
-        paddingRight: '1rem',
-      },
       '.cm-timestamp-gutter': {
         width: '6rem',
         paddingRight: '1rem',
@@ -218,7 +181,7 @@ const getStartTime = (started?: Date, message?: DataType.Message): Date | undefi
     return started;
   }
 
-  if (message?.blocks[0]?.type === 'transcription' && message.blocks[0].started) {
+  if (message?.blocks[0]?._tag === 'transcript' && message.blocks[0].started) {
     return new Date(message.blocks[0].started);
   }
 

package/src/components/TranscriptContainer.tsx

@@ -2,27 +2,33 @@
 // Copyright 2024 DXOS.org
 //
 
-import React, { type FC } from 'react';
+import React from 'react';
 
 import { fullyQualifiedId, getSpace } from '@dxos/client/echo';
 import { useMembers, useQueue } from '@dxos/react-client/echo';
 import { StackItem } from '@dxos/react-ui-stack';
 import { type DataType } from '@dxos/schema';
 
-import { Transcript, renderMarkdown } from './Transcript';
 import { useQueueModelAdapter } from '../hooks';
-import { type TranscriptType } from '../types';
+import { type Transcript } from '../types';
 
-export const TranscriptionContainer: FC<{ role: string; transcript: TranscriptType }> = ({ role, transcript }) => {
+import { TranscriptView, renderByline } from './Transcript';
+
+export type TranscriptionContainerProps = {
+  role: string;
+  transcript: Transcript.Transcript;
+};
+
+export const TranscriptionContainer = ({ transcript }: TranscriptionContainerProps) => {
   const attendableId = fullyQualifiedId(transcript);
   const space = getSpace(transcript);
   const members = useMembers(space?.key).map((member) => member.identity);
   const queue = useQueue<DataType.Message>(transcript.queue.dxn, { pollInterval: 1_000 });
-  const model = useQueueModelAdapter(renderMarkdown(members), queue);
+  const model = useQueueModelAdapter(renderByline(members), queue);
 
   return (
-    <StackItem.Content role={role} classNames='container-max-width pli-2'>
-      <Transcript attendableId={attendableId} space={space} model={model} transcript={transcript} />
+    <StackItem.Content>
+      <TranscriptView attendableId={attendableId} space={space} model={model} transcript={transcript} />
     </StackItem.Content>
   );
 };

package/src/components/stories/FileTranscription.stories.tsx

@@ -4,11 +4,9 @@
 
 import '@dxos-theme';
 
-import { type Meta, type StoryObj } from '@storybook/react';
+import { type Meta, type StoryObj } from '@storybook/react-vite';
 import React, { useCallback, useEffect, useMemo, useRef, useState } from 'react';
 
-import { EdgeAiServiceClient } from '@dxos/ai';
-import { AI_SERVICE_ENDPOINT } from '@dxos/ai/testing';
 import { Events, IntentPlugin, SettingsPlugin } from '@dxos/app-framework';
 import { withPluginManager } from '@dxos/app-framework/testing';
 import { scheduleTask } from '@dxos/async';
@@ -25,17 +23,18 @@ import { StorybookLayoutPlugin } from '@dxos/plugin-storybook-layout';
 import { ThemePlugin } from '@dxos/plugin-theme';
 import { defaultTx } from '@dxos/react-ui-theme';
 import { DataType } from '@dxos/schema';
-import { seedTestData, Testing } from '@dxos/schema/testing';
+import { Testing, seedTestData } from '@dxos/schema/testing';
 import { withLayout, withTheme } from '@dxos/storybook-utils';
 
-import { TranscriptionStory } from './TranscriptionStory';
-import { useIsSpeaking } from './useIsSpeaking';
-import { TranscriptionPlugin } from '../../TranscriptionPlugin';
 import { useAudioFile, useQueueModelAdapter, useTranscriber } from '../../hooks';
-import { getActorId, MessageNormalizer } from '../../segments-normalization';
+import { MessageNormalizer, getActorId } from '../../segments-normalization';
 import { TestItem } from '../../testing';
 import { type MediaStreamRecorderParams, type TranscriberParams } from '../../transcriber';
-import { renderMarkdown } from '../Transcript';
+import { TranscriptionPlugin } from '../../TranscriptionPlugin';
+import { renderByline } from '../Transcript';
+
+import { TranscriptionStory } from './TranscriptionStory';
+import { useIsSpeaking } from './useIsSpeaking';
 
 const AudioFile = ({
   detectSpeaking,
@@ -69,7 +68,6 @@ const AudioFile = ({
 
   useEffect(() => {
     if (!audio) {
-      log.warn('no audio');
       return;
     }
 
@@ -85,11 +83,16 @@ const AudioFile = ({
   const queueDxn = useMemo(() => createQueueDXN(), []);
   const queue = useMemo(() => new MemoryQueue<DataType.Message>(queueDxn), [queueDxn]);
 
-  const model = useQueueModelAdapter(renderMarkdown([]), queue);
+  const model = useQueueModelAdapter(renderByline([]), queue);
   const handleSegments = useCallback<TranscriberParams['onSegments']>(
     async (blocks) => {
-      const message = Obj.make(DataType.Message, { sender: actor, created: new Date().toISOString(), blocks });
-      void queue?.append([message]);
+      void queue?.append([
+        Obj.make(DataType.Message, {
+          created: new Date().toISOString(),
+          sender: actor,
+          blocks,
+        }),
+      ]);
     },
     [queue],
   );
@@ -108,14 +111,14 @@ const AudioFile = ({
     }
     const executor = new FunctionExecutor(
       new ServiceContainer().setServices({
-        ai: {
-          client: new EdgeAiServiceClient({
-            endpoint: AI_SERVICE_ENDPOINT.REMOTE,
-            defaultGenerationOptions: {
-              model: '@anthropic/claude-3-5-sonnet-20241022',
-            },
-          }),
-        },
+        // ai: {
+        //   client: new Edge AiServiceClient({
+        //     endpoint: AI_SERVICE_ENDPOINT.REMOTE,
+        //     defaultGenerationOptions: {
+        //       model: '@anthropic/claude-3-5-sonnet-20241022',
+        //     },
+        //   }),
+        // },
       }),
     );
 
@@ -143,10 +146,10 @@ const AudioFile = ({
   }, [normalizer]);
 
   const manageChunkRecording = () => {
-    if (running && isSpeaking) {
+    if (running && isSpeaking && transcriber) {
       log.info('starting transcription');
       transcriber?.startChunksRecording();
-    } else if (!isSpeaking || !running) {
+    } else if ((!isSpeaking || !running) && transcriber) {
       log.info('stopping transcription');
       transcriber?.stopChunksRecording();
     }
@@ -172,10 +175,18 @@ const AudioFile = ({
     manageChunkRecording();
   }, [isSpeaking, stream]);
 
-  return <TranscriptionStory model={model} running={running} onRunningChange={setRunning} audioRef={ref} />;
+  return (
+    <TranscriptionStory
+      disabled={!stream}
+      model={model}
+      running={running}
+      onRunningChange={setRunning}
+      audioRef={ref}
+    />
+  );
 };
 
-const meta: Meta<typeof AudioFile> = {
+const meta = {
   title: 'plugins/plugin-transcription/FileTranscription',
   decorators: [
     withPluginManager({
@@ -184,7 +195,7 @@ const meta: Meta<typeof AudioFile> = {
         StorybookLayoutPlugin(),
         ClientPlugin({
           types: [TestItem, DataType.Person, DataType.Organization, Testing.DocumentType],
-          onClientInitialized: async (_, client) => {
+          onClientInitialized: async ({ client }) => {
             await client.halo.createIdentity();
             await client.spaces.waitUntilReady();
             await client.spaces.default.waitUntilReady();
@@ -202,10 +213,12 @@ const meta: Meta<typeof AudioFile> = {
     withTheme,
     withLayout({ fullscreen: true, classNames: 'justify-center' }),
   ],
-};
+} satisfies Meta;
 
 export default meta;
 
+type Story = StoryObj<typeof meta>;
+
 const TRANSCRIBER_CONFIG = {
   transcribeAfterChunksAmount: 100,
   prefixBufferChunksAmount: 50,
@@ -228,15 +241,16 @@ export const Default: StoryObj<typeof AudioFile> = {
   },
 };
 
-export const WithSentenceNormalization: StoryObj<typeof AudioFile> = {
-  render: AudioFile,
-  args: {
-    detectSpeaking: true,
-    normalizeSentences: true,
-    // https://learnenglish.britishcouncil.org/general-english/audio-zone/living-london
-    audioUrl: 'https://dxos.network/audio-london.m4a',
-    // textUrl: 'https://dxos.network/audio-london.txt',
-    transcriberConfig: TRANSCRIBER_CONFIG,
-    recorderConfig: RECORDER_CONFIG,
-  },
-};
+// TODO(mykola): Fix sentence normalization.
+// export const WithSentenceNormalization: StoryObj<typeof AudioFile> = {
+//   render: AudioFile,
+//   args: {
+//     detectSpeaking: true,
+//     normalizeSentences: true,
+//     // https://learnenglish.britishcouncil.org/general-english/audio-zone/living-london
+//     audioUrl: 'https://dxos.network/audio-london.m4a',
+//     // textUrl: 'https://dxos.network/audio-london.txt',
+//     transcriberConfig: TRANSCRIBER_CONFIG,
+//     recorderConfig: RECORDER_CONFIG,
+//   },
+// };

package/src/components/stories/KeyWordRecognition.stories.tsx

@@ -4,7 +4,7 @@
 
 import '@dxos-theme';
 
-import { type StoryObj, type Meta } from '@storybook/react';
+import { type Meta, type StoryObj } from '@storybook/react-vite';
 import React, { useEffect, useMemo, useState } from 'react';
 
 import { log } from '@dxos/log';
@@ -17,7 +17,11 @@ type Word = {
   matched: boolean;
 };
 
-const SpeechRecognition = ({ keywords }: { keywords: string[] }) => {
+type StoryProps = {
+  keywords: string[];
+};
+
+const DefaultStory = ({ keywords }: StoryProps) => {
   const [running, setRunning] = useState(false);
   const [matchingWords, setMatchingWords] = useState<Word[]>(keywords.map((word) => ({ text: word, matched: false })));
 
@@ -113,15 +117,17 @@ const SpeechRecognition = ({ keywords }: { keywords: string[] }) => {
   );
 };
 
-const meta: Meta<typeof SpeechRecognition> = {
+const meta = {
   title: 'plugins/plugin-transcription/KeyWordDetection',
+  render: DefaultStory,
   decorators: [withTheme, withLayout({ fullscreen: true, classNames: 'justify-center' })],
-};
+} satisfies Meta<typeof DefaultStory>;
 
 export default meta;
 
-export const SpeechRecognitionAPI: StoryObj<typeof SpeechRecognition> = {
-  render: SpeechRecognition,
+type Story = StoryObj<typeof meta>;
+
+export const SpeechRecognitionAPI: Story = {
   args: {
     keywords: ['kai', 'computer', 'hello'],
   },

package/src/components/stories/MicrophoneTranscription.stories.tsx

@@ -4,20 +4,18 @@
 
 import '@dxos-theme';
 
-import { type Meta, type StoryObj } from '@storybook/react';
+import { type Meta, type StoryObj } from '@storybook/react-vite';
 import React, { useCallback, useEffect, useMemo, useState } from 'react';
 
-import { EdgeAiServiceClient } from '@dxos/ai';
-import { AI_SERVICE_ENDPOINT } from '@dxos/ai/testing';
 import { Events, IntentPlugin, SettingsPlugin } from '@dxos/app-framework';
 import { withPluginManager } from '@dxos/app-framework/testing';
 import {
-  extractionAnthropicFn,
   type ExtractionFunction,
-  extractionNerFn,
-  processTranscriptMessage,
+  extractionAnthropicFunction,
+  extractionNerFunction,
   getNer,
-} from '@dxos/assistant';
+  processTranscriptMessage,
+} from '@dxos/assistant/extraction';
 import { Filter, Obj, type Type } from '@dxos/echo';
 import { MemoryQueue } from '@dxos/echo-db';
 import { createQueueDXN } from '@dxos/echo-schema';
@@ -32,16 +30,17 @@ import { ThemePlugin } from '@dxos/plugin-theme';
 import { IndexKind, useSpace } from '@dxos/react-client/echo';
 import { defaultTx } from '@dxos/react-ui-theme';
 import { DataType } from '@dxos/schema';
-import { seedTestData, Testing } from '@dxos/schema/testing';
+import { Testing, seedTestData } from '@dxos/schema/testing';
 import { withLayout, withTheme } from '@dxos/storybook-utils';
 
-import { TranscriptionStory } from './TranscriptionStory';
-import { useIsSpeaking } from './useIsSpeaking';
-import { TranscriptionPlugin } from '../../TranscriptionPlugin';
 import { useAudioTrack, useQueueModelAdapter, useTranscriber } from '../../hooks';
 import { TestItem } from '../../testing';
 import { type MediaStreamRecorderParams, type TranscriberParams } from '../../transcriber';
-import { renderMarkdown } from '../Transcript';
+import { TranscriptionPlugin } from '../../TranscriptionPlugin';
+import { renderByline } from '../Transcript';
+
+import { TranscriptionStory } from './TranscriptionStory';
+import { useIsSpeaking } from './useIsSpeaking';
 
 const TRANSCRIBER_CONFIG = {
   transcribeAfterChunksAmount: 50,
@@ -79,7 +78,7 @@ const DefaultStory = ({
   // TODO(dmaretskyi): Use space.queues.create() instead.
   const queueDxn = useMemo(() => createQueueDXN(), []);
   const queue = useMemo(() => new MemoryQueue<DataType.Message>(queueDxn), [queueDxn]);
-  const model = useQueueModelAdapter(renderMarkdown([]), queue);
+  const model = useQueueModelAdapter(renderByline([]), queue);
   const space = useSpace();
 
   useEffect(() => {
@@ -102,9 +101,9 @@ const DefaultStory = ({
     if (entityExtraction === 'ner') {
       // Init model loading. Takes time.
       void getNer();
-      extractionFunction = extractionNerFn;
+      extractionFunction = extractionNerFunction;
     } else if (entityExtraction === 'llm') {
-      extractionFunction = extractionAnthropicFn;
+      extractionFunction = extractionAnthropicFunction;
       objects = space.db
         .query(
           Filter.or(
@@ -117,11 +116,15 @@ const DefaultStory = ({
         .then((result) => result.objects);
     }
     if (entityExtraction !== 'none') {
-      const AiService = new EdgeAiServiceClient({
-        endpoint: AI_SERVICE_ENDPOINT.REMOTE,
-      });
       executor = new FunctionExecutor(
-        new ServiceContainer().setServices({ ai: { client: AiService }, database: { db: space!.db } }),
+        new ServiceContainer().setServices({
+          // ai: {
+          //   client: new Edge AiServiceClient({
+          //     endpoint: AI_SERVICE_ENDPOINT.REMOTE,
+          //   }),
+          // },
+          // database: { db: space!.db },
+        }),
       );
     }
 
@@ -190,7 +193,7 @@ const DefaultStory = ({
   return <TranscriptionStory model={model} running={running} onRunningChange={setRunning} />;
 };
 
-const meta: Meta<typeof DefaultStory> = {
+const meta = {
   title: 'plugins/plugin-transcription/MicrophoneTranscription',
   render: DefaultStory,
   decorators: [
@@ -200,7 +203,7 @@ const meta: Meta<typeof DefaultStory> = {
         StorybookLayoutPlugin(),
         ClientPlugin({
           types: [TestItem, DataType.Person, DataType.Organization, Testing.DocumentType],
-          onClientInitialized: async (_, client) => {
+          onClientInitialized: async ({ client }) => {
             await client.halo.createIdentity();
             await client.spaces.waitUntilReady();
             await client.spaces.default.waitUntilReady();
@@ -231,11 +234,11 @@ const meta: Meta<typeof DefaultStory> = {
     withTheme,
     withLayout({ fullscreen: true, classNames: 'justify-center' }),
   ],
-};
+} satisfies Meta<typeof DefaultStory>;
 
 export default meta;
 
-type Story = StoryObj<typeof DefaultStory>;
+type Story = StoryObj<typeof meta>;
 
 export const Default: Story = {
   args: {
@@ -251,19 +254,21 @@ export const Default: Story = {
   },
 };
 
-export const EntityExtraction: Story = {
-  args: {
-    detectSpeaking: true,
-    entityExtraction: 'ner',
-    transcriberConfig: TRANSCRIBER_CONFIG,
-    recorderConfig: RECORDER_CONFIG,
-    audioConstraints: {
-      echoCancellation: true,
-      noiseSuppression: true,
-      autoGainControl: true,
-    },
-  },
-};
+// NOTE: We are running out of free quota on hugging face entity extraction.
+// TODO(mykola): Fix hugging face quota issues.
+// export const EntityExtraction: Story = {
+//   args: {
+//     detectSpeaking: true,
+//     entityExtraction: 'ner',
+//     transcriberConfig: TRANSCRIBER_CONFIG,
+//     recorderConfig: RECORDER_CONFIG,
+//     audioConstraints: {
+//       echoCancellation: true,
+//       noiseSuppression: true,
+//       autoGainControl: true,
+//     },
+//   },
+// };
 
 export const SpeechDetection: Story = {
   args: {

package/src/components/stories/TranscriptionStory.tsx

@@ -12,14 +12,15 @@ import { ScrollContainer } from '@dxos/react-ui-components';
 import { type DataType } from '@dxos/schema';
 
 import { type SerializationModel } from '../../model';
-import { Transcript } from '../Transcript';
+import { TranscriptView } from '../Transcript';
 
 export const TranscriptionStory: FC<{
   model: SerializationModel<DataType.Message>;
+  disabled?: boolean;
   running: boolean;
   onRunningChange: Dispatch<SetStateAction<boolean>>;
   audioRef?: RefObject<HTMLAudioElement>;
-}> = ({ model, running, onRunningChange, audioRef }) => {
+}> = ({ model, running, onRunningChange, audioRef, disabled }) => {
   const space = useSpace();
 
   return (
@@ -28,14 +29,17 @@ export const TranscriptionStory: FC<{
       <Toolbar.Root>
         <IconButton
           iconOnly
+          disabled={disabled}
           icon={running ? 'ph--pause--regular' : 'ph--play--regular'}
           label={running ? 'Pause' : 'Play'}
           onClick={() => onRunningChange((running) => !running)}
         />
       </Toolbar.Root>
-      <ScrollContainer>
-        <Transcript space={space} model={model} attendableId='story' />
-      </ScrollContainer>
+      <ScrollContainer.Root pin>
+        <ScrollContainer.Content>
+          <TranscriptView space={space} model={model} />
+        </ScrollContainer.Content>
+      </ScrollContainer.Root>
     </div>
   );
 };

package/src/functions/index.ts

@@ -0,0 +1,6 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+export { default as open } from './open';
+export { default as summarize } from './summarize';

package/src/functions/open.ts

@@ -0,0 +1,37 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import { Effect, Schema } from 'effect';
+
+import { ArtifactId } from '@dxos/assistant';
+import { Obj } from '@dxos/echo';
+import { DatabaseService, QueueService, defineFunction } from '@dxos/functions';
+import { DataType } from '@dxos/schema';
+
+import { renderByline } from '../components';
+import { Transcript } from '../types';
+
+export default defineFunction({
+  name: 'dxos.org/function/transcription/open',
+  description: 'Opens and reads the contents of a transcription object.',
+  inputSchema: Schema.Struct({
+    id: ArtifactId.annotations({
+      description: 'The ID of the transcription object.',
+    }),
+  }),
+  outputSchema: Schema.Struct({
+    content: Schema.String,
+  }),
+  handler: Effect.fn(function* ({ data: { id } }) {
+    const transcript = yield* DatabaseService.resolve(ArtifactId.toDXN(id), Transcript.Transcript);
+    const { dxn } = yield* Effect.promise(() => transcript.queue.load());
+    const queue = yield* QueueService.getQueue(dxn);
+    yield* Effect.promise(() => queue?.queryObjects());
+    const content = queue?.objects
+      .filter((message) => Obj.instanceOf(DataType.Message, message))
+      .flatMap((message, index) => renderByline([])(message, index))
+      .join('\n\n');
+    return { content };
+  }),
+});