@dxos/plugin-transcription 0.8.4-main.ae835ea → 0.8.4-main.bc2380dfbc

package/src/components/stories/FileTranscription.stories.tsx

@@ -1,256 +0,0 @@
-//
-// Copyright 2025 DXOS.org
-//
-
-import { type Meta, type StoryObj } from '@storybook/react-vite';
-import React, { useCallback, useEffect, useMemo, useRef, useState } from 'react';
-
-import { Events, IntentPlugin, SettingsPlugin } from '@dxos/app-framework';
-import { withPluginManager } from '@dxos/app-framework/testing';
-import { scheduleTask } from '@dxos/async';
-import { Context } from '@dxos/context';
-import { Obj } from '@dxos/echo';
-import { createQueueDXN } from '@dxos/echo/internal';
-import { MemoryQueue } from '@dxos/echo-db';
-import { FunctionExecutor, ServiceContainer } from '@dxos/functions';
-import { log } from '@dxos/log';
-import { ClientPlugin } from '@dxos/plugin-client';
-import { PreviewPlugin } from '@dxos/plugin-preview';
-import { SpacePlugin } from '@dxos/plugin-space';
-import { StorybookLayoutPlugin } from '@dxos/plugin-storybook-layout';
-import { ThemePlugin } from '@dxos/plugin-theme';
-import { withLayout, withTheme } from '@dxos/react-ui/testing';
-import { defaultTx } from '@dxos/react-ui-theme';
-import { DataType } from '@dxos/schema';
-import { Testing, seedTestData } from '@dxos/schema/testing';
-
-import { useAudioFile, useQueueModelAdapter, useTranscriber } from '../../hooks';
-import { MessageNormalizer, getActorId } from '../../segments-normalization';
-import { TestItem } from '../../testing';
-import { type MediaStreamRecorderParams, type TranscriberParams } from '../../transcriber';
-import { TranscriptionPlugin } from '../../TranscriptionPlugin';
-import { renderByline } from '../Transcript';
-
-import { TranscriptionStory } from './TranscriptionStory';
-import { useIsSpeaking } from './useIsSpeaking';
-
-const AudioFile = ({
-  detectSpeaking,
-  audioUrl,
-  normalizeSentences,
-  transcriberConfig,
-  recorderConfig,
-  audioConstraints,
-}: {
-  detectSpeaking?: boolean;
-  normalizeSentences?: boolean;
-  audioUrl: string;
-  transcriberConfig?: TranscriberParams['config'];
-  recorderConfig?: MediaStreamRecorderParams['config'];
-  audioConstraints?: MediaTrackConstraints;
-}) => {
-  const [running, setRunning] = useState(false);
-  const actor: DataType.Actor = useMemo(() => ({ name: 'You' }), []);
-
-  // Audio.
-  const { audio, track, stream } = useAudioFile(audioUrl, audioConstraints);
-
-  // Speaking.
-  const isSpeaking = detectSpeaking ? useIsSpeaking(track) : true;
-  const ref = useRef<HTMLAudioElement>(null);
-  useEffect(() => {
-    if (ref.current && stream) {
-      ref.current.srcObject = stream;
-    }
-  }, [stream]);
-
-  useEffect(() => {
-    if (!audio) {
-      return;
-    }
-
-    if (running) {
-      void audio.play();
-    } else {
-      void audio.pause();
-    }
-  }, [audio, running]);
-
-  // Transcriber.
-  // TODO(dmaretskyi): Use space.queues.create() instead.
-  const queueDxn = useMemo(() => createQueueDXN(), []);
-  const queue = useMemo(() => new MemoryQueue<DataType.Message>(queueDxn), [queueDxn]);
-
-  const model = useQueueModelAdapter(renderByline([]), queue);
-  const handleSegments = useCallback<TranscriberParams['onSegments']>(
-    async (blocks) => {
-      void queue?.append([
-        Obj.make(DataType.Message, {
-          created: new Date().toISOString(),
-          sender: actor,
-          blocks,
-        }),
-      ]);
-    },
-    [queue],
-  );
-
-  const transcriber = useTranscriber({
-    audioStreamTrack: track,
-    onSegments: handleSegments,
-    transcriberConfig,
-    recorderConfig,
-  });
-
-  // Normalize sentences.
-  const normalizer = useMemo(() => {
-    if (!normalizeSentences) {
-      return;
-    }
-    const executor = new FunctionExecutor(
-      new ServiceContainer().setServices({
-        // ai: {
-        //   client: new Edge AiServiceClient({
-        //     endpoint: AI_SERVICE_ENDPOINT.REMOTE,
-        //     defaultGenerationOptions: {
-        //       model: '@anthropic/claude-3-5-sonnet-20241022',
-        //     },
-        //   }),
-        // },
-      }),
-    );
-
-    return new MessageNormalizer({
-      functionExecutor: executor,
-      queue,
-      startingCursor: { actorId: getActorId(actor), timestamp: new Date().toISOString() },
-    });
-  }, [normalizeSentences, queue, actor]);
-
-  useEffect(() => {
-    if (!normalizer) {
-      return;
-    }
-    const ctx = new Context();
-    scheduleTask(ctx, async () => {
-      await normalizer.open();
-      ctx.onDispose(async () => {
-        await normalizer.close();
-      });
-    });
-    return () => {
-      void ctx.dispose();
-    };
-  }, [normalizer]);
-
-  const manageChunkRecording = () => {
-    if (running && isSpeaking && transcriber) {
-      log.info('starting transcription');
-      transcriber?.startChunksRecording();
-    } else if ((!isSpeaking || !running) && transcriber) {
-      log.info('stopping transcription');
-      transcriber?.stopChunksRecording();
-    }
-  };
-
-  useEffect(() => {
-    const ctx = new Context();
-    scheduleTask(ctx, async () => {
-      if (transcriber && running) {
-        await transcriber.open();
-      } else if (!running) {
-        await transcriber?.close();
-      }
-      manageChunkRecording();
-    });
-
-    return () => {
-      void ctx.dispose();
-    };
-  }, [transcriber, running]);
-
-  useEffect(() => {
-    manageChunkRecording();
-  }, [isSpeaking, stream]);
-
-  return (
-    <TranscriptionStory
-      disabled={!stream}
-      model={model}
-      running={running}
-      onRunningChange={setRunning}
-      audioRef={ref}
-    />
-  );
-};
-
-const meta = {
-  title: 'plugins/plugin-transcription/FileTranscription',
-  decorators: [
-    withTheme,
-    withLayout({ container: 'column' }),
-    withPluginManager({
-      plugins: [
-        ClientPlugin({
-          types: [TestItem, DataType.Person, DataType.Organization, Testing.DocumentType],
-          onClientInitialized: async ({ client }) => {
-            await client.halo.createIdentity();
-            await client.spaces.waitUntilReady();
-            await client.spaces.default.waitUntilReady();
-            await seedTestData(client.spaces.default);
-          },
-        }),
-        SpacePlugin({}),
-        IntentPlugin(),
-
-        // UI
-        ThemePlugin({ tx: defaultTx }),
-        SettingsPlugin(),
-        PreviewPlugin(),
-        TranscriptionPlugin(),
-        StorybookLayoutPlugin({}),
-      ],
-      fireEvents: [Events.SetupAppGraph],
-    }),
-  ],
-} satisfies Meta;
-
-export default meta;
-
-type Story = StoryObj<typeof meta>;
-
-const TRANSCRIBER_CONFIG = {
-  transcribeAfterChunksAmount: 100,
-  prefixBufferChunksAmount: 50,
-  normalizeSentences: true,
-};
-
-const RECORDER_CONFIG = {
-  interval: 200,
-};
-
-export const Default: StoryObj<typeof AudioFile> = {
-  render: AudioFile,
-  args: {
-    detectSpeaking: true,
-    // https://learnenglish.britishcouncil.org/general-english/audio-zone/living-london
-    audioUrl: 'https://dxos.network/audio-london.m4a',
-    // textUrl: 'https://dxos.network/audio-london.txt',
-    transcriberConfig: TRANSCRIBER_CONFIG,
-    recorderConfig: RECORDER_CONFIG,
-  },
-};
-
-// TODO(mykola): Fix sentence normalization.
-// export const WithSentenceNormalization: StoryObj<typeof AudioFile> = {
-//   render: AudioFile,
-//   args: {
-//     detectSpeaking: true,
-//     normalizeSentences: true,
-//     // https://learnenglish.britishcouncil.org/general-english/audio-zone/living-london
-//     audioUrl: 'https://dxos.network/audio-london.m4a',
-//     // textUrl: 'https://dxos.network/audio-london.txt',
-//     transcriberConfig: TRANSCRIBER_CONFIG,
-//     recorderConfig: RECORDER_CONFIG,
-//   },
-// };

package/src/components/stories/MicrophoneTranscription.stories.tsx

@@ -1,285 +0,0 @@
-//
-// Copyright 2025 DXOS.org
-//
-
-import { type Meta, type StoryObj } from '@storybook/react-vite';
-import React, { useCallback, useEffect, useMemo, useState } from 'react';
-
-import { Events, IntentPlugin, SettingsPlugin } from '@dxos/app-framework';
-import { withPluginManager } from '@dxos/app-framework/testing';
-import {
-  type ExtractionFunction,
-  extractionAnthropicFunction,
-  extractionNerFunction,
-  getNer,
-  processTranscriptMessage,
-} from '@dxos/assistant/extraction';
-import { Filter, Obj, type Type } from '@dxos/echo';
-import { createQueueDXN } from '@dxos/echo/internal';
-import { MemoryQueue } from '@dxos/echo-db';
-import { FunctionExecutor, ServiceContainer } from '@dxos/functions';
-import { invariant } from '@dxos/invariant';
-import { log } from '@dxos/log';
-import { ClientPlugin } from '@dxos/plugin-client';
-import { PreviewPlugin } from '@dxos/plugin-preview';
-import { SpacePlugin } from '@dxos/plugin-space';
-import { StorybookLayoutPlugin } from '@dxos/plugin-storybook-layout';
-import { ThemePlugin } from '@dxos/plugin-theme';
-import { IndexKind, useSpace } from '@dxos/react-client/echo';
-import { withLayout, withTheme } from '@dxos/react-ui/testing';
-import { defaultTx } from '@dxos/react-ui-theme';
-import { DataType } from '@dxos/schema';
-import { Testing, seedTestData } from '@dxos/schema/testing';
-
-import { useAudioTrack, useQueueModelAdapter, useTranscriber } from '../../hooks';
-import { TestItem } from '../../testing';
-import { type MediaStreamRecorderParams, type TranscriberParams } from '../../transcriber';
-import { TranscriptionPlugin } from '../../TranscriptionPlugin';
-import { renderByline } from '../Transcript';
-
-import { TranscriptionStory } from './TranscriptionStory';
-import { useIsSpeaking } from './useIsSpeaking';
-
-const TRANSCRIBER_CONFIG = {
-  transcribeAfterChunksAmount: 50,
-  prefixBufferChunksAmount: 10,
-};
-
-const RECORDER_CONFIG = {
-  interval: 200,
-};
-
-type StoryProps = {
-  detectSpeaking?: boolean;
-  transcriberConfig: TranscriberParams['config'];
-  recorderConfig: MediaStreamRecorderParams['config'];
-  audioConstraints?: MediaTrackConstraints;
-  entityExtraction: 'none' | 'ner' | 'llm';
-};
-
-const DefaultStory = ({
-  detectSpeaking,
-  entityExtraction = 'none',
-  transcriberConfig,
-  recorderConfig,
-  audioConstraints,
-}: StoryProps) => {
-  const [running, setRunning] = useState(false);
-
-  // Audio.
-  const track = useAudioTrack(running, audioConstraints);
-
-  // Speaking.
-  const isSpeaking = detectSpeaking ? useIsSpeaking(track) : true;
-
-  // Queue.
-  // TODO(dmaretskyi): Use space.queues.create() instead.
-  const queueDxn = useMemo(() => createQueueDXN(), []);
-  const queue = useMemo(() => new MemoryQueue<DataType.Message>(queueDxn), [queueDxn]);
-  const model = useQueueModelAdapter(renderByline([]), queue);
-  const space = useSpace();
-
-  useEffect(() => {
-    if (!space) {
-      log.warn('no space');
-    }
-  }, [space]);
-
-  // Entity extraction.
-  const { extractionFunction, executor, objects } = useMemo(() => {
-    if (!space) {
-      log.warn('no space');
-      return {};
-    }
-
-    let executor: FunctionExecutor | undefined;
-    let extractionFunction: ExtractionFunction | undefined;
-    let objects: Promise<Type.Expando[]> | undefined;
-
-    if (entityExtraction === 'ner') {
-      // Init model loading. Takes time.
-      void getNer();
-      extractionFunction = extractionNerFunction;
-    } else if (entityExtraction === 'llm') {
-      extractionFunction = extractionAnthropicFunction;
-      objects = space.db
-        .query(
-          Filter.or(
-            Filter.type(DataType.Person),
-            Filter.type(DataType.Organization),
-            Filter.type(Testing.DocumentType),
-          ),
-        )
-        .run()
-        .then((result) => result.objects);
-    }
-    if (entityExtraction !== 'none') {
-      executor = new FunctionExecutor(
-        new ServiceContainer().setServices({
-          // ai: {
-          //   client: new Edge AiServiceClient({
-          //     endpoint: AI_SERVICE_ENDPOINT.REMOTE,
-          //   }),
-          // },
-          // database: { db: space!.db },
-        }),
-      );
-    }
-
-    return { extractionFunction, executor, objects };
-  }, [entityExtraction, space]);
-
-  // Transcriber.
-  const handleSegments = useCallback<TranscriberParams['onSegments']>(
-    async (blocks) => {
-      const message = Obj.make(DataType.Message, {
-        sender: { name: 'You' },
-        created: new Date().toISOString(),
-        blocks,
-      });
-      if (!space) {
-        void queue.append([message]);
-        return;
-      }
-
-      if (entityExtraction !== 'none') {
-        invariant(extractionFunction, 'extractionFunction is required');
-        invariant(executor, 'executor is required');
-        const result = await processTranscriptMessage({
-          executor,
-          function: extractionFunction,
-          input: {
-            message,
-            objects: await objects,
-          },
-          options: {
-            fallbackToRaw: true,
-            timeout: 30_000,
-          },
-        });
-        void queue.append([result.message]);
-      } else {
-        void queue.append([message]);
-      }
-    },
-    [queue, space],
-  );
-
-  const transcriber = useTranscriber({
-    audioStreamTrack: track,
-    onSegments: handleSegments,
-    transcriberConfig,
-    recorderConfig,
-  });
-  const [isOpen, setIsOpen] = useState(false);
-  useEffect(() => {
-    void transcriber?.open().then(() => setIsOpen(true));
-    return () => {
-      void transcriber?.close().then(() => setIsOpen(false));
-    };
-  }, [transcriber]);
-
-  // Manage transcription state.
-  useEffect(() => {
-    if (running && transcriber?.isOpen && isSpeaking) {
-      transcriber?.startChunksRecording();
-    } else if (!running || !isSpeaking) {
-      transcriber?.stopChunksRecording();
-    }
-  }, [transcriber, running, isOpen, isSpeaking]);
-
-  return <TranscriptionStory model={model} running={running} onRunningChange={setRunning} />;
-};
-
-const meta = {
-  title: 'plugins/plugin-transcription/MicrophoneTranscription',
-  render: DefaultStory,
-  decorators: [
-    withTheme,
-    withLayout({ container: 'column' }),
-    withPluginManager({
-      plugins: [
-        ClientPlugin({
-          types: [TestItem, DataType.Person, DataType.Organization, Testing.DocumentType],
-          onClientInitialized: async ({ client }) => {
-            await client.halo.createIdentity();
-            await client.spaces.waitUntilReady();
-            await client.spaces.default.waitUntilReady();
-            // TODO(mykola): Make API easier to use.
-            // TODO(mykola): Delete after enabling vector indexing by default.
-            // Enable vector indexing.
-            await client.services.services.QueryService!.setConfig({
-              enabled: true,
-              indexes: [
-                //
-                { kind: IndexKind.Kind.SCHEMA_MATCH },
-                { kind: IndexKind.Kind.GRAPH },
-                { kind: IndexKind.Kind.VECTOR },
-              ],
-            });
-            await client.services.services.QueryService!.reindex();
-            await seedTestData(client.spaces.default);
-          },
-        }),
-        SpacePlugin({}),
-        IntentPlugin(),
-        SettingsPlugin(),
-
-        // UI
-        ThemePlugin({ tx: defaultTx }),
-        PreviewPlugin(),
-        TranscriptionPlugin(),
-        StorybookLayoutPlugin({}),
-      ],
-      fireEvents: [Events.SetupAppGraph],
-    }),
-  ],
-} satisfies Meta<typeof DefaultStory>;
-
-export default meta;
-
-type Story = StoryObj<typeof meta>;
-
-export const Default: Story = {
-  args: {
-    detectSpeaking: false,
-    entityExtraction: 'none',
-    transcriberConfig: TRANSCRIBER_CONFIG,
-    recorderConfig: RECORDER_CONFIG,
-    audioConstraints: {
-      echoCancellation: false,
-      noiseSuppression: false,
-      autoGainControl: false,
-    },
-  },
-};
-
-// NOTE: We are running out of free quota on hugging face entity extraction.
-// TODO(mykola): Fix hugging face quota issues.
-// export const EntityExtraction: Story = {
-//   args: {
-//     detectSpeaking: true,
-//     entityExtraction: 'ner',
-//     transcriberConfig: TRANSCRIBER_CONFIG,
-//     recorderConfig: RECORDER_CONFIG,
-//     audioConstraints: {
-//       echoCancellation: true,
-//       noiseSuppression: true,
-//       autoGainControl: true,
-//     },
-//   },
-// };
-
-export const SpeechDetection: Story = {
-  args: {
-    detectSpeaking: true,
-    entityExtraction: 'none',
-    transcriberConfig: TRANSCRIBER_CONFIG,
-    recorderConfig: RECORDER_CONFIG,
-    audioConstraints: {
-      echoCancellation: true,
-      noiseSuppression: true,
-      autoGainControl: true,
-    },
-  },
-};

package/src/components/stories/TranscriptionStory.tsx

@@ -1,45 +0,0 @@
-//
-// Copyright 2025 DXOS.org
-//
-
-import '@dxos-theme';
-
-import React, { type Dispatch, type FC, type RefObject, type SetStateAction } from 'react';
-
-import { useSpace } from '@dxos/react-client/echo';
-import { IconButton, Toolbar } from '@dxos/react-ui';
-import { ScrollContainer } from '@dxos/react-ui-components';
-import { type DataType } from '@dxos/schema';
-
-import { type SerializationModel } from '../../model';
-import { TranscriptView } from '../Transcript';
-
-export const TranscriptionStory: FC<{
-  model: SerializationModel<DataType.Message>;
-  disabled?: boolean;
-  running: boolean;
-  onRunningChange: Dispatch<SetStateAction<boolean>>;
-  audioRef?: RefObject<HTMLAudioElement | null>;
-}> = ({ model, running, onRunningChange, audioRef, disabled }) => {
-  const space = useSpace();
-
-  return (
-    <div className='flex flex-col w-[30rem]'>
-      {audioRef && <audio ref={audioRef} autoPlay />}
-      <Toolbar.Root>
-        <IconButton
-          iconOnly
-          disabled={disabled}
-          icon={running ? 'ph--pause--regular' : 'ph--play--regular'}
-          label={running ? 'Pause' : 'Play'}
-          onClick={() => onRunningChange((running) => !running)}
-        />
-      </Toolbar.Root>
-      <ScrollContainer.Root pin>
-        <ScrollContainer.Content>
-          <TranscriptView space={space} model={model} />
-        </ScrollContainer.Content>
-      </ScrollContainer.Root>
-    </div>
-  );
-};

package/src/components/stories/useIsSpeaking.tsx

@@ -1,40 +0,0 @@
-//
-// Copyright 2025 DXOS.org
-//
-
-import '@dxos-theme';
-
-import { useEffect, useMemo, useState } from 'react';
-
-import { scheduleTask } from '@dxos/async';
-import { Context } from '@dxos/context';
-import { SpeakingMonitor } from '@dxos/plugin-thread';
-
-export const useIsSpeaking = (track?: MediaStreamTrack) => {
-  const [isSpeaking, setIsSpeaking] = useState(false);
-  const speakingMonitor = useMemo(() => {
-    if (!track) {
-      return;
-    }
-    return new SpeakingMonitor(track);
-  }, [track]);
-
-  useEffect(() => {
-    if (!speakingMonitor) {
-      return;
-    }
-
-    const ctx = new Context();
-    scheduleTask(ctx, async () => {
-      speakingMonitor.speakingChanged.on(ctx, () => setIsSpeaking(speakingMonitor.isSpeaking));
-      await speakingMonitor.open();
-      ctx.onDispose(() => speakingMonitor.close());
-    });
-
-    return () => {
-      void ctx.dispose();
-    };
-  }, [speakingMonitor]);
-
-  return isSpeaking;
-};

package/src/functions/index.ts

@@ -1,6 +0,0 @@
-//
-// Copyright 2024 DXOS.org
-//
-
-export { default as open } from './open';
-export { default as summarize } from './summarize';

package/src/functions/open.ts

@@ -1,39 +0,0 @@
-//
-// Copyright 2025 DXOS.org
-//
-
-import * as Effect from 'effect/Effect';
-import * as Schema from 'effect/Schema';
-
-import { ArtifactId } from '@dxos/assistant';
-import { Obj } from '@dxos/echo';
-import { DatabaseService, QueueService, defineFunction } from '@dxos/functions';
-import { DataType } from '@dxos/schema';
-
-import { renderByline } from '../components';
-import { Transcript } from '../types';
-
-export default defineFunction({
-  key: 'dxos.org/function/transcription/open',
-  name: 'Open',
-  description: 'Opens and reads the contents of a transcription object.',
-  inputSchema: Schema.Struct({
-    id: ArtifactId.annotations({
-      description: 'The ID of the transcription object.',
-    }),
-  }),
-  outputSchema: Schema.Struct({
-    content: Schema.String,
-  }),
-  handler: Effect.fn(function* ({ data: { id } }) {
-    const transcript = yield* DatabaseService.resolve(ArtifactId.toDXN(id), Transcript.Transcript);
-    const { dxn } = yield* Effect.promise(() => transcript.queue.load());
-    const queue = yield* QueueService.getQueue(dxn);
-    yield* Effect.promise(() => queue?.queryObjects());
-    const content = queue?.objects
-      .filter((message) => Obj.instanceOf(DataType.Message, message))
-      .flatMap((message, index) => renderByline([])(message, index))
-      .join('\n\n');
-    return { content };
-  }),
-});