@dxos/plugin-transcription 0.8.3 → 0.8.4-main.03d5cd7b56

package/src/components/stories/MicrophoneTranscription.stories.tsx

@@ -1,280 +0,0 @@
-//
-// Copyright 2025 DXOS.org
-//
-
-import '@dxos-theme';
-
-import { type Meta, type StoryObj } from '@storybook/react';
-import React, { useCallback, useEffect, useMemo, useState } from 'react';
-
-import { EdgeAiServiceClient } from '@dxos/ai';
-import { AI_SERVICE_ENDPOINT } from '@dxos/ai/testing';
-import { Events, IntentPlugin, SettingsPlugin } from '@dxos/app-framework';
-import { withPluginManager } from '@dxos/app-framework/testing';
-import {
-  extractionAnthropicFn,
-  type ExtractionFunction,
-  extractionNerFn,
-  processTranscriptMessage,
-  getNer,
-} from '@dxos/assistant';
-import { Filter, Obj, type Type } from '@dxos/echo';
-import { MemoryQueue } from '@dxos/echo-db';
-import { createQueueDXN } from '@dxos/echo-schema';
-import { FunctionExecutor, ServiceContainer } from '@dxos/functions';
-import { invariant } from '@dxos/invariant';
-import { log } from '@dxos/log';
-import { ClientPlugin } from '@dxos/plugin-client';
-import { PreviewPlugin } from '@dxos/plugin-preview';
-import { SpacePlugin } from '@dxos/plugin-space';
-import { StorybookLayoutPlugin } from '@dxos/plugin-storybook-layout';
-import { ThemePlugin } from '@dxos/plugin-theme';
-import { IndexKind, useSpace } from '@dxos/react-client/echo';
-import { defaultTx } from '@dxos/react-ui-theme';
-import { DataType } from '@dxos/schema';
-import { seedTestData, Testing } from '@dxos/schema/testing';
-import { withLayout, withTheme } from '@dxos/storybook-utils';
-
-import { TranscriptionStory } from './TranscriptionStory';
-import { useIsSpeaking } from './useIsSpeaking';
-import { TranscriptionPlugin } from '../../TranscriptionPlugin';
-import { useAudioTrack, useQueueModelAdapter, useTranscriber } from '../../hooks';
-import { TestItem } from '../../testing';
-import { type MediaStreamRecorderParams, type TranscriberParams } from '../../transcriber';
-import { renderMarkdown } from '../Transcript';
-
-const TRANSCRIBER_CONFIG = {
-  transcribeAfterChunksAmount: 50,
-  prefixBufferChunksAmount: 10,
-};
-
-const RECORDER_CONFIG = {
-  interval: 200,
-};
-
-type StoryProps = {
-  detectSpeaking?: boolean;
-  transcriberConfig: TranscriberParams['config'];
-  recorderConfig: MediaStreamRecorderParams['config'];
-  audioConstraints?: MediaTrackConstraints;
-  entityExtraction: 'none' | 'ner' | 'llm';
-};
-
-const DefaultStory = ({
-  detectSpeaking,
-  entityExtraction = 'none',
-  transcriberConfig,
-  recorderConfig,
-  audioConstraints,
-}: StoryProps) => {
-  const [running, setRunning] = useState(false);
-
-  // Audio.
-  const track = useAudioTrack(running, audioConstraints);
-
-  // Speaking.
-  const isSpeaking = detectSpeaking ? useIsSpeaking(track) : true;
-
-  // Queue.
-  // TODO(dmaretskyi): Use space.queues.create() instead.
-  const queueDxn = useMemo(() => createQueueDXN(), []);
-  const queue = useMemo(() => new MemoryQueue<DataType.Message>(queueDxn), [queueDxn]);
-  const model = useQueueModelAdapter(renderMarkdown([]), queue);
-  const space = useSpace();
-
-  useEffect(() => {
-    if (!space) {
-      log.warn('no space');
-    }
-  }, [space]);
-
-  // Entity extraction.
-  const { extractionFunction, executor, objects } = useMemo(() => {
-    if (!space) {
-      log.warn('no space');
-      return {};
-    }
-
-    let executor: FunctionExecutor | undefined;
-    let extractionFunction: ExtractionFunction | undefined;
-    let objects: Promise<Type.Expando[]> | undefined;
-
-    if (entityExtraction === 'ner') {
-      // Init model loading. Takes time.
-      void getNer();
-      extractionFunction = extractionNerFn;
-    } else if (entityExtraction === 'llm') {
-      extractionFunction = extractionAnthropicFn;
-      objects = space.db
-        .query(
-          Filter.or(
-            Filter.type(DataType.Person),
-            Filter.type(DataType.Organization),
-            Filter.type(Testing.DocumentType),
-          ),
-        )
-        .run()
-        .then((result) => result.objects);
-    }
-    if (entityExtraction !== 'none') {
-      const AiService = new EdgeAiServiceClient({
-        endpoint: AI_SERVICE_ENDPOINT.REMOTE,
-      });
-      executor = new FunctionExecutor(
-        new ServiceContainer().setServices({ ai: { client: AiService }, database: { db: space!.db } }),
-      );
-    }
-
-    return { extractionFunction, executor, objects };
-  }, [entityExtraction, space]);
-
-  // Transcriber.
-  const handleSegments = useCallback<TranscriberParams['onSegments']>(
-    async (blocks) => {
-      const message = Obj.make(DataType.Message, {
-        sender: { name: 'You' },
-        created: new Date().toISOString(),
-        blocks,
-      });
-      if (!space) {
-        void queue.append([message]);
-        return;
-      }
-
-      if (entityExtraction !== 'none') {
-        invariant(extractionFunction, 'extractionFunction is required');
-        invariant(executor, 'executor is required');
-        const result = await processTranscriptMessage({
-          executor,
-          function: extractionFunction,
-          input: {
-            message,
-            objects: await objects,
-          },
-          options: {
-            fallbackToRaw: true,
-            timeout: 30_000,
-          },
-        });
-        void queue.append([result.message]);
-      } else {
-        void queue.append([message]);
-      }
-    },
-    [queue, space],
-  );
-
-  const transcriber = useTranscriber({
-    audioStreamTrack: track,
-    onSegments: handleSegments,
-    transcriberConfig,
-    recorderConfig,
-  });
-  const [isOpen, setIsOpen] = useState(false);
-  useEffect(() => {
-    void transcriber?.open().then(() => setIsOpen(true));
-    return () => {
-      void transcriber?.close().then(() => setIsOpen(false));
-    };
-  }, [transcriber]);
-
-  // Manage transcription state.
-  useEffect(() => {
-    if (running && transcriber?.isOpen && isSpeaking) {
-      transcriber?.startChunksRecording();
-    } else if (!running || !isSpeaking) {
-      transcriber?.stopChunksRecording();
-    }
-  }, [transcriber, running, isOpen, isSpeaking]);
-
-  return <TranscriptionStory model={model} running={running} onRunningChange={setRunning} />;
-};
-
-const meta: Meta<typeof DefaultStory> = {
-  title: 'plugins/plugin-transcription/MicrophoneTranscription',
-  render: DefaultStory,
-  decorators: [
-    withPluginManager({
-      plugins: [
-        ThemePlugin({ tx: defaultTx }),
-        StorybookLayoutPlugin(),
-        ClientPlugin({
-          types: [TestItem, DataType.Person, DataType.Organization, Testing.DocumentType],
-          onClientInitialized: async (_, client) => {
-            await client.halo.createIdentity();
-            await client.spaces.waitUntilReady();
-            await client.spaces.default.waitUntilReady();
-            // TODO(mykola): Make API easier to use.
-            // TODO(mykola): Delete after enabling vector indexing by default.
-            // Enable vector indexing.
-            await client.services.services.QueryService!.setConfig({
-              enabled: true,
-              indexes: [
-                //
-                { kind: IndexKind.Kind.SCHEMA_MATCH },
-                { kind: IndexKind.Kind.GRAPH },
-                { kind: IndexKind.Kind.VECTOR },
-              ],
-            });
-            await client.services.services.QueryService!.reindex();
-            await seedTestData(client.spaces.default);
-          },
-        }),
-        SpacePlugin(),
-        SettingsPlugin(),
-        PreviewPlugin(),
-        IntentPlugin(),
-        TranscriptionPlugin(),
-      ],
-      fireEvents: [Events.SetupAppGraph],
-    }),
-    withTheme,
-    withLayout({ fullscreen: true, classNames: 'justify-center' }),
-  ],
-};
-
-export default meta;
-
-type Story = StoryObj<typeof DefaultStory>;
-
-export const Default: Story = {
-  args: {
-    detectSpeaking: false,
-    entityExtraction: 'none',
-    transcriberConfig: TRANSCRIBER_CONFIG,
-    recorderConfig: RECORDER_CONFIG,
-    audioConstraints: {
-      echoCancellation: false,
-      noiseSuppression: false,
-      autoGainControl: false,
-    },
-  },
-};
-
-export const EntityExtraction: Story = {
-  args: {
-    detectSpeaking: true,
-    entityExtraction: 'ner',
-    transcriberConfig: TRANSCRIBER_CONFIG,
-    recorderConfig: RECORDER_CONFIG,
-    audioConstraints: {
-      echoCancellation: true,
-      noiseSuppression: true,
-      autoGainControl: true,
-    },
-  },
-};
-
-export const SpeechDetection: Story = {
-  args: {
-    detectSpeaking: true,
-    entityExtraction: 'none',
-    transcriberConfig: TRANSCRIBER_CONFIG,
-    recorderConfig: RECORDER_CONFIG,
-    audioConstraints: {
-      echoCancellation: true,
-      noiseSuppression: true,
-      autoGainControl: true,
-    },
-  },
-};

package/src/components/stories/TranscriptionStory.tsx

@@ -1,41 +0,0 @@
-//
-// Copyright 2025 DXOS.org
-//
-
-import '@dxos-theme';
-
-import React, { type Dispatch, type FC, type RefObject, type SetStateAction } from 'react';
-
-import { useSpace } from '@dxos/react-client/echo';
-import { IconButton, Toolbar } from '@dxos/react-ui';
-import { ScrollContainer } from '@dxos/react-ui-components';
-import { type DataType } from '@dxos/schema';
-
-import { type SerializationModel } from '../../model';
-import { Transcript } from '../Transcript';
-
-export const TranscriptionStory: FC<{
-  model: SerializationModel<DataType.Message>;
-  running: boolean;
-  onRunningChange: Dispatch<SetStateAction<boolean>>;
-  audioRef?: RefObject<HTMLAudioElement>;
-}> = ({ model, running, onRunningChange, audioRef }) => {
-  const space = useSpace();
-
-  return (
-    <div className='flex flex-col w-[30rem]'>
-      {audioRef && <audio ref={audioRef} autoPlay />}
-      <Toolbar.Root>
-        <IconButton
-          iconOnly
-          icon={running ? 'ph--pause--regular' : 'ph--play--regular'}
-          label={running ? 'Pause' : 'Play'}
-          onClick={() => onRunningChange((running) => !running)}
-        />
-      </Toolbar.Root>
-      <ScrollContainer>
-        <Transcript space={space} model={model} attendableId='story' />
-      </ScrollContainer>
-    </div>
-  );
-};

package/src/segments-normalization/normalization.ts

@@ -1,105 +0,0 @@
-//
-// Copyright 2025 DXOS.org
-//
-
-import { Schema } from 'effect';
-
-import { DEFAULT_EDGE_MODEL, Message } from '@dxos/ai';
-import { AISession } from '@dxos/assistant';
-import { Obj } from '@dxos/echo';
-import { AiService, defineFunction } from '@dxos/functions';
-import { ObjectId } from '@dxos/keys';
-import { log } from '@dxos/log';
-import { DataType } from '@dxos/schema';
-
-const MessageWithRangeId = Schema.extend(
-  DataType.Message,
-  Schema.Struct({
-    // TODO(mykola): Move to meta in DataType.Message. Use `DataType.Message` instead.
-    rangeId: Schema.optional(Schema.Array(Schema.String)).annotations({
-      description: 'The IDs of the messages that contain the sentences.',
-    }),
-  }),
-);
-
-export interface MessageWithRangeId extends Schema.Schema.Type<typeof MessageWithRangeId> {}
-
-export const NormalizationInput = Schema.Struct({
-  // TODO(mykola): Move to meta in DataType.Message. Use `DataType.Message` instead.
-  messages: Schema.Array(MessageWithRangeId).annotations({
-    description: 'Messages to normalize into sentences.',
-  }),
-});
-export interface NormalizationInput extends Schema.Schema.Type<typeof NormalizationInput> {}
-
-export const NormalizationOutput = Schema.Struct({
-  sentences: Schema.Array(MessageWithRangeId.pipe(Schema.mutable)).pipe(Schema.mutable).annotations({
-    description: 'The sentences of the transcript.',
-  }),
-});
-export interface NormalizationOutput extends Schema.Schema.Type<typeof NormalizationOutput> {}
-
-const prompt = `
-You are observing a real-time transcript of a single person speaking.
-The transcription is delivered in chunks of 10 seconds or less. As a result, individual sentences may be split across multiple messages, or multiple sentences may appear within a single message. Additionally, because this is real-time transcription, punctuation and capitalization may be incorrect or missing.
-
-# Task Description:
-  - Your task is to detect and reconstruct broken or incomplete sentences by merging segments into coherent, grammatically correct sentences where appropriate.
-
-# Input Format:
-  - The input is an array of messages that contain transcription blocks.
-  - Each message is a DataType.Message, as described in the output format.
-
-# Output Format:
-  - You have been provided with the tool that defines the output format; make sure to query it.
-  - Do not output anything other than the expected format.
-
-# Segment Handling Rules:
-  - Sort messages by timestamp.
-  - Leave ID of the first message in a sentence.
-  - Each divided sentence should be added to the output as a separate message or block within existing messages, preserving the original order.
-  - If one message contains multiple sentences, you should not split them.
-  - The last sentence may be incomplete; it should still be output as a separate block or message.
-  - Keep track of the original timestamps of the messages and blocks and, use the timestamp of the first message/block of a sentence as the 'started' field of the merged message.
-  - The 'rangeId' field of the merged message should include 'messageId'-s and 'rangeId'-s of all messages that have been used to construct this message.
-
-# Punctuation and Capitalization:
-  - Do not rely on the original punctuation and capitalization—they may be incorrect.
-  - Use logical reasoning to apply appropriate punctuation (e.g., period, comma, question mark, exclamation mark) and capitalization.
-
-# Restrictions:
-  - Do not alter the order of the original messages; maintain the natural flow of speech.
-  - Do not interpret or infer meaning beyond reconstructing sentences.
-  - Do not add or remove any words or phrases.
-`;
-
-export const sentenceNormalization = defineFunction<NormalizationInput, NormalizationOutput>({
-  description: 'Post process of transcription for sentence normalization',
-  inputSchema: NormalizationInput,
-  outputSchema: NormalizationOutput,
-  handler: async ({ data: { messages }, context }) => {
-    log.info('input', { messages });
-    const ai = context.getService(AiService);
-    const session = new AISession({ operationModel: 'configured' });
-
-    const response = await session.runStructured(NormalizationOutput, {
-      generationOptions: { model: DEFAULT_EDGE_MODEL },
-      client: ai.client,
-      tools: [],
-      artifacts: [],
-      history: [
-        Obj.make(Message, {
-          role: 'user',
-          content: messages.map((message) => ({ type: 'text', text: JSON.stringify(message) }) as const),
-        }),
-      ],
-      prompt,
-    });
-
-    response.sentences.forEach((sentence) => {
-      sentence.id = ObjectId.random();
-    });
-    log.info('response', { response });
-    return response;
-  },
-});

package/src/types/schema.ts

@@ -1,41 +0,0 @@
-//
-// Copyright 2025 DXOS.org
-//
-
-import { Schema } from 'effect';
-
-import { Queue } from '@dxos/client/echo';
-import { Type } from '@dxos/echo';
-import { TypedObject } from '@dxos/echo-schema';
-
-/**
- * Root transcript object created when the user starts a transcription.
- */
-export const TranscriptSchema = Schema.Struct({
-  // TODO(wittjosiah): Remove?
-  // TODO(burdon): Use Date or string?
-  started: Schema.optional(Schema.String),
-  ended: Schema.optional(Schema.String),
-
-  /**
-   * Queue containing TranscriptBlock objects.
-   */
-  queue: Type.Ref(Queue),
-});
-
-export class TranscriptType extends TypedObject({
-  typename: 'dxos.org/type/Transcript',
-  version: '0.1.0',
-})(TranscriptSchema.fields) {}
-
-// TODO(burdon): Do these need to be kept in sync with EDGE?
-
-/**
- * First message in queue.
- * Contains metadata for the recording and transcript.
- */
-const TranscriptHeader = Schema.Struct({
-  started: Schema.optional(Schema.String),
-});
-
-export type TranscriptHeader = Schema.Schema.Type<typeof TranscriptHeader>;

package/src/types/types.ts

@@ -1,32 +0,0 @@
-//
-// Copyright 2025 DXOS.org
-//
-
-import { Schema } from 'effect';
-
-import { SpaceId } from '@dxos/react-client/echo';
-
-import { TranscriptType } from './schema';
-import { TRANSCRIPTION_PLUGIN } from '../meta';
-
-// TODO(burdon): Move to separate proto.
-
-/**
- * Endpoint to the calls service.
- */
-// TODO(burdon): Reconfigure/move.
-export const TRANSCRIPTION_URL = 'https://calls-service.dxos.workers.dev';
-
-export namespace TranscriptionAction {
-  const TRANSCRIPTION_ACTION = `${TRANSCRIPTION_PLUGIN}/action`;
-
-  export class Create extends Schema.TaggedClass<Create>()(`${TRANSCRIPTION_ACTION}/create`, {
-    input: Schema.Struct({
-      name: Schema.optional(Schema.String),
-      spaceId: SpaceId,
-    }),
-    output: Schema.Struct({
-      object: TranscriptType,
-    }),
-  }) {}
-}

package/src/{segments-normalization → normalization}/index.ts

@@ -2,6 +2,6 @@
 // Copyright 2025 DXOS.org
 //
 
-export * from './normalization';
 export * from './message-normalizer';
+export * from './normalization';
 export * from './utils';