@dxos/plugin-transcription 0.8.2-staging.7ac8446 → 0.8.2

package/src/capabilities/intent-resolver.ts

@@ -2,21 +2,20 @@
 // Copyright 2025 DXOS.org
 //
 
-import { Capabilities, contributes, createResolver, type PluginsContext } from '@dxos/app-framework';
-import { create, refFromDXN } from '@dxos/live-object';
-import { type SpaceId } from '@dxos/react-client/echo';
+import { Capabilities, contributes, createResolver, type PluginContext } from '@dxos/app-framework';
+import { createQueueDxn } from '@dxos/echo-schema';
+import { type SpaceId } from '@dxos/keys';
+import { live, refFromDXN } from '@dxos/live-object';
 
 import { TranscriptionAction, TranscriptType } from '../types';
-import { getTimeStr, randomQueueDxn } from '../util';
 
-export default (context: PluginsContext) =>
+export default (context: PluginContext) =>
   contributes(Capabilities.IntentResolver, [
     createResolver({
       intent: TranscriptionAction.Create,
       resolve: ({ name, spaceId }) => {
-        const transcript = create(TranscriptType, {
-          queue: refFromDXN(randomQueueDxn(spaceId as SpaceId)),
-          name: name ?? `Transcript ${getTimeStr(Date.now())}`,
+        const transcript = live(TranscriptType, {
+          queue: refFromDXN(createQueueDxn(spaceId as SpaceId)),
         });
 
         return { data: { object: transcript } };

package/src/capabilities/react-surface.tsx

@@ -5,17 +5,18 @@
 import React from 'react';
 
 import { Capabilities, contributes, createSurface } from '@dxos/app-framework';
+import { isInstanceOf } from '@dxos/echo-schema';
 
 import { TranscriptionContainer } from '../components';
-import { TRANSCRIPTION_PLUGIN } from '../meta';
-import { isTranscript, type TranscriptType } from '../types';
+import { meta } from '../meta';
+import { TranscriptType } from '../types';
 
 export default () =>
   contributes(Capabilities.ReactSurface, [
     createSurface({
-      id: `${TRANSCRIPTION_PLUGIN}/article/transcript`,
+      id: `${meta.id}/article/transcript`,
       role: ['article', 'section'],
-      filter: (data): data is { subject: TranscriptType } => isTranscript(data.subject),
+      filter: (data): data is { subject: TranscriptType } => isInstanceOf(TranscriptType, data.subject),
       component: ({ data, role }) => <TranscriptionContainer transcript={data.subject} role={role} />,
     }),
   ]);

package/src/capabilities/transcriber.ts

@@ -2,10 +2,11 @@
 // Copyright 2025 DXOS.org
 //
 
-import { contributes } from '@dxos/app-framework';
+import { contributes, type PluginContext } from '@dxos/app-framework';
+import { ClientCapabilities } from '@dxos/plugin-client';
 
 import { TranscriptionCapabilities } from './capabilities';
-import { MediaStreamRecorder, Transcriber } from '../transcriber';
+import { MediaStreamRecorder, Transcriber, TranscriptionManager } from '../transcriber';
 
 // TODO(burdon): Move to config?
 
@@ -28,21 +29,48 @@ const TRANSCRIBE_AFTER_CHUNKS_AMOUNT = 50;
 /**
  * Records audio while user is speaking and transcribes it after user is done speaking.
  */
-export default () => {
-  const getTranscriber: TranscriptionCapabilities.GetTranscriber = ({ audioStreamTrack, onSegments }) => {
+export default (context: PluginContext) => {
+  const getTranscriber: TranscriptionCapabilities.GetTranscriber = ({
+    audioStreamTrack,
+    onSegments,
+    transcriberConfig,
+    recorderConfig,
+  }) => {
     // Initialize audio transcription.
     return new Transcriber({
       config: {
         transcribeAfterChunksAmount: TRANSCRIBE_AFTER_CHUNKS_AMOUNT,
         prefixBufferChunksAmount: PREFIXED_CHUNKS_AMOUNT,
+        ...transcriberConfig,
       },
       recorder: new MediaStreamRecorder({
         mediaStreamTrack: audioStreamTrack,
-        interval: RECORD_INTERVAL,
+        config: {
+          interval: RECORD_INTERVAL,
+          ...recorderConfig,
+        },
       }),
       onSegments,
     });
   };
 
-  return contributes(TranscriptionCapabilities.Transcriber, getTranscriber);
+  const getTranscriptionManager: TranscriptionCapabilities.GetTranscriptionManager = ({ messageEnricher }) => {
+    const client = context.getCapability(ClientCapabilities.Client);
+    const transcriptionManager = new TranscriptionManager({
+      edgeClient: client.edge,
+      messageEnricher,
+    });
+
+    const identity = client.halo.identity.get();
+    if (identity) {
+      transcriptionManager.setIdentityDid(identity.did);
+    }
+
+    return transcriptionManager;
+  };
+
+  return [
+    contributes(TranscriptionCapabilities.Transcriber, getTranscriber),
+    contributes(TranscriptionCapabilities.TranscriptionManager, getTranscriptionManager),
+  ];
 };

package/src/components/Transcript/Transcript.stories.tsx

@@ -4,60 +4,200 @@
 
 import '@dxos-theme';
 
-import { type StoryObj, type Meta } from '@storybook/react';
-import React, { useEffect, useState } from 'react';
+import { type Meta, type StoryObj } from '@storybook/react';
+import React, { type FC, useEffect, useMemo, useState } from 'react';
 
+import { IntentPlugin, SettingsPlugin } from '@dxos/app-framework';
+import { withPluginManager } from '@dxos/app-framework/testing';
+import { ObjectId } from '@dxos/echo-schema';
+import { ClientPlugin } from '@dxos/plugin-client';
+import { PreviewPlugin } from '@dxos/plugin-preview';
+import { SpacePlugin } from '@dxos/plugin-space';
+import { StorybookLayoutPlugin } from '@dxos/plugin-storybook-layout';
+import { ThemePlugin } from '@dxos/plugin-theme';
 import { faker } from '@dxos/random';
-import { hues } from '@dxos/react-ui-theme';
-import { withTheme, withLayout } from '@dxos/storybook-utils';
+import { useMembers, useSpace } from '@dxos/react-client/echo';
+import { IconButton, Toolbar } from '@dxos/react-ui';
+import { SyntaxHighlighter } from '@dxos/react-ui-syntax-highlighter';
+import { defaultTx } from '@dxos/react-ui-theme';
+import { DataType } from '@dxos/schema';
+import { Testing } from '@dxos/schema/testing';
+import { withLayout } from '@dxos/storybook-utils';
 
-import { Transcript } from './Transcript';
+import { renderMarkdown, Transcript, type TranscriptProps } from './Transcript';
+import { useQueueModelAdapter } from '../../hooks';
+import { SerializationModel } from '../../model';
+import {
+  MessageBuilder,
+  TestItem,
+  useTestTranscriptionQueue,
+  useTestTranscriptionQueueWithEntityExtraction,
+} from '../../testing';
 import translations from '../../translations';
 
-let start = new Date(Date.now() - 24 * 60 * 60 * 10_000);
-const next = () => {
-  start = new Date(start.getTime() + Math.random() * 10_000);
-  return start;
+faker.seed(1);
+
+/**
+ * Story wrapper with test controls.
+ */
+const TranscriptContainer: FC<
+  TranscriptProps & {
+    running: boolean;
+    onRunningChange: (running: boolean) => void;
+    onReset?: () => void;
+  }
+> = ({ space, model, running, onRunningChange, onReset }) => {
+  return (
+    <div className='grid grid-rows-[1fr_40px] grow divide-y divide-separator'>
+      <Transcript space={space} model={model} />
+      <div className='grid grid-cols-[1fr_16rem] overflow-hidden'>
+        <div className='flex items-center'>
+          <SyntaxHighlighter language='json' className='text-sm'>
+            {JSON.stringify(model.toJSON())}
+          </SyntaxHighlighter>
+        </div>
+        <Toolbar.Root classNames='justify-end'>
+          <IconButton
+            icon={running ? 'ph--pause--regular' : 'ph--play--regular'}
+            label={running ? 'Pause' : 'Start'}
+            onClick={() => onRunningChange(!running)}
+          />
+          {onReset && <IconButton icon='ph--x--regular' label='Reset' onClick={onReset} />}
+        </Toolbar.Root>
+      </div>
+    </div>
+  );
 };
 
-const users = Array.from({ length: 5 }, () => ({
-  authorName: faker.person.fullName(),
-  authorHue: faker.helpers.arrayElement(hues),
-}));
-
-const createBlock = () => {
-  const author = faker.helpers.arrayElement(users);
-  return {
-    id: faker.string.uuid(),
-    ...author,
-    segments: Array.from({ length: 1 + Math.floor(Math.random() * 2) }, () => ({
-      started: next(),
-      text: faker.lorem.paragraph(),
-    })),
+type StoryProps = { messages?: DataType.Message[] } & Pick<TranscriptProps, 'ignoreAttention' | 'attendableId'>;
+
+/**
+ * Basic story mutates array of messages.
+ */
+const BasicStory = ({ messages: initialMessages = [], ...props }: StoryProps) => {
+  const [reset, setReset] = useState({});
+  const builder = useMemo(() => new MessageBuilder(), []);
+  const model = useMemo(
+    () => new SerializationModel<DataType.Message>(renderMarkdown([]), initialMessages),
+    [initialMessages, reset],
+  );
+  const [running, setRunning] = useState(true);
+  const [currentMessage, setCurrentMessage] = useState<DataType.Message | null>(null);
+  useEffect(() => {
+    if (!running) {
+      return;
+    }
+
+    if (!currentMessage) {
+      void builder.createMessage().then((message) => {
+        model.appendChunk(message);
+        setCurrentMessage(message);
+      });
+      return;
+    }
+
+    const i = setInterval(() => {
+      if (currentMessage?.blocks.length && currentMessage.blocks.length >= 3) {
+        setCurrentMessage(null);
+        clearInterval(i);
+        return;
+      }
+
+      currentMessage.blocks.push(builder.createBlock());
+      model.updateChunk(currentMessage);
+    }, 3_000);
+
+    return () => clearInterval(i);
+  }, [model, currentMessage, running]);
+
+  const handleReset = () => {
+    setCurrentMessage(null);
+    setRunning(false);
+    setReset({});
   };
+
+  return (
+    <TranscriptContainer
+      model={model}
+      running={running}
+      onRunningChange={setRunning}
+      onReset={handleReset}
+      {...props}
+    />
+  );
 };
 
-const meta: Meta<typeof Transcript> = {
+/**
+ * Queue story mutates queue with model adapter.
+ */
+const QueueStory = ({
+  messages: initialMessages = [],
+  queueId,
+  onReset,
+  ...props
+}: StoryProps & { queueId: ObjectId; onReset: () => void }) => {
+  const [running, setRunning] = useState(true);
+  const space = useSpace();
+  const members = useMembers(space?.key).map((member) => member.identity);
+  const queue = useTestTranscriptionQueue(space, queueId, running, 2_000);
+  const model = useQueueModelAdapter(renderMarkdown(members), queue, initialMessages);
+
+  return (
+    <TranscriptContainer
+      space={space}
+      model={model}
+      running={running}
+      onRunningChange={setRunning}
+      onReset={onReset}
+      {...props}
+    />
+  );
+};
+
+// TODO(burdon): Reconcile with QueueStory.
+const EntityExtractionQueueStory = () => {
+  const [running, setRunning] = useState(true);
+  const space = useSpace();
+  const members = useMembers(space?.key).map((member) => member.identity);
+  const queue = useTestTranscriptionQueueWithEntityExtraction(space, undefined, running, 2_000);
+  const model = useQueueModelAdapter(renderMarkdown(members), queue, []);
+
+  return <TranscriptContainer space={space} model={model} running={running} onRunningChange={setRunning} />;
+};
+
+/**
+ * Wrapper remounts on refresh to reload queue.
+ */
+const QueueStoryWrapper = () => {
+  const [queueId] = useState(ObjectId.random());
+  const [key, setKey] = useState(ObjectId.random().toString());
+  const handleReset = () => {
+    setKey(ObjectId.random().toString());
+  };
+
+  return <QueueStory key={key} queueId={queueId} onReset={handleReset} />;
+};
+
+const meta: Meta<typeof QueueStory> = {
   title: 'plugins/plugin-transcription/Transcript',
-  component: Transcript,
-  render: ({ blocks: initialBlocks = [], ...args }) => {
-    const [blocks, setBlocks] = useState(initialBlocks);
-    useEffect(() => {
-      const i = setInterval(() => {
-        setBlocks((blocks) => [...blocks, createBlock()]);
-      }, 1_000);
-
-      return () => clearInterval(i);
-    }, []);
-
-    return <Transcript {...args} blocks={blocks} />;
-  },
   decorators: [
-    withTheme,
-    withLayout({
-      tooltips: true,
-      fullscreen: true,
+    withPluginManager({
+      plugins: [
+        ThemePlugin({ tx: defaultTx }),
+        StorybookLayoutPlugin(),
+        ClientPlugin({
+          types: [TestItem, Testing.DocumentType, DataType.Person, DataType.Organization],
+          onClientInitialized: async (_, client) => {
+            await client.halo.createIdentity();
+          },
+        }),
+        PreviewPlugin(),
+        SpacePlugin(),
+        SettingsPlugin(),
+        IntentPlugin(),
+      ],
     }),
+    withLayout({ fullscreen: true }),
   ],
   parameters: {
     translations,
@@ -66,17 +206,35 @@ const meta: Meta<typeof Transcript> = {
 
 export default meta;
 
-type Story = StoryObj<typeof Transcript>;
+type Story = StoryObj<typeof QueueStory>;
 
 export const Default: Story = {
+  render: BasicStory,
   args: {
     ignoreAttention: true,
     attendableId: 'story',
-    blocks: Array.from({ length: 10 }, createBlock),
+    messages: await Promise.all(Array.from({ length: 10 }, () => MessageBuilder.singleton.createMessage())),
   },
 };
 
 export const Empty: Story = {
+  render: BasicStory,
+  args: {
+    ignoreAttention: true,
+    attendableId: 'story',
+  },
+};
+
+export const WithQueue: Story = {
+  render: QueueStoryWrapper,
+  args: {
+    ignoreAttention: true,
+    attendableId: 'story',
+  },
+};
+
+export const WithEntityExtractionQueue: Story = {
+  render: EntityExtractionQueueStory,
   args: {
     ignoreAttention: true,
     attendableId: 'story',