@dxos/plugin-transcription 0.8.4-main.ae835ea → 0.8.4-main.bc2380dfbc

package/src/capabilities/transcriber.ts

@@ -2,12 +2,14 @@
 // Copyright 2025 DXOS.org
 //
 
-import { type PluginContext, contributes } from '@dxos/app-framework';
+import * as Effect from 'effect/Effect';
+
+import { Capabilities, Capability } from '@dxos/app-framework';
 import { ClientCapabilities } from '@dxos/plugin-client';
 
-import { MediaStreamRecorder, Transcriber, TranscriptionManager } from '../transcriber';
+import { TranscriptionCapabilities } from '#types';
 
-import { TranscriptionCapabilities } from './capabilities';
+import { MediaStreamRecorder, Transcriber, TranscriptionManager } from '../transcriber';
 
 // TODO(burdon): Move to config?
 
@@ -30,48 +32,57 @@ const TRANSCRIBE_AFTER_CHUNKS_AMOUNT = 50;
 /**
  * Records audio while user is speaking and transcribes it after user is done speaking.
  */
-export default (context: PluginContext) => {
-  const getTranscriber: TranscriptionCapabilities.GetTranscriber = ({
-    audioStreamTrack,
-    onSegments,
-    transcriberConfig,
-    recorderConfig,
-  }) => {
-    // Initialize audio transcription.
-    return new Transcriber({
-      config: {
-        transcribeAfterChunksAmount: TRANSCRIBE_AFTER_CHUNKS_AMOUNT,
-        prefixBufferChunksAmount: PREFIXED_CHUNKS_AMOUNT,
-        ...transcriberConfig,
-      },
-      recorder: new MediaStreamRecorder({
-        mediaStreamTrack: audioStreamTrack,
+export default Capability.makeModule(
+  Effect.fnUntraced(function* () {
+    // Get context for lazy capability access in callbacks.
+    const capabilities = yield* Capability.Service;
+    const registry = yield* Capability.get(Capabilities.AtomRegistry);
+
+    const getTranscriber: TranscriptionCapabilities.GetTranscriber = ({
+      audioStreamTrack,
+      onSegments,
+      transcriberConfig,
+      recorderConfig,
+      transcribe,
+    }) => {
+      // Initialize audio transcription.
+      return new Transcriber({
         config: {
-          interval: RECORD_INTERVAL,
-          ...recorderConfig,
+          transcribeAfterChunksAmount: TRANSCRIBE_AFTER_CHUNKS_AMOUNT,
+          prefixBufferChunksAmount: PREFIXED_CHUNKS_AMOUNT,
+          ...transcriberConfig,
         },
-      }),
-      onSegments,
-    });
-  };
+        recorder: new MediaStreamRecorder({
+          mediaStreamTrack: audioStreamTrack,
+          config: {
+            interval: RECORD_INTERVAL,
+            ...recorderConfig,
+          },
+        }),
+        onSegments,
+        transcribe,
+      });
+    };
 
-  const getTranscriptionManager: TranscriptionCapabilities.GetTranscriptionManager = ({ messageEnricher }) => {
-    const client = context.getCapability(ClientCapabilities.Client);
-    const transcriptionManager = new TranscriptionManager({
-      edgeClient: client.edge,
-      messageEnricher,
-    });
+    const getTranscriptionManager: TranscriptionCapabilities.GetTranscriptionManager = ({ messageEnricher }) => {
+      const client = capabilities.get(ClientCapabilities.Client);
+      const transcriptionManager = new TranscriptionManager({
+        edgeClient: client.edge.http,
+        messageEnricher,
+        registry,
+      });
 
-    const identity = client.halo.identity.get();
-    if (identity) {
-      transcriptionManager.setIdentityDid(identity.did);
-    }
+      const identity = client.halo.identity.get();
+      if (identity) {
+        transcriptionManager.setIdentityDid(identity.did);
+      }
 
-    return transcriptionManager;
-  };
+      return transcriptionManager;
+    };
 
-  return [
-    contributes(TranscriptionCapabilities.Transcriber, getTranscriber),
-    contributes(TranscriptionCapabilities.TranscriptionManager, getTranscriptionManager),
-  ];
-};
+    return [
+      Capability.contributes(TranscriptionCapabilities.Transcriber, getTranscriber),
+      Capability.contributes(TranscriptionCapabilities.TranscriptionManager, getTranscriptionManager),
+    ];
+  }),
+);

package/src/components/Oracle/Oracle.stories.tsx

@@ -0,0 +1,147 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+import { type Meta, type StoryObj } from '@storybook/react-vite';
+import * as Effect from 'effect/Effect';
+import React, { useCallback, useEffect, useState } from 'react';
+
+import { withPluginManager } from '@dxos/app-framework/testing';
+import { ClientPlugin, initializeIdentity } from '@dxos/plugin-client/testing';
+import { corePlugins } from '@dxos/plugin-testing';
+import { IconButton, Panel, Toolbar } from '@dxos/react-ui';
+import { withLayout } from '@dxos/react-ui/testing';
+import { type ContentBlock, type Message } from '@dxos/types';
+
+import { useAudioTrack, useTranscriber } from '#hooks';
+import { translations } from '#translations';
+
+import { TranscriptionPlugin } from '../../TranscriptionPlugin';
+import { Oracle } from './Oracle';
+
+type DefaultStoryProps = {
+  delay?: number;
+};
+
+const DefaultStory = ({ delay }: DefaultStoryProps) => {
+  const [recording, setRecording] = useState(false);
+  const [text, setText] = useState('');
+  const elapsed = useStopwatch(recording);
+  const track = useAudioTrack(recording);
+
+  // Build the latest transcribed text from incoming blocks.
+  const handleSegments = useCallback(async (blocks: Message.Message['blocks']) => {
+    const phrase = blocks
+      .filter((block): block is ContentBlock.Transcript => block._tag === 'transcript')
+      .map((block) => block.text.trim())
+      .filter(Boolean)
+      .join(' ');
+
+    setText((prev) => (prev ? `${prev} ${phrase}` : phrase));
+  }, []);
+
+  // Drive the transcriber off the recording flag.
+  // Default `transcribeAfterChunksAmount` is 50 × 200ms = 10s, so the first phrase only
+  // surfaces after ~10s. Drop it to 15 chunks (~3s) so the TextBlock starts updating fast.
+  const transcriber = useTranscriber({
+    audioStreamTrack: track,
+    onSegments: handleSegments,
+    transcriberConfig: { transcribeAfterChunksAmount: 15, prefixBufferChunksAmount: 5 },
+  });
+  useEffect(() => {
+    if (!transcriber) {
+      return;
+    }
+
+    let cancelled = false;
+    void transcriber.open().then(() => {
+      if (!cancelled && recording) {
+        transcriber.startChunksRecording();
+      }
+    });
+
+    return () => {
+      cancelled = true;
+      transcriber.stopChunksRecording();
+    };
+  }, [transcriber, recording]);
+
+  return (
+    <Panel.Root>
+      <Panel.Toolbar asChild>
+        <Toolbar.Root>
+          <IconButton
+            icon={recording ? 'ph--stop-circle--regular' : 'ph--microphone--regular'}
+            label={recording ? 'Stop' : 'Record'}
+            onClick={() => setRecording((value) => !value)}
+          />
+          <IconButton disabled={recording} icon='ph--x--regular' label='Clear' onClick={() => setText('')} />
+          <Toolbar.Separator />
+          <Toolbar.Text classNames='text-right tabular-nums text-subdued'>{elapsed}s</Toolbar.Text>
+        </Toolbar.Root>
+      </Panel.Toolbar>
+      <Oracle.Root text={text} delay={delay}>
+        <Panel.Content asChild>
+          <Oracle.Content>
+            <Oracle.Canvas />
+            <Oracle.Text />
+          </Oracle.Content>
+        </Panel.Content>
+      </Oracle.Root>
+    </Panel.Root>
+  );
+};
+
+const meta = {
+  title: 'plugins/plugin-transcription/components/Oracle',
+  render: DefaultStory,
+  decorators: [
+    withLayout({ layout: 'fullscreen' }),
+    withPluginManager({
+      plugins: [
+        ...corePlugins(),
+        ClientPlugin({
+          onClientInitialized: ({ client }) =>
+            Effect.gen(function* () {
+              yield* initializeIdentity(client);
+            }),
+        }),
+        TranscriptionPlugin(),
+      ],
+    }),
+  ],
+  parameters: {
+    layout: 'fullscreen',
+    translations,
+  },
+} satisfies Meta<typeof DefaultStory>;
+
+export default meta;
+
+type Story = StoryObj<typeof meta>;
+
+export const Default: Story = {
+  args: {
+    delay: 12,
+  },
+};
+
+export const Instant: Story = {
+  args: {
+    delay: 0,
+  },
+};
+
+/** Seconds elapsed while `running`; resets to 0 each time `running` toggles back to true. */
+const useStopwatch = (running: boolean): number => {
+  const [seconds, setSeconds] = useState(0);
+  useEffect(() => {
+    if (!running) {
+      return;
+    }
+    setSeconds(0);
+    const id = setInterval(() => setSeconds((value) => value + 1), 1_000);
+    return () => clearInterval(id);
+  }, [running]);
+  return seconds;
+};

package/src/components/Oracle/Oracle.tsx

@@ -0,0 +1,141 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+import { Primitive } from '@radix-ui/react-primitive';
+import { Slot } from '@radix-ui/react-slot';
+import React, { type PropsWithChildren, createContext, useContext, useMemo } from 'react';
+
+import '@dxos/react-ui-graph/styles/graph.css';
+import { type ThemedClassName } from '@dxos/react-ui';
+import { TextBlock } from '@dxos/react-ui-components';
+import { SVG } from '@dxos/react-ui-graph';
+import { composableProps, mx, slottable } from '@dxos/ui-theme';
+import { type SlottableProps } from '@dxos/ui-types';
+
+//
+// Context
+//
+
+type OracleContextValue = {
+  /** Latest utterance / phrase rendered by `Oracle.Text`. */
+  text: string;
+  /** Per-character reveal delay in ms for the streaming TextBlock. */
+  delay: number;
+};
+
+const OracleContext = createContext<OracleContextValue | null>(null);
+
+const useOracleContext = (consumer: string): OracleContextValue => {
+  const ctx = useContext(OracleContext);
+  if (!ctx) {
+    throw new Error(`<Oracle.${consumer}> must be rendered inside <Oracle.Root>`);
+  }
+  return ctx;
+};
+
+//
+// Root (headless — provides context only, renders no DOM)
+//
+
+export type OracleRootProps = PropsWithChildren<{
+  /** Latest utterance / phrase rendered by `Oracle.Text`. */
+  text?: string;
+  /** Per-character reveal delay in ms for the streaming TextBlock. @default 12 */
+  delay?: number;
+}>;
+
+/**
+ * Oracle surface root. Headless — provides text/delay context to subcomponents and renders
+ * no DOM of its own. Wrap subcomponents in `<Oracle.Content>` for the standard relative
+ * stacking layout, or place `<Oracle.Canvas>` / `<Oracle.Text>` inside any positioned
+ * container of your choice.
+ */
+const Root = ({ children, text = '', delay = 12 }: OracleRootProps) => {
+  const value = useMemo(() => ({ text, delay }), [text, delay]);
+  return <OracleContext.Provider value={value}>{children}</OracleContext.Provider>;
+};
+
+//
+// Content
+//
+
+export type OracleContentProps = SlottableProps;
+
+/**
+ * Positioned container that stacks `Oracle.Canvas` (the SVG graph viewport, absolutely
+ * positioned to fill) under `Oracle.Text` (the centered overlay). Use this for the standard
+ * Oracle layout; skip it if you need custom positioning.
+ *
+ * Pass `asChild` to merge the positioning + sizing onto a parent slot (e.g. `Panel.Content`).
+ */
+const Content = slottable<HTMLDivElement>(({ children, asChild, ...props }, forwardedRef) => {
+  const Comp = asChild ? Slot : Primitive.div;
+  return (
+    <Comp {...composableProps(props, { classNames: 'relative dx-expander' })} ref={forwardedRef}>
+      {children}
+    </Comp>
+  );
+});
+
+Content.displayName = 'Oracle.Content';
+
+//
+// Canvas
+//
+
+export type OracleCanvasProps = ThemedClassName<PropsWithChildren<{}>>;
+
+/**
+ * SVG graph viewport. Renders sensible defaults (markers, grid, pan/zoom, empty graph)
+ * when no children are supplied; consumers can replace those by passing their own
+ * SVG.* children (e.g. a populated `<SVG.Graph model={…} />`).
+ */
+const Canvas = ({ classNames, children }: OracleCanvasProps) => (
+  <SVG.Root centered classNames={mx('absolute inset-0', classNames)}>
+    {children ?? (
+      <>
+        <SVG.Markers />
+        <SVG.Grid />
+        <SVG.Zoom extent={[1 / 4, 4]}>
+          <SVG.Graph />
+        </SVG.Zoom>
+      </>
+    )}
+  </SVG.Root>
+);
+
+//
+// Text
+//
+
+export type OracleTextProps = ThemedClassName<{}>;
+
+/**
+ * Centered text overlay. Reads `text` and `delay` from `Oracle.Root` context and renders
+ * a streaming `TextBlock` floating above the canvas. Pointer-events are disabled so the
+ * underlying graph remains interactive.
+ */
+const Text = ({ classNames }: OracleTextProps) => {
+  const { text, delay } = useOracleContext('Text');
+  return (
+    <div className='pointer-events-none absolute inset-0 flex items-center justify-center'>
+      <TextBlock
+        classNames={mx('max-w-xl px-6 py-4 text-center text-2xl font-medium text-balance text-subdued', classNames)}
+        text={text}
+        delay={delay}
+      />
+    </div>
+  );
+};
+
+//
+// Namespace
+//
+
+export const Oracle = {
+  Root,
+  Content,
+  Canvas,
+  Text,
+};

package/src/components/Oracle/index.ts

@@ -0,0 +1,5 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+export * from './Oracle';

package/src/components/Transcription/Transcription.stories.tsx

@@ -0,0 +1,213 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import { type Meta, type StoryObj } from '@storybook/react-vite';
+import * as Effect from 'effect/Effect';
+import type * as Types from 'effect/Types';
+import React, { useCallback, useEffect, useMemo, useState } from 'react';
+
+import { withPluginManager } from '@dxos/app-framework/testing';
+import { Feed, Filter, Obj, Query } from '@dxos/echo';
+import { ClientPlugin, initializeIdentity } from '@dxos/plugin-client/testing';
+import { PreviewPlugin } from '@dxos/plugin-preview/testing';
+import { StorybookPlugin, corePlugins } from '@dxos/plugin-testing';
+import { random } from '@dxos/random';
+import { useMembers, useQuery, useSpaces } from '@dxos/react-client/echo';
+import { IconButton, Panel, Toolbar } from '@dxos/react-ui';
+import { JsonHighlighter } from '@dxos/react-ui-syntax-highlighter';
+import { withLayout } from '@dxos/react-ui/testing';
+import { type ContentBlock, Message, Organization, Person } from '@dxos/types';
+
+import { useFeedModelAdapter } from '#hooks';
+import { MessageBuilder, TestItem, useTestTranscriptionQueue } from '#testing';
+import { translations } from '#translations';
+
+import { SerializationModel } from '../../model';
+import { renderByline } from '../../util';
+import { Transcription, type TranscriptionProps } from './Transcription';
+
+random.seed(1);
+
+type Source = 'local' | 'feed';
+
+type DefaultStoryProps = {
+  /** Where transcript messages come from. */
+  source?: Source;
+  /** Number of synthetic messages to pre-populate (local source only). */
+  seed?: number;
+};
+
+const DefaultStory = ({ source = 'local', seed = 0, ...props }: DefaultStoryProps) => {
+  const [resetKey, setResetKey] = useState(0);
+  const handleReset = useCallback(() => setResetKey((value) => value + 1), []);
+  return source === 'feed' ? (
+    <FeedStory key={resetKey} onReset={handleReset} {...props} />
+  ) : (
+    <LocalStory key={resetKey} seed={seed} onReset={handleReset} {...props} />
+  );
+};
+
+type InnerProps = {
+  onReset: () => void;
+};
+
+const LocalStory = ({ seed = 0, onReset, ...props }: InnerProps & { seed?: number }) => {
+  const builder = useMemo(() => new MessageBuilder(), []);
+  const [initialMessages, setInitialMessages] = useState<Message.Message[]>([]);
+  const [ready, setReady] = useState(seed === 0);
+
+  useEffect(() => {
+    // Reset on every seed change so Storybook controls don't leave stale messages from a prior run.
+    let cancelled = false;
+    setInitialMessages([]);
+    setReady(seed === 0);
+
+    if (seed === 0) {
+      return () => {
+        cancelled = true;
+      };
+    }
+
+    void Promise.all(Array.from({ length: seed }, () => builder.createMessage())).then((messages) => {
+      if (cancelled) {
+        return;
+      }
+      setInitialMessages(messages);
+      setReady(true);
+    });
+
+    return () => {
+      cancelled = true;
+    };
+  }, [seed, builder]);
+
+  const model = useMemo(
+    () => new SerializationModel<Message.Message>(renderByline([]), initialMessages),
+    [initialMessages],
+  );
+
+  const [running, setRunning] = useState(true);
+  const [currentMessage, setCurrentMessage] = useState<Types.Mutable<Message.Message> | null>(null);
+
+  useEffect(() => {
+    if (!ready || !running) {
+      return;
+    }
+
+    if (!currentMessage) {
+      void builder.createMessage().then((message) => {
+        model.appendChunk(message);
+        setCurrentMessage(message);
+      });
+      return;
+    }
+
+    const id = setInterval(() => {
+      if (currentMessage.blocks.length >= 3) {
+        setCurrentMessage(null);
+        clearInterval(id);
+        return;
+      }
+      Obj.update(currentMessage, (currentMessage) => {
+        (currentMessage.blocks as ContentBlock.Any[]).push(builder.createBlock());
+      });
+      model.updateChunk(currentMessage);
+    }, 3_000);
+
+    return () => clearInterval(id);
+  }, [ready, running, currentMessage, model, builder]);
+
+  return <Shell model={model} running={running} onRunningChange={setRunning} onReset={onReset} {...props} />;
+};
+
+const FeedStory = ({ onReset, ...props }: InnerProps) => {
+  const [running, setRunning] = useState(true);
+  const [space] = useSpaces();
+  const members = useMembers(space?.id).map((member) => member.identity);
+  const feed = useTestTranscriptionQueue(space, running, 2_000);
+  const messages = useQuery(
+    space?.db,
+    feed ? Query.select(Filter.type(Message.Message)).from(feed) : Query.select(Filter.nothing()),
+  );
+  const model = useFeedModelAdapter(renderByline(members), messages, []);
+
+  return <Shell model={model} running={running} onRunningChange={setRunning} onReset={onReset} {...props} />;
+};
+
+type ShellProps = TranscriptionProps & {
+  running: boolean;
+  onRunningChange: (running: boolean) => void;
+  onReset: () => void;
+};
+
+const Shell = ({ model, running, onRunningChange, onReset, ...props }: ShellProps) => (
+  <Panel.Root>
+    <Panel.Toolbar asChild>
+      <Toolbar.Root classNames='justify-end'>
+        <IconButton
+          icon={running ? 'ph--pause--regular' : 'ph--play--regular'}
+          label={running ? 'Pause' : 'Start'}
+          onClick={() => onRunningChange(!running)}
+        />
+        <IconButton icon='ph--x--regular' label='Reset' onClick={onReset} />
+      </Toolbar.Root>
+    </Panel.Toolbar>
+    <Panel.Content className='dx-document'>
+      <Transcription model={model} {...props} />
+    </Panel.Content>
+    <Panel.Statusbar className='flex items-center justify-center'>
+      <JsonHighlighter data={model.toJSON()} indent={0} classNames='text-sm' />
+    </Panel.Statusbar>
+  </Panel.Root>
+);
+
+const meta = {
+  title: 'plugins/plugin-transcription/components/Transcription',
+  render: DefaultStory,
+  decorators: [
+    withLayout({ layout: 'fullscreen' }),
+    withPluginManager({
+      plugins: [
+        ...corePlugins(),
+        StorybookPlugin({}),
+        ClientPlugin({
+          types: [TestItem, Person.Person, Organization.Organization, Feed.Feed, Message.Message],
+          onClientInitialized: ({ client }) =>
+            Effect.gen(function* () {
+              yield* initializeIdentity(client);
+            }),
+        }),
+        PreviewPlugin(),
+      ],
+    }),
+  ],
+  parameters: {
+    layout: 'fullscreen',
+    translations,
+  },
+} satisfies Meta<typeof DefaultStory>;
+
+export default meta;
+
+type Story = StoryObj<typeof meta>;
+
+export const Default: Story = {
+  args: {
+    source: 'local',
+    seed: 10,
+  },
+};
+
+export const Empty: Story = {
+  args: {
+    source: 'local',
+    seed: 0,
+  },
+};
+
+export const WithQueue: Story = {
+  args: {
+    source: 'feed',
+  },
+};

package/src/components/Transcription/Transcription.tsx

@@ -0,0 +1,58 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import React from 'react';
+
+import { useThemeContext } from '@dxos/react-ui';
+import { useTextEditor } from '@dxos/react-ui-editor';
+import { type Message, type Transcript } from '@dxos/types';
+import {
+  createBasicExtensions,
+  createMarkdownExtensions,
+  createThemeExtensions,
+  decorateMarkdown,
+  documentSlots,
+  preview,
+  scroller,
+} from '@dxos/ui-editor';
+import { composable, composableProps } from '@dxos/ui-theme';
+
+import { type SerializationModel } from '../../model';
+import { renderByline } from '../../util';
+import { transcription } from './transcription-extension';
+
+export { renderByline };
+
+export type TranscriptionProps = {
+  transcript?: Transcript.Transcript;
+  model: SerializationModel<Message.Message>;
+};
+
+// TODO(burdon): Remove forwardedRef/use react-ui-editor.
+export const Transcription = composable<HTMLDivElement, TranscriptionProps>(
+  ({ transcript: object, model, children, ...props }, forwardedRef) => {
+    const { themeMode } = useThemeContext();
+    const { parentRef } = useTextEditor(() => {
+      return {
+        extensions: [
+          createBasicExtensions({ readOnly: true, lineWrapping: true, search: true }),
+          createThemeExtensions({ themeMode, slots: documentSlots }),
+          createMarkdownExtensions(),
+          decorateMarkdown(),
+          preview(),
+          transcription({ model, started: object?.started ? new Date(object.started) : undefined }),
+          scroller(),
+        ],
+      };
+    }, [model, themeMode]);
+
+    return (
+      <div
+        {...composableProps(props, { classNames: 'dx-container' })}
+        data-popover-collision-boundary={true}
+        ref={parentRef}
+      />
+    );
+  },
+);

package/src/components/{Transcript → Transcription}/index.ts

@@ -2,4 +2,4 @@
 // Copyright 2025 DXOS.org
 //
 
-export * from './Transcript';
+export * from './Transcription';