@dxos/plugin-transcription 0.8.4-main.c85a9c8dae → 0.8.4-main.dfabb4ec29

package/src/{components/stories → stories}/FileTranscription.stories.tsx

@@ -11,25 +11,26 @@ import { AppActivationEvents } from '@dxos/app-toolkit';
 import { scheduleTask } from '@dxos/async';
 import { Context } from '@dxos/context';
 import { Obj } from '@dxos/echo';
-import { createQueueDXN } from '@dxos/echo/internal';
 import { MemoryQueue } from '@dxos/echo-db';
+import { createQueueDXN } from '@dxos/echo/internal';
 import { FunctionExecutor, ServiceContainer } from '@dxos/functions-runtime';
 import { log } from '@dxos/log';
-import { ClientPlugin } from '@dxos/plugin-client';
-import { PreviewPlugin } from '@dxos/plugin-preview';
+import { ClientPlugin } from '@dxos/plugin-client/plugin';
+import { initializeIdentity } from '@dxos/plugin-client/testing';
+import { PreviewPlugin } from '@dxos/plugin-preview/testing';
 import { StorybookPlugin, corePlugins } from '@dxos/plugin-testing';
-import { withLayout, withTheme } from '@dxos/react-ui/testing';
+import { withLayout } from '@dxos/react-ui/testing';
 import { TestSchema } from '@dxos/schema/testing';
 import { type Actor, Message, Organization, Person } from '@dxos/types';
 import { seedTestData } from '@dxos/types/testing';
 
-import { useAudioFile, useQueueModelAdapter, useTranscriber } from '../../hooks';
-import { MessageNormalizer, getActorId } from '../../segments-normalization';
-import { TestItem } from '../../testing';
-import { type MediaStreamRecorderProps, type TranscriberProps } from '../../transcriber';
-import { TranscriptionPlugin } from '../../TranscriptionPlugin';
-import { renderByline } from '../../util';
+import { useAudioFile, useQueueModelAdapter, useTranscriber } from '#hooks';
+import { TestItem } from '#testing';
 
+import { MessageNormalizer, getActorId } from '../normalization';
+import { type MediaStreamRecorderProps, type TranscriberProps } from '../transcriber';
+import { TranscriptionPlugin } from '../TranscriptionPlugin';
+import { renderByline } from '../util';
 import { TranscriptionStory } from './TranscriptionStory';
 import { useIsSpeaking } from './useIsSpeaking';
 
@@ -51,8 +52,26 @@ const AudioFile = ({
   const [running, setRunning] = useState(false);
   const actor: Actor.Actor = useMemo(() => ({ name: 'You' }), []);
 
+  // Optional uploaded file overrides the default URL.
+  // The useEffect below revokes the previous URL whenever it changes (and on unmount).
+  const [uploadedUrl, setUploadedUrl] = useState<string>();
+  useEffect(() => {
+    return () => {
+      if (uploadedUrl) {
+        URL.revokeObjectURL(uploadedUrl);
+      }
+    };
+  }, [uploadedUrl]);
+
+  const handleUpload = useCallback((file: File) => {
+    setRunning(false);
+    setUploadedUrl(URL.createObjectURL(file));
+  }, []);
+
+  const sourceUrl = uploadedUrl ?? audioUrl;
+
   // Audio.
-  const { audio, track, stream } = useAudioFile(audioUrl, audioConstraints);
+  const { audio, track, stream } = useAudioFile(sourceUrl, audioConstraints);
 
   // Speaking.
   const isSpeaking = detectSpeaking ? useIsSpeaking(track) : true;
@@ -179,6 +198,7 @@ const AudioFile = ({
       running={running}
       onRunningChange={setRunning}
       audioRef={ref}
+      onUpload={handleUpload}
     />
   );
 };
@@ -186,7 +206,6 @@ const AudioFile = ({
 const meta = {
   title: 'plugins/plugin-transcription/components/FileTranscription',
   decorators: [
-    withTheme(),
     withLayout({ layout: 'column' }),
     withPluginManager({
       plugins: [
@@ -196,10 +215,8 @@ const meta = {
           types: [TestItem, Person.Person, Organization.Organization, TestSchema.DocumentType],
           onClientInitialized: ({ client }) =>
             Effect.gen(function* () {
-              yield* Effect.promise(() => client.halo.createIdentity());
-              yield* Effect.promise(() => client.spaces.waitUntilReady());
-              yield* Effect.promise(() => client.spaces.default.waitUntilReady());
-              yield* Effect.promise(() => seedTestData(client.spaces.default));
+              const { personalSpace } = yield* initializeIdentity(client);
+              yield* Effect.promise(() => seedTestData(personalSpace));
             }),
         }),
 

package/src/{components/stories → stories}/KeyWordRecognition.stories.tsx

@@ -6,7 +6,7 @@ import { type Meta, type StoryObj } from '@storybook/react-vite';
 import React, { useEffect, useMemo, useState } from 'react';
 
 import { log } from '@dxos/log';
-import { Toolbar } from '@dxos/react-ui';
+import { Panel, Toolbar } from '@dxos/react-ui';
 import { withLayout, withTheme } from '@dxos/react-ui/testing';
 import { mx } from '@dxos/ui-theme';
 
@@ -15,11 +15,11 @@ type Word = {
   matched: boolean;
 };
 
-type StoryProps = {
+type DefaultStoryProps = {
   keywords: string[];
 };
 
-const DefaultStory = ({ keywords }: StoryProps) => {
+const DefaultStory = ({ keywords }: DefaultStoryProps) => {
   const [running, setRunning] = useState(false);
   const [matchingWords, setMatchingWords] = useState<Word[]>(keywords.map((word) => ({ text: word, matched: false })));
 
@@ -86,32 +86,34 @@ const DefaultStory = ({ keywords }: StoryProps) => {
   }, [running, recognition]);
 
   return (
-    <div className={mx('flex flex-col gap-2 w-[30rem]')}>
-      <Toolbar.Root>
-        <Toolbar.IconButton
-          iconOnly
-          icon={running ? 'ph--pause--regular' : 'ph--play--regular'}
-          label={running ? 'Pause' : 'Play'}
-          onClick={() => setRunning((running) => !running)}
-        />
-      </Toolbar.Root>
-
-      <div className='flex flex-wrap'>
-        {matchingWords.map((word, index) => (
-          <span
-            key={index}
-            className={mx(
-              'px-3 py-1 m-2 rounded-sm border',
-              word.matched ? 'bg-red-200 text-red-800 border-red-300' : 'bg-gray-200 text-gray-600 border-gray-300',
-            )}
-          >
-            {word.text}
-          </span>
-        ))}
-      </div>
-
-      <span>{transcript || (running ? 'Listening...' : 'Click play to start speech recognition')}</span>
-    </div>
+    <Panel.Root>
+      <Panel.Toolbar asChild>
+        <Toolbar.Root>
+          <Toolbar.IconButton
+            iconOnly
+            icon={running ? 'ph--pause--regular' : 'ph--play--regular'}
+            label={running ? 'Pause' : 'Play'}
+            onClick={() => setRunning((running) => !running)}
+          />
+        </Toolbar.Root>
+      </Panel.Toolbar>
+
+      <Panel.Content>
+        <div className='p-4 grid grid-cols-[repeat(auto-fill,minmax(6rem,1fr))] gap-2'>
+          {matchingWords.map((word, index) => (
+            <span
+              key={index}
+              className={mx(
+                'grid place-items-center p-2 aspect-square rounded-sm border border-separator',
+                word.matched && 'border-red-300',
+              )}
+            >
+              {word.text}
+            </span>
+          ))}
+        </div>
+      </Panel.Content>
+    </Panel.Root>
   );
 };
 
@@ -127,6 +129,6 @@ type Story = StoryObj<typeof meta>;
 
 export const SpeechRecognitionAPI: Story = {
   args: {
-    keywords: ['kai', 'computer', 'hello'],
+    keywords: ['kai', 'computer', 'hello', 'dxos'],
   },
 };

package/src/{components/stories → stories}/MicrophoneTranscription.stories.tsx

@@ -16,26 +16,26 @@ import {
   processTranscriptMessage,
 } from '@dxos/assistant/extraction';
 import { Filter, type Obj } from '@dxos/echo';
-import { createQueueDXN } from '@dxos/echo/internal';
 import { MemoryQueue } from '@dxos/echo-db';
-import { FunctionExecutor, ServiceContainer } from '@dxos/functions-runtime';
+import { createQueueDXN } from '@dxos/echo/internal';
 import { invariant } from '@dxos/invariant';
 import { log } from '@dxos/log';
-import { ClientPlugin } from '@dxos/plugin-client';
-import { PreviewPlugin } from '@dxos/plugin-preview';
+import { ClientPlugin } from '@dxos/plugin-client/plugin';
+import { initializeIdentity } from '@dxos/plugin-client/testing';
+import { PreviewPlugin } from '@dxos/plugin-preview/testing';
 import { StorybookPlugin, corePlugins } from '@dxos/plugin-testing';
-import { IndexKind, useSpace } from '@dxos/react-client/echo';
-import { withLayout, withTheme } from '@dxos/react-ui/testing';
+import { IndexKind, useSpaces } from '@dxos/react-client/echo';
+import { withLayout } from '@dxos/react-ui/testing';
 import { TestSchema } from '@dxos/schema/testing';
 import { Message, Organization, Person } from '@dxos/types';
 import { seedTestData } from '@dxos/types/testing';
 
-import { useAudioTrack, useQueueModelAdapter, useTranscriber } from '../../hooks';
-import { TestItem } from '../../testing';
-import { type MediaStreamRecorderProps, type TranscriberProps } from '../../transcriber';
-import { TranscriptionPlugin } from '../../TranscriptionPlugin';
-import { renderByline } from '../../util';
+import { useAudioTrack, useQueueModelAdapter, useTranscriber } from '#hooks';
+import { TestItem } from '#testing';
 
+import { type MediaStreamRecorderProps, type TranscriberProps } from '../transcriber';
+import { TranscriptionPlugin } from '../TranscriptionPlugin';
+import { renderByline } from '../util';
 import { TranscriptionStory } from './TranscriptionStory';
 import { useIsSpeaking } from './useIsSpeaking';
 
@@ -48,7 +48,7 @@ const RECORDER_CONFIG = {
   interval: 200,
 };
 
-type StoryProps = {
+type DefaultStoryProps = {
   detectSpeaking?: boolean;
   transcriberConfig: TranscriberProps['config'];
   recorderConfig: MediaStreamRecorderProps['config'];
@@ -62,7 +62,7 @@ const DefaultStory = ({
   transcriberConfig,
   recorderConfig,
   audioConstraints,
-}: StoryProps) => {
+}: DefaultStoryProps) => {
   const [running, setRunning] = useState(false);
 
   // Audio.
@@ -76,7 +76,7 @@ const DefaultStory = ({
   const queueDxn = useMemo(() => createQueueDXN(), []);
   const queue = useMemo(() => new MemoryQueue<Message.Message>(queueDxn), [queueDxn]);
   const model = useQueueModelAdapter(renderByline([]), queue);
-  const space = useSpace();
+  const [space] = useSpaces();
 
   useEffect(() => {
     if (!space) {
@@ -85,13 +85,12 @@ const DefaultStory = ({
   }, [space]);
 
   // Entity extraction.
-  const { extractionFunction, executor, objects } = useMemo(() => {
+  const { extractionFunction, objects } = useMemo(() => {
     if (!space) {
       log.warn('no space');
       return {};
     }
 
-    let executor: FunctionExecutor | undefined;
     let extractionFunction: ExtractionFunction | undefined;
     let objects: Promise<Obj.Unknown[]> | undefined;
 
@@ -111,20 +110,8 @@ const DefaultStory = ({
         )
         .run();
     }
-    if (entityExtraction !== 'none') {
-      executor = new FunctionExecutor(
-        new ServiceContainer().setServices({
-          // ai: {
-          //   client: new Edge AiServiceClient({
-          //     endpoint: AI_SERVICE_ENDPOINT.REMOTE,
-          //   }),
-          // },
-          // database: { db: space!.db },
-        }),
-      );
-    }
 
-    return { extractionFunction, executor, objects };
+    return { extractionFunction, objects };
   }, [entityExtraction, space]);
 
   // Transcriber.
@@ -142,9 +129,7 @@ const DefaultStory = ({
 
       if (entityExtraction !== 'none') {
         invariant(extractionFunction, 'extractionFunction is required');
-        invariant(executor, 'executor is required');
         const result = await processTranscriptMessage({
-          executor,
           function: extractionFunction,
           input: {
             message,
@@ -193,7 +178,6 @@ const meta = {
   title: 'plugins/plugin-transcription/components/MicrophoneTranscription',
   render: DefaultStory,
   decorators: [
-    withTheme(),
     withLayout({ layout: 'column' }),
     withPluginManager({
       plugins: [
@@ -203,9 +187,7 @@ const meta = {
           types: [TestItem, Person.Person, Organization.Organization, TestSchema.DocumentType],
           onClientInitialized: ({ client }) =>
             Effect.gen(function* () {
-              yield* Effect.promise(() => client.halo.createIdentity());
-              yield* Effect.promise(() => client.spaces.waitUntilReady());
-              yield* Effect.promise(() => client.spaces.default.waitUntilReady());
+              const { personalSpace } = yield* initializeIdentity(client);
               // TODO(mykola): Make API easier to use.
               // TODO(mykola): Delete after enabling vector indexing by default.
               // Enable vector indexing.
@@ -221,7 +203,7 @@ const meta = {
                 }),
               );
               yield* Effect.promise(() => client.services.services.QueryService!.reindex());
-              yield* Effect.promise(() => seedTestData(client.spaces.default));
+              yield* Effect.promise(() => seedTestData(personalSpace));
             }),
         }),
 

package/src/stories/TranscriptionStory.tsx

@@ -0,0 +1,93 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import '@dxos-theme';
+
+import React, {
+  type ChangeEvent,
+  type Dispatch,
+  type FC,
+  type ReactNode,
+  type RefObject,
+  type SetStateAction,
+  useRef,
+} from 'react';
+
+import { IconButton, ScrollContainer, Toolbar } from '@dxos/react-ui';
+import { type Message } from '@dxos/types';
+
+import { Transcription } from '../components';
+import { type SerializationModel } from '../model';
+
+/**
+ * Story wrapper that renders a transcript with playback controls, an optional audio file
+ * upload affordance, and a toolbar slot for additional controls.
+ *
+ * @param onUpload - Callback fired when the user picks a local audio file via the upload button.
+ * @param uploadAccept - HTML `accept` filter for the hidden file input. Defaults to `audio/*`.
+ * @param toolbarSlot - Extra content rendered after the upload button inside the toolbar.
+ */
+export const TranscriptionStory: FC<{
+  model: SerializationModel<Message.Message>;
+  disabled?: boolean;
+  running: boolean;
+  onRunningChange: Dispatch<SetStateAction<boolean>>;
+  audioRef?: RefObject<HTMLAudioElement | null>;
+  onUpload?: (file: File) => void;
+  uploadAccept?: string;
+  toolbarSlot?: ReactNode;
+}> = ({ model, running, onRunningChange, audioRef, disabled, onUpload, uploadAccept = 'audio/*', toolbarSlot }) => {
+  const fileInputRef = useRef<HTMLInputElement>(null);
+
+  const handleFileChange = (event: ChangeEvent<HTMLInputElement>) => {
+    const file = event.target.files?.[0];
+    if (file && onUpload) {
+      onUpload(file);
+    }
+    // Reset so the same file can be re-selected.
+    event.target.value = '';
+  };
+
+  return (
+    <>
+      {audioRef && <audio ref={audioRef} autoPlay />}
+      <Toolbar.Root>
+        <IconButton
+          iconOnly
+          disabled={disabled}
+          icon={running ? 'ph--pause--regular' : 'ph--play--regular'}
+          label={running ? 'Pause' : 'Play'}
+          onClick={() => onRunningChange((running) => !running)}
+        />
+        {onUpload && (
+          <>
+            <input
+              ref={fileInputRef}
+              type='file'
+              accept={uploadAccept}
+              disabled={disabled}
+              className='hidden'
+              onChange={handleFileChange}
+            />
+            <IconButton
+              iconOnly
+              disabled={disabled}
+              icon='ph--upload--regular'
+              label='Upload audio'
+              onClick={() => fileInputRef.current?.click()}
+            />
+          </>
+        )}
+        {toolbarSlot}
+      </Toolbar.Root>
+      <ScrollContainer.Root pin>
+        <ScrollContainer.Content>
+          <ScrollContainer.Viewport>
+            <Transcription model={model} />
+          </ScrollContainer.Viewport>
+        </ScrollContainer.Content>
+      </ScrollContainer.Root>
+    </>
+  );
+};

package/src/testing/testing.ts

@@ -11,10 +11,9 @@ import { Filter, type Queue } from '@dxos/client/echo';
 import { Context } from '@dxos/context';
 import { type Key, Obj, Ref, Type } from '@dxos/echo';
 import { createQueueDXN } from '@dxos/echo/internal';
-import { FunctionExecutor, ServiceContainer } from '@dxos/functions-runtime';
 import { IdentityDid } from '@dxos/keys';
 import { log } from '@dxos/log';
-import { faker } from '@dxos/random';
+import { random } from '@dxos/random';
 import { type Space, useQueue } from '@dxos/react-client/echo';
 import { TestSchema } from '@dxos/schema/testing';
 import { type ContentBlock, Message, Organization, Person } from '@dxos/types';
@@ -32,7 +31,7 @@ export const TestItem = Schema.Struct({
   }),
 }).pipe(
   Type.object({
-    typename: 'dxos.org/type/Test',
+    typename: 'org.dxos.type.test',
     version: '0.1.0',
   }),
 );
@@ -50,8 +49,8 @@ export class MessageBuilder extends AbstractMessageBuilder {
 
   users = Array.from({ length: 5 }, () => ({
     identityDid: IdentityDid.random().toString(),
-    name: faker.person.fullName(),
-    email: faker.internet.email(),
+    name: random.person.fullName(),
+    email: random.internet.email(),
   }));
 
   start = new Date(Date.now() - 24 * 60 * 60 * 10_000);
@@ -63,19 +62,19 @@ export class MessageBuilder extends AbstractMessageBuilder {
   override async createMessage(numSegments = 1): Promise<Message.Message> {
     return Obj.make(Message.Message, {
       created: this.next().toISOString(),
-      sender: faker.helpers.arrayElement(this.users),
+      sender: random.helpers.arrayElement(this.users),
       blocks: Array.from({ length: numSegments }).map(() => this.createBlock()),
     });
   }
 
   createBlock(): ContentBlock.Transcript {
-    let text = faker.lorem.paragraph();
+    let text = random.lorem.paragraph();
     if (this._space) {
-      const label = faker.commerce.productName();
+      const label = random.commerce.productName();
       const obj = this._space.db.add(
         Obj.make(TestItem, {
           title: label,
-          description: faker.lorem.paragraph(),
+          description: random.lorem.paragraph(),
         }),
       );
       const dxn = Ref.make(obj).dxn.toString();
@@ -99,14 +98,6 @@ export class MessageBuilder extends AbstractMessageBuilder {
 
 // TODO(burdon): Reconcile with BlockBuilder.
 class EntityExtractionMessageBuilder extends AbstractMessageBuilder {
-  executor = new FunctionExecutor(
-    new ServiceContainer().setServices({
-      // ai: {
-      //   client: this.AiService,
-      // },
-    }),
-  );
-
   space: Space | undefined;
   currentMessage: number = 0;
   transcriptMessages: Message.Message[] = [];
@@ -139,7 +130,6 @@ class EntityExtractionMessageBuilder extends AbstractMessageBuilder {
 
     const { message: enhancedMessage } = await processTranscriptMessage({
       input: { message },
-      executor: this.executor,
       function: extractionAnthropicFunction,
     });
 

package/src/transcriber/index.ts

@@ -2,7 +2,7 @@
 // Copyright 2025 DXOS.org
 //
 
-export type * from './audio-recorder';
+export * from './audio-recorder';
 export * from './media-stream-recorder';
 export * from './transcriber';
 export * from './transcription-manager';

package/src/transcriber/transcriber.browser.test.ts

@@ -4,7 +4,6 @@
 
 import fs from 'node:fs';
 import path from 'node:path';
-
 import { describe, expect, test } from 'vitest';
 import { WaveFile } from 'wavefile';
 

package/src/transcriber/transcriber.test.ts

@@ -0,0 +1,104 @@
+//
+// Copyright 2026 DXOS.org
+//
+
+import { describe, test } from 'vitest';
+
+import { type AudioChunk } from './audio-recorder';
+import { alignWhisperSegments, type WhisperSegment } from './transcriber';
+
+const T0 = Date.parse('2026-01-01T00:00:00.000Z');
+
+const chunk = (offsetMs = 0, samples = 16): AudioChunk => ({
+  timestamp: T0 + offsetMs,
+  data: new Float64Array(samples),
+});
+
+const segment = (overrides: Partial<WhisperSegment>): WhisperSegment => ({
+  text: '',
+  start: 0,
+  end: 0,
+  no_speech_prob: 0,
+  words: [],
+  ...overrides,
+});
+
+describe('alignWhisperSegments', () => {
+  test('returns empty when no chunks were transcribed', ({ expect }) => {
+    const result = alignWhisperSegments(
+      [segment({ text: 'x', end: 1, words: [{ word: 'x', start: 0, end: 1 }] })],
+      [],
+      0,
+    );
+    expect(result.transcripts).toEqual([]);
+    expect(result.lastUsedTimestamp).toBeUndefined();
+  });
+
+  test('absolute-timestamps and deduplicates segments against lastUsedTimestamp', ({ expect }) => {
+    const segments: WhisperSegment[] = [
+      segment({
+        text: 'hello world',
+        start: 0,
+        end: 1,
+        words: [
+          { word: 'hello ', start: 0, end: 0.5 },
+          { word: 'world', start: 0.5, end: 1 },
+        ],
+      }),
+    ];
+
+    const first = alignWhisperSegments(segments, [chunk(0)], 0);
+    expect(first.transcripts).toHaveLength(1);
+    expect(first.transcripts[0]._tag).toBe('transcript');
+    expect(first.transcripts[0].text).toBe('hello world');
+    expect(new Date(first.transcripts[0].started).getTime()).toBe(T0);
+    expect(first.lastUsedTimestamp).toBe(T0 + 1_000);
+
+    // Same segments, same chunk start: every word's absolute start is < lastUsedTimestamp -> filtered out.
+    const second = alignWhisperSegments(segments, [chunk(0)], first.lastUsedTimestamp!);
+    expect(second.transcripts).toEqual([]);
+    expect(second.lastUsedTimestamp).toBeUndefined();
+  });
+
+  test('keeps later words when earlier words fall before lastUsedTimestamp', ({ expect }) => {
+    const segments: WhisperSegment[] = [
+      segment({
+        text: 'old new',
+        start: 0,
+        end: 2,
+        words: [
+          { word: 'old ', start: 0, end: 1 },
+          { word: 'new', start: 1.5, end: 2 },
+        ],
+      }),
+    ];
+
+    // First word starts at T0, second at T0+1500 — only the second survives a cursor at T0+1000.
+    const result = alignWhisperSegments(segments, [chunk(0)], T0 + 1_000);
+    expect(result.transcripts).toHaveLength(1);
+    expect(result.transcripts[0].text).toBe('new');
+    expect(new Date(result.transcripts[0].started).getTime()).toBe(T0 + 1_500);
+    expect(result.lastUsedTimestamp).toBe(T0 + 2_000);
+  });
+
+  test('drops segments that end before lastUsedTimestamp', ({ expect }) => {
+    const segments: WhisperSegment[] = [
+      segment({
+        text: 'stale',
+        start: 0,
+        end: 0.5,
+        words: [{ word: 'stale', start: 0, end: 0.5 }],
+      }),
+      segment({
+        text: 'fresh',
+        start: 1,
+        end: 2,
+        words: [{ word: 'fresh', start: 1.5, end: 2 }],
+      }),
+    ];
+
+    const result = alignWhisperSegments(segments, [chunk(0)], T0 + 1_000);
+    expect(result.transcripts).toHaveLength(1);
+    expect(result.transcripts[0].text).toBe('fresh');
+  });
+});