@dxos/plugin-transcription 0.9.0 → 0.9.1-staging.ee54ba693a

package/src/model/model.ts

@@ -1,229 +0,0 @@
-//
-// Copyright 2025 DXOS.org
-//
-
-import { Text } from '@codemirror/state';
-import { type EditorView } from '@codemirror/view';
-
-import { Event } from '@dxos/async';
-import { invariant } from '@dxos/invariant';
-import { log } from '@dxos/log';
-
-export type Chunk = { id: string };
-
-/**
- * Covert block to a set of markdown lines.
- */
-export type ChunkRenderer<T extends Chunk> = (chunk: T, index: number, debug?: boolean) => string[];
-
-/**
- * Abstract representation of the document model.
- */
-export interface ChunkDocument {
-  lineCount(): number;
-  replaceLines(from: number, remove: number, insert: string[]): void;
-}
-
-/**
- * Codemirror document adapter.
- */
-export class DocumentAdapter implements ChunkDocument {
-  constructor(private readonly _view: EditorView) {}
-
-  /** The document must always have at least one line. */
-  lineCount(): number {
-    return this._view.state.doc.lines - 1;
-  }
-
-  replaceLines(from: number, remove: number, insert: string[]): void {
-    log('replaceLines', { count: this._view.state.doc.lines, from, remove, insert });
-
-    const numLines = this._view.state.doc.lines;
-    const posFrom = this._view.state.doc.line(from).from;
-    const posTo = remove === 0 ? posFrom : this._view.state.doc.line(Math.min(numLines, from + remove)).from;
-
-    // Don't remove the last line.
-    let text = insert.join('\n');
-    if (posTo === this._view.state.doc.length) {
-      text += '\n';
-    }
-
-    this._view.dispatch({
-      changes: { from: posFrom, to: posTo, insert: text },
-    });
-  }
-}
-
-type ChunkChange<T extends Chunk> =
-  | {
-      type: 'append';
-      index: number;
-      chunk: T;
-    }
-  | {
-      type: 'update';
-      index: number;
-      chunk: T;
-    }
-  | {
-      type: 'delete';
-      index: number;
-      id: string;
-    };
-
-/**
- * Each Chunk can be converted into a set of lines.
- * Chunks can be added, updated or deleted from the Queue.
- * A Document is an immutable projection that represents a flat list of lines corresponding to the Queue.
- *
- * Ideally we would implement a custom virtual Text model for the View, but this currently isn't possible in Codemirror.
- * Instead this model tracks changes and syncs them with the ChunkDocument.
- */
-export class SerializationModel<T extends Chunk> {
-  /** Emits when the document is updated. */
-  public readonly update = new Event<void>();
-
-  /** Ordered set of chunks. */
-  private readonly _chunks: T[] = [];
-
-  /** Map of chunk IDs to their current line counts. */
-  private readonly _chunkLineCounts = new Map<string, number>();
-
-  /** Map of line numbers to chunk IDs. */
-  private readonly _lineToChunk = new Map<number, T>();
-
-  /** Track chunk changes since last sync. */
-  private readonly _changes: Array<ChunkChange<T>> = [];
-
-  constructor(
-    private readonly _renderer: ChunkRenderer<T>,
-    initialChunks: T[] = [],
-  ) {
-    initialChunks.forEach((chunk) => {
-      this.appendChunk(chunk);
-    });
-  }
-
-  toJSON(): { chunks: number; changes: number } {
-    return {
-      chunks: this._chunks.length,
-      changes: this._changes.length,
-    };
-  }
-
-  get doc() {
-    return Text.of([...this._chunks.flatMap((chunk, index) => this._renderer(chunk, index)), '']);
-  }
-
-  get chunks() {
-    return this._chunks;
-  }
-
-  /**
-   * Get the block ID at the given line number.
-   */
-  getChunkAtLine(line: number): T | undefined {
-    return this._lineToChunk.get(line);
-  }
-
-  reset(): this {
-    this._chunks.length = 0;
-    this._chunkLineCounts.clear();
-    this._lineToChunk.clear();
-    this.update.emit();
-    return this;
-  }
-
-  appendChunk(chunk: T): this {
-    const index = this._chunks.length;
-    this._chunks.push(chunk);
-    this._changes.push({ type: 'append', index, chunk });
-    this.update.emit();
-    return this;
-  }
-
-  updateChunk(chunk: T): this {
-    const index = this._chunks.findIndex((c) => c.id === chunk.id);
-    invariant(index !== -1);
-    this._chunks[index] = chunk;
-    this._changes.push({ type: 'update', index, chunk });
-    this.update.emit();
-    return this;
-  }
-
-  deleteBlock(id: string): this {
-    const index = this._chunks.findIndex((c) => c.id === id);
-    invariant(index !== -1);
-    this._chunks.splice(index, 1);
-    this._changes.push({ type: 'delete', index, id });
-    this.update.emit();
-    return this;
-  }
-
-  /**
-   * Syncs the tracked changes with the ChunkDocument.
-   */
-  sync(document: ChunkDocument): this {
-    log('sync', { changes: this._changes });
-
-    // Process each change in order.
-    for (const change of this._changes) {
-      switch (change.type) {
-        case 'append': {
-          // For appends, convert block to lines and add at the end.
-          const lines = this._renderer(change.chunk, change.index);
-          const lastLine = document.lineCount();
-          document.replaceLines(lastLine + 1, 0, lines);
-          break;
-        }
-
-        case 'update': {
-          // For updates, find the block's line range and replace with new lines.
-          let lineStart = 1;
-          for (let i = 0; i < change.index; i++) {
-            lineStart += this._chunkLineCounts.get(this._chunks[i].id)!;
-          }
-          const previousLineCount = this._chunkLineCounts.get(change.chunk.id)!;
-          const newLines = this._renderer(change.chunk, change.index);
-          document.replaceLines(lineStart, previousLineCount, newLines);
-          break;
-        }
-
-        case 'delete': {
-          // For deletes, find the block's line range and remove those lines.
-          let lineStart = 1;
-          for (let i = 0; i < change.index; i++) {
-            lineStart += this._chunkLineCounts.get(this._chunks[i].id)!;
-          }
-          const previousLineCount = this._chunkLineCounts.get(change.id)!;
-          document.replaceLines(lineStart, previousLineCount, []);
-          break;
-        }
-      }
-    }
-
-    // Clear the line-to-block mapping.
-    this._lineToChunk.clear();
-
-    // Update all block line counts and line-to-block mapping after processing changes.
-    let currentLine = 1;
-    for (const [index, chunk] of this._chunks.entries()) {
-      const lineCount = this._renderer(chunk, index).length;
-      this._chunkLineCounts.set(chunk.id, lineCount);
-      this._lineToChunk.set(currentLine, chunk);
-      currentLine += lineCount;
-    }
-
-    // Clean up deleted blocks.
-    const chunkIds = new Set(this._chunks.map((chunk) => chunk.id));
-    for (const id of this._chunkLineCounts.keys()) {
-      if (!chunkIds.has(id)) {
-        this._chunkLineCounts.delete(id);
-      }
-    }
-
-    // Clear changes after sync.
-    this._changes.length = 0;
-    return this;
-  }
-}

package/src/stories/FileTranscription.stories.tsx

@@ -1,143 +0,0 @@
-//
-// Copyright 2025 DXOS.org
-//
-
-import { type Meta, type StoryObj } from '@storybook/react-vite';
-import React, { useCallback, useEffect, useRef, useState } from 'react';
-
-import { useAudioFile } from '#hooks';
-
-import { type MediaStreamRecorderProps, type TranscriberProps } from '../transcriber';
-import {
-  useIsSpeaking,
-  createStoryDecorators,
-  useStoryAppendSegments,
-  useStoryMessageModel,
-  useStoryTranscriber,
-} from './common';
-import { TranscriptionStory } from './TranscriptionStory';
-
-const DEFAULT_TRANSCRIBER_CONFIG = {
-  transcribeAfterChunksAmount: 100,
-  prefixBufferChunksAmount: 50,
-  normalizeSentences: true,
-};
-
-const DEFAULT_RECORDER_CONFIG = {
-  interval: 200,
-};
-
-type DefaultStoryProps = {
-  detectSpeaking?: boolean;
-  normalizeSentences?: boolean;
-  audioUrl: string;
-  transcriberConfig?: TranscriberProps['config'];
-  recorderConfig?: MediaStreamRecorderProps['config'];
-  audioConstraints?: MediaTrackConstraints;
-};
-
-const DefaultStory = ({
-  detectSpeaking,
-  audioUrl,
-  normalizeSentences,
-  transcriberConfig = DEFAULT_TRANSCRIBER_CONFIG,
-  recorderConfig = DEFAULT_RECORDER_CONFIG,
-  audioConstraints,
-}: DefaultStoryProps) => {
-  const [running, setRunning] = useState(false);
-
-  // Optional uploaded file overrides the default URL; revoke the previous URL on change/unmount.
-  const [uploadedUrl, setUploadedUrl] = useState<string>();
-  useEffect(() => {
-    return () => {
-      if (uploadedUrl) {
-        URL.revokeObjectURL(uploadedUrl);
-      }
-    };
-  }, [uploadedUrl]);
-
-  const handleUpload = useCallback((file: File) => {
-    setRunning(false);
-    setUploadedUrl(URL.createObjectURL(file));
-  }, []);
-
-  const sourceUrl = uploadedUrl ?? audioUrl;
-  const { audio, track, stream } = useAudioFile(sourceUrl, audioConstraints);
-  const isSpeaking = detectSpeaking ? useIsSpeaking(track) : true;
-
-  // Pipe the decoded stream into the <audio> element so the user hears playback.
-  const ref = useRef<HTMLAudioElement>(null);
-  useEffect(() => {
-    if (ref.current && stream) {
-      ref.current.srcObject = stream;
-    }
-  }, [stream]);
-
-  // Mirror the toolbar's running flag to the underlying HTMLAudioElement.
-  useEffect(() => {
-    if (!audio) {
-      return;
-    }
-
-    if (running) {
-      void audio.play();
-    } else {
-      void audio.pause();
-    }
-  }, [audio, running]);
-
-  const { model, appendMessage } = useStoryMessageModel();
-  const handleSegments = useStoryAppendSegments(appendMessage);
-
-  useStoryTranscriber({
-    audioStreamTrack: track,
-    running,
-    isSpeaking,
-    transcriberConfig,
-    recorderConfig,
-    onSegments: handleSegments,
-  });
-
-  // TODO(burdon): Sentence normalization moved to require a real space-backed Feed + FeedService runtime.
-  //  Re-enable here once the story has access to one.
-  void normalizeSentences;
-
-  return (
-    <TranscriptionStory
-      audioRef={ref}
-      disabled={!stream}
-      model={model}
-      running={running}
-      onRunningChange={setRunning}
-      onUpload={handleUpload}
-    />
-  );
-};
-
-const meta = {
-  title: 'plugins/plugin-transcription/stories/FileTranscription',
-  render: DefaultStory,
-  decorators: createStoryDecorators(),
-} satisfies Meta<DefaultStoryProps>;
-
-export default meta;
-
-type Story = StoryObj<typeof meta>;
-
-export const Default: Story = {
-  args: {
-    detectSpeaking: true,
-    // https://learnenglish.britishcouncil.org/general-english/audio-zone/living-london
-    audioUrl: 'https://dxos.network/audio-london.m4a',
-    // textUrl: 'https://dxos.network/audio-london.txt',
-  },
-};
-
-// TODO(mykola): Fix sentence normalization.
-// export const WithSentenceNormalization: StoryObj<typeof AudioFile> = {
-//   args: {
-//     detectSpeaking: true,
-//     normalizeSentences: true,
-//     audioUrl: 'https://dxos.network/audio-london.m4a',
-//   },
-// };

package/src/stories/KeyWordRecognition.stories.tsx

@@ -1,132 +0,0 @@
-//
-// Copyright 2025 DXOS.org
-//
-
-import { type Meta, type StoryObj } from '@storybook/react-vite';
-import React, { useEffect, useMemo, useState } from 'react';
-
-import { log } from '@dxos/log';
-import { Panel, Toolbar } from '@dxos/react-ui';
-import { withLayout, withTheme } from '@dxos/react-ui/testing';
-import { mx } from '@dxos/ui-theme';
-
-type Word = {
-  text: string;
-  matched: boolean;
-};
-
-type DefaultStoryProps = {
-  keywords: string[];
-};
-
-const DefaultStory = ({ keywords }: DefaultStoryProps) => {
-  const [running, setRunning] = useState(false);
-  const [matchingWords, setMatchingWords] = useState<Word[]>(keywords.map((word) => ({ text: word, matched: false })));
-
-  const recognition = useMemo(() => {
-    // TODO(mykola): Fix types
-    const SpeechRecognition = (window as any).SpeechRecognition || (window as any).webkitSpeechRecognition;
-    const SpeechGrammarList = (window as any).SpeechGrammarList || (window as any).webkitSpeechGrammarList;
-    if (!SpeechRecognition || !SpeechGrammarList) {
-      log.error('Speech recognition not supported in this browser');
-      return;
-    }
-
-    const recognition = new SpeechRecognition();
-    recognition.continuous = true;
-    recognition.interimResults = true;
-    recognition.lang = 'en-US';
-    recognition.maxAlternatives = 1;
-    // Add this line to enable punctuation.
-    recognition.grammars = new SpeechGrammarList();
-
-    // TODO(mykola): Fix types.
-    recognition.onresult = (event: any) => {
-      log.info('recognition result', { event });
-      const current = event.resultIndex;
-      const transcript = event.results[current][0].transcript;
-
-      // Remove punctuation and normalize whitespace for comparison.
-      const normalizeText = (text: string) =>
-        text
-          .toLowerCase()
-          .replace(/[.,!?]/g, '')
-          .replace(/\s+/g, ' ')
-          .trim();
-
-      const normalizedTranscript = normalizeText(transcript);
-
-      // Check each keyword and update matching words.
-      const updatedWords = matchingWords.map((word) => {
-        const normalizedKeyword = normalizeText(word.text);
-        const matched = normalizedTranscript.includes(normalizedKeyword);
-        return { ...word, matched };
-      });
-
-      setMatchingWords(updatedWords);
-    };
-
-    // TODO(mykola): Fix types.
-    recognition.onerror = (event: any) => {
-      log.error('Speech recognition error:', { error: event.error });
-      setRunning(false);
-    };
-
-    return recognition;
-  }, []);
-
-  useEffect(() => {
-    if (running) {
-      recognition?.start();
-    } else {
-      recognition?.stop();
-    }
-  }, [running, recognition]);
-
-  return (
-    <Panel.Root>
-      <Panel.Toolbar asChild>
-        <Toolbar.Root>
-          <Toolbar.IconButton
-            iconOnly
-            icon={running ? 'ph--pause--regular' : 'ph--play--regular'}
-            label={running ? 'Pause' : 'Play'}
-            onClick={() => setRunning((running) => !running)}
-          />
-        </Toolbar.Root>
-      </Panel.Toolbar>
-
-      <Panel.Content>
-        <div className='p-2 grid grid-cols-[repeat(auto-fill,minmax(6rem,1fr))] gap-2'>
-          {matchingWords.map((word, index) => (
-            <span
-              key={index}
-              className={mx(
-                'grid place-items-center p-2 aspect-square rounded-sm border border-separator',
-                word.matched && 'border-red-300',
-              )}
-            >
-              {word.text}
-            </span>
-          ))}
-        </div>
-      </Panel.Content>
-    </Panel.Root>
-  );
-};
-
-const meta = {
-  title: 'plugins/plugin-transcription/stories/KeyWordDetection',
-  render: DefaultStory,
-  decorators: [withTheme(), withLayout({ layout: 'column' })],
-} satisfies Meta<typeof DefaultStory>;
-
-export default meta;
-
-type Story = StoryObj<typeof meta>;
-
-export const SpeechRecognitionAPI: Story = {
-  args: {
-    keywords: ['kai', 'computer', 'hello', 'dxos'],
-  },
-};

package/src/stories/LiveTranscription.stories.tsx

@@ -1,153 +0,0 @@
-//
-// Copyright 2025 DXOS.org
-//
-
-import { type Meta, type StoryObj } from '@storybook/react-vite';
-import React, { useCallback, useMemo, useState } from 'react';
-
-import {
-  type ExtractionFunction,
-  extractionAnthropicFunction,
-  extractionNerFunction,
-  getNer,
-  processTranscriptMessage,
-} from '@dxos/assistant/extraction';
-import { Filter, type Obj } from '@dxos/echo';
-import { invariant } from '@dxos/invariant';
-import { useSpaces } from '@dxos/react-client/echo';
-import { Message, Organization, Person } from '@dxos/types';
-
-import { useAudioTrack } from '#hooks';
-
-import { type MediaStreamRecorderProps, type TranscriberProps } from '../transcriber';
-import { useIsSpeaking, createStoryDecorators, useStoryMessageModel, useStoryTranscriber } from './common';
-import { TranscriptionStory } from './TranscriptionStory';
-
-const DEFAULT_TRANSCRIBER_CONFIG = {
-  transcribeAfterChunksAmount: 50,
-  prefixBufferChunksAmount: 10,
-};
-
-const DEFAULT_RECORDER_CONFIG = {
-  interval: 200,
-};
-
-type DefaultStoryProps = {
-  detectSpeaking?: boolean;
-  transcriberConfig?: TranscriberProps['config'];
-  recorderConfig?: MediaStreamRecorderProps['config'];
-  audioConstraints?: MediaTrackConstraints;
-  entityExtraction?: 'ner' | 'llm';
-};
-
-const DefaultStory = ({
-  detectSpeaking,
-  entityExtraction,
-  transcriberConfig = DEFAULT_TRANSCRIBER_CONFIG,
-  recorderConfig = DEFAULT_RECORDER_CONFIG,
-  audioConstraints,
-}: DefaultStoryProps) => {
-  const [running, setRunning] = useState(false);
-  const track = useAudioTrack(running, audioConstraints);
-  const isSpeaking = detectSpeaking ? useIsSpeaking(track) : true;
-  const { model, appendMessage } = useStoryMessageModel();
-  const [space] = useSpaces();
-
-  // Resolve the entity-extraction function + nearby objects once per `entityExtraction` toggle.
-  const { extractionFunction, objects } = useMemo(() => {
-    if (!space || !entityExtraction) {
-      return {};
-    }
-
-    let extractionFunction: ExtractionFunction | undefined;
-    let objects: Promise<Obj.Unknown[]> | undefined;
-    if (entityExtraction === 'ner') {
-      void getNer(); // Init model loading. Takes time.
-      extractionFunction = extractionNerFunction;
-    } else if (entityExtraction === 'llm') {
-      extractionFunction = extractionAnthropicFunction;
-      objects = space.db.query(Filter.or(Filter.type(Person.Person), Filter.type(Organization.Organization))).run();
-    }
-    return { extractionFunction, objects };
-  }, [entityExtraction, space]);
-
-  // Build a Message from each transcribed batch; optionally enrich via entity extraction.
-  const handleSegments = useCallback<TranscriberProps['onSegments']>(
-    async (blocks) => {
-      const message = Message.make({
-        sender: { name: 'You' },
-        created: new Date().toISOString(),
-        blocks,
-      });
-
-      if (entityExtraction && space) {
-        invariant(extractionFunction, 'extractionFunction is required');
-        const result = await processTranscriptMessage({
-          function: extractionFunction,
-          input: { message, objects: await objects },
-          options: { fallbackToRaw: true, timeout: 30_000 },
-        });
-        appendMessage(result.message);
-      } else {
-        appendMessage(message);
-      }
-    },
-    [appendMessage, space, entityExtraction, extractionFunction, objects],
-  );
-
-  useStoryTranscriber({
-    audioStreamTrack: track,
-    running,
-    isSpeaking,
-    transcriberConfig,
-    recorderConfig,
-    onSegments: handleSegments,
-  });
-
-  return <TranscriptionStory model={model} running={running} onRunningChange={setRunning} />;
-};
-
-const meta = {
-  title: 'plugins/plugin-transcription/stories/LiveTranscription',
-  render: DefaultStory,
-  decorators: createStoryDecorators({ enableVectorIndex: true }),
-} satisfies Meta<typeof DefaultStory>;
-
-export default meta;
-
-type Story = StoryObj<typeof meta>;
-
-export const Default: Story = {
-  args: {
-    detectSpeaking: false,
-    audioConstraints: {
-      echoCancellation: false,
-      noiseSuppression: false,
-      autoGainControl: false,
-    },
-  },
-};
-
-export const SpeechDetection: Story = {
-  args: {
-    detectSpeaking: true,
-    audioConstraints: {
-      echoCancellation: true,
-      noiseSuppression: true,
-      autoGainControl: true,
-    },
-  },
-};
-
-// TODO(mykola): Fix hugging face quota issues.
-// export const EntityExtraction: Story = {
-//   args: {
-//     detectSpeaking: true,
-//     entityExtraction: 'ner',
-//     audioConstraints: {
-//       echoCancellation: true,
-//       noiseSuppression: true,
-//       autoGainControl: true,
-//     },
-//   },
-// };