@dxos/plugin-transcription 0.9.1-main.c7dcc2e112 → 0.10.0

package/src/skills/transcription-skill.ts

@@ -0,0 +1,29 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import { Skill, Template } from '@dxos/compute';
+import { trim } from '@dxos/util';
+
+import { TranscriptOperation } from '#types';
+
+const SKILL_KEY = 'org.dxos.skill.transcription';
+
+const make = () =>
+  Skill.make({
+    key: SKILL_KEY,
+    name: 'Transcription',
+    tools: Skill.toolDefinitions({ operations: [TranscriptOperation.Open, TranscriptOperation.Summarize] }),
+    instructions: Template.make({
+      source: trim`
+        You can open and summarize a meeting transcript.
+      `,
+    }),
+  });
+
+const skill: Skill.Definition = {
+  key: SKILL_KEY,
+  make,
+};
+
+export default skill;

package/src/stories/Keyword.stories.tsx

@@ -0,0 +1,98 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+/**
+ * Keyword-spotting demo on the browser's Web Speech API (no DXOS transcriber), driven by the real
+ * `Mic` button so the mic UX matches the rest of the suite.
+ *
+ * - `Default` lists the keywords and checks each as it is heard.
+ * - The toolbar `Mic` toggles the shared `RecordingSession`; the story observes it (`useRecordingSession`)
+ *   to start/stop `window.SpeechRecognition` (continuous + interim results). The editor driver stays
+ *   idle (no editor view for this session).
+ * - Matching normalizes punctuation/whitespace; matched rows show a check via `react-ui-list` `Listbox`.
+ */
+
+import { type Meta, type StoryObj } from '@storybook/react-vite';
+import React, { useState } from 'react';
+
+import { useAtomCapabilityState } from '@dxos/app-framework/ui';
+import { Panel, Toolbar } from '@dxos/react-ui';
+import { Listbox } from '@dxos/react-ui-list';
+import { useSpeechRecognition } from '@dxos/react-ui-transcription';
+
+import { Mic } from '#components';
+import { translations } from '#translations';
+import { TranscriptionCapabilities } from '#types';
+
+import { createStoryDecorators } from '../testing';
+
+// Stable session key for the Mic button; any non-editor id works (the editor driver ignores it).
+const DOC_ID = 'keyword-detection';
+
+// Reads the shared recording flag toggled by the `Mic` button (the editor driver stays idle when no
+// editor view is registered for this session).
+const useRecordingSession = (docId: string): boolean => {
+  const [session] = useAtomCapabilityState(TranscriptionCapabilities.RecordingSession);
+  return !!session?.recording && session.id === docId;
+};
+
+type StoryArgs = {
+  keywords: string[];
+};
+
+const DefaultStory = ({ keywords }: StoryArgs) => {
+  const [matched, setMatched] = useState<string | undefined>(undefined);
+  const handleTranscript = (transcript: string) => {
+    const match = keywords.find((keyword) => transcript.includes(keyword));
+    setMatched(match ?? undefined);
+  };
+
+  const recording = useRecordingSession(DOC_ID);
+  useSpeechRecognition({ active: recording, onTranscript: handleTranscript });
+
+  return (
+    <Panel.Root>
+      <Panel.Toolbar asChild>
+        <Toolbar.Root>
+          <Mic docId={DOC_ID} />
+        </Toolbar.Root>
+      </Panel.Toolbar>
+
+      <Panel.Content>
+        <Listbox.Root value={matched}>
+          <Listbox.Viewport thin padding>
+            <Listbox.Content aria-label='Keywords'>
+              {keywords.map((keyword) => (
+                <Listbox.Item key={keyword} id={keyword}>
+                  <Listbox.ItemLabel>{keyword}</Listbox.ItemLabel>
+                </Listbox.Item>
+              ))}
+            </Listbox.Content>
+          </Listbox.Viewport>
+        </Listbox.Root>
+      </Panel.Content>
+    </Panel.Root>
+  );
+};
+
+const meta = {
+  title: 'plugins/plugin-transcription/stories/Keyword',
+  render: DefaultStory,
+  decorators: createStoryDecorators(),
+  parameters: {
+    layout: 'fullscreen',
+    controls: { disable: true },
+    translations,
+  },
+} satisfies Meta<typeof DefaultStory>;
+
+export default meta;
+
+type Story = StoryObj<typeof meta>;
+
+export const Default: Story = {
+  args: {
+    keywords: ['hello', 'hey kai', 'kai', 'computer', 'dxos', 'composer'].sort(),
+  },
+};

package/src/stories/Pipeline.stories.tsx

@@ -0,0 +1,361 @@
+//
+// Copyright 2026 DXOS.org
+//
+
+/**
+ * End-to-end transcription into a real plugin-markdown editor: the single editor-output integration
+ * test, unifying the former MarkdownTranscription, Pipeline and MicPipeline stories.
+ *
+ * - `Live` — live microphone via the real `TranscriptionDriver` (toolbar `Mic` button), streaming
+ *   pending text into the document with inline confirm/cancel + entity linking (requires a mic).
+ * - `Recording` / `RecordingIndicator` — seed the pending-text decoration (no mic) to show the
+ *   recording preview / placeholder deterministically.
+ * - `CorrectionOnly` / `WithExtraction` / `Full` — drive `PipelineRuntime` over a scripted transcript
+ *   (correction → extraction → summarization), writing each stage into the doc; status + per-stage
+ *   telemetry + summary render in the side panel via the shared `PipelineStatus` component.
+ * - Space is seeded with Person/Organization entities so extraction links recognized names.
+ */
+
+import { type StateEffect } from '@codemirror/state';
+import { type Meta, type StoryObj } from '@storybook/react-vite';
+import * as Effect from 'effect/Effect';
+import * as Stream from 'effect/Stream';
+import React, { useEffect, useMemo, useState } from 'react';
+
+import { Capability, Plugin } from '@dxos/app-framework';
+import { withPluginManager } from '@dxos/app-framework/testing';
+import { Surface, useAtomCapability, useCapabilities } from '@dxos/app-framework/ui';
+import { AppActivationEvents, AppCapabilities, AppNode, AppPlugin, AppSpace } from '@dxos/app-toolkit';
+import { AppSurface, useAppGraph } from '@dxos/app-toolkit/ui';
+import { Filter, Query } from '@dxos/echo';
+import { Doc } from '@dxos/echo-doc';
+import { EffectEx } from '@dxos/effect';
+import { DXN } from '@dxos/keys';
+import { ClientCapabilities } from '@dxos/plugin-client';
+import { ClientPlugin, initializeIdentity } from '@dxos/plugin-client/testing';
+import { Graph, GraphBuilder, Node, NodeMatcher, qualifyId } from '@dxos/plugin-graph';
+import { Markdown, MarkdownCapabilities, MarkdownEvents } from '@dxos/plugin-markdown';
+import { MarkdownPlugin } from '@dxos/plugin-markdown/testing';
+import { SpacePlugin } from '@dxos/plugin-space/testing';
+import { corePlugins } from '@dxos/plugin-testing';
+import { useQuery, useSpaces } from '@dxos/react-client/echo';
+import { useAttentionAttributes } from '@dxos/react-ui-attention';
+import { PipelineStatus } from '@dxos/react-ui-transcription';
+import { Loading, withLayout } from '@dxos/react-ui/testing';
+import { Text } from '@dxos/schema';
+import {
+  type CommitFn,
+  PipelineRuntime,
+  type Stage,
+  type TelemetryEvent,
+  TranscriptEvent,
+  makeCorrectionStage,
+  makeDatabaseLookup,
+  makeExtractionStage,
+  makeSummarizationStage,
+} from '@dxos/transcription-pipeline';
+import { type ContentBlock, Organization, Person } from '@dxos/types';
+import { seedTestData } from '@dxos/types/testing';
+import { appendPendingText, cancelPendingText, setPendingAnchor, setPendingInterim } from '@dxos/ui-editor';
+import { isNonNullable, trim } from '@dxos/util';
+
+import { translations } from '#translations';
+import { TranscriptionCapabilities } from '#types';
+
+import { enableQueryIndexes } from '../testing';
+import { TranscriptionPlugin } from '../TranscriptionPlugin';
+
+const SAMPLE_CONTENT = trim`
+  # Test
+
+  Place the cursor here, then click the microphone in the toolbar to start recording.
+  Live transcription appears inline as greyed text — confirm (✓) to insert it into the document, or cancel (✕) to discard.
+
+`;
+
+const TRANSCRIPT = trim`
+  So I caught up with Sarah Johnson this morning
+  We talked about the DXOS and Cyberdyne partnership
+  Amco might join the project later this year
+  I should follow up with Michael Chen next week
+`.split('\n');
+
+// Story-only plugin exposing Markdown documents in the personal space as direct children of the graph
+// root, so TranscriptionPlugin's toolbar extension can attach the record action to the doc's node.
+const StoryGraphPlugin = () =>
+  Plugin.define(
+    Plugin.makeMeta({
+      key: DXN.make('org.dxos.plugin.transcription.story.pipelineGraph'),
+      name: 'Transcription Pipeline Story Graph',
+    }),
+  ).pipe(
+    AppPlugin.addAppGraphModule({
+      activate: Effect.fnUntraced(function* () {
+        const capabilities = yield* Capability.Service;
+        const extensions = yield* GraphBuilder.createExtension({
+          id: 'storyDocs',
+          match: NodeMatcher.whenRoot,
+          connector: (_, get) =>
+            Effect.gen(function* () {
+              // Tolerate the teardown window when stories swap: the Client capability may already be
+              // removed while this reactive connector recomputes once more (use `getAll`, not the
+              // throwing `get`).
+              const [client] = capabilities.getAll(ClientCapabilities.Client);
+              const space = client && AppSpace.getPersonalSpace(client);
+              if (!space) {
+                return [];
+              }
+
+              const docs = get(space.db.query(Filter.type(Markdown.Document)).atom);
+              return docs
+                .map((object) => AppNode.makeObject({ get, db: space.db, object, droppable: false }))
+                .filter(isNonNullable);
+            }),
+        });
+        return Capability.contributes(AppCapabilities.AppGraphBuilder, extensions);
+      }),
+    }),
+    Plugin.make,
+  )();
+
+type StageId = 'correct' | 'extract' | 'summarize';
+const STAGE_FACTORY: Record<StageId, () => Stage<any, any>> = {
+  correct: makeCorrectionStage,
+  extract: makeExtractionStage,
+  summarize: makeSummarizationStage,
+};
+
+type StoryArgs = {
+  /**
+   * When set, drives `PipelineRuntime` over the scripted transcript through these stages.
+   */
+  stages?: readonly StageId[];
+  /**
+   * Seeds the pending-text decoration directly (final/interim), bypassing audio capture, to
+   * showcase the recording-state preview deterministically (no microphone).
+   */
+  seed?: { final?: string; interim?: string };
+};
+
+const DefaultStory = ({ stages, seed }: StoryArgs) => {
+  const { graph } = useAppGraph();
+  const [space] = useSpaces();
+  const [doc] = useQuery(space?.db, Query.type(Markdown.Document));
+  const attendableId = doc && qualifyId(Node.RootId, doc.id);
+  // Mark the editor attended so its toolbar (and the contributed record action) are active.
+  const attentionAttrs = useAttentionAttributes(attendableId);
+  const [editorViews] = useCapabilities(MarkdownCapabilities.EditorViews);
+  const status = useAtomCapability(TranscriptionCapabilities.PipelineStatus);
+  const [telemetry, setTelemetry] = useState<TelemetryEvent[]>([]);
+  const [summary, setSummary] = useState<string>();
+
+  // The status panel is relevant for the live (driver phase) and scripted (telemetry) variants; the
+  // seeded recording-preview variants are about the inline editor affordance only.
+  const showStatus = !seed;
+
+  // Story renders the surface directly (no deck), so expand the doc node's actions.
+  useEffect(() => {
+    if (attendableId) {
+      void Graph.expand(graph, attendableId, 'action');
+    }
+  }, [graph, attendableId]);
+
+  // Seed the pending-text decoration once the editor view has registered (no mic required).
+  useEffect(() => {
+    if (!attendableId || !editorViews) {
+      return;
+    }
+    // Clear any previously-seeded decoration when seeding stops.
+    if (!seed) {
+      editorViews.get(attendableId)?.view.dispatch({ effects: cancelPendingText.of() });
+      return;
+    }
+    let cancelled = false;
+    const trySeed = () => {
+      const entry = editorViews.get(attendableId);
+      if (!entry) {
+        return false;
+      }
+      // Anchor the pending block at the end of the document.
+      const anchor = entry.view.state.doc.length;
+      const effects: StateEffect<unknown>[] = [setPendingAnchor.of({ anchor, placeholder: 'Recording…' })];
+      if (seed.final) {
+        effects.push(appendPendingText.of(seed.final));
+      }
+      if (seed.interim) {
+        effects.push(setPendingInterim.of(seed.interim));
+      }
+      entry.view.dispatch({ effects });
+      return true;
+    };
+    if (trySeed()) {
+      return;
+    }
+
+    const interval = setInterval(() => {
+      if (cancelled || trySeed()) {
+        clearInterval(interval);
+      }
+    }, 50);
+    return () => {
+      cancelled = true;
+      clearInterval(interval);
+    };
+  }, [seed, attendableId, editorViews]);
+
+  // Run the pipeline over the scripted transcript, writing each stage's output into the document.
+  useEffect(() => {
+    const target = doc?.content?.target;
+    if (!stages || !space || !target) {
+      return;
+    }
+    let cancelled = false;
+    setTelemetry([]);
+    setSummary(undefined);
+
+    const blocks: ContentBlock.Transcript[] = TRANSCRIPT.map((text, index) => ({
+      _tag: 'transcript',
+      started: new Date(index * 1000).toISOString(),
+      text,
+    }));
+    const stageList = stages.map((id) => STAGE_FACTORY[id]());
+
+    // Re-render the whole document body from the (mutated-in-place) blocks; preferring corrected text.
+    const render = () => {
+      const body = blocks.map((block) => block.corrected ?? block.text).join('\n\n');
+      Doc.updateText(target, 'content', `# Transcript\n\n${body}\n`);
+    };
+    render();
+
+    const commit: CommitFn = (write, window) =>
+      Effect.sync(() => {
+        if (cancelled) {
+          return;
+        }
+        for (const update of write.blockUpdates ?? []) {
+          const block = window[update.index];
+          if (block) {
+            const { index: _index, ...patch } = update;
+            Object.assign(block, patch);
+          }
+        }
+        if (write.transcriptUpdate?.summary) {
+          setSummary(write.transcriptUpdate.summary);
+        }
+        render();
+      });
+
+    const source = Stream.fromIterable([
+      ...blocks.map((block) => TranscriptEvent.block(block)),
+      TranscriptEvent.silence(5_000),
+    ]);
+    void EffectEx.runPromise(
+      PipelineRuntime.run({
+        source,
+        stages: stageList,
+        lookup: makeDatabaseLookup(space.db),
+        commit,
+        onTelemetry: (event) => {
+          if (!cancelled) {
+            setTelemetry((prev) => [...prev, event]);
+          }
+        },
+      }),
+    );
+
+    return () => {
+      cancelled = true;
+    };
+  }, [space, doc, stages]);
+
+  const data = useMemo(() => ({ subject: doc, attendableId: attendableId ?? 'story' }), [doc, attendableId]);
+
+  if (!doc) {
+    return <Loading data={{ doc: !!doc }} />;
+  }
+
+  if (!showStatus) {
+    return (
+      <div className='contents' {...attentionAttrs}>
+        <Surface.Surface type={AppSurface.Article} data={data} limit={1} />
+      </div>
+    );
+  }
+
+  return (
+    <div role='none' className='dx-container grid grid-cols-[1fr_20rem] gap-2' {...attentionAttrs}>
+      <div role='none' className='dx-expander'>
+        <Surface.Surface type={AppSurface.Article} data={data} limit={1} />
+      </div>
+      <PipelineStatus
+        phase={stages ? 'idle' : (status?.phase ?? 'idle')}
+        stages={stages}
+        telemetry={telemetry}
+        summary={summary}
+      />
+    </div>
+  );
+};
+
+const meta = {
+  title: 'plugins/plugin-transcription/stories/Pipeline',
+  render: DefaultStory,
+  decorators: [
+    withLayout({ layout: 'fullscreen' }),
+    withPluginManager({
+      setupEvents: [AppActivationEvents.SetupSettings, MarkdownEvents.SetupExtensions],
+      plugins: [
+        ...corePlugins(),
+        ClientPlugin({
+          types: [Markdown.Document, Text.Text, Person.Person, Organization.Organization],
+          onClientInitialized: ({ client }) =>
+            Effect.gen(function* () {
+              const { personalSpace } = yield* initializeIdentity(client);
+              // Vector indexes so extraction can link recognized Person/Organization names.
+              yield* enableQueryIndexes(client.services.services);
+              yield* Effect.promise(() => seedTestData(personalSpace));
+              personalSpace.db.add(Markdown.make({ name: 'Transcript', content: SAMPLE_CONTENT }));
+              yield* Effect.promise(() => personalSpace.db.flush({ indexes: true }));
+            }),
+        }),
+        SpacePlugin({}),
+        MarkdownPlugin(),
+        StoryGraphPlugin(),
+        TranscriptionPlugin(),
+      ],
+    }),
+  ],
+  parameters: {
+    layout: 'fullscreen',
+    controls: { disable: true },
+    translations,
+  },
+} satisfies Meta<typeof DefaultStory>;
+
+export default meta;
+
+type Story = StoryObj<typeof meta>;
+
+/** Live microphone via the real driver: streams transcription into the doc (requires a mic). */
+export const Live: Story = {};
+
+/** Scripted: correction only (punctuation / capitalization). */
+export const WithCorrection: Story = {
+  args: {
+    stages: ['correct'],
+  },
+};
+
+/** Scripted: correction + entity extraction (recognized entities become dx-anchor links). */
+export const WithExtraction: Story = {
+  args: {
+    stages: ['correct', 'extract'],
+  },
+};
+
+/** Scripted: correction + extraction + a cumulative summary (shown in the side panel). */
+export const Full: Story = {
+  args: {
+    stages: ['correct', 'extract', 'summarize'],
+  },
+};

package/src/stories/Transcription.stories.tsx

@@ -0,0 +1,178 @@
+//
+// Copyright 2026 DXOS.org
+//
+
+/**
+ * Streaming file transcription into a message list. Loads an audio file and streams its transcript
+ * through the recording engine (Transcriber → runLivePipeline → correction stage) **as it plays** —
+ * appending each corrected block as a message rather than waiting for playback to stop.
+ *
+ * - `Default` plays a hosted m4a; the play/pause + upload toolbar gates playback and capture.
+ * - A small chunk threshold drives incremental transcription every few seconds (the streaming stack).
+ */
+
+import { type Meta, type StoryObj } from '@storybook/react-vite';
+import * as Effect from 'effect/Effect';
+import React, { type ChangeEvent, useCallback, useEffect, useMemo, useRef, useState } from 'react';
+
+import { IconButton, Panel, ScrollContainer, Toolbar } from '@dxos/react-ui';
+import {
+  Transcription,
+  renderByline,
+  useAudioFile,
+  useFeedModelAdapter,
+  useRecordingPipeline,
+} from '@dxos/react-ui-transcription';
+import { type CommitFn, type TranscribeConfig, makeCorrectionStage } from '@dxos/transcription-pipeline';
+import { type ContentBlock, Message } from '@dxos/types';
+
+import { createStoryDecorators } from '../testing';
+
+// Small chunk threshold so the transcriber emits every few seconds while the file plays (streaming),
+// instead of only flushing the whole buffer on stop.
+const STREAMING_TRANSCRIBE_CONFIG: Partial<TranscribeConfig> = {
+  transcribeAfterChunksAmount: 25,
+  prefixBufferChunksAmount: 10,
+};
+
+// In-memory message buffer + model adapter (production wires up a real space-backed `Feed`).
+const useStoryMessageModel = () => {
+  const [messages, setMessages] = useState<Message.Message[]>([]);
+  const appendMessage = useCallback((message: Message.Message) => setMessages((prev) => [...prev, message]), []);
+  const model = useFeedModelAdapter(renderByline([]), messages);
+  return { model, appendMessage };
+};
+
+type StoryArgs = {
+  audioUrl: string;
+  audioConstraints?: MediaTrackConstraints;
+};
+
+const DefaultStory = ({ audioUrl, audioConstraints }: StoryArgs) => {
+  // Decode the file/URL into a track; a play/pause button gates playback + capture.
+  const [running, setRunning] = useState(false);
+  const [uploadedUrl, setUploadedUrl] = useState<string>();
+  useEffect(() => {
+    return () => {
+      if (uploadedUrl) {
+        URL.revokeObjectURL(uploadedUrl);
+      }
+    };
+  }, [uploadedUrl]);
+
+  const fileInputRef = useRef<HTMLInputElement>(null);
+  const handleFileChange = useCallback((event: ChangeEvent<HTMLInputElement>) => {
+    const file = event.target.files?.[0];
+    if (file) {
+      setRunning(false);
+      setUploadedUrl(URL.createObjectURL(file));
+    }
+    // Reset so the same file can be re-selected.
+    event.target.value = '';
+  }, []);
+  const { audio, stream, track } = useAudioFile(uploadedUrl ?? audioUrl, audioConstraints);
+
+  // Pipe the decoded stream into the <audio> element so the user hears playback, mirroring `running`.
+  const audioRef = useRef<HTMLAudioElement>(null);
+  useEffect(() => {
+    if (audioRef.current && stream) {
+      audioRef.current.srcObject = stream;
+    }
+  }, [stream]);
+  useEffect(() => {
+    if (!audio) {
+      return;
+    }
+    if (running) {
+      void audio.play();
+    } else {
+      audio.pause();
+    }
+  }, [audio, running]);
+
+  // Append one message per committed block (keyed by block identity so re-commits don't duplicate).
+  const { model, appendMessage } = useStoryMessageModel();
+  const seen = useRef(new WeakSet<ContentBlock.Transcript>());
+  const commit = useCallback<CommitFn>(
+    (write, window) =>
+      Effect.sync(() => {
+        for (const update of write.blockUpdates ?? []) {
+          const block = window[update.index];
+          if (!block || seen.current.has(block)) {
+            continue;
+          }
+          seen.current.add(block);
+          appendMessage(
+            Message.make({
+              sender: { name: 'You' },
+              created: new Date().toISOString(),
+              blocks: [{ ...block, text: update.corrected ?? block.text }],
+            }),
+          );
+        }
+      }),
+    [appendMessage],
+  );
+
+  const stages = useMemo(() => [makeCorrectionStage()], []);
+  useRecordingPipeline({
+    config: STREAMING_TRANSCRIBE_CONFIG,
+    segmentSentences: true,
+    active: running,
+    track,
+    stages,
+    commit,
+  });
+
+  return (
+    <>
+      <audio ref={audioRef} autoPlay />
+      <Panel.Root>
+        <Panel.Toolbar asChild>
+          <Toolbar.Root>
+            <IconButton
+              iconOnly
+              disabled={!stream}
+              icon={running ? 'ph--pause--regular' : 'ph--play--regular'}
+              label={running ? 'Pause' : 'Play'}
+              onClick={() => setRunning((value) => !value)}
+            />
+            <input ref={fileInputRef} type='file' accept='audio/*' className='hidden' onChange={handleFileChange} />
+            <IconButton
+              iconOnly
+              icon='ph--upload--regular'
+              label='Upload audio'
+              onClick={() => fileInputRef.current?.click()}
+            />
+          </Toolbar.Root>
+        </Panel.Toolbar>
+        <ScrollContainer.Root pin>
+          <Panel.Content asChild>
+            <ScrollContainer.Content>
+              <ScrollContainer.Viewport>
+                <Transcription model={model} />
+              </ScrollContainer.Viewport>
+            </ScrollContainer.Content>
+          </Panel.Content>
+        </ScrollContainer.Root>
+      </Panel.Root>
+    </>
+  );
+};
+
+const meta = {
+  title: 'plugins/plugin-transcription/stories/Transcription',
+  render: DefaultStory,
+  decorators: createStoryDecorators(),
+} satisfies Meta<StoryArgs>;
+
+export default meta;
+
+type Story = StoryObj<typeof meta>;
+
+export const Default: Story = {
+  args: {
+    // https://learnenglish.britishcouncil.org/general-english/audio-zone/living-london
+    audioUrl: 'https://dxos.network/audio-london.m4a',
+  },
+};