@dxos/plugin-transcription 0.9.0 → 0.9.1-staging.ee54ba693a

package/src/hooks/useTranscriptionRecording.ts

@@ -5,16 +5,16 @@
 import * as Effect from 'effect/Effect';
 import { useCallback, useEffect, useState } from 'react';
 
-import { createFeedServiceLayer } from '@dxos/client/echo';
-import { Feed, Obj } from '@dxos/echo';
+import { useAtomCapability, useOperationInvoker } from '@dxos/app-framework/ui';
+import { Database, Feed, Obj } from '@dxos/echo';
 import { EffectEx } from '@dxos/effect';
 import { log } from '@dxos/log';
 import { getSpace } from '@dxos/react-client/echo';
 import { useIdentity } from '@dxos/react-client/halo';
+import { useAudioTrack, useTranscriber } from '@dxos/react-ui-transcription';
 import { Message, type Transcript } from '@dxos/types';
 
-import { useAudioTrack } from './useAudioTrack';
-import { useTranscriber } from './useTranscriber';
+import { TranscriptionCapabilities, TranscriptOperation } from '#types';
 
 export type TranscriptionRecording = {
   recording: boolean;
@@ -33,23 +33,41 @@ export const useTranscriptionRecording = (transcript: Transcript.Transcript): Tr
 
   const [recording, setRecording] = useState(false);
   const track = useAudioTrack(recording);
+  const { invokePromise } = useOperationInvoker();
+  const settings = useAtomCapability(TranscriptionCapabilities.Settings);
 
-  // Append a transcribed message batch to the transcript's feed.
+  // Append a transcribed message batch to the transcript's feed, enriching it first (when entity
+  // extraction is enabled) with references to known entities mentioned in the transcript.
   const handleSegments = useCallback(
     async (blocks: Message.Message['blocks']) => {
       if (!space || !feed || blocks.length === 0) {
         return;
       }
       const sender = identity ? { identityDid: identity.did, name: identity.profile?.displayName } : {};
-      const message = Obj.make(Message.Message, {
+      let message = Obj.make(Message.Message, {
         sender,
         created: new Date().toISOString(),
         blocks,
       });
-      const feedServiceLayer = createFeedServiceLayer(space.queues);
-      await Feed.append(feed, [message]).pipe(Effect.provide(feedServiceLayer), EffectEx.runAndForwardErrors);
+
+      // `spaceId` provides the space context required by the process-affinity AiService/Database.
+      // Fall back to the raw message if extraction fails (e.g. AI service unavailable).
+      if (settings?.entityExtraction ?? true) {
+        const { data, error } = await invokePromise(
+          TranscriptOperation.EnrichMessage,
+          { message },
+          { spaceId: space.id },
+        );
+        if (error) {
+          log.warn('entity extraction failed; using raw transcript', { error });
+        } else if (data?.message) {
+          message = data.message;
+        }
+      }
+
+      await Feed.append(feed, [message]).pipe(Effect.provide(Database.layer(space.db)), EffectEx.runAndForwardErrors);
     },
-    [space, feed, identity],
+    [space, feed, identity, invokePromise, settings],
   );
 
   // Drive the transcriber lifecycle off the recording flag + audio track.

package/src/index.ts

@@ -2,10 +2,6 @@
 // Copyright 2025 DXOS.org
 //
 
-// TODO(wittjosiah): Hooks should not be exported from the plugin package at all.
-//   Either refactor callers to not need them or factor them out to a shared package.
-export * from './blueprints';
-export * from './hooks';
 export * from './meta';
-export * from './transcriber';
+export * from './skills';
 export * from './types';

package/src/meta.ts

@@ -3,40 +3,7 @@
 //
 
 import { Plugin } from '@dxos/app-framework';
-import { DXN } from '@dxos/keys';
-import { trim } from '@dxos/util';
 
-export const meta = Plugin.makeMeta({
-  key: DXN.make('org.dxos.plugin.transcription'),
-  name: 'Transcription',
-  author: 'DXOS',
-  description: trim`
-    Real-time voice-to-text transcription for DXOS Composer.
-    Captures microphone audio in short PCM chunks, batches them into WAV payloads, and streams
-    the results to a Whisper-compatible speech-recognition endpoint.
-    Transcribed segments are timestamped, deduplicated across chunk boundaries, and persisted
-    as Message objects in an ECHO Feed so that every participant in a shared space sees the
-    live transcript without any manual sync step.
+import config from '../dx.config';
 
-    Each recording session is represented as a Transcript object linked to its Feed.
-    The read-only editor renders messages with speaker bylines and wall-clock timestamps,
-    supporting markdown decoration and document preview.
-    An optional message enricher pipeline lets plugins annotate messages before they are
-    committed — for example, to attach speaker identity from a calls service.
-
-    The plugin ships two blueprint-compatible operations: Open (loads and formats a transcript
-    for reading by an AI assistant) and Summarize (sends the transcript text to an LLM and
-    returns a structured Markdown summary with key points, verbatim quotes, and actionable
-    tasks extracted from the conversation).
-
-    A sentence-normalization pass merges raw word-level tokens from the recognizer into
-    grammatically complete sentences, improving readability of the stored transcript without
-    altering the underlying audio timeline.
-  `,
-  icon: 'ph--microphone--regular',
-  iconHue: 'sky',
-  source: 'https://github.com/dxos/dxos/tree/main/packages/plugins/plugin-transcription',
-  spec: 'PLUGIN.mdl',
-  tags: ['labs'],
-  screenshots: [],
-});
+export const meta = Plugin.getMetaFromConfig(config);

package/src/normalization/message-normalizer.ts

@@ -7,7 +7,7 @@ import type * as Runtime from 'effect/Runtime';
 
 import { DeferredTask, asyncTimeout } from '@dxos/async';
 import { LifecycleState, Resource } from '@dxos/context';
-import { Feed, Filter } from '@dxos/echo';
+import { Database, Feed, Filter } from '@dxos/echo';
 import { EffectEx } from '@dxos/effect';
 import { type FunctionExecutor } from '@dxos/functions-runtime';
 import { log } from '@dxos/log';
@@ -22,7 +22,7 @@ const MAX_RANGE_ID_COUNT = 10;
 export type SegmentsNormalizerProps = {
   functionExecutor: FunctionExecutor;
   feed: Feed.Feed;
-  feedRuntime: Runtime.Runtime<Feed.FeedService>;
+  feedRuntime: Runtime.Runtime<Database.Service>;
   startingCursor: QueueCursor;
 };
 
@@ -36,7 +36,7 @@ export class MessageNormalizer extends Resource {
   private readonly _functionExecutor: FunctionExecutor;
 
   private _feed: Feed.Feed;
-  private _feedRuntime: Runtime.Runtime<Feed.FeedService>;
+  private _feedRuntime: Runtime.Runtime<Database.Service>;
   private _cursor: QueueCursor;
   private _messagesToProcess: MessageWithRangeId[] = [];
   private _normalizationTask?: DeferredTask;

package/src/normalization/normalization.ts

@@ -7,6 +7,9 @@ import * as Schema from 'effect/Schema';
 
 import { Operation } from '@dxos/compute';
 import { log } from '@dxos/log';
+// Imported so TypeScript can name these types in the emitted .d.ts.
+// eslint-disable-next-line unused-imports/no-unused-imports
+import { type Message, type Person } from '@dxos/types';
 import { trim } from '@dxos/util';
 
 import { TranscriptOperation } from '../types';

package/src/operations/enrich-message.ts

@@ -0,0 +1,27 @@
+//
+// Copyright 2026 DXOS.org
+//
+
+import * as Effect from 'effect/Effect';
+
+import { enrichTranscriptMessage } from '@dxos/assistant/extraction';
+import { Operation } from '@dxos/compute';
+import { Database } from '@dxos/echo';
+
+import { TranscriptOperation } from '../types';
+
+/**
+ * Extracts proper nouns from a transcript message and links them to objects in the space via
+ * full-text search, rewriting the text with reference links.
+ */
+const handler: Operation.WithHandler<typeof TranscriptOperation.EnrichMessage> = TranscriptOperation.EnrichMessage.pipe(
+  Operation.withHandler(
+    Effect.fnUntraced(function* ({ message }) {
+      const { db } = yield* Database.Service;
+      const enriched = yield* enrichTranscriptMessage(message, { db });
+      return { message: enriched };
+    }),
+  ),
+);
+
+export default handler;

package/src/operations/index.ts

@@ -8,4 +8,5 @@ export const TranscriptionOperationHandlerSet = OperationHandlerSet.lazy(
   () => import('./create'),
   () => import('./open'),
   () => import('./summarize'),
+  () => import('./enrich-message'),
 );

package/src/operations/open.ts

@@ -6,17 +6,17 @@ import * as Effect from 'effect/Effect';
 
 import { Operation } from '@dxos/compute';
 import { Database, Feed, Filter } from '@dxos/echo';
+import { renderByline } from '@dxos/react-ui-transcription';
 import { Message } from '@dxos/types';
 
 import { TranscriptOperation } from '../types';
-import { renderByline } from '../util';
 
 const handler: Operation.WithHandler<typeof TranscriptOperation.Open> = TranscriptOperation.Open.pipe(
   Operation.withHandler(
     Effect.fn(function* ({ transcript }) {
       const transcriptObj = yield* Database.load(transcript);
       const feed = yield* Database.load(transcriptObj.feed);
-      const messages = yield* Feed.runQuery(feed, Filter.type(Message.Message));
+      const messages = yield* Feed.query(feed, Filter.type(Message.Message)).run;
       const content = messages
         .flatMap((message: Message.Message, index: number) => renderByline([])(message, index))
         .join('\n\n');

package/src/operations/summarize.ts

@@ -10,7 +10,7 @@ import * as Option from 'effect/Option';
 
 import { AiService, ConsolePrinter, ToolExecutionService, ToolResolverService } from '@dxos/ai';
 import { AiRequest, GenerationObserver } from '@dxos/assistant';
-import { Trace, Operation } from '@dxos/compute';
+import { Operation, Trace } from '@dxos/compute';
 import { Database } from '@dxos/echo';
 import { registryLayerNoop } from '@dxos/echo/testing';
 import { trim } from '@dxos/util';
@@ -49,7 +49,7 @@ const handler: Operation.WithHandler<typeof TranscriptOperation.Summarize> = Tra
       },
       Effect.provide(
         Layer.mergeAll(
-          AiService.model('ai.claude.model.claude-sonnet-4-0'),
+          AiService.model('com.anthropic.model.claude-sonnet-4-6.default'),
           ToolResolverService.layerEmpty,
           ToolExecutionService.layerEmpty,
           Trace.writerLayerNoop,

package/src/skills/index.ts

@@ -0,0 +1,5 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+export { default as TranscriptionSkill } from './transcription-skill';

package/src/skills/transcription-skill.ts

@@ -0,0 +1,29 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import { Skill, Template } from '@dxos/compute';
+import { trim } from '@dxos/util';
+
+import { TranscriptOperation } from '#types';
+
+const SKILL_KEY = 'org.dxos.skill.transcription';
+
+const make = () =>
+  Skill.make({
+    key: SKILL_KEY,
+    name: 'Transcription',
+    tools: Skill.toolDefinitions({ operations: [TranscriptOperation.Open, TranscriptOperation.Summarize] }),
+    instructions: Template.make({
+      source: trim`
+        You can open and summarize a meeting transcript.
+      `,
+    }),
+  });
+
+const skill: Skill.Definition = {
+  key: SKILL_KEY,
+  make,
+};
+
+export default skill;

package/src/stories/Keyword.stories.tsx

@@ -0,0 +1,98 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+/**
+ * Keyword-spotting demo on the browser's Web Speech API (no DXOS transcriber), driven by the real
+ * `Mic` button so the mic UX matches the rest of the suite.
+ *
+ * - `Default` lists the keywords and checks each as it is heard.
+ * - The toolbar `Mic` toggles the shared `RecordingSession`; the story observes it (`useRecordingSession`)
+ *   to start/stop `window.SpeechRecognition` (continuous + interim results). The editor driver stays
+ *   idle (no editor view for this session).
+ * - Matching normalizes punctuation/whitespace; matched rows show a check via `react-ui-list` `Listbox`.
+ */
+
+import { type Meta, type StoryObj } from '@storybook/react-vite';
+import React, { useState } from 'react';
+
+import { useAtomCapabilityState } from '@dxos/app-framework/ui';
+import { Panel, Toolbar } from '@dxos/react-ui';
+import { Listbox } from '@dxos/react-ui-list';
+import { useSpeechRecognition } from '@dxos/react-ui-transcription';
+
+import { Mic } from '#components';
+import { translations } from '#translations';
+import { TranscriptionCapabilities } from '#types';
+
+import { createStoryDecorators } from '../testing';
+
+// Stable session key for the Mic button; any non-editor id works (the editor driver ignores it).
+const DOC_ID = 'keyword-detection';
+
+// Reads the shared recording flag toggled by the `Mic` button (the editor driver stays idle when no
+// editor view is registered for this session).
+const useRecordingSession = (docId: string): boolean => {
+  const [session] = useAtomCapabilityState(TranscriptionCapabilities.RecordingSession);
+  return !!session?.recording && session.id === docId;
+};
+
+type StoryArgs = {
+  keywords: string[];
+};
+
+const DefaultStory = ({ keywords }: StoryArgs) => {
+  const [matched, setMatched] = useState<string | undefined>(undefined);
+  const handleTranscript = (transcript: string) => {
+    const match = keywords.find((keyword) => transcript.includes(keyword));
+    setMatched(match ?? undefined);
+  };
+
+  const recording = useRecordingSession(DOC_ID);
+  useSpeechRecognition({ active: recording, onTranscript: handleTranscript });
+
+  return (
+    <Panel.Root>
+      <Panel.Toolbar asChild>
+        <Toolbar.Root>
+          <Mic docId={DOC_ID} />
+        </Toolbar.Root>
+      </Panel.Toolbar>
+
+      <Panel.Content>
+        <Listbox.Root value={matched}>
+          <Listbox.Viewport thin padding>
+            <Listbox.Content aria-label='Keywords'>
+              {keywords.map((keyword) => (
+                <Listbox.Item key={keyword} id={keyword}>
+                  <Listbox.ItemLabel>{keyword}</Listbox.ItemLabel>
+                </Listbox.Item>
+              ))}
+            </Listbox.Content>
+          </Listbox.Viewport>
+        </Listbox.Root>
+      </Panel.Content>
+    </Panel.Root>
+  );
+};
+
+const meta = {
+  title: 'plugins/plugin-transcription/stories/Keyword',
+  render: DefaultStory,
+  decorators: createStoryDecorators(),
+  parameters: {
+    layout: 'fullscreen',
+    controls: { disable: true },
+    translations,
+  },
+} satisfies Meta<typeof DefaultStory>;
+
+export default meta;
+
+type Story = StoryObj<typeof meta>;
+
+export const Default: Story = {
+  args: {
+    keywords: ['hello', 'hey kai', 'kai', 'computer', 'dxos', 'composer'].sort(),
+  },
+};