@dxos/plugin-transcription 0.8.4-main.fd6878d → 0.9.0

package/src/components/{Transcript/transcript-extension.ts → Transcription/transcription-extension.ts}

@@ -8,24 +8,22 @@ import { format } from 'date-fns/format';
 import { intervalToDuration } from 'date-fns/intervalToDuration';
 
 import { type CleanupFn, addEventListener, combine } from '@dxos/async';
-import { type RenderCallback } from '@dxos/react-ui-editor';
-import { type DataType } from '@dxos/schema';
+import { type Message } from '@dxos/types';
 
 import { DocumentAdapter, type SerializationModel } from '../../model';
 
 /**
  * Data structure that maps Chunks queue to lines with transcript state.
  */
-export type TranscriptOptions = {
-  model: SerializationModel<DataType.Message>;
+export type TranscriptionOptions = {
+  model: SerializationModel<Message.Message>;
   started?: Date;
-  renderButton?: RenderCallback<{ onClick: () => void }>;
 };
 
 /**
  * Scrolling transcript with timestamps.
  */
-export const transcript = ({ model, started, renderButton }: TranscriptOptions): Extension => {
+export const transcription = ({ model, started }: TranscriptionOptions): Extension => {
   return [
     // Show timestamps in the gutter.
     gutter({
@@ -90,22 +88,6 @@ export const transcript = ({ model, started, renderButton }: TranscriptOptions):
             });
           };
 
-          // Scroll button.
-          if (renderButton) {
-            this._controls = document.createElement('div');
-            this._controls.classList.add('cm-controls', 'transition-opacity', 'duration-300', 'opacity-0');
-            view.dom.appendChild(this._controls);
-            renderButton(
-              this._controls,
-              {
-                onClick: () => {
-                  scrollToBottom(false);
-                },
-              },
-              view,
-            );
-          }
-
           // Event listeners.
           this._cleanup = combine(
             addEventListener(view.scrollDOM, 'scroll', () => {
@@ -131,7 +113,7 @@ export const transcript = ({ model, started, renderButton }: TranscriptOptions):
           );
         }
 
-        update(update: ViewUpdate) {
+        update(_update: ViewUpdate) {
           // Initial sync.
           if (!this._initialized) {
             this._initialized = true;
@@ -149,27 +131,12 @@ export const transcript = ({ model, started, renderButton }: TranscriptOptions):
     ),
 
     EditorView.theme({
-      '.cm-scroller': {
-        overflowY: 'scroll',
-      },
-      '.cm-hide-scrollbar': {
-        scrollbarWidth: 'none',
-        '-ms-overflow-style': 'none',
-      },
-      '.cm-hide-scrollbar::-webkit-scrollbar': {
-        display: 'none',
-      },
-      '.cm-controls': {
-        position: 'absolute',
-        bottom: '0.5rem',
-        right: '0.5rem',
-        zIndex: 1000,
-      },
-      '.cm-line': {
-        paddingRight: '1rem',
+      '.cm-gutters': {
+        backgroundColor: 'var(--color-base-surface)',
       },
       '.cm-timestamp-gutter': {
         width: '6rem',
+        paddingLeft: '0.5rem',
         paddingRight: '1rem',
       },
       '.cm-timestamp-gutter > .cm-gutterElement > div': {
@@ -193,13 +160,13 @@ class TimestampMarker extends GutterMarker {
     super();
   }
 
-  override eq(other: this): boolean {
-    return other._timestamp === this._timestamp;
+  override eq(other: this) {
+    return this._timestamp === other._timestamp;
   }
 
-  override toDOM(view: EditorView): HTMLDivElement {
+  override toDOM(view: EditorView) {
     const el = document.createElement('div');
-    el.className = 'text-sm text-subdued hover:bg-hoverSurface cursor-pointer';
+    el.className = 'text-sm text-subdued hover:bg-hover-surface cursor-pointer';
     el.textContent = formatTimestamp(this._timestamp, this._started);
     // TODO(burdon): Click to bookmark or copy hyperlink.
     el.onclick = () => {
@@ -213,7 +180,7 @@ class TimestampMarker extends GutterMarker {
   }
 }
 
-const getStartTime = (started?: Date, message?: DataType.Message): Date | undefined => {
+const getStartTime = (started?: Date, message?: Message.Message): Date | undefined => {
   if (started) {
     return started;
   }

package/src/components/index.ts

@@ -2,9 +2,5 @@
 // Copyright 2025 DXOS.org
 //
 
-import { lazy } from 'react';
-
-export * from './Transcript';
-export * from './TranscriptContainer';
-
-export const TranscriptContainer = lazy(() => import('./TranscriptContainer'));
+export * from './Oracle';
+export * from './Transcription';

package/src/containers/TranscriptionArticle/TranscriptionArticle.stories.tsx

@@ -0,0 +1,61 @@
+//
+// Copyright 2026 DXOS.org
+//
+
+import { type Meta, type StoryObj } from '@storybook/react-vite';
+import * as Effect from 'effect/Effect';
+import React from 'react';
+
+import { withPluginManager } from '@dxos/app-framework/testing';
+import { Feed, Query, Ref } from '@dxos/echo';
+import { ClientPlugin, initializeIdentity } from '@dxos/plugin-client/testing';
+import { corePlugins } from '@dxos/plugin-testing';
+import { useQuery, useSpaces } from '@dxos/react-client/echo';
+import { Loading, withLayout } from '@dxos/react-ui/testing';
+import { Message, Transcript } from '@dxos/types';
+
+import { translations } from '#translations';
+
+import { TestItem } from '../../testing';
+import { TranscriptionPlugin } from '../../TranscriptionPlugin';
+import { TranscriptionArticle } from './TranscriptionArticle';
+
+const DefaultStory = () => {
+  const [space] = useSpaces();
+  const [transcript] = useQuery(space?.db, Query.type(Transcript.Transcript));
+  if (!space || !transcript) {
+    return <Loading />;
+  }
+
+  return <TranscriptionArticle role='article' attendableId='story' subject={transcript} />;
+};
+
+const meta = {
+  title: 'plugins/plugin-transcription/containers/TranscriptionArticle',
+  render: DefaultStory,
+  decorators: [
+    withLayout({ layout: 'column' }),
+    withPluginManager({
+      plugins: [
+        ...corePlugins(),
+        ClientPlugin({
+          types: [Transcript.Transcript, Feed.Feed, Message.Message, TestItem],
+          onClientInitialized: ({ client }) =>
+            Effect.gen(function* () {
+              const { personalSpace } = yield* initializeIdentity(client);
+              const feed = personalSpace.db.add(Feed.make({ name: 'transcription' }));
+              personalSpace.db.add(Transcript.make(Ref.make(feed)));
+            }),
+        }),
+        TranscriptionPlugin(),
+      ],
+    }),
+  ],
+  parameters: { layout: 'fullscreen', translations },
+} satisfies Meta<typeof DefaultStory>;
+
+export default meta;
+
+type Story = StoryObj<typeof meta>;
+
+export const Default: Story = {};

package/src/containers/TranscriptionArticle/TranscriptionArticle.tsx

@@ -0,0 +1,62 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+import React from 'react';
+
+import { type AppSurface } from '@dxos/app-toolkit/ui';
+import { Filter, Obj, Query } from '@dxos/echo';
+import { useMembers, useQuery } from '@dxos/react-client/echo';
+import { Panel } from '@dxos/react-ui';
+import { Menu, MenuBuilder, useMenuBuilder } from '@dxos/react-ui-menu';
+import { Message, type Transcript } from '@dxos/types';
+
+import { Transcription } from '#components';
+import { useFeedModelAdapter, useTranscriptionRecording } from '#hooks';
+import { meta } from '#meta';
+
+import { renderByline } from '../../util';
+
+export type TranscriptionArticleProps = AppSurface.ObjectArticleProps<Transcript.Transcript>;
+
+export const TranscriptionArticle = ({ role, subject: transcript, attendableId }: TranscriptionArticleProps) => {
+  const db = Obj.getDatabase(transcript);
+  const members = useMembers(db?.spaceId).map((member) => member.identity);
+  const feed = transcript.feed.target;
+  const messages = useQuery(
+    db,
+    feed ? Query.select(Filter.type(Message.Message)).from(feed) : Query.select(Filter.nothing()),
+  );
+  const model = useFeedModelAdapter(renderByline(members), messages);
+  const { recording, toggleRecording } = useTranscriptionRecording(transcript);
+
+  const menuActions = useMenuBuilder(
+    () =>
+      MenuBuilder.make()
+        .action(
+          'toggle-recording',
+          {
+            label: [recording ? 'stop-recording.label' : 'start-recording.label', { ns: meta.id }],
+            icon: recording ? 'ph--stop-circle--regular' : 'ph--microphone--regular',
+            disposition: 'toolbar',
+            testId: 'transcription.toggle-recording',
+          },
+          toggleRecording,
+        )
+        .build(),
+    [recording, toggleRecording],
+  );
+
+  return (
+    <Panel.Root role={role}>
+      <Menu.Root {...menuActions} attendableId={attendableId}>
+        <Panel.Toolbar asChild>
+          <Menu.Toolbar />
+        </Panel.Toolbar>
+      </Menu.Root>
+      <Panel.Content asChild>
+        <Transcription model={model} transcript={transcript} />
+      </Panel.Content>
+    </Panel.Root>
+  );
+};

package/src/containers/TranscriptionArticle/index.ts

@@ -0,0 +1,5 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+export { TranscriptionArticle as default } from './TranscriptionArticle';

package/src/containers/index.ts

@@ -0,0 +1,7 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+import { type ComponentType, lazy } from 'react';
+
+export const TranscriptionArticle: ComponentType<any> = lazy(() => import('./TranscriptionArticle'));

package/src/enrichment/conversation.test.ts

@@ -0,0 +1,162 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+// Headless test that simulates a two-speaker conversation as a pair of Whisper streams and
+// drives them through the (currently stubbed) Pass A enrichment and Pass B summarization
+// operations. The handlers are deterministic placeholders — they don't call an LLM — so the
+// test runs offline; switching to the real LLM-backed handlers later doesn't change the
+// harness, just the assertions.
+//
+// What this verifies today:
+//   - Conversation fixture parses to entities + speaker-tagged utterances.
+//   - Stream simulator emits per-batch transcript blocks in chronological order.
+//   - Pass A's sliding-window enrichment links seeded ECHO entities by name and surfaces
+//     remaining capitalised tokens as candidates.
+//   - Pass B accumulates a per-speaker summary and resolves "I" → speaker display name.
+
+import { describe, it } from '@effect/vitest';
+import * as Effect from 'effect/Effect';
+import * as fs from 'node:fs';
+import * as path from 'node:path';
+import * as url from 'node:url';
+
+import { Blueprint, Operation, OperationHandlerSet } from '@dxos/compute';
+import { Database, Feed, Obj, Type } from '@dxos/echo';
+import { TestHelpers } from '@dxos/effect/testing';
+import { AssistantTestLayer } from '@dxos/functions-runtime/testing';
+import { EntityId } from '@dxos/keys';
+import { Organization, Person } from '@dxos/types';
+
+import { EnrichTranscript, EnrichmentHandlers, SummarizeConversation } from './operations';
+import { parseConversation } from './parser';
+import { simulateStream } from './stream-simulator';
+
+EntityId.dangerouslyDisableRandomness();
+
+const TestLayer = AssistantTestLayer({
+  operationHandlers: OperationHandlerSet.make(...EnrichmentHandlers),
+  types: [Person.Person, Organization.Organization, Blueprint.Blueprint, Feed.Feed],
+  disableLlmMemoization: true,
+});
+
+describe('Transcript enrichment — conversation simulation', () => {
+  it.scoped(
+    'streams a two-speaker conversation through Pass A and Pass B (stub handlers)',
+    Effect.fnUntraced(
+      function* ({ expect }) {
+        // 1. Load and parse the fixture.
+        const fixturePath = path.join(path.dirname(url.fileURLToPath(import.meta.url)), 'conversation.txt');
+        const conversation = parseConversation(fs.readFileSync(fixturePath, 'utf8'));
+        expect(conversation.speakers).toEqual(['A', 'B']);
+        expect(conversation.utterances.length).toBeGreaterThan(0);
+        expect(conversation.entities.length).toBeGreaterThan(0);
+
+        // 2. Seed the ECHO space with the fixture's entities. Map fixture types → schema types.
+        const seeded: { id: string; typename: string; name: string }[] = [];
+        for (const entity of conversation.entities) {
+          if (entity.kind === 'Person') {
+            const fullName = entity.fields.fullName ?? entity.fields.name;
+            if (!fullName) {
+              continue;
+            }
+            const person = yield* Database.add(Obj.make(Person.Person, { fullName }));
+            seeded.push({ id: person.id, typename: Type.getTypename(Person.Person), name: fullName });
+          } else if (entity.kind === 'Organization') {
+            const name = entity.fields.name;
+            if (!name) {
+              continue;
+            }
+            const org = yield* Database.add(Obj.make(Organization.Organization, { name }));
+            seeded.push({ id: org.id, typename: Type.getTypename(Organization.Organization), name });
+          }
+        }
+        yield* Database.flush();
+        expect(seeded.map((s) => s.name)).toContain('Chad');
+        expect(seeded.map((s) => s.name)).toContain('Blue Yard');
+
+        // 3. Generate the per-batch Whisper schedule. Batch by speaker so each batch maps to
+        //    a single Pass-A invocation, mimicking what the live `useTranscriptEnrichment`
+        //    hook will receive from the transcriber.
+        const batches = simulateStream(conversation, { wordsPerSecond: 2.5, gapSeconds: 0.5 });
+        expect(batches.length).toBe(conversation.utterances.length);
+        expect(batches[0].offsetMs).toBe(0);
+        expect(batches.every((b) => b.blocks.length > 0)).toBe(true);
+
+        // 4. Run Pass A over each batch — sliding window of the last 4 blocks across speakers.
+        const slidingWindow: { speaker: string; text: string; started: string }[] = [];
+        const enrichments: {
+          speaker: string;
+          corrected: string;
+          referenceIds: string[];
+          candidates: { text: string }[];
+        }[] = [];
+        for (const batch of batches) {
+          for (const block of batch.blocks) {
+            slidingWindow.push({ speaker: batch.speaker, text: block.text, started: block.started });
+            if (slidingWindow.length > 4) {
+              slidingWindow.shift();
+            }
+          }
+          const result = yield* Operation.invoke(EnrichTranscript, {
+            window: slidingWindow,
+            knownEntities: seeded,
+          });
+          // The new batch's entries are the *last* `batch.blocks.length` items in the window.
+          const newEntries = result.blocks.slice(-batch.blocks.length);
+          for (const block of newEntries) {
+            const windowEntry = slidingWindow[block.index];
+            enrichments.push({
+              speaker: windowEntry.speaker,
+              corrected: block.corrected,
+              referenceIds: [...block.referenceIds],
+              candidates: block.candidates.map((c) => ({ text: c.text })),
+            });
+          }
+        }
+
+        // 4a. Every block got a terminal-punctuated `corrected` value.
+        for (const e of enrichments) {
+          expect(e.corrected).toMatch(/[.!?]$/);
+        }
+
+        // 4b. The block containing "Chad" links to the seeded Person.
+        const chadId = seeded.find((s) => s.name === 'Chad')!.id;
+        const chadMention = enrichments.find((e) => /\bChad\b/.test(e.corrected));
+        expect(chadMention).toBeDefined();
+        expect(chadMention!.referenceIds).toContain(chadId);
+
+        // 4c. Place names surface as candidates (not in our seed).
+        const allCandidates = enrichments.flatMap((e) => e.candidates.map((c) => c.text));
+        expect(allCandidates).toContain('Munich');
+        expect(allCandidates).toContain('Los Angeles');
+
+        // 5. Run Pass B once at the end of the conversation (the real pipeline would fire it
+        //    after silence + interval; here we trigger it manually after the last batch).
+        const speakerNames = { A: 'Speaker A', B: 'Speaker B' };
+        const summary = yield* Operation.invoke(SummarizeConversation, {
+          previousSummary: '',
+          newUtterances: enrichments.map((e) => ({ speaker: e.speaker, text: e.corrected })),
+          speakerNames,
+          recentReferences: seeded.map((s) => ({ id: s.id, name: s.name })),
+        });
+
+        // 5a. Summary is non-empty and mentions both speakers.
+        expect(summary.summary.length).toBeGreaterThan(0);
+        expect(summary.summary).toContain('Speaker A');
+        expect(summary.summary).toContain('Speaker B');
+
+        // 5b. Resolved referents: every utterance that contains "I" picks up the speaker's name.
+        const iUtterances = enrichments.filter((e) => /\bI\b/.test(e.corrected));
+        expect(summary.resolvedReferents.length).toBe(iUtterances.length);
+        for (const referent of summary.resolvedReferents) {
+          expect(referent.surface).toBe('I');
+          expect([speakerNames.A, speakerNames.B]).toContain(referent.referent);
+        }
+      },
+      Effect.provide(TestLayer),
+      TestHelpers.provideTestContext,
+    ),
+    { timeout: 30_000 },
+  );
+});

package/src/enrichment/conversation.txt

@@ -0,0 +1,23 @@
+# Conversation fixture for transcription enrichment tests.
+# Format:
+#   `Entities:` block — declares ECHO objects to seed into the test space.
+#     Each entry is `<Type>:` followed by indented `<field>: <value>` lines.
+#   `Conversation:` block — one utterance per line, `<speaker>: <text>`.
+#   Speakers map to distinct virtual Whisper streams in the simulator.
+#   Lines starting with `#` are comments and are ignored by the parser.
+
+Entities:
+  Person:
+    fullName: Chad
+    employer: Blue Yard
+  Organization:
+    name: Blue Yard
+
+Conversation:
+  A: I think I'm going to go.
+  B: Are you meeting with them?
+  A: Yes I think so.
+  B: But I thought they were in Munich?
+  A: They are, but the offsite is in Los Angeles.
+  B: Is Chad going to be there?
+  A: Yes, the whole team. It's their annual offsite.