@dxos/plugin-transcription 0.8.4-main.c85a9c8dae → 0.8.4-main.dfabb4ec29

package/src/capabilities/operation-handler/index.ts

@@ -1,7 +0,0 @@
-//
-// Copyright 2025 DXOS.org
-//
-
-import { Capability } from '@dxos/app-framework';
-
-export const OperationResolver = Capability.lazy('OperationResolver', () => import('./operation-handler'));

package/src/capabilities/operation-handler/operation-handler.ts

@@ -1,25 +0,0 @@
-//
-// Copyright 2025 DXOS.org
-//
-
-import * as Effect from 'effect/Effect';
-
-import { Capabilities, Capability } from '@dxos/app-framework';
-import { OperationResolver } from '@dxos/operation';
-import { Transcript } from '@dxos/types';
-
-import { TranscriptOperation } from '../../types';
-
-export default Capability.makeModule(() =>
-  Effect.succeed(
-    Capability.contributes(Capabilities.OperationResolver, [
-      OperationResolver.make({
-        operation: TranscriptOperation.Create,
-        handler: ({ space }) =>
-          Effect.succeed({
-            object: Transcript.make(space.queues.create().dxn),
-          }),
-      }),
-    ]),
-  ),
-);

package/src/capabilities/operation-resolver/index.ts

@@ -1,7 +0,0 @@
-//
-// Copyright 2025 DXOS.org
-//
-
-import { Capability } from '@dxos/app-framework';
-
-export const OperationResolver = Capability.lazy('OperationResolver', () => import('./operation-resolver'));

package/src/capabilities/operation-resolver/operation-resolver.ts

@@ -1,25 +0,0 @@
-//
-// Copyright 2025 DXOS.org
-//
-
-import * as Effect from 'effect/Effect';
-
-import { Capabilities, Capability } from '@dxos/app-framework';
-import { OperationResolver } from '@dxos/operation';
-import { Transcript } from '@dxos/types';
-
-import { TranscriptOperation } from '../../types';
-
-export default Capability.makeModule(() =>
-  Effect.succeed(
-    Capability.contributes(Capabilities.OperationResolver, [
-      OperationResolver.make({
-        operation: TranscriptOperation.Create,
-        handler: ({ space }) =>
-          Effect.succeed({
-            object: Transcript.make(space.queues.create().dxn),
-          }),
-      }),
-    ]),
-  ),
-);

package/src/capabilities/react-surface/index.ts

@@ -1,7 +0,0 @@
-//
-// Copyright 2025 DXOS.org
-//
-
-import { Capability } from '@dxos/app-framework';
-
-export const ReactSurface = Capability.lazy('ReactSurface', () => import('./react-surface'));

package/src/capabilities/react-surface/react-surface.tsx

@@ -1,28 +0,0 @@
-//
-// Copyright 2025 DXOS.org
-//
-
-import * as Effect from 'effect/Effect';
-import React from 'react';
-
-import { Capabilities, Capability } from '@dxos/app-framework';
-import { Surface } from '@dxos/app-framework/ui';
-import { Obj } from '@dxos/echo';
-import { Transcript } from '@dxos/types';
-
-import { TranscriptionContainer } from '../../containers';
-import { meta } from '../../meta';
-
-export default Capability.makeModule(() =>
-  Effect.succeed(
-    Capability.contributes(Capabilities.ReactSurface, [
-      Surface.create({
-        id: `${meta.id}/article/transcript`,
-        role: ['article', 'section'],
-        filter: (data): data is { subject: Transcript.Transcript } =>
-          Obj.instanceOf(Transcript.Transcript, data.subject),
-        component: ({ data, role }) => <TranscriptionContainer role={role} subject={data.subject} />,
-      }),
-    ]),
-  ),
-);

package/src/capabilities/transcriber/index.ts

@@ -1,7 +0,0 @@
-//
-// Copyright 2025 DXOS.org
-//
-
-import { Capability } from '@dxos/app-framework';
-
-export const Transcriber = Capability.lazy('Transcriber', () => import('./transcriber'));

package/src/components/stories/TranscriptionStory.tsx

@@ -1,41 +0,0 @@
-//
-// Copyright 2025 DXOS.org
-//
-
-import '@dxos-theme';
-
-import React, { type Dispatch, type FC, type RefObject, type SetStateAction } from 'react';
-
-import { IconButton, ScrollContainer, Toolbar } from '@dxos/react-ui';
-import { type Message } from '@dxos/types';
-
-import { type SerializationModel } from '../../model';
-import { Transcription } from '../Transcription';
-
-export const TranscriptionStory: FC<{
-  model: SerializationModel<Message.Message>;
-  disabled?: boolean;
-  running: boolean;
-  onRunningChange: Dispatch<SetStateAction<boolean>>;
-  audioRef?: RefObject<HTMLAudioElement | null>;
-}> = ({ model, running, onRunningChange, audioRef, disabled }) => {
-  return (
-    <div className='flex flex-col w-[30rem]'>
-      {audioRef && <audio ref={audioRef} autoPlay />}
-      <Toolbar.Root>
-        <IconButton
-          iconOnly
-          disabled={disabled}
-          icon={running ? 'ph--pause--regular' : 'ph--play--regular'}
-          label={running ? 'Pause' : 'Play'}
-          onClick={() => onRunningChange((running) => !running)}
-        />
-      </Toolbar.Root>
-      <ScrollContainer.Root pin>
-        <ScrollContainer.Viewport>
-          <Transcription model={model} />
-        </ScrollContainer.Viewport>
-      </ScrollContainer.Root>
-    </div>
-  );
-};

package/src/functions/index.ts

@@ -1,11 +0,0 @@
-//
-// Copyright 2024 DXOS.org
-//
-
-import Open from './open';
-import Summarize from './summarize';
-
-export const TranscriptionFunctions = {
-  Open,
-  Summarize,
-};

package/src/functions/open.ts

@@ -1,37 +0,0 @@
-//
-// Copyright 2025 DXOS.org
-//
-
-import * as Effect from 'effect/Effect';
-import * as Schema from 'effect/Schema';
-
-import { Database, Obj, Ref } from '@dxos/echo';
-import { QueueService, defineFunction } from '@dxos/functions';
-import { Message, Transcript } from '@dxos/types';
-
-import { renderByline } from '../util';
-
-export default defineFunction({
-  key: 'dxos.org/function/transcription/open',
-  name: 'Open',
-  description: 'Opens and reads the contents of a transcription object.',
-  inputSchema: Schema.Struct({
-    transcript: Ref.Ref(Transcript.Transcript).annotations({
-      description: 'The ID of the transcription object.',
-    }),
-  }),
-  outputSchema: Schema.Struct({
-    content: Schema.String,
-  }),
-  handler: Effect.fn(function* ({ data: { transcript } }) {
-    const transcriptObj = yield* Database.load(transcript);
-    const { dxn } = yield* Effect.promise(() => transcriptObj.queue.load());
-    const queue = yield* QueueService.getQueue(dxn);
-    yield* Effect.promise(() => queue?.queryObjects());
-    const content = queue?.objects
-      .filter((message) => Obj.instanceOf(Message.Message, message))
-      .flatMap((message, index) => renderByline([])(message, index))
-      .join('\n\n');
-    return { content };
-  }),
-});

package/src/functions/summarize.ts

@@ -1,103 +0,0 @@
-//
-// Copyright 2025 DXOS.org
-//
-
-import * as Array from 'effect/Array';
-import * as Effect from 'effect/Effect';
-import * as Function from 'effect/Function';
-import * as Layer from 'effect/Layer';
-import * as Option from 'effect/Option';
-import * as Schema from 'effect/Schema';
-
-import { AiService, ConsolePrinter, ToolExecutionService, ToolResolverService } from '@dxos/ai';
-import { AiSession, GenerationObserver } from '@dxos/assistant';
-import { TracingService, defineFunction } from '@dxos/functions';
-import { trim } from '@dxos/util';
-
-/**
- * Summarize a transcript of a meeting.
- */
-export default defineFunction({
-  key: 'dxos.org/function/transcription/summarize',
-  name: 'Summarize',
-  description: 'Summarize a transcript of a meeting.',
-  inputSchema: Schema.Struct({
-    transcript: Schema.String.annotations({
-      description: 'The transcript of the meeting.',
-    }),
-    notes: Schema.optional(Schema.String).annotations({
-      description: 'Additional notes from the participants.',
-    }),
-  }),
-  outputSchema: Schema.Struct({
-    summary: Schema.String.annotations({
-      description: 'The summary of the transcript.',
-    }),
-  }),
-  handler: Effect.fnUntraced(
-    function* ({ data: { transcript, notes } }) {
-      const result = yield* new AiSession().run({
-        prompt: `Transcript: ${transcript}\n\nNotes: ${notes}`,
-        history: [],
-        system: systemPrompt,
-        observer: GenerationObserver.fromPrinter(new ConsolePrinter({ tag: 'summarize' })),
-      });
-
-      const summary = Function.pipe(
-        result,
-        Array.findLast((msg) => msg.sender.role === 'assistant' && msg.blocks.some((block) => block._tag === 'text')),
-        Option.flatMap((msg) =>
-          Function.pipe(
-            msg.blocks,
-            Array.findLast((block) => block._tag === 'text'),
-            Option.map((block) => block.text),
-          ),
-        ),
-        Option.getOrThrowWith(() => new Error('No summary found')),
-      );
-
-      return { summary };
-    },
-    Effect.provide(
-      Layer.mergeAll(
-        AiService.model('@anthropic/claude-sonnet-4-0'),
-        ToolResolverService.layerEmpty,
-        ToolExecutionService.layerEmpty,
-        TracingService.layerNoop,
-      ),
-    ),
-  ),
-});
-
-const systemPrompt = trim`
-  You are a helpful assistant that summarizes transcripts of meetings.
-
-  # Goal
-  Create a markdown summary of the meeting transcript with text notes provided.
-  Notes are very important so make sure to include them in the summary if they contain meaningful information.
-
-  # Formatting
-  - Format the summary as a markdown document without extra comments like "Here is the summary of the transcript:".
-  - Use markdown formatting for headings and bullet points.
-  - Format the summary as a list of key points and takeaways.
-  - All names of people should be in bold.
-
-  # Note Taking
-  - Correlate items in the summary with the person of origin to build a coherent narrative.
-  - Include short quotes verbatim where appropriate. Especially when concerned with design decisions and problem descriptions.
-
-  # Tasks
-  At the end of the summary include tasks.
-  Extract only the tasks that are:
-  - Directly actionable.
-  - Clearly assigned to a person or team (or can easily be inferred).
-  - Strongly implied by the conversation and/or user note (no speculative tasks).
-  - Specific enough that someone reading them would know exactly what to do next.
-
-  Format all tasks as markdown checkboxes using the syntax:
-  - [ ] Task description.
-
-  Additional information can be included (indented).
-
-  If no actionable tasks are found, omit this tasks section.
-`;

package/src/segments-normalization/normalization.ts

@@ -1,115 +0,0 @@
-//
-// Copyright 2025 DXOS.org
-//
-
-// ISSUE(burdon): defineFunction
-// @ts-nocheck
-
-import * as Schema from 'effect/Schema';
-
-import { AiService, DEFAULT_EDGE_MODEL } from '@dxos/ai';
-import { AiSession } from '@dxos/assistant';
-import { Obj } from '@dxos/echo';
-import { defineFunction } from '@dxos/functions';
-import { ObjectId } from '@dxos/keys';
-import { log } from '@dxos/log';
-import { Message } from '@dxos/types';
-import { trim } from '@dxos/util';
-
-const MessageWithRangeId = Schema.extend(
-  Message.Message,
-  Schema.Struct({
-    // TODO(mykola): Move to meta in Message. Use `Message.Message` instead.
-    rangeId: Schema.optional(Schema.Array(Schema.String)).annotations({
-      description: 'The IDs of the messages that contain the sentences.',
-    }),
-  }),
-);
-
-export interface MessageWithRangeId extends Schema.Schema.Type<typeof MessageWithRangeId> {}
-
-export const NormalizationInput = Schema.Struct({
-  // TODO(mykola): Move to meta in Message. Use `Message.Message` instead.
-  messages: Schema.Array(MessageWithRangeId).annotations({
-    description: 'Messages to normalize into sentences.',
-  }),
-});
-export interface NormalizationInput extends Schema.Schema.Type<typeof NormalizationInput> {}
-
-export const NormalizationOutput = Schema.Struct({
-  sentences: Schema.Array(MessageWithRangeId.pipe(Schema.mutable)).pipe(Schema.mutable).annotations({
-    description: 'The sentences of the transcript.',
-  }),
-});
-export interface NormalizationOutput extends Schema.Schema.Type<typeof NormalizationOutput> {}
-
-export const sentenceNormalization = defineFunction<NormalizationInput, NormalizationOutput>({
-  description: 'Post process of transcription for sentence normalization',
-  inputSchema: NormalizationInput,
-  outputSchema: NormalizationOutput,
-  handler: async ({ data: { messages }, context }) => {
-    log.info('input', { messages });
-    const ai = context.getService(AiService);
-    const session = new AiSession({ operationModel: 'configured' });
-
-    // TODO(dmaretskyi): This got broken after effect-ai transition.
-    const response = session.runStructured(NormalizationOutput, {
-      generationOptions: {
-        model: DEFAULT_EDGE_MODEL,
-      },
-      client: ai.client,
-      tools: [],
-      artifacts: [],
-      history: [
-        Obj.make(Message.Message, {
-          created: new Date().toISOString(),
-          sender: {
-            role: 'user',
-          },
-          blocks: messages.map((message) => ({ _tag: 'text', text: JSON.stringify(message) }) as const),
-        }),
-      ],
-      prompt,
-    }) as any;
-
-    response.sentences.forEach((sentence) => {
-      sentence.id = ObjectId.random();
-    });
-    log.info('response', { response });
-    return response;
-  },
-});
-
-const prompt = trim`
-  You are observing a real-time transcript of a single person speaking.
-  The transcription is delivered in chunks of 10 seconds or less. As a result, individual sentences may be split across multiple messages, or multiple sentences may appear within a single message. Additionally, because this is real-time transcription, punctuation and capitalization may be incorrect or missing.
-
-  # Task Description:
-    - Your task is to detect and reconstruct broken or incomplete sentences by merging segments into coherent, grammatically correct sentences where appropriate.
-
-  # Input Format:
-    - The input is an array of messages that contain transcription blocks.
-    - Each message is a Message.Message, as described in the output format.
-
-  # Output Format:
-    - You have been provided with the tool that defines the output format; make sure to query it.
-    - Do not output anything other than the expected format.
-
-  # Segment Handling Rules:
-    - Sort messages by timestamp.
-    - Leave ID of the first message in a sentence.
-    - Each divided sentence should be added to the output as a separate message or block within existing messages, preserving the original order.
-    - If one message contains multiple sentences, you should not split them.
-    - The last sentence may be incomplete; it should still be output as a separate block or message.
-    - Keep track of the original timestamps of the messages and blocks and, use the timestamp of the first message/block of a sentence as the 'started' field of the merged message.
-    - The 'rangeId' field of the merged message should include 'messageId'-s and 'rangeId'-s of all messages that have been used to construct this message.
-
-  # Punctuation and Capitalization:
-    - Do not rely on the original punctuation and capitalization—they may be incorrect.
-    - Use logical reasoning to apply appropriate punctuation (e.g., period, comma, question mark, exclamation mark) and capitalization.
-
-  # Restrictions:
-    - Do not alter the order of the original messages; maintain the natural flow of speech.
-    - Do not interpret or infer meaning beyond reconstructing sentences.
-    - Do not add or remove any words or phrases.
-`;

package/src/types/TranscriptAction.ts

@@ -1,32 +0,0 @@
-//
-// Copyright 2025 DXOS.org
-//
-
-import * as Schema from 'effect/Schema';
-
-import { SpaceSchema } from '@dxos/client/echo';
-import { Operation } from '@dxos/operation';
-import { Transcript } from '@dxos/types';
-
-import { meta } from '../meta';
-
-//
-// Operations
-//
-
-const TRANSCRIPT_OPERATION = `${meta.id}/operation`;
-
-export namespace TranscriptOperation {
-  export const Create = Operation.make({
-    meta: { key: `${TRANSCRIPT_OPERATION}/create`, name: 'Create Transcript' },
-    schema: {
-      input: Schema.Struct({
-        name: Schema.optional(Schema.String),
-        space: SpaceSchema,
-      }),
-      output: Schema.Struct({
-        object: Transcript.Transcript,
-      }),
-    },
-  });
-}

package/src/types/capabilities.ts

@@ -1,33 +0,0 @@
-//
-// Copyright 2025 DXOS.org
-//
-
-import { Capability } from '@dxos/app-framework';
-
-import { meta } from '../meta';
-import {
-  type MediaStreamRecorderProps,
-  type Transcriber,
-  type TranscriberProps,
-  type TranscriptMessageEnricher,
-  type TranscriptionManager,
-} from '../transcriber';
-
-export namespace TranscriptionCapabilities {
-  export type GetTranscriberProps = {
-    audioStreamTrack: MediaStreamTrack;
-    recorderConfig?: Partial<MediaStreamRecorderProps['config']>;
-    transcriberConfig?: Partial<TranscriberProps['config']>;
-    onSegments: TranscriberProps['onSegments'];
-  };
-  export type GetTranscriber = (props: GetTranscriberProps) => Transcriber;
-  export const Transcriber = Capability.make<GetTranscriber>(`${meta.id}/capability/transcriber`);
-
-  export type GetTranscriptionManagerProps = {
-    messageEnricher?: TranscriptMessageEnricher;
-  };
-  export type GetTranscriptionManager = (props: GetTranscriptionManagerProps) => TranscriptionManager;
-  export const TranscriptionManager = Capability.make<GetTranscriptionManager>(
-    `${meta.id}/capability/transcription-manager`,
-  );
-}

package/src/{segments-normalization → normalization}/index.ts

@@ -2,6 +2,6 @@
 // Copyright 2025 DXOS.org
 //
 
-export * from './normalization';
 export * from './message-normalizer';
+export * from './normalization';
 export * from './utils';

package/src/{segments-normalization → normalization}/normalization.test.ts

@@ -7,8 +7,8 @@ import { describe, test } from 'vitest';
 import { scheduleTaskInterval } from '@dxos/async';
 import { Context } from '@dxos/context';
 import { Obj } from '@dxos/echo';
-import { createQueueDXN } from '@dxos/echo/internal';
 import { MemoryQueue } from '@dxos/echo-db';
+import { createQueueDXN } from '@dxos/echo/internal';
 import { FunctionExecutor, ServiceContainer } from '@dxos/functions-runtime';
 import { log } from '@dxos/log';
 import { type Actor, Message } from '@dxos/types';