@dxos/plugin-transcription 0.8.3 → 0.8.4-main.28f8d3d

package/src/model/model.test.ts

@@ -9,7 +9,7 @@ import { Obj } from '@dxos/echo';
 import type { ObjectId } from '@dxos/keys';
 import { DataType } from '@dxos/schema';
 
-import { SerializationModel, DocumentAdapter, type ChunkRenderer } from './model';
+import { type ChunkRenderer, DocumentAdapter, SerializationModel } from './model';
 
 const blockToMarkdown: ChunkRenderer<DataType.Message> = (
   message: DataType.Message,
@@ -18,7 +18,7 @@ const blockToMarkdown: ChunkRenderer<DataType.Message> = (
 ): string[] => {
   return [
     `###### ${message.sender.name}`,
-    ...message.blocks.filter((block) => block.type === 'transcription').map((block) => block.text),
+    ...message.blocks.filter((block) => block._tag === 'transcript').map((block) => block.text),
     '',
   ];
 };
@@ -37,7 +37,7 @@ describe('SerializationModel', () => {
       sender: { name: 'Alice' },
       blocks: [
         {
-          type: 'transcription',
+          _tag: 'transcript',
           started: createDate(),
           text: 'Hello world!',
         },
@@ -51,7 +51,7 @@ describe('SerializationModel', () => {
     }
 
     // Update message.
-    message.blocks.push({ type: 'transcription', started: createDate(), text: 'Hello again!' });
+    message.blocks.push({ _tag: 'transcript', started: createDate(), text: 'Hello again!' });
     model.updateChunk(message);
     {
       const text = model.doc.toString();
@@ -74,7 +74,7 @@ describe('SerializationModel', () => {
         sender: { name: 'Alice' },
         blocks: [
           {
-            type: 'transcription',
+            _tag: 'transcript',
             started: createDate(),
             text: 'Hello world!',
           },
@@ -92,7 +92,7 @@ describe('SerializationModel', () => {
         sender: { name: 'Bob' },
         blocks: [
           {
-            type: 'transcription',
+            _tag: 'transcript',
             started: createDate(),
             text: 'Hello world!',
           },
@@ -119,7 +119,7 @@ describe('SerializationModel', () => {
         sender: { name: 'Alice' },
         blocks: [
           {
-            type: 'transcription',
+            _tag: 'transcript',
             started: createDate(),
             text: 'Hello world!',
           },
@@ -131,7 +131,7 @@ describe('SerializationModel', () => {
       expect(view.state.doc.toString()).to.eq('###### Alice\nHello world!\n\n');
 
       // Update message (add block).
-      message.blocks.push({ type: 'transcription', started: createDate(), text: 'Hello again!' });
+      message.blocks.push({ _tag: 'transcript', started: createDate(), text: 'Hello again!' });
       model.updateChunk(message);
       model.sync(adapter);
       expect(view.state.doc.toString()).to.eq('###### Alice\nHello world!\nHello again!\n\n');
@@ -150,7 +150,7 @@ describe('SerializationModel', () => {
       const message = Obj.make(DataType.Message, {
         created: createDate(),
         sender: { name: 'Bob' },
-        blocks: [{ type: 'transcription', started: createDate(), text: 'Hello again!' }],
+        blocks: [{ _tag: 'transcript', started: createDate(), text: 'Hello again!' }],
       });
       model.appendChunk(message);
       model.sync(adapter);

package/src/segments-normalization/message-normalizer.ts

@@ -8,7 +8,7 @@
 
 import { effect } from '@preact/signals-core';
 
-import { asyncTimeout, DeferredTask } from '@dxos/async';
+import { DeferredTask, asyncTimeout } from '@dxos/async';
 import { LifecycleState, Resource } from '@dxos/context';
 import { type Queue } from '@dxos/echo-db';
 import { type FunctionExecutor } from '@dxos/functions';

package/src/segments-normalization/normalization.test.ts

@@ -5,8 +5,6 @@
 import { effect } from '@preact/signals-core';
 import { describe, test } from 'vitest';
 
-import { EdgeAiServiceClient, OllamaAiServiceClient } from '@dxos/ai';
-import { AI_SERVICE_ENDPOINT } from '@dxos/ai/testing';
 import { scheduleTaskInterval } from '@dxos/async';
 import { Context } from '@dxos/context';
 import { Obj } from '@dxos/echo';
@@ -51,33 +49,32 @@ const messages: MessageWithRangeId[] = [
   'in classical physics objects have well-defined properties such as position speed and momentum',
 ].map((string, index) =>
   Obj.make(DataType.Message, {
-    created: new Date(Date.now() + 1000 * index).toISOString(),
+    created: new Date(Date.now() + 1_000 * index).toISOString(),
     sender,
-    blocks: [{ type: 'transcription', started: new Date(Date.now() + 1000 * index).toISOString(), text: string }],
-    rangeId: [],
-  } as any),
+    blocks: [{ _tag: 'transcript', started: new Date(Date.now() + 1_000 * index).toISOString(), text: string }],
+  }),
 );
 
-const REMOTE_AI = true;
+// const REMOTE_AI = true;
 
 describe.skip('SentenceNormalization', () => {
   const getExecutor = () => {
     return new FunctionExecutor(
       new ServiceContainer().setServices({
-        ai: {
-          client: REMOTE_AI
-            ? new EdgeAiServiceClient({
-                endpoint: AI_SERVICE_ENDPOINT.REMOTE,
-                defaultGenerationOptions: {
-                  model: '@anthropic/claude-3-5-sonnet-20241022',
-                },
-              })
-            : new OllamaAiServiceClient({
-                overrides: {
-                  model: 'llama3.1:8b',
-                },
-              }),
-        },
+        // ai: {
+        //   client: REMOTE_AI
+        //     ? new Edge AiServiceClient({
+        //         endpoint: AI_SERVICE_ENDPOINT.REMOTE,
+        //         defaultGenerationOptions: {
+        //           model: '@anthropic/claude-3-5-sonnet-20241022',
+        //         },
+        //       })
+        //     : new Ollama AiServiceClient({
+        //         overrides: {
+        //           model: 'llama3.1:8b',
+        //         },
+        //       }),
+        // },
       }),
     );
   };
@@ -100,6 +97,7 @@ describe.skip('SentenceNormalization', () => {
         buffer = sentences.slice(activeSentenceIndex);
       }
     }
+
     log.info('sentences', {
       originalMessages: JSON.stringify(messages, null, 2),
       sentences: JSON.stringify(sentences, null, 2),

package/src/segments-normalization/normalization.ts

@@ -2,15 +2,19 @@
 // Copyright 2025 DXOS.org
 //
 
+// ISSUE(burdon): defineFunction
+// @ts-nocheck
+
 import { Schema } from 'effect';
 
-import { DEFAULT_EDGE_MODEL, Message } from '@dxos/ai';
-import { AISession } from '@dxos/assistant';
+import { AiService, DEFAULT_EDGE_MODEL } from '@dxos/ai';
+import { AiSession } from '@dxos/assistant';
 import { Obj } from '@dxos/echo';
-import { AiService, defineFunction } from '@dxos/functions';
+import { defineFunction } from '@dxos/functions';
 import { ObjectId } from '@dxos/keys';
 import { log } from '@dxos/log';
 import { DataType } from '@dxos/schema';
+import { trim } from '@dxos/util';
 
 const MessageWithRangeId = Schema.extend(
   DataType.Message,
@@ -39,40 +43,6 @@ export const NormalizationOutput = Schema.Struct({
 });
 export interface NormalizationOutput extends Schema.Schema.Type<typeof NormalizationOutput> {}
 
-const prompt = `
-You are observing a real-time transcript of a single person speaking.
-The transcription is delivered in chunks of 10 seconds or less. As a result, individual sentences may be split across multiple messages, or multiple sentences may appear within a single message. Additionally, because this is real-time transcription, punctuation and capitalization may be incorrect or missing.
-
-# Task Description:
-  - Your task is to detect and reconstruct broken or incomplete sentences by merging segments into coherent, grammatically correct sentences where appropriate.
-
-# Input Format:
-  - The input is an array of messages that contain transcription blocks.
-  - Each message is a DataType.Message, as described in the output format.
-
-# Output Format:
-  - You have been provided with the tool that defines the output format; make sure to query it.
-  - Do not output anything other than the expected format.
-
-# Segment Handling Rules:
-  - Sort messages by timestamp.
-  - Leave ID of the first message in a sentence.
-  - Each divided sentence should be added to the output as a separate message or block within existing messages, preserving the original order.
-  - If one message contains multiple sentences, you should not split them.
-  - The last sentence may be incomplete; it should still be output as a separate block or message.
-  - Keep track of the original timestamps of the messages and blocks and, use the timestamp of the first message/block of a sentence as the 'started' field of the merged message.
-  - The 'rangeId' field of the merged message should include 'messageId'-s and 'rangeId'-s of all messages that have been used to construct this message.
-
-# Punctuation and Capitalization:
-  - Do not rely on the original punctuation and capitalization—they may be incorrect.
-  - Use logical reasoning to apply appropriate punctuation (e.g., period, comma, question mark, exclamation mark) and capitalization.
-
-# Restrictions:
-  - Do not alter the order of the original messages; maintain the natural flow of speech.
-  - Do not interpret or infer meaning beyond reconstructing sentences.
-  - Do not add or remove any words or phrases.
-`;
-
 export const sentenceNormalization = defineFunction<NormalizationInput, NormalizationOutput>({
   description: 'Post process of transcription for sentence normalization',
   inputSchema: NormalizationInput,
@@ -80,21 +50,27 @@ export const sentenceNormalization = defineFunction<NormalizationInput, Normaliz
   handler: async ({ data: { messages }, context }) => {
     log.info('input', { messages });
     const ai = context.getService(AiService);
-    const session = new AISession({ operationModel: 'configured' });
+    const session = new AiSession({ operationModel: 'configured' });
 
-    const response = await session.runStructured(NormalizationOutput, {
-      generationOptions: { model: DEFAULT_EDGE_MODEL },
+    // TODO(dmaretskyi): This got broken after effect-ai transition.
+    const response = session.runStructured(NormalizationOutput, {
+      generationOptions: {
+        model: DEFAULT_EDGE_MODEL,
+      },
       client: ai.client,
       tools: [],
       artifacts: [],
       history: [
-        Obj.make(Message, {
-          role: 'user',
-          content: messages.map((message) => ({ type: 'text', text: JSON.stringify(message) }) as const),
+        Obj.make(DataType.Message, {
+          created: new Date().toISOString(),
+          sender: {
+            role: 'user',
+          },
+          blocks: messages.map((message) => ({ _tag: 'text', text: JSON.stringify(message) }) as const),
         }),
       ],
       prompt,
-    });
+    }) as any;
 
     response.sentences.forEach((sentence) => {
       sentence.id = ObjectId.random();
@@ -103,3 +79,37 @@ export const sentenceNormalization = defineFunction<NormalizationInput, Normaliz
     return response;
   },
 });
+
+const prompt = trim`
+  You are observing a real-time transcript of a single person speaking.
+  The transcription is delivered in chunks of 10 seconds or less. As a result, individual sentences may be split across multiple messages, or multiple sentences may appear within a single message. Additionally, because this is real-time transcription, punctuation and capitalization may be incorrect or missing.
+
+  # Task Description:
+    - Your task is to detect and reconstruct broken or incomplete sentences by merging segments into coherent, grammatically correct sentences where appropriate.
+
+  # Input Format:
+    - The input is an array of messages that contain transcription blocks.
+    - Each message is a DataType.Message, as described in the output format.
+
+  # Output Format:
+    - You have been provided with the tool that defines the output format; make sure to query it.
+    - Do not output anything other than the expected format.
+
+  # Segment Handling Rules:
+    - Sort messages by timestamp.
+    - Leave ID of the first message in a sentence.
+    - Each divided sentence should be added to the output as a separate message or block within existing messages, preserving the original order.
+    - If one message contains multiple sentences, you should not split them.
+    - The last sentence may be incomplete; it should still be output as a separate block or message.
+    - Keep track of the original timestamps of the messages and blocks and, use the timestamp of the first message/block of a sentence as the 'started' field of the merged message.
+    - The 'rangeId' field of the merged message should include 'messageId'-s and 'rangeId'-s of all messages that have been used to construct this message.
+
+  # Punctuation and Capitalization:
+    - Do not rely on the original punctuation and capitalization—they may be incorrect.
+    - Use logical reasoning to apply appropriate punctuation (e.g., period, comma, question mark, exclamation mark) and capitalization.
+
+  # Restrictions:
+    - Do not alter the order of the original messages; maintain the natural flow of speech.
+    - Do not interpret or infer meaning beyond reconstructing sentences.
+    - Do not add or remove any words or phrases.
+`;

package/src/testing/testing.ts

@@ -5,9 +5,7 @@
 import { Schema } from 'effect';
 import { useEffect, useMemo, useState } from 'react';
 
-import { EdgeAiServiceClient } from '@dxos/ai';
-import { AI_SERVICE_ENDPOINT } from '@dxos/ai/testing';
-import { extractionAnthropicFn, processTranscriptMessage } from '@dxos/assistant';
+import { extractionAnthropicFn, processTranscriptMessage } from '@dxos/assistant/extraction';
 import { scheduleTaskInterval } from '@dxos/async';
 import { Filter, type Queue } from '@dxos/client/echo';
 import { Context } from '@dxos/context';
@@ -17,7 +15,7 @@ import { FunctionExecutor, ServiceContainer } from '@dxos/functions';
 import { IdentityDid } from '@dxos/keys';
 import { log } from '@dxos/log';
 import { faker } from '@dxos/random';
-import { useQueue, type Space } from '@dxos/react-client/echo';
+import { type Space, useQueue } from '@dxos/react-client/echo';
 import { DataType } from '@dxos/schema';
 import { Testing, seedTestData } from '@dxos/schema/testing';
 
@@ -69,7 +67,7 @@ export class MessageBuilder extends AbstractMessageBuilder {
     });
   }
 
-  createBlock(): DataType.MessageBlock.Transcription {
+  createBlock(): DataType.MessageBlock.Transcript {
     let text = faker.lorem.paragraph();
     if (this._space) {
       const label = faker.commerce.productName();
@@ -81,7 +79,7 @@ export class MessageBuilder extends AbstractMessageBuilder {
     }
 
     return {
-      type: 'transcription',
+      _tag: 'transcript',
       started: this.next().toISOString(),
       text,
     };
@@ -95,11 +93,13 @@ export class MessageBuilder extends AbstractMessageBuilder {
 
 // TODO(burdon): Reconcile with BlockBuilder.
 class EntityExtractionMessageBuilder extends AbstractMessageBuilder {
-  AiService = new EdgeAiServiceClient({
-    endpoint: AI_SERVICE_ENDPOINT.REMOTE,
-  });
-
-  executor = new FunctionExecutor(new ServiceContainer().setServices({ ai: { client: this.AiService } }));
+  executor = new FunctionExecutor(
+    new ServiceContainer().setServices({
+      // ai: {
+      //   client: this.AiService,
+      // },
+    }),
+  );
 
   space: Space | undefined;
   currentMessage: number = 0;

package/src/transcriber/index.ts

@@ -2,7 +2,7 @@
 // Copyright 2025 DXOS.org
 //
 
-export * from './audio-recorder';
+export type * from './audio-recorder';
 export * from './media-stream-recorder';
 export * from './transcriber';
 export * from './transcription-manager';

package/src/transcriber/media-stream-recorder.ts

@@ -10,7 +10,7 @@ import { invariant } from '@dxos/invariant';
 import { log } from '@dxos/log';
 import { trace } from '@dxos/tracing';
 
-import { type WavConfig, type AudioChunk, type AudioRecorder } from './audio-recorder';
+import { type AudioChunk, type AudioRecorder, type WavConfig } from './audio-recorder';
 
 let initializingPromise: Promise<void> | undefined;
 

package/src/transcriber/transcriber.browser.test.ts

@@ -4,6 +4,7 @@
 
 import fs from 'node:fs';
 import path from 'node:path';
+
 import { describe, expect, test } from 'vitest';
 import { WaveFile } from 'wavefile';
 
@@ -11,7 +12,7 @@ import { Trigger } from '@dxos/async';
 import { log } from '@dxos/log';
 import { type DataType } from '@dxos/schema';
 import { openAndClose } from '@dxos/test-utils';
-import { trace, TRACE_PROCESSOR } from '@dxos/tracing';
+import { TRACE_PROCESSOR, trace } from '@dxos/tracing';
 
 import { type AudioChunk, type AudioRecorder, Transcriber } from '../transcriber';
 import { mergeFloat64Arrays } from '../util';
@@ -93,7 +94,7 @@ describe.skip('Transcriber', () => {
   });
 
   test.skip('transcription of audio recording', { timeout: 10_000 }, async () => {
-    const trigger = new Trigger<DataType.MessageBlock.Transcription[]>({ autoReset: true });
+    const trigger = new Trigger<DataType.MessageBlock.Transcript[]>({ autoReset: true });
     const recorder = new MockAudioRecorder({
       buffer: await readFile('test.wav'),
       chunkDuration: 3_000,
@@ -127,7 +128,7 @@ describe.skip('Transcriber', () => {
   });
 
   test.skip('transcription of audio recording with overlapping chunks', { timeout: 20_000 }, async () => {
-    const trigger = new Trigger<DataType.MessageBlock.Transcription[]>({ autoReset: true });
+    const trigger = new Trigger<DataType.MessageBlock.Transcript[]>({ autoReset: true });
     const recorder = new MockAudioRecorder({
       buffer: await readFile('test.wav'),
       chunkDuration: 3_000,

package/src/transcriber/transcriber.ts

@@ -10,10 +10,11 @@ import { log } from '@dxos/log';
 import { type DataType } from '@dxos/schema';
 import { trace } from '@dxos/tracing';
 
-import { type AudioRecorder, type AudioChunk } from './audio-recorder';
 import { TRANSCRIPTION_URL } from '../types';
 import { mergeFloat64Arrays } from '../util';
 
+import { type AudioChunk, type AudioRecorder } from './audio-recorder';
+
 type WhisperWord = {
   word: string;
 
@@ -69,7 +70,7 @@ export type TranscriberParams = {
    * Callback to handle the transcribed segments, after all segment transformers are applied.
    * @param segments - The transcribed segments.
    */
-  onSegments: (segments: DataType.MessageBlock.Transcription[]) => Promise<void>;
+  onSegments: (segments: DataType.MessageBlock.Transcript[]) => Promise<void>;
 };
 
 /**
@@ -220,10 +221,7 @@ export class Transcriber extends Resource {
     return segments;
   }
 
-  private _alignSegments(
-    segments: WhisperSegment[],
-    originalChunks: AudioChunk[],
-  ): DataType.MessageBlock.Transcription[] {
+  private _alignSegments(segments: WhisperSegment[], originalChunks: AudioChunk[]): DataType.MessageBlock.Transcript[] {
     // Absolute zero for all relative timestamps in the segments.
     const zeroTimestamp = originalChunks.at(0)!.timestamp;
 
@@ -252,7 +250,7 @@ export class Transcriber extends Resource {
 
     // Add absolute timestamp to each segment.
     return filteredSegments.map((segment) => ({
-      type: 'transcription',
+      _tag: 'transcript',
       started: new Date(zeroTimestamp + segment.start * 1_000).toISOString(),
       text: segment.text.trim(),
     }));

package/src/transcriber/transcription-manager.ts

@@ -133,7 +133,7 @@ export class TranscriptionManager extends Resource {
       // TODO(burdon): Started and stopped blocks appear twice.
       const block = Obj.make(DataType.Message, {
         created: new Date().toISOString(),
-        blocks: [{ type: 'transcription', text: 'Started', started: new Date().toISOString() }],
+        blocks: [{ _tag: 'transcript', text: 'Started', started: new Date().toISOString() }],
         sender: { role: 'assistant' },
       });
       await this._queue?.append([block]);
@@ -141,7 +141,7 @@ export class TranscriptionManager extends Resource {
       await this._transcriber?.close();
       const block = Obj.make(DataType.Message, {
         created: new Date().toISOString(),
-        blocks: [{ type: 'transcription', text: 'Stopped', started: new Date().toISOString() }],
+        blocks: [{ _tag: 'transcript', text: 'Stopped', started: new Date().toISOString() }],
         sender: { role: 'assistant' },
       });
       await this._queue?.append([block]);
@@ -175,7 +175,7 @@ export class TranscriptionManager extends Resource {
     }
   }
 
-  private async _onSegments(segments: DataType.MessageBlock.Transcription[]): Promise<void> {
+  private async _onSegments(segments: DataType.MessageBlock.Transcript[]): Promise<void> {
     if (!this.isOpen || !this._queue) {
       return;
     }

package/src/translations.ts

@@ -2,12 +2,14 @@
 // Copyright 2023 DXOS.org
 //
 
-import { TRANSCRIPTION_PLUGIN } from './meta';
+import { type Resource } from '@dxos/react-ui';
 
-export default [
+import { meta } from './meta';
+
+export const translations = [
   {
     'en-US': {
-      [TRANSCRIPTION_PLUGIN]: {
+      [meta.id]: {
         'plugin name': 'Transcription',
         'transcript companion label': 'Transcript',
 
@@ -19,4 +21,4 @@ export default [
       },
     },
   },
-];
+] as const satisfies Resource[];

package/src/types/types.ts

@@ -6,9 +6,10 @@ import { Schema } from 'effect';
 
 import { SpaceId } from '@dxos/react-client/echo';
 
-import { TranscriptType } from './schema';
 import { TRANSCRIPTION_PLUGIN } from '../meta';
 
+import { TranscriptType } from './schema';
+
 // TODO(burdon): Move to separate proto.
 
 /**

package/dist/lib/browser/TranscriptContainer-H3AZQ4CQ.mjs

@@ -1,11 +0,0 @@
-import {
-  TranscriptContainer_default,
-  TranscriptionContainer
-} from "./chunk-53AXUEIQ.mjs";
-import "./chunk-W2KNYJD6.mjs";
-import "./chunk-GOF7MJNV.mjs";
-export {
-  TranscriptionContainer,
-  TranscriptContainer_default as default
-};
-//# sourceMappingURL=TranscriptContainer-H3AZQ4CQ.mjs.map