@dxos/plugin-transcription 0.8.2-staging.7ac8446 → 0.8.2

package/src/model/model.test.ts

@@ -0,0 +1,169 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import { EditorView } from '@codemirror/view';
+import { describe, test } from 'vitest';
+
+import { type DataType } from '@dxos/schema';
+
+import { SerializationModel, DocumentAdapter, type ChunkRenderer } from './model';
+
+const blockToMarkdown: ChunkRenderer<DataType.Message> = (
+  message: DataType.Message,
+  index: number,
+  debug = true,
+): string[] => {
+  return [
+    `###### ${message.sender.name}`,
+    ...message.blocks.filter((block) => block.type === 'transcription').map((block) => block.text),
+    '',
+  ];
+};
+
+const createDate = () => new Date().toISOString();
+
+describe('SerializationModel', () => {
+  test('basic', ({ expect }) => {
+    const model = new SerializationModel<DataType.Message>(blockToMarkdown);
+    expect(model.chunks.length).to.eq(0);
+    expect(model.doc.toString()).to.eq('');
+
+    // Create message.
+    const message: DataType.Message = {
+      id: '1',
+      created: createDate(),
+      sender: { name: 'Alice' },
+      blocks: [
+        {
+          type: 'transcription',
+          started: createDate(),
+          text: 'Hello world!',
+        },
+      ],
+    };
+    model.appendChunk(message);
+    {
+      const text = model.doc.toString();
+      expect(text).to.eq('###### Alice\nHello world!\n\n');
+      expect(model.chunks.length).to.eq(1);
+    }
+
+    // Update message.
+    message.blocks.push({ type: 'transcription', started: createDate(), text: 'Hello again!' });
+    model.updateChunk(message);
+    {
+      const text = model.doc.toString();
+      expect(text).to.eq('###### Alice\nHello world!\nHello again!\n\n');
+      expect(model.chunks.length).to.eq(1);
+    }
+  });
+
+  test('sync - append', ({ expect }) => {
+    const view = new EditorView({ extensions: [], doc: '' });
+    const model = new SerializationModel<DataType.Message>(blockToMarkdown);
+    const adapter = new DocumentAdapter(view);
+    expect(adapter.lineCount()).to.eq(0);
+    expect(view.state.doc.toString()).to.eq('');
+
+    // Append message.
+    {
+      const message: DataType.Message = {
+        id: '1',
+        created: createDate(),
+        sender: { name: 'Alice' },
+        blocks: [
+          {
+            type: 'transcription',
+            started: createDate(),
+            text: 'Hello world!',
+          },
+        ],
+      };
+      model.appendChunk(message);
+      model.sync(adapter);
+      expect(view.state.doc.toString()).to.eq('###### Alice\nHello world!\n\n');
+    }
+
+    // Append message.
+    {
+      const message: DataType.Message = {
+        id: '2',
+        created: createDate(),
+        sender: { name: 'Bob' },
+        blocks: [
+          {
+            type: 'transcription',
+            started: createDate(),
+            text: 'Hello world!',
+          },
+        ],
+      };
+      model.appendChunk(message);
+      model.sync(adapter);
+      expect(view.state.doc.toString()).to.eq('###### Alice\nHello world!\n\n###### Bob\nHello world!\n\n');
+    }
+  });
+
+  test('sync - append, update, delete', ({ expect }) => {
+    const view = new EditorView({ extensions: [], doc: '' });
+    const model = new SerializationModel<DataType.Message>(blockToMarkdown);
+    const adapter = new DocumentAdapter(view);
+    expect(adapter.lineCount()).to.eq(0);
+
+    // Append message.
+    {
+      const message: DataType.Message = {
+        id: '1',
+        created: createDate(),
+        sender: { name: 'Alice' },
+        blocks: [
+          {
+            type: 'transcription',
+            started: createDate(),
+            text: 'Hello world!',
+          },
+        ],
+      };
+      model.appendChunk(message);
+      model.sync(adapter);
+      expect(view.state.doc.toString()).to.eq('###### Alice\nHello world!\n\n');
+
+      // Update message (add block).
+      message.blocks.push({ type: 'transcription', started: createDate(), text: 'Hello again!' });
+      model.updateChunk(message);
+      model.sync(adapter);
+      expect(view.state.doc.toString()).to.eq('###### Alice\nHello world!\nHello again!\n\n');
+
+      // Update message (remove block).
+      message.blocks.splice(0, 1);
+      model.updateChunk(message);
+      model.sync(adapter);
+      expect(view.state.doc.toString()).to.eq('###### Alice\nHello again!\n\n');
+
+      expect(model.getChunkAtLine(1)).to.eq(message);
+    }
+
+    // Append message.
+    {
+      const message: DataType.Message = {
+        id: '2',
+        created: createDate(),
+        sender: { name: 'Bob' },
+        blocks: [{ type: 'transcription', started: createDate(), text: 'Hello again!' }],
+      };
+      model.appendChunk(message);
+      model.sync(adapter);
+      expect(view.state.doc.toString()).to.eq('###### Alice\nHello again!\n\n###### Bob\nHello again!\n\n');
+
+      expect(model.getChunkAtLine(4)).to.eq(message);
+    }
+
+    // Delete block.
+    {
+      model.deleteBlock('1');
+      model.sync(adapter);
+      expect(view.state.doc.toString()).to.eq('###### Bob\nHello again!\n\n');
+    }
+  });
+});

package/src/model/model.ts

@@ -0,0 +1,229 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import { Text } from '@codemirror/state';
+import { type EditorView } from '@codemirror/view';
+
+import { Event } from '@dxos/async';
+import { invariant } from '@dxos/invariant';
+import { log } from '@dxos/log';
+
+export type Chunk = { id: string };
+
+/**
+ * Covert block to a set of markdown lines.
+ */
+export type ChunkRenderer<T extends Chunk> = (chunk: T, index: number, debug?: boolean) => string[];
+
+/**
+ * Abstract representation of the document model.
+ */
+export interface ChunkDocument {
+  lineCount(): number;
+  replaceLines(from: number, remove: number, insert: string[]): void;
+}
+
+/**
+ * Codemirror document adapter.
+ */
+export class DocumentAdapter implements ChunkDocument {
+  constructor(private readonly _view: EditorView) {}
+
+  /** The document must always have at least one line. */
+  lineCount(): number {
+    return this._view.state.doc.lines - 1;
+  }
+
+  replaceLines(from: number, remove: number, insert: string[]): void {
+    log('replaceLines', { count: this._view.state.doc.lines, from, remove, insert });
+
+    const numLines = this._view.state.doc.lines;
+    const posFrom = this._view.state.doc.line(from).from;
+    const posTo = remove === 0 ? posFrom : this._view.state.doc.line(Math.min(numLines, from + remove)).from;
+
+    // Don't remove the last line.
+    let text = insert.join('\n');
+    if (posTo === this._view.state.doc.length) {
+      text += '\n';
+    }
+
+    this._view.dispatch({
+      changes: { from: posFrom, to: posTo, insert: text },
+    });
+  }
+}
+
+type ChunkChange<T extends Chunk> =
+  | {
+      type: 'append';
+      index: number;
+      chunk: T;
+    }
+  | {
+      type: 'update';
+      index: number;
+      chunk: T;
+    }
+  | {
+      type: 'delete';
+      index: number;
+      id: string;
+    };
+
+/**
+ * Each Chunk can be converted into a set of lines.
+ * Chunks can be added, updated or deleted from the Queue.
+ * A Document is an immutable projection that represents a flat list of lines corresponding to the Queue.
+ *
+ * Ideally we would implement a custom virtual Text model for the View, but this currently isn't possible in Codemirror.
+ * Instead this model tracks changes and syncs them with the ChunkDocument.
+ */
+export class SerializationModel<T extends Chunk> {
+  /** Ordered set of chunks. */
+  private readonly _chunks: T[] = [];
+
+  /** Map of chunk IDs to their current line counts. */
+  private readonly _chunkLineCounts = new Map<string, number>();
+
+  /** Map of line numbers to chunk IDs. */
+  private readonly _lineToChunk = new Map<number, T>();
+
+  /** Track chunk changes since last sync. */
+  private readonly _changes: Array<ChunkChange<T>> = [];
+
+  /** Emits when the document is updated. */
+  public readonly update = new Event<void>();
+
+  constructor(
+    private readonly _renderer: ChunkRenderer<T>,
+    initialChunks: T[] = [],
+  ) {
+    initialChunks.forEach((chunk) => {
+      this.appendChunk(chunk);
+    });
+  }
+
+  toJSON(): { chunks: number; changes: number } {
+    return {
+      chunks: this._chunks.length,
+      changes: this._changes.length,
+    };
+  }
+
+  get doc() {
+    return Text.of([...this._chunks.flatMap((chunk, index) => this._renderer(chunk, index)), '']);
+  }
+
+  get chunks() {
+    return this._chunks;
+  }
+
+  /**
+   * Get the block ID at the given line number.
+   */
+  getChunkAtLine(line: number): T | undefined {
+    return this._lineToChunk.get(line);
+  }
+
+  reset(): this {
+    this._chunks.length = 0;
+    this._chunkLineCounts.clear();
+    this._lineToChunk.clear();
+    this.update.emit();
+    return this;
+  }
+
+  appendChunk(chunk: T): this {
+    const index = this._chunks.length;
+    this._chunks.push(chunk);
+    this._changes.push({ type: 'append', index, chunk });
+    this.update.emit();
+    return this;
+  }
+
+  updateChunk(chunk: T): this {
+    const index = this._chunks.findIndex((c) => c.id === chunk.id);
+    invariant(index !== -1);
+    this._chunks[index] = chunk;
+    this._changes.push({ type: 'update', index, chunk });
+    this.update.emit();
+    return this;
+  }
+
+  deleteBlock(id: string): this {
+    const index = this._chunks.findIndex((c) => c.id === id);
+    invariant(index !== -1);
+    this._chunks.splice(index, 1);
+    this._changes.push({ type: 'delete', index, id });
+    this.update.emit();
+    return this;
+  }
+
+  /**
+   * Syncs the tracked changes with the ChunkDocument.
+   */
+  sync(document: ChunkDocument): this {
+    log('sync', { changes: this._changes });
+
+    // Process each change in order.
+    for (const change of this._changes) {
+      switch (change.type) {
+        case 'append': {
+          // For appends, convert block to lines and add at the end.
+          const lines = this._renderer(change.chunk, change.index);
+          const lastLine = document.lineCount();
+          document.replaceLines(lastLine + 1, 0, lines);
+          break;
+        }
+
+        case 'update': {
+          // For updates, find the block's line range and replace with new lines.
+          let lineStart = 1;
+          for (let i = 0; i < change.index; i++) {
+            lineStart += this._chunkLineCounts.get(this._chunks[i].id)!;
+          }
+          const previousLineCount = this._chunkLineCounts.get(change.chunk.id)!;
+          const newLines = this._renderer(change.chunk, change.index);
+          document.replaceLines(lineStart, previousLineCount, newLines);
+          break;
+        }
+
+        case 'delete': {
+          // For deletes, find the block's line range and remove those lines.
+          let lineStart = 1;
+          for (let i = 0; i < change.index; i++) {
+            lineStart += this._chunkLineCounts.get(this._chunks[i].id)!;
+          }
+          const previousLineCount = this._chunkLineCounts.get(change.id)!;
+          document.replaceLines(lineStart, previousLineCount, []);
+          break;
+        }
+      }
+    }
+
+    // Clear the line-to-block mapping.
+    this._lineToChunk.clear();
+
+    // Update all block line counts and line-to-block mapping after processing changes.
+    let currentLine = 1;
+    for (const [index, chunk] of this._chunks.entries()) {
+      const lineCount = this._renderer(chunk, index).length;
+      this._chunkLineCounts.set(chunk.id, lineCount);
+      this._lineToChunk.set(currentLine, chunk);
+      currentLine += lineCount;
+    }
+
+    // Clean up deleted blocks.
+    const chunkIds = new Set(this._chunks.map((chunk) => chunk.id));
+    for (const id of this._chunkLineCounts.keys()) {
+      if (!chunkIds.has(id)) {
+        this._chunkLineCounts.delete(id);
+      }
+    }
+
+    // Clear changes after sync.
+    this._changes.length = 0;
+    return this;
+  }
+}

package/src/segments-normalization/index.ts

@@ -0,0 +1,7 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+export * from './normalization';
+export * from './message-normalizer';
+export * from './utils';

package/src/segments-normalization/message-normalizer.ts

@@ -0,0 +1,108 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+//
+// Copyright 2025 DXOS.org
+//
+
+import { effect } from '@preact/signals-core';
+
+import { asyncTimeout, DeferredTask } from '@dxos/async';
+import { LifecycleState, Resource } from '@dxos/context';
+import { type Queue } from '@dxos/echo-db';
+import { type FunctionExecutor } from '@dxos/functions';
+import { log } from '@dxos/log';
+import { type DataType } from '@dxos/schema';
+
+import { type MessageWithRangeId, sentenceNormalization } from './normalization';
+import { getActorId } from './utils';
+
+const PROCESSING_TIMEOUT = 20_000; // ms
+const MAX_RANGE_ID_COUNT = 10;
+
+export type SegmentsNormalizerParams = {
+  functionExecutor: FunctionExecutor;
+  queue: Queue<DataType.Message>;
+  startingCursor: QueueCursor;
+};
+
+// TODO(mykola): Use generic queue cursor from @dxos/protocols.
+export type QueueCursor = {
+  actorId: string;
+  timestamp: string;
+};
+
+// TODO(mykola): .
+export class MessageNormalizer extends Resource {
+  private readonly _functionExecutor: FunctionExecutor;
+  private _queue: Queue<DataType.Message>;
+  private _cursor: QueueCursor;
+  private _messagesToProcess: MessageWithRangeId[] = [];
+  private _normalizationTask?: DeferredTask;
+  private _lastProcessedMessageIds?: string[];
+
+  constructor({ functionExecutor, queue, startingCursor }: SegmentsNormalizerParams) {
+    super();
+    this._functionExecutor = functionExecutor;
+    this._queue = queue;
+    this._cursor = startingCursor;
+  }
+
+  protected override async _open(): Promise<void> {
+    this._normalizationTask = new DeferredTask(this._ctx, () => this._processMessages());
+    const unsubscribe = effect(() => {
+      if (this._lifecycleState !== LifecycleState.OPEN) {
+        return;
+      }
+
+      this._messagesToProcess = this._queue.objects.filter((message) => {
+        const actorId = getActorId(message.sender);
+        return actorId === this._cursor.actorId && message.created >= this._cursor.timestamp;
+      });
+
+      this._normalizationTask!.schedule();
+    });
+    this._ctx.onDispose(() => unsubscribe());
+  }
+
+  // Need to unpack strings from blocks from messages run them through the function and then pack them back into blocks into messages.
+  private async _processMessages(): Promise<void> {
+    const messages = this._messagesToProcess;
+    this._messagesToProcess = [];
+    if (
+      this._lastProcessedMessageIds &&
+      messages.every((message) => this._lastProcessedMessageIds!.includes(message.id))
+    ) {
+      return;
+    }
+
+    this._lastProcessedMessageIds = messages.map((message) => message.id);
+
+    try {
+      // TODO(mykola): Executor should support timeout.
+      const response = await asyncTimeout(
+        this._functionExecutor.invoke(sentenceNormalization, { messages }),
+        PROCESSING_TIMEOUT,
+      );
+
+      this._writeMessages(response.sentences);
+      this._lastProcessedMessageIds.push(...response.sentences.map((sentence) => sentence.id));
+      if (response.sentences.length === 1 && response.sentences.at(-1)!.rangeId!.length > MAX_RANGE_ID_COUNT) {
+        log.warn('Sentence is too long', { messages });
+        this._cursor.timestamp = new Date(new Date(response.sentences[0].created).getTime() + 1).toISOString();
+      }
+    } catch {
+      log.error('Failed to normalize segments', { messages });
+      // If processing failed, emit the segments as is.
+      this._writeMessages(messages);
+    }
+  }
+
+  private _writeMessages(messages: MessageWithRangeId[]): void {
+    log.info('writing messages', { messages });
+    const lastMessage = messages[messages.length - 1];
+    this._cursor.timestamp = lastMessage.created;
+    this._queue.append(messages);
+  }
+}

package/src/segments-normalization/normalization.test.ts

@@ -0,0 +1,142 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import { effect } from '@preact/signals-core';
+import { describe, test } from 'vitest';
+
+import { AIServiceEdgeClient, OllamaClient } from '@dxos/ai';
+import { AI_SERVICE_ENDPOINT } from '@dxos/ai/testing';
+import { scheduleTaskInterval } from '@dxos/async';
+import { Context } from '@dxos/context';
+import { MemoryQueue } from '@dxos/echo-db';
+import { createQueueDxn, ObjectId } from '@dxos/echo-schema';
+import { FunctionExecutor, ServiceContainer } from '@dxos/functions';
+import { log } from '@dxos/log';
+import { type DataType } from '@dxos/schema';
+
+import { MessageNormalizer } from './message-normalizer';
+import { type MessageWithRangeId, sentenceNormalization } from './normalization';
+import { getActorId } from './utils';
+
+const sender: DataType.Actor = {
+  identityDid: 'did:key:123',
+};
+
+// Generate bunch of complex messages.
+const messages: MessageWithRangeId[] = [
+  // Control message.
+  'Hello, every body. We will talk about quantum entanglement today',
+
+  // Two sentences in one message.
+  'hello every body we will talk about quantum entanglement today',
+
+  // Sentence broken into pieces.
+  'quantum entanglement',
+  'is one of the most...',
+  'perplexing and fascinating phenomena',
+  'in modern physics',
+
+  // Two sentences without separators.
+  'it challenges our classical intuitions about how the universe works and forces us to reconsider our assumptions about space, time, and information',
+  'at its core, quantum entanglement refers to a peculiar connection between two or more particles in such a way that the state of one particle instantly...',
+  'determines the state of the other, no matter how far apart they may be — even if they are light-years away',
+
+  // 2 sentences are merged.
+  'this seemingly instantaneous correlation between distant particles has baffled physicists since the early 20th century to understand entanglement, we',
+  'first need to touch on the fundamentals of quantum mechanics',
+
+  // No punctuation.
+  'in classical physics objects have well-defined properties such as position speed and momentum',
+].map((string, index) => ({
+  id: ObjectId.random(),
+  created: new Date(Date.now() + 1000 * index).toISOString(),
+  sender,
+  blocks: [{ type: 'transcription', started: new Date(Date.now() + 1000 * index).toISOString(), text: string }],
+  rangeId: [],
+}));
+
+const REMOTE_AI = true;
+
+describe.skip('SentenceNormalization', () => {
+  const getExecutor = () => {
+    return new FunctionExecutor(
+      new ServiceContainer().setServices({
+        ai: {
+          client: REMOTE_AI
+            ? new AIServiceEdgeClient({
+                endpoint: AI_SERVICE_ENDPOINT.REMOTE,
+                defaultGenerationOptions: {
+                  model: '@anthropic/claude-3-5-sonnet-20241022',
+                },
+              })
+            : new OllamaClient({
+                overrides: {
+                  model: 'llama3.1:8b',
+                },
+              }),
+        },
+      }),
+    );
+  };
+
+  test('messages merging', { timeout: 120_000 }, async () => {
+    const executor = getExecutor();
+    const sentences: MessageWithRangeId[] = [];
+    let buffer: MessageWithRangeId[] = [];
+    let activeSentenceIndex = 0;
+    for (const message of messages) {
+      const result = await executor.invoke(sentenceNormalization, {
+        messages: [...buffer, message],
+      });
+      log.info('BEFORE sentence splicing', { activeSentenceIndex, sentences, inserting: result.sentences });
+      sentences.splice(activeSentenceIndex, sentences.length - activeSentenceIndex, ...result.sentences);
+      log.info('AFTER sentence splicing', { activeSentenceIndex, sentences });
+
+      if (sentences.length > 0) {
+        activeSentenceIndex = sentences.length - 1;
+        buffer = sentences.slice(activeSentenceIndex);
+      }
+    }
+    log.info('sentences', {
+      originalMessages: JSON.stringify(messages, null, 2),
+      sentences: JSON.stringify(sentences, null, 2),
+    });
+    throw new Error('test');
+  });
+
+  test.only('queue', { timeout: 120_000 }, async () => {
+    // Create queue.
+    const queue = new MemoryQueue<DataType.Message>(createQueueDxn());
+    const ctx = new Context();
+    let idx = 0;
+    scheduleTaskInterval(
+      ctx,
+      async () => {
+        if (idx >= messages.length) {
+          void ctx.dispose();
+          return;
+        }
+        await queue.append([messages[idx]]);
+        idx++;
+      },
+      2_000,
+    );
+
+    // Create normalizer.
+    const normalizer = new MessageNormalizer({
+      functionExecutor: getExecutor(),
+      queue,
+      startingCursor: { actorId: getActorId(sender), timestamp: '0' },
+    });
+
+    // Start normalizer.
+    await normalizer.open();
+    effect(() => {
+      log.info('normalizer');
+      log.info(JSON.stringify(queue.objects, null, 2));
+    });
+
+    await new Promise(() => {});
+  });
+});