@dxos/plugin-transcription 0.9.0 → 0.9.1-staging.ee54ba693a

package/src/components/Transcription/transcription-extension.ts

@@ -1,203 +0,0 @@
-//
-// Copyright 2025 DXOS.org
-//
-
-import { type Extension, RangeSetBuilder } from '@codemirror/state';
-import { EditorView, GutterMarker, ViewPlugin, type ViewUpdate, gutter } from '@codemirror/view';
-import { format } from 'date-fns/format';
-import { intervalToDuration } from 'date-fns/intervalToDuration';
-
-import { type CleanupFn, addEventListener, combine } from '@dxos/async';
-import { type Message } from '@dxos/types';
-
-import { DocumentAdapter, type SerializationModel } from '../../model';
-
-/**
- * Data structure that maps Chunks queue to lines with transcript state.
- */
-export type TranscriptionOptions = {
-  model: SerializationModel<Message.Message>;
-  started?: Date;
-};
-
-/**
- * Scrolling transcript with timestamps.
- */
-export const transcription = ({ model, started }: TranscriptionOptions): Extension => {
-  return [
-    // Show timestamps in the gutter.
-    gutter({
-      class: 'cm-timestamp-gutter',
-      lineMarkerChange: (update) => update.docChanged || update.viewportChanged,
-      markers: (view) => {
-        const start = getStartTime(started, model.chunks[0]);
-        const builder = new RangeSetBuilder<GutterMarker>();
-        for (const { from, to } of view.visibleRanges) {
-          let line = view.state.doc.lineAt(from);
-          while (line.from <= to) {
-            const block = model.getChunkAtLine(line.number)?.blocks[0];
-            const timestamp = block?._tag === 'transcript' && block.started;
-            if (timestamp) {
-              builder.add(line.from, line.from, new TimestampMarker(line.number, new Date(timestamp), start));
-            }
-
-            if (line.to + 1 > view.state.doc.length) {
-              break;
-            }
-
-            line = view.state.doc.lineAt(line.to + 1);
-          }
-        }
-
-        return builder.finish();
-      },
-    }),
-
-    // Listen for model updates.
-    ViewPlugin.fromClass(
-      class {
-        private readonly _controls?: HTMLDivElement;
-        private readonly _cleanup: CleanupFn;
-        private readonly _adapter: DocumentAdapter;
-        private _initialized = false;
-
-        constructor(view: EditorView) {
-          this._adapter = new DocumentAdapter(view);
-
-          const scroller = view.scrollDOM;
-          let isAutoScrolling = false;
-          let hasScrolled = false;
-
-          let timeout: NodeJS.Timeout | undefined;
-          const scrollToBottom = (smooth = false) => {
-            scroller.style.scrollBehavior = smooth ? 'smooth' : '';
-
-            // Temporarily hide scrollbar to prevent flicker.
-            scroller.classList.add('cm-hide-scrollbar');
-            isAutoScrolling = true;
-            clearTimeout(timeout);
-            timeout = setTimeout(() => {
-              this._controls?.classList.add('opacity-0');
-              scroller.classList.remove('cm-hide-scrollbar');
-              isAutoScrolling = false;
-            }, 500);
-
-            // Scroll to bottom.
-            view.dispatch({
-              effects: EditorView.scrollIntoView(view.state.doc.length, { y: 'end' }),
-            });
-          };
-
-          // Event listeners.
-          this._cleanup = combine(
-            addEventListener(view.scrollDOM, 'scroll', () => {
-              if (!isAutoScrolling) {
-                hasScrolled = true;
-                this._controls?.classList.remove('opacity-0');
-              }
-            }),
-
-            model.update.on(() => {
-              // Check if clamped to bottom.
-              const autoScroll =
-                scroller.scrollHeight - scroller.scrollTop - scroller.clientHeight === 0 || !hasScrolled;
-
-              // Sync.
-              model.sync(this._adapter);
-
-              // Scroll.
-              if (autoScroll) {
-                scrollToBottom(true);
-              }
-            }),
-          );
-        }
-
-        update(_update: ViewUpdate) {
-          // Initial sync.
-          if (!this._initialized) {
-            this._initialized = true;
-            setTimeout(() => {
-              model.sync(this._adapter);
-            });
-          }
-        }
-
-        destroy() {
-          this._controls?.remove();
-          this._cleanup();
-        }
-      },
-    ),
-
-    EditorView.theme({
-      '.cm-gutters': {
-        backgroundColor: 'var(--color-base-surface)',
-      },
-      '.cm-timestamp-gutter': {
-        width: '6rem',
-        paddingLeft: '0.5rem',
-        paddingRight: '1rem',
-      },
-      '.cm-timestamp-gutter > .cm-gutterElement > div': {
-        display: 'inline-flex',
-        textAlign: 'right',
-        padding: '3px',
-      },
-    }),
-  ];
-};
-
-/**
- * Gutter marker that displays a timestamp.
- */
-class TimestampMarker extends GutterMarker {
-  constructor(
-    readonly _line: number,
-    readonly _timestamp: Date,
-    readonly _started?: Date,
-  ) {
-    super();
-  }
-
-  override eq(other: this) {
-    return this._timestamp === other._timestamp;
-  }
-
-  override toDOM(view: EditorView) {
-    const el = document.createElement('div');
-    el.className = 'text-sm text-subdued hover:bg-hover-surface cursor-pointer';
-    el.textContent = formatTimestamp(this._timestamp, this._started);
-    // TODO(burdon): Click to bookmark or copy hyperlink.
-    el.onclick = () => {
-      const pos = view.state.doc.line(this._line).from;
-      view.dispatch({
-        effects: EditorView.scrollIntoView(pos, { y: 'start' }),
-      });
-    };
-
-    return el;
-  }
-}
-
-const getStartTime = (started?: Date, message?: Message.Message): Date | undefined => {
-  if (started) {
-    return started;
-  }
-
-  if (message?.blocks[0]?._tag === 'transcript' && message.blocks[0].started) {
-    return new Date(message.blocks[0].started);
-  }
-
-  return undefined;
-};
-
-const formatTimestamp = (timestamp: Date, relative?: Date) => {
-  if (relative) {
-    const pad = (n = 0) => String(n).padStart(2, '0');
-    const { hours, minutes, seconds } = intervalToDuration({ start: relative, end: timestamp });
-    return `${pad(hours)}:${pad(minutes)}:${pad(seconds)}`;
-  } else {
-    return format(timestamp, 'HH:mm:ss');
-  }
-};

package/src/enrichment/conversation.test.ts

@@ -1,162 +0,0 @@
-//
-// Copyright 2024 DXOS.org
-//
-
-// Headless test that simulates a two-speaker conversation as a pair of Whisper streams and
-// drives them through the (currently stubbed) Pass A enrichment and Pass B summarization
-// operations. The handlers are deterministic placeholders — they don't call an LLM — so the
-// test runs offline; switching to the real LLM-backed handlers later doesn't change the
-// harness, just the assertions.
-//
-// What this verifies today:
-//   - Conversation fixture parses to entities + speaker-tagged utterances.
-//   - Stream simulator emits per-batch transcript blocks in chronological order.
-//   - Pass A's sliding-window enrichment links seeded ECHO entities by name and surfaces
-//     remaining capitalised tokens as candidates.
-//   - Pass B accumulates a per-speaker summary and resolves "I" → speaker display name.
-
-import { describe, it } from '@effect/vitest';
-import * as Effect from 'effect/Effect';
-import * as fs from 'node:fs';
-import * as path from 'node:path';
-import * as url from 'node:url';
-
-import { Blueprint, Operation, OperationHandlerSet } from '@dxos/compute';
-import { Database, Feed, Obj, Type } from '@dxos/echo';
-import { TestHelpers } from '@dxos/effect/testing';
-import { AssistantTestLayer } from '@dxos/functions-runtime/testing';
-import { EntityId } from '@dxos/keys';
-import { Organization, Person } from '@dxos/types';
-
-import { EnrichTranscript, EnrichmentHandlers, SummarizeConversation } from './operations';
-import { parseConversation } from './parser';
-import { simulateStream } from './stream-simulator';
-
-EntityId.dangerouslyDisableRandomness();
-
-const TestLayer = AssistantTestLayer({
-  operationHandlers: OperationHandlerSet.make(...EnrichmentHandlers),
-  types: [Person.Person, Organization.Organization, Blueprint.Blueprint, Feed.Feed],
-  disableLlmMemoization: true,
-});
-
-describe('Transcript enrichment — conversation simulation', () => {
-  it.scoped(
-    'streams a two-speaker conversation through Pass A and Pass B (stub handlers)',
-    Effect.fnUntraced(
-      function* ({ expect }) {
-        // 1. Load and parse the fixture.
-        const fixturePath = path.join(path.dirname(url.fileURLToPath(import.meta.url)), 'conversation.txt');
-        const conversation = parseConversation(fs.readFileSync(fixturePath, 'utf8'));
-        expect(conversation.speakers).toEqual(['A', 'B']);
-        expect(conversation.utterances.length).toBeGreaterThan(0);
-        expect(conversation.entities.length).toBeGreaterThan(0);
-
-        // 2. Seed the ECHO space with the fixture's entities. Map fixture types → schema types.
-        const seeded: { id: string; typename: string; name: string }[] = [];
-        for (const entity of conversation.entities) {
-          if (entity.kind === 'Person') {
-            const fullName = entity.fields.fullName ?? entity.fields.name;
-            if (!fullName) {
-              continue;
-            }
-            const person = yield* Database.add(Obj.make(Person.Person, { fullName }));
-            seeded.push({ id: person.id, typename: Type.getTypename(Person.Person), name: fullName });
-          } else if (entity.kind === 'Organization') {
-            const name = entity.fields.name;
-            if (!name) {
-              continue;
-            }
-            const org = yield* Database.add(Obj.make(Organization.Organization, { name }));
-            seeded.push({ id: org.id, typename: Type.getTypename(Organization.Organization), name });
-          }
-        }
-        yield* Database.flush();
-        expect(seeded.map((s) => s.name)).toContain('Chad');
-        expect(seeded.map((s) => s.name)).toContain('Blue Yard');
-
-        // 3. Generate the per-batch Whisper schedule. Batch by speaker so each batch maps to
-        //    a single Pass-A invocation, mimicking what the live `useTranscriptEnrichment`
-        //    hook will receive from the transcriber.
-        const batches = simulateStream(conversation, { wordsPerSecond: 2.5, gapSeconds: 0.5 });
-        expect(batches.length).toBe(conversation.utterances.length);
-        expect(batches[0].offsetMs).toBe(0);
-        expect(batches.every((b) => b.blocks.length > 0)).toBe(true);
-
-        // 4. Run Pass A over each batch — sliding window of the last 4 blocks across speakers.
-        const slidingWindow: { speaker: string; text: string; started: string }[] = [];
-        const enrichments: {
-          speaker: string;
-          corrected: string;
-          referenceIds: string[];
-          candidates: { text: string }[];
-        }[] = [];
-        for (const batch of batches) {
-          for (const block of batch.blocks) {
-            slidingWindow.push({ speaker: batch.speaker, text: block.text, started: block.started });
-            if (slidingWindow.length > 4) {
-              slidingWindow.shift();
-            }
-          }
-          const result = yield* Operation.invoke(EnrichTranscript, {
-            window: slidingWindow,
-            knownEntities: seeded,
-          });
-          // The new batch's entries are the *last* `batch.blocks.length` items in the window.
-          const newEntries = result.blocks.slice(-batch.blocks.length);
-          for (const block of newEntries) {
-            const windowEntry = slidingWindow[block.index];
-            enrichments.push({
-              speaker: windowEntry.speaker,
-              corrected: block.corrected,
-              referenceIds: [...block.referenceIds],
-              candidates: block.candidates.map((c) => ({ text: c.text })),
-            });
-          }
-        }
-
-        // 4a. Every block got a terminal-punctuated `corrected` value.
-        for (const e of enrichments) {
-          expect(e.corrected).toMatch(/[.!?]$/);
-        }
-
-        // 4b. The block containing "Chad" links to the seeded Person.
-        const chadId = seeded.find((s) => s.name === 'Chad')!.id;
-        const chadMention = enrichments.find((e) => /\bChad\b/.test(e.corrected));
-        expect(chadMention).toBeDefined();
-        expect(chadMention!.referenceIds).toContain(chadId);
-
-        // 4c. Place names surface as candidates (not in our seed).
-        const allCandidates = enrichments.flatMap((e) => e.candidates.map((c) => c.text));
-        expect(allCandidates).toContain('Munich');
-        expect(allCandidates).toContain('Los Angeles');
-
-        // 5. Run Pass B once at the end of the conversation (the real pipeline would fire it
-        //    after silence + interval; here we trigger it manually after the last batch).
-        const speakerNames = { A: 'Speaker A', B: 'Speaker B' };
-        const summary = yield* Operation.invoke(SummarizeConversation, {
-          previousSummary: '',
-          newUtterances: enrichments.map((e) => ({ speaker: e.speaker, text: e.corrected })),
-          speakerNames,
-          recentReferences: seeded.map((s) => ({ id: s.id, name: s.name })),
-        });
-
-        // 5a. Summary is non-empty and mentions both speakers.
-        expect(summary.summary.length).toBeGreaterThan(0);
-        expect(summary.summary).toContain('Speaker A');
-        expect(summary.summary).toContain('Speaker B');
-
-        // 5b. Resolved referents: every utterance that contains "I" picks up the speaker's name.
-        const iUtterances = enrichments.filter((e) => /\bI\b/.test(e.corrected));
-        expect(summary.resolvedReferents.length).toBe(iUtterances.length);
-        for (const referent of summary.resolvedReferents) {
-          expect(referent.surface).toBe('I');
-          expect([speakerNames.A, speakerNames.B]).toContain(referent.referent);
-        }
-      },
-      Effect.provide(TestLayer),
-      TestHelpers.provideTestContext,
-    ),
-    { timeout: 30_000 },
-  );
-});

package/src/enrichment/conversation.txt

@@ -1,23 +0,0 @@
-# Conversation fixture for transcription enrichment tests.
-# Format:
-#   `Entities:` block — declares ECHO objects to seed into the test space.
-#     Each entry is `<Type>:` followed by indented `<field>: <value>` lines.
-#   `Conversation:` block — one utterance per line, `<speaker>: <text>`.
-#   Speakers map to distinct virtual Whisper streams in the simulator.
-#   Lines starting with `#` are comments and are ignored by the parser.
-
-Entities:
-  Person:
-    fullName: Chad
-    employer: Blue Yard
-  Organization:
-    name: Blue Yard
-
-Conversation:
-  A: I think I'm going to go.
-  B: Are you meeting with them?
-  A: Yes I think so.
-  B: But I thought they were in Munich?
-  A: They are, but the offsite is in Los Angeles.
-  B: Is Chad going to be there?
-  A: Yes, the whole team. It's their annual offsite.

package/src/enrichment/operations.ts

@@ -1,271 +0,0 @@
-//
-// Copyright 2024 DXOS.org
-//
-
-// Operation definitions for the transcript enrichment pipeline (see ENRICHMENT.md).
-//
-// **Stub status.** The handlers below return deterministic dummy values rather than calling
-// an LLM; they exist so the end-to-end test harness (`conversation.test.ts`) can exercise
-// the schemas, the operation wiring, and the streaming/orchestration code paths without
-// requiring an Anthropic key on first run. The real handlers will replace these stubs in
-// a follow-up PR (calling `LanguageModel.generateObject(...)`) while keeping the same
-// inputs / outputs.
-
-import * as Effect from 'effect/Effect';
-import * as Schema from 'effect/Schema';
-
-import { Operation } from '@dxos/compute';
-import { DXN } from '@dxos/keys';
-
-import { meta } from '#meta';
-
-const NAMESPACE = `${meta.id}.enrichment`;
-
-//
-// Shared sub-schemas.
-//
-
-/** A noun / proper noun surfaced for follow-up actions (create object, research, link…). */
-export const CandidateMention = Schema.Struct({
-  text: Schema.String.annotations({ description: 'Surface form as it appears in the transcript.' }),
-  kind: Schema.Literal('noun', 'proper-noun').annotations({
-    description: 'Whether this is a common noun (e.g. "team") or a proper noun (e.g. "Munich").',
-  }),
-  start: Schema.Number.annotations({ description: 'Character offset within the corrected text.' }),
-  end: Schema.Number.annotations({ description: 'Character offset within the corrected text.' }),
-  suggested: Schema.optional(
-    Schema.Struct({
-      typename: Schema.String.annotations({
-        description: 'Optional hint for the ECHO type the user might create (e.g. "org.dxos.type.organization").',
-      }),
-    }),
-  ),
-});
-
-/** Per-block enrichment output produced by Pass A. */
-export const EnrichmentBlock = Schema.Struct({
-  /** Index into the input window; lets the caller match enrichment to the originating block. */
-  index: Schema.Number,
-  /** Corrected text — punctuation, capitalization, words split across batch boundaries reassembled. */
-  corrected: Schema.String,
-  /**
-   * IDs of ECHO objects the LLM is confident the block mentions. Strings rather than `Ref<T>`
-   * to keep the schema serialisable; the consumer resolves them via `Database.resolve(...)`.
-   */
-  referenceIds: Schema.Array(Schema.String),
-  /** Candidate nouns / proper nouns not already covered by `referenceIds`. */
-  candidates: Schema.Array(CandidateMention),
-});
-
-//
-// Pass A — enrichment (per Whisper batch / sliding window).
-//
-
-export const EnrichmentInput = Schema.Struct({
-  /** Sliding-window blocks. Each carries the speaker that produced it. */
-  window: Schema.Array(
-    Schema.Struct({
-      speaker: Schema.String,
-      text: Schema.String,
-      started: Schema.String,
-    }),
-  ),
-  /** ECHO objects available in the space that the LLM may pick from when linking references. */
-  knownEntities: Schema.optional(
-    Schema.Array(
-      Schema.Struct({
-        id: Schema.String,
-        typename: Schema.String,
-        name: Schema.String,
-      }),
-    ),
-  ),
-});
-export type EnrichmentInputType = Schema.Schema.Type<typeof EnrichmentInput>;
-
-export const EnrichmentOutput = Schema.Struct({
-  blocks: Schema.Array(EnrichmentBlock),
-});
-export type EnrichmentOutputType = Schema.Schema.Type<typeof EnrichmentOutput>;
-
-export const EnrichTranscript = Operation.make({
-  meta: {
-    key: DXN.make(`${NAMESPACE}.enrich`),
-    name: 'Enrich Transcript',
-    description: 'Correct Whisper output and surface entity references and candidates over a sliding window.',
-    icon: 'ph--microphone--regular',
-  },
-  input: EnrichmentInput,
-  output: EnrichmentOutput,
-});
-
-/**
- * Deterministic stub: echoes the input text as `corrected`, links every block whose text contains
- * a `knownEntities[i].name` substring to that entity's id, and surfaces every capitalised word
- * not already linked as a `proper-noun` candidate. Sentence-end punctuation is added if missing.
- */
-const enrichHandler: Operation.WithHandler<typeof EnrichTranscript> = EnrichTranscript.pipe(
-  Operation.withHandler(
-    Effect.fnUntraced(function* ({ window, knownEntities }) {
-      const known = knownEntities ?? [];
-      const blocks = window.map((entry, index) => {
-        const corrected = ensureTerminalPunctuation(entry.text);
-        const referenceIds: string[] = [];
-        const covered: Array<{ start: number; end: number }> = [];
-        for (const entity of known) {
-          const haystack = corrected.toLowerCase();
-          const needle = entity.name.toLowerCase();
-          const idx = haystack.indexOf(needle);
-          if (idx >= 0) {
-            referenceIds.push(entity.id);
-            covered.push({ start: idx, end: idx + entity.name.length });
-          }
-        }
-
-        const candidates = [...findCapitalisedWords(corrected)].filter(
-          ({ start, end }) => !covered.some((c) => c.start <= start && end <= c.end),
-        );
-
-        return { index, corrected, referenceIds, candidates };
-      });
-      return { blocks };
-    }),
-  ),
-);
-
-//
-// Pass B — summary (end-of-utterance + periodic).
-//
-
-export const SummaryInput = Schema.Struct({
-  /** The cumulative summary as of the last invocation (empty string on first call). */
-  previousSummary: Schema.String,
-  /** New corrected utterances appended since the previous summary. */
-  newUtterances: Schema.Array(
-    Schema.Struct({
-      speaker: Schema.String,
-      text: Schema.String,
-    }),
-  ),
-  /** Map of speaker label → human-readable display name. Drives "I" resolution. */
-  speakerNames: Schema.optional(Schema.Record({ key: Schema.String, value: Schema.String })),
-  /** IDs and surface forms of references emitted by recent Pass A invocations. */
-  recentReferences: Schema.optional(
-    Schema.Array(
-      Schema.Struct({
-        id: Schema.String,
-        name: Schema.String,
-      }),
-    ),
-  ),
-});
-export type SummaryInputType = Schema.Schema.Type<typeof SummaryInput>;
-
-export const ResolvedReferent = Schema.Struct({
-  surface: Schema.String.annotations({ description: 'The deictic / anaphoric form, e.g. "I", "there".' }),
-  referent: Schema.String.annotations({ description: 'Canonical name or description the surface resolves to.' }),
-  refId: Schema.optional(Schema.String).annotations({
-    description: 'ECHO object id when the referent maps to a known entity.',
-  }),
-});
-
-export const SummaryOutput = Schema.Struct({
-  summary: Schema.String,
-  resolvedReferents: Schema.Array(ResolvedReferent),
-});
-export type SummaryOutputType = Schema.Schema.Type<typeof SummaryOutput>;
-
-export const SummarizeConversation = Operation.make({
-  meta: {
-    key: DXN.make(`${NAMESPACE}.summarize`),
-    name: 'Summarize Conversation',
-    description: 'Maintain a cumulative summary with referent resolution over a transcript stream.',
-    icon: 'ph--text-align-left--regular',
-  },
-  input: SummaryInput,
-  output: SummaryOutput,
-});
-
-/**
- * Deterministic stub: appends each new utterance to the previous summary on its own line,
- * prefixed with the speaker's display name (resolving the speaker label via `speakerNames`).
- * If any utterance contains "I" as a standalone word, emits a `resolvedReferents` entry binding
- * it to the speaker's display name — this is the minimal demonstration of the Pass-C contract.
- */
-const summarizeHandler: Operation.WithHandler<typeof SummarizeConversation> = SummarizeConversation.pipe(
-  Operation.withHandler(
-    Effect.fnUntraced(function* ({ previousSummary, newUtterances, speakerNames, recentReferences }) {
-      const names = speakerNames ?? {};
-      const lines = newUtterances.map((u) => `${names[u.speaker] ?? u.speaker}: ${u.text}`);
-      const summary = [previousSummary.trim(), ...lines].filter(Boolean).join('\n');
-
-      const resolvedReferents = newUtterances
-        .filter((u) => /\bI\b/.test(u.text))
-        .map((u) => {
-          const displayName = names[u.speaker] ?? u.speaker;
-          const ref = (recentReferences ?? []).find((r) => r.name === displayName);
-          return { surface: 'I', referent: displayName, refId: ref?.id };
-        });
-
-      return { summary, resolvedReferents };
-    }),
-  ),
-);
-
-//
-// Exports.
-//
-
-export const EnrichmentHandlers = [enrichHandler, summarizeHandler];
-
-//
-// Helpers.
-//
-
-const ensureTerminalPunctuation = (text: string): string => {
-  const trimmed = text.trim();
-  if (trimmed.length === 0) {
-    return trimmed;
-  }
-  return /[.!?]$/.test(trimmed) ? trimmed : `${trimmed}.`;
-};
-
-function* findCapitalisedWords(
-  text: string,
-): Generator<{ text: string; kind: 'proper-noun'; start: number; end: number }> {
-  const re = /\b[A-Z][a-zA-Z]+(?:\s+[A-Z][a-zA-Z]+)*\b/g;
-  let match: RegExpExecArray | null;
-  while ((match = re.exec(text)) !== null) {
-    // Skip pure sentence-initial single words ("Yes", "I", "But", "They") to keep candidates focused on
-    // proper nouns. Heuristic for the stub: drop common stop-words.
-    const word = match[0];
-    if (STOP_WORDS.has(word)) {
-      continue;
-    }
-    yield { text: word, kind: 'proper-noun', start: match.index, end: match.index + word.length };
-  }
-}
-
-const STOP_WORDS = new Set([
-  'I',
-  'Yes',
-  'No',
-  'But',
-  'And',
-  'Or',
-  'So',
-  'They',
-  'We',
-  'He',
-  'She',
-  'It',
-  'The',
-  'A',
-  'An',
-  'Is',
-  'Are',
-  'Was',
-  'Were',
-  'Be',
-  'Been',
-]);