@dxos/plugin-transcription 0.8.4-main.fd6878d → 0.9.0

package/src/enrichment/operations.ts

@@ -0,0 +1,271 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+// Operation definitions for the transcript enrichment pipeline (see ENRICHMENT.md).
+//
+// **Stub status.** The handlers below return deterministic dummy values rather than calling
+// an LLM; they exist so the end-to-end test harness (`conversation.test.ts`) can exercise
+// the schemas, the operation wiring, and the streaming/orchestration code paths without
+// requiring an Anthropic key on first run. The real handlers will replace these stubs in
+// a follow-up PR (calling `LanguageModel.generateObject(...)`) while keeping the same
+// inputs / outputs.
+
+import * as Effect from 'effect/Effect';
+import * as Schema from 'effect/Schema';
+
+import { Operation } from '@dxos/compute';
+import { DXN } from '@dxos/keys';
+
+import { meta } from '#meta';
+
+const NAMESPACE = `${meta.id}.enrichment`;
+
+//
+// Shared sub-schemas.
+//
+
+/** A noun / proper noun surfaced for follow-up actions (create object, research, link…). */
+export const CandidateMention = Schema.Struct({
+  text: Schema.String.annotations({ description: 'Surface form as it appears in the transcript.' }),
+  kind: Schema.Literal('noun', 'proper-noun').annotations({
+    description: 'Whether this is a common noun (e.g. "team") or a proper noun (e.g. "Munich").',
+  }),
+  start: Schema.Number.annotations({ description: 'Character offset within the corrected text.' }),
+  end: Schema.Number.annotations({ description: 'Character offset within the corrected text.' }),
+  suggested: Schema.optional(
+    Schema.Struct({
+      typename: Schema.String.annotations({
+        description: 'Optional hint for the ECHO type the user might create (e.g. "org.dxos.type.organization").',
+      }),
+    }),
+  ),
+});
+
+/** Per-block enrichment output produced by Pass A. */
+export const EnrichmentBlock = Schema.Struct({
+  /** Index into the input window; lets the caller match enrichment to the originating block. */
+  index: Schema.Number,
+  /** Corrected text — punctuation, capitalization, words split across batch boundaries reassembled. */
+  corrected: Schema.String,
+  /**
+   * IDs of ECHO objects the LLM is confident the block mentions. Strings rather than `Ref<T>`
+   * to keep the schema serialisable; the consumer resolves them via `Database.resolve(...)`.
+   */
+  referenceIds: Schema.Array(Schema.String),
+  /** Candidate nouns / proper nouns not already covered by `referenceIds`. */
+  candidates: Schema.Array(CandidateMention),
+});
+
+//
+// Pass A — enrichment (per Whisper batch / sliding window).
+//
+
+export const EnrichmentInput = Schema.Struct({
+  /** Sliding-window blocks. Each carries the speaker that produced it. */
+  window: Schema.Array(
+    Schema.Struct({
+      speaker: Schema.String,
+      text: Schema.String,
+      started: Schema.String,
+    }),
+  ),
+  /** ECHO objects available in the space that the LLM may pick from when linking references. */
+  knownEntities: Schema.optional(
+    Schema.Array(
+      Schema.Struct({
+        id: Schema.String,
+        typename: Schema.String,
+        name: Schema.String,
+      }),
+    ),
+  ),
+});
+export type EnrichmentInputType = Schema.Schema.Type<typeof EnrichmentInput>;
+
+export const EnrichmentOutput = Schema.Struct({
+  blocks: Schema.Array(EnrichmentBlock),
+});
+export type EnrichmentOutputType = Schema.Schema.Type<typeof EnrichmentOutput>;
+
+export const EnrichTranscript = Operation.make({
+  meta: {
+    key: DXN.make(`${NAMESPACE}.enrich`),
+    name: 'Enrich Transcript',
+    description: 'Correct Whisper output and surface entity references and candidates over a sliding window.',
+    icon: 'ph--microphone--regular',
+  },
+  input: EnrichmentInput,
+  output: EnrichmentOutput,
+});
+
+/**
+ * Deterministic stub: echoes the input text as `corrected`, links every block whose text contains
+ * a `knownEntities[i].name` substring to that entity's id, and surfaces every capitalised word
+ * not already linked as a `proper-noun` candidate. Sentence-end punctuation is added if missing.
+ */
+const enrichHandler: Operation.WithHandler<typeof EnrichTranscript> = EnrichTranscript.pipe(
+  Operation.withHandler(
+    Effect.fnUntraced(function* ({ window, knownEntities }) {
+      const known = knownEntities ?? [];
+      const blocks = window.map((entry, index) => {
+        const corrected = ensureTerminalPunctuation(entry.text);
+        const referenceIds: string[] = [];
+        const covered: Array<{ start: number; end: number }> = [];
+        for (const entity of known) {
+          const haystack = corrected.toLowerCase();
+          const needle = entity.name.toLowerCase();
+          const idx = haystack.indexOf(needle);
+          if (idx >= 0) {
+            referenceIds.push(entity.id);
+            covered.push({ start: idx, end: idx + entity.name.length });
+          }
+        }
+
+        const candidates = [...findCapitalisedWords(corrected)].filter(
+          ({ start, end }) => !covered.some((c) => c.start <= start && end <= c.end),
+        );
+
+        return { index, corrected, referenceIds, candidates };
+      });
+      return { blocks };
+    }),
+  ),
+);
+
+//
+// Pass B — summary (end-of-utterance + periodic).
+//
+
+export const SummaryInput = Schema.Struct({
+  /** The cumulative summary as of the last invocation (empty string on first call). */
+  previousSummary: Schema.String,
+  /** New corrected utterances appended since the previous summary. */
+  newUtterances: Schema.Array(
+    Schema.Struct({
+      speaker: Schema.String,
+      text: Schema.String,
+    }),
+  ),
+  /** Map of speaker label → human-readable display name. Drives "I" resolution. */
+  speakerNames: Schema.optional(Schema.Record({ key: Schema.String, value: Schema.String })),
+  /** IDs and surface forms of references emitted by recent Pass A invocations. */
+  recentReferences: Schema.optional(
+    Schema.Array(
+      Schema.Struct({
+        id: Schema.String,
+        name: Schema.String,
+      }),
+    ),
+  ),
+});
+export type SummaryInputType = Schema.Schema.Type<typeof SummaryInput>;
+
+export const ResolvedReferent = Schema.Struct({
+  surface: Schema.String.annotations({ description: 'The deictic / anaphoric form, e.g. "I", "there".' }),
+  referent: Schema.String.annotations({ description: 'Canonical name or description the surface resolves to.' }),
+  refId: Schema.optional(Schema.String).annotations({
+    description: 'ECHO object id when the referent maps to a known entity.',
+  }),
+});
+
+export const SummaryOutput = Schema.Struct({
+  summary: Schema.String,
+  resolvedReferents: Schema.Array(ResolvedReferent),
+});
+export type SummaryOutputType = Schema.Schema.Type<typeof SummaryOutput>;
+
+export const SummarizeConversation = Operation.make({
+  meta: {
+    key: DXN.make(`${NAMESPACE}.summarize`),
+    name: 'Summarize Conversation',
+    description: 'Maintain a cumulative summary with referent resolution over a transcript stream.',
+    icon: 'ph--text-align-left--regular',
+  },
+  input: SummaryInput,
+  output: SummaryOutput,
+});
+
+/**
+ * Deterministic stub: appends each new utterance to the previous summary on its own line,
+ * prefixed with the speaker's display name (resolving the speaker label via `speakerNames`).
+ * If any utterance contains "I" as a standalone word, emits a `resolvedReferents` entry binding
+ * it to the speaker's display name — this is the minimal demonstration of the Pass-C contract.
+ */
+const summarizeHandler: Operation.WithHandler<typeof SummarizeConversation> = SummarizeConversation.pipe(
+  Operation.withHandler(
+    Effect.fnUntraced(function* ({ previousSummary, newUtterances, speakerNames, recentReferences }) {
+      const names = speakerNames ?? {};
+      const lines = newUtterances.map((u) => `${names[u.speaker] ?? u.speaker}: ${u.text}`);
+      const summary = [previousSummary.trim(), ...lines].filter(Boolean).join('\n');
+
+      const resolvedReferents = newUtterances
+        .filter((u) => /\bI\b/.test(u.text))
+        .map((u) => {
+          const displayName = names[u.speaker] ?? u.speaker;
+          const ref = (recentReferences ?? []).find((r) => r.name === displayName);
+          return { surface: 'I', referent: displayName, refId: ref?.id };
+        });
+
+      return { summary, resolvedReferents };
+    }),
+  ),
+);
+
+//
+// Exports.
+//
+
+export const EnrichmentHandlers = [enrichHandler, summarizeHandler];
+
+//
+// Helpers.
+//
+
+const ensureTerminalPunctuation = (text: string): string => {
+  const trimmed = text.trim();
+  if (trimmed.length === 0) {
+    return trimmed;
+  }
+  return /[.!?]$/.test(trimmed) ? trimmed : `${trimmed}.`;
+};
+
+function* findCapitalisedWords(
+  text: string,
+): Generator<{ text: string; kind: 'proper-noun'; start: number; end: number }> {
+  const re = /\b[A-Z][a-zA-Z]+(?:\s+[A-Z][a-zA-Z]+)*\b/g;
+  let match: RegExpExecArray | null;
+  while ((match = re.exec(text)) !== null) {
+    // Skip pure sentence-initial single words ("Yes", "I", "But", "They") to keep candidates focused on
+    // proper nouns. Heuristic for the stub: drop common stop-words.
+    const word = match[0];
+    if (STOP_WORDS.has(word)) {
+      continue;
+    }
+    yield { text: word, kind: 'proper-noun', start: match.index, end: match.index + word.length };
+  }
+}
+
+const STOP_WORDS = new Set([
+  'I',
+  'Yes',
+  'No',
+  'But',
+  'And',
+  'Or',
+  'So',
+  'They',
+  'We',
+  'He',
+  'She',
+  'It',
+  'The',
+  'A',
+  'An',
+  'Is',
+  'Are',
+  'Was',
+  'Were',
+  'Be',
+  'Been',
+]);

package/src/enrichment/parser.ts

@@ -0,0 +1,101 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+// Parser for the human-authored conversation fixture (see `conversation.txt`).
+// Captures both the seed entities (Persons, Organizations, …) and the speaker-tagged
+// dialogue used by `stream-simulator.ts` to feed the enrichment pipeline.
+
+export type ConversationEntity = {
+  /** Type name as written in the fixture, e.g. `Person`, `Organization`. */
+  kind: string;
+  /** Free-form field values; consumers map these to the relevant ECHO schema. */
+  fields: Record<string, string>;
+};
+
+export type ConversationUtterance = {
+  /** Speaker label as written in the fixture (e.g. `A`, `B`, `Rich`). */
+  speaker: string;
+  /** Raw utterance text. */
+  text: string;
+};
+
+export type ParsedConversation = {
+  entities: ConversationEntity[];
+  utterances: ConversationUtterance[];
+  /** Distinct speaker labels in first-seen order. */
+  speakers: string[];
+};
+
+/**
+ * Parse the on-disk conversation format into structured data.
+ *
+ * Tolerates leading whitespace inside blocks and ignores blank / `#` comment lines.
+ * Throws if the input doesn't contain a `Conversation:` block — `Entities:` is optional.
+ */
+export const parseConversation = (input: string): ParsedConversation => {
+  const lines = input.split('\n').map((line) => line.replace(/\s+$/, ''));
+
+  const entities: ConversationEntity[] = [];
+  const utterances: ConversationUtterance[] = [];
+  const speakerOrder = new Set<string>();
+
+  type Section = 'none' | 'entities' | 'conversation';
+  let section: Section = 'none';
+  let currentEntity: ConversationEntity | undefined;
+
+  for (const raw of lines) {
+    const line = raw.trim();
+    if (line.length === 0 || line.startsWith('#')) {
+      continue;
+    }
+
+    // Section headers (must be left-aligned to switch sections).
+    if (raw.startsWith('Entities:') || raw.startsWith('Entitites:')) {
+      // tolerate the typo present in earlier fixtures
+      section = 'entities';
+      currentEntity = undefined;
+      continue;
+    }
+    if (raw.startsWith('Conversation:')) {
+      section = 'conversation';
+      currentEntity = undefined;
+      continue;
+    }
+
+    if (section === 'entities') {
+      // `<TypeName>:` starts a new entity; nested `<field>: <value>` lines fill it.
+      const typeMatch = line.match(/^([A-Za-z][A-Za-z0-9_-]*):$/);
+      if (typeMatch) {
+        currentEntity = { kind: typeMatch[1], fields: {} };
+        entities.push(currentEntity);
+        continue;
+      }
+      const fieldMatch = line.match(/^([A-Za-z][A-Za-z0-9_-]*):\s*(.+)$/);
+      if (fieldMatch && currentEntity) {
+        currentEntity.fields[fieldMatch[1]] = fieldMatch[2];
+        continue;
+      }
+      // Silently skip unrecognised lines so authors can add free-form annotations.
+      continue;
+    }
+
+    if (section === 'conversation') {
+      const utteranceMatch = line.match(/^([^:]+):\s*(.+)$/);
+      if (utteranceMatch) {
+        const speaker = utteranceMatch[1].trim();
+        const text = utteranceMatch[2].trim();
+        if (text.length > 0) {
+          utterances.push({ speaker, text });
+          speakerOrder.add(speaker);
+        }
+      }
+    }
+  }
+
+  if (utterances.length === 0) {
+    throw new Error('parseConversation: no utterances found');
+  }
+
+  return { entities, utterances, speakers: Array.from(speakerOrder) };
+};

package/src/enrichment/stream-simulator.ts

@@ -0,0 +1,125 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+// Turns a parsed conversation into a chronologically-ordered sequence of Whisper-shaped
+// transcript batches, one per (speaker, utterance) pair. Each batch carries a virtual
+// timestamp (ms since the conversation start) and the speaker label so consumers can fan
+// out per-stream — analogous to two microphones being transcribed in parallel.
+//
+// The simulator does NOT do any wall-clock waiting. Tests drive virtual time via
+// `vi.useFakeTimers()` or step through the batches directly; production code wouldn't use
+// this module at all.
+
+import { type ContentBlock } from '@dxos/types';
+
+import type { ConversationUtterance, ParsedConversation } from './parser';
+
+export type StreamSimulatorOptions = {
+  /**
+   * Average speaking rate, words per second. Used to assign a virtual duration to each
+   * utterance so consecutive utterances land at realistic offsets.
+   * @default 2.5 (≈150 wpm — typical conversational pace)
+   */
+  wordsPerSecond?: number;
+
+  /**
+   * Gap between consecutive utterances in seconds. Models a brief pause for turn-taking
+   * even when speakers don't actually overlap.
+   * @default 0.5
+   */
+  gapSeconds?: number;
+
+  /**
+   * Whisper batches typically arrive in chunks of N seconds rather than per-utterance.
+   * When set, utterances shorter than `batchSeconds` are merged into the same batch as
+   * the following utterance from the same speaker. `0` disables merging — every utterance
+   * is its own batch.
+   * @default 0
+   */
+  batchSeconds?: number;
+};
+
+export type SimulatedBatch = {
+  /** Speaker label from the fixture (`A`, `B`, …). Maps to a virtual `identityDid`. */
+  speaker: string;
+  /** ms offset from the conversation start when this batch is delivered. */
+  offsetMs: number;
+  /** Whisper-shaped transcript blocks (raw text, no enrichment). */
+  blocks: ContentBlock.Transcript[];
+};
+
+/**
+ * Build the chronological batch schedule for a parsed conversation. The result is a
+ * pure data structure — consumers iterate and drive virtual time themselves.
+ */
+export const simulateStream = (
+  conversation: ParsedConversation,
+  options: StreamSimulatorOptions = {},
+): SimulatedBatch[] => {
+  const wordsPerSecond = options.wordsPerSecond ?? 2.5;
+  const gapSeconds = options.gapSeconds ?? 0.5;
+  const batchSeconds = options.batchSeconds ?? 0;
+
+  // Guard against divide-by-zero / NaN / negative inputs — `wordsPerSecond` divides into a
+  // duration below, and the other knobs feed into offsets that must be finite.
+  if (!Number.isFinite(wordsPerSecond) || wordsPerSecond <= 0) {
+    throw new Error(`simulateStream: wordsPerSecond must be a finite number > 0 (got ${wordsPerSecond}).`);
+  }
+  if (!Number.isFinite(gapSeconds) || gapSeconds < 0) {
+    throw new Error(`simulateStream: gapSeconds must be a finite number >= 0 (got ${gapSeconds}).`);
+  }
+  if (!Number.isFinite(batchSeconds) || batchSeconds < 0) {
+    throw new Error(`simulateStream: batchSeconds must be a finite number >= 0 (got ${batchSeconds}).`);
+  }
+
+  const conversationStart = Date.UTC(2024, 0, 1, 12, 0, 0); // deterministic, ignored by callers
+  let cursorMs = 0;
+  const batches: SimulatedBatch[] = [];
+
+  // Optionally merge same-speaker consecutive utterances until they reach `batchSeconds`.
+  const grouped: { speaker: string; utterances: ConversationUtterance[] }[] = [];
+  if (batchSeconds === 0) {
+    for (const u of conversation.utterances) {
+      grouped.push({ speaker: u.speaker, utterances: [u] });
+    }
+  } else {
+    let pending: { speaker: string; utterances: ConversationUtterance[]; words: number } | undefined;
+    for (const u of conversation.utterances) {
+      const words = utteranceWordCount(u.text);
+      if (pending && pending.speaker === u.speaker && pending.words / wordsPerSecond < batchSeconds) {
+        pending.utterances.push(u);
+        pending.words += words;
+      } else {
+        if (pending) {
+          grouped.push({ speaker: pending.speaker, utterances: pending.utterances });
+        }
+        pending = { speaker: u.speaker, utterances: [u], words };
+      }
+    }
+    if (pending) {
+      grouped.push({ speaker: pending.speaker, utterances: pending.utterances });
+    }
+  }
+
+  for (const group of grouped) {
+    const startMs = cursorMs;
+    const blocks: ContentBlock.Transcript[] = [];
+    let segmentCursorMs = cursorMs;
+    for (const u of group.utterances) {
+      const durationMs = Math.max(500, (utteranceWordCount(u.text) / wordsPerSecond) * 1_000);
+      blocks.push({
+        _tag: 'transcript',
+        started: new Date(conversationStart + segmentCursorMs).toISOString(),
+        text: u.text,
+      });
+      segmentCursorMs += durationMs;
+    }
+    batches.push({ speaker: group.speaker, offsetMs: startMs, blocks });
+    cursorMs = segmentCursorMs + gapSeconds * 1_000;
+  }
+
+  return batches;
+};
+
+const utteranceWordCount = (text: string): number => text.trim().split(/\s+/).filter(Boolean).length;

package/src/hooks/index.ts

@@ -4,6 +4,7 @@
 
 export * from './useAudioFile';
 export * from './useAudioTrack';
-export * from './useQueueModelAdapter';
+export * from './useFeedModelAdapter';
 export * from './useTranscriber';
+export * from './useTranscriptionRecording';
 export * from './useVoiceInput';

package/src/hooks/useAudioFile.ts

@@ -20,67 +20,63 @@ export const useAudioFile = (audioUrl: string, constraints?: MediaTrackConstrain
     const ctx = new Context();
 
     scheduleTask(ctx, async () => {
-      const response = await fetch(audioUrl);
-      const blob = await response.blob();
-      const objectUrl = URL.createObjectURL(blob);
-      ctx.onDispose(() => {
-        URL.revokeObjectURL(objectUrl);
-      });
+      // TODO(wittjosiah): Fetch to external url fails in headless storybook test.
+      try {
+        const response = await fetch(audioUrl);
+        const blob = await response.blob();
+        const objectUrl = URL.createObjectURL(blob);
+        ctx.onDispose(() => {
+          URL.revokeObjectURL(objectUrl);
+        });
 
-      const audio = new Audio();
-      audio.src = objectUrl;
+        const audio = new Audio();
+        audio.src = objectUrl;
 
-      await new Promise<void>((resolve, reject) => {
-        audio.addEventListener(
-          'error',
-          (err) => {
-            log.error('error', { err });
-            reject(err);
-          },
-          { once: true },
-        );
-        audio.addEventListener(
-          'canplay',
-          async () => {
-            log.info('starting...');
-            try {
-              // Try to play the audio
-              await audio.play();
+        await new Promise<void>((resolve, reject) => {
+          audio.addEventListener(
+            'error',
+            (err) => {
+              log.error('error', { err });
+              reject(err);
+            },
+            { once: true },
+          );
+          audio.addEventListener(
+            'canplay',
+            async () => {
               resolve();
-            } catch (playError) {
-              log.error('Play failed', { playError });
-              // Still resolve as the audio is ready, even if autoplay failed.
-              resolve();
-            }
-          },
-          { once: true },
-        );
-        audio.load();
-      });
+            },
+            { once: true },
+          );
+          audio.load();
+        });
 
-      const audioCtx = new AudioContext();
+        const audioCtx = new AudioContext();
 
-      // Resume AudioContext if it's suspended.
-      if (audioCtx.state === 'suspended') {
-        await audioCtx.resume();
-      }
+        // Resume AudioContext if it's suspended.
+        if (audioCtx.state === 'suspended') {
+          await audioCtx.resume();
+        }
 
-      const destination = audioCtx.createMediaStreamDestination();
-      destination.channelCount = 1;
+        const destination = audioCtx.createMediaStreamDestination();
+        destination.channelCount = 1;
 
-      const source = audioCtx.createMediaElementSource(audio);
-      source.connect(destination);
-      const track = destination.stream.getAudioTracks()[0];
-      if (constraints) {
-        await track.applyConstraints(constraints).catch((err) => log.catch(err));
-      }
+        const source = audioCtx.createMediaElementSource(audio);
+        source.connect(destination);
+        const track = destination.stream.getAudioTracks()[0];
+        if (constraints) {
+          await track.applyConstraints(constraints).catch((err) => log.catch(err));
+        }
 
-      // Also connect to speakers so audio is audible.
-      setStream({
-        audio,
-        stream: destination.stream,
-        track: destination.stream.getAudioTracks()[0],
-      });
+        // Also connect to speakers so audio is audible.
+        setStream({
+          audio,
+          stream: destination.stream,
+          track: destination.stream.getAudioTracks()[0],
+        });
+      } catch (err) {
+        log.catch(err);
+      }
     });
 
     return () => {

package/src/hooks/useAudioTrack.ts

@@ -7,7 +7,7 @@ import { useEffect, useState } from 'react';
 /**
  *
  */
-// TODO(burdon): Reconcile with react-ui-sfx and plugin-calls.
+// TODO(burdon): Reconcile with react-ui-experimental and plugin-calls.
 export const useAudioTrack = (active?: boolean, constraints?: MediaTrackConstraints): MediaStreamTrack | undefined => {
   const [track, setTrack] = useState<MediaStreamTrack>();
   useEffect(() => {

package/src/hooks/useFeedModelAdapter.ts

@@ -0,0 +1,46 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import { useEffect, useMemo } from 'react';
+
+import type { Obj } from '@dxos/echo';
+
+import { type ChunkRenderer, SerializationModel } from '../model';
+
+/**
+ * Feeds an array of objects (typically from `useQuery(db, Query.from(feed))`) into a
+ * `SerializationModel`, appending each new tail item as the array grows.
+ *
+ * @param renderer - chunk renderer.
+ * @param objects - current snapshot of feed items.
+ * @param initialChunks - chunks to seed the model with on first render.
+ */
+export const useFeedModelAdapter = <T extends Obj.Unknown>(
+  renderer: ChunkRenderer<T>,
+  objects: readonly T[],
+  initialChunks: T[] = [],
+): SerializationModel<T> => {
+  const model = useMemo(() => new SerializationModel<T>(renderer, initialChunks), [renderer]);
+
+  // Seed the model with the current snapshot on first render and whenever the
+  // identity of `objects` changes (e.g. feed swapped).
+  useEffect(() => {
+    for (const chunk of objects) {
+      model.appendChunk(chunk);
+    }
+  }, [model, objects]);
+
+  // Append new tail items as the array grows.
+  useEffect(() => {
+    if (model.chunks.length === objects.length) {
+      return;
+    }
+    const chunk = objects.at(-1);
+    if (chunk) {
+      model.appendChunk(chunk);
+    }
+  }, [model, objects.length]);
+
+  return model;
+};