@dxos/pipeline-transcription 0.0.0

package/src/stages/extraction.test.ts

@@ -0,0 +1,79 @@
+//
+// Copyright 2026 DXOS.org
+//
+
+import { afterEach, beforeEach, describe, test } from 'vitest';
+
+import { Obj } from '@dxos/echo';
+import { EchoTestBuilder } from '@dxos/echo-client/testing';
+import { EffectEx } from '@dxos/effect';
+import { Organization } from '@dxos/types';
+
+import { makeDatabaseLookup } from '../lookup';
+import { extractProperNouns, makeExtractionStage } from './extraction';
+
+describe('extraction', () => {
+  test('extractProperNouns finds capitalized runs minus stop-words', ({ expect }) => {
+    expect(extractProperNouns('We should ship Munich with Anna')).toEqual(['Munich', 'Anna']);
+  });
+
+  test('reports unmatched proper nouns as candidates when no db is provided', async ({ expect }) => {
+    const stage = makeExtractionStage();
+    const block = { _tag: 'transcript' as const, started: 's', text: 'ship Munich on Friday' };
+    const write = await EffectEx.runPromise(stage.run({ window: [block] }, { model: stage.model! }));
+    const candidates = (write.blockUpdates?.[0].candidates ?? []).map((candidate) => candidate.text);
+    expect(candidates).toContain('Munich');
+    expect(candidates).toContain('Friday');
+  });
+});
+
+describe('extraction (with full-text index)', () => {
+  let builder: EchoTestBuilder;
+  beforeEach(async () => {
+    builder = await new EchoTestBuilder().open();
+  });
+  afterEach(async () => {
+    await builder.close();
+  });
+
+  test('links an unambiguous match and reports the rest as candidates', async ({ expect }) => {
+    const { db } = await builder.createDatabase({ types: [Organization.Organization] });
+    db.add(Obj.make(Organization.Organization, { name: 'Amco' }));
+    await db.flush({ indexes: true });
+
+    const stage = makeExtractionStage();
+    const block = { _tag: 'transcript' as const, started: 's', text: 'we met Amco and Globex today' };
+    const write = await EffectEx.runPromise(
+      stage.run({ window: [block] }, { lookup: makeDatabaseLookup(db), model: stage.model! }),
+    );
+
+    const update = write.blockUpdates![0];
+    expect(update.references).toHaveLength(1);
+    // Matched entity is rewritten as an inline echo link (dx-anchor) in the corrected text.
+    expect(update.corrected).toMatch(/\[Amco\]\(echo:\/[^)]+\)/);
+    const candidates = (update.candidates ?? []).map((candidate) => candidate.text);
+    expect(candidates).toContain('Globex');
+    expect(candidates).not.toContain('Amco');
+  });
+
+  test('leaves an ambiguous noun (multiple matches) as a candidate rather than linking', async ({ expect }) => {
+    const { db } = await builder.createDatabase({ types: [Organization.Organization] });
+    // Two objects match the same surface noun, so the match is ambiguous.
+    db.add(Obj.make(Organization.Organization, { name: 'Acme' }));
+    db.add(Obj.make(Organization.Organization, { name: 'Acme' }));
+    await db.flush({ indexes: true });
+
+    const stage = makeExtractionStage();
+    const block = { _tag: 'transcript' as const, started: 's', text: 'we met Acme today' };
+    const write = await EffectEx.runPromise(
+      stage.run({ window: [block] }, { lookup: makeDatabaseLookup(db), model: stage.model! }),
+    );
+
+    const update = write.blockUpdates?.[0];
+    // Ambiguous noun is surfaced as a candidate, not auto-linked.
+    expect(update?.references ?? []).toHaveLength(0);
+    expect(update?.corrected).toBeUndefined();
+    const candidates = (update?.candidates ?? []).map((candidate) => candidate.text);
+    expect(candidates).toContain('Acme');
+  });
+});

package/src/stages/extraction.ts

@@ -0,0 +1,151 @@
+//
+// Copyright 2026 DXOS.org
+//
+
+import * as Effect from 'effect/Effect';
+
+import { insertReferences } from '@dxos/assistant/extraction';
+import { type Ref } from '@dxos/echo';
+import { type ContentBlock } from '@dxos/types';
+
+import { type EntityLookup } from '../lookup';
+import { DEFAULT_STAGE_MODEL } from '../PipelineConfig';
+import { type BlockUpdate, type Stage, StageWrite } from '../Stage';
+
+export type ExtractionInput = { window: ContentBlock.Transcript[] };
+
+const STOP_WORDS = new Set([
+  'I',
+  'Yes',
+  'No',
+  'But',
+  'And',
+  'Or',
+  'So',
+  'They',
+  'We',
+  'He',
+  'She',
+  'It',
+  'The',
+  'A',
+  'An',
+  'Is',
+  'Are',
+  'Was',
+  'Were',
+  'Be',
+  'Been',
+  'This',
+  'That',
+]);
+
+/**
+ * Deterministic proper-noun heuristic: capitalized word runs, minus sentence-initial stop-words.
+ * The LLM seam (see `run`) replaces this with model-driven NER.
+ */
+export const extractProperNouns = (text: string): string[] => {
+  const matches = text.match(/\b[A-Z][a-zA-Z]+(?:\s+[A-Z][a-zA-Z]+)*\b/g) ?? [];
+  return Array.from(new Set(matches.filter((word) => !STOP_WORDS.has(word))));
+};
+
+/** Marker that a block's text already carries inline echo links, so extraction can skip it. */
+export const isLinked = (text: string): boolean => text.includes('](echo:');
+
+/** Result of resolving a block's proper nouns against the {@link EntityLookup}. */
+export type ExtractedEntities = {
+  references: Ref.Ref<any>[];
+  quotes: { quote: string; id: string }[];
+  candidates: ContentBlock.Candidate[];
+};
+
+/** Resolve a block's proper nouns: matched → references/quotes, unmatched → candidates. */
+export const resolveEntities = async (text: string, lookup?: EntityLookup): Promise<ExtractedEntities> => {
+  const nouns = extractProperNouns(text);
+  const references: Ref.Ref<any>[] = [];
+  const quotes: { quote: string; id: string }[] = [];
+  const matched = new Set<string>();
+  if (lookup) {
+    for (const noun of nouns) {
+      // TODO(LLM seam): replace heuristic nouns with model NER, and disambiguate tied candidates
+      // against the conversation window (the stage's `window` + `resolvedReferents`) instead of
+      // leaving them as candidates. The lookup backend stays injected.
+      const candidates = await lookup(noun); // ranked best-first.
+      // Auto-link only a clear winner: the top candidate must strictly out-rank the runner-up. A tie
+      // at the top is genuine ambiguity, left for the context-aware resolver / the user — so we never
+      // link the wrong object on a guess. (Text search returns weak spurious matches too, so counting
+      // candidates is not a reliable ambiguity signal; the rank gap is.)
+      const unambiguous =
+        candidates.length === 1 || (candidates.length > 1 && candidates[0].score > candidates[1].score);
+      if (candidates.length > 0 && unambiguous) {
+        references.push(candidates[0].ref);
+        quotes.push({ quote: noun, id: candidates[0].id });
+        matched.add(noun.toLowerCase());
+      }
+    }
+  }
+  const candidates: ContentBlock.Candidate[] = nouns
+    .filter((noun) => !matched.has(noun.toLowerCase()))
+    .map((noun) => {
+      const start = Math.max(0, text.indexOf(noun));
+      return { text: noun, kind: 'proper-noun' as const, start, end: start + noun.length };
+    });
+  return { references, quotes, candidates };
+};
+
+/**
+ * Rewrite text with inline `[noun](echo:/<id>)` links for entities matched by `lookup`, so markdown
+ * consumers decorate them as dx-anchors. Reused by the live recording driver's post-process seam.
+ */
+export const linkEntities = async (text: string, lookup: EntityLookup): Promise<string> => {
+  if (isLinked(text)) {
+    return text;
+  }
+  const { quotes } = await resolveEntities(text, lookup);
+  return quotes.length > 0 ? insertReferences(text, { references: quotes }) : text;
+};
+
+/**
+ * Stage ②: surface proper nouns across the window, link those that match objects (via the injected
+ * {@link EntityLookup}), and report the rest as candidates the user can act on.
+ *
+ * Processes every not-yet-linked block in the window (not just the newest). Under `latest-wins`
+ * with a fast stream, earlier per-block invocations are interrupted, so the surviving invocation
+ * must cover the whole window — otherwise blocks behind the head never get extracted.
+ */
+export const makeExtractionStage = (): Stage<ExtractionInput> => ({
+  id: 'extract',
+  trigger: 'per-block',
+  window: { blocks: 8 },
+  concurrency: 'latest-wins',
+  model: DEFAULT_STAGE_MODEL,
+  run: ({ window }, ctx) =>
+    Effect.gen(function* () {
+      const updates: BlockUpdate[] = [];
+      for (let index = 0; index < window.length; index++) {
+        const block = window[index];
+        const text = block.corrected ?? block.text;
+        // Idempotent across windowed re-invocations: already-linked blocks need no further work.
+        if (isLinked(text)) {
+          continue;
+        }
+        const { references, quotes, candidates } = yield* Effect.promise(() => resolveEntities(text, ctx.lookup));
+
+        const update: BlockUpdate = { index };
+        if (references.length > 0) {
+          update.references = references;
+        }
+        if (candidates.length > 0) {
+          update.candidates = candidates;
+        }
+        // Inline links so markdown consumers render dx-anchors; structured references stay for others.
+        if (quotes.length > 0) {
+          update.corrected = insertReferences(text, { references: quotes });
+        }
+        if (update.references || update.candidates || update.corrected) {
+          updates.push(update);
+        }
+      }
+      return updates.length > 0 ? StageWrite.blocks(updates) : StageWrite.empty();
+    }),
+});

package/src/stages/index.ts

@@ -0,0 +1,44 @@
+//
+// Copyright 2026 DXOS.org
+//
+
+import { log } from '@dxos/log';
+
+import { type Stage } from '../Stage';
+import { makeCorrectionStage } from './correction';
+import { makeDiarizationStage } from './diarization';
+import { makeExtractionStage } from './extraction';
+import { makeSummarizationStage } from './summarization';
+import { makeTranslationStage } from './translation';
+
+export * from './correction';
+export * from './correction-llm';
+export * from './extraction';
+export * from './summarization';
+export * from './translation';
+export * from './diarization';
+
+/**
+ * The default registry of stage factories keyed by stage id. Consumers build a stage list for a
+ * `PipelineConfig` by selecting enabled ids in config order.
+ */
+export const STAGE_FACTORIES: Readonly<Record<string, () => Stage<any, any>>> = Object.freeze({
+  correct: makeCorrectionStage,
+  extract: makeExtractionStage,
+  summarize: makeSummarizationStage,
+  translate: makeTranslationStage,
+  diarize: makeDiarizationStage,
+});
+
+/** Build the ordered, enabled stage list for a set of stage configs. */
+export const buildStages = (configs: readonly { id: string; enabled: boolean }[]): Stage<any, any>[] => {
+  // Surface unknown ids (stale / mistyped persisted config) rather than silently dropping them, so a
+  // config migration miss is visible at setup instead of leaving the runtime quietly under-configured.
+  const unknown = configs.filter((config) => config.enabled && !STAGE_FACTORIES[config.id]).map((config) => config.id);
+  if (unknown.length > 0) {
+    log.warn('ignoring unknown pipeline stage ids', { unknown, known: Object.keys(STAGE_FACTORIES) });
+  }
+  return configs
+    .filter((config) => config.enabled && STAGE_FACTORIES[config.id])
+    .map((config) => STAGE_FACTORIES[config.id]());
+};

package/src/stages/summarization.ts

@@ -0,0 +1,37 @@
+//
+// Copyright 2026 DXOS.org
+//
+
+import * as Effect from 'effect/Effect';
+
+import { type ContentBlock } from '@dxos/types';
+
+import { DEFAULT_STAGE_MODEL } from '../PipelineConfig';
+import { type Stage, StageWrite } from '../Stage';
+
+export type SummarizationInput = { window: ContentBlock.Transcript[] };
+
+/**
+ * Stage ④: maintain a cumulative summary. Fires on the silence edge, skips if a summary is already
+ * in flight. The deterministic implementation concatenates corrected utterances; the LLM seam
+ * (see `run`) replaces it with a cumulative model summary plus referent resolution.
+ */
+export const makeSummarizationStage = (): Stage<SummarizationInput> => ({
+  id: 'summarize',
+  trigger: 'on-silence',
+  concurrency: 'skip-if-busy',
+  model: DEFAULT_STAGE_MODEL,
+  run: ({ window }) =>
+    Effect.sync(() => {
+      const utterances = window.map((block) => (block.corrected ?? block.text).trim()).filter(Boolean);
+      if (utterances.length === 0) {
+        return StageWrite.empty();
+      }
+      // TODO(LLM seam): replace with a cumulative LLM summary that resolves deictic referents
+      // ("I" → speaker, "there" → prior entity) and writes `resolvedReferents`.
+      return StageWrite.transcript({
+        summary: utterances.join(' '),
+        summaryUpdatedAt: new Date().toISOString(),
+      });
+    }),
+});

package/src/stages/translation.ts

@@ -0,0 +1,26 @@
+//
+// Copyright 2026 DXOS.org
+//
+
+import * as Effect from 'effect/Effect';
+
+import { type ContentBlock } from '@dxos/types';
+
+import { DEFAULT_STAGE_MODEL } from '../PipelineConfig';
+import { type Stage, StageWrite } from '../Stage';
+
+export type TranslationInput = { window: ContentBlock.Transcript[] };
+
+/**
+ * Stage ③ (interface only / deferred): translate the newest block into `targetLanguage`.
+ * Conforms to the `Stage` contract and is inert until a translation backend is wired.
+ */
+export const makeTranslationStage = (_targetLanguage = 'en'): Stage<TranslationInput> => ({
+  id: 'translate',
+  trigger: 'per-block',
+  window: { blocks: 1 },
+  concurrency: 'latest-wins',
+  model: DEFAULT_STAGE_MODEL,
+  // TODO: implement translation (per-block; writes block.translation via a model call).
+  run: () => Effect.succeed(StageWrite.empty()),
+});

package/src/testing/conversation-parser.ts

@@ -0,0 +1,101 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+// Parser for the human-authored conversation fixture (see `conversation.txt`).
+// Captures both the seed entities (Persons, Organizations, …) and the speaker-tagged
+// dialogue used by `stream-simulator.ts` to feed the pipeline in tests.
+
+export type ConversationEntity = {
+  /** Type name as written in the fixture, e.g. `Person`, `Organization`. */
+  kind: string;
+  /** Free-form field values; consumers map these to the relevant ECHO schema. */
+  fields: Record<string, string>;
+};
+
+export type ConversationUtterance = {
+  /** Speaker label as written in the fixture (e.g. `A`, `B`, `Rich`). */
+  speaker: string;
+  /** Raw utterance text. */
+  text: string;
+};
+
+export type ParsedConversation = {
+  entities: ConversationEntity[];
+  utterances: ConversationUtterance[];
+  /** Distinct speaker labels in first-seen order. */
+  speakers: string[];
+};
+
+/**
+ * Parse the on-disk conversation format into structured data.
+ *
+ * Tolerates leading whitespace inside blocks and ignores blank / `#` comment lines.
+ * Throws if the input doesn't contain a `Conversation:` block — `Entities:` is optional.
+ */
+export const parseConversation = (input: string): ParsedConversation => {
+  const lines = input.split('\n').map((line) => line.replace(/\s+$/, ''));
+
+  const entities: ConversationEntity[] = [];
+  const utterances: ConversationUtterance[] = [];
+  const speakerOrder = new Set<string>();
+
+  type Section = 'none' | 'entities' | 'conversation';
+  let section: Section = 'none';
+  let currentEntity: ConversationEntity | undefined;
+
+  for (const raw of lines) {
+    const line = raw.trim();
+    if (line.length === 0 || line.startsWith('#')) {
+      continue;
+    }
+
+    // Section headers (must be left-aligned to switch sections).
+    if (raw.startsWith('Entities:') || raw.startsWith('Entitites:')) {
+      // tolerate the typo present in earlier fixtures
+      section = 'entities';
+      currentEntity = undefined;
+      continue;
+    }
+    if (raw.startsWith('Conversation:')) {
+      section = 'conversation';
+      currentEntity = undefined;
+      continue;
+    }
+
+    if (section === 'entities') {
+      // `<TypeName>:` starts a new entity; nested `<field>: <value>` lines fill it.
+      const typeMatch = line.match(/^([A-Za-z][A-Za-z0-9_-]*):$/);
+      if (typeMatch) {
+        currentEntity = { kind: typeMatch[1], fields: {} };
+        entities.push(currentEntity);
+        continue;
+      }
+      const fieldMatch = line.match(/^([A-Za-z][A-Za-z0-9_-]*):\s*(.+)$/);
+      if (fieldMatch && currentEntity) {
+        currentEntity.fields[fieldMatch[1]] = fieldMatch[2];
+        continue;
+      }
+      // Silently skip unrecognised lines so authors can add free-form annotations.
+      continue;
+    }
+
+    if (section === 'conversation') {
+      const utteranceMatch = line.match(/^([^:]+):\s*(.+)$/);
+      if (utteranceMatch) {
+        const speaker = utteranceMatch[1].trim();
+        const text = utteranceMatch[2].trim();
+        if (text.length > 0) {
+          utterances.push({ speaker, text });
+          speakerOrder.add(speaker);
+        }
+      }
+    }
+  }
+
+  if (utterances.length === 0) {
+    throw new Error('parseConversation: no utterances found');
+  }
+
+  return { entities, utterances, speakers: Array.from(speakerOrder) };
+};

package/src/testing/conversation.test.ts

@@ -0,0 +1,53 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+import * as fs from 'node:fs';
+import * as path from 'node:path';
+import * as url from 'node:url';
+import { describe, test } from 'vitest';
+
+import { parseConversation } from './conversation-parser';
+import { simulateStream } from './stream-simulator';
+
+const fixture = () =>
+  fs.readFileSync(path.join(path.dirname(url.fileURLToPath(import.meta.url)), 'conversation.txt'), 'utf8');
+
+describe('conversation fixture', () => {
+  test('parses entities + speaker-tagged utterances', ({ expect }) => {
+    const conversation = parseConversation(fixture());
+    expect(conversation.speakers).toEqual(['A', 'B']);
+    expect(conversation.utterances.length).toBe(7);
+    expect(conversation.entities.map((entity) => entity.fields.fullName ?? entity.fields.name)).toContain('Chad');
+    expect(conversation.entities.map((entity) => entity.fields.name)).toContain('Blue Yard');
+  });
+
+  test('simulates a chronological, per-utterance batch schedule', ({ expect }) => {
+    const conversation = parseConversation(fixture());
+    const batches = simulateStream(conversation, { wordsPerSecond: 2.5, gapSeconds: 0.5 });
+
+    expect(batches.length).toBe(conversation.utterances.length);
+    expect(batches[0].offsetMs).toBe(0);
+    expect(batches.every((batch) => batch.blocks.length > 0)).toBe(true);
+    // Offsets are strictly increasing (chronological delivery).
+    for (let index = 1; index < batches.length; index++) {
+      expect(batches[index].offsetMs).toBeGreaterThan(batches[index - 1].offsetMs);
+    }
+  });
+
+  test('merges consecutive same-speaker utterances when batchSeconds is set', ({ expect }) => {
+    const conversation = {
+      entities: [],
+      speakers: ['A', 'B'],
+      utterances: [
+        { speaker: 'A', text: 'one two three' },
+        { speaker: 'A', text: 'four five six' },
+        { speaker: 'B', text: 'seven eight' },
+      ],
+    };
+    // A large window coalesces the two consecutive A turns into one batch (A, B).
+    const merged = simulateStream(conversation, { batchSeconds: 60 });
+    expect(merged.length).toBe(2);
+    expect(merged[0].blocks.length).toBe(2);
+  });
+});

package/src/testing/conversation.txt

@@ -0,0 +1,23 @@
+# Conversation fixture for transcription pipeline tests.
+# Format:
+#   `Entities:` block — declares ECHO objects to seed into the test space.
+#     Each entry is `<Type>:` followed by indented `<field>: <value>` lines.
+#   `Conversation:` block — one utterance per line, `<speaker>: <text>`.
+#   Speakers map to distinct virtual Whisper streams in the simulator.
+#   Lines starting with `#` are comments and are ignored by the parser.
+
+Entities:
+  Person:
+    fullName: Chad
+    employer: Blue Yard
+  Organization:
+    name: Blue Yard
+
+Conversation:
+  A: I think I'm going to go.
+  B: Are you meeting with them?
+  A: Yes I think so.
+  B: But I thought they were in Munich?
+  A: They are, but the offsite is in Los Angeles.
+  B: Is Chad going to be there?
+  A: Yes, the whole team. It's their annual offsite.

package/src/testing/fixtures.ts

@@ -0,0 +1,18 @@
+//
+// Copyright 2026 DXOS.org
+//
+
+import { type ScriptedBlock } from './scripted-source';
+
+/** A short multi-utterance meeting transcript (raw, lower-cased — as Whisper tends to emit). */
+export const SAMPLE_MEETING: readonly ScriptedBlock[] = Object.freeze([
+  { text: 'so i think we should ship munich on friday' },
+  { text: 'lets confirm the plan with anna first' },
+  { text: 'agreed we can review it tomorrow' },
+]);
+
+/** A single-speaker dictation transcript (for the Notes preset). */
+export const SAMPLE_NOTES: readonly ScriptedBlock[] = Object.freeze([
+  { text: 'remember to email the quarterly report' },
+  { text: 'and schedule a follow up with the design team' },
+]);

package/src/testing/index.ts

@@ -0,0 +1,8 @@
+//
+// Copyright 2026 DXOS.org
+//
+
+export * from './conversation-parser';
+export * from './fixtures';
+export * from './scripted-source';
+export * from './stream-simulator';

package/src/testing/scripted-source.ts

@@ -0,0 +1,35 @@
+//
+// Copyright 2026 DXOS.org
+//
+
+import * as Stream from 'effect/Stream';
+
+import { TranscriptEvent } from '../TranscriptEvent';
+
+export type ScriptedBlock = { text: string; started?: string };
+
+export type ScriptedSourceOptions = {
+  /** Append a trailing silence event (drives on-silence stages). Default: true. */
+  readonly silence?: boolean;
+};
+
+/**
+ * Build a deterministic {@link TranscriptEvent} stream from a fixture of blocks. Emits a `block`
+ * event per entry, then (by default) a trailing `silence`. No audio or network — for testbench and CI.
+ */
+export const scriptedSource = (
+  blocks: readonly ScriptedBlock[],
+  options: ScriptedSourceOptions = {},
+): Stream.Stream<TranscriptEvent> => {
+  const events: TranscriptEvent[] = blocks.map((block, index) =>
+    TranscriptEvent.block({
+      _tag: 'transcript',
+      started: block.started ?? new Date(index * 1000).toISOString(),
+      text: block.text,
+    }),
+  );
+  if (options.silence !== false) {
+    events.push(TranscriptEvent.silence(5000));
+  }
+  return Stream.fromIterable(events);
+};