@mentra/engine 3.4.0-dev.1182 → 3.4.0-dev.1183

package/src/services/LocalSttFallbackCoordinator.ts

@@ -1,7 +1,6 @@
 import BluetoothSdk from "@mentra/bluetooth-sdk/internal"
 import {ISLAND_SETTINGS_KEYS} from "../runtime/config"
 import {useSettingsStore} from "../stores/settings"
-import {cloudClientService} from "./CloudClientService"
 import {ModelRetry} from "./ModelRetry"
 import offlineSpeechModelService from "./OfflineSpeechModelService"
 import sttModelManager from "./STTModelManager"
@@ -23,10 +22,10 @@ export interface LocalSttModelStatus {
 }
 
 /**
- * Decides whether on-device Sherpa STT should produce transcripts for local
- * miniapps, instead of (or in addition to) cloud transcription. Local STT is
- * active when the cloud is unavailable or any subscription requires
- * `forceLocal`.
+ * Runs on-device Sherpa STT for local miniapps while LocalMiniappRuntime says
+ * a transcription listener needs it (`setDemand`): a `localOnly` listener, a
+ * `preferLocal` one the picked model can serve, any listener while the cloud
+ * is unreachable, or one still finishing an on-device utterance.
  *
  * There is one recognizer and it always runs the model picked on the Speech
  * page. When on-device transcription is needed and that model is missing, it
@@ -39,22 +38,14 @@ export interface LocalSttModelStatus {
  * `local_transcription` Bridge event; the host wires that event to
  * `localMiniappRuntime.forwardEvent`, with this coordinator providing the
  * subscription-active gate so we never publish transcripts to miniapps that
- * aren't asking for them. LocalMiniappRuntime filters mixed local/cloud
- * delivery per miniapp.
+ * aren't asking for them. LocalMiniappRuntime routes local and cloud
+ * transcripts to each listener.
  */
 class LocalSttFallbackCoordinator {
   private static instance: LocalSttFallbackCoordinator
 
-  private hasTranscriptionSubscription = false
-  private hasForceLocalSubscription = false
-  /**
-   * Default to "cloud is up" so we never accidentally activate local STT before
-   * the cloud client has had a chance to start. We attach lazily on the first
-   * subscription/reconcile pass; once attached, this field reflects the real
-   * cloud-client runtime status.
-   */
-  private cloudConnected = true
-  private cloudAdapterAttached = false
+  /** A miniapp transcription listener needs on-device transcripts. */
+  private demand = false
   private selectionAttached = false
   private hydration?: Promise<unknown>
   private hydrated = false
@@ -86,24 +77,6 @@ class LocalSttFallbackCoordinator {
     return LocalSttFallbackCoordinator.instance
   }
 
-  /**
-   * Lazy-attach the engine cloud connection listener. The coordinator is
-   * constructed at module load, before engine.start() brings up the cloud
-   * client, so we defer until the first reconcile.
-   */
-  private attachCloudAdapterIfReady(): void {
-    if (this.cloudAdapterAttached) return
-    this.cloudConnected = cloudClientService.isConnected()
-    cloudClientService.onStatusChanged((status) => {
-      const connected = status.status === "connected"
-      if (this.cloudConnected === connected) return
-      this.log(`cloud connection -> ${connected ? "up" : "down"}`)
-      this.cloudConnected = connected
-      this.reconcile()
-    })
-    this.cloudAdapterAttached = true
-  }
-
   /** A model change on the Speech page reloads the recognizer while it is in use. */
   private attachSelectionIfReady(): void {
     if (this.selectionAttached) return
@@ -134,32 +107,17 @@ class LocalSttFallbackCoordinator {
     }
   }
 
-  onSubscriptionChange(hasTranscription: boolean, hasForceLocal = false): void {
-    if (
-      this.hasTranscriptionSubscription !== hasTranscription ||
-      this.hasForceLocalSubscription !== (hasTranscription && hasForceLocal)
-    ) {
-      this.log(`onSubscriptionChange(hasTx=${hasTranscription}, forceLocal=${hasForceLocal})`)
-    }
-    this.hasTranscriptionSubscription = hasTranscription
-    this.hasForceLocalSubscription = hasTranscription && hasForceLocal
+  /** Whether a miniapp transcription listener needs on-device transcripts. */
+  setDemand(demand: boolean): void {
+    if (this.demand !== demand) this.log(`on-device demand -> ${demand}`)
+    this.demand = demand
     this.reconcile()
   }
 
-  /**
-   * Informational hook, retained for API stability. No longer called: the
-   * authoritative switching signal is the cloud-client adapter (local miniapps
-   * are powered only by V2). v1 cloud transcripts deliberately do NOT drive this
-   * coordinator anymore. Kept for future hysteresis if connection-state alone
-   * proves too coarse.
-   */
-  onCloudTranscript(): void {}
-
   private reconcile(): void {
-    this.attachCloudAdapterIfReady()
     this.attachSelectionIfReady()
-    if (!this.shouldUseLocalStt()) {
-      this.stop(this.hasTranscriptionSubscription ? "cloud reconnected" : "subscription gone")
+    if (!this.demand) {
+      this.stop("no listener needs it")
       return
     }
     if (!this.hydrated) {
@@ -191,8 +149,7 @@ class LocalSttFallbackCoordinator {
 
   private async start(code: string): Promise<void> {
     const generation = ++this.generation
-    const current = () =>
-      generation === this.generation && this.shouldUseLocalStt() && sttModelManager.getCurrentLanguage() === code
+    const current = () => generation === this.generation && this.demand && sttModelManager.getCurrentLanguage() === code
 
     if (this.localActive) {
       // Switching models: hold transcripts until the new model is loaded.
@@ -297,10 +254,6 @@ class LocalSttFallbackCoordinator {
     }
   }
 
-  private shouldUseLocalStt(): boolean {
-    return this.hasTranscriptionSubscription && (this.hasForceLocalSubscription || !this.cloudConnected)
-  }
-
   private log(msg: string): void {
     console.log(`[LocalSttFallback] ${msg}`)
   }

package/src/services/TranscriptionRouting.ts

@@ -1,37 +1,207 @@
-export interface TranscriptionRouteSummary {
-  hasCloudSubscriber: boolean
-  hasForceLocalSubscriber: boolean
+/**
+ * Where each miniapp transcription listener gets its text: the cloud or the
+ * phone's on-device recognizer.
+ *
+ * A listener asks for one of three behaviours (the SDK's `processing`):
+ *   - preferCloud (default): the cloud; on-device only while the cloud is
+ *     unreachable.
+ *   - preferLocal: on-device while the running model transcribes the
+ *     listener's language, otherwise the cloud.
+ *   - localOnly: on-device only; never the cloud.
+ *
+ * Each listener receives one source at a time (`handoffStep`). A planned
+ * switch, while its current source still works, waits for a pause in which
+ * both sources have settled (`UtteranceTracker`), then for a drain of the
+ * on-device recognizer (it reports everything it heard so far and starts
+ * over). The drain is the boundary: on-device text from before it stays with
+ * the old side, on-device text after it is new speech, and the listener's
+ * cloud text starts at the cloud audio position where the drain began. So a
+ * listener never misses or repeats speech. A model that cannot be drained
+ * leaves the quiet window as the boundary. A forced switch, when the current
+ * source stops, happens at once; a move to the cloud then starts at the
+ * cloud audio position of the last on-device final, so an utterance cut short
+ * on-device is completed in the cloud and finished ones are not repeated.
+ * The listeners' sources and targets decide what runs: a stream is requested
+ * from the cloud only while a listener on it needs the cloud, and the
+ * on-device recognizer runs only while a listener needs it.
+ */
+
+export type SpeechProcessing = "preferCloud" | "preferLocal" | "localOnly"
+export type TranscriptionSource = "cloud" | "local"
+
+const PROCESSING: ReadonlySet<unknown> = new Set<SpeechProcessing>(["preferCloud", "preferLocal", "localOnly"])
+const AUTO_STREAM = "transcription:auto"
+const STREAM_PREFIX = "transcription:"
+
+/**
+ * A source has settled once it has produced nothing for this long since its
+ * last final (or since it started): a pause in speech, longer than the
+ * recognizers' own end-of-speech delays.
+ */
+export const SOURCE_QUIET_MS = 3000
+/** An utterance whose final never came has ended after this long without transcripts. */
+export const UTTERANCE_STALE_MS = 6000
+
+/**
+ * The processing a listener, subscription or speak request asks for.
+ * `processing` wins; without it, the `forceLocal: true` of SDKs before it
+ * means localOnly. Anything else is preferCloud.
+ */
+export function readSpeechProcessing(request: {processing?: unknown; forceLocal?: unknown}): SpeechProcessing {
+  if (PROCESSING.has(request.processing)) return request.processing as SpeechProcessing
+  return request.forceLocal === true ? "localOnly" : "preferCloud"
+}
+
+export interface RoutedListener {
+  /** `transcription:<language>` or `transcription:auto`. */
+  stream: string
+  processing: SpeechProcessing
+}
+
+/** What the phone can transcribe with right now, as one miniapp sees it. */
+export interface TranscriptionRoutingState {
+  cloudConnected: boolean
+  /** On-device speech is allowed by this deployment. */
+  onDevice: boolean
+  /** BCP-47 language of the running on-device model, or null while none runs. */
+  localLanguage: string | null
+  /** BCP-47 language of the model picked on the Speech page. */
+  pickedLanguage: string
+  /** The miniapp's language hints (bare codes, from transcription.configure). */
+  hints: readonly string[]
+}
+
+/**
+ * Whether an on-device model in `language` transcribes what the listener asks
+ * for. A specific language must be the model's. Auto-detect follows the
+ * model's language, the one the user picked; but a preferLocal listener whose
+ * miniapp hinted at another language keeps the cloud, which tells them apart.
+ */
+export function onDeviceServes(listener: RoutedListener, language: string, hints: readonly string[]): boolean {
+  const requested = listener.stream.slice(STREAM_PREFIX.length).toLowerCase()
+  if (requested === language.toLowerCase()) return true
+  if (listener.stream !== AUTO_STREAM) return false
+  if (listener.processing !== "preferLocal") return true
+  const base = language.split("-")[0].toLowerCase()
+  return hints.every((hint) => hint.toLowerCase() === base)
+}
+
+/** The source the listener should receive now. */
+export function targetSource(listener: RoutedListener, state: TranscriptionRoutingState): TranscriptionSource {
+  if (listener.processing === "localOnly" || !state.cloudConnected) return "local"
+  if (listener.processing === "preferCloud") return "cloud"
+  return state.onDevice && state.localLanguage !== null && onDeviceServes(listener, state.localLanguage, state.hints)
+    ? "local"
+    : "cloud"
+}
+
+/**
+ * What a listener on `current` does on its way to `target`:
+ * - "settle": it has no source yet or is already there; take `target`.
+ * - "forced": `current` can no longer deliver; move now.
+ * - "wait": a source is still busy on its streams; stay.
+ * - "drain": both settled, but the on-device recognizer has spoken since its
+ *   last drain; stay until a drain completes.
+ * - "planned": both settled and drained; move now.
+ */
+export type HandoffStep = "settle" | "forced" | "wait" | "drain" | "planned"
+
+export function handoffStep(
+  current: TranscriptionSource | null,
+  target: TranscriptionSource,
+  canDeliver: (source: TranscriptionSource) => boolean,
+  busy: (source: TranscriptionSource) => boolean,
+  drained: boolean,
+): HandoffStep {
+  if (current === null || current === target) return "settle"
+  if (!canDeliver(current)) return "forced"
+  if (busy(current) || busy(target)) return "wait"
+  return drained ? "planned" : "drain"
+}
+
+/** Whether the listener needs its stream from the cloud. */
+export function needsCloud(
+  listener: RoutedListener,
+  source: TranscriptionSource | null,
+  target: TranscriptionSource,
+): boolean {
+  if (listener.processing === "localOnly") return false
+  return listener.processing === "preferCloud" || source === "cloud" || target === "cloud"
 }
 
-export interface TranscriptionSubscriberRoutes {
-  cloud: boolean
-  forceLocal: boolean
+/**
+ * Whether the listener needs the on-device recognizer running. preferLocal
+ * asks for it whenever the picked model transcribes its language, which is
+ * what starts (and, when missing, downloads) that model.
+ */
+export function needsLocal(
+  listener: RoutedListener,
+  source: TranscriptionSource | null,
+  state: TranscriptionRoutingState,
+): boolean {
+  if (source === "local" || listener.processing === "localOnly" || !state.cloudConnected) return true
+  return (
+    listener.processing === "preferLocal" &&
+    state.onDevice &&
+    onDeviceServes(listener, state.pickedLanguage, state.hints)
+  )
 }
 
-export type TranscriptionDeliveryRoute = "default" | "forceLocal" | "all"
+/** When each source last produced a transcript on each stream, to tell when it has settled. */
+export class UtteranceTracker {
+  /** `${source} ${stream}` -> its last transcript; stream "*" marks a source start, for every stream. */
+  private readonly last = new Map<string, {at: number; final: boolean}>()
+
+  constructor(
+    private readonly quietMs = SOURCE_QUIET_MS,
+    private readonly staleMs = UTTERANCE_STALE_MS,
+  ) {}
+
+  /** Records a transcript: an interim opens (or continues) an utterance, a final ends it. */
+  note(source: TranscriptionSource, stream: string, isFinal: boolean, now: number): void {
+    this.last.set(`${source} ${stream}`, {at: now, final: isFinal})
+  }
+
+  /**
+   * A source started (the cloud connected, or the on-device model loaded): it
+   * may already be working on speech it has not reported, so it settles only
+   * after a quiet window.
+   */
+  started(source: TranscriptionSource, now: number): void {
+    this.clear(source)
+    this.note(source, "*", true, now)
+  }
+
+  /** Forgets a source that stopped: it never reports anything more. */
+  clear(source: TranscriptionSource): void {
+    for (const key of [...this.last.keys()]) {
+      if (key.startsWith(`${source} `)) this.last.delete(key)
+    }
+  }
+
+  /** Whether `source` may still report speech that a listener on `stream` hears. */
+  busy(source: TranscriptionSource, stream: string, now: number): boolean {
+    for (const [key, last] of this.last) {
+      if (now >= this.settlesAt(last)) continue
+      const [keySource, keyStream] = key.split(" ")
+      if (keySource !== source) continue
+      if (keyStream === "*" || stream === AUTO_STREAM || keyStream === stream) return true
+    }
+    return false
+  }
+
+  /** When the next busy stream settles, or null when every one has. */
+  nextSettle(now: number): number | null {
+    let next: number | null = null
+    for (const [key, last] of this.last) {
+      const at = this.settlesAt(last)
+      if (now >= at) this.last.delete(key)
+      else if (next === null || at < next) next = at
+    }
+    return next
+  }
 
-export function summarizeTranscriptionRoutes<T>(
-  subscribers: Iterable<T>,
-  routesFor: (subscriber: T) => TranscriptionSubscriberRoutes,
-): TranscriptionRouteSummary {
-  let hasCloudSubscriber = false
-  let hasForceLocalSubscriber = false
-  for (const subscriber of subscribers) {
-    const routes = routesFor(subscriber)
-    if (routes.cloud) hasCloudSubscriber = true
-    if (routes.forceLocal) hasForceLocalSubscriber = true
+  private settlesAt(last: {at: number; final: boolean}): number {
+    return last.at + (last.final ? this.quietMs : this.staleMs)
   }
-  return {hasCloudSubscriber, hasForceLocalSubscriber}
-}
-
-export function transcriptionDeliveryRoute(
-  source: "cloud" | "local",
-  routes: TranscriptionSubscriberRoutes,
-  cloudConnected: boolean,
-): TranscriptionDeliveryRoute | null {
-  if (source === "cloud") return routes.cloud ? "default" : null
-  if (cloudConnected) return routes.forceLocal ? "forceLocal" : null
-  if (routes.cloud && routes.forceLocal) return "all"
-  if (routes.forceLocal) return "forceLocal"
-  return routes.cloud ? "default" : null
 }

package/src/services/TranscriptionSubscriptions.ts

@@ -1,20 +1,132 @@
 import type {AudioPosition, TranscriptionToken} from "@mentra/cloud-protocol"
+import {
+  handoffStep,
+  readSpeechProcessing,
+  type RoutedListener,
+  type SpeechProcessing,
+  type TranscriptionSource,
+} from "./TranscriptionRouting"
 
 export interface TranscriptionListenerRegistration {
   id: string
   stream: string
-  forceLocal?: boolean
+  processing: SpeechProcessing
+  /**
+   * Registered for an SDK that predates listener ids. It gets un-addressed
+   * events, which that SDK fans out to every listener of the same class, so
+   * all of a class's legacy registrations move between sources together.
+   */
   legacy?: boolean
 }
 
+/** A transcription stream's entry in SUBSCRIBE `subscriptions`. */
+export interface TranscriptionStreamEntry {
+  stream: string
+  processing?: unknown
+  /** SDKs before `processing`: the stream has localOnly listeners. */
+  forceLocal?: unknown
+  /** SDKs before `processing`, with forceLocal: the stream also has cloud listeners. */
+  includeCloud?: unknown
+}
+
+/** Records what one transcription stream entry asks for. */
+export function addStreamProcessing(
+  byStream: Map<string, Set<SpeechProcessing>>,
+  entry: TranscriptionStreamEntry,
+): void {
+  const processing = byStream.get(entry.stream) ?? new Set<SpeechProcessing>()
+  byStream.set(entry.stream, processing)
+  processing.add(readSpeechProcessing(entry))
+  if (entry.processing === undefined && entry.forceLocal === true && entry.includeCloud === true) {
+    processing.add("preferCloud")
+  }
+}
+
+/**
+ * The transcription listeners of one SUBSCRIBE. SDKs register each listener
+ * (`transcriptionListeners`: `{id, stream, processing}`, or `forceLocal`
+ * before `processing`). A subscribed stream without registered listeners, as
+ * from SDKs that predate registration, gets one legacy registration per
+ * processing its stream entries ask for.
+ */
+export function transcriptionRegistrations(
+  registered: unknown,
+  streams: string[],
+  streamProcessing: Map<string, Set<SpeechProcessing>>,
+): TranscriptionListenerRegistration[] {
+  const subscribed = new Set(streams.filter((stream) => stream.startsWith("transcription:")))
+  const registrations: TranscriptionListenerRegistration[] = []
+  for (const value of Array.isArray(registered) ? registered : []) {
+    if (typeof value !== "object" || value === null) continue
+    const listener = value as {id?: unknown; stream?: unknown; processing?: unknown; forceLocal?: unknown}
+    if (typeof listener.id !== "string" || typeof listener.stream !== "string") continue
+    if (!subscribed.has(listener.stream)) continue
+    registrations.push({id: listener.id, stream: listener.stream, processing: readSpeechProcessing(listener)})
+  }
+  const registeredStreams = new Set(registrations.map((registration) => registration.stream))
+  for (const stream of subscribed) {
+    if (registeredStreams.has(stream)) continue
+    for (const processing of streamProcessing.get(stream) ?? ["preferCloud" as const]) {
+      registrations.push({id: `legacy:${processing}:${stream}`, stream, processing, legacy: true})
+    }
+  }
+  return registrations
+}
+
+/** What `TranscriptionSubscriptions.route` moves listeners by. */
+export interface TranscriptionHandoff {
+  target: (listener: RoutedListener) => TranscriptionSource
+  canDeliver: (source: TranscriptionSource) => boolean
+  /** Whether `source` is still busy on `stream` (see UtteranceTracker). */
+  busy: (source: TranscriptionSource, stream: string) => boolean
+  /** The on-device recognizer has been drained since it last spoke. */
+  drained: boolean
+  /** Cloud audio position where the last drain began. */
+  plannedWatermark: AudioPosition | null
+  /** Cloud audio position of the last on-device final. */
+  forcedWatermark: AudioPosition | null
+}
+
+/**
+ * Class an SDK from before `processing` filters routed events on: localOnly
+ * listeners take "forceLocal", the others "default".
+ */
+export type TranscriptionDeliveryRoute = "default" | "forceLocal"
+
+export interface TranscriptionDelivery {
+  /** Absent for a legacy class: the SDK fans the event out by `route`. */
+  listenerId?: string
+  route: TranscriptionDeliveryRoute
+  text: string
+}
+
+export interface TranscriptionListenerRoute extends RoutedListener {
+  id: string
+  /** Where the listener gets its text now; null before its first routing. */
+  source: TranscriptionSource | null
+  /** Where it is going (its source, unless a switch waits for a pause). */
+  target: TranscriptionSource
+}
+
 interface Listener extends TranscriptionListenerRegistration {
   cutoff: AudioPosition | null
   sessions: Set<number>
 }
 
-/** Runtime-owned text projection. Token detail never crosses the miniapp bridge. */
+interface GroupRoute {
+  source: TranscriptionSource
+  target: TranscriptionSource
+}
+
+/**
+ * One miniapp's transcription listeners: which source each one gets its text
+ * from, and its runtime-owned text projection. Token detail never crosses the
+ * miniapp bridge.
+ */
 export class TranscriptionSubscriptions {
   private listeners = new Map<string, Listener>()
+  /** Routing of each group: a listener, or one legacy class. */
+  private routing = new Map<string, GroupRoute>()
 
   observe(current: AudioPosition | null): void {
     if (!current) return
@@ -28,41 +140,149 @@ export class TranscriptionSubscriptions {
       const existing = this.listeners.get(registration.id)
       next.set(
         registration.id,
-        existing?.stream === registration.stream && !!existing.forceLocal === !!registration.forceLocal
+        existing?.stream === registration.stream &&
+          existing.processing === registration.processing &&
+          !!existing.legacy === !!registration.legacy
           ? existing
           : {...registration, cutoff, sessions: new Set(cutoff ? [cutoff.sessionTag] : [])},
       )
     }
     this.listeners = next
+    const groups = new Set([...next.values()].map(groupOf))
+    for (const group of [...this.routing.keys()]) {
+      if (!groups.has(group)) this.routing.delete(group)
+    }
+  }
+
+  /**
+   * Moves each group toward its target source (see `handoffStep`). A group
+   * that moves from on-device to the cloud starts its positioned cloud text
+   * at a watermark, so cloud tokens for speech it already got on-device are
+   * never projected to it: where the drain began after a planned move, the
+   * last on-device final after a forced one.
+   */
+  route(handoff: TranscriptionHandoff): {changed: boolean; needsDrain: boolean} {
+    let changed = false
+    let needsDrain = false
+    for (const [group, members] of this.groups()) {
+      // A legacy class moves on-device only when that serves all its streams.
+      const target = members.every((member) => handoff.target(member) === "local") ? "local" : "cloud"
+      const previous = this.routing.get(group)
+      const step = handoffStep(
+        previous?.source ?? null,
+        target,
+        handoff.canDeliver,
+        (candidate) => members.some((member) => handoff.busy(candidate, member.stream)),
+        handoff.drained,
+      )
+      if (step === "drain") needsDrain = true
+      const source = step === "wait" || step === "drain" ? previous!.source : target
+      if (previous?.source === source && previous.target === target) continue
+      changed = true
+      this.routing.set(group, {source, target})
+      const watermark = step === "planned" ? handoff.plannedWatermark : handoff.forcedWatermark
+      if (previous?.source === "local" && source === "cloud" && watermark) {
+        for (const member of members) advanceCutoff(member, watermark)
+      }
+    }
+    return {changed, needsDrain}
   }
 
-  project(
+  /** Whether a listener is waiting for its sources to settle before it switches. */
+  switching(): boolean {
+    for (const routing of this.routing.values()) {
+      if (routing.source !== routing.target) return true
+    }
+    return false
+  }
+
+  /** Every listener with its current routing. */
+  routes(): TranscriptionListenerRoute[] {
+    return [...this.listeners.values()].map((listener) => {
+      const routing = this.routing.get(groupOf(listener))
+      return {
+        id: listener.id,
+        stream: listener.stream,
+        processing: listener.processing,
+        source: routing?.source ?? null,
+        target: routing?.target ?? (listener.processing === "localOnly" ? "local" : "cloud"),
+      }
+    })
+  }
+
+  /**
+   * The events one transcript produces for this miniapp: one per listener
+   * that hears `stream` and currently gets its text from `source`. Cloud
+   * transcripts with token positions are trimmed to what each listener may
+   * hear; without positions (`tokens` undefined) the text goes as is.
+   */
+  deliveries(
     stream: string,
-    tokens: TranscriptionToken[],
+    source: TranscriptionSource,
+    text: string,
+    tokens: TranscriptionToken[] | undefined,
     current: AudioPosition | null,
-    route: "default" | "forceLocal" | "all",
-  ) {
+  ): TranscriptionDelivery[] {
     this.observe(current)
-    const events: Array<{listenerId?: string; text: string}> = []
+    const deliveries: TranscriptionDelivery[] = []
     for (const listener of this.listeners.values()) {
       if (listener.stream !== stream && listener.stream !== "transcription:auto") continue
-      if (listener.forceLocal ? route !== "forceLocal" && route !== "all" : route === "forceLocal") continue
-      const eligible = tokens.filter((token) => {
-        const position = token.audioPosition
-        if (!position) return false
-        if (position.sessionTag === listener.cutoff?.sessionTag) return position.offsetMs >= listener.cutoff.offsetMs
-        return listener.sessions.has(position.sessionTag)
-      })
-      const text = eligible
-        .map((token) => token.text)
-        .join("")
-        .trim()
-      if (text && listener.legacy) {
-        const existing = events.find((event) => event.listenerId === undefined)
-        if (!existing) events.push({text})
-        else if (text.length > existing.text.length) existing.text = text
-      } else if (text) events.push({listenerId: listener.id, text})
+      if (this.routing.get(groupOf(listener))?.source !== source) continue
+      const listenerText = tokens ? projectTokens(listener, tokens) : text
+      if (tokens && !listenerText) continue
+      const route: TranscriptionDeliveryRoute = listener.processing === "localOnly" ? "forceLocal" : "default"
+      if (!listener.legacy) {
+        deliveries.push({listenerId: listener.id, route, text: listenerText})
+        continue
+      }
+      const existing = deliveries.find((delivery) => delivery.listenerId === undefined && delivery.route === route)
+      if (!existing) deliveries.push({route, text: listenerText})
+      else if (listenerText.length > existing.text.length) existing.text = listenerText
     }
-    return events
+    return deliveries
   }
+
+  private groups(): Map<string, Listener[]> {
+    const groups = new Map<string, Listener[]>()
+    for (const listener of this.listeners.values()) {
+      const group = groupOf(listener)
+      const members = groups.get(group)
+      if (members) members.push(listener)
+      else groups.set(group, [listener])
+    }
+    return groups
+  }
+}
+
+function groupOf(listener: TranscriptionListenerRegistration): string {
+  return listener.legacy ? `legacy ${listener.processing}` : `listener ${listener.id}`
+}
+
+/**
+ * Moves a listener's cloud text start up to `watermark`, never back before
+ * where it already starts (its registration, or an earlier handoff). A
+ * watermark in a session the listener never saw predates it.
+ */
+function advanceCutoff(listener: Listener, watermark: AudioPosition): void {
+  const cutoff = listener.cutoff
+  if (cutoff?.sessionTag === watermark.sessionTag) {
+    if (watermark.offsetMs > cutoff.offsetMs) listener.cutoff = watermark
+    return
+  }
+  if (!listener.sessions.has(watermark.sessionTag)) return
+  listener.cutoff = watermark
+  listener.sessions = new Set([watermark.sessionTag])
+}
+
+function projectTokens(listener: Listener, tokens: TranscriptionToken[]): string {
+  return tokens
+    .filter((token) => {
+      const position = token.audioPosition
+      if (!position) return false
+      if (position.sessionTag === listener.cutoff?.sessionTag) return position.offsetMs >= listener.cutoff.offsetMs
+      return listener.sessions.has(position.sessionTag)
+    })
+    .map((token) => token.text)
+    .join("")
+    .trim()
 }