@linto-ai/transcript-ui-plugin-audio 0.9.0

package/README.md

@@ -0,0 +1,28 @@
+# @linto-ai/transcript-ui-plugin-audio
+
+Audio playback synced with the transcript: waveform display, active-word/active-turn highlighting as playback progresses, click-to-seek, and per-speaker region coloring on the waveform.
+
+## Usage
+
+```ts
+import { createAudioPlugin } from "@linto-ai/transcript-ui-plugin-audio"
+
+core.use(
+  createAudioPlugin({
+    // Resolve a Channel/Translation's `audio.src` into a playable URL —
+    // add auth, fetch as a blob, whatever your backend requires.
+    // (generateUrl/fileId below are just this host's own routing — not
+    // part of this library.)
+    resolveSrc: async (source) => {
+      const response = await fetch(generateUrl(`apps/linto/api/audio/${fileId}`))
+      if (!response.ok) throw new Error("Audio unavailable")
+      const blob = await response.blob()
+      return URL.createObjectURL(blob)
+    },
+  }),
+)
+```
+
+Any `blob:` URL returned by `resolveSrc` is revoked automatically when the source changes or the plugin is destroyed.
+
+`resolveWaveform` is the same idea for precomputed waveform peaks (e.g. fetched from your API) — return `null` to fall back to client-side decoding.

package/package.json

@@ -0,0 +1,46 @@
+{
+  "name": "@linto-ai/transcript-ui-plugin-audio",
+  "version": "0.9.0",
+  "description": "Audio playback plugin for @linto-ai/transcript-ui — waveform, active-word highlighting, seek",
+  "keywords": [
+    "vue",
+    "vue3",
+    "transcript",
+    "audio",
+    "waveform",
+    "linto-plugin"
+  ],
+  "sideEffects": [
+    "**/*.vue"
+  ],
+  "homepage": "https://github.com/linto-ai/linto-studio",
+  "bugs": "https://github.com/linto-ai/linto-studio/issues",
+  "author": "Tom Darboux <tdarboux@linagora.com>",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/linto-ai/linto-studio.git",
+    "directory": "studio-sdk/components/transcript-ui/packages/plugin-audio"
+  },
+  "type": "module",
+  "license": "AGPL-3.0",
+  "publishConfig": {
+    "access": "public"
+  },
+  "files": [
+    "src"
+  ],
+  "main": "./src/index.ts",
+  "types": "./src/index.ts",
+  "exports": {
+    ".": "./src/index.ts"
+  },
+  "peerDependencies": {
+    "vue": "^3.5.0"
+  },
+  "dependencies": {
+    "@linto-ai/transcript-ui-core": "0.9.0",
+    "@linto-ai/transcript-ui-i18n": "0.9.0",
+    "@linto-ai/transcript-ui-ui": "0.9.0",
+    "wavesurfer.js": "^7.12.1"
+  }
+}

package/src/AudioPlayer.vue

@@ -0,0 +1,96 @@
+<script setup lang="ts">
+import { ref, toRef } from 'vue'
+import AudioPlayerControls from './AudioPlayerControls.vue'
+import { useAudioPlayer } from './useAudioPlayer'
+
+const props = defineProps<{
+  audioSrc?: string
+}>()
+
+const waveformRef = ref<HTMLElement | null>(null)
+
+const {
+  isPlaying,
+  isReady,
+  isLoading,
+  volume,
+  playbackRate,
+  isMuted,
+  formattedCurrentTime,
+  formattedDuration,
+  togglePlay,
+  seekTo,
+  pause,
+  skip,
+  setVolume,
+  cyclePlaybackRate,
+  toggleMute,
+} = useAudioPlayer({
+  containerRef: waveformRef,
+  audioSrc: toRef(() => props.audioSrc),
+})
+
+defineExpose({ seekTo, pause })
+</script>
+
+<template>
+  <footer class="audio-player">
+    <div
+      ref="waveformRef"
+      class="waveform-container"
+      :class="{ 'waveform-container--loading': isLoading }" />
+    <AudioPlayerControls
+      :is-playing="isPlaying"
+      :current-time="formattedCurrentTime"
+      :duration="formattedDuration"
+      :volume="volume"
+      :playback-rate="playbackRate"
+      :is-muted="isMuted"
+      :is-ready="isReady"
+      @toggle-play="togglePlay"
+      @skip-back="skip(-10)"
+      @skip-forward="skip(10)"
+      @update:volume="setVolume"
+      @toggle-mute="toggleMute"
+      @cycle-playback-rate="cyclePlaybackRate" />
+  </footer>
+</template>
+
+<style scoped>
+.audio-player {
+  border-top: 1px solid var(--color-border);
+  background-color: var(--color-surface);
+  flex-shrink: 0;
+}
+
+.waveform-container {
+  min-height: 32px;
+}
+
+.waveform-container--loading {
+  background: linear-gradient(
+    90deg,
+    var(--color-border-light, var(--color-border)) 25%,
+    var(--color-border) 50%,
+    var(--color-border-light, var(--color-border)) 75%
+  );
+  background-size: 200% 100%;
+  animation: shimmer 1.5s ease-in-out infinite;
+  border-radius: var(--radius-sm);
+}
+
+@keyframes shimmer {
+  0% {
+    background-position: 200% 0;
+  }
+  100% {
+    background-position: -200% 0;
+  }
+}
+
+@media (prefers-reduced-motion: reduce) {
+  .waveform-container--loading {
+    animation: none;
+  }
+}
+</style>

package/src/AudioPlayerControls.vue

@@ -0,0 +1,207 @@
+<script setup lang="ts">
+import { ref } from 'vue'
+import { Play, Pause, SkipBack, SkipForward, Volume2, VolumeX } from 'lucide-vue-next'
+import { Button } from '@linto-ai/transcript-ui-ui'
+import { useI18n } from '@linto-ai/transcript-ui-i18n'
+
+defineProps<{
+  isPlaying: boolean
+  currentTime: string
+  duration: string
+  volume: number
+  playbackRate: number
+  isMuted: boolean
+  isReady: boolean
+}>()
+
+const emit = defineEmits<{
+  togglePlay: []
+  skipBack: []
+  skipForward: []
+  'update:volume': [value: number]
+  toggleMute: []
+  cyclePlaybackRate: []
+}>()
+
+const { t } = useI18n()
+
+const showVolumeSlider = ref(false)
+
+function onVolumeInput(event: Event) {
+  const target = event.target as HTMLInputElement
+  emit('update:volume', parseFloat(target.value))
+}
+</script>
+
+<template>
+  <div class="player-controls">
+    <div class="controls-left">
+      <Button
+        variant="transparent"
+        size="md"
+        class="skip-button"
+        :aria-label="t('player.skipBack')"
+        :disabled="!isReady"
+        @click="emit('skipBack')"
+      >
+        <template #icon><SkipBack :size="16" /></template>
+      </Button>
+
+      <Button
+        variant="transparent"
+        size="md"
+        class="play-button"
+        :aria-label="isPlaying ? t('player.pause') : t('player.play')"
+        :disabled="!isReady"
+        @click="emit('togglePlay')"
+      >
+        <template #icon>
+          <Pause v-if="isPlaying" :size="20" />
+          <Play v-else :size="20" />
+        </template>
+      </Button>
+
+      <Button
+        variant="transparent"
+        size="md"
+        class="skip-button"
+        :aria-label="t('player.skipForward')"
+        :disabled="!isReady"
+        @click="emit('skipForward')"
+      >
+        <template #icon><SkipForward :size="16" /></template>
+      </Button>
+    </div>
+
+    <div class="controls-time">
+      <time class="time-display">{{ currentTime }}</time>
+      <span class="time-separator">/</span>
+      <time class="time-display">{{ duration }}</time>
+    </div>
+
+    <div class="controls-right">
+      <div
+        class="volume-group"
+        @mouseenter="showVolumeSlider = true"
+        @mouseleave="showVolumeSlider = false"
+      >
+        <Button
+          variant="transparent"
+          size="md"
+          :aria-label="isMuted ? t('player.unmute') : t('player.mute')"
+          :disabled="!isReady"
+          @click="emit('toggleMute')"
+        >
+          <template #icon>
+            <VolumeX v-if="isMuted" :size="16" />
+            <Volume2 v-else :size="16" />
+          </template>
+        </Button>
+        <input
+          v-show="showVolumeSlider"
+          type="range"
+          class="volume-slider"
+          min="0"
+          max="1"
+          step="0.05"
+          :value="volume"
+          :aria-label="t('player.volume')"
+          :disabled="!isReady"
+          @input="onVolumeInput"
+        >
+      </div>
+
+      <Button
+        variant="transparent"
+        size="md"
+        class="speed-button"
+        :aria-label="t('player.speed')"
+        :disabled="!isReady"
+        @click="emit('cyclePlaybackRate')"
+      >
+        {{ playbackRate }}x
+      </Button>
+    </div>
+  </div>
+</template>
+
+<style scoped>
+.player-controls {
+  display: flex;
+  align-items: center;
+  gap: var(--spacing-md);
+  padding: var(--spacing-xs) var(--spacing-lg);
+  height: 44px;
+}
+
+.controls-left {
+  display: flex;
+  align-items: center;
+  gap: var(--spacing-xs);
+}
+
+.controls-time {
+  display: flex;
+  align-items: center;
+  gap: var(--spacing-xxs);
+  font-family: var(--font-family-mono);
+  font-size: var(--font-size-sm);
+  color: var(--color-text-muted);
+  user-select: none;
+}
+
+.time-separator {
+  color: var(--color-text-muted);
+  opacity: 0.5;
+}
+
+.controls-right {
+  display: flex;
+  align-items: center;
+  gap: var(--spacing-xs);
+  margin-left: auto;
+}
+
+.volume-group {
+  display: flex;
+  align-items: center;
+  gap: var(--spacing-xs);
+}
+
+.volume-slider {
+  width: 80px;
+  height: 4px;
+  accent-color: var(--color-primary);
+  cursor: pointer;
+}
+
+.volume-slider:disabled {
+  opacity: 0.5;
+  cursor: default;
+}
+
+.play-button {
+  width: 40px;
+  height: 40px;
+}
+
+.speed-button {
+  font-size: var(--font-size-sm);
+  font-family: var(--font-family-mono);
+}
+
+@media (max-width: 767px) {
+  .skip-button {
+    display: none;
+  }
+
+  .volume-slider {
+    display: none;
+  }
+
+  .player-controls {
+    padding: var(--spacing-xs) var(--spacing-md);
+    gap: var(--spacing-sm);
+  }
+}
+</style>

package/src/index.ts

@@ -0,0 +1,196 @@
+import { ref, computed, watch, watchEffect } from "vue"
+import type { Core, CorePlugin, AudioPluginApi, AudioSource } from "@linto-ai/transcript-ui-core"
+import { utils } from "@linto-ai/transcript-ui-core"
+import AudioPlayer from "./AudioPlayer.vue"
+
+const { findActiveWord, firstWordStart, lastWordEnd } = utils
+
+export type { AudioPluginApi }
+export { AudioPlayer }
+
+/**
+ * Minimum playback progress (in seconds of media time) between two
+ * activeWordId computations. Playback ticks ~60 Hz but the active word only
+ * changes at word boundaries (~4 Hz), so recomputing every frame is wasted
+ * work. Keep it well under a spoken word's duration so the highlight never
+ * lags perceptibly.
+ */
+const WORD_TRACK_INTERVAL = 0.05
+
+export interface AudioPluginOptions {
+  /**
+   * Resolves an `AudioSource` into a playable URL. Lets the host add a
+   * bearer token, fetch as a blob then `URL.createObjectURL`, etc.
+   * When absent, `source.src` is used as is.
+   *
+   * Any returned `blob:` URL is revoked automatically when the source
+   * changes or the plugin is destroyed.
+   */
+  resolveSrc?: (source: AudioSource) => string | Promise<string>
+
+  /**
+   * Resolves precomputed waveform peaks for an `AudioSource` (e.g. fetched
+   * from the API). Raw amplitude values, any scale — the player normalizes
+   * them. Return null (or throw) to fall back to client-side decoding.
+   */
+  resolveWaveform?: (
+    source: AudioSource,
+  ) => number[] | null | Promise<number[] | null>
+}
+
+export function createAudioPlugin(
+  options: AudioPluginOptions = {},
+): CorePlugin {
+  return {
+    name: "audio",
+    components: { player: AudioPlayer },
+
+    install(core: Core) {
+      const currentTime = ref(0)
+      const isPlaying = ref(false)
+      const activeWordId = ref<string | null>(null)
+      const activeTurnId = ref<string | null>(null)
+      let seekHandler: ((time: number) => void) | null = null
+      let pauseHandler: (() => void) | null = null
+
+      const rawSource = computed(
+        () => core.activeChannel.value?.activeTranslation.value.audio ?? null,
+      )
+
+      const resolvedSrc = ref<string | null>(null)
+      const waveform = ref<number[] | null>(null)
+      let ownedObjectUrl: string | null = null
+
+      function revokeOwned() {
+        if (ownedObjectUrl) {
+          URL.revokeObjectURL(ownedObjectUrl)
+          ownedObjectUrl = null
+        }
+      }
+
+      const stopSourceWatch = watch(
+        rawSource,
+        async (source) => {
+          revokeOwned()
+          resolvedSrc.value = null
+          waveform.value = null
+          if (!source) return
+
+          // Resolved alongside the src; a failure here only disables the
+          // precomputed waveform, never the audio itself.
+          const waveformPromise = options.resolveWaveform
+            ? Promise.resolve(options.resolveWaveform(source)).catch((err) => {
+                console.warn("[audio] resolveWaveform failed", err)
+                return null
+              })
+            : Promise.resolve(null)
+
+          try {
+            const [url, peaks] = await Promise.all([
+              options.resolveSrc
+                ? options.resolveSrc(source)
+                : Promise.resolve(source.src),
+              waveformPromise,
+            ])
+            // Peaks are set before the src: the player creates WaveSurfer
+            // when the src changes and reads the waveform at that point.
+            waveform.value = peaks?.length ? peaks : null
+            resolvedSrc.value = url
+            if (url.startsWith("blob:")) ownedObjectUrl = url
+          } catch (err) {
+            console.error("[audio] resolveSrc failed", err)
+          }
+        },
+        { immediate: true },
+      )
+
+      const src = computed(() => resolvedSrc.value)
+
+      // Media time of the last activeWordId computation, used to throttle.
+      // -Infinity forces a compute on the first tick after playback starts.
+      let lastComputeTime = Number.NEGATIVE_INFINITY
+
+      // Single source of truth: computes activeTurnId / activeWordId.
+      // No reset to null on pause: the last known position is kept.
+      const stopTracker = watchEffect(() => {
+        // Read BOTH refs unconditionally so the effect tracks currentTime even
+        // while paused — otherwise seeking/scrubbing (e.g. clicking a word)
+        // while paused would never move the highlight.
+        const time = currentTime.value
+        const playing = isPlaying.value
+
+        // Throttle only during continuous playback (the word can't have changed
+        // within WORD_TRACK_INTERVAL). A backward jump is negative and falls
+        // through; while paused, any seek recomputes.
+        if (playing) {
+          const elapsed = time - lastComputeTime
+          if (elapsed >= 0 && elapsed < WORD_TRACK_INTERVAL) return
+        }
+        lastComputeTime = time
+
+        const translation = core.activeChannel.value?.activeTranslation.value
+        if (!translation) return
+
+        for (const turn of translation.turns.value) {
+          // Derive the turn's span from its words: after a split, the turn
+          // attrs go stale (the first half keeps the whole original span, the
+          // second half has none), which would overlap and pick the wrong turn.
+          // The words carry the correct per-wid timestamps, so trust them and
+          // only fall back to the turn attrs for word-less (live) turns.
+          // First/last DEFINED word times: robust to words with no timestamp
+          // (freshly typed, or split/merge products) sitting anywhere, and to
+          // the stale turn attrs after a split. Fall back to the turn attrs for
+          // fully word-less (live text-only) turns.
+          const words = turn.words
+          const start = firstWordStart(words) ?? turn.startTime
+          const end = lastWordEnd(words) ?? turn.endTime
+          if (start != null && end != null && time >= start && time <= end) {
+            activeTurnId.value = turn.id
+            // Returns null when no timestamped word matches (e.g. the playhead
+            // sits over an untimed, just-typed word) — no stale highlight.
+            activeWordId.value = findActiveWord(words, time)
+            return
+          }
+        }
+      })
+
+      function seekTo(time: number) {
+        seekHandler?.(time)
+      }
+
+      function setSeekHandler(fn: ((time: number) => void) | null) {
+        seekHandler = fn
+      }
+
+      function pause() {
+        pauseHandler?.()
+      }
+
+      function setPauseHandler(fn: (() => void) | null) {
+        pauseHandler = fn
+      }
+
+      const api: AudioPluginApi = {
+        currentTime,
+        isPlaying,
+        src,
+        waveform,
+        activeWordId,
+        activeTurnId,
+        seekTo,
+        setSeekHandler,
+        pause,
+        setPauseHandler,
+      }
+
+      core.audio = api
+
+      return () => {
+        stopSourceWatch()
+        stopTracker()
+        revokeOwned()
+        core.audio = undefined
+      }
+    },
+  }
+}