@lincyaw/addon-lecture 0.0.0-stage → 1.0.0-rc.1

package/src/client/PlayerPanel.tsx

@@ -0,0 +1,194 @@
+import { PmCaption, PmDrawer, PmIconButton, PmMenu, PmTabs, PRODUCTION_PLAYBACK, TIMED_CAPTIONS, useMessages, useServer, useService, useStore } from '@lincyaw/host'
+import type { MenuItem } from '@lincyaw/host'
+import { isPreviewEmbedded } from '@lincyaw/kernel'
+import { useEffect, useMemo, useState } from 'react'
+import type { CSSProperties } from 'react'
+import {
+  captionLanguages,
+  captionGroups,
+  captionGroupAt,
+  captionText,
+  clockText,
+  isStale,
+  LECTURE_ID,
+  LECTURE_MESSAGES,
+  SCRIPT_TRACK,
+  trackChapters,
+  utteranceAt,
+} from '../domain/index.ts'
+import type { Utterance } from '../domain/index.ts'
+import ChapterList from './ChapterList.tsx'
+import { useLecturePlayback } from './playback.ts'
+import { lectureSource } from './source.ts'
+import { PLAYBACK_RATES, ProductionPlaybackController } from './production-playback.ts'
+import {
+  autoplay,
+  burnedCaptions,
+  exporting,
+  keyboardPlayer,
+  play,
+  playing,
+  useLecturesAvailable,
+  useViewer,
+} from './state.ts'
+import TranscriptList from './TranscriptList.tsx'
+import './PlayerPanel.css'
+/** `&autoplay` starts the first lecture that becomes ready on this page, once. */
+let autoplayPending = autoplay
+export default function PlayerPanel(_props: { view?: string }) {
+  const t = useMessages(LECTURE_MESSAGES),
+    viewer = useViewer(),
+    server = useServer()
+  const available = useLecturesAvailable(),
+    id = useStore(playing),
+    doc = useStore(viewer.doc)
+  const { player, status, failure, language, setLanguage } =
+    useLecturePlayback()
+  const timedCaptions = useService(TIMED_CAPTIONS)
+  const production = useService(PRODUCTION_PLAYBACK)
+  const controlled = useStore(production instanceof ProductionPlaybackController ? production.engaged : undefined, false)
+  useEffect(() => player && status === 'ready' ? timedCaptions?.claim() : undefined, [player, status, timedCaptions])
+  useEffect(() => {
+    if (!player || status !== 'ready' || failure || controlled || exporting) return
+    keyboardPlayer.set(player)
+    if (autoplayPending) {
+      autoplayPending = false
+      void player.play()
+    }
+    return () => { if (keyboardPlayer.getSnapshot() === player) keyboardPlayer.set(undefined) }
+  }, [player, status, failure, controlled])
+  const time = useStore(player?.time, 0),
+    clockPlaying = useStore(player?.clock.playing, false),
+    rate = useStore(player?.clock.rate, 1)
+  const groupedCaptions = useMemo(() => new Map(player?.track.utterances.map(utterance => [utterance.id, captionGroups(utterance)])), [player?.track])
+  const cursor = useStore(player?.cursor), positionMode = useStore(player?.positionMode, 'time')
+  const [compact, setCompact] = useState(() => matchMedia('(max-width: 479px)').matches)
+  useEffect(() => {
+    const media = matchMedia('(max-width: 479px)')
+    const changed = () => setCompact(media.matches)
+    media.addEventListener('change', changed)
+    return () => media.removeEventListener('change', changed)
+  }, [])
+  const [captions, setCaptions] = useState(false),
+    [panel, setPanel] = useState<'transcript' | 'chapters'>()
+  useEffect(() => {
+    let active = true
+    if (
+      playing.getSnapshot() ||
+      exporting ||
+      (!server.available && (isPreviewEmbedded() || !available))
+    )
+      return
+    void lectureSource(server, doc.config.presmium[LECTURE_ID])
+      .list()
+      .then((tracks) => {
+        if (active && !playing.getSnapshot() && !(production instanceof ProductionPlaybackController && production.engaged.getSnapshot()) && tracks.some(track => track.id === SCRIPT_TRACK))
+          play(SCRIPT_TRACK)
+      })
+      .catch((error) =>
+        console.warn(
+          '[presmium] the presentation could not list its lectures',
+          error,
+        ),
+      )
+    return () => {
+      active = false
+    }
+  }, [server.available, available, doc, production])
+  const state = !player
+    ? undefined
+    : clockPlaying
+      ? 'playing'
+      : 'paused'
+  const titles = new Map(
+    doc.sections.map((section) => [section.id, section.title]),
+  )
+  const chapters = player
+    ? trackChapters(player.events, (key) => titles.get(key))
+    : []
+  const languages = player ? captionLanguages(player.track) : []
+  const timed = player ? utteranceAt(player.track, time) : undefined
+  const boundary = player && cursor ? player.events.findIndex(event => event.type === 'go' && event.data.section === cursor.sectionId && Number(event.data.step ?? 0) === cursor.step) : -1
+  const start = player?.events[boundary]?.t, end = player?.events.slice(boundary + 1).find(event => event.type === 'go')?.t ?? player?.duration
+  // A positive utterance may continue through a cue into another step. Preserve that real timing.
+  const continuing = timed && cursor && timed.d > 0 && timed.section === cursor.sectionId && (timed.step ?? 0) <= cursor.step
+  const selectedZero = player && cursor && positionMode === 'step' && !clockPlaying && boundary >= 0 && start === end && time === start && !continuing
+  const selectedCaptions = selectedZero ? player.track.utterances.filter(utterance => utterance.section === cursor.sectionId && (utterance.step ?? 0) === cursor.step && utterance.d === 0 && utterance.t === time) : undefined
+  const current = selectedZero ? selectedCaptions?.[0] : timed
+  const voice = player?.track.voice
+  const unsaid = player?.track.utterances.filter((u) => u.unsaid) ?? []
+  const synthetic =
+    voice?.kind === 'synthetic' && voice.provider
+      ? t.player.syntheticBy(voice.provider)
+      : t.player.synthetic
+  const lang = burnedCaptions ?? language
+  const text = (u: Utterance) =>
+    lang && lang !== player?.track.lang
+      ? (u.translations?.[lang]?.text ?? u.text)
+      : captionGroupAt(groupedCaptions.get(u.id) ?? [], time)?.text ?? u.text
+  const manual = player?.track.captionOverrides?.find(own => own.startMs <= time && time < own.endMs), manualText = manual && captionGroupAt(manual.groups, time)?.text
+  const originalLanguage = !lang || lang === player?.track.lang
+  const caption = manual && originalLanguage ? manualText : selectedCaptions ? selectedCaptions.map(text).join(' ') : current ? text(current) : undefined
+  const burned =
+    burnedCaptions && player && (current || manual)
+      ? burnedCaptions === player.track.lang ? manual ? manualText : current && text(current) : current && captionText(player.track, current, burnedCaptions)
+      : undefined
+  const stale = new Set(
+    player && lang && lang !== player.track.lang
+      ? player.track.utterances.filter((u) => isStale(u, lang)).map((u) => u.id)
+      : [],
+  )
+  if (controlled) return player && (current || manual) && caption ? <PmCaption overlay label={t.player.captions} lines={[caption]} lang={lang} /> : null
+  if (!id) return null
+  if (exporting) return player ? <div className="pm-lecture-layer">
+    <div className="pm-lecture-export" data-time={Math.round(time)} data-clip={current?.clip}>
+      {voice?.kind === 'synthetic' && <span className="pm-lecture-player__ai pm-ui-badge pm-ui-badge--accent pm-ui-badge--pill">{synthetic}</span>}
+    </div>
+    {burned && <p className="pm-lecture-caption" lang={burnedCaptions}>{burned}</p>}
+  </div> : null
+  const problem = failure ? t.player.failed(failure) : status === 'missing' ? t.player.missing(id) : undefined
+  if (!player || problem) return <div className="pm-lecture-player__status" role={problem ? 'alert' : 'status'}>
+    <span>{problem ?? t.player.loading}</span>
+    {problem && <PmIconButton small icon="close" label={t.player.close} onClick={() => play(undefined)} />}
+  </div>
+  const options: MenuItem[] = [
+    ...(compact ? [{ id: 'previous', label: t.player.previous, icon: 'chevron-left' as const }, { id: 'next', label: t.player.next, icon: 'chevron-right' as const }] : []),
+    ...(compact && viewer.options.openPresenter ? [{ id: 'presenter', label: t.player.presenter, icon: 'slides' as const }] : []),
+    { id: 'transcript', label: t.player.transcript, icon: 'document', checked: panel === 'transcript' },
+    { id: 'chapters', label: t.player.chapters, icon: 'overview', checked: panel === 'chapters' },
+    ...PLAYBACK_RATES.map((value, index) => ({ id: `rate:${value}`, label: `${t.player.rate} · ${value}×`, checked: rate === value, separator: index === 0 })),
+    ...(languages.length > 1 ? languages.map((value, index) => ({ id: `language:${value}`, label: `${t.player.language} · ${value}`, checked: lang === value, separator: index === 0 })) : []),
+    ...(player.dropped ? [{ id: 'dropped', label: t.player.dropped(player.dropped), disabled: true, separator: true }] : []),
+    ...(unsaid.length ? [{ id: 'unsaid', label: t.player.unsaid(unsaid.length), disabled: true, separator: true }] : []),
+    { id: 'close', label: t.player.close, icon: 'close', separator: true },
+  ]
+  const selectOption = (option: string) => {
+    if (option === 'transcript' || option === 'chapters') setPanel(value => value === option ? undefined : option)
+    else if (option === 'previous') viewer.prev()
+    else if (option === 'next') viewer.next()
+    else if (option === 'presenter') viewer.options.openPresenter?.(viewer.position.getSnapshot())
+    else if (option.startsWith('rate:')) player.setRate(Number(option.slice(5)))
+    else if (option.startsWith('language:')) setLanguage(option.slice(9))
+    else if (option === 'close') play(undefined)
+  }
+  return <>
+    {(captions || current?.unsaid) && (current || manual) && caption && <PmCaption overlay label={t.player.captions} lines={[caption]} lang={lang}
+      data-stale={current && stale.has(current.id) ? 'true' : undefined} hint={current && stale.has(current.id) ? t.player.stale : undefined} />}
+    {panel && <PmDrawer open label={panel === 'transcript' ? t.player.transcript : t.player.chapters} onClose={() => setPanel(undefined)}>
+      <PmTabs value={panel} onValueChange={value => setPanel(value as 'transcript' | 'chapters')}
+        tabs={[{ value: 'transcript', label: t.player.transcript }, { value: 'chapters', label: t.player.chapters }]} label={t.player.sideTabs} />
+      {panel === 'transcript' ? <TranscriptList track={player.track} language={lang} current={current?.id} stale={stale} onSeek={at => player.seek(at)} />
+        : <ChapterList chapters={chapters} events={player.events} time={time} onSeek={at => player.seek(at)} />}
+    </PmDrawer>}
+    <div className="pm-lecture-player" role="region" aria-label={t.player.region} data-state={state} data-time={Math.round(time)} data-clip={current?.clip} data-unsaid={unsaid.length || undefined}>
+      <PmIconButton small className="pm-lecture-player__play" icon={clockPlaying ? 'pause' : 'play'} label={clockPlaying ? t.player.pause : t.player.play}
+        hint={`${clockPlaying ? t.player.pause : t.player.play} · Space`} aria-keyshortcuts="Space"
+        onClick={() => player.playing ? player.pause() : void player.play()} />
+      <span className="pm-lecture-player__time" title={t.player.time(clockText(time), clockText(player.duration))}>{clockText(time)}<span className="pm-lecture-player__time-total"> / {clockText(player.duration)}</span></span>
+      <input className="pm-lecture-player__seek pm-ui-range" style={{ '--pm-range-progress': `${player.duration ? Math.max(0, Math.min(100, time / player.duration * 100)) : 0}%` } as CSSProperties} type="range" min="0" max={player.duration} step="100" value={Math.round(time)}
+        aria-label={t.player.seek} aria-valuetext={t.player.time(clockText(time), clockText(player.duration))} onChange={event => player.seek(Number(event.target.value))} />
+      <PmIconButton small icon="captions" label={t.player.captions} aria-pressed={captions} onClick={() => setCaptions(value => !value)} />
+      <PmMenu label={t.player.more} items={options} placement="above-end" onSelect={selectOption} />
+    </div>
+  </>
+}

package/src/client/RecorderPanel.css

@@ -0,0 +1,46 @@
+/* Recording is for presenting, not for the preview of a workbench. */
+html.pm-in-workbench .pm-lecture-recorder {
+  display: none;
+}
+
+.pm-lecture-recorder {
+  position: absolute;
+  top: 0.5rem;
+  right: 0.5rem;
+  z-index: 5;
+  box-sizing: border-box;
+  max-width: calc(100% - 1rem);
+  padding: var(--pm-space-2) var(--pm-space-3);
+  border: var(--pm-size-hairline) solid var(--pm-color-line);
+  border-radius: var(--pm-radius-card);
+  background: var(--pm-color-surface);
+  color: var(--pm-color-ink);
+  box-shadow: var(--pm-shadow-float);
+  font-family: var(--pm-font-sans);
+  font-size: var(--pm-text-sm);
+}
+.pm-lecture-recorder__row {
+  display: flex;
+  flex-wrap: wrap;
+  align-items: center;
+  gap: 0.4rem;
+}
+.pm-lecture-recorder__field {
+  display: inline-flex;
+  align-items: center;
+  gap: 0.25rem;
+}
+.pm-lecture-recorder__status {
+  margin: 0.25rem 0 0;
+}
+.pm-lecture-recorder__dot {
+  margin-inline-end: 0.35em;
+  color: var(--pm-color-danger);
+}
+.pm-lecture-surface {
+  position: fixed;
+  inset: 0 0 var(--pm-overlay-bar, 3rem);
+  z-index: -1;
+  cursor: crosshair;
+  touch-action: none;
+}

package/src/client/RecorderPanel.tsx

@@ -0,0 +1,282 @@
+import {
+  blockAt,
+  linesText,
+  OPERATIONS,
+  PmIcon,
+  useMessages,
+  useServer,
+  useService,
+  useStore,
+} from '@lincyaw/host'
+import type { BlockPoint } from '@lincyaw/host'
+import { useEffect, useRef, useState } from 'react'
+import { clockText, LECTURE_MESSAGES, simplifyPath } from '../domain/index.ts'
+import type { PointerEvent as ReactPointerEvent } from 'react'
+import type { PathPoint } from '../domain/index.ts'
+import { applyInk } from './pointer-ink.ts'
+import { LectureRecorder } from './recorder.ts'
+import { serverSource } from './source.ts'
+import { playing, refreshLectures, useViewer } from './state.ts'
+import './RecorderPanel.css'
+const cls = 'pm-lecture-button pm-ui-button pm-ui-button--small'
+export default function RecorderPanel(_props: { view?: string }) {
+  const t = useMessages(LECTURE_MESSAGES),
+    viewer = useViewer(),
+    server = useServer(),
+    operations = useService(OPERATIONS)
+  const id = useStore(playing),
+    member = useStore(server.member, undefined)
+  const [recorder, setRecorder] = useState<LectureRecorder>(),
+    [trackIds, setTrackIds] = useState<string[]>([]),
+    [trackId, setTrackId] = useState(''),
+    [camera, setCamera] = useState(false),
+    [screen, setScreen] = useState(false),
+    [tool, setTool] = useState<'laser' | 'marker'>()
+  const state = useStore(recorder?.state, undefined),
+    elapsed = useStore(recorder?.elapsed, 0),
+    failure = useStore(recorder?.error, undefined)
+  const shown = server.available && !id,
+    recording = state === 'recording',
+    busy = ['starting', 'recording', 'saving'].includes(state ?? '')
+  const surface = useRef<HTMLDivElement>(null),
+    stroke = useRef<BlockPoint>()
+  const loadTrackIds = async () =>
+    setTrackIds(
+      (
+        await serverSource(server)
+          .list()
+          .catch(() => [])
+      ).map((item) => item.id),
+    )
+  useEffect(() => {
+    if (shown) void loadTrackIds()
+  }, [shown])
+  async function start() {
+    if (!operations || busy) return
+    const next = new LectureRecorder({
+      server,
+      operations,
+      doc: () => viewer.doc.getSnapshot(),
+      position: () => viewer.position.getSnapshot(),
+      speaker: member?.name,
+      camera,
+      screen,
+      track: trackId || undefined,
+    })
+    setRecorder(next)
+    await next.start()
+  }
+  async function stop() {
+    await recorder?.stop()
+    await Promise.all([loadTrackIds(), refreshLectures()])
+  }
+  useEffect(() => {
+    let path:
+      | { block: string; line?: number; from: number; points: PathPoint[] }
+      | undefined
+    let sampled = 0,
+      idle: ReturnType<typeof setTimeout> | undefined
+    function flush() {
+      clearTimeout(idle)
+      const done = path
+      path = undefined
+      if (!done || !operations) return
+      const data = {
+        block: done.block,
+        path: simplifyPath(done.points),
+        ...(done.line ? { line: done.line } : {}),
+      }
+      recorder?.notePointerPathStart(data, done.from)
+      operations.report({ type: 'point', data })
+    }
+    function pointer(event: PointerEvent) {
+      if (!recording || event.timeStamp - sampled < 50) return
+      sampled = event.timeStamp
+      const at = blockAt(event.clientX, event.clientY, {
+        skip: surface.current ?? undefined,
+        within: '.pm-present',
+      })
+      if (!at || (path && path.block !== at.key)) flush()
+      if (!at) return
+      path ??= { block: at.key, from: event.timeStamp, points: [] }
+      path.points.push([event.timeStamp - path.from, at.x, at.y])
+      if (at.line) path.line = at.line
+      clearTimeout(idle)
+      idle = setTimeout(flush, 400)
+      if (event.timeStamp - path.from >= 1000) flush()
+    }
+    window.addEventListener('pointermove', pointer, { passive: true })
+    return () => {
+      window.removeEventListener('pointermove', pointer)
+      flush()
+    }
+  }, [recording, recorder, operations])
+  useEffect(
+    () => () => {
+      void recorder?.stop()
+    },
+    [recorder],
+  )
+  function onUp(event: ReactPointerEvent) {
+    const start = stroke.current
+    stroke.current = undefined
+    if (!start || !tool) return
+    const box = start.element.getBoundingClientRect()
+    const clamp = (n: number) =>
+      Math.round(Math.max(0, Math.min(1, n)) * 1000) / 1000
+    const last = blockAt(event.clientX, event.clientY, {
+      skip: surface.current ?? undefined,
+      within: '.pm-present',
+    })
+    const lines =
+      start.line && last && last.key === start.key && last.line
+        ? linesText(
+            Array.from(
+              { length: Math.abs(last.line - start.line) + 1 },
+              (_, k) => Math.min(start.line!, last.line!) + k,
+            ),
+          )
+        : undefined
+    const data = {
+      block: start.key,
+      tool,
+      ...(lines
+        ? { lines }
+        : {
+            range: [
+              clamp(start.x),
+              clamp(start.y),
+              clamp((event.clientX - box.left) / box.width),
+              clamp((event.clientY - box.top) / box.height),
+            ],
+          }),
+    }
+    applyInk(data, false)
+    operations?.report({ type: 'ink', data })
+  }
+  function clear() {
+    const data = { clear: true }
+    applyInk(data, false)
+    operations?.report({ type: 'ink', data })
+  }
+  const status =
+    state === 'recording'
+      ? t.recorder.recording(clockText(elapsed))
+      : state === 'saving'
+        ? t.recorder.saving
+        : state === 'saved'
+          ? t.recorder.saved(recorder?.id ?? '')
+          : state === 'failed'
+            ? t.recorder.failed(failure ?? '')
+            : ''
+  return shown ? (
+    <div
+      className="pm-lecture-recorder"
+      role="region"
+      aria-label={t.recorder.region}
+      data-state={state}
+    >
+      {tool && (
+        <div
+          ref={surface}
+          className="pm-lecture-surface"
+          data-tool={tool}
+          onPointerDown={(event) => {
+            stroke.current = blockAt(event.clientX, event.clientY, {
+              skip: surface.current ?? undefined,
+              within: '.pm-present',
+            })
+          }}
+          onPointerUp={onUp}
+        />
+      )}
+      <div className="pm-lecture-recorder__row">
+        {!busy ? (
+          <>
+            {!!trackIds.length && (
+              <label className="pm-lecture-recorder__field">
+                {t.recorder.track}
+                <select
+                  value={trackId}
+                  onChange={(e) => setTrackId(e.target.value)}
+                  className="pm-ui-select"
+                >
+                  <option value="">{t.recorder.newTrack}</option>
+                  {trackIds.map((value) => (
+                    <option key={value} value={value}>
+                      {value}
+                    </option>
+                  ))}
+                </select>
+              </label>
+            )}
+            <label className="pm-lecture-recorder__field">
+              <input
+                checked={camera}
+                onChange={(e) => setCamera(e.target.checked)}
+                className="pm-ui-checkbox"
+                type="checkbox"
+              />{' '}
+              {t.recorder.camera}
+            </label>
+            <label className="pm-lecture-recorder__field">
+              <input
+                checked={screen}
+                onChange={(e) => setScreen(e.target.checked)}
+                className="pm-ui-checkbox"
+                type="checkbox"
+              />{' '}
+              {t.recorder.screen}
+            </label>
+            <button
+              type="button"
+              className={`${cls} pm-ui-button--primary pm-lecture-recorder__record`}
+              onClick={() => void start()}
+            >
+              {trackId ? t.recorder.rerecord : t.recorder.record}
+            </button>
+          </>
+        ) : recording ? (
+          <button
+            type="button"
+            className={`${cls} pm-ui-button--danger pm-lecture-recorder__stop`}
+            onClick={() => void stop()}
+          >
+            {t.recorder.stop}
+          </button>
+        ) : null}
+        <button
+          type="button"
+          className={cls}
+          aria-pressed={tool === 'laser'}
+          onClick={() =>
+            setTool((value) => (value === 'laser' ? undefined : 'laser'))
+          }
+        >
+          {t.recorder.laser}
+        </button>
+        <button
+          type="button"
+          className={cls}
+          aria-pressed={tool === 'marker'}
+          onClick={() =>
+            setTool((value) => (value === 'marker' ? undefined : 'marker'))
+          }
+        >
+          {t.recorder.marker}
+        </button>
+        <button type="button" className={cls} onClick={clear}>
+          {t.recorder.clear}
+        </button>
+      </div>
+      {status && (
+        <p className="pm-lecture-recorder__status" role="status">
+          {recording && (
+            <PmIcon className="pm-lecture-recorder__dot" name="record" />
+          )}
+          {status}
+        </p>
+      )}
+    </div>
+  ) : null
+}

package/src/client/TranscriptList.tsx

@@ -0,0 +1,28 @@
+import { PmIcon, useMessages } from '@lincyaw/host'
+import { clockText, LECTURE_MESSAGES, textPieces } from '../domain/index.ts'
+import type { LectureTrack, Utterance } from '../domain/index.ts'
+
+/** Published captions are a playback view. Authoring belongs to the document transaction API. */
+export default function TranscriptList({ track, language, current, stale, onSeek }: {
+  track: LectureTrack
+  language?: string
+  current?: string
+  stale: ReadonlySet<string>
+  onSeek(t: number): void
+}) {
+  const t = useMessages(LECTURE_MESSAGES)
+  const text = (utterance: Utterance) => language && language !== track.lang ? utterance.translations?.[language]?.text ?? utterance.text : utterance.text
+  return (
+    <ol className="pm-lecture-transcript" lang={language}>
+      {track.utterances.map(utterance => (
+        <li key={utterance.id} className="pm-lecture-transcript__item" data-utterance={utterance.id} data-stale={stale.has(utterance.id) ? 'true' : undefined} aria-current={current === utterance.id ? 'true' : undefined}>
+          <button type="button" className="pm-lecture-transcript__time" aria-label={t.player.jump(clockText(utterance.t))} onClick={() => onSeek(utterance.t)}>{clockText(utterance.t)}</button>
+          <span className="pm-lecture-transcript__text" onClick={() => onSeek(utterance.t)}>
+            {language === track.lang ? textPieces(utterance)?.map((piece, index) => piece.t !== undefined ? <span key={index} className="pm-lecture-word" onClick={event => { event.stopPropagation(); onSeek(piece.t!) }}>{piece.text}</span> : piece.text) ?? text(utterance) : text(utterance)}
+          </span>
+          {stale.has(utterance.id) && <span className="pm-lecture-stale" role="img" title={t.player.stale} aria-label={t.player.stale}><PmIcon name="warning" /></span>}
+        </li>
+      ))}
+    </ol>
+  )
+}