@lincyaw/addon-lecture 0.0.0-stage → 1.0.0-rc.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +202 -0
- package/README.md +31 -2
- package/bin/presmium-lecture.mjs +12 -0
- package/package.json +68 -3
- package/src/client/ChapterList.tsx +50 -0
- package/src/client/LectureView.css +21 -0
- package/src/client/LectureView.tsx +84 -0
- package/src/client/PlayLectureButton.tsx +21 -0
- package/src/client/PlayerPanel.css +153 -0
- package/src/client/PlayerPanel.tsx +198 -0
- package/src/client/RecorderPanel.css +46 -0
- package/src/client/RecorderPanel.tsx +282 -0
- package/src/client/TranscriptList.tsx +28 -0
- package/src/client/clock.ts +179 -0
- package/src/client/env.d.ts +7 -0
- package/src/client/media.ts +86 -0
- package/src/client/playback.ts +140 -0
- package/src/client/player.ts +202 -0
- package/src/client/plugin.ts +33 -0
- package/src/client/pointer-ink.ts +143 -0
- package/src/client/production-playback.ts +188 -0
- package/src/client/recorder.ts +245 -0
- package/src/client/source.ts +109 -0
- package/src/client/state.ts +45 -0
- package/src/client/viewer.ts +37 -0
- package/src/domain/audience.ts +108 -0
- package/src/domain/caption-timing.ts +59 -0
- package/src/domain/captions.ts +59 -0
- package/src/domain/config.ts +67 -0
- package/src/domain/drift.ts +159 -0
- package/src/domain/editing.ts +112 -0
- package/src/domain/format.ts +13 -0
- package/src/domain/index.ts +50 -0
- package/src/domain/messages.ts +324 -0
- package/src/domain/narration.ts +160 -0
- package/src/domain/plan.ts +319 -0
- package/src/domain/production-audio.ts +39 -0
- package/src/domain/production.ts +136 -0
- package/src/domain/recording.ts +130 -0
- package/src/domain/script-track.ts +176 -0
- package/src/domain/timing.ts +107 -0
- package/src/domain/track.ts +302 -0
- package/src/domain/translation.ts +34 -0
- package/src/domain/vtt.ts +68 -0
- package/src/domain/words.ts +114 -0
- package/src/domain/writeback.ts +129 -0
- package/src/node/audio-authoring.ts +53 -0
- package/src/node/check.ts +93 -0
- package/src/node/cli.ts +82 -0
- package/src/node/clips.ts +10 -0
- package/src/node/engines.ts +43 -0
- package/src/node/export.ts +331 -0
- package/src/node/ffmpeg.ts +76 -0
- package/src/node/latest-ir.ts +54 -0
- package/src/node/plan.ts +35 -0
- package/src/node/presmium.ts +59 -0
- package/src/node/production-assets.ts +112 -0
- package/src/node/production.ts +102 -0
- package/src/node/renderer.ts +182 -0
- package/src/node/roles.ts +23 -0
- package/src/node/routes/production.ts +51 -0
- package/src/node/routes/recordings.ts +122 -0
- package/src/node/routes/serve-file.ts +38 -0
- package/src/node/routes/served-deck.ts +61 -0
- package/src/node/routes/server.ts +12 -0
- package/src/node/routes/tracks.ts +98 -0
- package/src/node/routes/voiceover.ts +23 -0
- package/src/node/store.ts +192 -0
- package/src/node/synthesize.ts +190 -0
- package/src/node/tools.ts +170 -0
- package/src/node/transcribe.ts +53 -0
- package/src/node/voice-generation.ts +15 -0
- package/src/node/voice-source.ts +34 -0
- package/src/node/voiceover-execution.ts +223 -0
|
@@ -0,0 +1,143 @@
|
|
|
1
|
+
import { blockElement, codeLinesBox } from '@lincyaw/host'
|
|
2
|
+
import type { PathPoint } from '../domain/index.ts'
|
|
3
|
+
|
|
4
|
+
const LASER_MS = 1500
|
|
5
|
+
|
|
6
|
+
interface Pointer {
|
|
7
|
+
block: string
|
|
8
|
+
path: PathPoint[]
|
|
9
|
+
from: number
|
|
10
|
+
animate: boolean
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
interface Ink {
|
|
14
|
+
block: string
|
|
15
|
+
tool: 'laser' | 'marker'
|
|
16
|
+
lines?: string
|
|
17
|
+
range?: [number, number, number, number]
|
|
18
|
+
until?: number
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
let pointer: Pointer | undefined
|
|
22
|
+
let inks: Ink[] = []
|
|
23
|
+
let layer: HTMLElement | undefined
|
|
24
|
+
let frame = 0
|
|
25
|
+
|
|
26
|
+
function path(value: unknown): PathPoint[] {
|
|
27
|
+
return Array.isArray(value)
|
|
28
|
+
? value.filter((p): p is PathPoint => Array.isArray(p) && p.length === 3 && p.every(n => typeof n === 'number'))
|
|
29
|
+
: []
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
function range(value: unknown): Ink['range'] {
|
|
33
|
+
return Array.isArray(value) && value.length === 4 && value.every(n => typeof n === 'number') ? value as Ink['range'] : undefined
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
function along(points: PathPoint[], elapsed: number): PathPoint {
|
|
37
|
+
const next = points.findIndex(([t]) => t > elapsed)
|
|
38
|
+
if (next <= 0) return next === 0 ? points[0]! : points.at(-1)!
|
|
39
|
+
const [t0, x0, y0] = points[next - 1]!
|
|
40
|
+
const [t1, x1, y1] = points[next]!
|
|
41
|
+
const k = (elapsed - t0) / Math.max(1, t1 - t0)
|
|
42
|
+
return [elapsed, x0 + (x1 - x0) * k, y0 + (y1 - y0) * k]
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
function ensureLayer(): HTMLElement {
|
|
46
|
+
if (layer?.isConnected) return layer
|
|
47
|
+
layer = document.createElement('div')
|
|
48
|
+
layer.className = 'pm-lecture-pointer-ink'
|
|
49
|
+
layer.setAttribute('aria-hidden', 'true')
|
|
50
|
+
Object.assign(layer.style, { position: 'fixed', inset: '0', pointerEvents: 'none', zIndex: 'calc(var(--pm-cue-z, 1) + 2)' })
|
|
51
|
+
document.body.append(layer)
|
|
52
|
+
return layer
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
function inkBox(ink: Ink, block: HTMLElement): DOMRect | undefined {
|
|
56
|
+
if (ink.lines) return codeLinesBox(block, ink.lines)
|
|
57
|
+
if (!ink.range) return undefined
|
|
58
|
+
const box = block.getBoundingClientRect()
|
|
59
|
+
const [x0, y0, x1, y1] = ink.range
|
|
60
|
+
return new DOMRect(box.left + Math.min(x0, x1) * box.width, box.top + Math.min(y0, y1) * box.height, Math.abs(x1 - x0) * box.width, Math.abs(y1 - y0) * box.height)
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
const STYLES: Record<string, Partial<CSSStyleDeclaration>> = {
|
|
64
|
+
marker: { background: 'rgb(250 204 21 / 0.35)', borderRadius: '3px', mixBlendMode: 'multiply' },
|
|
65
|
+
laser: { boxShadow: '0 0 0 3px rgb(239 68 68 / 0.85), 0 0 18px rgb(239 68 68 / 0.6)', borderRadius: '6px' },
|
|
66
|
+
pointer: { width: '16px', height: '16px', borderRadius: '50%', background: 'rgb(239 68 68 / 0.9)', boxShadow: '0 0 0 4px rgb(239 68 68 / 0.25)' },
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
function slot(root: HTMLElement, k: number, kind: string, block: string): HTMLElement {
|
|
70
|
+
let el = root.children[k] as HTMLElement | undefined
|
|
71
|
+
if (!el || el.dataset.kind !== kind) {
|
|
72
|
+
const made = document.createElement('div')
|
|
73
|
+
made.className = kind === 'pointer' ? 'pm-lecture-pointer' : `pm-lecture-ink pm-lecture-ink--${kind}`
|
|
74
|
+
made.dataset.kind = kind
|
|
75
|
+
Object.assign(made.style, { position: 'absolute', ...STYLES[kind] })
|
|
76
|
+
if (el) el.replaceWith(made)
|
|
77
|
+
else root.append(made)
|
|
78
|
+
el = made
|
|
79
|
+
}
|
|
80
|
+
el.dataset.block = block
|
|
81
|
+
return el
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
function draw(): void {
|
|
85
|
+
frame = 0
|
|
86
|
+
const now = performance.now()
|
|
87
|
+
inks = inks.filter(ink => ink.until === undefined || ink.until > now)
|
|
88
|
+
const root = pointer || inks.length > 0 ? ensureLayer() : layer
|
|
89
|
+
let k = 0
|
|
90
|
+
for (const ink of inks) {
|
|
91
|
+
const block = blockElement(ink.block)
|
|
92
|
+
const box = block && inkBox(ink, block)
|
|
93
|
+
if (!box || !root) continue
|
|
94
|
+
const el = slot(root, k++, ink.tool, ink.block)
|
|
95
|
+
Object.assign(el.style, { left: `${box.left}px`, top: `${box.top}px`, width: `${box.width}px`, height: `${box.height}px` })
|
|
96
|
+
if (ink.until !== undefined) el.style.opacity = String(Math.min(1, (ink.until - now) / 400))
|
|
97
|
+
}
|
|
98
|
+
const block = pointer && blockElement(pointer.block)
|
|
99
|
+
if (pointer && block && root) {
|
|
100
|
+
const [, x, y] = pointer.animate ? along(pointer.path, now - pointer.from) : pointer.path.at(-1)!
|
|
101
|
+
const box = block.getBoundingClientRect()
|
|
102
|
+
const dot = slot(root, k++, 'pointer', pointer.block)
|
|
103
|
+
Object.assign(dot.style, { left: `${box.left + x * box.width - 8}px`, top: `${box.top + y * box.height - 8}px` })
|
|
104
|
+
}
|
|
105
|
+
while (root && root.children.length > k) root.lastElementChild!.remove()
|
|
106
|
+
if (pointer || inks.length > 0) frame = requestAnimationFrame(draw)
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
function redraw(): void {
|
|
110
|
+
if (!frame) frame = requestAnimationFrame(draw)
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
export function applyPoint(data: Record<string, unknown>, seeking: boolean): void {
|
|
114
|
+
const points = path(data.path)
|
|
115
|
+
if (typeof data.block !== 'string' || points.length === 0) return
|
|
116
|
+
pointer = { block: data.block, path: points, from: performance.now(), animate: !seeking }
|
|
117
|
+
redraw()
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
export function applyInk(data: Record<string, unknown>, seeking: boolean): void {
|
|
121
|
+
if (data.clear === true) {
|
|
122
|
+
inks = []
|
|
123
|
+
redraw()
|
|
124
|
+
return
|
|
125
|
+
}
|
|
126
|
+
const tool = data.tool === 'laser' ? 'laser' : 'marker'
|
|
127
|
+
if (typeof data.block !== 'string' || (tool === 'laser' && seeking)) return
|
|
128
|
+
const ink: Ink = { block: data.block, tool, ...typeof data.lines === 'string' ? { lines: data.lines } : {}, ...range(data.range) ? { range: range(data.range) } : {} }
|
|
129
|
+
if (!ink.lines && !ink.range) return
|
|
130
|
+
if (tool === 'laser') ink.until = performance.now() + LASER_MS
|
|
131
|
+
inks.push(ink)
|
|
132
|
+
redraw()
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
export function clearPointer(): void {
|
|
136
|
+
pointer = undefined
|
|
137
|
+
redraw()
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
export function clearInks(): void {
|
|
141
|
+
inks = []
|
|
142
|
+
redraw()
|
|
143
|
+
}
|
|
@@ -0,0 +1,188 @@
|
|
|
1
|
+
import { createStore } from '@lincyaw/host'
|
|
2
|
+
import { PRODUCTION_PLAYBACK_RATES as PLAYBACK_RATES, sameProductionBinding } from '@lincyaw/kernel'
|
|
3
|
+
import { flushSync } from 'react-dom'
|
|
4
|
+
import type { ProductionPlayback, StepClock } from '@lincyaw/host'
|
|
5
|
+
import type { ProductionBinding, ProductionPlan, ProductionPlaybackCommand, ProductionPlaybackState } from '@lincyaw/kernel'
|
|
6
|
+
import type { LecturePlayer } from './player.ts'
|
|
7
|
+
|
|
8
|
+
export { PLAYBACK_RATES }
|
|
9
|
+
interface Loader { load(binding: ProductionBinding, signal: AbortSignal): Promise<void>, current(binding: ProductionBinding): boolean }
|
|
10
|
+
interface Driver { plan: ProductionPlan, player: LecturePlayer, clock?: StepClock }
|
|
11
|
+
|
|
12
|
+
/** Let newly mounted pictures register their preparation, then await the real shared step clock. */
|
|
13
|
+
export async function settleProductionPlayback(clock?: StepClock, signal?: AbortSignal): Promise<void> {
|
|
14
|
+
const frame = () => new Promise<void>((resolve, reject) => {
|
|
15
|
+
signal?.throwIfAborted()
|
|
16
|
+
const abort = () => { cancelAnimationFrame(id); reject(signal?.reason ?? new DOMException('Playback cancelled', 'AbortError')) }
|
|
17
|
+
const id = requestAnimationFrame(() => { signal?.removeEventListener('abort', abort); resolve() })
|
|
18
|
+
signal?.addEventListener('abort', abort, { once: true })
|
|
19
|
+
})
|
|
20
|
+
await frame(); signal?.throwIfAborted()
|
|
21
|
+
await clock?.settled(); signal?.throwIfAborted()
|
|
22
|
+
await frame(); signal?.throwIfAborted()
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
/** A versioned handle to one existing LecturePlayer, with no scheduler or independent playback clock. */
|
|
26
|
+
export class ProductionPlaybackController implements ProductionPlayback {
|
|
27
|
+
readonly version = 1 as const
|
|
28
|
+
readonly state = createStore<ProductionPlaybackState | undefined>(undefined)
|
|
29
|
+
readonly engaged = createStore(false)
|
|
30
|
+
#loader?: Loader
|
|
31
|
+
#driver?: Driver
|
|
32
|
+
#request?: AbortController
|
|
33
|
+
#command?: ProductionPlaybackCommand
|
|
34
|
+
#generation = 0
|
|
35
|
+
#stopped = false
|
|
36
|
+
#waiting = new Set<() => void>()
|
|
37
|
+
|
|
38
|
+
register(loader: Loader): () => void {
|
|
39
|
+
this.#loader = loader
|
|
40
|
+
for (const ready of [...this.#waiting]) ready()
|
|
41
|
+
return () => {
|
|
42
|
+
if (this.#loader !== loader) return
|
|
43
|
+
this.#loader = undefined; this.#request?.abort(); this.#driver?.player.pause()
|
|
44
|
+
const before = this.state.getSnapshot()
|
|
45
|
+
if (before) this.state.set({ binding: before.binding, status: 'unavailable', error: 'production-playback-unavailable' })
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
loading(binding: ProductionBinding): void {
|
|
50
|
+
this.state.set({ binding, status: 'loading' })
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
#publish(driver: Driver): void {
|
|
54
|
+
if (this.#stopped || this.#driver !== driver || !this.#loader?.current(driver.plan.binding)) return
|
|
55
|
+
const shown = this.state.getSnapshot()
|
|
56
|
+
if (shown && !sameProductionBinding(shown.binding, driver.plan.binding)) return
|
|
57
|
+
const { player, plan } = driver, error = player.clock.error.getSnapshot()
|
|
58
|
+
const at = player.time.getSnapshot(), cursor = player.cursor.getSnapshot()
|
|
59
|
+
const page = plan.pages.find(page => page.sectionId === cursor?.sectionId)
|
|
60
|
+
const step = page?.steps.find(step => step.index === cursor?.step)
|
|
61
|
+
this.state.set({ binding: plan.binding, status: error ? 'failed' : 'ready', position: {
|
|
62
|
+
timeMs: at, durationMs: player.duration, stepTimeMs: player.stepTime.getSnapshot(),
|
|
63
|
+
...page ? { sectionId: page.sectionId } : {}, ...step ? { stepId: step.stepId } : {},
|
|
64
|
+
playing: player.clock.playing.getSnapshot(), rate: player.clock.rate.getSnapshot(), trying: player.trying.getSnapshot(),
|
|
65
|
+
...player.waitingFor.getSnapshot() ? { waitingFor: player.waitingFor.getSnapshot() } : {},
|
|
66
|
+
}, ...error ? { error: 'production-playback-failed' } : {} })
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
attach(plan: ProductionPlan, player?: LecturePlayer, clock?: StepClock): () => void {
|
|
70
|
+
if (!player) {
|
|
71
|
+
this.#driver = undefined
|
|
72
|
+
this.state.set({ binding: plan.binding, status: plan.timing === 'estimated' ? 'estimated' : 'unavailable', error: plan.canPreview ? undefined : 'production-plan-unavailable' })
|
|
73
|
+
return () => {}
|
|
74
|
+
}
|
|
75
|
+
const driver = { plan, player, clock }
|
|
76
|
+
this.#driver = driver
|
|
77
|
+
let queued = false
|
|
78
|
+
const report = () => { if (!queued) { queued = true; queueMicrotask(() => { queued = false; this.#publish(driver) }) } }
|
|
79
|
+
const stops = [player.time.subscribe(report), player.stepTime.subscribe(report), player.cursor.subscribe(report), player.trying.subscribe(report), player.waitingFor.subscribe(report), player.clock.playing.subscribe(report), player.clock.rate.subscribe(report), player.clock.error.subscribe(report)]
|
|
80
|
+
this.#publish(driver)
|
|
81
|
+
return () => {
|
|
82
|
+
stops.forEach(stop => stop())
|
|
83
|
+
if (this.#driver !== driver) return
|
|
84
|
+
this.#driver = undefined
|
|
85
|
+
this.state.set({ binding: plan.binding, status: 'unavailable', error: 'production-playback-unavailable' })
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
async #ready(signal: AbortSignal): Promise<Loader> {
|
|
90
|
+
if (this.#loader) return this.#loader
|
|
91
|
+
return new Promise((resolve, reject) => {
|
|
92
|
+
const finish = () => {
|
|
93
|
+
if (!this.#waiting.delete(finish)) return
|
|
94
|
+
signal.removeEventListener('abort', finish)
|
|
95
|
+
signal.aborted || this.#stopped ? reject(new DOMException('Playback cancelled', 'AbortError')) : this.#loader ? resolve(this.#loader) : reject(new Error('production-playback-unavailable'))
|
|
96
|
+
}
|
|
97
|
+
this.#waiting.add(finish); signal.addEventListener('abort', finish, { once: true })
|
|
98
|
+
if (signal.aborted || this.#stopped) finish()
|
|
99
|
+
})
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
async control(command: ProductionPlaybackCommand, signal?: AbortSignal): Promise<ProductionPlaybackState> {
|
|
103
|
+
signal?.throwIfAborted()
|
|
104
|
+
if (this.#stopped) throw new Error('production-playback-unavailable')
|
|
105
|
+
if (this.#loader && !this.#loader.current(command.binding)) throw new Error('production-binding-stale')
|
|
106
|
+
const cancellingLoad = command.action === 'pause' && this.#request && this.#command
|
|
107
|
+
&& (this.#command.action === 'load' || this.state.getSnapshot()?.status === 'loading') && sameProductionBinding(this.#command.binding, command.binding)
|
|
108
|
+
if (command.action !== 'load' && command.action !== 'step' && !cancellingLoad && !sameProductionBinding(this.state.getSnapshot()?.binding ?? command.binding, command.binding)) throw new Error('production-binding-stale')
|
|
109
|
+
const request = new AbortController(), generation = ++this.#generation
|
|
110
|
+
this.#request?.abort(); this.#request = request; this.#command = command
|
|
111
|
+
const combined = signal ? AbortSignal.any([signal, request.signal]) : request.signal
|
|
112
|
+
let driver = this.#driver
|
|
113
|
+
const loads = command.action === 'load' || command.action === 'step'
|
|
114
|
+
&& (!driver || !sameProductionBinding(driver.plan.binding, command.binding) || this.state.getSnapshot()?.status !== 'ready')
|
|
115
|
+
const cancel = () => { if (driver && this.#driver === driver) driver.player.pause() }
|
|
116
|
+
combined.addEventListener('abort', cancel, { once: true })
|
|
117
|
+
const check = () => {
|
|
118
|
+
combined.throwIfAborted()
|
|
119
|
+
if (this.#stopped || generation !== this.#generation || !this.#loader?.current(command.binding)) throw new Error('production-binding-stale')
|
|
120
|
+
}
|
|
121
|
+
try {
|
|
122
|
+
if (command.action === 'load' && driver && sameProductionBinding(driver.plan.binding, command.binding) && this.state.getSnapshot()?.status === 'ready') {
|
|
123
|
+
this.engaged.set(true); driver.player.pause(); check(); this.#publish(driver)
|
|
124
|
+
return this.state.getSnapshot()!
|
|
125
|
+
}
|
|
126
|
+
if (cancellingLoad || command.action === 'pause' && !this.#loader) {
|
|
127
|
+
this.#driver?.player.pause()
|
|
128
|
+
const state: ProductionPlaybackState = { binding: command.binding, status: 'unavailable', error: 'production-playback-cancelled' }
|
|
129
|
+
this.state.set(state)
|
|
130
|
+
return state
|
|
131
|
+
}
|
|
132
|
+
if (loads) { this.engaged.set(true); this.loading(command.binding) }
|
|
133
|
+
const loader = await this.#ready(combined)
|
|
134
|
+
if (!loader.current(command.binding)) throw new Error('production-binding-stale')
|
|
135
|
+
if (loads) {
|
|
136
|
+
await loader.load(command.binding, combined); check()
|
|
137
|
+
if (!sameProductionBinding(this.state.getSnapshot()?.binding ?? command.binding, command.binding)) throw new Error('production-binding-stale')
|
|
138
|
+
}
|
|
139
|
+
if (command.action !== 'load') {
|
|
140
|
+
check()
|
|
141
|
+
driver = this.#driver
|
|
142
|
+
if (driver && !sameProductionBinding(driver.plan.binding, command.binding)) throw new Error('production-binding-stale')
|
|
143
|
+
if (command.action === 'pause') driver?.player.pause()
|
|
144
|
+
else {
|
|
145
|
+
if (!driver || !driver.plan.canPreview || this.state.getSnapshot()?.status !== 'ready') throw new Error('production-plan-unavailable')
|
|
146
|
+
if (command.action === 'seek') {
|
|
147
|
+
if (driver.plan.timing !== 'confirmed') throw new Error('production-plan-unavailable')
|
|
148
|
+
if (!Number.isFinite(command.timeMs) || command.timeMs < 0 || command.timeMs > driver.player.duration) throw new Error('production-playback-time')
|
|
149
|
+
flushSync(() => driver!.player.seek(command.timeMs))
|
|
150
|
+
} else if (command.action === 'step') {
|
|
151
|
+
const step = driver.plan.pages.find(page => page.sectionId === command.sectionId)?.steps.find(step => step.stepId === command.stepId)
|
|
152
|
+
if (!step || step.startMs === undefined) throw new Error('production-binding-stale')
|
|
153
|
+
const utterance = command.utteranceId ? driver.player.track.utterances.find(unit => unit.id === command.utteranceId && unit.section === command.sectionId && !unit.unsaid) : undefined
|
|
154
|
+
if (command.utteranceId && !utterance) throw new Error('production-binding-stale')
|
|
155
|
+
let selected = false
|
|
156
|
+
flushSync(() => {
|
|
157
|
+
if (utterance && utterance.step === step.index) { driver!.player.seek(utterance.t); selected = true }
|
|
158
|
+
else selected = driver!.player.seekStep(command.sectionId, step.index)
|
|
159
|
+
})
|
|
160
|
+
if (!selected) throw new Error('production-binding-stale')
|
|
161
|
+
} else if (command.action === 'rate') {
|
|
162
|
+
if (!PLAYBACK_RATES.some(rate => rate === command.rate)) throw new Error('production-playback-rate')
|
|
163
|
+
driver.player.setRate(command.rate)
|
|
164
|
+
} else if (!await driver.player.play()) throw new Error('production-playback-cancelled')
|
|
165
|
+
await settleProductionPlayback(driver.clock, combined); check()
|
|
166
|
+
}
|
|
167
|
+
if (driver) this.#publish(driver)
|
|
168
|
+
}
|
|
169
|
+
check()
|
|
170
|
+
const state = this.state.getSnapshot()
|
|
171
|
+
if (!state || !sameProductionBinding(state.binding, command.binding)) throw new Error('production-playback-unavailable')
|
|
172
|
+
return state
|
|
173
|
+
} catch (error) {
|
|
174
|
+
if (this.#request === request && (command.action === 'load' || loads && this.state.getSnapshot()?.status === 'loading')) this.state.set({ binding: command.binding, status: combined.aborted ? 'unavailable' : 'failed', error: combined.aborted ? 'production-playback-cancelled' : 'production-playback-failed' })
|
|
175
|
+
throw error
|
|
176
|
+
} finally {
|
|
177
|
+
combined.removeEventListener('abort', cancel)
|
|
178
|
+
if (this.#request === request) { this.#request = undefined; this.#command = undefined }
|
|
179
|
+
}
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
dispose(): void {
|
|
183
|
+
this.#stopped = true; this.#generation++; this.#request?.abort(); this.#driver?.player.pause()
|
|
184
|
+
for (const ready of [...this.#waiting]) ready()
|
|
185
|
+
this.#loader = undefined; this.#driver = undefined; this.engaged.set(false)
|
|
186
|
+
this.state.set(undefined)
|
|
187
|
+
}
|
|
188
|
+
}
|
|
@@ -0,0 +1,245 @@
|
|
|
1
|
+
import type { Operations, ServerClient } from '@lincyaw/host'
|
|
2
|
+
import type { IRDocument, Operation, TrackEvent } from '@lincyaw/kernel'
|
|
3
|
+
import { createStore } from '@lincyaw/host'
|
|
4
|
+
import { irHash, LECTURE_ID, referencedBlocks, sectionHashes } from '../domain/index.ts'
|
|
5
|
+
import type { MediaSegment, Recording, VideoSegment } from '../domain/index.ts'
|
|
6
|
+
import type { Capture, Chunk } from './media.ts'
|
|
7
|
+
import { captureAudio, captureVideo } from './media.ts'
|
|
8
|
+
import { readAnswer } from './source.ts'
|
|
9
|
+
|
|
10
|
+
const UPLOAD_MS = 5000
|
|
11
|
+
|
|
12
|
+
export type RecorderState = 'starting' | 'recording' | 'saving' | 'saved' | 'failed'
|
|
13
|
+
|
|
14
|
+
export interface RecorderOptions {
|
|
15
|
+
server: ServerClient
|
|
16
|
+
operations: Operations
|
|
17
|
+
doc: () => IRDocument
|
|
18
|
+
position: () => { section: string, step: number }
|
|
19
|
+
speaker?: string
|
|
20
|
+
camera?: boolean
|
|
21
|
+
screen?: boolean
|
|
22
|
+
track?: string
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
type Source = 'audio' | 'camera' | 'screen'
|
|
26
|
+
|
|
27
|
+
interface Segment {
|
|
28
|
+
source: Source
|
|
29
|
+
section: string
|
|
30
|
+
start: number
|
|
31
|
+
end?: number
|
|
32
|
+
capture: Capture
|
|
33
|
+
src: Promise<string>
|
|
34
|
+
pending: Chunk[]
|
|
35
|
+
stopped?: Promise<void>
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
export class LectureRecorder {
|
|
39
|
+
readonly state = createStore<RecorderState>('starting')
|
|
40
|
+
readonly error = createStore<string | undefined>(undefined)
|
|
41
|
+
readonly elapsed = createStore(0)
|
|
42
|
+
id?: string
|
|
43
|
+
readonly rerecordSection?: string
|
|
44
|
+
private current?: string
|
|
45
|
+
|
|
46
|
+
private readonly options: RecorderOptions
|
|
47
|
+
private readonly events: TrackEvent[] = []
|
|
48
|
+
private readonly paths = new WeakMap<object, number>()
|
|
49
|
+
private readonly segments: Segment[] = []
|
|
50
|
+
private streams: Partial<Record<Source, MediaStream>> = {}
|
|
51
|
+
private t0 = 0
|
|
52
|
+
private timers: ReturnType<typeof setInterval>[] = []
|
|
53
|
+
private unsubscribe?: () => void
|
|
54
|
+
private uploading = Promise.resolve()
|
|
55
|
+
private rotating = Promise.resolve()
|
|
56
|
+
private stopping?: Promise<void>
|
|
57
|
+
|
|
58
|
+
constructor(options: RecorderOptions) {
|
|
59
|
+
this.options = options
|
|
60
|
+
if (options.track) this.rerecordSection = options.position().section
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
now(): number {
|
|
64
|
+
return Math.round(performance.now() - this.t0)
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
async start(): Promise<void> {
|
|
68
|
+
try {
|
|
69
|
+
if (this.options.screen) this.streams.screen = await navigator.mediaDevices.getDisplayMedia({ video: true })
|
|
70
|
+
this.streams.audio = await navigator.mediaDevices.getUserMedia({ audio: { echoCancellation: true, noiseSuppression: true, autoGainControl: true } })
|
|
71
|
+
if (this.options.camera) this.streams.camera = await navigator.mediaDevices.getUserMedia({ video: true })
|
|
72
|
+
const { id } = await readAnswer<{ id: string }>(this.post('recordings', { track: this.options.track }))
|
|
73
|
+
this.id = id
|
|
74
|
+
this.t0 = performance.now()
|
|
75
|
+
const position = this.options.position()
|
|
76
|
+
this.current = position.section
|
|
77
|
+
this.events.push([0, 'go', { section: position.section, step: position.step, view: 'present' }])
|
|
78
|
+
await this.open(position.section)
|
|
79
|
+
this.unsubscribe = this.options.operations.subscribe(operation => this.record(operation))
|
|
80
|
+
this.timers.push(setInterval(() => void this.upload(), UPLOAD_MS), setInterval(() => this.elapsed.set(this.now()), 250))
|
|
81
|
+
this.state.set('recording')
|
|
82
|
+
} catch (error) {
|
|
83
|
+
this.fail(error)
|
|
84
|
+
this.release()
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
notePointerPathStart(data: Record<string, unknown>, at: number): void {
|
|
89
|
+
this.paths.set(data, at)
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
stop(): Promise<void> {
|
|
93
|
+
this.stopping ??= this.finish()
|
|
94
|
+
return this.stopping
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
private post(path: string, body: unknown): Promise<Response> {
|
|
98
|
+
return this.options.server.fetch(LECTURE_ID, path, { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(body) })
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
private record({ type, data }: Operation): void {
|
|
102
|
+
if (this.state.getSnapshot() !== 'recording') return
|
|
103
|
+
const began = this.paths.get(data)
|
|
104
|
+
const path = type === 'point' && Array.isArray(data.path) ? data.path as [number, number, number][] : undefined
|
|
105
|
+
const t = Math.max(0, Math.round(began !== undefined ? began - this.t0 : this.now() - (path?.at(-1)?.[0] ?? 0)))
|
|
106
|
+
if (type === 'go' && typeof data.section === 'string' && data.section !== this.current) {
|
|
107
|
+
if (this.rerecordSection) {
|
|
108
|
+
void this.stop()
|
|
109
|
+
return
|
|
110
|
+
}
|
|
111
|
+
this.current = data.section
|
|
112
|
+
this.events.push([t, type, data])
|
|
113
|
+
const section = data.section
|
|
114
|
+
this.rotating = this.rotating.then(() => this.rotate(section)).catch(error => this.fail(error))
|
|
115
|
+
return
|
|
116
|
+
}
|
|
117
|
+
this.events.push([t, type, data])
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
private async open(section: string): Promise<void> {
|
|
121
|
+
const start = this.now()
|
|
122
|
+
const sources: Source[] = ['audio', ...this.streams.camera ? ['camera' as const] : [], ...this.streams.screen ? ['screen' as const] : []]
|
|
123
|
+
for (const source of sources) {
|
|
124
|
+
const stream = this.streams[source]!
|
|
125
|
+
const track = source === 'audio' ? stream.getAudioTracks()[0] : stream.getVideoTracks()[0]
|
|
126
|
+
if (!track) continue
|
|
127
|
+
const segment = { source, section, start, pending: [] as Chunk[] } as Omit<Segment, 'capture' | 'src'> as Segment
|
|
128
|
+
const capture = await (source === 'audio' ? captureAudio : captureVideo)(track, chunk => segment.pending.push(chunk))
|
|
129
|
+
if (!capture) {
|
|
130
|
+
if (source === 'audio') throw new Error('no audio encoder')
|
|
131
|
+
continue
|
|
132
|
+
}
|
|
133
|
+
capture.failed.catch(error => this.fail(error))
|
|
134
|
+
segment.capture = capture
|
|
135
|
+
segment.src = readAnswer<{ src: string }>(this.post(`recordings/${this.id}/media`, {
|
|
136
|
+
folder: source === 'audio' ? 'audio' : 'video',
|
|
137
|
+
section,
|
|
138
|
+
...source === 'audio' ? {} : { kind: source },
|
|
139
|
+
ext: capture.ext,
|
|
140
|
+
})).then(({ src }) => src)
|
|
141
|
+
segment.src.catch(error => this.fail(error))
|
|
142
|
+
this.segments.push(segment)
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
private close(segment: Segment): Promise<void> {
|
|
147
|
+
segment.end ??= this.now()
|
|
148
|
+
segment.stopped ??= segment.capture.stop()
|
|
149
|
+
return segment.stopped
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
private async rotate(section: string): Promise<void> {
|
|
153
|
+
const open = this.segments.filter(s => s.end === undefined)
|
|
154
|
+
open.forEach(s => s.end = this.now())
|
|
155
|
+
await this.open(section)
|
|
156
|
+
await Promise.all(open.map(s => this.close(s)))
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
private upload(): Promise<void> {
|
|
160
|
+
this.uploading = this.uploading.then(async () => {
|
|
161
|
+
for (const segment of this.segments) {
|
|
162
|
+
const chunks = segment.pending.splice(0)
|
|
163
|
+
if (chunks.length === 0) continue
|
|
164
|
+
const src = await segment.src
|
|
165
|
+
const runs: Chunk[] = []
|
|
166
|
+
for (const chunk of chunks) {
|
|
167
|
+
const last = runs.at(-1)
|
|
168
|
+
if (last && last.position + last.data.byteLength === chunk.position) {
|
|
169
|
+
const joined = new Uint8Array(last.data.byteLength + chunk.data.byteLength)
|
|
170
|
+
joined.set(last.data)
|
|
171
|
+
joined.set(chunk.data, last.data.byteLength)
|
|
172
|
+
runs[runs.length - 1] = { data: joined, position: last.position }
|
|
173
|
+
} else {
|
|
174
|
+
runs.push(chunk)
|
|
175
|
+
}
|
|
176
|
+
}
|
|
177
|
+
for (const run of runs) {
|
|
178
|
+
await readAnswer(this.options.server.fetch(LECTURE_ID, `recordings/${this.id}/files/${src.split('/').map(encodeURIComponent).join('/')}?at=${run.position}`, {
|
|
179
|
+
method: 'PUT',
|
|
180
|
+
body: run.data,
|
|
181
|
+
}))
|
|
182
|
+
}
|
|
183
|
+
}
|
|
184
|
+
}).catch(error => this.fail(error))
|
|
185
|
+
return this.uploading
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
private async finish(): Promise<void> {
|
|
189
|
+
if (this.state.getSnapshot() !== 'recording') {
|
|
190
|
+
this.release()
|
|
191
|
+
return
|
|
192
|
+
}
|
|
193
|
+
this.state.set('saving')
|
|
194
|
+
this.unsubscribe?.()
|
|
195
|
+
const duration = this.now()
|
|
196
|
+
try {
|
|
197
|
+
await this.rotating
|
|
198
|
+
await Promise.all(this.segments.map(s => this.close(s)))
|
|
199
|
+
await this.upload()
|
|
200
|
+
if (this.failed()) return
|
|
201
|
+
const media = async (s: Segment) => ({ section: s.section, start: s.start / 1000, src: await s.src, duration: (s.end! - s.start) / 1000 })
|
|
202
|
+
const recording: Recording = {
|
|
203
|
+
events: this.events,
|
|
204
|
+
audio: await Promise.all(this.segments.filter(s => s.source === 'audio').map(media)) as MediaSegment[],
|
|
205
|
+
video: await Promise.all(this.segments.filter(s => s.source !== 'audio').map(async s => ({ ...await media(s), kind: s.source }))) as VideoSegment[],
|
|
206
|
+
utterances: [],
|
|
207
|
+
duration,
|
|
208
|
+
}
|
|
209
|
+
const doc = this.options.doc()
|
|
210
|
+
const head = {
|
|
211
|
+
doc: doc.meta.entry,
|
|
212
|
+
lang: doc.meta.lang ?? 'zh-CN',
|
|
213
|
+
irHash: irHash(doc),
|
|
214
|
+
sections: sectionHashes(doc),
|
|
215
|
+
voice: { kind: 'recorded', ...this.options.speaker ? { speaker: this.options.speaker } : {} },
|
|
216
|
+
blocks: referencedBlocks(doc, this.events),
|
|
217
|
+
}
|
|
218
|
+
await readAnswer(this.post(`recordings/${this.id}/finish`, { head, recording, ...this.rerecordSection ? { section: this.rerecordSection } : {} }))
|
|
219
|
+
this.state.set('saved')
|
|
220
|
+
} catch (error) {
|
|
221
|
+
this.fail(error)
|
|
222
|
+
} finally {
|
|
223
|
+
this.release()
|
|
224
|
+
}
|
|
225
|
+
}
|
|
226
|
+
|
|
227
|
+
private failed(): boolean {
|
|
228
|
+
return this.state.getSnapshot() === 'failed'
|
|
229
|
+
}
|
|
230
|
+
|
|
231
|
+
private fail(error: unknown): void {
|
|
232
|
+
if (this.state.getSnapshot() === 'failed') return
|
|
233
|
+
console.error('[presmium] lecture recording failed', error)
|
|
234
|
+
this.error.set(error instanceof Error ? error.message : String(error))
|
|
235
|
+
this.state.set('failed')
|
|
236
|
+
}
|
|
237
|
+
|
|
238
|
+
private release(): void {
|
|
239
|
+
this.timers.forEach(clearInterval)
|
|
240
|
+
this.timers = []
|
|
241
|
+
this.unsubscribe?.()
|
|
242
|
+
for (const stream of Object.values(this.streams)) stream?.getTracks().forEach(track => track.stop())
|
|
243
|
+
this.streams = {}
|
|
244
|
+
}
|
|
245
|
+
}
|