@lincyaw/addon-lecture 0.0.0-stage → 1.0.0-rc.1

package/src/domain/plan.ts

@@ -0,0 +1,319 @@
+import { irBlocks, parseDuration } from '@lincyaw/kernel'
+import type { CodeNode, IRBlock, IRDocument, IRSection, RecordedRun, SourcePos, SpeechVoice, TimeRange, VoiceTarget, WordSpan } from '@lincyaw/kernel'
+import { narrationVoice } from './config.ts'
+import { blockKey, codeLineCount, lastNamedLine } from './drift.ts'
+import type { CueCommand, CueMark, SpokenText } from './narration.ts'
+import { cueProblem, parseCue, spokenText, TARGETED } from './narration.ts'
+import { LECTURE_RENDERER, PLAYER_RENDERER } from './track.ts'
+
+export interface PlannedCue {
+  name: CueCommand
+  offset: number
+  data: Record<string, unknown>
+}
+
+export interface PlannedPiece {
+  id?: string
+  text: string
+  spoken: string
+  spans: WordSpan[]
+  cues: PlannedCue[]
+  pause?: number
+}
+
+export interface PlannedParagraph {
+  narrationId?: string
+  sourceKind?: 'none' | 'tts' | 'recording' | 'unset'
+  step: number
+  mode?: 'during' | 'after'
+  pieces: PlannedPiece[]
+  source?: SourcePos
+  voice?: SpeechVoice
+  voiceIssue?: string
+  voiceTarget?: VoiceTarget
+}
+
+export interface PlannedRecording {
+  id: string
+  step: number
+  stepId: string
+  assetId: string
+  sourceRange?: TimeRange
+  mode: 'during' | 'after'
+  text: string
+  narrationIds: string[]
+  cues: PlannedCue[]
+  issue?: 'recording-cue-unsupported' | 'recording-start-conflict'
+}
+
+export interface PlannedSection {
+  id: string
+  title: string
+  steps: number
+  from: 'narration'
+  paragraphs: PlannedParagraph[]
+  recordings?: PlannedRecording[]
+  holds: number[]
+  trailingPauseMs?: number[]
+  webStep(step: number): number
+}
+
+export interface PlanProblem {
+  message: string
+  source?: SourcePos
+  reportedAtCompile?: boolean
+}
+
+export interface LecturePlan {
+  sections: PlannedSection[]
+  problems: PlanProblem[]
+}
+
+function isSaid(block: IRBlock): boolean {
+  return !block.only || block.only.includes(LECTURE_RENDERER)
+}
+
+function isShown(block: IRBlock): boolean {
+  return !block.only || block.only.includes(PLAYER_RENDERER)
+}
+
+function named(doc: IRDocument, section: IRSection, to: string): { section: IRSection, block: IRBlock } | undefined {
+  const find = (s: IRSection) => [...irBlocks(s.blocks)].find(b => b.id === to && b.role === 'visual')
+  const own = find(section)
+  if (own) return { section, block: own }
+  for (const other of doc.sections) {
+    const block = find(other)
+    if (block) return { section: other, block }
+  }
+  return undefined
+}
+
+function webSteps(section: IRSection): (step: number) => number {
+  const map = section.stepMap
+  if (!map) return step => step
+  return (step) => {
+    const original = map.indexOf(step)
+    return original === -1 ? step : original
+  }
+}
+
+function cssString(value: string): string {
+  return `"${value.replace(/["\\]/g, '\\$&').replace(/\n/g, '\\a ')}"`
+}
+
+function cueData(name: CueCommand, arg: string, mark: CueMark, section: IRSection, block: IRBlock): Record<string, unknown> {
+  const key = blockKey(section.id, block.id)
+  const lines = mark.lines ? { lines: mark.lines } : {}
+  switch (name) {
+    case 'spot':
+    case 'hint':
+      return mark.sel ? { selector: `[data-section=${cssString(section.id)}] .pm-block[data-block=${cssString(block.id)}] ${mark.sel}` } : { block: key, ...lines }
+    case 'point': {
+      const first = mark.lines ? Number(/\d+/.exec(mark.lines)?.[0]) : undefined
+      return { block: key, path: [[0, 0.5, 0.5]], ...first ? { line: first } : {} }
+    }
+    case 'ink':
+      return mark.lines ? { block: key, tool: 'marker', lines: mark.lines } : { block: key, tool: 'marker', range: [0, 0, 1, 1] }
+    case 'click':
+      return { block: key, action: arg }
+    case 'run': {
+      const recorded = (block.node as CodeNode).outputs?.[0] as RecordedRun | undefined
+      return { section: section.id, block: block.id, result: recorded?.cacheKey ? `freeze:${recorded.cacheKey}` : 'live' }
+    }
+    default:
+      return {}
+  }
+}
+
+interface Reader {
+  doc: IRDocument
+  section: IRSection
+  target?: IRBlock
+  source?: SourcePos
+  problems: PlanProblem[]
+}
+
+function planCues(spoken: SpokenText, at: Reader): PlannedCue[] {
+  const cues: PlannedCue[] = []
+  const problem = (message: string, reportedAtCompile = false) => at.problems.push({ message: `section ${at.section.id}: ${message}`, source: at.source, ...reportedAtCompile ? { reportedAtCompile } : {} })
+  for (const mark of spoken.cues) {
+    const parsed = parseCue(mark.command)
+    if ('problem' in parsed) {
+      problem(cueProblem(mark.command, parsed.problem, at.doc.meta.lang), true)
+      continue
+    }
+    const { name, arg } = parsed
+    if (name === 'next') {
+      cues.push({ name, offset: mark.offset, data: {} })
+      continue
+    }
+    if (name === 'pause' || name === 'mark') {
+      cues.push({ name, offset: mark.offset, data: name === 'pause' ? { ms: parseDuration(arg)! } : { label: arg } })
+      continue
+    }
+    const found = mark.to ? named(at.doc, at.section, mark.to) : at.target && { section: at.section, block: at.target }
+    if (!found) {
+      problem(mark.to ? `{cue to=${mark.to}}\`${mark.command}\`: the deck has no block ${mark.to}` : `{cue}\`${mark.command}\` has no block before it to act on; name one with to=<label>`)
+      continue
+    }
+    if (!isShown(found.block)) {
+      problem(`{cue}\`${mark.command}\`: block ${found.block.id} is not shown in the web form, where the lecture plays`)
+      continue
+    }
+    const lines = codeLineCount(found.block)
+    const last = mark.lines ? lastNamedLine(mark.lines) : undefined
+    if (last !== undefined && lines !== undefined && last > lines) {
+      problem(`{cue}\`${mark.command}\` names lines ${mark.lines} of ${found.block.id}, which has ${lines}`)
+      continue
+    }
+    if (name === 'run' && found.block.node.type !== 'code') {
+      problem(`{cue}\`run\` on ${found.block.id}, which is no code block`)
+      continue
+    }
+    if (!TARGETED.has(name)) continue
+    cues.push({ name, offset: mark.offset, data: cueData(name, arg, mark, found.section, found.block) })
+  }
+  return cues
+}
+
+function pieces(spoken: SpokenText, cues: PlannedCue[]): PlannedPiece[] {
+  const list: PlannedPiece[] = []
+  let from = 0
+  let pending: PlannedCue[] = []
+  const cut = (to: number, pause?: number) => {
+    const text = spoken.text.slice(from, to)
+    const lead = text.length - text.trimStart().length
+    const start = from + lead
+    const end = from + text.trimEnd().length
+    const shift = (offset: number) => spoken.spans.filter(s => s.end <= offset).reduce((sum, s) => sum + (s.to - s.from) - (s.end - s.start), 0)
+    const spans = spoken.spans.filter(s => s.start >= start && s.end <= end)
+      .map(s => ({ from: s.from - (start + shift(start)), to: s.to - (start + shift(start)), start: s.start - start, end: s.end - start }))
+    list.push({
+      text: spoken.text.slice(start, end),
+      spoken: spoken.spoken.slice(start + shift(start), end + shift(end)),
+      spans,
+      cues: pending.map(c => ({ ...c, offset: Math.max(0, Math.min(c.offset - start, end - start)) })),
+      ...pause ? { pause } : {},
+    })
+    pending = []
+    from = to
+  }
+  for (const cue of cues) {
+    pending.push(cue)
+    if (cue.name === 'pause') cut(cue.offset, cue.data.ms as number)
+  }
+  if (from < spoken.text.length || pending.length > 0 || list.length === 0) cut(spoken.text.length)
+  return list.filter(piece => piece.text || piece.cues.length > 0 || piece.pause)
+}
+
+function narration(section: IRSection): { block: IRBlock, target?: IRBlock }[] {
+  const list: { block: IRBlock, target?: IRBlock }[] = []
+  const walk = (blocks: IRBlock[], target: IRBlock | undefined) => {
+    for (const block of blocks) {
+      if (block.role === 'visual') {
+        if (isShown(block)) target = block
+        if (block.node.type === 'component' && block.node.children) walk(block.node.children, target)
+      } else if (isSaid(block)) {
+        list.push({ block, target })
+      }
+    }
+  }
+  walk(section.blocks, undefined)
+  return list
+}
+
+export interface PlannedAnimation { blockId: string, step: number, durationMs?: number }
+
+/** Changes that start an animation; static built-ins do not invent an animation duration. */
+export function plannedAnimations(section: IRSection): PlannedAnimation[] {
+  const animations: PlannedAnimation[] = []
+  for (const block of irBlocks(section.blocks)) {
+    if (block.node.type !== 'component' || !isShown(block) || !block.node.durations && !(block.node.plugin === 'presmium.motion' && block.node.name === 'PmScene')) continue
+    const duration = (step: number, local: number) => {
+      const ms = block.node.type === 'component' ? block.node.durations?.[local] : undefined
+      animations.push({ blockId: block.id, step, ...typeof ms === 'number' && Number.isFinite(ms) && ms >= 0 ? { durationMs: ms } : {} })
+    }
+    if (block.stepStates) {
+      for (const [step, state] of block.stepStates.entries()) {
+        const previous = block.stepStates[step - 1]
+        if (!state.visible || previous?.visible && previous.localStep === state.localStep && previous.state === state.state && JSON.stringify(previous.parameters) === JSON.stringify(state.parameters)) continue
+        duration(step, state.localStep)
+      }
+    } else for (let local = 0; local <= (block.span ?? 0); local++) if (block.at + local <= section.steps) duration(block.at + local, local)
+  }
+  return animations
+}
+
+/** Effective minimum step holds: declared animations and authored Production stay durations. */
+function stepHolds(section: IRSection, doc: IRDocument): number[] {
+  const holds = Array.from({ length: section.steps + 1 }, () => 0)
+  for (const animation of plannedAnimations(section)) holds[animation.step] = Math.max(holds[animation.step]!, animation.durationMs ?? 0)
+  for (const [step, id] of (section.stepIds ?? []).entries()) {
+    const production = { ...doc.production?.defaults, ...doc.production?.sectionOverrides?.[section.id], ...doc.production?.stepOverrides[id] }
+    holds[step] = Math.max(holds[step] ?? 0, production.minimumHoldMs ?? 0)
+  }
+  return holds
+}
+
+export function lecturePlan(doc: IRDocument): LecturePlan {
+  const problems: PlanProblem[] = []
+  const sections: PlannedSection[] = doc.sections.map((section) => {
+    const paragraphs: PlannedParagraph[] = []
+    for (const { block, target } of narration(section)) {
+      if (block.node.type !== 'markdown') continue
+      const spoken = spokenText(block.node.mdast)
+      const cues = planCues(spoken, { doc, section, target, source: block.source, problems })
+      const voice = narrationVoice(doc, section, block.id, block.at)
+      const stepId = section.stepIds?.[block.at]
+      const source = (stepId ? doc.production?.stepOverrides[stepId]?.voiceSource : undefined) ?? doc.production?.sectionOverrides?.[section.id]?.voiceSource ?? doc.production?.defaults.voiceSource
+      const pointer = block.source.pointer
+      const narrationId = pointer?.startsWith('/narrations/') ? pointer.slice('/narrations/'.length).replaceAll('~1', '/').replaceAll('~0', '~') : undefined
+      paragraphs.push({ ...narrationId ? { narrationId } : {}, sourceKind: source?.type ?? (voice.voice ? 'tts' : 'unset'), step: Math.min(block.at, section.steps), mode: block.node.narration?.mode, pieces: pieces(spoken, cues), source: block.source, voice: voice.voice, voiceIssue: voice.issue, voiceTarget: voice.target })
+    }
+    paragraphs.sort((a, b) => a.step - b.step)
+    const settings = (section.stepIds ?? []).map(id => ({ ...doc.production?.defaults, ...doc.production?.sectionOverrides?.[section.id], ...doc.production?.stepOverrides[id] }))
+    const trailingPauseMs = settings.some(value => value.trailingPauseMs !== undefined) ? settings.map(value => value.trailingPauseMs ?? 500) : undefined
+    const recordings: PlannedRecording[] = settings.flatMap((setting, step) => {
+      if (setting.voiceSource?.type !== 'recording') return []
+      const own = paragraphs.filter(paragraph => paragraph.step === step), cues: PlannedCue[] = []
+      let text = false, issue: PlannedRecording['issue']
+      const mode = own[0]?.mode ?? (setting.narrationStart === 'after-animation' ? 'after' : 'during')
+      for (const paragraph of own) {
+        if ((paragraph.mode ?? 'during') !== mode) issue = 'recording-start-conflict'
+        for (const piece of paragraph.pieces) {
+          if (piece.pause || piece.cues.some(cue => cue.name === 'next' || cue.name === 'pause' || cue.offset > 0 || text)) issue = 'recording-cue-unsupported'
+          cues.push(...piece.cues)
+          if (piece.text.trim()) text = true
+        }
+      }
+      const stepId = section.stepIds![step]!
+      return [{ id: `recording:${stepId}`, step, stepId, assetId: setting.voiceSource.assetId, ...setting.voiceSource.sourceRange ? { sourceRange: { ...setting.voiceSource.sourceRange } } : {}, mode, text: own.flatMap(paragraph => paragraph.pieces.map(piece => piece.text)).filter(Boolean).join('\n\n'), narrationIds: [...new Set(own.flatMap(paragraph => paragraph.narrationId ?? []))], cues, ...issue ? { issue } : {} }]
+    })
+    return { id: section.id, title: section.title, steps: section.steps, from: 'narration', paragraphs, recordings, holds: stepHolds(section, doc), ...trailingPauseMs ? { trailingPauseMs } : {}, webStep: webSteps(section) }
+  })
+  let n = 0
+  for (const section of sections) {
+    for (const paragraph of section.paragraphs) for (const piece of paragraph.pieces) if (piece.text) piece.id = `u${++n}`
+  }
+  return { sections, problems }
+}
+
+/** The most words of a section that transcription takes as hints (an engine's hotwords, or Whisper's prompt). */
+const HINT_WORDS = 100
+
+/**
+ * The words a speaker is likely to say in a section, its title's and its narration's, as hints for the
+ * transcription of a recording: they tell homophones apart (趟 and 烫, 数 and 树).
+ */
+export function sectionHints(section: PlannedSection, lang: string): string[] {
+  const segmenter = new Intl.Segmenter(lang, { granularity: 'word' })
+  const text = [section.title, ...section.paragraphs.flatMap(paragraph => paragraph.pieces.map(piece => piece.text))].join('\n')
+  const words = [...segmenter.segment(text)].filter(part => part.isWordLike).map(part => part.segment)
+  return [...new Set(words)].slice(0, HINT_WORDS)
+}
+
+export function plannedUtterances(plan: LecturePlan): (PlannedPiece & { id: string, section: string, step: number, source?: SourcePos, narrationId?: string, sourceKind?: PlannedParagraph['sourceKind'], voice?: SpeechVoice, voiceIssue?: string, voiceTarget?: VoiceTarget })[] {
+  return plan.sections.flatMap(section => section.paragraphs.flatMap(paragraph => paragraph.pieces
+    .filter((piece): piece is PlannedPiece & { id: string } => Boolean(piece.id))
+    .map(piece => ({ ...piece, section: section.id, step: paragraph.step, narrationId: paragraph.narrationId, sourceKind: paragraph.sourceKind, voice: paragraph.voice, voiceIssue: paragraph.voiceIssue, voiceTarget: paragraph.voiceTarget, ...paragraph.source ? { source: paragraph.source } : {} }))))
+}

package/src/domain/production-audio.ts

@@ -0,0 +1,39 @@
+import type { IRDocument, ProductionAssetInspection, ProductionAudioClip, ProductionPage } from '@lincyaw/kernel'
+import { deckLocale } from '@lincyaw/kernel'
+
+export interface ProductionAudioIssue { code: string, sectionId?: string, stepId: string, assetId: string, mediaClipId: string }
+
+const messages: Record<string, { 'zh-CN': string, en: string }> = {
+  'production.asset-missing': { 'zh-CN': '找不到此音频资产声明。', en: 'This audio asset is not declared in the document.' },
+  'production.asset-kind': { 'zh-CN': '当前只支持本地音频；视频尚未支持。', en: 'Only local audio is supported; video is unavailable.' },
+  'production.asset-path': { 'zh-CN': '音频须为同文稿的非隐藏本地文件；远程或跨文稿素材尚未支持。', en: 'Audio must be a non-hidden local file in this document; remote and shared assets are unsupported.' },
+  'production.asset-file': { 'zh-CN': '音频文件不存在、不可读取或不是本地常规文件。', en: 'The audio is missing, unreadable, or not a local regular file.' },
+  'production.asset-size': { 'zh-CN': '音频文件须非空且不超过 16 MiB。', en: 'The audio file must be nonempty and at most 16 MiB.' },
+  'production.asset-count': { 'zh-CN': '本次制作最多引用 32 份音频资产。', en: 'A production can reference at most 32 audio assets.' },
+  'production.asset-decoded-size': { 'zh-CN': '引用音频的解码总量超过 256 MiB，请减少或拆分素材。', en: 'The referenced audio exceeds the 256 MiB decoded-data limit; reduce or split the assets.' },
+  'production.asset-format': { 'zh-CN': '此文件不是受支持的单音轨音频，或媒体数据已损坏。', en: 'The file is not supported single-track audio, or its media data is damaged.' },
+  'production.asset-duration': { 'zh-CN': '无法确认音频时长，或时长超过 10 分钟。', en: 'The audio duration is unconfirmed or exceeds ten minutes.' },
+  'production.asset-fingerprint': { 'zh-CN': '音频字节与声明的 SHA-256 不符，请检查或重新采用素材。', en: 'The audio bytes do not match the declared SHA-256; review or adopt the asset again.' },
+  'production.asset-changed': { 'zh-CN': '音频文件已变化，请重新准备当前编排。', en: 'The audio file changed; prepare the current sequence again.' },
+  'production.media-range': { 'zh-CN': '裁切出点须晚于入点且不超过实际音频时长。', en: 'The out point must follow the in point and stay within the actual audio duration.' },
+  'production.media-step-unknown': { 'zh-CN': '音频只能附着到时段已确认的步骤，请先修正配音或时序。', en: 'Audio requires a confirmed step range; resolve voiceover or timing first.' },
+  'production.media-step-range': { 'zh-CN': '音频片段超出本步时段；请调整裁切、偏移或本步停留，不会自动跨步。', en: 'The audio exceeds this step; adjust its range, offset, or step hold. It will not cross steps automatically.' },
+}
+export function productionAudioMessage(doc: IRDocument, code: string): string { return (messages[code] ?? messages['production.asset-format']!)[deckLocale(doc.meta.lang)] }
+
+/** Media never retimes the lecture. Every use must fit an existing confirmed step range. */
+export function placeProductionAudio(doc: IRDocument, pages: readonly ProductionPage[], assets: ReadonlyMap<string, ProductionAssetInspection>, errors: ReadonlyMap<string, string>): { clips: ProductionAudioClip[], issues: ProductionAudioIssue[] } {
+  const clips: ProductionAudioClip[] = [], issues: ProductionAudioIssue[] = []
+  for (const clip of doc.production?.mediaClips ?? []) {
+    const page = pages.find(page => page.steps.some(step => step.stepId === clip.attachment.stepId))
+    const step = page?.steps.find(step => step.stepId === clip.attachment.stepId), asset = assets.get(clip.assetId)
+    const issue = (code: string) => issues.push({ code, ...page ? { sectionId: page.sectionId } : {}, stepId: clip.attachment.stepId, assetId: clip.assetId, mediaClipId: clip.id })
+    if (!asset) { issue(errors.get(clip.assetId) ?? 'production.asset-missing'); continue }
+    if (clip.sourceRange.startMs < 0 || clip.sourceRange.endMs <= clip.sourceRange.startMs || clip.sourceRange.endMs > asset.durationMs) { issue('production.media-range'); continue }
+    if (!step || step.timing !== 'confirmed' || step.startMs === undefined || step.endMs === undefined) { issue('production.media-step-unknown'); continue }
+    const duration = clip.sourceRange.endMs - clip.sourceRange.startMs
+    if (clip.attachment.offsetMs < 0 || clip.attachment.offsetMs + duration > step.endMs - step.startMs) { issue('production.media-step-range'); continue }
+    clips.push({ id: clip.id, assetId: clip.assetId, mediaId: asset.mediaId, sectionId: page!.sectionId, stepId: step.stepId, startMs: step.startMs + clip.attachment.offsetMs, endMs: step.startMs + clip.attachment.offsetMs + duration, sourceRange: { ...clip.sourceRange }, volume: clip.volume, muted: clip.muted ?? false })
+  }
+  return { clips, issues }
+}

package/src/domain/production.ts

@@ -0,0 +1,136 @@
+import { deckLocale } from '@lincyaw/kernel'
+import type { IRDocument, ProductionAssetInspection, ProductionAudio, ProductionDiagnostic, ProductionPage, ProductionPlan, ProductionRange, ProductionSegment, ProductionTiming } from '@lincyaw/kernel'
+import { plannedAnimations, plannedUtterances } from './plan.ts'
+import type { LecturePlan } from './plan.ts'
+import { trackDuration } from './track.ts'
+import { captionTimingSources } from './caption-timing.ts'
+import { captionGroups } from './captions.ts'
+import type { LectureTrack } from './track.ts'
+
+const messages = {
+  'zh-CN': {
+    animation: '动画未声明有效时长，不能用于按时播放或导出。',
+    recording: '录音中的提示位置或讲稿起点尚无确认的时间；请使用不跨步、无内部停顿且起点一致的讲稿。',
+    recordingCaptions: '配音已裁切；字幕仍为本步完整讲稿，未按原音对齐，请核对。',
+    fingerprint: '采用此原音前须确认实际 SHA-256。',
+    media: '当前只支持同文稿本地音频；视频尚未支持。',
+    cue: '无配音讲稿中的文字位置提示没有确认的时间；请移到步骤起点或提供音频。',
+    next: '含跨步提示的同一段配音尚不能插入步骤结束停顿；请先拆为步骤绑定的讲稿。',
+    cache: '当前讲稿与音色缺少已生成的配音；显示的时长为估算，不能用于确认播放或导出。',
+    unset: '此讲稿尚未选择音频来源；请选择无配音或实际音色。',
+    step: (step: number) => `第 ${step} 步`,
+  },
+  en: {
+    animation: 'The animation has no valid declared duration; timed playback and export are unavailable.',
+    recording: 'Recording cues or script starts have no confirmed timing; use step-bound scripts without internal pauses and with one common start.',
+    recordingCaptions: 'The voiceover is trimmed. Captions still display the complete step script without audio alignment; review them.',
+    fingerprint: 'Confirm the actual SHA-256 before adopting this recording.',
+    media: 'Only local audio from this document is supported; video is unavailable.',
+    cue: 'Text-anchored cues without audio have no confirmed timing; move them to the step start or provide audio.',
+    next: 'A voiceover clip with a next-step cue can’t insert trailing step pauses yet; split it into step-bound scripts first.',
+    cache: 'No generated voiceover matches the current script and voice; the displayed duration is estimated and can’t confirm playback or export.',
+    unset: 'This script has no audio source; choose silence or an actual voice.',
+    step: (step: number) => `Step ${step}`,
+  },
+}
+export function productionMessages(doc: IRDocument) { return messages[deckLocale(doc.meta.lang)] }
+
+/** Capability findings shared by the renderer and the read-only production plan. */
+export function productionDiagnostics(doc: IRDocument, plan: LecturePlan): ProductionDiagnostic[] {
+  const result: ProductionDiagnostic[] = [], text = productionMessages(doc)
+  for (const clip of doc.production?.mediaClips ?? []) {
+    const section = doc.sections.find(section => section.stepIds?.includes(clip.attachment.stepId))
+    if (doc.productionAssets?.[clip.assetId]?.kind !== 'audio') result.push({ code: 'production.media-unsupported', severity: 'error', message: text.media, ...section ? { sectionId: section.id } : {}, stepId: clip.attachment.stepId })
+  }
+  for (const section of doc.sections) {
+    for (const animation of plannedAnimations(section)) if (animation.durationMs === undefined) result.push({ code: 'production.animation-duration-unknown', severity: 'error', message: text.animation, sectionId: section.id, stepId: section.stepIds?.[animation.step], blockId: animation.blockId })
+    const own = plan.sections.find(own => own.id === section.id)
+    if (!own) continue
+    if (own.trailingPauseMs && own.paragraphs.some(paragraph => paragraph.pieces.some(piece => piece.cues.some(cue => cue.name === 'next')))) result.push({ code: 'production.next-pause-unsupported', severity: 'error', message: text.next, sectionId: section.id })
+    for (const [index, stepId] of (section.stepIds ?? []).entries()) {
+      const source = doc.production?.stepOverrides[stepId]?.voiceSource ?? doc.production?.sectionOverrides?.[section.id]?.voiceSource ?? doc.production?.defaults.voiceSource
+      if (source?.type === 'recording') {
+        const recording = own.recordings?.find(recording => recording.stepId === stepId), asset = doc.productionAssets?.[source.assetId]
+        if (recording?.issue) result.push({ code: `production.${recording.issue}`, severity: 'error', message: text.recording, sectionId: section.id, stepId, assetId: source.assetId })
+        if (!asset?.fingerprint || !/^(?:sha256:)?[a-f\d]{64}$/i.test(asset.fingerprint)) result.push({ code: 'production.recording-fingerprint-required', severity: 'error', message: text.fingerprint, sectionId: section.id, stepId, assetId: source.assetId })
+      }
+      for (const paragraph of own.paragraphs.filter(paragraph => paragraph.step === index)) if (source?.type === 'none' && paragraph.pieces.some(piece => piece.cues.some(cue => cue.name !== 'pause' && cue.offset > 0))) result.push({ code: 'production.silent-cue-unknown', severity: 'error', message: text.cue, sectionId: section.id, stepId, narrationId: paragraph.narrationId })
+    }
+  }
+  return result
+}
+
+export function projectProductionPlan(doc: IRDocument, plan: LecturePlan, track: LectureTrack, audio: ReadonlyMap<string, ProductionAudio>, failures: readonly { id: string, section: string, text: string, reason: string }[], mediaDiagnostics: readonly ProductionDiagnostic[] = [], assets: ReadonlyMap<string, ProductionAssetInspection> = new Map()): Omit<ProductionPlan, 'binding'> {
+  const duration = trackDuration(track), units = plannedUtterances(plan), byId = new Map(units.map(unit => [unit.id, unit]))
+  const diagnostics: ProductionDiagnostic[] = [...productionDiagnostics(doc, plan), ...mediaDiagnostics]
+  const messages = productionMessages(doc)
+  for (const failure of failures) {
+    const unit = byId.get(failure.id), section = doc.sections.find(section => section.id === failure.section)
+    diagnostics.push({ code: unit?.sourceKind === 'unset' ? 'production.voice-unset' : 'production.cache-missing', severity: 'warning', message: unit?.sourceKind === 'unset' ? messages.unset : messages.cache, sectionId: failure.section, ...unit ? { stepId: section?.stepIds?.[unit.step], narrationId: unit.narrationId } : {} })
+  }
+  for (const finding of plan.problems) diagnostics.push({ code: 'production.cue-invalid', severity: 'warning', message: finding.message })
+  const starts = track.events.flatMap(([at, type, data]) => type === 'go' && typeof data.section === 'string' && typeof data.step === 'number' ? [{ at, section: data.section, step: data.step }] : [])
+  const startFor = (section: string, step: number) => starts.find(at => at.section === section && at.step === step)?.at ?? 0
+  const unknown = diagnostics.filter(issue => issue.severity === 'error')
+  const unknownAt = unknown.length ? Math.min(...unknown.map(issue => {
+    const section = doc.sections.find(section => section.id === issue.sectionId)
+    return section ? startFor(section.id, issue.stepId ? section.stepIds?.indexOf(issue.stepId) ?? 0 : 0) : 0
+  })) : Infinity
+  const estimatedAt = Math.min(Infinity, ...track.utterances.filter(utterance => utterance.unsaid).map(utterance => utterance.t))
+  const range = (start: number, end: number): ProductionRange => {
+    if (start >= unknownAt || end > unknownAt) return { ...start < unknownAt ? { startMs: start } : {}, timing: 'unknown' }
+    return { startMs: start, endMs: end, timing: end > estimatedAt || start >= estimatedAt ? 'estimated' : 'confirmed' }
+  }
+  const pages: ProductionPage[] = doc.sections.map(section => {
+    const planned = plan.sections.find(planned => planned.id === section.id), animations = plannedAnimations(section)
+    const steps = (section.stepIds ?? []).map((stepId, index) => {
+      const start = startFor(section.id, index), at = starts.findIndex(at => at.section === section.id && at.step === index)
+      const end = at < 0 ? start : starts[at + 1]?.at ?? duration
+      const own = animations.filter(animation => animation.step === index)
+      return { stepId, index, ...range(start, end), holdMs: planned?.holds[index] ?? 0, ...own.every(animation => animation.durationMs !== undefined) ? { animationMs: Math.max(0, ...own.map(animation => animation.durationMs!)) } : {} }
+    })
+    const start = startFor(section.id, 0), last = steps.at(-1)
+    return { sectionId: section.id, title: section.title, ...range(start, last?.endMs ?? duration), ...last?.endMs === undefined && last?.timing === 'unknown' ? { endMs: undefined, timing: 'unknown' as const } : {}, steps }
+  })
+  const captionSources = captionTimingSources(doc, track, pages)
+  for (const page of pages) for (const step of page.steps) step.captionTiming = captionSources.get(step.stepId)
+  const visual: ProductionSegment[] = pages.flatMap(page => page.steps.map(step => ({ id: `visual:${step.stepId}`, sectionId: page.sectionId, stepId: step.stepId, label: messages.step(step.index + 1), startMs: step.startMs, endMs: step.endMs, timing: step.timing })))
+  const voice: ProductionSegment[] = [], caption: ProductionSegment[] = []
+  for (const utterance of track.utterances) {
+    const section = doc.sections.find(section => section.id === utterance.section), stepId = section?.stepIds?.[utterance.step ?? 0]
+    if (!section || !stepId) continue
+    const unit = byId.get(utterance.id), file = utterance.clipId ? audio.get(utterance.clipId) : undefined, groups = captionGroups(utterance)
+    const covered = pages.find(page => page.sectionId === section.id)?.steps.filter(step => step.startMs !== undefined && step.endMs !== undefined && step.startMs < utterance.t + utterance.d && step.endMs > utterance.t).map(step => step.stepId) ?? []
+    const segment: ProductionSegment = {
+      id: utterance.id, sectionId: section.id, stepId, stepIds: [...new Set([stepId, ...covered])], ...unit ? { sourceStepId: section.stepIds?.[unit.step] } : {},
+      ...utterance.narrationIds ? { narrationIds: [...utterance.narrationIds] } : {}, utteranceId: utterance.id, label: utterance.text,
+      ...range(utterance.t, utterance.t + utterance.d), status: utterance.recording ? 'recorded' : file ? 'cached' : utterance.silent ? 'silent' : unit?.sourceKind === 'recording' || unit?.sourceKind === 'unset' ? 'blocked' : 'missing',
+      alignment: groups[0]?.alignment ?? 'none',
+    }
+    if (!utterance.recording) voice.push({ ...segment, id: `voice:${utterance.id}`, ...file ? { audio: file } : {} })
+    for (const group of track.captionOverrides?.some(own => own.section === utterance.section && own.step === utterance.step) ? [] : groups) {
+      const covered = pages.find(page => page.sectionId === section.id)?.steps.filter(step => step.startMs !== undefined && step.endMs !== undefined && step.startMs < group.t + group.d && step.endMs > group.t).map(step => step.stepId) ?? []
+      caption.push({ ...segment, id: `caption:${group.id}`, stepId: covered[0] ?? stepId, stepIds: covered.length ? covered : [stepId], label: group.text,
+        ...range(group.t, group.t + group.d), alignment: group.alignment })
+    }
+  }
+  for (const own of track.captionOverrides ?? []) for (const group of own.groups) caption.push({ id: `caption:${group.id}`, sectionId: own.section, stepId: own.stepId, stepIds: [own.stepId], label: group.text, alignment: 'manual', status: track.audio?.some(a => a.role === 'recording' && a.stepId === own.stepId) ? 'recorded' : 'cached', ...range(group.t, group.t + group.d) })
+  for (const segment of track.audio ?? []) {
+    if (segment.role === 'recording' && segment.assetId && segment.stepId) {
+      const media = assets.get(segment.assetId)
+      if (media) {
+        const sourceRange = { startMs: segment.sourceStartMs ?? 0, endMs: (segment.sourceStartMs ?? 0) + segment.duration * 1000 }
+        voice.push({ id: `voice:recording:${segment.stepId}`, sectionId: segment.section, stepId: segment.stepId, stepIds: [segment.stepId], narrationIds: segment.narrationIds ?? [], sourceStepId: segment.stepId, assetId: segment.assetId, media, label: doc.productionAssets?.[segment.assetId]?.title || segment.assetId, status: 'recorded', alignment: 'none', sourceRange, ...range(segment.start * 1000, (segment.start + segment.duration) * 1000) })
+        if ((sourceRange.startMs > 0 || sourceRange.endMs < media.durationMs) && track.utterances.some(utterance => utterance.recording && utterance.section === segment.section && utterance.t === segment.start * 1000)) diagnostics.push({ code: 'production.recording-captions-unverified', severity: 'warning', message: messages.recordingCaptions, sectionId: segment.section, stepId: segment.stepId, assetId: segment.assetId })
+      }
+      continue
+    }
+    if (!segment.assetId || !segment.mediaClipId) continue
+    const clip = doc.production?.mediaClips.find(clip => clip.id === segment.mediaClipId), media = assets.get(segment.assetId)
+    if (!clip || !media) continue
+    voice.push({ id: `media:${clip.id}`, sectionId: segment.section, stepId: clip.attachment.stepId, stepIds: [clip.attachment.stepId], mediaClipId: clip.id, assetId: clip.assetId, label: doc.productionAssets?.[clip.assetId]?.title || clip.assetId, media, sourceRange: { ...clip.sourceRange }, volume: clip.volume, muted: clip.muted ?? false, alignment: 'none', ...range(segment.start * 1000, (segment.start + segment.duration) * 1000) })
+  }
+  const timing: ProductionTiming = unknown.length ? 'unknown' : Number.isFinite(estimatedAt) ? 'estimated' : 'confirmed'
+  const ready = timing === 'confirmed' && failures.length === 0 && duration > 0
+  return { ...timing !== 'unknown' ? { durationMs: duration } : {}, timing, pages, tracks: [{ kind: 'visual', segments: visual }, { kind: 'voice', segments: voice }, { kind: 'caption', segments: caption }], events: timing === 'unknown' ? [] : track.events, diagnostics, canPreview: timing !== 'unknown' && duration > 0, canExport: ready }
+}

package/src/domain/recording.ts

@@ -0,0 +1,130 @@
+import type { TrackEvent } from '@lincyaw/kernel'
+import { blockOf } from './drift.ts'
+import { timedEvents } from './timing.ts'
+import type { LectureTrack, MediaSegment, TrackBlock, Utterance, VideoSegment, Voice } from './track.ts'
+import { sortEvents, TRACK_VERSION, trackDuration } from './track.ts'
+
+export type PathPoint = [dt: number, x: number, y: number]
+
+function distance([, x, y]: PathPoint, [, x1, y1]: PathPoint, [, x2, y2]: PathPoint): number {
+  const dx = x2 - x1
+  const dy = y2 - y1
+  const length = dx * dx + dy * dy
+  if (length === 0) return Math.hypot(x - x1, y - y1)
+  const k = Math.max(0, Math.min(1, ((x - x1) * dx + (y - y1) * dy) / length))
+  return Math.hypot(x - (x1 + k * dx), y - (y1 + k * dy))
+}
+
+export function simplifyPath(path: readonly PathPoint[], tolerance = 0.01): PathPoint[] {
+  const round = ([t, x, y]: PathPoint): PathPoint => [Math.round(t), Math.round(x * 1000) / 1000, Math.round(y * 1000) / 1000]
+  if (path.length <= 2) return path.map(round)
+  const keep = new Set([0, path.length - 1])
+  const split = (from: number, to: number) => {
+    let farthest = -1
+    let worst = tolerance
+    for (let i = from + 1; i < to; i++) {
+      const d = distance(path[i]!, path[from]!, path[to]!)
+      if (d > worst) {
+        worst = d
+        farthest = i
+      }
+    }
+    if (farthest === -1) return
+    keep.add(farthest)
+    split(from, farthest)
+    split(farthest, to)
+  }
+  split(0, path.length - 1)
+  return [...keep].sort((a, b) => a - b).map(i => round(path[i]!))
+}
+
+export interface Recording {
+  events: TrackEvent[]
+  audio: MediaSegment[]
+  video: VideoSegment[]
+  utterances: Utterance[]
+  duration: number
+}
+
+export interface TrackHead {
+  id: string
+  doc: string
+  lang: string
+  irHash: string
+  sections: Record<string, string>
+  voice: Voice
+  blocks: Record<string, TrackBlock>
+}
+
+export function recordedTrack(head: TrackHead, recording: Recording): LectureTrack {
+  return {
+    track: TRACK_VERSION,
+    id: head.id,
+    doc: head.doc,
+    lang: head.lang,
+    ir: { hash: head.irHash, sections: head.sections },
+    voice: head.voice,
+    ...recording.audio.length ? { audio: recording.audio } : {},
+    ...recording.video.length ? { video: recording.video } : {},
+    blocks: pruneBlocks(head.blocks, recording.events),
+    utterances: recording.utterances,
+    events: sortEvents(recording.events),
+  }
+}
+
+function pruneBlocks(blocks: Record<string, TrackBlock>, events: readonly TrackEvent[]): Record<string, TrackBlock> {
+  const used = new Set(events.flatMap(([, , data]) => blockOf(data) ?? []))
+  return Object.fromEntries(Object.entries(blocks).filter(([key]) => used.has(key)))
+}
+
+export function sectionSpan(track: LectureTrack, section: string, end: number): [number, number] | undefined {
+  const goes = timedEvents(track).filter(e => e.type === 'go' && typeof e.data.section === 'string')
+  const first = goes.findIndex(e => e.data.section === section)
+  if (first === -1) return undefined
+  const next = goes.slice(first + 1).find(e => e.data.section !== section)
+  return [goes[first]!.t, next?.t ?? end]
+}
+
+function splicedIr(track: LectureTrack, section: string, head: TrackHead): LectureTrack['ir'] {
+  const before = track.ir.sections ?? (track.ir.hash === head.irHash ? head.sections : {})
+  const unknown = Object.fromEntries(Object.keys(head.sections).filter(id => !(id in before)).map(id => [id, '']))
+  const sections = { ...unknown, ...before, ...section in head.sections ? { [section]: head.sections[section]! } : {} }
+  const current = Object.entries(sections).every(([id, hash]) => head.sections[id] === hash)
+  return { hash: current ? head.irHash : track.ir.hash, sections }
+}
+
+export function spliceSection(track: LectureTrack, section: string, recording: Recording, head: TrackHead): { track: LectureTrack, unused: string[] } {
+  const end = Math.round(trackDuration(track))
+  const [from, to] = sectionSpan(track, section, end) ?? [end, end]
+  const shift = recording.duration - (to - from)
+  const inside = (t: number) => t >= from && t < to
+  const moved = (t: number) => (t >= to ? t + shift : t)
+  const keepMedia = <T extends MediaSegment>(segments: T[] = []) => segments
+    .filter(s => !(s.section === section && inside(s.start * 1000)))
+    .map(s => ({ ...s, start: Math.round(moved(s.start * 1000)) / 1000 }))
+  const offset = <T extends MediaSegment>(segments: T[]) => segments.map(s => ({ ...s, start: Math.round(s.start * 1000 + from) / 1000 }))
+  const removed = new Set(track.utterances.filter(u => inside(u.t)).map(u => u.id))
+  const events: TrackEvent[] = [
+    ...track.events.filter(([t]) => !inside(t)).map(([t, type, data]): TrackEvent => [moved(t), type, data]),
+    ...recording.events.map(([t, type, data]): TrackEvent => [t + from, type, data]),
+  ]
+  const audio = [...keepMedia(track.audio), ...offset(recording.audio)].sort((a, b) => a.start - b.start)
+  const video = [...keepMedia(track.video), ...offset(recording.video)].sort((a, b) => a.start - b.start)
+  const kept = new Set([...audio, ...video].map(s => s.src))
+  const unused = [...track.audio ?? [], ...track.video ?? []].map(s => s.src).filter(src => !kept.has(src))
+  const next: LectureTrack = {
+    ...track,
+    ir: splicedIr(track, section, head),
+    ...audio.length ? { audio } : { audio: undefined },
+    ...video.length ? { video } : { video: undefined },
+    blocks: pruneBlocks({ ...track.blocks, ...head.blocks }, events),
+    utterances: [
+      ...track.utterances.filter(u => !removed.has(u.id)).map(u => ({ ...u, t: moved(u.t), words: u.words?.map(([t, d, w]): [number, number, string] => [moved(t), d, w]) })),
+      ...recording.utterances.map(u => ({ ...u, t: u.t + from, words: u.words?.map(([t, d, w]): [number, number, string] => [t + from, d, w]) })),
+    ].sort((a, b) => a.t - b.t),
+    events: sortEvents(events),
+  }
+  if (!next.audio) delete next.audio
+  if (!next.video) delete next.video
+  return { track: next, unused }
+}