@lincyaw/addon-lecture 0.0.0-stage → 1.0.0-rc.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +202 -0
- package/README.md +31 -2
- package/bin/presmium-lecture.mjs +12 -0
- package/package.json +68 -3
- package/src/client/ChapterList.tsx +50 -0
- package/src/client/LectureView.css +21 -0
- package/src/client/LectureView.tsx +84 -0
- package/src/client/PlayLectureButton.tsx +21 -0
- package/src/client/PlayerPanel.css +120 -0
- package/src/client/PlayerPanel.tsx +194 -0
- package/src/client/RecorderPanel.css +46 -0
- package/src/client/RecorderPanel.tsx +282 -0
- package/src/client/TranscriptList.tsx +28 -0
- package/src/client/clock.ts +169 -0
- package/src/client/env.d.ts +7 -0
- package/src/client/media.ts +86 -0
- package/src/client/playback.ts +140 -0
- package/src/client/player.ts +202 -0
- package/src/client/plugin.ts +33 -0
- package/src/client/pointer-ink.ts +143 -0
- package/src/client/production-playback.ts +188 -0
- package/src/client/recorder.ts +245 -0
- package/src/client/source.ts +109 -0
- package/src/client/state.ts +45 -0
- package/src/client/viewer.ts +37 -0
- package/src/domain/audience.ts +108 -0
- package/src/domain/caption-timing.ts +59 -0
- package/src/domain/captions.ts +59 -0
- package/src/domain/config.ts +67 -0
- package/src/domain/drift.ts +159 -0
- package/src/domain/editing.ts +112 -0
- package/src/domain/format.ts +13 -0
- package/src/domain/index.ts +50 -0
- package/src/domain/messages.ts +324 -0
- package/src/domain/narration.ts +160 -0
- package/src/domain/plan.ts +319 -0
- package/src/domain/production-audio.ts +39 -0
- package/src/domain/production.ts +136 -0
- package/src/domain/recording.ts +130 -0
- package/src/domain/script-track.ts +176 -0
- package/src/domain/timing.ts +107 -0
- package/src/domain/track.ts +302 -0
- package/src/domain/translation.ts +34 -0
- package/src/domain/vtt.ts +68 -0
- package/src/domain/words.ts +114 -0
- package/src/domain/writeback.ts +129 -0
- package/src/node/audio-authoring.ts +53 -0
- package/src/node/check.ts +93 -0
- package/src/node/cli.ts +82 -0
- package/src/node/clips.ts +10 -0
- package/src/node/engines.ts +43 -0
- package/src/node/export.ts +331 -0
- package/src/node/ffmpeg.ts +76 -0
- package/src/node/latest-ir.ts +54 -0
- package/src/node/plan.ts +35 -0
- package/src/node/presmium.ts +59 -0
- package/src/node/production-assets.ts +112 -0
- package/src/node/production.ts +102 -0
- package/src/node/renderer.ts +182 -0
- package/src/node/roles.ts +23 -0
- package/src/node/routes/production.ts +51 -0
- package/src/node/routes/recordings.ts +122 -0
- package/src/node/routes/serve-file.ts +38 -0
- package/src/node/routes/served-deck.ts +61 -0
- package/src/node/routes/server.ts +12 -0
- package/src/node/routes/tracks.ts +98 -0
- package/src/node/routes/voiceover.ts +23 -0
- package/src/node/store.ts +192 -0
- package/src/node/synthesize.ts +190 -0
- package/src/node/tools.ts +170 -0
- package/src/node/transcribe.ts +53 -0
- package/src/node/voice-generation.ts +15 -0
- package/src/node/voice-source.ts +34 -0
- package/src/node/voiceover-execution.ts +223 -0
|
@@ -0,0 +1,170 @@
|
|
|
1
|
+
import { isAbsolute, relative, resolve } from 'node:path'
|
|
2
|
+
import { deckLocale, defineService, DOCUMENT_FILES, SPEECH } from '@lincyaw/kernel'
|
|
3
|
+
import type { AgentTool, AgentToolContext, IRDocument, SourceRange, SpeechService } from '@lincyaw/kernel'
|
|
4
|
+
import { LECTURES_DIR, LECTURE_MESSAGES, LECTURE_TOOL_ERRORS, narrationDraft, narrationSource, SCRIPT_TRACK, trackDuration, translationItems } from '../domain/index.ts'
|
|
5
|
+
import type { LecturePlan, LectureTrack } from '../domain/index.ts'
|
|
6
|
+
import { latestView } from './latest-ir.ts'
|
|
7
|
+
import { deckPlan } from './plan.ts'
|
|
8
|
+
import { deckFiles, readTrack, TRACK_FILE, trackDir } from './store.ts'
|
|
9
|
+
import type { DeckFiles } from './store.ts'
|
|
10
|
+
import { synthesisFailureMessage, synthesizeLecture } from './synthesize.ts'
|
|
11
|
+
import { assertVoiceSource, VoiceSourceRefusal } from './voice-source.ts'
|
|
12
|
+
import { claimVoiceGeneration } from './voice-generation.ts'
|
|
13
|
+
import { transcribeLecture } from './transcribe.ts'
|
|
14
|
+
|
|
15
|
+
export interface LectureDeck {
|
|
16
|
+
deckDir: string
|
|
17
|
+
workspaceRoot: string
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
export const LECTURE_DECK = defineService<LectureDeck>('presmium.lecture.deck')
|
|
21
|
+
|
|
22
|
+
function deckOf(ctx: AgentToolContext): { deckDir: string, doc: IRDocument, files: DeckFiles } {
|
|
23
|
+
const deck = ctx.service(LECTURE_DECK)
|
|
24
|
+
if (!deck) throw new Error('lecture-plugin-missing')
|
|
25
|
+
const doc = latestView(deck.deckDir)
|
|
26
|
+
if (!doc) throw new Error('document-not-ready')
|
|
27
|
+
return { deckDir: deck.deckDir, doc, files: deckFiles(deck.workspaceRoot, () => ctx.service(DOCUMENT_FILES)) }
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
function speechOf(ctx: AgentToolContext): SpeechService {
|
|
31
|
+
const speech = ctx.service(SPEECH)
|
|
32
|
+
if (!speech) throw new Error('speech-unavailable')
|
|
33
|
+
return speech
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
function planView(plan: LecturePlan, section?: string) {
|
|
37
|
+
return plan.sections.filter(s => !section || s.id === section).map(s => ({
|
|
38
|
+
section: s.id,
|
|
39
|
+
title: s.title,
|
|
40
|
+
from: s.from,
|
|
41
|
+
steps: s.steps,
|
|
42
|
+
paragraphs: s.paragraphs.map(p => ({
|
|
43
|
+
step: p.step,
|
|
44
|
+
said: p.pieces.map(piece => ({
|
|
45
|
+
...piece.id ? { utterance: piece.id } : {},
|
|
46
|
+
text: piece.text,
|
|
47
|
+
...piece.spoken !== piece.text ? { spoken: piece.spoken } : {},
|
|
48
|
+
cues: piece.cues.map(c => ({ cue: c.name, before: piece.text.slice(c.offset, c.offset + 8) || '(end)', ...c.data })),
|
|
49
|
+
...piece.pause ? { pauseMs: piece.pause } : {},
|
|
50
|
+
})),
|
|
51
|
+
})),
|
|
52
|
+
}))
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
export const lecturePreview: AgentTool = {
|
|
56
|
+
name: 'lecture_preview',
|
|
57
|
+
description: 'The lecture plan of the deck: per section and step, the canonical lecture.json narration the synthesized lecture says and the {cue} marks in it with the operation each makes; the voice; and problems (unknown cues, missing blocks). Reads only; nothing is synthesized.',
|
|
58
|
+
input: { type: 'object', properties: { section: { type: 'string', description: 'Only this section id' } } },
|
|
59
|
+
kind: 'read',
|
|
60
|
+
async run(input, ctx) {
|
|
61
|
+
const { doc } = deckOf(ctx)
|
|
62
|
+
const deck = deckPlan(doc)
|
|
63
|
+
return {
|
|
64
|
+
voice: deck.voice ?? null,
|
|
65
|
+
gapMs: deck.gap,
|
|
66
|
+
sections: planView(deck.plan, typeof input.section === 'string' ? input.section : undefined),
|
|
67
|
+
problems: [...deck.findings.map(f => f.message), ...deck.plan.problems.filter(p => p.reportedAtCompile).map(p => p.message)],
|
|
68
|
+
play: deck.voice ? `?view=lecture&track=${SCRIPT_TRACK}` : null,
|
|
69
|
+
}
|
|
70
|
+
},
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
export const lectureSynthesize: AgentTool = {
|
|
74
|
+
name: 'lecture_synthesize',
|
|
75
|
+
description: 'Explicitly generates voiceover for the canonical narrations and Production voice sources: clips missing from the exact synthesis cache (.presmium/speech/) are synthesized by the speech engine, which costs money; matching clips come from cache. Returns the synthesized lecture\'s length and parts; play it with ?view=lecture&track=script.',
|
|
76
|
+
input: { type: 'object', properties: {} },
|
|
77
|
+
kind: 'spend',
|
|
78
|
+
async run(_input, ctx) {
|
|
79
|
+
const { deckDir, doc } = deckOf(ctx)
|
|
80
|
+
const source = await assertVoiceSource(doc, deckDir)
|
|
81
|
+
const assertCurrent = async () => {
|
|
82
|
+
if (latestView(deckDir) !== doc) throw new VoiceSourceRefusal('voice-source-changed')
|
|
83
|
+
await assertVoiceSource(doc, deckDir, source)
|
|
84
|
+
if (latestView(deckDir) !== doc) throw new VoiceSourceRefusal('voice-source-changed')
|
|
85
|
+
}
|
|
86
|
+
const release = claimVoiceGeneration(deckDir)
|
|
87
|
+
try {
|
|
88
|
+
await assertCurrent()
|
|
89
|
+
const { deck, track, failures } = await synthesizeLecture(doc, deckDir, speechOf(ctx), ctx.signal, { assertCurrent })
|
|
90
|
+
await assertCurrent()
|
|
91
|
+
ctx.signal.throwIfAborted()
|
|
92
|
+
if (!deck.voice) throw new Error(LECTURE_MESSAGES[deckLocale(doc.meta.lang)].production.noVoice)
|
|
93
|
+
if (!track) throw new Error(failures.map(failure => synthesisFailureMessage(failure)).join('\n'))
|
|
94
|
+
return {
|
|
95
|
+
track: SCRIPT_TRACK,
|
|
96
|
+
voice: track.voice,
|
|
97
|
+
durationMs: trackDuration(track),
|
|
98
|
+
utterances: track.utterances.length,
|
|
99
|
+
events: track.events.length,
|
|
100
|
+
problems: deck.findings.map(f => f.message),
|
|
101
|
+
source,
|
|
102
|
+
}
|
|
103
|
+
} finally { release() }
|
|
104
|
+
},
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
export const trackTranscribe: AgentTool = {
|
|
108
|
+
name: 'track_transcribe',
|
|
109
|
+
description: 'Transcribes a recorded lecture (lectures/<track>/): its audio goes to the speech engine\'s provider, which costs money; the transcript becomes the lecture\'s utterances, and the actions taken while a sentence was said are bound to its words. Writes lectures/<track>/track.json.',
|
|
110
|
+
input: { type: 'object', properties: { track: { type: 'string', description: 'The lecture id (a directory of lectures/)' } }, required: ['track'] },
|
|
111
|
+
kind: 'spend',
|
|
112
|
+
async run(input, ctx) {
|
|
113
|
+
const { deckDir, doc, files } = deckOf(ctx)
|
|
114
|
+
if (typeof input.track !== 'string' || !input.track) throw new Error('track-required')
|
|
115
|
+
const { track, bound, provider } = await transcribeLecture(files, deckDir, input.track, speechOf(ctx), doc, ctx.signal)
|
|
116
|
+
return { track: track.id, provider, utterances: track.utterances.map(u => ({ id: u.id, section: u.section, t: u.t, text: u.text })), bound }
|
|
117
|
+
},
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
function storedTrack(deckDir: string, input: Record<string, unknown>): LectureTrack {
|
|
121
|
+
if (typeof input.track !== 'string' || !input.track) throw new Error('track-required')
|
|
122
|
+
const track = trackDir(deckDir, input.track) ? readTrack(deckDir, input.track)?.track : undefined
|
|
123
|
+
if (!track) throw new Error(`no lecture ${input.track} (lectures/${input.track}/${TRACK_FILE})`)
|
|
124
|
+
return track
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
const TRACK_INPUT = { track: { type: 'string', description: 'The lecture id (a directory of lectures/)' } }
|
|
128
|
+
|
|
129
|
+
export const trackNarration: AgentTool = {
|
|
130
|
+
name: 'track_narration',
|
|
131
|
+
description: 'A recorded lecture\'s transcript as narration, to write it back into the deck: per section and step, the sentences said there joined (draft), with a {cue …}`…` role before the word each action of the lecture (spotlight, pointer, next step, run …) was bound to; and where that step\'s narration is in the source (narration: file and lines of its paragraphs; for a step without narration, the on-screen block it follows: after). Reads only.',
|
|
132
|
+
input: { type: 'object', properties: { ...TRACK_INPUT, section: { type: 'string', description: 'Only this section id' } }, required: ['track'] },
|
|
133
|
+
kind: 'read',
|
|
134
|
+
async run(input, ctx) {
|
|
135
|
+
const { deckDir, doc } = deckOf(ctx)
|
|
136
|
+
const track = storedTrack(deckDir, input)
|
|
137
|
+
const draft = narrationDraft(track, typeof input.section === 'string' ? input.section : undefined)
|
|
138
|
+
const { plan } = deckPlan(doc)
|
|
139
|
+
const file = (range: SourceRange): SourceRange => ({ ...range, file: relative(deckDir, isAbsolute(range.file) ? range.file : resolve(deckDir, range.file)) })
|
|
140
|
+
return {
|
|
141
|
+
track: track.id,
|
|
142
|
+
lang: track.lang,
|
|
143
|
+
sections: draft.sections.map(s => ({
|
|
144
|
+
section: s.section,
|
|
145
|
+
title: doc.sections.find(own => own.id === s.section)?.title ?? '(the deck has no such section now)',
|
|
146
|
+
steps: s.steps.map((step) => {
|
|
147
|
+
const source = narrationSource(doc, plan, s.section, step.step)
|
|
148
|
+
return { ...step, narration: source.narration.map(file), ...source.after ? { after: { ...file(source.after), block: source.after.block } } : {} }
|
|
149
|
+
}),
|
|
150
|
+
})),
|
|
151
|
+
...draft.skipped ? { skipped: `${draft.skipped} actions bound to words have no {cue}` } : {},
|
|
152
|
+
}
|
|
153
|
+
},
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
export const trackTranslations: AgentTool = {
|
|
157
|
+
name: 'track_translations',
|
|
158
|
+
description: 'The sentences (utterances) of a recorded lecture to translate: each with its id, text and source, the digest of the text that its translation records; with the translation into lang it has, if any, and whether that is stale. Write a translation into the utterance\'s line of lectures/<track>/track.json as "translations": { "<lang>": { "text": "…", "source": "<source>" } }. Reads only.',
|
|
159
|
+
input: { type: 'object', properties: { ...TRACK_INPUT, lang: { type: 'string', description: 'The language to translate into (BCP 47), e.g. en' } }, required: ['track', 'lang'] },
|
|
160
|
+
kind: 'read',
|
|
161
|
+
async run(input, ctx) {
|
|
162
|
+
const { deckDir } = deckOf(ctx)
|
|
163
|
+
const track = storedTrack(deckDir, input)
|
|
164
|
+
if (typeof input.lang !== 'string' || !input.lang) throw new Error('lang-required')
|
|
165
|
+
return { track: track.id, file: `${LECTURES_DIR}/${track.id}/${TRACK_FILE}`, lang: track.lang, into: input.lang, utterances: translationItems(track, input.lang) }
|
|
166
|
+
},
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
// Generation uses shared host start_voiceover tasks; the existing export CLI is unchanged.
|
|
170
|
+
export const lectureTools: AgentTool[] = [lecturePreview, trackTranscribe, trackNarration, trackTranslations].map(tool => ({ ...tool, errors: LECTURE_TOOL_ERRORS }))
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
import type { IRDocument, SpeechService } from '@lincyaw/kernel'
|
|
2
|
+
import { sectionHints, transcribedTrack } from '../domain/index.ts'
|
|
3
|
+
import type { LectureTrack, MediaSegment, SegmentTranscript } from '../domain/index.ts'
|
|
4
|
+
import { whyNoTranscriptionEngine, transcriptionEngine } from './engines.ts'
|
|
5
|
+
import { deckPlan } from './plan.ts'
|
|
6
|
+
import { mediaFile, readInDeck, readTrack, trackDir, writeTrack } from './store.ts'
|
|
7
|
+
import type { DeckFiles } from './store.ts'
|
|
8
|
+
|
|
9
|
+
const MIME: Record<string, string> = { wav: 'audio/wav', ogg: 'audio/ogg', opus: 'audio/ogg', webm: 'audio/webm', mp4: 'audio/mp4', m4a: 'audio/mp4' }
|
|
10
|
+
|
|
11
|
+
function audioBlob(file: string, bytes: Uint8Array): Blob {
|
|
12
|
+
return new Blob([new Uint8Array(bytes)], { type: MIME[file.split('.').at(-1) ?? ''] ?? 'application/octet-stream' })
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
interface TranscriptionOptions {
|
|
16
|
+
engine?: string
|
|
17
|
+
lang: string
|
|
18
|
+
/** The words likely said in a section (its narration's), which the engine takes as hints. */
|
|
19
|
+
hints: (section: string) => string[]
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
async function transcribeSegments(speech: SpeechService, segments: readonly MediaSegment[], audio: (segment: MediaSegment) => Blob | undefined, options: TranscriptionOptions, signal?: AbortSignal): Promise<SegmentTranscript[]> {
|
|
23
|
+
const transcripts: SegmentTranscript[] = []
|
|
24
|
+
for (const segment of segments) {
|
|
25
|
+
const local = audio(segment)
|
|
26
|
+
if (!local) throw new Error(`the audio of section ${segment.section} (${segment.src}) is not on this machine`)
|
|
27
|
+
const hints = options.hints(segment.section)
|
|
28
|
+
const transcript = await speech.transcribe({ ...options.engine ? { engine: options.engine } : {}, audio: local, lang: options.lang, ...hints.length ? { hints } : {} }, signal)
|
|
29
|
+
transcripts.push({ segment, segments: transcript.segments })
|
|
30
|
+
}
|
|
31
|
+
return transcripts
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
export async function transcribeLecture(files: DeckFiles, deckDir: string, id: string, speech: SpeechService, doc: IRDocument | undefined, signal?: AbortSignal): Promise<{ track: LectureTrack, bound: number, provider: string }> {
|
|
35
|
+
const stored = trackDir(deckDir, id) ? readTrack(deckDir, id) : undefined
|
|
36
|
+
if (!stored) throw new Error(`no lecture ${id}`)
|
|
37
|
+
const engine = await transcriptionEngine(speech, doc)
|
|
38
|
+
if (!engine) throw new Error(whyNoTranscriptionEngine(doc))
|
|
39
|
+
const audio = (segment: { src: string }) => {
|
|
40
|
+
const file = mediaFile(deckDir, id, segment.src)
|
|
41
|
+
const bytes = file && readInDeck(deckDir, file)
|
|
42
|
+
return file && bytes ? audioBlob(file, bytes) : undefined
|
|
43
|
+
}
|
|
44
|
+
const plan = doc && deckPlan(doc).plan
|
|
45
|
+
const hints = (section: string) => {
|
|
46
|
+
const planned = plan?.sections.find(s => s.id === section)
|
|
47
|
+
return planned ? sectionHints(planned, stored.track.lang) : []
|
|
48
|
+
}
|
|
49
|
+
const transcripts = await transcribeSegments(speech, stored.track.audio ?? [], audio, { engine: engine.id, lang: stored.track.lang, hints }, signal)
|
|
50
|
+
const { track, bound } = transcribedTrack(stored.track, transcripts)
|
|
51
|
+
await writeTrack(files, deckDir, track)
|
|
52
|
+
return { track, bound, provider: engine.provider }
|
|
53
|
+
}
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
import { resolve } from 'node:path'
|
|
2
|
+
|
|
3
|
+
const active = new Map<string, symbol>()
|
|
4
|
+
export class VoiceGenerationBusy extends Error {
|
|
5
|
+
constructor() { super('voice-job-busy') }
|
|
6
|
+
}
|
|
7
|
+
|
|
8
|
+
/** One spending operation per document, including preflight awaits and cancellation acknowledgement. */
|
|
9
|
+
export function claimVoiceGeneration(deckDir: string): () => void {
|
|
10
|
+
const key = resolve(deckDir)
|
|
11
|
+
if (active.has(key)) throw new VoiceGenerationBusy()
|
|
12
|
+
const owner = Symbol()
|
|
13
|
+
active.set(key, owner)
|
|
14
|
+
return () => { if (active.get(key) === owner) active.delete(key) }
|
|
15
|
+
}
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
import { createHash } from 'node:crypto'
|
|
2
|
+
import { readFile } from 'node:fs/promises'
|
|
3
|
+
import { join, resolve } from 'node:path'
|
|
4
|
+
import { DOCUMENT_ENTRY, fnv1a64, isRecord } from '@lincyaw/kernel'
|
|
5
|
+
import type { CanonicalSource, IRDocument } from '@lincyaw/kernel'
|
|
6
|
+
|
|
7
|
+
export class VoiceSourceRefusal extends Error {}
|
|
8
|
+
|
|
9
|
+
export function canonicalSource(value: unknown): value is CanonicalSource {
|
|
10
|
+
return isRecord(value) && typeof value.documentId === 'string' && value.documentId.length > 0
|
|
11
|
+
&& typeof value.version === 'string' && /^[\da-f]{20}$/.test(value.version)
|
|
12
|
+
&& typeof value.fingerprint === 'string' && /^[\da-f]{16}$/.test(value.fingerprint)
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
export function sameCanonicalSource(a: CanonicalSource, b: CanonicalSource): boolean {
|
|
16
|
+
return a.documentId === b.documentId && a.version === b.version && a.fingerprint === b.fingerprint
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
/** Generation and audition verify disk provenance; renderers continue consuming IR and exact cache only. */
|
|
20
|
+
export async function assertVoiceSource(doc: IRDocument, deckDir: string, expected?: CanonicalSource): Promise<CanonicalSource> {
|
|
21
|
+
const source = doc.source
|
|
22
|
+
if (!canonicalSource(source)) throw new VoiceSourceRefusal('voice-source-unavailable')
|
|
23
|
+
if (expected && !sameCanonicalSource(expected, source)) throw new VoiceSourceRefusal('voice-source-changed')
|
|
24
|
+
const file = join(resolve(deckDir), DOCUMENT_ENTRY)
|
|
25
|
+
if (resolve(doc.meta.entry) !== file || doc.meta.documentId !== source.documentId) throw new VoiceSourceRefusal('voice-source-changed')
|
|
26
|
+
let raw: string
|
|
27
|
+
try { raw = await readFile(file, 'utf8') } catch { throw new VoiceSourceRefusal('voice-source-unavailable') }
|
|
28
|
+
if (createHash('sha256').update(raw).digest('hex').slice(0, 20) !== source.version || fnv1a64(raw) !== source.fingerprint) throw new VoiceSourceRefusal('voice-source-changed')
|
|
29
|
+
try {
|
|
30
|
+
const value: unknown = JSON.parse(raw.replace(/^\uFEFF/, ''))
|
|
31
|
+
if (!isRecord(value) || value.id !== source.documentId) throw new Error('identity')
|
|
32
|
+
} catch { throw new VoiceSourceRefusal('voice-source-changed') }
|
|
33
|
+
return { ...source }
|
|
34
|
+
}
|
|
@@ -0,0 +1,223 @@
|
|
|
1
|
+
import { createHash } from 'node:crypto'
|
|
2
|
+
import { readFile } from 'node:fs/promises'
|
|
3
|
+
import { setImmediate } from 'node:timers/promises'
|
|
4
|
+
import { deckLocale, errorMessage, isRecord, resolvedVoice, speechEngineId, speechFailureCode, speechVoice, voiceTarget } from '@lincyaw/kernel'
|
|
5
|
+
import { irRevision, VoiceoverOperationError } from '@lincyaw/kernel/node'
|
|
6
|
+
import type { CanonicalSource, IRDocument, NodeServiceContext, SpeechService, SpeechVoice, Synthesis, SynthesisRequest, VoiceoverExecution, VoiceoverProgress, VoiceoverRunResult, VoiceoverSnapshot, VoiceoverStatusQuery, VoicePreviewQuery, VoicePreviewQuote, VoiceScope, VoiceTarget } from '@lincyaw/kernel'
|
|
7
|
+
import { plannedUtterances } from '../domain/index.ts'
|
|
8
|
+
import { latestDoc, lectureView } from './latest-ir.ts'
|
|
9
|
+
import { deckPlan } from './plan.ts'
|
|
10
|
+
import { claimVoiceGeneration, VoiceGenerationBusy } from './voice-generation.ts'
|
|
11
|
+
import { assertVoiceSource, VoiceSourceRefusal } from './voice-source.ts'
|
|
12
|
+
import { createServedDeck, RouteRefusal } from './routes/served-deck.ts'
|
|
13
|
+
|
|
14
|
+
const PREVIEW_TEXT = {
|
|
15
|
+
'zh-CN': '你好,这是一段配音试听。你可以用它比较音色,再选择文稿的配音音色。',
|
|
16
|
+
'en': 'Hello. This is a short voice preview for your presentation.',
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
type Unit = ReturnType<typeof plannedUtterances>[number]
|
|
20
|
+
interface Context { source: CanonicalSource, doc: IRDocument, revision: string, units: Unit[], lang: string, lexicon?: Record<string, string> }
|
|
21
|
+
const digest = (value: unknown) => createHash('sha256').update(JSON.stringify(value)).digest('hex')
|
|
22
|
+
const validTarget = (value: unknown): value is VoiceTarget => isRecord(value) && typeof value.section === 'string' && Number.isSafeInteger(value.step) && (value.step as number) >= 0 && typeof value.binding === 'string'
|
|
23
|
+
|
|
24
|
+
function scoped(context: Context, scope: VoiceScope): Unit[] {
|
|
25
|
+
if (scope.kind === 'document') return context.units
|
|
26
|
+
if (scope.kind === 'page') {
|
|
27
|
+
if (!context.doc.sections.some(section => section.id === scope.section)) throw new RouteRefusal(409, 'voice-target-changed')
|
|
28
|
+
return context.units.filter(unit => unit.section === scope.section)
|
|
29
|
+
}
|
|
30
|
+
const targets = scope.kind === 'step' ? [scope.target] : scope.targets
|
|
31
|
+
for (const at of targets) {
|
|
32
|
+
const section = context.doc.sections.find(section => section.id === at.section)
|
|
33
|
+
if (!section || at.step > section.steps || voiceTarget(section, at.step).binding !== at.binding) throw new RouteRefusal(409, 'voice-step-binding-stale')
|
|
34
|
+
const setting = resolvedVoice(context.doc, section, at.step)
|
|
35
|
+
if (!setting.editable) throw new RouteRefusal(409, `voice-${setting.reason}`)
|
|
36
|
+
}
|
|
37
|
+
return context.units.filter(unit => targets.some(at => unit.voiceTarget?.section === at.section && unit.voiceTarget.step === at.step && unit.voiceTarget.binding === at.binding))
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
function requestOf(context: Context, unit: Unit): SynthesisRequest {
|
|
41
|
+
if (unit.voiceIssue || !unit.voice) throw new RouteRefusal(409, unit.voiceIssue ?? 'voice-unset')
|
|
42
|
+
return { ...unit.voice, text: unit.spoken, lang: context.lang, ...context.lexicon ? { lexicon: context.lexicon } : {} }
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
/** Real lecture execution; the shared host owner retains all jobs and intents. */
|
|
47
|
+
export function voiceoverExecution(ctx: NodeServiceContext): VoiceoverExecution {
|
|
48
|
+
const served = createServedDeck({ ...ctx, service: token => ctx.service?.(token) })
|
|
49
|
+
const context = async (expected?: CanonicalSource): Promise<Context> => {
|
|
50
|
+
const doc = latestDoc(served.deckDir)
|
|
51
|
+
if (!doc) throw new RouteRefusal(503, 'voice-document-unavailable')
|
|
52
|
+
const source = await assertVoiceSource(doc, served.deckDir, expected)
|
|
53
|
+
if (latestDoc(served.deckDir) !== doc) throw new RouteRefusal(409, 'voice-revision-changed')
|
|
54
|
+
const deck = deckPlan(lectureView(doc))
|
|
55
|
+
return { source, doc, revision: irRevision(doc), units: plannedUtterances(deck.plan), lang: deck.lang, lexicon: deck.lexicon }
|
|
56
|
+
}
|
|
57
|
+
const same = (now: Context) => latestDoc(served.deckDir) === now.doc
|
|
58
|
+
const assertCurrent = async (now: Context) => {
|
|
59
|
+
if (!same(now)) throw new RouteRefusal(409, 'voice-revision-changed')
|
|
60
|
+
await assertVoiceSource(now.doc, served.deckDir, now.source)
|
|
61
|
+
if (!same(now)) throw new RouteRefusal(409, 'voice-revision-changed')
|
|
62
|
+
}
|
|
63
|
+
const stale = (error: unknown) => error instanceof VoiceSourceRefusal || error instanceof RouteRefusal && error.message === 'voice-revision-changed'
|
|
64
|
+
const assertRevision = async (revision: unknown, expected?: CanonicalSource): Promise<Context> => {
|
|
65
|
+
const now = await context(expected)
|
|
66
|
+
if (revision !== now.revision) throw new RouteRefusal(409, 'voice-revision-changed')
|
|
67
|
+
return now
|
|
68
|
+
}
|
|
69
|
+
const engines = async (speech: SpeechService) => (await speech.engines()).filter(engine => engine.tts)
|
|
70
|
+
/** Naming an engine that does not synthesize, or that is not configured, is a bad request; only having no engine at all is the service's state. */
|
|
71
|
+
const noEngine = async (speech: SpeechService, preferred: string | undefined) => {
|
|
72
|
+
if (!preferred) return new RouteRefusal(503, 'voice-not-configured')
|
|
73
|
+
const id = speechEngineId(preferred), all = await speech.engines()
|
|
74
|
+
if (all.some(engine => engine.id === id)) return new RouteRefusal(422, 'voice-engine-unsupported')
|
|
75
|
+
return all.some(engine => engine.tts) ? new RouteRefusal(422, 'voice-engine-unknown') : new RouteRefusal(503, 'voice-not-configured')
|
|
76
|
+
}
|
|
77
|
+
const providerFor = async (speech: SpeechService, voice: SpeechVoice, now: Context, signal?: AbortSignal) => {
|
|
78
|
+
const list = await engines(speech)
|
|
79
|
+
const preferred = voice.engine ?? await speech.defaultSynthesisEngine?.()
|
|
80
|
+
const engine = preferred ? list.find(engine => engine.id === speechEngineId(preferred)) : undefined
|
|
81
|
+
if (!engine) throw await noEngine(speech, preferred)
|
|
82
|
+
const catalog = await speech.voiceCatalog?.(engine.id, undefined, signal)
|
|
83
|
+
if (catalog?.status !== 'available') throw new RouteRefusal(409, 'voice-catalog-unavailable')
|
|
84
|
+
const chosen = catalog.voices.find(candidate => candidate.id === voice.voice)
|
|
85
|
+
if (!chosen || chosen.lang.length && !chosen.lang.some(lang => lang.toLowerCase().split('-')[0] === now.lang.toLowerCase().split('-')[0])) throw new RouteRefusal(409, 'voice-unavailable')
|
|
86
|
+
return engine
|
|
87
|
+
}
|
|
88
|
+
const snapshot = async (query: VoiceoverStatusQuery, signal: AbortSignal): Promise<Omit<VoiceoverSnapshot, 'jobs'>> => {
|
|
89
|
+
const now = await context(query.source)
|
|
90
|
+
if (query.revision !== undefined && query.revision !== now.revision) throw new RouteRefusal(409, 'voice-revision-changed')
|
|
91
|
+
let speech: SpeechService | undefined
|
|
92
|
+
try { speech = served.speech() } catch { /* Readability and editing remain available without speech. */ }
|
|
93
|
+
const providers = speech ? await engines(speech) : []
|
|
94
|
+
const defaultEngine = await speech?.defaultSynthesisEngine?.()
|
|
95
|
+
const cached = new Map<string, boolean>()
|
|
96
|
+
const unavailable = new Map<string, string>()
|
|
97
|
+
const usable = new Set<string>()
|
|
98
|
+
await Promise.all(now.units.map(async unit => {
|
|
99
|
+
if (!speech || !unit.voice || unit.voiceIssue) return
|
|
100
|
+
try {
|
|
101
|
+
const hit = Boolean(await speech.lookup?.(requestOf(now, unit), signal))
|
|
102
|
+
cached.set(unit.id, hit)
|
|
103
|
+
if (!hit) await providerFor(speech, unit.voice, now, signal)
|
|
104
|
+
usable.add(unit.id)
|
|
105
|
+
} catch (error) { unavailable.set(unit.id, errorMessage(error)) }
|
|
106
|
+
}))
|
|
107
|
+
await assertCurrent(now)
|
|
108
|
+
const steps = now.doc.sections.flatMap(section => Array.from({ length: section.steps + 1 }, (_, step) => {
|
|
109
|
+
const selected = resolvedVoice(now.doc, section, step)
|
|
110
|
+
const units = now.units.filter(unit => unit.voiceTarget?.section === section.id && unit.voiceTarget.step === step)
|
|
111
|
+
const count = units.filter(unit => cached.get(unit.id)).length
|
|
112
|
+
const reason = units.find(unit => unit.voiceIssue)?.voiceIssue ?? units.map(unit => unavailable.get(unit.id)).find(Boolean) ?? (units.length > 0 && !selected.voice ? 'voice-unset' : undefined)
|
|
113
|
+
return { target: selected.target, ...section.stepIds?.[step] ? { stepId: section.stepIds[step] } : {}, state: reason ? 'blocked' as const : units.length === 0 ? 'empty' as const : count === units.length ? 'ready' as const : 'missing' as const, utterances: units.length, cached: count, ...reason ? { reason } : {} }
|
|
114
|
+
}))
|
|
115
|
+
const unboundByPage = now.doc.sections.flatMap(section => {
|
|
116
|
+
const units = now.units.filter(unit => !unit.voiceTarget && unit.section === section.id)
|
|
117
|
+
if (!units.length) return []
|
|
118
|
+
const count = units.filter(unit => cached.get(unit.id)).length
|
|
119
|
+
const reason = units.find(unit => unit.voiceIssue)?.voiceIssue ?? units.map(unit => unavailable.get(unit.id)).find(Boolean)
|
|
120
|
+
return [{ section: section.id, state: reason ? 'blocked' as const : count === units.length ? 'ready' as const : 'missing' as const, utterances: units.length, cached: count, ...reason ? { reason } : {} }]
|
|
121
|
+
})
|
|
122
|
+
return { source: now.source, revision: now.revision, capability: { available: Boolean(speech && usable.size), providers, ...defaultEngine ? { defaultEngine } : {}, ...!speech ? { reason: 'speech-unavailable' as const } : !usable.size && unavailable.size ? { reason: [...unavailable.values()].some(reason => reason === 'voice-not-configured') ? 'not-configured' as const : [...unavailable.values()].some(reason => reason === 'voice-catalog-unavailable') ? 'catalog-unavailable' as const : 'voice-unavailable' as const } : !usable.size ? { reason: 'voice-unset' as const } : {} }, steps, unboundByPage }
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
const quote = async (query: VoicePreviewQuery, signal: AbortSignal, expected?: CanonicalSource): Promise<{ quote: VoicePreviewQuote, request: SynthesisRequest, hit?: Synthesis, context: Context }> => {
|
|
126
|
+
const now = await assertRevision(query.revision, expected), voice = speechVoice(query.voice)
|
|
127
|
+
if (!voice) throw new RouteRefusal(400, 'voice-preview-invalid')
|
|
128
|
+
let text = PREVIEW_TEXT[deckLocale(now.lang)]
|
|
129
|
+
if (query.mode === 'step') {
|
|
130
|
+
if (!validTarget(query.target)) throw new RouteRefusal(400, 'voice-target')
|
|
131
|
+
text = scoped(now, { kind: 'step', target: query.target }).map(unit => unit.spoken).join('\n\n')
|
|
132
|
+
if (!text.trim()) throw new RouteRefusal(409, 'voice-script-empty')
|
|
133
|
+
}
|
|
134
|
+
if (text.length > 2000) throw new RouteRefusal(413, 'voice-preview-too-long')
|
|
135
|
+
const speech = served.speech(), available = await engines(speech)
|
|
136
|
+
const preferred = voice.engine ?? await speech.defaultSynthesisEngine?.()
|
|
137
|
+
const engine = preferred ? available.find(engine => engine.id === speechEngineId(preferred)) : undefined
|
|
138
|
+
const request = { ...voice, ...engine ? { engine: engine.id } : {}, text, lang: now.lang, ...now.lexicon ? { lexicon: now.lexicon } : {} }
|
|
139
|
+
const hit = await speech.lookup?.(request, signal)
|
|
140
|
+
if (!engine && !hit) throw await noEngine(speech, preferred)
|
|
141
|
+
if (!hit) await providerFor(speech, voice, now, signal)
|
|
142
|
+
await assertCurrent(now)
|
|
143
|
+
const provider = hit?.provider ?? engine!.provider, local = Boolean(hit || engine?.local)
|
|
144
|
+
const token = digest({ source: now.source, revision: now.revision, request, provider, local, target: query.target, mode: query.mode })
|
|
145
|
+
return { quote: { ...query, source: now.source, voice, text, provider, local, cached: Boolean(hit), quote: token }, request, hit, context: now }
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
|
|
149
|
+
const guarded = async <T>(run: () => Promise<T>): Promise<T> => {
|
|
150
|
+
try { return await run() }
|
|
151
|
+
catch (error) {
|
|
152
|
+
if (error instanceof VoiceSourceRefusal || error instanceof VoiceGenerationBusy) throw new VoiceoverOperationError(409, error.message)
|
|
153
|
+
throw error
|
|
154
|
+
}
|
|
155
|
+
}
|
|
156
|
+
return {
|
|
157
|
+
status: (query, signal) => guarded(() => snapshot(query, signal)),
|
|
158
|
+
prepare: (request, signal) => guarded(async () => {
|
|
159
|
+
signal.throwIfAborted()
|
|
160
|
+
const now = await assertRevision(request.revision, request.source), units = scoped(now, request.scope)
|
|
161
|
+
signal.throwIfAborted()
|
|
162
|
+
if (!units.length) throw new RouteRefusal(409, 'voice-script-empty')
|
|
163
|
+
const release = claimVoiceGeneration(served.deckDir)
|
|
164
|
+
try {
|
|
165
|
+
const speech = served.speech(), issues = new Map<string, string>()
|
|
166
|
+
for (const unit of units) {
|
|
167
|
+
signal.throwIfAborted(); await assertCurrent(now); signal.throwIfAborted()
|
|
168
|
+
try { if (!await speech.lookup?.(requestOf(now, unit), signal)) await providerFor(speech, unit.voice!, now, signal) }
|
|
169
|
+
catch (error) { signal.throwIfAborted(); if (stale(error)) throw error; issues.set(unit.id, errorMessage(error)) }
|
|
170
|
+
await assertCurrent(now)
|
|
171
|
+
}
|
|
172
|
+
signal.throwIfAborted()
|
|
173
|
+
if (issues.size === units.length) throw new RouteRefusal(409, issues.values().next().value ?? 'voice-unavailable')
|
|
174
|
+
return {
|
|
175
|
+
source: now.source, revision: now.revision, scope: structuredClone(request.scope), total: units.length, dispose: release,
|
|
176
|
+
async run(signal: AbortSignal, progress: (value: VoiceoverProgress) => void): Promise<VoiceoverRunResult> {
|
|
177
|
+
let completed = 0, failed = 0, failure: string | undefined, yielded = performance.now()
|
|
178
|
+
try {
|
|
179
|
+
for (const unit of units) {
|
|
180
|
+
signal.throwIfAborted(); await assertCurrent(now)
|
|
181
|
+
try {
|
|
182
|
+
if (issues.has(unit.id)) throw new Error(issues.get(unit.id))
|
|
183
|
+
const request = requestOf(now, unit)
|
|
184
|
+
if (!await speech.lookup?.(request, signal)) {
|
|
185
|
+
await providerFor(speech, unit.voice!, now, signal)
|
|
186
|
+
await assertCurrent(now); signal.throwIfAborted()
|
|
187
|
+
await speech.synthesize(request, signal)
|
|
188
|
+
}
|
|
189
|
+
signal.throwIfAborted(); await assertCurrent(now); signal.throwIfAborted(); completed++
|
|
190
|
+
} catch (error) {
|
|
191
|
+
if (signal.aborted || stale(error) || !same(now)) throw error
|
|
192
|
+
failed++; failure = speechFailureCode(error) ?? errorMessage(error)
|
|
193
|
+
}
|
|
194
|
+
progress({ completed, failed, ...failure ? { error: failure } : {} })
|
|
195
|
+
if (performance.now() - yielded >= 8) { await setImmediate(); yielded = performance.now() }
|
|
196
|
+
}
|
|
197
|
+
await assertCurrent(now); signal.throwIfAborted()
|
|
198
|
+
return { state: failed ? 'failed' : 'done', completed, failed, ...failure ? { error: failure } : {} }
|
|
199
|
+
} catch (error) {
|
|
200
|
+
const state = stale(error) || !same(now) ? 'stale' : signal.aborted ? 'cancelled' : 'failed'
|
|
201
|
+
return { state, completed, failed, ...state !== 'cancelled' ? { error: errorMessage(error) } : {} }
|
|
202
|
+
}
|
|
203
|
+
},
|
|
204
|
+
}
|
|
205
|
+
} catch (error) { release(); throw error }
|
|
206
|
+
}),
|
|
207
|
+
quote: (query, signal) => guarded(async () => (await quote(query, signal, query.source)).quote),
|
|
208
|
+
preview: (request, signal) => guarded(async () => {
|
|
209
|
+
const queried = await quote(request, signal, request.source)
|
|
210
|
+
if (request.quote !== queried.quote.quote) throw new RouteRefusal(409, 'voice-preview-changed')
|
|
211
|
+
const release = queried.hit ? undefined : claimVoiceGeneration(served.deckDir)
|
|
212
|
+
try {
|
|
213
|
+
signal.throwIfAborted(); await assertCurrent(queried.context); signal.throwIfAborted()
|
|
214
|
+
const result = queried.hit ?? await served.speech().synthesize(queried.request, signal)
|
|
215
|
+
await assertCurrent(queried.context)
|
|
216
|
+
const bytes = typeof result.audio === 'string' ? await readFile(result.audio) : Buffer.from(await result.audio.arrayBuffer())
|
|
217
|
+
if (bytes.byteLength > 8 << 20) throw new RouteRefusal(413, 'voice-preview-too-large')
|
|
218
|
+
signal.throwIfAborted(); await assertCurrent(queried.context); signal.throwIfAborted()
|
|
219
|
+
return { source: queried.context.source, revision: request.revision, voice: queried.quote.voice, text: queried.quote.text, cached: Boolean(queried.hit), key: result.key, bytes: new Uint8Array(bytes), mediaType: result.mime, duration: result.duration, provider: result.provider }
|
|
220
|
+
} finally { release?.() }
|
|
221
|
+
}),
|
|
222
|
+
}
|
|
223
|
+
}
|