@lincyaw/addon-speech 0.0.0-stage → 1.0.0-rc.2

package/src/domain/lexicon.ts

@@ -0,0 +1,29 @@
+import type { WordSpan } from '@lincyaw/kernel'
+
+export interface LexiconText {
+  text: string
+  spans: WordSpan[]
+}
+
+export function applyLexiconWithSpans(text: string, lexicon: Record<string, string> | undefined): LexiconText {
+  const entries = Object.entries(lexicon ?? {}).filter(([written]) => written.length > 0).sort((a, b) => b[0].length - a[0].length)
+  if (entries.length === 0) return { text, spans: [] }
+  const escape = (value: string) => value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')
+  const pattern = new RegExp(entries.map(([written]) => escape(written)).join('|'), 'g')
+  const spokenForm = new Map(entries)
+  const spans: LexiconText['spans'] = []
+  let spoken = ''
+  let last = 0
+  for (const match of text.matchAll(pattern)) {
+    const replacement = spokenForm.get(match[0]) ?? match[0]
+    spoken += text.slice(last, match.index)
+    spans.push({ from: spoken.length, to: spoken.length + replacement.length, start: match.index, end: match.index + match[0].length })
+    spoken += replacement
+    last = match.index + match[0].length
+  }
+  return { text: spoken + text.slice(last), spans }
+}
+
+export function applyLexicon(text: string, lexicon: Record<string, string> | undefined): string {
+  return applyLexiconWithSpans(text, lexicon).text
+}

package/src/domain/protocol.ts

@@ -0,0 +1,81 @@
+import { isRecord } from '@lincyaw/kernel'
+import type { AigcLabel, SpeechEngineInfo, SpeechWord, SynthesisRequest, Transcript, VoiceInfo } from '@lincyaw/kernel'
+import type { UnavailableEngine } from './engines.ts'
+
+export interface ListedEngine extends SpeechEngineInfo {
+  model: string
+  transcribeModel: string
+}
+
+export interface EnginesResponseBody {
+  engines: ListedEngine[]
+  unavailable: UnavailableEngine[]
+}
+
+export interface VoicesResponseBody {
+  voices: VoiceInfo[]
+}
+
+export interface SynthesisRequestBody {
+  engine?: string
+  voice: string
+  input: string
+  lang?: string
+  speed?: number
+  style?: string
+  lexicon?: Record<string, string>
+}
+
+export interface SynthesisResponseBody {
+  audio: string
+  format: string
+  mime: string
+  duration: number
+  words?: SpeechWord[]
+  engine: string
+  model: string
+  voice: string
+  provider: string
+  cloned: boolean
+  key: string
+  synthetic: true
+  AIGC: AigcLabel
+}
+
+export interface TranscriptionResponseBody extends Transcript {
+  language: string
+  words: SpeechWord[]
+}
+
+export function synthesisRequestBody(request: SynthesisRequest): SynthesisRequestBody {
+  return {
+    ...request.engine ? { engine: request.engine } : {},
+    voice: request.voice,
+    input: request.text,
+    lang: request.lang,
+    ...request.rate !== undefined ? { speed: request.rate } : {},
+    ...request.style ? { style: request.style } : {},
+    ...request.lexicon ? { lexicon: request.lexicon } : {},
+  }
+}
+
+export function toSynthesisRequest(body: unknown): SynthesisRequest | string {
+  if (!isRecord(body)) return 'a JSON object { voice, input, lang?, engine?, speed?, style?, lexicon? }'
+  const text = typeof body.input === 'string' ? body.input : typeof body.text === 'string' ? body.text : undefined
+  if (typeof body.voice !== 'string' || !body.voice) return 'voice is needed'
+  if (!text?.trim()) return 'input (the text to speak) is needed'
+  const lexicon = isRecord(body.lexicon) ? Object.fromEntries(Object.entries(body.lexicon).filter((entry): entry is [string, string] => typeof entry[1] === 'string')) : undefined
+  return {
+    ...typeof body.engine === 'string' && body.engine ? { engine: body.engine } : {},
+    voice: body.voice,
+    text,
+    lang: typeof body.lang === 'string' && body.lang ? body.lang : 'zh-CN',
+    ...typeof body.speed === 'number' && body.speed > 0 ? { rate: body.speed } : {},
+    ...typeof body.style === 'string' && body.style ? { style: body.style } : {},
+    ...lexicon && Object.keys(lexicon).length ? { lexicon } : {},
+  }
+}
+
+export function transcriptOf(body: TranscriptionResponseBody): Transcript {
+  return { lang: body.lang ?? body.language, duration: body.duration, text: body.text, segments: body.segments }
+}

package/src/domain/transcript.ts

@@ -0,0 +1,17 @@
+import type { SpeechWord, Transcript, TranscriptSegment } from '@lincyaw/kernel'
+
+export function oneSegment(text: string, words: SpeechWord[], lang: string, duration: number): Transcript {
+  const segment: TranscriptSegment = { start: words[0]?.start ?? 0, end: words.at(-1)?.end ?? 0, text, words }
+  return { lang, duration, text, segments: text || words.length ? [segment] : [] }
+}
+
+export function wordsIntoSegments(segments: { start: number, end: number, text: string }[], words: SpeechWord[]): TranscriptSegment[] {
+  return segments.map((segment, index) => {
+    const last = index === segments.length - 1
+    const inside = words.filter((word) => {
+      const middle = (word.start + word.end) / 2
+      return middle >= segment.start && (middle < segment.end || (last && middle <= segment.end))
+    })
+    return { start: segment.start, end: segment.end, text: segment.text.trim(), words: inside }
+  })
+}

package/src/node/audio/label.ts

@@ -0,0 +1,32 @@
+import { aigcString } from '@lincyaw/kernel'
+import type { AigcLabel } from '@lincyaw/kernel'
+import { SpeechError } from '../errors.ts'
+import { isOgg, oggOpusDuration, withOpusComment } from './ogg.ts'
+import { isWav, wavDuration, withInfo } from './riff.ts'
+
+export interface LabeledAudio {
+  audio: Uint8Array
+  mime: string
+  ext: string
+  duration: number
+}
+
+export function labelAudio(bytes: Uint8Array, label: AigcLabel): LabeledAudio {
+  if (isWav(bytes)) {
+    return { audio: withInfo(bytes, [['AIGC', JSON.stringify(label)], ['ICMT', aigcString(label)]]), mime: 'audio/wav', ext: 'wav', duration: wavDuration(bytes) }
+  }
+  if (isOgg(bytes)) {
+    try {
+      return { audio: withOpusComment(bytes, aigcString(label)), mime: 'audio/ogg', ext: 'ogg', duration: oggOpusDuration(bytes) }
+    } catch (error) {
+      throw new SpeechError(502, `the engine's Ogg audio cannot carry the AI-generated label: ${error instanceof Error ? error.message : String(error)}`)
+    }
+  }
+  throw new SpeechError(502, 'the engine returned audio that is neither WAV nor Ogg/Opus, so the AI-generated label cannot be written into it')
+}
+
+export function audioMime(bytes: Uint8Array, given?: string): string {
+  if (isWav(bytes)) return 'audio/wav'
+  if (isOgg(bytes)) return 'audio/ogg'
+  return given?.split(';')[0]?.trim() || 'application/octet-stream'
+}

package/src/node/audio/ogg.ts

@@ -0,0 +1,153 @@
+interface Page {
+  flags: number
+  granule: bigint
+  serial: number
+  sequence: number
+  segments: number[]
+  body: Uint8Array
+}
+
+const encoder = new TextEncoder()
+const decoder = new TextDecoder()
+
+const CRC = (() => {
+  const table = new Uint32Array(256)
+  for (let i = 0; i < 256; i++) {
+    let r = i << 24
+    for (let j = 0; j < 8; j++) r = r & 0x80000000 ? (r << 1) ^ 0x04c11db7 : r << 1
+    table[i] = r >>> 0
+  }
+  return table
+})()
+
+function crc(bytes: Uint8Array): number {
+  let value = 0
+  for (const byte of bytes) value = ((value << 8) ^ CRC[((value >>> 24) ^ byte) & 0xff]!) >>> 0
+  return value
+}
+
+export function isOgg(bytes: Uint8Array): boolean {
+  return bytes.length >= 4 && String.fromCharCode(...bytes.subarray(0, 4)) === 'OggS'
+}
+
+function pages(bytes: Uint8Array): Page[] {
+  const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength)
+  const out: Page[] = []
+  let at = 0
+  while (at + 27 <= bytes.length) {
+    if (String.fromCharCode(...bytes.subarray(at, at + 4)) !== 'OggS') throw new Error(`not an Ogg page at byte ${at}`)
+    const count = bytes[at + 26]!
+    const segments = [...bytes.subarray(at + 27, at + 27 + count)]
+    const length = segments.reduce((n, s) => n + s, 0)
+    const start = at + 27 + count
+    if (start + length > bytes.length) throw new Error('the last Ogg page is cut short')
+    out.push({ flags: bytes[at + 5]!, granule: view.getBigInt64(at + 6, true), serial: view.getUint32(at + 14, true), sequence: view.getUint32(at + 18, true), segments, body: bytes.subarray(start, start + length) })
+    at = start + length
+  }
+  return out
+}
+
+function pageBytes(page: Page): Uint8Array {
+  const out = new Uint8Array(27 + page.segments.length + page.body.length)
+  const view = new DataView(out.buffer)
+  out.set(encoder.encode('OggS'), 0)
+  out[5] = page.flags
+  view.setBigInt64(6, page.granule, true)
+  view.setUint32(14, page.serial, true)
+  view.setUint32(18, page.sequence, true)
+  out[26] = page.segments.length
+  out.set(page.segments, 27)
+  out.set(page.body, 27 + page.segments.length)
+  view.setUint32(22, crc(out), true)
+  return out
+}
+
+function commentHeader(all: Page[]): { end: number, packet: Uint8Array } {
+  const parts: Uint8Array[] = []
+  for (let i = 1; i < all.length; i++) {
+    const page = all[i]!
+    if (page.serial !== all[0]!.serial) throw new Error('an Ogg file of several streams')
+    parts.push(page.body)
+    if (page.segments.at(-1)! < 255) {
+      const packet = new Uint8Array(parts.reduce((n, p) => n + p.length, 0))
+      let at = 0
+      for (const part of parts) {
+        packet.set(part, at)
+        at += part.length
+      }
+      if (String.fromCharCode(...packet.subarray(0, 8)) !== 'OpusTags') throw new Error('the second Ogg packet is not an Opus comment header')
+      return { end: i, packet }
+    }
+  }
+  throw new Error('the Ogg file ends inside its comment header')
+}
+
+function parseTags(packet: Uint8Array): { vendor: Uint8Array, comments: Uint8Array[], rest: Uint8Array } {
+  const view = new DataView(packet.buffer, packet.byteOffset, packet.byteLength)
+  let at = 8
+  const vendorLength = view.getUint32(at, true)
+  const vendor = packet.subarray(at + 4, at + 4 + vendorLength)
+  at += 4 + vendorLength
+  const count = view.getUint32(at, true)
+  at += 4
+  const comments: Uint8Array[] = []
+  for (let i = 0; i < count; i++) {
+    const length = view.getUint32(at, true)
+    comments.push(packet.subarray(at + 4, at + 4 + length))
+    at += 4 + length
+  }
+  return { vendor, comments, rest: packet.subarray(at) }
+}
+
+function u32(value: number): Uint8Array {
+  const out = new Uint8Array(4)
+  new DataView(out.buffer).setUint32(0, value, true)
+  return out
+}
+
+export function opusComments(bytes: Uint8Array): string[] {
+  return parseTags(commentHeader(pages(bytes)).packet).comments.map(comment => decoder.decode(comment))
+}
+
+export function withOpusComment(bytes: Uint8Array, comment: string): Uint8Array {
+  const all = pages(bytes)
+  const { end, packet } = commentHeader(all)
+  const tags = parseTags(packet)
+  const comments = [...tags.comments, encoder.encode(comment)]
+  const parts = [encoder.encode('OpusTags'), u32(tags.vendor.length), tags.vendor, u32(comments.length), ...comments.flatMap(c => [u32(c.length), c]), tags.rest]
+  const header = new Uint8Array(parts.reduce((n, p) => n + p.length, 0))
+  let at = 0
+  for (const part of parts) {
+    header.set(part, at)
+    at += part.length
+  }
+  const lacing = [...Array<number>(Math.floor(header.length / 255)).fill(255), header.length % 255]
+  const rebuilt: Page[] = []
+  let offset = 0
+  for (let i = 0; i < lacing.length; i += 255) {
+    const segments = lacing.slice(i, i + 255)
+    const length = segments.reduce((n, s) => n + s, 0)
+    const last = i + 255 >= lacing.length
+    rebuilt.push({ flags: i === 0 ? 0 : 1, granule: last ? 0n : -1n, serial: all[0]!.serial, sequence: 1 + rebuilt.length, segments, body: header.subarray(offset, offset + length) })
+    offset += length
+  }
+  const shift = rebuilt.length - end
+  const later = all.slice(end + 1).map(page => ({ ...page, sequence: page.sequence + shift }))
+  const out = [all[0]!, ...rebuilt, ...later].map(pageBytes)
+  const result = new Uint8Array(out.reduce((n, p) => n + p.length, 0))
+  at = 0
+  for (const page of out) {
+    result.set(page, at)
+    at += page.length
+  }
+  return result
+}
+
+export function oggOpusDuration(bytes: Uint8Array): number {
+  const all = pages(bytes)
+  const head = all[0]?.body
+  if (!head || String.fromCharCode(...head.subarray(0, 8)) !== 'OpusHead') return 0
+  const preSkip = new DataView(head.buffer, head.byteOffset, head.byteLength).getUint16(10, true)
+  const granule = all.filter(page => page.granule >= 0n).at(-1)?.granule ?? 0n
+  return Math.max(0, Number(granule) - preSkip) / 48000
+}

package/src/node/audio/riff.ts

@@ -0,0 +1,96 @@
+export interface RiffChunk {
+  id: string
+  data: Uint8Array
+}
+
+const encoder = new TextEncoder()
+const decoder = new TextDecoder()
+
+function ascii(bytes: Uint8Array, at: number, length: number): string {
+  return String.fromCharCode(...bytes.subarray(at, at + length))
+}
+
+function u32(value: number): Uint8Array {
+  const out = new Uint8Array(4)
+  new DataView(out.buffer).setUint32(0, value, true)
+  return out
+}
+
+function concat(parts: Uint8Array[]): Uint8Array {
+  const out = new Uint8Array(parts.reduce((n, part) => n + part.length, 0))
+  let at = 0
+  for (const part of parts) {
+    out.set(part, at)
+    at += part.length
+  }
+  return out
+}
+
+function chunkBytes(id: string, data: Uint8Array): Uint8Array {
+  return concat([encoder.encode(id.padEnd(4, ' ').slice(0, 4)), u32(data.length), data, data.length % 2 ? new Uint8Array(1) : new Uint8Array(0)])
+}
+
+export function isWav(bytes: Uint8Array): boolean {
+  return bytes.length >= 12 && ascii(bytes, 0, 4) === 'RIFF' && ascii(bytes, 8, 4) === 'WAVE'
+}
+
+export function wavChunks(bytes: Uint8Array): RiffChunk[] {
+  const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength)
+  const chunks: RiffChunk[] = []
+  let at = 12
+  while (at + 8 <= bytes.length) {
+    const size = Math.min(view.getUint32(at + 4, true), bytes.length - at - 8)
+    chunks.push({ id: ascii(bytes, at, 4), data: bytes.subarray(at + 8, at + 8 + size) })
+    at += 8 + size + (size % 2)
+  }
+  return chunks
+}
+
+function riff(chunks: RiffChunk[]): Uint8Array {
+  const body = concat([encoder.encode('WAVE'), ...chunks.map(chunk => chunkBytes(chunk.id, chunk.data))])
+  return concat([encoder.encode('RIFF'), u32(body.length), body])
+}
+
+export function withInfo(bytes: Uint8Array, entries: [string, string][]): Uint8Array {
+  const info = concat([encoder.encode('INFO'), ...entries.map(([id, value]) => chunkBytes(id, encoder.encode(`${value}\0`)))])
+  return riff([...wavChunks(bytes), { id: 'LIST', data: info }])
+}
+
+export function infoEntries(bytes: Uint8Array): Record<string, string> {
+  const entries: Record<string, string> = {}
+  for (const chunk of wavChunks(bytes)) {
+    if (chunk.id !== 'LIST' || ascii(chunk.data, 0, 4) !== 'INFO') continue
+    const view = new DataView(chunk.data.buffer, chunk.data.byteOffset, chunk.data.byteLength)
+    let at = 4
+    while (at + 8 <= chunk.data.length) {
+      const size = Math.min(view.getUint32(at + 4, true), chunk.data.length - at - 8)
+      entries[ascii(chunk.data, at, 4)] = decoder.decode(chunk.data.subarray(at + 8, at + 8 + size)).replace(/\0+$/, '')
+      at += 8 + size + (size % 2)
+    }
+  }
+  return entries
+}
+
+export function wavDuration(bytes: Uint8Array): number {
+  const chunks = wavChunks(bytes)
+  const fmt = chunks.find(chunk => chunk.id === 'fmt ')
+  const data = chunks.find(chunk => chunk.id === 'data')
+  if (!fmt || !data || fmt.data.length < 12) return 0
+  const byteRate = new DataView(fmt.data.buffer, fmt.data.byteOffset, fmt.data.byteLength).getUint32(8, true)
+  return byteRate ? data.data.length / byteRate : 0
+}
+
+export function makeWav(samples: Int16Array, rate: number, extra: RiffChunk[] = []): Uint8Array {
+  const fmt = new Uint8Array(16)
+  const view = new DataView(fmt.buffer)
+  view.setUint16(0, 1, true)
+  view.setUint16(2, 1, true)
+  view.setUint32(4, rate, true)
+  view.setUint32(8, rate * 2, true)
+  view.setUint16(12, 2, true)
+  view.setUint16(14, 16, true)
+  const data = new Uint8Array(samples.length * 2)
+  const pcm = new DataView(data.buffer)
+  samples.forEach((sample, i) => pcm.setInt16(i * 2, sample, true))
+  return riff([{ id: 'fmt ', data: fmt }, ...extra, { id: 'data', data }])
+}

package/src/node/backend.ts

@@ -0,0 +1,47 @@
+import type { AigcLabel, SpeechWord, SynthesisRequest, Transcript, TranscriptionRequest, VoiceCatalog, VoiceCloneStatus, VoiceInfo } from '@lincyaw/kernel'
+import type { EnginesResponseBody } from '../domain/index.ts'
+
+export interface SynthesisResult {
+  audio: Uint8Array
+  mime: string
+  ext: string
+  duration: number
+  words?: SpeechWord[]
+  engine: string
+  model: string
+  voice: string
+  provider: string
+  cloned: boolean
+  key: string
+  requestKey?: string
+  profile?: string
+  AIGC: AigcLabel
+}
+
+/** Training a cloned voice: the sample, and the speaker's consent with their recorded statement. */
+export interface CloneInput {
+  engine?: string
+  voice: string
+  sample: Uint8Array
+  sampleMime: string
+  lang?: string
+  text?: string
+  consent: { speaker: string, scope: string, statement: Uint8Array, statementMime: string }
+}
+
+export type CacheProfile = { status: 'configured', profile: string, model: string, provider: string } | { status: 'absent' | 'unavailable' }
+
+export interface SpeechBackend {
+  listEngines(lang?: string): Promise<EnginesResponseBody>
+  defaultSynthesisEngine(): Promise<string | undefined>
+  /** Configuration identity only: never contacts a provider and never includes credentials. */
+  cacheProfile(engine: string | undefined, use: 'synthesize' | 'transcribe', lang?: string): Promise<CacheProfile>
+  voices(engine: string | undefined, lang?: string, signal?: AbortSignal): Promise<VoiceInfo[]>
+  voiceCatalog?(engine: string | undefined, lang?: string, signal?: AbortSignal): Promise<VoiceCatalog>
+  synthesize(request: SynthesisRequest, signal?: AbortSignal): Promise<SynthesisResult>
+  transcribe(request: TranscriptionRequest, signal?: AbortSignal): Promise<Transcript>
+  consent(engine: string, voice: string, recorded: boolean, signal?: AbortSignal): Promise<void>
+  /** Registers the consent, then starts training; absent when cloning is not wired. */
+  cloneVoice?(input: CloneInput, signal?: AbortSignal): Promise<VoiceCloneStatus>
+  cloneStatus?(engine: string | undefined, voice: string, signal?: AbortSignal): Promise<VoiceCloneStatus>
+}