egaki 0.6.0 → 0.7.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +145 -9
- package/dist/cli/cache-utils.d.ts +33 -0
- package/dist/cli/cache-utils.d.ts.map +1 -0
- package/dist/cli/cache-utils.js +157 -0
- package/dist/cli/cached-generate.d.ts +105 -0
- package/dist/cli/cached-generate.d.ts.map +1 -0
- package/dist/cli/cached-generate.js +208 -0
- package/dist/cli/cached-generate.test.d.ts +11 -0
- package/dist/cli/cached-generate.test.d.ts.map +1 -0
- package/dist/cli/cached-generate.test.js +431 -0
- package/dist/cli/cartesia-provider.d.ts +16 -0
- package/dist/cli/cartesia-provider.d.ts.map +1 -0
- package/dist/cli/cartesia-provider.js +184 -0
- package/dist/cli/chatgpt-auth.d.ts.map +1 -0
- package/dist/cli/cli.d.ts +4 -0
- package/dist/cli/cli.d.ts.map +1 -0
- package/dist/{cli.js → cli/cli.js} +626 -600
- package/dist/cli/clone-generate.d.ts +26 -0
- package/dist/cli/clone-generate.d.ts.map +1 -0
- package/dist/cli/clone-generate.js +148 -0
- package/dist/cli/credentials.d.ts.map +1 -0
- package/dist/{credentials.js → cli/credentials.js} +20 -0
- package/dist/cli/elevenlabs-speech-provider.d.ts +26 -0
- package/dist/cli/elevenlabs-speech-provider.d.ts.map +1 -0
- package/dist/cli/elevenlabs-speech-provider.js +199 -0
- package/dist/cli/generate.d.ts +140 -0
- package/dist/cli/generate.d.ts.map +1 -0
- package/dist/cli/generate.js +604 -0
- package/dist/cli/generate.test.d.ts +2 -0
- package/dist/cli/generate.test.d.ts.map +1 -0
- package/dist/cli/generate.test.js +131 -0
- package/dist/cli/login.d.ts.map +1 -0
- package/dist/cli/main.d.ts +3 -0
- package/dist/cli/main.d.ts.map +1 -0
- package/dist/cli/main.js +7 -0
- package/dist/{model-catalog.d.ts → cli/model-catalog.d.ts} +6 -0
- package/dist/cli/model-catalog.d.ts.map +1 -0
- package/dist/{model-catalog.js → cli/model-catalog.js} +714 -4
- package/dist/{models.d.ts → cli/models.d.ts} +12 -5
- package/dist/cli/models.d.ts.map +1 -0
- package/dist/{models.js → cli/models.js} +58 -57
- package/dist/cli/open-browser.d.ts.map +1 -0
- package/dist/cli/openai-speech-provider.d.ts +3 -0
- package/dist/cli/openai-speech-provider.d.ts.map +1 -0
- package/dist/cli/openai-speech-provider.js +28 -0
- package/dist/cli/separation-generate.d.ts +34 -0
- package/dist/cli/separation-generate.d.ts.map +1 -0
- package/dist/cli/separation-generate.js +117 -0
- package/dist/cli/speech-catalog.d.ts +35 -0
- package/dist/cli/speech-catalog.d.ts.map +1 -0
- package/dist/cli/speech-catalog.js +195 -0
- package/dist/cli/speech-generate.d.ts +86 -0
- package/dist/cli/speech-generate.d.ts.map +1 -0
- package/dist/cli/speech-generate.js +156 -0
- package/dist/cli/speech-models.d.ts +8 -0
- package/dist/cli/speech-models.d.ts.map +1 -0
- package/dist/cli/speech-models.js +109 -0
- package/dist/cli/speech-provider.test.d.ts +2 -0
- package/dist/cli/speech-provider.test.d.ts.map +1 -0
- package/dist/cli/speech-provider.test.js +197 -0
- package/dist/cli/subscription.d.ts.map +1 -0
- package/dist/cli/transcription-catalog.d.ts +35 -0
- package/dist/cli/transcription-catalog.d.ts.map +1 -0
- package/dist/cli/transcription-catalog.js +192 -0
- package/dist/cli/transcription-generate.d.ts +62 -0
- package/dist/cli/transcription-generate.d.ts.map +1 -0
- package/dist/cli/transcription-generate.js +144 -0
- package/dist/cli/transcription-models.d.ts +9 -0
- package/dist/cli/transcription-models.d.ts.map +1 -0
- package/dist/cli/transcription-models.js +100 -0
- package/dist/cli/transcription-providers.d.ts +22 -0
- package/dist/cli/transcription-providers.d.ts.map +1 -0
- package/dist/cli/transcription-providers.js +290 -0
- package/dist/cli/video-model-catalog.d.ts.map +1 -0
- package/dist/cli/xai-auth.d.ts.map +1 -0
- package/dist/{xai-auth.js → cli/xai-auth.js} +1 -1
- package/dist/vite/angled-screen.d.ts +57 -0
- package/dist/vite/angled-screen.d.ts.map +1 -0
- package/dist/vite/angled-screen.js +115 -0
- package/dist/vite/app.d.ts +51 -0
- package/dist/vite/app.d.ts.map +1 -0
- package/dist/vite/app.js +150 -0
- package/dist/vite/bands-shader.d.ts +47 -0
- package/dist/vite/bands-shader.d.ts.map +1 -0
- package/dist/vite/bands-shader.js +267 -0
- package/dist/vite/code-block.d.ts +107 -0
- package/dist/vite/code-block.d.ts.map +1 -0
- package/dist/vite/code-block.js +641 -0
- package/dist/vite/components.d.ts +36 -0
- package/dist/vite/components.d.ts.map +1 -0
- package/dist/vite/components.js +113 -0
- package/dist/vite/dispersion-rings-shader.d.ts +44 -0
- package/dist/vite/dispersion-rings-shader.d.ts.map +1 -0
- package/dist/vite/dispersion-rings-shader.js +282 -0
- package/dist/vite/easing-curves.d.ts +129 -0
- package/dist/vite/easing-curves.d.ts.map +1 -0
- package/dist/vite/easing-curves.js +584 -0
- package/dist/vite/easing-curves.test.d.ts +2 -0
- package/dist/vite/easing-curves.test.d.ts.map +1 -0
- package/dist/vite/easing-curves.test.js +200 -0
- package/dist/vite/generated-media-client.d.ts +35 -0
- package/dist/vite/generated-media-client.d.ts.map +1 -0
- package/dist/vite/generated-media-client.js +76 -0
- package/dist/vite/keyframes.d.ts +129 -0
- package/dist/vite/keyframes.d.ts.map +1 -0
- package/dist/vite/keyframes.js +194 -0
- package/dist/vite/layout-editor.d.ts +45 -0
- package/dist/vite/layout-editor.d.ts.map +1 -0
- package/dist/vite/layout-editor.js +587 -0
- package/dist/vite/layout-transition.d.ts +136 -0
- package/dist/vite/layout-transition.d.ts.map +1 -0
- package/dist/vite/layout-transition.js +581 -0
- package/dist/vite/liquid-gradient-shader.d.ts +62 -0
- package/dist/vite/liquid-gradient-shader.d.ts.map +1 -0
- package/dist/vite/liquid-gradient-shader.js +413 -0
- package/dist/vite/mdx-client.d.ts +15 -0
- package/dist/vite/mdx-client.d.ts.map +1 -0
- package/dist/vite/mdx-client.js +265 -0
- package/dist/vite/mdx-parse.d.ts +102 -0
- package/dist/vite/mdx-parse.d.ts.map +1 -0
- package/dist/vite/mdx-parse.js +251 -0
- package/dist/vite/mdx-provided-components.d.ts +18 -0
- package/dist/vite/mdx-provided-components.d.ts.map +1 -0
- package/dist/vite/mdx-provided-components.js +14 -0
- package/dist/vite/mdx-video.d.ts +327 -0
- package/dist/vite/mdx-video.d.ts.map +1 -0
- package/dist/vite/mdx-video.js +596 -0
- package/dist/vite/mdx-video.test.d.ts +9 -0
- package/dist/vite/mdx-video.test.d.ts.map +1 -0
- package/dist/vite/mdx-video.test.js +1945 -0
- package/dist/vite/media-components.d.ts +45 -0
- package/dist/vite/media-components.d.ts.map +1 -0
- package/dist/vite/media-components.js +355 -0
- package/dist/vite/media-duration-store.d.ts +66 -0
- package/dist/vite/media-duration-store.d.ts.map +1 -0
- package/dist/vite/media-duration-store.js +204 -0
- package/dist/vite/player-page.d.ts +35 -0
- package/dist/vite/player-page.d.ts.map +1 -0
- package/dist/vite/player-page.js +720 -0
- package/dist/vite/render-client.d.ts +18 -0
- package/dist/vite/render-client.d.ts.map +1 -0
- package/dist/vite/render-client.js +67 -0
- package/dist/vite/sdk.d.ts +221 -0
- package/dist/vite/sdk.d.ts.map +1 -0
- package/dist/vite/sdk.js +355 -0
- package/dist/vite/server-components.d.ts +55 -0
- package/dist/vite/server-components.d.ts.map +1 -0
- package/dist/vite/server-components.js +156 -0
- package/dist/vite/server-mdx.d.ts +74 -0
- package/dist/vite/server-mdx.d.ts.map +1 -0
- package/dist/vite/server-mdx.js +259 -0
- package/dist/vite/shader-renderer.d.ts +48 -0
- package/dist/vite/shader-renderer.d.ts.map +1 -0
- package/dist/vite/shader-renderer.js +411 -0
- package/dist/vite/store-hooks.d.ts +12 -0
- package/dist/vite/store-hooks.d.ts.map +1 -0
- package/dist/vite/store-hooks.js +93 -0
- package/dist/vite/store.d.ts +75 -0
- package/dist/vite/store.d.ts.map +1 -0
- package/dist/vite/store.js +52 -0
- package/dist/vite/store.test.d.ts +6 -0
- package/dist/vite/store.test.d.ts.map +1 -0
- package/dist/vite/store.test.js +172 -0
- package/dist/vite/tweakpane-hook.d.ts +94 -0
- package/dist/vite/tweakpane-hook.d.ts.map +1 -0
- package/dist/vite/tweakpane-hook.js +450 -0
- package/dist/vite/vite-plugin.d.ts +28 -0
- package/dist/vite/vite-plugin.d.ts.map +1 -0
- package/dist/vite/vite-plugin.js +539 -0
- package/dist/vite/wave-gradient-shader.d.ts +41 -0
- package/dist/vite/wave-gradient-shader.d.ts.map +1 -0
- package/dist/vite/wave-gradient-shader.js +207 -0
- package/package.json +101 -24
- package/skills/egaki/SKILL.md +162 -1
- package/src/cli/cache-utils.ts +169 -0
- package/src/cli/cached-generate.test.ts +531 -0
- package/src/cli/cached-generate.ts +348 -0
- package/src/cli/cartesia-provider.ts +249 -0
- package/src/{cli.ts → cli/cli.ts} +836 -884
- package/src/cli/clone-generate.ts +202 -0
- package/src/{credentials.ts → cli/credentials.ts} +20 -0
- package/src/cli/elevenlabs-speech-provider.ts +243 -0
- package/src/cli/generate.test.ts +166 -0
- package/src/cli/generate.ts +912 -0
- package/src/cli/main.ts +8 -0
- package/src/{model-catalog.ts → cli/model-catalog.ts} +787 -58
- package/src/{models.ts → cli/models.ts} +65 -86
- package/src/cli/openai-speech-provider.ts +34 -0
- package/src/cli/separation-generate.ts +187 -0
- package/src/cli/speech-catalog.ts +253 -0
- package/src/cli/speech-generate.ts +275 -0
- package/src/cli/speech-models.ts +146 -0
- package/src/cli/speech-provider.test.ts +226 -0
- package/src/cli/transcription-catalog.ts +250 -0
- package/src/cli/transcription-generate.ts +214 -0
- package/src/cli/transcription-models.ts +136 -0
- package/src/cli/transcription-providers.ts +427 -0
- package/src/{xai-auth.ts → cli/xai-auth.ts} +1 -1
- package/src/vite/angled-screen.tsx +214 -0
- package/src/vite/app.tsx +197 -0
- package/src/vite/bands-shader.tsx +319 -0
- package/src/vite/code-block.tsx +939 -0
- package/src/vite/components.tsx +278 -0
- package/src/vite/dispersion-rings-shader.tsx +331 -0
- package/src/vite/easing-curves.test.ts +232 -0
- package/src/vite/easing-curves.ts +743 -0
- package/src/vite/generated-media-client.tsx +121 -0
- package/src/vite/keyframes.tsx +347 -0
- package/src/vite/layout-editor.tsx +709 -0
- package/src/vite/layout-transition.tsx +706 -0
- package/src/vite/liquid-gradient-shader.tsx +474 -0
- package/src/vite/mdx-client.tsx +420 -0
- package/src/vite/mdx-parse.ts +343 -0
- package/src/vite/mdx-provided-components.ts +19 -0
- package/src/vite/mdx-video.test.tsx +2219 -0
- package/src/vite/mdx-video.tsx +976 -0
- package/src/vite/media-components.tsx +473 -0
- package/src/vite/media-duration-store.ts +225 -0
- package/src/vite/motion-timing.ts +194 -0
- package/src/vite/player-page.tsx +1107 -0
- package/src/vite/render-client.ts +84 -0
- package/src/vite/sdk.ts +598 -0
- package/src/vite/server-components.tsx +241 -0
- package/src/vite/server-mdx.ts +285 -0
- package/src/vite/shader-renderer.tsx +535 -0
- package/src/vite/store-hooks.ts +117 -0
- package/src/vite/store.test.ts +202 -0
- package/src/vite/store.ts +115 -0
- package/src/vite/styles.css +21 -0
- package/src/vite/tweakpane-hook.tsx +593 -0
- package/src/vite/virtual-modules.d.ts +30 -0
- package/src/vite/vite-plugin.ts +585 -0
- package/src/vite/wave-gradient-shader.tsx +253 -0
- package/README.md +0 -370
- package/dist/antigravity-auth.d.ts +0 -15
- package/dist/antigravity-auth.d.ts.map +0 -1
- package/dist/antigravity-auth.js +0 -544
- package/dist/antigravity-image.d.ts +0 -11
- package/dist/antigravity-image.d.ts.map +0 -1
- package/dist/antigravity-image.js +0 -285
- package/dist/chatgpt-auth.d.ts.map +0 -1
- package/dist/cli.d.ts +0 -3
- package/dist/cli.d.ts.map +0 -1
- package/dist/credentials.d.ts.map +0 -1
- package/dist/gateway-model-candidates.generated.d.ts +0 -11
- package/dist/gateway-model-candidates.generated.d.ts.map +0 -1
- package/dist/gateway-model-candidates.generated.js +0 -64
- package/dist/login.d.ts.map +0 -1
- package/dist/model-catalog.d.ts.map +0 -1
- package/dist/models.d.ts.map +0 -1
- package/dist/open-browser.d.ts.map +0 -1
- package/dist/subscription.d.ts.map +0 -1
- package/dist/video-model-catalog.d.ts.map +0 -1
- package/dist/xai-auth.d.ts.map +0 -1
- /package/dist/{chatgpt-auth.d.ts → cli/chatgpt-auth.d.ts} +0 -0
- /package/dist/{chatgpt-auth.js → cli/chatgpt-auth.js} +0 -0
- /package/dist/{credentials.d.ts → cli/credentials.d.ts} +0 -0
- /package/dist/{login.d.ts → cli/login.d.ts} +0 -0
- /package/dist/{login.js → cli/login.js} +0 -0
- /package/dist/{open-browser.d.ts → cli/open-browser.d.ts} +0 -0
- /package/dist/{open-browser.js → cli/open-browser.js} +0 -0
- /package/dist/{subscription.d.ts → cli/subscription.d.ts} +0 -0
- /package/dist/{subscription.js → cli/subscription.js} +0 -0
- /package/dist/{video-model-catalog.d.ts → cli/video-model-catalog.d.ts} +0 -0
- /package/dist/{video-model-catalog.js → cli/video-model-catalog.js} +0 -0
- /package/dist/{xai-auth.d.ts → cli/xai-auth.d.ts} +0 -0
- /package/src/{chatgpt-auth.ts → cli/chatgpt-auth.ts} +0 -0
- /package/src/{login.ts → cli/login.ts} +0 -0
- /package/src/{open-browser.ts → cli/open-browser.ts} +0 -0
- /package/src/{subscription.ts → cli/subscription.ts} +0 -0
- /package/src/{video-model-catalog.ts → cli/video-model-catalog.ts} +0 -0
|
@@ -0,0 +1,226 @@
|
|
|
1
|
+
// Tests for the speech provider timestamp parsing logic.
|
|
2
|
+
// Tests pure functions (no API calls): ElevenLabs character-to-word grouping,
|
|
3
|
+
// ElevenLabs NDJSON stream parsing, and Cartesia SSE stream parsing.
|
|
4
|
+
|
|
5
|
+
import { describe, it, expect } from 'vitest'
|
|
6
|
+
import { groupCharactersToWords, parseElevenLabsStream } from './elevenlabs-speech-provider.js'
|
|
7
|
+
import { parseCartesiaSseStream } from './cartesia-provider.js'
|
|
8
|
+
|
|
9
|
+
describe('ElevenLabs groupCharactersToWords', () => {
|
|
10
|
+
it('groups simple words separated by spaces', () => {
|
|
11
|
+
const result = groupCharactersToWords({
|
|
12
|
+
characters: ['H', 'e', 'l', 'l', 'o', ' ', 'w', 'o', 'r', 'l', 'd'],
|
|
13
|
+
character_start_times_seconds: [0, 0.05, 0.1, 0.15, 0.2, 0.25, 0.3, 0.35, 0.4, 0.45, 0.5],
|
|
14
|
+
character_end_times_seconds: [0.05, 0.1, 0.15, 0.2, 0.25, 0.3, 0.35, 0.4, 0.45, 0.5, 0.55],
|
|
15
|
+
})
|
|
16
|
+
expect(result).toMatchInlineSnapshot(`
|
|
17
|
+
[
|
|
18
|
+
{
|
|
19
|
+
"endSecond": 0.25,
|
|
20
|
+
"startSecond": 0,
|
|
21
|
+
"word": "Hello",
|
|
22
|
+
},
|
|
23
|
+
{
|
|
24
|
+
"endSecond": 0.55,
|
|
25
|
+
"startSecond": 0.3,
|
|
26
|
+
"word": "world",
|
|
27
|
+
},
|
|
28
|
+
]
|
|
29
|
+
`)
|
|
30
|
+
})
|
|
31
|
+
|
|
32
|
+
it('handles multiple spaces between words', () => {
|
|
33
|
+
const result = groupCharactersToWords({
|
|
34
|
+
characters: ['a', ' ', ' ', 'b'],
|
|
35
|
+
character_start_times_seconds: [0, 0.1, 0.2, 0.3],
|
|
36
|
+
character_end_times_seconds: [0.1, 0.2, 0.3, 0.4],
|
|
37
|
+
})
|
|
38
|
+
expect(result).toEqual([
|
|
39
|
+
{ word: 'a', startSecond: 0, endSecond: 0.1 },
|
|
40
|
+
{ word: 'b', startSecond: 0.3, endSecond: 0.4 },
|
|
41
|
+
])
|
|
42
|
+
})
|
|
43
|
+
|
|
44
|
+
it('handles punctuation attached to words', () => {
|
|
45
|
+
const result = groupCharactersToWords({
|
|
46
|
+
characters: ['H', 'i', '!', ' ', 'O', 'k', '.'],
|
|
47
|
+
character_start_times_seconds: [0, 0.05, 0.1, 0.15, 0.2, 0.25, 0.3],
|
|
48
|
+
character_end_times_seconds: [0.05, 0.1, 0.15, 0.2, 0.25, 0.3, 0.35],
|
|
49
|
+
})
|
|
50
|
+
expect(result).toEqual([
|
|
51
|
+
{ word: 'Hi!', startSecond: 0, endSecond: 0.15 },
|
|
52
|
+
{ word: 'Ok.', startSecond: 0.2, endSecond: 0.35 },
|
|
53
|
+
])
|
|
54
|
+
})
|
|
55
|
+
|
|
56
|
+
it('handles single character input', () => {
|
|
57
|
+
const result = groupCharactersToWords({
|
|
58
|
+
characters: ['a'],
|
|
59
|
+
character_start_times_seconds: [0],
|
|
60
|
+
character_end_times_seconds: [0.1],
|
|
61
|
+
})
|
|
62
|
+
expect(result).toEqual([{ word: 'a', startSecond: 0, endSecond: 0.1 }])
|
|
63
|
+
})
|
|
64
|
+
|
|
65
|
+
it('returns empty array for empty input', () => {
|
|
66
|
+
const result = groupCharactersToWords({
|
|
67
|
+
characters: [],
|
|
68
|
+
character_start_times_seconds: [],
|
|
69
|
+
character_end_times_seconds: [],
|
|
70
|
+
})
|
|
71
|
+
expect(result).toEqual([])
|
|
72
|
+
})
|
|
73
|
+
|
|
74
|
+
it('handles leading/trailing whitespace', () => {
|
|
75
|
+
const result = groupCharactersToWords({
|
|
76
|
+
characters: [' ', 'a', ' '],
|
|
77
|
+
character_start_times_seconds: [0, 0.1, 0.2],
|
|
78
|
+
character_end_times_seconds: [0.1, 0.2, 0.3],
|
|
79
|
+
})
|
|
80
|
+
expect(result).toEqual([{ word: 'a', startSecond: 0.1, endSecond: 0.2 }])
|
|
81
|
+
})
|
|
82
|
+
})
|
|
83
|
+
|
|
84
|
+
describe('ElevenLabs parseElevenLabsStream', () => {
|
|
85
|
+
function createNdjsonStream(lines: object[]): ReadableStream<Uint8Array> {
|
|
86
|
+
const text = lines.map((l) => JSON.stringify(l)).join('\n')
|
|
87
|
+
const encoder = new TextEncoder()
|
|
88
|
+
return new ReadableStream({
|
|
89
|
+
start(controller) {
|
|
90
|
+
controller.enqueue(encoder.encode(text))
|
|
91
|
+
controller.close()
|
|
92
|
+
},
|
|
93
|
+
})
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
it('parses audio chunks and alignment', async () => {
|
|
97
|
+
const lines = [
|
|
98
|
+
{
|
|
99
|
+
audio_base64: Buffer.from([1, 2, 3]).toString('base64'),
|
|
100
|
+
normalized_alignment: {
|
|
101
|
+
characters: ['H', 'i'],
|
|
102
|
+
character_start_times_seconds: [0, 0.1],
|
|
103
|
+
character_end_times_seconds: [0.1, 0.2],
|
|
104
|
+
},
|
|
105
|
+
},
|
|
106
|
+
{ audio_base64: Buffer.from([4, 5, 6]).toString('base64') },
|
|
107
|
+
]
|
|
108
|
+
|
|
109
|
+
const result = await parseElevenLabsStream(createNdjsonStream(lines))
|
|
110
|
+
|
|
111
|
+
expect(result.audioChunks.length).toBe(2)
|
|
112
|
+
expect(result.alignment).toEqual({
|
|
113
|
+
characters: ['H', 'i'],
|
|
114
|
+
character_start_times_seconds: [0, 0.1],
|
|
115
|
+
character_end_times_seconds: [0.1, 0.2],
|
|
116
|
+
})
|
|
117
|
+
})
|
|
118
|
+
|
|
119
|
+
it('handles stream with no alignment', async () => {
|
|
120
|
+
const lines = [
|
|
121
|
+
{ audio_base64: Buffer.from([1, 2, 3]).toString('base64') },
|
|
122
|
+
]
|
|
123
|
+
|
|
124
|
+
const result = await parseElevenLabsStream(createNdjsonStream(lines))
|
|
125
|
+
|
|
126
|
+
expect(result.audioChunks.length).toBe(1)
|
|
127
|
+
expect(result.alignment).toBeUndefined()
|
|
128
|
+
})
|
|
129
|
+
|
|
130
|
+
it('prefers normalized_alignment over alignment', async () => {
|
|
131
|
+
const lines = [
|
|
132
|
+
{
|
|
133
|
+
audio_base64: Buffer.from([1]).toString('base64'),
|
|
134
|
+
alignment: {
|
|
135
|
+
characters: ['4', '2'],
|
|
136
|
+
character_start_times_seconds: [0, 0.1],
|
|
137
|
+
character_end_times_seconds: [0.1, 0.2],
|
|
138
|
+
},
|
|
139
|
+
normalized_alignment: {
|
|
140
|
+
characters: ['f', 'o', 'r', 't', 'y', ' ', 't', 'w', 'o'],
|
|
141
|
+
character_start_times_seconds: [0, 0.02, 0.04, 0.06, 0.08, 0.1, 0.12, 0.14, 0.16],
|
|
142
|
+
character_end_times_seconds: [0.02, 0.04, 0.06, 0.08, 0.1, 0.12, 0.14, 0.16, 0.18],
|
|
143
|
+
},
|
|
144
|
+
},
|
|
145
|
+
]
|
|
146
|
+
|
|
147
|
+
const result = await parseElevenLabsStream(createNdjsonStream(lines))
|
|
148
|
+
|
|
149
|
+
expect(result.alignment!.characters[0]).toBe('f') // normalized, not '4'
|
|
150
|
+
})
|
|
151
|
+
})
|
|
152
|
+
|
|
153
|
+
describe('Cartesia parseCartesiaSseStream', () => {
|
|
154
|
+
function createSseStream(events: string[]): ReadableStream<Uint8Array> {
|
|
155
|
+
const text = events.map((e) => `data: ${e}\n\n`).join('')
|
|
156
|
+
const encoder = new TextEncoder()
|
|
157
|
+
return new ReadableStream({
|
|
158
|
+
start(controller) {
|
|
159
|
+
controller.enqueue(encoder.encode(text))
|
|
160
|
+
controller.close()
|
|
161
|
+
},
|
|
162
|
+
})
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
it('parses audio chunks and timestamps', async () => {
|
|
166
|
+
const events = [
|
|
167
|
+
JSON.stringify({ type: 'chunk', done: false, status_code: 206, data: 'AAAA', step_time: 10 }),
|
|
168
|
+
JSON.stringify({
|
|
169
|
+
type: 'timestamps', done: false, status_code: 206,
|
|
170
|
+
word_timestamps: { words: ['Hello', 'world'], start: [0, 0.5], end: [0.4, 0.9] },
|
|
171
|
+
}),
|
|
172
|
+
JSON.stringify({ type: 'chunk', done: false, status_code: 206, data: 'BBBB', step_time: 10 }),
|
|
173
|
+
JSON.stringify({ type: 'done', done: true, status_code: 206 }),
|
|
174
|
+
]
|
|
175
|
+
|
|
176
|
+
const result = await parseCartesiaSseStream(createSseStream(events))
|
|
177
|
+
|
|
178
|
+
expect(result.audioChunks.length).toBe(2)
|
|
179
|
+
expect(result.timestamps).toEqual([
|
|
180
|
+
{ word: 'Hello', startSecond: 0, endSecond: 0.4 },
|
|
181
|
+
{ word: 'world', startSecond: 0.5, endSecond: 0.9 },
|
|
182
|
+
])
|
|
183
|
+
})
|
|
184
|
+
|
|
185
|
+
it('handles stream with no timestamps', async () => {
|
|
186
|
+
const events = [
|
|
187
|
+
JSON.stringify({ type: 'chunk', done: false, status_code: 206, data: 'AAAA', step_time: 10 }),
|
|
188
|
+
JSON.stringify({ type: 'done', done: true, status_code: 206 }),
|
|
189
|
+
]
|
|
190
|
+
|
|
191
|
+
const result = await parseCartesiaSseStream(createSseStream(events))
|
|
192
|
+
|
|
193
|
+
expect(result.audioChunks.length).toBe(1)
|
|
194
|
+
expect(result.timestamps).toEqual([])
|
|
195
|
+
})
|
|
196
|
+
|
|
197
|
+
it('throws on error events', async () => {
|
|
198
|
+
const events = [
|
|
199
|
+
JSON.stringify({ type: 'error', done: true, status_code: 400, message: 'Invalid model', title: 'Bad Request' }),
|
|
200
|
+
]
|
|
201
|
+
|
|
202
|
+
await expect(parseCartesiaSseStream(createSseStream(events)))
|
|
203
|
+
.rejects.toThrow('Cartesia TTS SSE error 400: Invalid model')
|
|
204
|
+
})
|
|
205
|
+
|
|
206
|
+
it('handles multiple timestamp events', async () => {
|
|
207
|
+
const events = [
|
|
208
|
+
JSON.stringify({
|
|
209
|
+
type: 'timestamps', done: false, status_code: 206,
|
|
210
|
+
word_timestamps: { words: ['Hello'], start: [0], end: [0.4] },
|
|
211
|
+
}),
|
|
212
|
+
JSON.stringify({
|
|
213
|
+
type: 'timestamps', done: false, status_code: 206,
|
|
214
|
+
word_timestamps: { words: ['world'], start: [0.5], end: [0.9] },
|
|
215
|
+
}),
|
|
216
|
+
JSON.stringify({ type: 'done', done: true, status_code: 206 }),
|
|
217
|
+
]
|
|
218
|
+
|
|
219
|
+
const result = await parseCartesiaSseStream(createSseStream(events))
|
|
220
|
+
|
|
221
|
+
expect(result.timestamps).toEqual([
|
|
222
|
+
{ word: 'Hello', startSecond: 0, endSecond: 0.4 },
|
|
223
|
+
{ word: 'world', startSecond: 0.5, endSecond: 0.9 },
|
|
224
|
+
])
|
|
225
|
+
})
|
|
226
|
+
})
|
|
@@ -0,0 +1,250 @@
|
|
|
1
|
+
// Transcription (speech-to-text) model catalog — single source of truth.
|
|
2
|
+
// Mirrors the speech catalog pattern: every model's provider, pricing,
|
|
3
|
+
// features live here. Other modules derive their data from this catalog.
|
|
4
|
+
//
|
|
5
|
+
// Pricing sources:
|
|
6
|
+
// OpenAI: https://platform.openai.com/docs/pricing (per minute of audio)
|
|
7
|
+
// ElevenLabs: https://elevenlabs.io/pricing
|
|
8
|
+
// Deepgram: https://deepgram.com/pricing
|
|
9
|
+
// Groq: https://groq.com/pricing
|
|
10
|
+
|
|
11
|
+
import type { ProviderOption } from './model-catalog.js'
|
|
12
|
+
|
|
13
|
+
// ─── cost types ──────────────────────────────────────────────────────────────
|
|
14
|
+
|
|
15
|
+
export type PerSecondCost = {
|
|
16
|
+
type: 'per-second'
|
|
17
|
+
/** USD per second of audio */
|
|
18
|
+
perSecond: number
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
export type TranscriptionModelCost = PerSecondCost
|
|
22
|
+
|
|
23
|
+
// ─── feature types ───────────────────────────────────────────────────────────
|
|
24
|
+
|
|
25
|
+
export type TranscriptionModelFeatures = {
|
|
26
|
+
/** Whether word-level timestamps are returned */
|
|
27
|
+
wordTimestamps: boolean
|
|
28
|
+
/** Whether speaker diarization is supported */
|
|
29
|
+
diarization: boolean
|
|
30
|
+
/** Whether language detection is automatic */
|
|
31
|
+
languageDetection: boolean
|
|
32
|
+
/** Whether a language hint can be provided */
|
|
33
|
+
languageHint: boolean
|
|
34
|
+
/** Supported input audio formats */
|
|
35
|
+
inputFormats: string[]
|
|
36
|
+
/** Max audio duration in seconds (0 = unknown/unlimited) */
|
|
37
|
+
maxDurationSec: number
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
// ─── entry type ──────────────────────────────────────────────────────────────
|
|
41
|
+
|
|
42
|
+
export type TranscriptionModelEntry = {
|
|
43
|
+
id: string
|
|
44
|
+
name: string
|
|
45
|
+
description?: string
|
|
46
|
+
provider: string
|
|
47
|
+
strategy: 'transcription'
|
|
48
|
+
released: string
|
|
49
|
+
cost: TranscriptionModelCost
|
|
50
|
+
features: TranscriptionModelFeatures
|
|
51
|
+
providerOptions?: ProviderOption[]
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
// ─── shared fragments ────────────────────────────────────────────────────────
|
|
55
|
+
|
|
56
|
+
const openaiBase = {
|
|
57
|
+
provider: 'openai',
|
|
58
|
+
strategy: 'transcription' as const,
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
const elevenlabsBase = {
|
|
62
|
+
provider: 'elevenlabs',
|
|
63
|
+
strategy: 'transcription' as const,
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
const deepgramBase = {
|
|
67
|
+
provider: 'deepgram',
|
|
68
|
+
strategy: 'transcription' as const,
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
const groqBase = {
|
|
72
|
+
provider: 'groq',
|
|
73
|
+
strategy: 'transcription' as const,
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
const cartesiaBase = {
|
|
77
|
+
provider: 'cartesia',
|
|
78
|
+
strategy: 'transcription' as const,
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
const commonInputFormats = ['mp3', 'mp4', 'mpeg', 'mpga', 'm4a', 'wav', 'webm', 'ogg', 'flac']
|
|
82
|
+
|
|
83
|
+
// ─── catalog ─────────────────────────────────────────────────────────────────
|
|
84
|
+
|
|
85
|
+
export const TRANSCRIPTION_CATALOG: TranscriptionModelEntry[] = [
|
|
86
|
+
// ── OpenAI ─────────────────────────────────────────────────────────────
|
|
87
|
+
{
|
|
88
|
+
id: 'whisper-1',
|
|
89
|
+
name: 'Whisper v1',
|
|
90
|
+
description: 'OpenAI Whisper large-v2 model. Reliable, supports 57 languages.',
|
|
91
|
+
...openaiBase,
|
|
92
|
+
released: '2023-03',
|
|
93
|
+
cost: { type: 'per-second', perSecond: 0.0001 }, // $0.006/min
|
|
94
|
+
features: {
|
|
95
|
+
wordTimestamps: true,
|
|
96
|
+
diarization: false,
|
|
97
|
+
languageDetection: true,
|
|
98
|
+
languageHint: true,
|
|
99
|
+
inputFormats: commonInputFormats,
|
|
100
|
+
maxDurationSec: 0,
|
|
101
|
+
},
|
|
102
|
+
},
|
|
103
|
+
{
|
|
104
|
+
id: 'gpt-4o-transcribe',
|
|
105
|
+
name: 'GPT-4o Transcribe',
|
|
106
|
+
description: 'GPT-4o powered transcription. Better accuracy than Whisper on complex audio. No word timestamps (OpenAI API only supports response_format=json for this model, verbose_json is required for timestamps).',
|
|
107
|
+
...openaiBase,
|
|
108
|
+
released: '2025-03',
|
|
109
|
+
cost: { type: 'per-second', perSecond: 0.0001 }, // $0.006/min
|
|
110
|
+
features: {
|
|
111
|
+
// OpenAI API limitation: gpt-4o-transcribe only supports response_format=json,
|
|
112
|
+
// not verbose_json. Word timestamps require verbose_json.
|
|
113
|
+
wordTimestamps: false,
|
|
114
|
+
diarization: false,
|
|
115
|
+
languageDetection: true,
|
|
116
|
+
languageHint: true,
|
|
117
|
+
inputFormats: commonInputFormats,
|
|
118
|
+
maxDurationSec: 0,
|
|
119
|
+
},
|
|
120
|
+
},
|
|
121
|
+
{
|
|
122
|
+
id: 'gpt-4o-mini-transcribe',
|
|
123
|
+
name: 'GPT-4o Mini Transcribe',
|
|
124
|
+
description: 'GPT-4o Mini powered transcription. Cheaper, still better than Whisper. No word timestamps (same OpenAI API limitation as gpt-4o-transcribe).',
|
|
125
|
+
...openaiBase,
|
|
126
|
+
released: '2025-03',
|
|
127
|
+
cost: { type: 'per-second', perSecond: 0.00005 }, // $0.003/min
|
|
128
|
+
features: {
|
|
129
|
+
// Same OpenAI API limitation as gpt-4o-transcribe.
|
|
130
|
+
wordTimestamps: false,
|
|
131
|
+
diarization: false,
|
|
132
|
+
languageDetection: true,
|
|
133
|
+
languageHint: true,
|
|
134
|
+
inputFormats: commonInputFormats,
|
|
135
|
+
maxDurationSec: 0,
|
|
136
|
+
},
|
|
137
|
+
},
|
|
138
|
+
|
|
139
|
+
// ── ElevenLabs ─────────────────────────────────────────────────────────
|
|
140
|
+
{
|
|
141
|
+
id: 'scribe_v1',
|
|
142
|
+
name: 'ElevenLabs Scribe v1',
|
|
143
|
+
description: 'ElevenLabs speech-to-text with word-level timestamps and speaker diarization.',
|
|
144
|
+
...elevenlabsBase,
|
|
145
|
+
released: '2024-12',
|
|
146
|
+
cost: { type: 'per-second', perSecond: 0.0001 },
|
|
147
|
+
features: {
|
|
148
|
+
wordTimestamps: true,
|
|
149
|
+
diarization: true,
|
|
150
|
+
languageDetection: true,
|
|
151
|
+
languageHint: true,
|
|
152
|
+
inputFormats: commonInputFormats,
|
|
153
|
+
maxDurationSec: 0,
|
|
154
|
+
},
|
|
155
|
+
},
|
|
156
|
+
|
|
157
|
+
// ── Deepgram ───────────────────────────────────────────────────────────
|
|
158
|
+
{
|
|
159
|
+
id: 'nova-3',
|
|
160
|
+
name: 'Deepgram Nova 3',
|
|
161
|
+
description: 'Deepgram Nova 3. Fast and accurate with word-level timestamps and diarization.',
|
|
162
|
+
...deepgramBase,
|
|
163
|
+
released: '2025-01',
|
|
164
|
+
cost: { type: 'per-second', perSecond: 0.0000717 }, // $0.0043/min
|
|
165
|
+
features: {
|
|
166
|
+
wordTimestamps: true,
|
|
167
|
+
diarization: true,
|
|
168
|
+
languageDetection: true,
|
|
169
|
+
languageHint: true,
|
|
170
|
+
inputFormats: commonInputFormats,
|
|
171
|
+
maxDurationSec: 0,
|
|
172
|
+
},
|
|
173
|
+
},
|
|
174
|
+
|
|
175
|
+
// ── Groq ───────────────────────────────────────────────────────────────
|
|
176
|
+
{
|
|
177
|
+
id: 'whisper-large-v3',
|
|
178
|
+
name: 'Whisper Large v3 (Groq)',
|
|
179
|
+
description: 'Whisper large-v3 on Groq. Extremely fast inference with word-level timestamps.',
|
|
180
|
+
...groqBase,
|
|
181
|
+
released: '2024-01',
|
|
182
|
+
cost: { type: 'per-second', perSecond: 0.00000185 }, // $0.000111/min
|
|
183
|
+
features: {
|
|
184
|
+
wordTimestamps: true,
|
|
185
|
+
diarization: false,
|
|
186
|
+
languageDetection: true,
|
|
187
|
+
languageHint: true,
|
|
188
|
+
inputFormats: commonInputFormats,
|
|
189
|
+
maxDurationSec: 7200,
|
|
190
|
+
},
|
|
191
|
+
},
|
|
192
|
+
{
|
|
193
|
+
id: 'whisper-large-v3-turbo',
|
|
194
|
+
name: 'Whisper Large v3 Turbo (Groq)',
|
|
195
|
+
description: 'Whisper large-v3-turbo on Groq. Faster variant with slightly less accuracy.',
|
|
196
|
+
...groqBase,
|
|
197
|
+
released: '2024-06',
|
|
198
|
+
cost: { type: 'per-second', perSecond: 0.000000667 }, // $0.00004/min
|
|
199
|
+
features: {
|
|
200
|
+
wordTimestamps: true,
|
|
201
|
+
diarization: false,
|
|
202
|
+
languageDetection: true,
|
|
203
|
+
languageHint: true,
|
|
204
|
+
inputFormats: commonInputFormats,
|
|
205
|
+
maxDurationSec: 7200,
|
|
206
|
+
},
|
|
207
|
+
},
|
|
208
|
+
{
|
|
209
|
+
id: 'distil-whisper-large-v3-en',
|
|
210
|
+
name: 'Distil Whisper Large v3 EN (Groq)',
|
|
211
|
+
description: 'Distilled English-only Whisper on Groq. Cheapest option, English only.',
|
|
212
|
+
...groqBase,
|
|
213
|
+
released: '2024-01',
|
|
214
|
+
cost: { type: 'per-second', perSecond: 0.000000333 }, // $0.00002/min
|
|
215
|
+
features: {
|
|
216
|
+
wordTimestamps: true,
|
|
217
|
+
diarization: false,
|
|
218
|
+
languageDetection: false,
|
|
219
|
+
languageHint: false,
|
|
220
|
+
inputFormats: commonInputFormats,
|
|
221
|
+
maxDurationSec: 7200,
|
|
222
|
+
},
|
|
223
|
+
},
|
|
224
|
+
|
|
225
|
+
// ── Cartesia ───────────────────────────────────────────────────────────
|
|
226
|
+
// Pricing: 1 credit per 2 seconds of audio on /stt batch endpoint.
|
|
227
|
+
// At Startup plan ($49/1.25M credits) ≈ $0.00002/sec.
|
|
228
|
+
{
|
|
229
|
+
id: 'ink-whisper',
|
|
230
|
+
name: 'Cartesia Ink Whisper',
|
|
231
|
+
description: 'Cartesia batch STT. Word-level timestamps, 99+ languages. Based on Whisper.',
|
|
232
|
+
...cartesiaBase,
|
|
233
|
+
released: '2026-01',
|
|
234
|
+
cost: { type: 'per-second', perSecond: 0.00002 },
|
|
235
|
+
features: {
|
|
236
|
+
wordTimestamps: true,
|
|
237
|
+
diarization: false,
|
|
238
|
+
languageDetection: false,
|
|
239
|
+
languageHint: true,
|
|
240
|
+
inputFormats: ['mp3', 'mp4', 'm4a', 'wav', 'webm', 'ogg', 'flac'],
|
|
241
|
+
maxDurationSec: 0,
|
|
242
|
+
},
|
|
243
|
+
},
|
|
244
|
+
]
|
|
245
|
+
|
|
246
|
+
// ─── lookup helpers ──────────────────────────────────────────────────────────
|
|
247
|
+
|
|
248
|
+
export function findTranscriptionModel(modelId: string): TranscriptionModelEntry | undefined {
|
|
249
|
+
return TRANSCRIPTION_CATALOG.find((m) => m.id === modelId)
|
|
250
|
+
}
|
|
@@ -0,0 +1,214 @@
|
|
|
1
|
+
// Programmatic API for egaki audio transcription (speech-to-text).
|
|
2
|
+
// Returns Error | Result (errore style) instead of process.exit/console.error.
|
|
3
|
+
//
|
|
4
|
+
// Two variants:
|
|
5
|
+
// - `transcribeAudio(opts)` — cached, writes JSON to public/generated/transcription/, returns WordTimestamp[]
|
|
6
|
+
// - `transcribeAudioUncached(opts)` — raw, returns TranscribeResult with text + segments
|
|
7
|
+
//
|
|
8
|
+
// Uses direct HTTP calls to each provider API (no AI SDK dependency).
|
|
9
|
+
// Each provider implementation lives in transcription-providers.ts.
|
|
10
|
+
//
|
|
11
|
+
// Usage:
|
|
12
|
+
// import { transcribeAudio, transcribeAudioUncached } from 'egaki/generate'
|
|
13
|
+
// const cached = await transcribeAudio({ audio: fs.readFileSync('audio.mp3') })
|
|
14
|
+
// if (!(cached instanceof Error)) cached // WordTimestamp[]
|
|
15
|
+
// const raw = await transcribeAudioUncached({ audio: fs.readFileSync('audio.mp3') })
|
|
16
|
+
// if (!(raw instanceof Error)) raw.text // full transcript
|
|
17
|
+
import { injectCredentialsToEnv } from './credentials.js'
|
|
18
|
+
import {
|
|
19
|
+
getTranscriptionModelConfig,
|
|
20
|
+
getTranscriptionProvider,
|
|
21
|
+
DEFAULT_TRANSCRIPTION_MODEL,
|
|
22
|
+
} from './transcription-models.js'
|
|
23
|
+
|
|
24
|
+
// ─── autocomplete-friendly union types ───────────────────────────────────────
|
|
25
|
+
|
|
26
|
+
/** Transcription model IDs from the catalog. Accepts arbitrary strings too. */
|
|
27
|
+
export type TranscriptionModelId =
|
|
28
|
+
| 'whisper-1' | 'gpt-4o-transcribe' | 'gpt-4o-mini-transcribe'
|
|
29
|
+
| 'scribe_v1'
|
|
30
|
+
| 'nova-3'
|
|
31
|
+
| 'whisper-large-v3' | 'whisper-large-v3-turbo' | 'distil-whisper-large-v3-en'
|
|
32
|
+
| 'ink-whisper'
|
|
33
|
+
| (string & {})
|
|
34
|
+
|
|
35
|
+
// ─── public types ────────────────────────────────────────────────────────────
|
|
36
|
+
|
|
37
|
+
export interface TranscribeOptions {
|
|
38
|
+
/** Audio data as a Uint8Array or Buffer. */
|
|
39
|
+
audio: Uint8Array
|
|
40
|
+
/** Transcription model ID. Defaults to 'whisper-1' if omitted. */
|
|
41
|
+
model?: TranscriptionModelId
|
|
42
|
+
/** ISO 639-1 language hint (e.g. "en", "es", "fr"). */
|
|
43
|
+
language?: string
|
|
44
|
+
/** Original audio filename, used as the human-readable cache file prefix. */
|
|
45
|
+
filename?: string
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
export interface TranscriptionSegment {
|
|
49
|
+
text: string
|
|
50
|
+
startSecond: number
|
|
51
|
+
endSecond: number
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
export interface TranscribeResult {
|
|
55
|
+
text: string
|
|
56
|
+
segments: TranscriptionSegment[]
|
|
57
|
+
language: string | undefined
|
|
58
|
+
durationInSeconds: number | undefined
|
|
59
|
+
model: string
|
|
60
|
+
cost: number | null
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
// ─── word-level timestamp types ──────────────────────────────────────────────
|
|
64
|
+
|
|
65
|
+
/** A single word with its start/end time in seconds. */
|
|
66
|
+
export interface WordTimestamp {
|
|
67
|
+
word: string
|
|
68
|
+
startSecond: number
|
|
69
|
+
endSecond: number
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
/**
|
|
73
|
+
* Convert TranscriptionSegment[] to WordTimestamp[].
|
|
74
|
+
*
|
|
75
|
+
* When segments are word-level (one word per segment), maps directly.
|
|
76
|
+
* When segments are phrase/sentence-level, splits by whitespace and
|
|
77
|
+
* distributes time evenly across words as a fallback.
|
|
78
|
+
*/
|
|
79
|
+
export function segmentsToWordTimestamps(segments: TranscriptionSegment[]): WordTimestamp[] {
|
|
80
|
+
const result: WordTimestamp[] = []
|
|
81
|
+
for (const seg of segments) {
|
|
82
|
+
const trimmed = seg.text.trim()
|
|
83
|
+
// Skip empty/whitespace-only segments (ElevenLabs returns space segments)
|
|
84
|
+
if (!trimmed) continue
|
|
85
|
+
const words = trimmed.split(/\s+/).filter(Boolean)
|
|
86
|
+
if (words.length <= 1) {
|
|
87
|
+
result.push({ word: trimmed, startSecond: seg.startSecond, endSecond: seg.endSecond })
|
|
88
|
+
continue
|
|
89
|
+
}
|
|
90
|
+
// Phrase-level segment: distribute time evenly across words
|
|
91
|
+
const duration = seg.endSecond - seg.startSecond
|
|
92
|
+
const wordDuration = duration / words.length
|
|
93
|
+
for (let i = 0; i < words.length; i++) {
|
|
94
|
+
result.push({
|
|
95
|
+
word: words[i]!,
|
|
96
|
+
startSecond: seg.startSecond + i * wordDuration,
|
|
97
|
+
endSecond: seg.startSecond + (i + 1) * wordDuration,
|
|
98
|
+
})
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
return result
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
/**
|
|
105
|
+
* Convert WordTimestamp[] to @remotion/captions Caption[] format.
|
|
106
|
+
* First word has no leading space; subsequent words get a leading space
|
|
107
|
+
* (required by createTikTokStyleCaptions).
|
|
108
|
+
*/
|
|
109
|
+
export function wordTimestampsToCaptions(words: WordTimestamp[]): Array<{
|
|
110
|
+
text: string
|
|
111
|
+
startMs: number
|
|
112
|
+
endMs: number
|
|
113
|
+
timestampMs: number
|
|
114
|
+
confidence: number
|
|
115
|
+
}> {
|
|
116
|
+
return words.map((w, i) => ({
|
|
117
|
+
text: i === 0 ? w.word : ` ${w.word}`,
|
|
118
|
+
startMs: w.startSecond * 1000,
|
|
119
|
+
endMs: w.endSecond * 1000,
|
|
120
|
+
timestampMs: ((w.startSecond + w.endSecond) / 2) * 1000,
|
|
121
|
+
confidence: 1,
|
|
122
|
+
}))
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
// ─── cost calculation ────────────────────────────────────────────────────────
|
|
126
|
+
|
|
127
|
+
export function calculateTranscriptionCost(
|
|
128
|
+
cost: { type: 'per-second'; perSecond: number },
|
|
129
|
+
durationInSeconds: number | undefined,
|
|
130
|
+
): number | null {
|
|
131
|
+
if (durationInSeconds == null) return null
|
|
132
|
+
return durationInSeconds * cost.perSecond
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
// ─── transcribeAudioUncached ─────────────────────────────────────────────────
|
|
136
|
+
|
|
137
|
+
/**
|
|
138
|
+
* Transcribe audio to text. Auto-detects which provider to use
|
|
139
|
+
* based on model ID. Shares credentials with the CLI.
|
|
140
|
+
*/
|
|
141
|
+
export async function transcribeAudioUncached(opts: TranscribeOptions): Promise<Error | TranscribeResult> {
|
|
142
|
+
injectCredentialsToEnv()
|
|
143
|
+
|
|
144
|
+
const model = opts.model ?? DEFAULT_TRANSCRIPTION_MODEL
|
|
145
|
+
const config = getTranscriptionModelConfig(model)
|
|
146
|
+
if (config instanceof Error) return config
|
|
147
|
+
|
|
148
|
+
if (opts.language && !config.features.languageHint) {
|
|
149
|
+
return new Error(`Model ${model} does not support a language hint`)
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
const provider = getTranscriptionProvider(config.provider)
|
|
153
|
+
if (provider instanceof Error) return provider
|
|
154
|
+
|
|
155
|
+
let result
|
|
156
|
+
try {
|
|
157
|
+
result = await provider.transcribe({
|
|
158
|
+
audio: opts.audio,
|
|
159
|
+
modelId: model,
|
|
160
|
+
language: opts.language,
|
|
161
|
+
})
|
|
162
|
+
} catch (err) {
|
|
163
|
+
return err instanceof Error ? err : new Error(String(err))
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
const cost = calculateTranscriptionCost(config.cost, result.durationInSeconds)
|
|
167
|
+
|
|
168
|
+
return {
|
|
169
|
+
text: result.text,
|
|
170
|
+
segments: result.segments,
|
|
171
|
+
language: result.language,
|
|
172
|
+
durationInSeconds: result.durationInSeconds,
|
|
173
|
+
model,
|
|
174
|
+
cost,
|
|
175
|
+
}
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
// ─── cached transcribeAudio ─────────────────────────────────────────────────
|
|
179
|
+
|
|
180
|
+
import fs from 'node:fs'
|
|
181
|
+
import path from 'node:path'
|
|
182
|
+
import { cachedGenerate } from './cached-generate.js'
|
|
183
|
+
|
|
184
|
+
export const transcribeAudio = cachedGenerate<TranscribeOptions, { wordTimestamps: WordTimestamp[]; text: string }, WordTimestamp[]>({
|
|
185
|
+
namespace: 'transcription',
|
|
186
|
+
prefixFrom: (p) => {
|
|
187
|
+
if (p.filename) {
|
|
188
|
+
const ext = path.extname(p.filename)
|
|
189
|
+
return path.basename(p.filename, ext)
|
|
190
|
+
}
|
|
191
|
+
return 'transcription'
|
|
192
|
+
},
|
|
193
|
+
// Exclude filename from cache key — it only affects the human-readable prefix,
|
|
194
|
+
// not the identity of the transcription result.
|
|
195
|
+
cacheKey: ({ filename, ...rest }) => rest as Record<string, unknown>,
|
|
196
|
+
modelFrom: (p) => p.model,
|
|
197
|
+
generate: async (params) => {
|
|
198
|
+
const result = await transcribeAudioUncached(params)
|
|
199
|
+
if (result instanceof Error) throw result
|
|
200
|
+
return {
|
|
201
|
+
wordTimestamps: segmentsToWordTimestamps(result.segments),
|
|
202
|
+
text: result.text,
|
|
203
|
+
}
|
|
204
|
+
},
|
|
205
|
+
serialize: (result) => ({
|
|
206
|
+
json: result.wordTimestamps,
|
|
207
|
+
extension: '.json',
|
|
208
|
+
// Use the transcript text as a more readable filename prefix
|
|
209
|
+
prefix: result.text || undefined,
|
|
210
|
+
}),
|
|
211
|
+
deserialize: ({ filePath }) => {
|
|
212
|
+
return JSON.parse(fs.readFileSync(filePath, 'utf-8'))
|
|
213
|
+
},
|
|
214
|
+
})
|