hyperframes 0.8.69 → 0.8.70
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.js +18 -13
- package/dist/commands/layout-audit.browser.js +36 -4
- package/dist/fontLocalizeCli.js +2 -2
- package/dist/hyperframe-runtime.js +37 -37
- package/dist/hyperframe.manifest.json +1 -1
- package/dist/hyperframe.runtime.iife.js +37 -37
- package/dist/hyperframes-player.global.js +1 -1
- package/dist/skills/media-use/audio/references/tts-to-captions.md +11 -2
- package/dist/skills/media-use/audio/references/tts.md +84 -6
- package/dist/skills/media-use/audio/scripts/audio.mjs +10 -3
- package/dist/skills/media-use/audio/scripts/gemini-pipeline.test.mjs +113 -0
- package/dist/skills/media-use/audio/scripts/lib/gemini-auth.mjs +43 -0
- package/dist/skills/media-use/audio/scripts/lib/gemini-auth.py +52 -0
- package/dist/skills/media-use/audio/scripts/lib/gemini-auth.test.mjs +86 -0
- package/dist/skills/media-use/audio/scripts/lib/gemini-auth_test.py +90 -0
- package/dist/skills/media-use/audio/scripts/lib/gemini-tts.mjs +122 -0
- package/dist/skills/media-use/audio/scripts/lib/gemini-tts.test.mjs +243 -0
- package/dist/skills/media-use/audio/scripts/lib/tts.mjs +13 -2
- package/dist/skills/media-use/references/audio.md +2 -1
- package/dist/studio/assets/{hyperframes-player-B8bDHi2F.js → hyperframes-player-CYdwxTwz.js} +1 -1
- package/dist/studio/assets/{index-784N6Mag.js → index-BPafHV3q.js} +1 -1
- package/dist/studio/assets/{index-D-JPuFj7.js → index-D7hgjzBq.js} +1 -1
- package/dist/studio/assets/{index-Bmd-35nQ.js → index-Egnc_kWw.js} +4 -4
- package/dist/studio/index.html +1 -1
- package/package.json +1 -1
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
"use strict";var HyperframesPlayer=(()=>{var Se=Object.defineProperty;var fn=Object.getOwnPropertyDescriptor;var hn=Object.getOwnPropertyNames;var bn=Object.prototype.hasOwnProperty;var gn=(n,e)=>{for(var t in e)Se(n,t,{get:e[t],enumerable:!0})},_n=(n,e,t,i)=>{if(e&&typeof e=="object"||typeof e=="function")for(let r of hn(e))!bn.call(n,r)&&r!==t&&Se(n,r,{get:()=>e[r],enumerable:!(i=fn(e,r))||i.enumerable});return n};var yn=n=>_n(Se({},"__esModule",{value:!0}),n);var Xi={};gn(Xi,{HyperframesPlayer:()=>Ee,SPEED_PRESETS:()=>we,formatSpeed:()=>G,formatTime:()=>ie});function ct(n){return n.hasRuntime||n.runtimeInjected?!1:!!(n.hasNestedCompositions||n.hasTimelines&&n.attempts>=5)}function H(n){return typeof n=="object"&&n!==null}function mt(n){return H(n)&&typeof n.getDuration=="function"}function pt(n){return H(n)&&typeof n.duration=="function"&&typeof n.time=="function"&&typeof n.seek=="function"&&typeof n.play=="function"&&typeof n.pause=="function"}var L="https://cdn.jsdelivr.net/npm/@hyperframes/core@0.8.
|
|
1
|
+
"use strict";var HyperframesPlayer=(()=>{var Se=Object.defineProperty;var fn=Object.getOwnPropertyDescriptor;var hn=Object.getOwnPropertyNames;var bn=Object.prototype.hasOwnProperty;var gn=(n,e)=>{for(var t in e)Se(n,t,{get:e[t],enumerable:!0})},_n=(n,e,t,i)=>{if(e&&typeof e=="object"||typeof e=="function")for(let r of hn(e))!bn.call(n,r)&&r!==t&&Se(n,r,{get:()=>e[r],enumerable:!(i=fn(e,r))||i.enumerable});return n};var yn=n=>_n(Se({},"__esModule",{value:!0}),n);var Xi={};gn(Xi,{HyperframesPlayer:()=>Ee,SPEED_PRESETS:()=>we,formatSpeed:()=>G,formatTime:()=>ie});function ct(n){return n.hasRuntime||n.runtimeInjected?!1:!!(n.hasNestedCompositions||n.hasTimelines&&n.attempts>=5)}function H(n){return typeof n=="object"&&n!==null}function mt(n){return H(n)&&typeof n.getDuration=="function"}function pt(n){return H(n)&&typeof n.duration=="function"&&typeof n.time=="function"&&typeof n.seek=="function"&&typeof n.play=="function"&&typeof n.pause=="function"}var L="https://cdn.jsdelivr.net/npm/@hyperframes/core@0.8.70/dist/hyperframe.runtime.iife.js";function V(n){if(n===null)return null;let e=Number.parseInt(n,10);return Number.isFinite(e)&&e>0?e:null}function vn(n){let e=n?.querySelector("[data-composition-id][data-width][data-height]")??n?.querySelector("[data-width][data-height]");if(!e)return null;let t=V(e.getAttribute("data-width")),i=V(e.getAttribute("data-height"));return t!==null&&i!==null?{width:t,height:i}:null}var ne=class{constructor(e,t){this._iframe=e;this._callbacks=t}_iframe;_callbacks;_interval=null;_runtimeInjected=!1;get runtimeInjected(){return this._runtimeInjected}start(){this.stop(),this._runtimeInjected=!1;let e=0;this._interval=setInterval(()=>{e++;try{let t=this._iframe.contentWindow;if(!t)return;let i=!!(t.__hf||t.__player),r=!!(t.__timelines&&Object.keys(t.__timelines).length>0),o=!!this._iframe.contentDocument?.querySelector("[data-composition-src]");if(ct({hasRuntime:i,hasTimelines:r,hasNestedCompositions:o,runtimeInjected:this._runtimeInjected,attempts:e})){this._injectRuntime();return}if(this._runtimeInjected&&!i)return;let a=this._resolvePlaybackDurationAdapter(t);if(a&&a.getDuration()>0){this.stop();let s=vn(this._iframe.contentDocument);this._callbacks.onReady({duration:a.getDuration(),adapter:a,compositionSize:s});return}}catch{}e>=40&&(this.stop(),this._callbacks.onError("Composition timeline not found after 8s"))},200)}stop(){this._interval!==null&&(clearInterval(this._interval),this._interval=null)}resolveDirectTimelineAdapter(){try{let e=this._iframe.contentWindow;return e?this._resolveDirectTimelineAdapterFromWindow(e):null}catch{return null}}resolveDirectTimelineAdapterFromWindow(e){return this._resolveDirectTimelineAdapterFromWindow(e)}hasRuntimeBridge(e){return Reflect.get(e,"__hf")!==void 0||H(Reflect.get(e,"__player"))}_injectRuntime(){this._runtimeInjected=!0;try{let e=this._iframe.contentDocument;if(!e)return;let t=e.createElement("script");t.src=L,(e.head||e.documentElement).appendChild(t),this._callbacks.onRuntimeInjected?.()}catch{}}_resolveDirectTimelineAdapterFromWindow(e){if(this.hasRuntimeBridge(e))return null;let t=Reflect.get(e,"__timelines");if(!H(t))return null;let i=Object.keys(t);if(i.length===0)return null;let r=this._iframe.contentDocument?.querySelector("[data-composition-id]")?.getAttribute("data-composition-id"),o=r&&r in t?r:i[i.length-1],a=t[o];return pt(a)?a:null}_resolvePlaybackDurationAdapter(e){let t=Reflect.get(e,"__player");if(mt(t))return{kind:"runtime",getDuration:()=>t.getDuration()};let i=this._resolveDirectTimelineAdapterFromWindow(e);return i?{kind:"direct-timeline",timeline:i,getDuration:()=>i.duration()}:null}};var ft=`
|
|
2
2
|
:host {
|
|
3
3
|
display: block;
|
|
4
4
|
position: relative;
|
|
@@ -14,9 +14,14 @@ node skills/media-use/audio/scripts/heygen-tts.mjs \
|
|
|
14
14
|
|
|
15
15
|
`narration.words.json` is already in the `[{ id, text, start, end }]` shape the captions pipeline consumes — no separate transcribe pass.
|
|
16
16
|
|
|
17
|
-
## Path B — ElevenLabs / Kokoro (TTS →
|
|
17
|
+
## Path B — Gemini / ElevenLabs / Kokoro (TTS → transcription)
|
|
18
18
|
|
|
19
|
-
These
|
|
19
|
+
These adapters supply audio without word data. The shared audio engine runs
|
|
20
|
+
transcription automatically when timings are absent. For Gemini, use the
|
|
21
|
+
request in [Text to speech](tts.md#gemini-narration), then consume
|
|
22
|
+
`audio_meta.json` → `voices[].words`.
|
|
23
|
+
|
|
24
|
+
For a standalone local Kokoro generation, generate the audio, then transcribe:
|
|
20
25
|
|
|
21
26
|
```bash
|
|
22
27
|
npx hyperframes tts script.txt --voice af_heart --output narration.wav
|
|
@@ -24,3 +29,7 @@ npx hyperframes transcribe narration.wav --model small.en # voice af_heart is
|
|
|
24
29
|
```
|
|
25
30
|
|
|
26
31
|
Whisper extracts precise word boundaries from the generated audio, so caption timing matches delivery without hand-tuning. Match `--model` to the voice's language (use `small.en` for `a`/`b` prefixes, `small --language <code>` otherwise). Then consume `transcript.json` via the caption references in `captions/`.
|
|
32
|
+
|
|
33
|
+
For Gemini, verify that transcription preserved the script, especially names,
|
|
34
|
+
numbers, and delivery pauses. If `words` is empty, resolve the transcription
|
|
35
|
+
failure before captioning. Generate and align again after changing the read.
|
|
@@ -36,6 +36,10 @@ Use another voice only for a documented reason, and write the reason down.
|
|
|
36
36
|
|
|
37
37
|
## Available routes
|
|
38
38
|
|
|
39
|
+
Gemini is an explicit alternative to the automatic provider order below. A
|
|
40
|
+
request to use Gemini already chooses the provider; do not redirect that user
|
|
41
|
+
to HeyGen sign-in. Read the Gemini section for its credential requirement.
|
|
42
|
+
|
|
39
43
|
| Order | Provider | Env trigger | Voice IDs | Word timestamps | Audio format |
|
|
40
44
|
| ----- | ----------------- | ------------------------------------------- | ------------------------------------------- | ----------------------------------------- | -------------------- |
|
|
41
45
|
| 1 | HeyGen (Starfish) | `$HEYGEN_API_KEY` / `~/.heygen/credentials` | UUIDs from `GET /v3/voices?engine=starfish` | **Yes** (`word_timestamps[]` in response) | mp3 → wav via ffmpeg |
|
|
@@ -83,12 +87,86 @@ node skills/media-use/audio/scripts/heygen-tts.mjs --list # public starfish vo
|
|
|
83
87
|
|
|
84
88
|
## When to use which provider
|
|
85
89
|
|
|
86
|
-
| Goal | Use
|
|
87
|
-
| --------------------------------------------------------- |
|
|
88
|
-
| Best voice quality + word timestamps in one call | **HeyGen**
|
|
89
|
-
| Drop-in cloud TTS, big voice catalog | **ElevenLabs**
|
|
90
|
-
| Offline, no API key, fast iteration | **Kokoro**
|
|
91
|
-
|
|
|
90
|
+
| Goal | Use |
|
|
91
|
+
| --------------------------------------------------------- | --------------------------------------------------------- |
|
|
92
|
+
| Best voice quality + word timestamps in one call | **HeyGen** |
|
|
93
|
+
| Drop-in cloud TTS, big voice catalog | **ElevenLabs** |
|
|
94
|
+
| Offline, no API key, fast iteration | **Kokoro** |
|
|
95
|
+
| Directed delivery with Gemini prebuilt or custom voices | **Gemini** (explicit selection; transcription for timing) |
|
|
96
|
+
| Non-English multilingual with deterministic phonemization | **Kokoro** (`ef_dora`, `jf_alpha`, `zf_xiaobei`, …) |
|
|
97
|
+
|
|
98
|
+
## Gemini narration
|
|
99
|
+
|
|
100
|
+
Use the shared audio engine, not `hyperframes tts`. Authenticate with either:
|
|
101
|
+
|
|
102
|
+
- `GEMINI_API_KEY` or `GOOGLE_API_KEY` (first set key wins).
|
|
103
|
+
- A service-account JSON file at `GOOGLE_APPLICATION_CREDENTIALS`, or injected
|
|
104
|
+
JSON in `GCS_CREDS`. Install `google-auth requests` in the Python 3 environment
|
|
105
|
+
used by the helper. A configured file takes precedence over injected JSON.
|
|
106
|
+
|
|
107
|
+
API keys take precedence over service accounts: unset both key variables to use
|
|
108
|
+
OAuth. Never put credentials in a request file or composition. The helper
|
|
109
|
+
obtains a fresh OAuth token for each generation with the
|
|
110
|
+
`generative-language.retriever` scope. The quota project resolves from
|
|
111
|
+
`GOOGLE_CLOUD_PROJECT`, then `GCLOUD_PROJECT_ID`, then the service-account JSON.
|
|
112
|
+
User ADC files and metadata-server authentication are not supported.
|
|
113
|
+
|
|
114
|
+
Both routes call the Gemini Developer Interactions API, not Cloud TTS or Vertex
|
|
115
|
+
AI. Those APIs have separate model catalogs and access requirements.
|
|
116
|
+
|
|
117
|
+
Save this as `audio_request.json` in the project:
|
|
118
|
+
|
|
119
|
+
```json
|
|
120
|
+
{
|
|
121
|
+
"provider": "gemini",
|
|
122
|
+
"tts_model": "gemini-3.8-flash-tts",
|
|
123
|
+
"voice": "Kore",
|
|
124
|
+
"lang": "en",
|
|
125
|
+
"style": "Warm, clear, conversational. Leave a short pause between sentences.",
|
|
126
|
+
"lines": [
|
|
127
|
+
{ "id": "intro", "text": "Every word has a moment. Let the picture follow the voice." }
|
|
128
|
+
],
|
|
129
|
+
"bgm": { "mode": "none" }
|
|
130
|
+
}
|
|
131
|
+
```
|
|
132
|
+
|
|
133
|
+
```bash
|
|
134
|
+
node <SKILL_DIR>/audio/scripts/audio.mjs \
|
|
135
|
+
--request ./audio_request.json --hyperframes . --out ./audio_meta.json --only tts
|
|
136
|
+
```
|
|
137
|
+
|
|
138
|
+
The engine saves `assets/voice/intro.wav`, measures its duration, and transcribes
|
|
139
|
+
it into `voices[].words` in `audio_meta.json`. Check that every requested line
|
|
140
|
+
has audio and nonempty word timings before building a captioned video. Review
|
|
141
|
+
the timings against the actual audio; transcription is estimated alignment,
|
|
142
|
+
not native TTS timestamps. Do not distribute words evenly across a clip.
|
|
143
|
+
|
|
144
|
+
- **Models:** `gemini-3.8-flash-tts` (default), `gemini-3.8-flash-lite-tts`,
|
|
145
|
+
`gemini-3.1-flash-tts-preview`, `gemini-2.5-pro-preview-tts`, and
|
|
146
|
+
`gemini-2.5-flash-preview-tts`. Use these exact Developer API IDs; Cloud TTS
|
|
147
|
+
aliases such as `gemini-2.5-flash-tts` are not accepted.
|
|
148
|
+
- **Voice:** `Kore` by default; pass another prebuilt voice or an existing custom
|
|
149
|
+
voice ID for 3.8. Older models require prebuilt voices. Creating or replicating
|
|
150
|
+
voices is outside this helper.
|
|
151
|
+
- **Delivery:** Put directions in `style`, not in spoken `text`. Each line can
|
|
152
|
+
override `style`. 3.8 uses structured annotations; older models receive a
|
|
153
|
+
delivery prompt before the transcript. Check that directions were not spoken.
|
|
154
|
+
Use style for pacing; numeric `speed` must be omitted or 1.
|
|
155
|
+
- **Audio:** 3.8 returns a complete WAV. Older models return mono 16-bit PCM,
|
|
156
|
+
which the helper wraps as WAV at the returned sample rate without resampling.
|
|
157
|
+
No generation calls run during playback or rendering.
|
|
158
|
+
- **Timing:** This adapter requests no native word timestamps. It uses the
|
|
159
|
+
existing transcription pass. `lang` selects transcription language; Gemini
|
|
160
|
+
infers speech language from the text.
|
|
161
|
+
- **Workflow adapters:** Product-launch, faceless-explainer, and PR-video audio
|
|
162
|
+
scripts accept `--provider gemini --voice Kore --tts-model gemini-3.8-flash-tts
|
|
163
|
+
--style "Warm and clear"`.
|
|
164
|
+
|
|
165
|
+
An API error is reported as a failed line; an explicitly chosen Gemini voice
|
|
166
|
+
never silently falls back to another provider. Check the engine's anomalies
|
|
167
|
+
and output metadata, not only its exit code.
|
|
168
|
+
|
|
169
|
+
API contract: [Google's speech generation guide](https://ai.google.dev/gemini-api/docs/speech-generation).
|
|
92
170
|
|
|
93
171
|
## ffmpeg requirement
|
|
94
172
|
|
|
@@ -15,7 +15,7 @@
|
|
|
15
15
|
//
|
|
16
16
|
// ── audio_request.json (input) ────────────────────────────────────────────────
|
|
17
17
|
// {
|
|
18
|
-
// "provider": "auto", // auto|heygen|elevenlabs|kokoro (override: --provider)
|
|
18
|
+
// "provider": "auto", // auto|heygen|elevenlabs|kokoro|gemini (override: --provider)
|
|
19
19
|
// "lang": "en", "speed": 1.0,
|
|
20
20
|
// "lines": [ // one TTS unit each; id joins back to the caller's model
|
|
21
21
|
// { "id": "01", "text": "...", "sfx": ["whoosh", "ui click"] }
|
|
@@ -113,7 +113,6 @@ const speed = Number(speedOverride ?? request.speed ?? 1.0) || 1.0;
|
|
|
113
113
|
// ── env + HeyGen availability (the single switch) ─────────────────────────────
|
|
114
114
|
loadEnvFromDir(hyperframesDir);
|
|
115
115
|
const heygenOK = heygenCredential() !== null;
|
|
116
|
-
const headers = heygenOK ? heygenAuthHeaders() : null;
|
|
117
116
|
|
|
118
117
|
// ── merge base: preserve sections not selected by --only ──────────────────────
|
|
119
118
|
const audioMeta = openAudioMeta(outPath);
|
|
@@ -153,6 +152,8 @@ if (only.has("tts") && lines.length) {
|
|
|
153
152
|
voiceId,
|
|
154
153
|
lang,
|
|
155
154
|
speed,
|
|
155
|
+
model: flag("tts-model", request.tts_model),
|
|
156
|
+
style: flag("style", line.style ?? request.style),
|
|
156
157
|
wavAbs: abs,
|
|
157
158
|
hyperframesDir,
|
|
158
159
|
});
|
|
@@ -210,7 +211,12 @@ if (only.has("bgm")) {
|
|
|
210
211
|
console.error(`· bgm: disabled`);
|
|
211
212
|
} else if (mode === "retrieve") {
|
|
212
213
|
try {
|
|
213
|
-
bgm = await retrieveBgm({
|
|
214
|
+
bgm = await retrieveBgm({
|
|
215
|
+
query: request.bgm?.query,
|
|
216
|
+
headers: heygenAuthHeaders(),
|
|
217
|
+
hyperframesDir,
|
|
218
|
+
hasVoice,
|
|
219
|
+
});
|
|
214
220
|
if (bgm) {
|
|
215
221
|
bgmFields.bgm_provider = "heygen";
|
|
216
222
|
bgmFields.bgm_mode = "retrieve";
|
|
@@ -262,6 +268,7 @@ if (only.has("sfx")) {
|
|
|
262
268
|
.map((name) => ({ id: String(l.id), name: String(name).trim() }))
|
|
263
269
|
.filter((c) => c.name),
|
|
264
270
|
);
|
|
271
|
+
const headers = heygenOK && cues.length ? heygenAuthHeaders() : null;
|
|
265
272
|
const res = await resolveSfx({ cues, heygenOK, headers, hyperframesDir, sfxLibDir });
|
|
266
273
|
sfx = res.sfx;
|
|
267
274
|
anomalies.push(...res.anomalies);
|
|
@@ -0,0 +1,113 @@
|
|
|
1
|
+
import assert from "node:assert/strict";
|
|
2
|
+
import { test } from "node:test";
|
|
3
|
+
import { mkdtempSync, writeFileSync, readFileSync, mkdirSync, rmSync } from "node:fs";
|
|
4
|
+
import { join } from "node:path";
|
|
5
|
+
import { tmpdir } from "node:os";
|
|
6
|
+
import { spawnSync } from "node:child_process";
|
|
7
|
+
|
|
8
|
+
// Exercise the actual engine and provider together; only the paid API and
|
|
9
|
+
// external audio tools are fixtures. This is not a live synthesis/render test.
|
|
10
|
+
for (const expired of [false, true]) {
|
|
11
|
+
for (const only of ["tts", "tts,bgm,sfx"]) {
|
|
12
|
+
test(`Gemini engine returns caption metadata with ${expired ? "expired" : "absent"} HeyGen credentials (${only})`, (t) => {
|
|
13
|
+
const dir = mkdtempSync(join(tmpdir(), "hf-gemini-pipeline-"));
|
|
14
|
+
t.after(() => rmSync(dir, { recursive: true, force: true }));
|
|
15
|
+
const config = join(dir, "heygen");
|
|
16
|
+
mkdirSync(config);
|
|
17
|
+
if (expired)
|
|
18
|
+
writeFileSync(
|
|
19
|
+
join(config, "credentials"),
|
|
20
|
+
JSON.stringify({
|
|
21
|
+
oauth: { access_token: "expired-fixture", expires_at: "2000-01-01T00:00:00Z" },
|
|
22
|
+
}),
|
|
23
|
+
);
|
|
24
|
+
const bin = join(dir, "bin");
|
|
25
|
+
mkdirSync(bin);
|
|
26
|
+
const preload = join(dir, "fetch.mjs");
|
|
27
|
+
writeFileSync(
|
|
28
|
+
preload,
|
|
29
|
+
`
|
|
30
|
+
import assert from 'node:assert/strict';
|
|
31
|
+
globalThis.fetch = async (url, options) => {
|
|
32
|
+
assert.equal(url, 'https://generativelanguage.googleapis.com/v1beta/interactions');
|
|
33
|
+
const body = JSON.parse(options.body);
|
|
34
|
+
assert.equal(body.model, 'gemini-3.8-flash-lite-tts');
|
|
35
|
+
assert.equal(body.input[0].content[0].annotations[0].style, 'Line direction');
|
|
36
|
+
const bytes = Buffer.alloc(48);
|
|
37
|
+
bytes.write('RIFF'); bytes.write('WAVE', 8);
|
|
38
|
+
return Response.json({status:'completed', steps:[{type:'model_output', content:[
|
|
39
|
+
{type:'audio', mime_type:'audio/wav', data:bytes.toString('base64')}
|
|
40
|
+
]}]});
|
|
41
|
+
};
|
|
42
|
+
`,
|
|
43
|
+
);
|
|
44
|
+
writeFileSync(
|
|
45
|
+
join(bin, "npx"),
|
|
46
|
+
`#!${process.execPath}
|
|
47
|
+
const assert = require('node:assert/strict');
|
|
48
|
+
const fs = require('node:fs');
|
|
49
|
+
const path = require('node:path');
|
|
50
|
+
const args = process.argv.slice(2);
|
|
51
|
+
assert.deepEqual(args.slice(0, 3), ['hyperframes', 'transcribe', 'assets/voice/intro.wav']);
|
|
52
|
+
assert.ok(fs.existsSync(args[2]));
|
|
53
|
+
assert.equal(args[args.indexOf('--model')+1], 'small.en');
|
|
54
|
+
fs.writeFileSync(path.join(args[args.indexOf('--dir')+1], 'transcript.json'), JSON.stringify([
|
|
55
|
+
{text:'Hello',start:0.1,end:0.4}, {text:'there.',start:0.5,end:0.9}
|
|
56
|
+
]));
|
|
57
|
+
`,
|
|
58
|
+
{ mode: 0o755 },
|
|
59
|
+
);
|
|
60
|
+
writeFileSync(join(bin, "ffprobe"), "#!/bin/sh\necho 1.25\n", { mode: 0o755 });
|
|
61
|
+
writeFileSync(join(dir, ".env"), "");
|
|
62
|
+
writeFileSync(
|
|
63
|
+
join(dir, "audio_request.json"),
|
|
64
|
+
JSON.stringify({
|
|
65
|
+
provider: "gemini",
|
|
66
|
+
tts_model: "gemini-3.8-flash-lite-tts",
|
|
67
|
+
style: "Global direction",
|
|
68
|
+
lines: [{ id: "intro", text: "Hello there.", style: "Line direction" }],
|
|
69
|
+
bgm: { mode: "none" },
|
|
70
|
+
}),
|
|
71
|
+
);
|
|
72
|
+
const result = spawnSync(
|
|
73
|
+
process.execPath,
|
|
74
|
+
[
|
|
75
|
+
"--import",
|
|
76
|
+
preload,
|
|
77
|
+
new URL("./audio.mjs", import.meta.url).pathname,
|
|
78
|
+
"--hyperframes",
|
|
79
|
+
dir,
|
|
80
|
+
"--only",
|
|
81
|
+
only,
|
|
82
|
+
],
|
|
83
|
+
{
|
|
84
|
+
encoding: "utf8",
|
|
85
|
+
env: {
|
|
86
|
+
...process.env,
|
|
87
|
+
HEYGEN_CONFIG_DIR: config,
|
|
88
|
+
HEYGEN_API_KEY: "",
|
|
89
|
+
HYPERFRAMES_API_KEY: "",
|
|
90
|
+
GEMINI_API_KEY: "fixture-key",
|
|
91
|
+
PATH: `${bin}:${process.env.PATH}`,
|
|
92
|
+
},
|
|
93
|
+
},
|
|
94
|
+
);
|
|
95
|
+
assert.equal(result.status, 0, result.stderr);
|
|
96
|
+
const meta = JSON.parse(readFileSync(join(dir, "audio_meta.json"), "utf8"));
|
|
97
|
+
assert.equal(meta.tts_provider, "gemini");
|
|
98
|
+
assert.equal(meta.voice_id, "Kore");
|
|
99
|
+
assert.deepEqual(meta.voices, [
|
|
100
|
+
{
|
|
101
|
+
id: "intro",
|
|
102
|
+
path: "assets/voice/intro.wav",
|
|
103
|
+
duration_s: 1.25,
|
|
104
|
+
words: [
|
|
105
|
+
{ id: "w0", text: "Hello", start: 0.1, end: 0.4 },
|
|
106
|
+
{ id: "w1", text: "there.", start: 0.5, end: 0.9 },
|
|
107
|
+
],
|
|
108
|
+
},
|
|
109
|
+
]);
|
|
110
|
+
assert.equal(meta.total_duration_s, 1.25);
|
|
111
|
+
});
|
|
112
|
+
}
|
|
113
|
+
}
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
import { spawnSync } from "node:child_process";
|
|
2
|
+
import { fileURLToPath } from "node:url";
|
|
3
|
+
import { pythonInvocation } from "./python.mjs";
|
|
4
|
+
|
|
5
|
+
export function geminiConfigured() {
|
|
6
|
+
return Boolean(
|
|
7
|
+
process.env.GEMINI_API_KEY ||
|
|
8
|
+
process.env.GOOGLE_API_KEY ||
|
|
9
|
+
process.env.GOOGLE_APPLICATION_CREDENTIALS ||
|
|
10
|
+
process.env.GCS_CREDS,
|
|
11
|
+
);
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
export function geminiAuth({ run = spawnSync } = {}) {
|
|
15
|
+
const key = process.env.GEMINI_API_KEY || process.env.GOOGLE_API_KEY;
|
|
16
|
+
if (key) return { headers: { "x-goog-api-key": key }, secret: key };
|
|
17
|
+
if (!geminiConfigured()) {
|
|
18
|
+
throw new Error(
|
|
19
|
+
"Gemini TTS needs GEMINI_API_KEY or GOOGLE_API_KEY, or service-account credentials (GOOGLE_APPLICATION_CREDENTIALS or GCS_CREDS)",
|
|
20
|
+
);
|
|
21
|
+
}
|
|
22
|
+
const { cmd, args } = pythonInvocation([
|
|
23
|
+
fileURLToPath(new URL("./gemini-auth.py", import.meta.url)),
|
|
24
|
+
]);
|
|
25
|
+
const result = run(cmd, args, { encoding: "utf8", timeout: 60_000, maxBuffer: 1024 * 1024 });
|
|
26
|
+
let data;
|
|
27
|
+
try {
|
|
28
|
+
data = JSON.parse(result.stdout);
|
|
29
|
+
} catch {
|
|
30
|
+
throw new Error(
|
|
31
|
+
"Gemini service-account authentication failed; check Python and install google-auth requests",
|
|
32
|
+
);
|
|
33
|
+
}
|
|
34
|
+
if (result.status !== 0 || !data.token) {
|
|
35
|
+
// Do not echo subprocess output: it could contain credentials.
|
|
36
|
+
throw new Error(
|
|
37
|
+
"Gemini service-account authentication failed; check service-account JSON, permissions, and Python packages google-auth requests",
|
|
38
|
+
);
|
|
39
|
+
}
|
|
40
|
+
const headers = { Authorization: `Bearer ${data.token}` };
|
|
41
|
+
if (data.project) headers["x-goog-user-project"] = data.project;
|
|
42
|
+
return { headers, secret: data.token };
|
|
43
|
+
}
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
"""Obtain Gemini OAuth headers without persisting credentials or tokens."""
|
|
2
|
+
|
|
3
|
+
import json
|
|
4
|
+
import os
|
|
5
|
+
import sys
|
|
6
|
+
|
|
7
|
+
# Keep the credential subprocess protocol UTF-8 on every platform.
|
|
8
|
+
for _stream in (sys.stdout, sys.stderr):
|
|
9
|
+
if hasattr(_stream, "reconfigure"):
|
|
10
|
+
_stream.reconfigure(encoding="utf-8", errors=_stream.errors)
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def authenticate():
|
|
14
|
+
try:
|
|
15
|
+
from google.auth.transport.requests import Request
|
|
16
|
+
from google.oauth2 import service_account
|
|
17
|
+
except ImportError:
|
|
18
|
+
return {"error": "Service-account auth needs Python packages: pip install google-auth requests"}
|
|
19
|
+
|
|
20
|
+
try:
|
|
21
|
+
path = os.environ.get("GOOGLE_APPLICATION_CREDENTIALS")
|
|
22
|
+
if path:
|
|
23
|
+
with open(path, encoding="utf-8") as source:
|
|
24
|
+
info = json.load(source)
|
|
25
|
+
else:
|
|
26
|
+
info = json.loads(os.environ["GCS_CREDS"])
|
|
27
|
+
if info.get("type") != "service_account":
|
|
28
|
+
return {"error": "Gemini requires service-account JSON, not user ADC credentials"}
|
|
29
|
+
# Only exchange credentials with Google's OAuth server.
|
|
30
|
+
info["token_uri"] = "https://oauth2.googleapis.com/token"
|
|
31
|
+
credentials = service_account.Credentials.from_service_account_info(
|
|
32
|
+
info,
|
|
33
|
+
scopes=[
|
|
34
|
+
"https://www.googleapis.com/auth/generative-language.retriever",
|
|
35
|
+
],
|
|
36
|
+
)
|
|
37
|
+
credentials.refresh(Request())
|
|
38
|
+
project = (
|
|
39
|
+
os.environ.get("GOOGLE_CLOUD_PROJECT")
|
|
40
|
+
or os.environ.get("GCLOUD_PROJECT_ID")
|
|
41
|
+
or info.get("project_id")
|
|
42
|
+
)
|
|
43
|
+
return {"token": credentials.token, "project": project}
|
|
44
|
+
except Exception:
|
|
45
|
+
# Library exceptions can contain credential data. Never forward them.
|
|
46
|
+
return {"error": "Gemini service-account authentication failed; check credentials, permissions, and network access"}
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
if __name__ == "__main__":
|
|
50
|
+
result = authenticate()
|
|
51
|
+
print(json.dumps(result))
|
|
52
|
+
sys.exit(1 if "error" in result else 0)
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
import { spawnSync } from "node:child_process";
|
|
2
|
+
import { fileURLToPath } from "node:url";
|
|
3
|
+
import { pythonInvocation } from "./python.mjs";
|
|
4
|
+
import { test } from "node:test";
|
|
5
|
+
import assert from "node:assert/strict";
|
|
6
|
+
import { geminiAuth, geminiConfigured } from "./gemini-auth.mjs";
|
|
7
|
+
import { pickProvider } from "./tts.mjs";
|
|
8
|
+
|
|
9
|
+
function env(t) {
|
|
10
|
+
const saved = { ...process.env };
|
|
11
|
+
for (const key of [
|
|
12
|
+
"GEMINI_API_KEY",
|
|
13
|
+
"GOOGLE_API_KEY",
|
|
14
|
+
"GOOGLE_APPLICATION_CREDENTIALS",
|
|
15
|
+
"GCS_CREDS",
|
|
16
|
+
])
|
|
17
|
+
delete process.env[key];
|
|
18
|
+
t.after(() => {
|
|
19
|
+
process.env = saved;
|
|
20
|
+
});
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
test("API keys take precedence over service accounts without invoking Python", (t) => {
|
|
24
|
+
env(t);
|
|
25
|
+
process.env.GEMINI_API_KEY = "gemini-key";
|
|
26
|
+
process.env.GOOGLE_API_KEY = "google-key";
|
|
27
|
+
process.env.GCS_CREDS = "private-json";
|
|
28
|
+
const run = () => assert.fail("must not spawn Python");
|
|
29
|
+
assert.deepEqual(geminiAuth({ run }).headers, { "x-goog-api-key": "gemini-key" });
|
|
30
|
+
delete process.env.GEMINI_API_KEY;
|
|
31
|
+
assert.deepEqual(geminiAuth({ run }).headers, { "x-goog-api-key": "google-key" });
|
|
32
|
+
});
|
|
33
|
+
|
|
34
|
+
test("service-account configuration selects Gemini and returns scoped helper headers", (t) => {
|
|
35
|
+
env(t);
|
|
36
|
+
assert.equal(geminiConfigured(), false);
|
|
37
|
+
assert.throws(() => geminiAuth(), /needs GEMINI_API_KEY/);
|
|
38
|
+
for (const key of ["GOOGLE_APPLICATION_CREDENTIALS", "GCS_CREDS"]) {
|
|
39
|
+
process.env[key] = "credential-source";
|
|
40
|
+
assert.equal(pickProvider("gemini"), "gemini");
|
|
41
|
+
const auth = geminiAuth({
|
|
42
|
+
run: (_, args, options) => {
|
|
43
|
+
assert.ok(args.at(-1).endsWith("gemini-auth.py"));
|
|
44
|
+
assert.ok(!args.includes("credential-source"));
|
|
45
|
+
assert.equal(options.timeout, 60000);
|
|
46
|
+
return {
|
|
47
|
+
status: 0,
|
|
48
|
+
stdout: JSON.stringify({ token: "access-token", project: "quota-project" }),
|
|
49
|
+
};
|
|
50
|
+
},
|
|
51
|
+
});
|
|
52
|
+
assert.deepEqual(auth.headers, {
|
|
53
|
+
Authorization: "Bearer access-token",
|
|
54
|
+
"x-goog-user-project": "quota-project",
|
|
55
|
+
});
|
|
56
|
+
delete process.env[key];
|
|
57
|
+
}
|
|
58
|
+
});
|
|
59
|
+
|
|
60
|
+
test("failed, malformed, or missing Python output cannot leak secrets or fall back", (t) => {
|
|
61
|
+
env(t);
|
|
62
|
+
process.env.GCS_CREDS = "private-json";
|
|
63
|
+
for (const result of [
|
|
64
|
+
{ status: 1, stdout: '{"error":"private-json"}', stderr: "private-json" },
|
|
65
|
+
{ status: null, stdout: "private-json", error: new Error("private-json") },
|
|
66
|
+
{ status: 0, stdout: "{}" },
|
|
67
|
+
]) {
|
|
68
|
+
assert.throws(
|
|
69
|
+
() => geminiAuth({ run: () => result }),
|
|
70
|
+
(error) => {
|
|
71
|
+
assert.match(error.message, /authentication failed/);
|
|
72
|
+
assert.ok(!error.message.includes("private-json"));
|
|
73
|
+
return true;
|
|
74
|
+
},
|
|
75
|
+
);
|
|
76
|
+
}
|
|
77
|
+
});
|
|
78
|
+
|
|
79
|
+
// Keep the Python credential-boundary suite reachable from the normal Node CI runner.
|
|
80
|
+
test("Python service-account boundary validates credentials, scope, transport and errors", () => {
|
|
81
|
+
const { cmd, args } = pythonInvocation([
|
|
82
|
+
fileURLToPath(new URL("./gemini-auth_test.py", import.meta.url)),
|
|
83
|
+
]);
|
|
84
|
+
const result = spawnSync(cmd, args, { encoding: "utf8", timeout: 30_000 });
|
|
85
|
+
assert.equal(result.status, 0, result.stderr || result.error?.message);
|
|
86
|
+
});
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
"""Credential-boundary tests; stub Google's transport, never use real secrets."""
|
|
2
|
+
|
|
3
|
+
import importlib.util
|
|
4
|
+
import json
|
|
5
|
+
import os
|
|
6
|
+
import sys
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
import tempfile
|
|
9
|
+
import types
|
|
10
|
+
import unittest
|
|
11
|
+
from unittest.mock import Mock, patch
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
# Keep the credential subprocess protocol UTF-8 on every platform.
|
|
15
|
+
for _stream in (sys.stdout, sys.stderr):
|
|
16
|
+
if hasattr(_stream, "reconfigure"):
|
|
17
|
+
_stream.reconfigure(encoding="utf-8", errors=_stream.errors)
|
|
18
|
+
|
|
19
|
+
spec = importlib.util.spec_from_file_location("gemini_auth", Path(__file__).with_name("gemini-auth.py"))
|
|
20
|
+
auth = importlib.util.module_from_spec(spec)
|
|
21
|
+
spec.loader.exec_module(auth)
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
class AuthenticationTests(unittest.TestCase):
|
|
25
|
+
def setUp(self):
|
|
26
|
+
self.info = {"type": "service_account", "project_id": "account-project", "token_uri": "https://untrusted.invalid/token"}
|
|
27
|
+
self.credentials = Mock(token="fixture-token")
|
|
28
|
+
self.factory = Mock(return_value=self.credentials)
|
|
29
|
+
self.request = Mock(return_value=object())
|
|
30
|
+
transport = types.ModuleType("google.auth.transport.requests")
|
|
31
|
+
transport.Request = self.request
|
|
32
|
+
oauth = types.ModuleType("google.oauth2")
|
|
33
|
+
oauth.service_account = types.SimpleNamespace(Credentials=types.SimpleNamespace(from_service_account_info=self.factory))
|
|
34
|
+
modules = {name: types.ModuleType(name) for name in ("google", "google.auth", "google.auth.transport")}
|
|
35
|
+
modules.update({"google.auth.transport.requests": transport, "google.oauth2": oauth})
|
|
36
|
+
self.addCleanup(patch.stopall)
|
|
37
|
+
patch.dict("sys.modules", modules).start()
|
|
38
|
+
patch.dict(os.environ, {"GCS_CREDS": json.dumps(self.info)}, clear=True).start()
|
|
39
|
+
|
|
40
|
+
def test_scope_endpoint_and_refresh(self):
|
|
41
|
+
self.assertEqual(auth.authenticate(), {"token": "fixture-token", "project": "account-project"})
|
|
42
|
+
info = self.factory.call_args.args[0]
|
|
43
|
+
self.assertEqual(info["token_uri"], "https://oauth2.googleapis.com/token")
|
|
44
|
+
self.assertEqual(self.factory.call_args.kwargs["scopes"], ["https://www.googleapis.com/auth/generative-language.retriever"])
|
|
45
|
+
self.credentials.refresh.assert_called_once_with(self.request.return_value)
|
|
46
|
+
|
|
47
|
+
def test_file_takes_precedence_over_json_environment(self):
|
|
48
|
+
with tempfile.TemporaryDirectory() as directory:
|
|
49
|
+
path = Path(directory) / "service.json"
|
|
50
|
+
path.write_text(json.dumps({**self.info, "project_id": "file-project"}), encoding="utf-8")
|
|
51
|
+
os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = str(path)
|
|
52
|
+
os.environ["GCS_CREDS"] = "invalid-json"
|
|
53
|
+
self.assertEqual(auth.authenticate()["project"], "file-project")
|
|
54
|
+
|
|
55
|
+
def test_quota_project_precedence(self):
|
|
56
|
+
os.environ["GCLOUD_PROJECT_ID"] = "alias-project"
|
|
57
|
+
self.assertEqual(auth.authenticate()["project"], "alias-project")
|
|
58
|
+
os.environ["GOOGLE_CLOUD_PROJECT"] = "explicit-project"
|
|
59
|
+
self.assertEqual(auth.authenticate()["project"], "explicit-project")
|
|
60
|
+
|
|
61
|
+
def test_user_adc_is_rejected_before_exchange(self):
|
|
62
|
+
os.environ["GCS_CREDS"] = json.dumps({"type": "authorized_user"})
|
|
63
|
+
self.assertIn("service-account JSON", auth.authenticate()["error"])
|
|
64
|
+
self.factory.assert_not_called()
|
|
65
|
+
|
|
66
|
+
def test_invalid_json_or_missing_file_does_not_fall_back(self):
|
|
67
|
+
os.environ["GCS_CREDS"] = "private-invalid-json"
|
|
68
|
+
self.assertIn("error", auth.authenticate())
|
|
69
|
+
os.environ["GCS_CREDS"] = json.dumps(self.info)
|
|
70
|
+
os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = "/missing/service-account-fixture.json"
|
|
71
|
+
self.assertIn("error", auth.authenticate())
|
|
72
|
+
self.factory.assert_not_called()
|
|
73
|
+
|
|
74
|
+
def test_library_errors_are_redacted(self):
|
|
75
|
+
for boundary in (self.factory, self.credentials.refresh):
|
|
76
|
+
with self.subTest(boundary=boundary):
|
|
77
|
+
boundary.side_effect = RuntimeError("private-key-and-token")
|
|
78
|
+
result = auth.authenticate()
|
|
79
|
+
self.assertIn("error", result)
|
|
80
|
+
self.assertNotIn("private-key-and-token", json.dumps(result))
|
|
81
|
+
boundary.side_effect = None
|
|
82
|
+
|
|
83
|
+
def test_missing_dependency_has_actionable_error(self):
|
|
84
|
+
with patch.dict("sys.modules", {"google.auth.transport.requests": None}):
|
|
85
|
+
self.assertIn("pip install google-auth requests", auth.authenticate()["error"])
|
|
86
|
+
self.factory.assert_not_called()
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
if __name__ == "__main__":
|
|
90
|
+
unittest.main()
|