egaki 0.5.0 → 0.7.0

package/dist/vite/wave-gradient-shader.js

@@ -0,0 +1,207 @@
+'use client';
+import { defineShader } from "./shader-renderer.js";
+// ---------------------------------------------------------------------------
+// Fragment shader body
+// ---------------------------------------------------------------------------
+/**
+ * Raw GLSL fragment body from GradientWave.js.
+ * The defineShader engine prepends #version, precision, v_uv, fragColor,
+ * uniform declarations, and built-in uniforms automatically.
+ *
+ * Algorithm overview:
+ *   1. Noise rotation: seed-derived hash noise determines a rotation angle
+ *      per pixel, creating an organic swirling base pattern
+ *   2. Wave warp: UV coordinates are distorted by a seed-modulated wave
+ *      function with configurable frequency, angle, amplitude, and softness
+ *   3. Layer blend: the rotated UVs and warped UVs are mixed via blendAmount,
+ *      then two color layers (each a smoothstep gradient between 2 colors)
+ *      are combined with a vertical smoothstep for the final output
+ *   4. Tone mapping: a subtle S-curve (col*col + 0.5*sqrt(col)) is mixed in
+ *      at 30% to add richness
+ */
+const FRAGMENT_SOURCE = `
+#define S(a,b,t) smoothstep(a,b,t)
+
+mat2 Rot(float a) {
+    float s = sin(a), c = cos(a);
+    return mat2(c, -s, s, c);
+}
+
+vec2 hash(vec2 p) {
+    float s = u_seed;
+    vec2 k1 = vec2(2127.1 + s * 13.37, 81.17 + s * 7.31);
+    vec2 k2 = vec2(1269.5 + s * 11.13, 283.37 + s * 5.79);
+    p = vec2(dot(p, k1), dot(p, k2));
+    return fract(sin(p) * (43758.5453 + s * 1.618));
+}
+
+float noise(in vec2 p) {
+    vec2 i = floor(p);
+    vec2 f = fract(p);
+    vec2 u = f * f * (3.0 - 2.0 * f);
+    float n = mix(
+        mix(dot(-1.0 + 2.0 * hash(i), f),
+            dot(-1.0 + 2.0 * hash(i + vec2(1.0, 0.0)), f - vec2(1.0, 0.0)), u.x),
+        mix(dot(-1.0 + 2.0 * hash(i + vec2(0.0, 1.0)), f - vec2(0.0, 1.0)),
+            dot(-1.0 + 2.0 * hash(i + vec2(1.0, 1.0)), f - vec2(1.0, 1.0)), u.x),
+        u.y
+    );
+    return 0.5 + 0.5 * n;
+}
+
+vec3 getColor(int idx) {
+    if (u_colors_length < 1) return vec3(0.0);
+    int safeIdx = clamp(idx, 0, u_colors_length - 1);
+    return u_colors[safeIdx].rgb;
+}
+
+float seedF(float base) {
+    return base * (1.0 + 0.5 * sin(u_seed * 3.17 + base));
+}
+
+vec2 warpUV(vec2 uv) {
+    float t = u_time * u_waveSpeed;
+
+    float angleOffset = sin(u_seed * 2.73) * 30.0;
+    mat2 dirRot = Rot(radians(u_waveAngle + angleOffset));
+    vec2 ruv = dirRot * uv;
+
+    float fxMod = seedF(u_waveFreqX);
+    float fyMod = seedF(u_waveFreqY);
+
+    float phaseX = fract(sin(u_seed * 7.19) * 437.58) * 6.2832;
+    float phaseY = fract(cos(u_seed * 3.41) * 291.37) * 6.2832;
+
+    // Core wave with seed-dependent harmonics
+    float harmonic = sin(u_seed * 1.23) * 0.5;
+    float a = fyMod * ruv.y - sin(ruv.x * fxMod + ruv.y - t + phaseX);
+    a += harmonic * sin(ruv.x * fxMod * 2.0 + ruv.y * 0.5 + t * 0.7 + phaseY);
+
+    // Smoothstep mask
+    a = smoothstep(
+        cos(a) * u_maskSoftness,
+        sin(a) * u_maskSoftness + 3.,
+        cos(a - fyMod * ruv.y) - sin(a - fxMod * ruv.x)
+    );
+
+    a *= u_waveAmplitude;
+
+    uv = cos(a) * uv + sin(a) * vec2(-uv.y, uv.x);
+    return uv;
+}
+
+void main() {
+    vec2 fragCoord = v_uv * u_resolution;
+    vec2 uv = fragCoord / u_resolution.xy;
+    float ratio = u_resolution.x / u_resolution.y;
+    float t = u_time * u_waveSpeed;
+
+    vec2 tuv = uv - 0.5;
+
+    vec2 seedShift = vec2(sin(u_seed * 4.37), cos(u_seed * 5.91)) * 100.0;
+    float degree = noise(vec2(t * 0.1, tuv.x * tuv.y) + seedShift);
+    tuv.y *= 1.0 / ratio;
+    tuv *= Rot(radians((degree - 0.5) * 720.0 + 180.0));
+    tuv.y *= ratio;
+
+    // Seed-rotate uv2 before warping
+    vec2 uv2 = (fragCoord * 2.0 - u_resolution.xy) / (u_resolution.x + u_resolution.y) * 2.0;
+    float preRotAngle = fract(sin(u_seed * 5.63) * 173.29) * 6.2832;
+    uv2 *= Rot(preRotAngle);
+    vec2 warped = warpUV(uv2) * 0.5 + 0.5;
+
+    vec2 blendUV = mix(tuv, warped - 0.5, u_blendAmount);
+
+    float layerRot1 = -5.0 + sin(u_seed * 1.83) * 20.0;
+    float layerRot2 = 10.0 + cos(u_seed * 2.47) * 20.0;
+
+    vec3 c0 = getColor(0);
+    vec3 c1 = getColor(1);
+    vec3 c2 = getColor(2);
+    vec3 c3 = getColor(3);
+
+    vec3 layer1 = mix(c0, c2, S(-0.3, 0.3, (blendUV * Rot(radians(layerRot1))).x));
+    vec3 layer2 = mix(c3, c1, S(-0.3, 0.3, (blendUV * Rot(radians(layerRot2))).x));
+    vec3 col = mix(layer1, layer2, S(0.3, -0.3, blendUV.y));
+
+    col = mix(col, col * col + 0.5 * sqrt(col), 0.3);
+
+    fragColor = vec4(col, 1.0);
+}
+`;
+// ---------------------------------------------------------------------------
+// Component
+// ---------------------------------------------------------------------------
+/**
+ * Property controls matching the original Framer GradientWave module.
+ * Ranges, defaults, and step sizes are preserved from the original.
+ */
+export const WaveGradientShader = defineShader({
+    title: 'WaveGradientShader',
+    fragment: FRAGMENT_SOURCE,
+    propertyControls: {
+        colors: {
+            type: 'array',
+            control: { type: 'color' },
+            maxCount: 4,
+            maxVisible: 4,
+            defaultValue: ['#FF3624', '#9EABFF', '#FFAE00', '#E29EFF'],
+        },
+        seed: {
+            type: 'number',
+            defaultValue: 32,
+            min: 0,
+            max: 100,
+            step: 1,
+        },
+        waveSpeed: {
+            type: 'number',
+            defaultValue: 1.5,
+            min: 0,
+            max: 3,
+            step: 0.01,
+        },
+        waveFreqX: {
+            type: 'number',
+            defaultValue: 0.9,
+            min: 0.1,
+            max: 6,
+            step: 0.1,
+        },
+        waveFreqY: {
+            type: 'number',
+            defaultValue: 6,
+            min: 0.1,
+            max: 6,
+            step: 0.1,
+        },
+        waveAngle: {
+            type: 'number',
+            defaultValue: 105,
+            min: -180,
+            max: 180,
+            step: 1,
+        },
+        waveAmplitude: {
+            type: 'number',
+            defaultValue: 2.1,
+            min: 0.5,
+            max: 3,
+            step: 0.01,
+        },
+        maskSoftness: {
+            type: 'number',
+            defaultValue: 0.74,
+            min: 0.01,
+            max: 2,
+            step: 0.01,
+        },
+        blendAmount: {
+            type: 'number',
+            defaultValue: 0.54,
+            min: 0,
+            max: 1,
+            step: 0.01,
+        },
+    },
+});

package/package.json

@@ -1,30 +1,58 @@
 {
   "name": "egaki",
-  "version": "0.5.0",
+  "version": "0.7.0",
   "type": "module",
-  "main": "./dist/cli.js",
-  "types": "./dist/cli.d.ts",
+  "description": "AI image, video, and speech generation CLI with MDX-to-video framework",
+  "main": "./dist/cli/main.js",
+  "types": "./dist/cli/cli.d.ts",
   "exports": {
     "./package.json": "./package.json",
     ".": {
-      "types": "./dist/cli.d.ts",
-      "default": "./dist/cli.js"
+      "types": "./dist/cli/cli.d.ts",
+      "default": "./dist/cli/cli.js"
     },
-    "./src": {
-      "types": "./src/cli.ts",
-      "default": "./src/cli.ts"
+    "./generate": {
+      "types": "./dist/cli/generate.d.ts",
+      "default": "./dist/cli/generate.js"
     },
-    "./src/*": {
-      "types": "./src/*.ts",
-      "default": "./src/*.ts"
+    "./video": {
+      "types": "./src/vite/mdx-video.tsx",
+      "default": "./src/vite/mdx-video.tsx"
+    },
+    "./text-to-speech": {
+      "types": "./src/vite/server-components.tsx",
+      "default": "./src/vite/server-components.tsx"
+    },
+    "./generate-media": {
+      "types": "./src/vite/server-components.tsx",
+      "default": "./src/vite/server-components.tsx"
+    },
+    "./cached-generate": {
+      "types": "./src/cli/cached-generate.ts",
+      "default": "./src/cli/cached-generate.ts"
+    },
+    "./vite": {
+      "types": "./src/vite/vite-plugin.ts",
+      "default": "./src/vite/vite-plugin.ts"
+    },
+    "./mdx-components": {
+      "types": "./src/vite/mdx-provided-components.ts",
+      "default": "./src/vite/mdx-provided-components.ts"
+    },
+    "./src/cli/*": {
+      "types": "./src/cli/*.ts",
+      "default": "./src/cli/*.ts"
+    },
+    "./src/vite/*": {
+      "types": "./src/vite/*.tsx",
+      "default": "./src/vite/*.tsx"
     }
   },
-  "description": "AI image and video generation CLI",
-  "bin": "dist/cli.js",
+  "bin": "dist/cli/main.js",
   "repository": {
     "type": "git",
     "url": "https://github.com/remorses/egaki.git",
-    "directory": "."
+    "directory": "cli"
   },
   "homepage": "https://github.com/remorses/egaki#readme",
   "bugs": {
@@ -34,41 +62,91 @@
     "src",
     "dist",
     "skills",
-    "README.md",
     "CHANGELOG.md"
   ],
   "keywords": [
     "ai",
     "image",
     "video",
+    "speech",
+    "tts",
     "generation",
     "cli",
-    "gemini"
+    "gemini",
+    "remotion",
+    "mdx",
+    "vite"
   ],
   "license": "MIT",
   "dependencies": {
-    "@ai-sdk/fal": "2.0.21",
+    "@ai-sdk/bytedance": "^1.0.14",
+    "@ai-sdk/fal": "2.0.34",
     "@ai-sdk/google": "3.0.33",
-    "@ai-sdk/google-vertex": "^4.0.95",
+    "@ai-sdk/google-vertex": "^4.0.137",
     "@ai-sdk/openai": "3.0.36",
     "@ai-sdk/replicate": "2.0.20",
+    "@ai-sdk/xai": "^3.0.92",
     "@clack/prompts": "^1.0.1",
+    "@fal-ai/client": "^1.10.1",
+    "@remotion/media": "4.0.475",
+    "@remotion/player": "4.0.475",
+    "@remotion/web-renderer": "4.0.475",
+    "@tailwindcss/vite": "^4.2.2",
+    "@tweakpane/core": "^2.0.5",
+    "@tweakpane/plugin-essentials": "^0.2.1",
     "ai": "6.0.103",
     "eventsource-parser": "^3.0.6",
-    "goke": "^6.6.1",
+    "goke": "^6.12.1",
     "js-yaml": "^4.1.1",
+    "mediabunny": "^1.46.0",
+    "moveable": "^0.53.0",
     "picocolors": "^1.1.1",
+    "react": "19.2.7",
+    "react-dom": "19.2.7",
+    "remotion": "4.0.475",
+    "safe-mdx": "^1.11.3",
+    "shiki": "^4.2.0",
     "string-dedent": "^3.0.2",
-    "zod": "^4.3.6"
+    "tailwindcss": "^4.3.0",
+    "tweakpane": "^4.0.5",
+    "yaml": "^2.9.0",
+    "zod": "^4.3.6",
+    "zustand": "^5.0.14"
+  },
+  "peerDependencies": {
+    "@vitejs/plugin-react": ">=6",
+    "motion": ">=12",
+    "motion-dom": ">=12",
+    "motion-utils": ">=12",
+    "spiceflow": ">=1.26.0-rsc.5"
+  },
+  "peerDependenciesMeta": {
+    "@vitejs/plugin-react": {
+      "optional": true
+    },
+    "motion": {
+      "optional": true
+    },
+    "motion-dom": {
+      "optional": true
+    },
+    "motion-utils": {
+      "optional": true
+    },
+    "spiceflow": {
+      "optional": true
+    }
   },
   "devDependencies": {
     "@types/js-yaml": "^4.0.9",
-    "@types/node": "^25.3.0",
-    "tsx": "^4.21.0",
-    "typescript": "^5.9.3"
+    "@types/react": "^19.2.17",
+    "@vitejs/plugin-react": "^6.0.2",
+    "vite": "^8.0.10",
+    "vitest": "^4.1.8"
   },
   "scripts": {
-    "dev": "tsx src/cli.ts",
-    "build": "tsc && chmod +x dist/cli.js"
+    "dev": "tsx src/cli/cli.ts",
+    "build": "tsc && chmod +x dist/cli/main.js",
+    "test": "vitest run"
   }
 }

package/skills/egaki/SKILL.md

@@ -10,7 +10,7 @@ description: >
 
 Generate AI images and videos from the terminal.
 Use this for text-to-image, image editing, mask-based edits, text-to-video,
-image-to-video, and model discovery.
+image-to-video, audio stem separation, and model discovery.
 
 ## Install
 
@@ -54,6 +54,23 @@ The `--model` / `-m` flag is **optional** on both `egaki image` and `egaki video
 
 Agents should always pass `-m` explicitly to avoid the interactive picker.
 
+### Preferred image models
+
+When the user does not specify a model, prefer these in order:
+
+1. **`gpt-image-1`** (or `gpt-image-2` / `chatgpt-image-latest` if available)
+2. **`grok-imagine-image`** (xAI Grok)
+3. **`nano-banana-pro-preview`**
+
+Always try the first available model. Fall back to the next if auth is missing
+for the preferred one.
+
+### Preferred video models
+
+When the user does not specify a video model, prefer:
+
+1. **`grok-imagine-video-1.5`** (xAI Grok)
+
 ## Example commands
 
 ```bash
@@ -102,6 +119,150 @@ egaki models --type video
 egaki models --type image
 ```
 
+## Audio stem separation (demucs)
+
+Separate audio into individual stems using Meta's **Demucs** model via fal.ai.
+Requires a fal.ai API key (`egaki login --provider fal`).
+
+The default model `htdemucs_6s` splits into 6 stems: **vocals, drums, bass,
+guitar, piano, other**. By default only vocals and other (background) are
+extracted.
+
+```bash
+# Extract vocals and background (default)
+egaki demucs song.mp3
+
+# Extract only vocals
+egaki demucs song.mp3 --stems vocals
+
+# All 6 stems as WAV
+egaki demucs song.mp3 --stems vocals,drums,bass,other,guitar,piano --output-format wav
+
+# Save to a directory
+egaki demucs song.mp3 -o stems/
+
+# Higher quality (slower, more random shifts)
+egaki demucs song.mp3 --shifts 5
+
+# Read from stdin (e.g. ffmpeg pipe)
+ffmpeg -i video.mp4 -f mp3 - | egaki demucs --stdin -o stems/
+```
+
+Output files are named `{input}-{stem}.{format}` (e.g. `song-vocals.mp3`,
+`song-other.mp3`).
+
+**Model variants:** `htdemucs_6s` (default, 6 stems), `htdemucs` (4 stems),
+`htdemucs_ft` (fine-tuned 4 stems), `hdemucs_mmi` (4 stems), `mdx` (4 stems,
+MDX architecture), `mdx_extra` (4 stems, MDX), `mdx_q` (4 stems, quantized),
+`mdx_extra_q` (4 stems, quantized). Guitar and piano stems are only available
+with `htdemucs_6s`.
+
+## Voice cloning
+
+Clone a voice from any audio. Returns a voice ID for use with `egaki speech --voice <id>`.
+
+Two providers:
+- **Cartesia** (default): fast, free, best with short clean clips (5-10s)
+- **ElevenLabs**: accepts longer audio (1-3 min), has `--remove-background-noise`
+
+### Preparing the audio
+
+Raw recordings, YouTube clips, podcasts, and interviews almost always have
+background music, room noise, or multiple speakers. Feeding dirty audio to the
+clone API produces a muddy, inaccurate voice. Always clean it up first.
+
+**Step 1: Isolate vocals with demucs.**
+Strip everything except the target speaker's voice. This removes background
+music, ambient noise, sound effects, and other speakers.
+
+```bash
+egaki demucs recording.mp3 --stems vocals
+# outputs: recording-vocals.mp3
+```
+
+For higher quality separation (slower), add `--shifts 5`.
+
+**Step 2: Transcribe to find the best segment.**
+Run transcription to get word-level timestamps. Look for a segment where the
+speaker says a **complete phrase or sentence** clearly, with no hesitations,
+filler words ("um", "uh"), crosstalk, or long pauses.
+
+```bash
+egaki transcribe recording-vocals.mp3
+```
+
+Review the `wordTimestamps` array in the JSON output. Pick a window that is:
+- **Cartesia**: 5-10 seconds (max 10s). One clean sentence is ideal.
+- **ElevenLabs**: 30s-3 min. Multiple sentences give better results.
+
+The segment should start and end mid-speech, not on silence. The clone mimics
+the energy and pacing of the source, so pick a segment that matches the tone
+you want (calm narration, energetic presentation, conversational, etc.).
+
+**Step 3: Trim with ffmpeg.**
+Use the timestamps from step 2 to cut the exact segment. The `-c copy` flag
+avoids re-encoding so there's no quality loss.
+
+```bash
+# Cartesia: trim a 10s clip (timestamps from transcription)
+ffmpeg -i recording-vocals.mp3 -ss 12.5 -to 22.0 -c copy clip.mp3
+
+# ElevenLabs: trim a longer segment
+ffmpeg -i recording-vocals.mp3 -ss 5.0 -to 65.0 -c copy clip-long.mp3
+```
+
+### Cloning
+
+```bash
+# Cartesia (default, fast, free)
+egaki voice clone clip.mp3 --name "Speaker Name"
+
+# ElevenLabs (longer clips, optional noise removal)
+egaki voice clone clip-long.mp3 --name "Speaker Name" --provider elevenlabs
+
+# ElevenLabs with AI noise removal (if source still has some noise)
+egaki voice clone clip.mp3 --name "Speaker Name" --provider elevenlabs --remove-background-noise
+
+# Specify language for non-English Cartesia clones (ignored by ElevenLabs)
+egaki voice clone clip.mp3 --name "Narrador" --language es
+```
+
+The command prints the voice ID on success. Save it.
+
+### Using the cloned voice
+
+```bash
+egaki speech "Any text you want spoken in the cloned voice." --voice <voice-id> -m sonic-3.5
+```
+
+The voice ID works with any Cartesia TTS model (`sonic-3`, `sonic-3.5`) or
+ElevenLabs model (`eleven_v3`, `eleven_flash_v2_5`) depending on which
+provider you cloned with. Cartesia voice IDs only work with Cartesia models
+and vice versa.
+
+### Full pipeline example
+
+```bash
+# 1. Download audio (e.g. from YouTube)
+yt-dlp -x --audio-format mp3 -o source.mp3 'https://youtube.com/watch?v=...'
+
+# 2. Isolate vocals
+egaki demucs source.mp3 --stems vocals
+
+# 3. Transcribe to find timestamps
+egaki transcribe source-vocals.mp3
+
+# 4. Trim to a clean 10s segment (adjust -ss and -to from timestamps)
+ffmpeg -i source-vocals.mp3 -ss 0 -to 10 -c copy clip.mp3
+
+# 5. Clone
+egaki voice clone clip.mp3 --name "Narrator"
+# Voice ID: abc123-def456-...
+
+# 6. Generate speech
+egaki speech "Your text here." --voice abc123-def456-... -m sonic-3.5 -o output.wav
+```
+
 ## Video generation note for agents
 
 Video generation can be very slow — some models take 1–3 minutes per request.