openclacky 1.2.17 → 1.3.0

data/lib/clacky/cli.rb

@@ -290,6 +290,54 @@ module Clacky
         ui_controller.append_output("")
       end
 
+      # Handle the `/model` slash command — a quick model-card switcher.
+      #
+      # This is the lightweight counterpart to /config: it only lets the user
+      # pick an already-configured model and switches to it (no add/edit/delete).
+      # Switching goes through the unified Agent#switch_model_by_id path and
+      # also updates the global default so the choice sticks across launches,
+      # matching /config's :switch behavior.
+      private def handle_model_command(ui_controller, agent_config, agent, session_manager = nil)
+        config = agent_config
+
+        if config.models.empty?
+          ui_controller.show_error("No models configured. Run /config to add one.")
+          return
+        end
+
+        # Resolve a card's provider sub-models so the picker can offer them in
+        # the card's sub-model drawer.
+        submodels_for = lambda do |model|
+          base_url = model["base_url"]
+          provider_id = base_url && Clacky::Providers.find_by_base_url(base_url)
+          provider_id ? Clacky::Providers.models(provider_id) : []
+        end
+
+        result = ui_controller.show_model_switch_modal(config, submodels_for)
+        return if result.nil?
+
+        target_id = result[:model_id]
+        sub_model = result[:model_name]
+
+        agent.switch_model_by_id(target_id)
+        config.set_default_model_by_id(target_id)
+        config.save
+
+        # Pin (or clear) the per-session sub-model overlay for the chosen card.
+        agent.set_session_sub_model(sub_model)
+
+        # The overlay lives in the session file (not config.yml), so persist it
+        # now — otherwise it would be lost if the user quits before the next task.
+        session_manager&.save(agent.to_session_data)
+
+        ui_controller.config[:model] = config.model_name
+        ui_controller.update_sessionbar(
+          tasks: agent.total_tasks,
+          cost: agent.total_cost
+        )
+        ui_controller.show_success("Switched to model: #{config.model_name}")
+      end
+
       private def handle_time_machine_command(ui_controller, agent, session_manager)
         # Get task history from agent
         history = agent.get_task_history(limit: 10)
@@ -892,10 +940,11 @@ module Clacky
               ui_controller.append_output("")
             end
 
-            # Stop UI and exit
+            # Stop UI and exit. Each UI decides whether to clear the screen on
+            # exit (UI2 keeps it so the resume hint survives; Rich clears).
             shutting_down = true
             idle_timer.shutdown
-            ui_controller.stop(clear_screen: true)
+            ui_controller.stop
             exit(0)
           end
 
@@ -913,6 +962,9 @@ module Clacky
           when "/config"
             handle_config_command(ui_controller, agent_config, agent)
             next
+          when "/model"
+            handle_model_command(ui_controller, agent_config, agent, session_manager)
+            next
           when "/undo"
             handle_time_machine_command(ui_controller, agent, session_manager)
             next
@@ -948,7 +1000,7 @@ module Clacky
           when "/exit", "/quit"
             shutting_down = true
             idle_timer.shutdown
-            ui_controller.stop(clear_screen: true)
+            ui_controller.stop
             exit(0)
           when "/help"
             sleep 0.1

data/lib/clacky/default_skills/media-gen/SKILL.md

@@ -1,8 +1,9 @@
 ---
 name: media-gen
-description: 'Generate images (and later videos / audio) inside the current task. Use this skill whenever the user asks to create, generate, or produce a picture / image / illustration / cover / poster / icon / artwork — including phrases like 生成图片, 画一张, 做封面, 来张配图, generate image, make a picture, draw, create artwork, design a cover. Also use when building documents (slides, PPT, posters, marketing pages, README hero shots) where an image is needed inline. Routes calls through the local Clacky HTTP server, which uses the user-configured `type=image` model — you do NOT need to know which provider; the server handles it.'
+description: 'Generate images, videos, or audio (text-to-speech) in the current task. Use whenever the user asks to create/generate/produce a picture / image / illustration / cover / poster / icon / artwork, a video / clip / animation, or speech / voiceover / narration / TTS — e.g. 生成图片, 画一张, 做封面, 配图, generate image, make a picture, draw, design a cover, 生成视频, 做个视频, text-to-video, 朗读, 配音, 旁白, 文字转语音, generate speech, voiceover. Also use when a document (slides, poster, README hero) needs an inline image.'
 disable-model-invocation: false
 user-invocable: true
+always-show: true
 ---
 
 # media-gen
@@ -26,13 +27,29 @@ curl -s http://${CLACKY_SERVER_HOST}:${CLACKY_SERVER_PORT}/api/media/types
 
 If the response shows `image.configured = false`, stop and tell the user:
 
-> 还没有配置生图模型。请打开 Clacky 设置页 → 添加模型 → 类型选 `image`（推荐 `or-gemini-3-pro-image` 或 `or-gpt-image-1`）。配好后再让我生图。
+> 还没有配置生图模型。请打开设置页 → 添加模型 → 类型选 `image`（走 openclacky 官方网关时推荐 `or-gemini-3-pro-image` 或 `or-gpt-image-2`）。配好后再让我生图。
 
 Do NOT try to fall back to `terminal` + a hand-written `curl https://api.openai.com/...` — that bypasses the user's configured backend and won't be billed correctly.
 
+**You do NOT configure models — the user does, in the settings page.** Never
+edit the user's `config.yml` to add or change a model, and never invent a model
+name from memory (e.g. `or-gpt-5.4-image-2` does not exist). The real, current
+model is whatever `/api/media/types` reports under `image.model`. If you think a
+different model is needed, tell the user which one to set in the settings page —
+don't touch the config file yourself.
+
 ## Step 2 — Generate the image
 
-### ⚠️  Important: generation speed & concurrency
+### The model does NOT honor exact pixel sizes
+
+There is no `size` / `width` / `height` field — the only shape control is
+`aspect_ratio` (`landscape` / `square` / `portrait`), and even that is just a
+rough hint (ask for `576x96` and you may get `1408x768`). When the user needs an
+**exact pixel size, a grid, an icon at NxN, or a spritesheet**, generate first at
+whatever size the model gives, then resize / crop / tile to the exact pixels with
+ImageMagick (`magick`). Verify with `magick identify` before reporting done.
+
+### Important: generation speed & concurrency
 
 - **Image generation can be slow — up to 2 minutes per image depending on the model.** Before calling the API, warn the user that it may take a minute or two. The curl request blocks until the image is ready; do NOT run it in the background.
 - **One at a time only.** Never generate multiple images concurrently (e.g. by running several `curl` commands simultaneously or in a script loop). Each call consumes significant server-side resources, and parallel requests will almost certainly cause timeouts. If the user wants several images, generate them **sequentially**, one after another.
@@ -46,6 +63,10 @@ curl -s -X POST http://${CLACKY_SERVER_HOST}:${CLACKY_SERVER_PORT}/api/media/ima
   }'
 ```
 
+- The terminal blocks multi-line commands — write the request into a `.sh` file and run it, don't paste a multi-line `curl`.
+- If a call fails with `400 / INVALID_ARGUMENT`, drop the `aspect_ratio` field and retry once before reporting the error.
+- If a call fails with `unknown image model` (400), the configured model name isn't recognized by its backend — tell the user to fix the model name in the settings page; do NOT guess another name and retry.
+
 ### Request fields
 
 | Field          | Required | Values                              | Notes |
@@ -128,6 +149,153 @@ When the user gives a vague request like "给我配张图", ask one clarifying q
 - The user wants a **diagram / chart** with specific data — use a charting library (matplotlib, mermaid, etc.) instead; image gen is for illustrations, not data viz
 - The user asks for **screenshots** of real software — use the browser tool
 
-## Future modalities
+## Generating video (Veo)
+
+The same `/api/media/` namespace serves video generation. The user must
+configure a `type=video` model in settings (recommended: `or-veo-3-1`).
+
+### Endpoint
+
+```
+POST http://${CLACKY_SERVER_HOST}:${CLACKY_SERVER_PORT}/api/media/video
+```
+
+Check `GET /api/media/types` first — if `video.configured = false`, tell the
+user to add a `type=video` model in settings before generating.
+
+### Video is slow and expensive
+
+- **A single clip can take 1–3 minutes (sometimes longer).** Warn the user
+  before calling, and run the curl in the foreground — it blocks until the
+  MP4 is ready. Do NOT background it.
+- **One at a time.** Never run multiple video generations concurrently.
+- Each clip costs real money (billed per output-second). Confirm the prompt
+  with the user before generating.
+
+### Request
+
+```bash
+curl -s -X POST http://${CLACKY_SERVER_HOST}:${CLACKY_SERVER_PORT}/api/media/video \
+  -H "Content-Type: application/json" \
+  -d '{
+    "prompt": "A cinematic drone shot flying over a misty mountain range at sunrise, golden light, 4K.",
+    "aspect_ratio": "landscape",
+    "duration_seconds": 8
+  }'
+```
+
+| Field              | Required | Values                          | Notes |
+|--------------------|----------|---------------------------------|-------|
+| `prompt`           | yes      | string                          | Same prompt-craft tips as images apply. |
+| `aspect_ratio`     | no       | `landscape` / `portrait`        | Defaults to `landscape` (16:9). |
+| `duration_seconds` | no       | 4–8                             | Defaults to 8. |
+| `image`            | no       | `{ "b64_json": "...", "mime_type": "image/png" }` | Optional first frame for image-to-video. |
+| `output_dir`       | no       | absolute path                   | MP4 saved under `<output_dir>/assets/generated/`. |
+
+### Response (success)
+
+```json
+{
+  "success": true,
+  "video": "/abs/path/to/working_dir/assets/generated/vid_20260615_011820_a1b2c3d4.mp4",
+  "model": "or-veo-3-1",
+  "provider": "openclacky",
+  "prompt": "A cinematic drone shot ...",
+  "aspect_ratio": "landscape",
+  "duration_seconds": 8,
+  "cost_usd": 2.688
+}
+```
+
+The `video` field is an absolute path on disk. Show it to the user with a
+markdown link or an HTML5 `<video>` tag pointing at the `file://` path; embed
+it in documents with a relative path under `./assets/generated/`.
+
+### Response (failure)
+
+Same shape and `error_type` values as image generation, but with `"video": null`.
+`not_configured` means no `type=video` model is set up.
+
+## Generating speech (Gemini TTS)
+
+The same `/api/media/` namespace serves text-to-speech. The user must
+configure a `type=audio` model in settings (recommended:
+`or-tts-gemini-2-5-flash`, the cheap+fast default).
+
+### Endpoint
+
+```
+POST http://${CLACKY_SERVER_HOST}:${CLACKY_SERVER_PORT}/api/media/audio/speech
+```
+
+Check `GET /api/media/types` first — if `audio.configured = false`, tell the
+user to add a `type=audio` model in settings before generating.
+
+### Request
+
+```bash
+curl -s -X POST http://${CLACKY_SERVER_HOST}:${CLACKY_SERVER_PORT}/api/media/audio/speech \
+  -H "Content-Type: application/json" \
+  -d '{
+    "input": "Hello and welcome to openclacky. Today we will explore...",
+    "voice": "Kore"
+  }'
+```
+
+| Field        | Required | Values                          | Notes |
+|--------------|----------|---------------------------------|-------|
+| `input`      | yes      | string                          | The text to speak. Plain prose works best; you can prefix with style cues like "Say cheerfully:" or "In a calm tone:". |
+| `voice`      | no       | string voice name               | Defaults to `Kore`. Common Gemini voices: `Kore`, `Puck`, `Charon`, `Fenrir`, `Aoede`. |
+| `output_dir` | no       | absolute path                   | WAV saved under `<output_dir>/assets/generated/`. |
+
+Generation typically takes 2–10 seconds depending on length. The request
+blocks until the WAV is ready.
+
+### Response (success)
+
+```json
+{
+  "success": true,
+  "audio": "/abs/path/to/working_dir/assets/generated/tts_20260615_233522_4ff02705.wav",
+  "model": "or-tts-gemini-2-5-flash",
+  "provider": "openclacky",
+  "input": "Hello and welcome to openclacky...",
+  "voice": "Kore",
+  "mime_type": "audio/wav",
+  "usage": { "prompt_tokens": 13, "completion_tokens": 122, "total_tokens": 135 },
+  "cost_usd": 0.000259
+}
+```
+
+The `audio` field is an absolute path on disk. Output is mono 16-bit PCM at
+24 kHz wrapped in a standard WAV container — playable by any browser, OS
+player, or `<audio>` tag without conversion.
+
+To let the user hear it, write a markdown link in your reply:
+
+```markdown
+[🔊 听一下](file:///abs/path/from/response.wav)
+```
+
+For embedding in HTML documents, use:
+
+```html
+<audio controls src="./assets/generated/xxx.wav"></audio>
+```
+
+### Response (failure)
+
+Same shape and `error_type` values as image generation, but with `"audio": null`.
+`not_configured` means no `type=audio` model is set up.
+
+### Cost & length tips
+
+- Gemini TTS bills by tokens (input text + generated audio). A typical
+  one-paragraph narration costs well under $0.001.
+- For long-form audio (>1 minute), split the script into paragraphs and
+  generate each separately, then concatenate locally — avoids upstream
+  truncation and gives you finer control over pacing.
+- Voice consistency: Gemini TTS does not currently support voice cloning;
+  use the same `voice` name across calls in one project to keep the
+  narrator consistent.
 
-The same `/api/media/` namespace will gain `video` and `audio` endpoints. The pattern is identical: the user configures `type=video` / `type=audio` models in settings, this skill (or its successor) calls the matching endpoint.

data/lib/clacky/default_skills/skill-creator/SKILL.md

@@ -1,6 +1,7 @@
 ---
 name: skill-creator
 description: Create new skills, modify and improve existing skills, and measure skill performance. Use when users want to create a skill from scratch, edit, or optimize an existing skill, run evals to test a skill, benchmark skill performance with variance analysis, or optimize a skill's description for better triggering accuracy.
+always-show: true
 ---
 
 # Skill Creator

data/lib/clacky/media/base.rb

@@ -3,6 +3,7 @@
 require "fileutils"
 require "base64"
 require "securerandom"
+require "faraday"
 
 module Clacky
   module Media
@@ -28,6 +29,28 @@ module Clacky
         raise NotImplementedError, "#{self.class.name} must implement #generate_image"
       end
 
+      # @return [Hash] either video_success_response(...) or
+      #   video_error_response(...)
+      def generate_video(prompt:, aspect_ratio: "landscape", duration_seconds: nil, output_dir: nil, **_kwargs)
+        video_error_response(
+          error: "Video generation is not supported by #{self.class.name.split("::").last}. Use the openclacky gateway with a video model such as or-veo-3-1.",
+          error_type: "not_implemented",
+          provider: "",
+          prompt: prompt,
+          aspect_ratio: aspect_ratio
+        )
+      end
+
+      # @return [Hash] either audio_success_response(...) or audio_error_response(...)
+      def generate_speech(input:, voice: nil, output_dir: nil, **_kwargs)
+        audio_error_response(
+          error: "Speech synthesis is not supported by #{self.class.name.split("::").last}. Use the openclacky gateway with a TTS model such as or-tts-gemini-2-5-flash.",
+          error_type: "not_implemented",
+          provider: "",
+          input: input
+        )
+      end
+
       # Persist a base64-encoded image under <output_dir>/assets/generated/.
       # Returns the absolute path on disk.
       private def save_b64_image(b64_data, output_dir:, prefix: "img", extension: "png")
@@ -40,6 +63,60 @@ module Clacky
         path
       end
 
+      # Persist a base64-encoded video under <output_dir>/assets/generated/.
+      # Returns the absolute path on disk. Mirrors #save_b64_image; the only
+      # difference is the default extension (mp4).
+      private def save_b64_video(b64_data, output_dir:, prefix: "vid", extension: "mp4")
+        target_dir = File.join(output_dir, "assets", "generated")
+        FileUtils.mkdir_p(target_dir)
+        ts    = Time.now.strftime("%Y%m%d_%H%M%S")
+        short = SecureRandom.hex(4)
+        path  = File.join(target_dir, "#{prefix}_#{ts}_#{short}.#{extension}")
+        File.binwrite(path, Base64.decode64(b64_data))
+        path
+      end
+
+      private def save_b64_audio(b64_data, output_dir:, prefix: "tts", extension: "wav")
+        target_dir = File.join(output_dir, "assets", "generated")
+        FileUtils.mkdir_p(target_dir)
+        ts    = Time.now.strftime("%Y%m%d_%H%M%S")
+        short = SecureRandom.hex(4)
+        path  = File.join(target_dir, "#{prefix}_#{ts}_#{short}.#{extension}")
+        File.binwrite(path, Base64.decode64(b64_data))
+        path
+      end
+
+      # Download a remote image URL and persist it under
+      # <output_dir>/assets/generated/, mirroring save_b64_image so providers
+      # that return URLs (e.g. DashScope, whose links expire after 24h) land
+      # local files at the same path shape as base64 providers.
+      # Returns the absolute path on disk, or nil if the download fails.
+      private def save_image_from_url(url, output_dir:, prefix: "img", extension: "png")
+        body = download_url(url)
+        return nil if body.nil? || body.empty?
+
+        target_dir = File.join(output_dir, "assets", "generated")
+        FileUtils.mkdir_p(target_dir)
+        ts    = Time.now.strftime("%Y%m%d_%H%M%S")
+        short = SecureRandom.hex(4)
+        path  = File.join(target_dir, "#{prefix}_#{ts}_#{short}.#{extension}")
+        File.binwrite(path, body)
+        path
+      end
+
+      # Fetch raw bytes from a URL. Isolated so specs can stub it without a
+      # live HTTP call. Returns the response body String, or nil on failure.
+      private def download_url(url)
+        conn = Faraday.new do |f|
+          f.options.timeout      = 120
+          f.options.open_timeout = 10
+        end
+        resp = conn.get(url)
+        resp.success? ? resp.body : nil
+      rescue Faraday::Error
+        nil
+      end
+
       private def success_response(image:, prompt:, aspect_ratio:, provider:, extra: {})
         {
           "success"      => true,
@@ -63,6 +140,54 @@ module Clacky
           "provider"     => provider
         }
       end
+
+      private def video_success_response(video:, prompt:, aspect_ratio:, provider:, extra: {})
+        {
+          "success"      => true,
+          "video"        => video,
+          "model"        => @model,
+          "prompt"       => prompt,
+          "aspect_ratio" => aspect_ratio,
+          "provider"     => provider
+        }.merge(extra)
+      end
+
+      private def video_error_response(error:, error_type: "provider_error", provider: "", prompt: "", aspect_ratio: "landscape")
+        {
+          "success"      => false,
+          "video"        => nil,
+          "error"        => error,
+          "error_type"   => error_type,
+          "model"        => @model,
+          "prompt"       => prompt,
+          "aspect_ratio" => aspect_ratio,
+          "provider"     => provider
+        }
+      end
+
+      private def audio_success_response(audio:, input:, voice:, provider:, extra: {})
+        {
+          "success"  => true,
+          "audio"    => audio,
+          "model"    => @model,
+          "input"    => input,
+          "voice"    => voice,
+          "provider" => provider
+        }.merge(extra)
+      end
+
+      private def audio_error_response(error:, error_type: "provider_error", provider: "", input: "", voice: "")
+        {
+          "success"    => false,
+          "audio"      => nil,
+          "error"      => error,
+          "error_type" => error_type,
+          "model"      => @model,
+          "input"      => input,
+          "voice"      => voice,
+          "provider"   => provider
+        }
+      end
     end
   end
 end