openclacky 1.2.8 → 1.2.9

data/lib/clacky/media/openai_compat.rb

@@ -0,0 +1,168 @@
+# frozen_string_literal: true
+
+require "faraday"
+require "json"
+require_relative "base"
+
+module Clacky
+  module Media
+    # OpenAI-compatible image generation provider.
+    #
+    # Talks to POST <base_url>/images/generations with the standard OpenAI
+    # request shape. Handles three providers under one class because they
+    # all expose the same endpoint: OpenAI, OpenRouter, and the openclacky
+    # platform gateway. Provider-specific quirks (model id naming, billing)
+    # live in PRESETS, not here.
+    class OpenAICompat < Base
+      ASPECT_TO_SIZE = {
+        "landscape" => "1536x1024",
+        "square"    => "1024x1024",
+        "portrait"  => "1024x1536"
+      }.freeze
+
+      DEFAULT_ASPECT = "landscape"
+
+      def generate_image(prompt:, aspect_ratio: DEFAULT_ASPECT, output_dir: nil, n: 1, **_kwargs)
+        provider_id = Clacky::Providers.find_by_base_url(@base_url) || "custom"
+        aspect      = ASPECT_TO_SIZE.key?(aspect_ratio) ? aspect_ratio : DEFAULT_ASPECT
+        size        = ASPECT_TO_SIZE[aspect]
+
+        if prompt.to_s.strip.empty?
+          return error_response(
+            error: "Prompt is required and must be a non-empty string",
+            error_type: "invalid_argument",
+            provider: provider_id,
+            aspect_ratio: aspect
+          )
+        end
+
+        if @api_key.to_s.empty?
+          return error_response(
+            error: "api_key not configured for image model '#{@model}'",
+            error_type: "auth_required",
+            provider: provider_id,
+            prompt: prompt,
+            aspect_ratio: aspect
+          )
+        end
+
+        payload = { model: @model, n: n }
+        if gemini_family?(@model)
+          # Gemini image models (routed via openclacky / openrouter gateway)
+          # don't accept the OpenAI `size` parameter — they infer aspect from
+          # the prompt text. Embedding a hint keeps the user's aspect choice
+          # honoured without breaking the gateway request validator.
+          payload[:prompt] = "#{prompt}\n\n[aspect: #{aspect}]"
+        else
+          payload[:prompt] = prompt
+          payload[:size]   = size
+        end
+
+        begin
+          response = connection.post("images/generations") do |req|
+            req.headers["Content-Type"]  = "application/json"
+            req.headers["Authorization"] = "Bearer #{@api_key}"
+            req.body = JSON.generate(payload)
+          end
+        rescue Faraday::Error => e
+          return error_response(
+            error: "HTTP request failed: #{e.message}",
+            error_type: "network_error",
+            provider: provider_id,
+            prompt: prompt,
+            aspect_ratio: aspect
+          )
+        end
+
+        unless response.success?
+          return error_response(
+            error: "Upstream #{response.status}: #{truncate(response.body, 500)}",
+            error_type: "api_error",
+            provider: provider_id,
+            prompt: prompt,
+            aspect_ratio: aspect
+          )
+        end
+
+        body = parse_json(response.body)
+        return error_response(
+          error: "Invalid JSON response from upstream",
+          error_type: "invalid_response",
+          provider: provider_id,
+          prompt: prompt,
+          aspect_ratio: aspect
+        ) unless body.is_a?(Hash)
+
+        data = body["data"] || []
+        first = data.first
+        if first.nil?
+          return error_response(
+            error: "Upstream returned no image data",
+            error_type: "empty_response",
+            provider: provider_id,
+            prompt: prompt,
+            aspect_ratio: aspect
+          )
+        end
+
+        image_ref =
+          if first["b64_json"]
+            save_b64_image(first["b64_json"], output_dir: output_dir || Dir.pwd, prefix: "img")
+          elsif first["url"]
+            first["url"]
+          end
+
+        if image_ref.nil?
+          return error_response(
+            error: "Response contained neither b64_json nor url",
+            error_type: "empty_response",
+            provider: provider_id,
+            prompt: prompt,
+            aspect_ratio: aspect
+          )
+        end
+
+        success_response(
+          image: image_ref,
+          prompt: prompt,
+          aspect_ratio: aspect,
+          provider: provider_id,
+          extra: {
+            "size"     => size,
+            "usage"    => body["usage"],
+            "cost_usd" => body["cost_usd"]
+          }.compact
+        )
+      end
+
+      private def connection
+        Faraday.new(url: normalized_base_url) do |f|
+          f.options.timeout      = 240
+          f.options.open_timeout = 10
+        end
+      end
+
+      private def gemini_family?(model_name)
+        model_name.to_s.match?(/gemini|imagen/i)
+      end
+
+      # base_url is taken verbatim from PRESETS (each provider already
+      # includes the API version segment when needed). We only ensure a
+      # trailing slash so Faraday's relative-path join behaves.
+      private def normalized_base_url
+        "#{@base_url.to_s.chomp("/")}/"
+      end
+
+      private def parse_json(body)
+        JSON.parse(body)
+      rescue JSON::ParserError
+        nil
+      end
+
+      private def truncate(str, max)
+        s = str.to_s
+        s.length > max ? "#{s[0, max]}..." : s
+      end
+    end
+  end
+end

data/lib/clacky/providers.rb

@@ -41,6 +41,17 @@ module Clacky
           "dsk-deepseek-v4-flash",
           "or-gemini-3-1-pro"
         ],
+        # Image generation models served by the openclacky platform
+        # gateway. The gateway exposes a standard OpenAI-compatible
+        # /v1/images/generations endpoint, so the same OpenAICompat
+        # provider class handles them. `or-` prefix mirrors the chat
+        # model naming — these are routed through the OpenRouter
+        # backend by the platform.
+        "image_models" => [
+          "or-gemini-3-pro-image",
+          "or-gpt-image-2"
+        ],
+        "default_image_model" => "or-gpt-image-2",
         # Provider-level default: the Claude family served here is vision-capable.
         "capabilities" => { "vision" => true }.freeze,
         # Model-level overrides: DeepSeek models routed through this provider
@@ -123,6 +134,13 @@ module Clacky
           /\Aanthropic\// => "anthropic-messages",
           /\Aclaude[-.]/  => "anthropic-messages"
         }.freeze,
+        # Image generation via OpenRouter is currently routed through the
+        # openclacky platform gateway (see "openclacky" provider above) which
+        # handles the OpenRouter chat-completions + modalities translation.
+        # Direct OpenRouter image config is not exposed here — leave empty
+        # until we ship a dedicated client-side adapter for that protocol.
+        "image_models" => [],
+        "default_image_model" => nil,
         "website_url" => "https://openrouter.ai/keys"
       }.freeze,
 
@@ -305,6 +323,12 @@ module Clacky
           "gpt-5.5" => "gpt-5.4-mini",
           "gpt-5.4" => "gpt-5.4-mini"
         },
+        # OpenAI's image generation model — same /v1/images/generations
+        # endpoint, so the OpenAICompat image provider handles it.
+        "image_models" => [
+          "gpt-image-2"
+        ],
+        "default_image_model" => "gpt-image-2",
         "website_url" => "https://platform.openai.com/api-keys"
       }.freeze,
 
@@ -342,6 +366,8 @@ module Clacky
 
     }.freeze
 
+    MEDIA_KINDS = %w[image video audio].freeze
+
     class << self
       # Check if a provider preset exists
       # @param provider_id [String] The provider identifier (e.g., "anthropic", "openrouter")
@@ -446,6 +472,62 @@ module Clacky
         preset&.dig("models") || []
       end
 
+      # Get available image generation models for a provider.
+      # Returns an empty array when the provider doesn't declare any —
+      # callers should treat that as "image generation not supported by this provider".
+      # @param provider_id [String] The provider identifier
+      # @return [Array<String>] List of image model names
+      def image_models(provider_id)
+        preset = PRESETS[provider_id]
+        preset&.dig("image_models") || []
+      end
+
+      # Video generation models — placeholder. No provider supports video
+      # via Clacky yet; once they do, declare "video_models" alongside
+      # "image_models" in the relevant PRESETS entry and this returns it.
+      def video_models(provider_id)
+        preset = PRESETS[provider_id]
+        preset&.dig("video_models") || []
+      end
+
+      # Audio generation models — same placeholder pattern as video_models.
+      def audio_models(provider_id)
+        preset = PRESETS[provider_id]
+        preset&.dig("audio_models") || []
+      end
+
+      # Unified entry for media model lookup by kind.
+      # @param provider_id [String]
+      # @param kind [String] one of "image" / "video" / "audio"
+      # @return [Array<String>]
+      def media_models(provider_id, kind)
+        case kind.to_s
+        when "image" then image_models(provider_id)
+        when "video" then video_models(provider_id)
+        when "audio" then audio_models(provider_id)
+        else []
+        end
+      end
+
+      # Default media model for a kind under a provider. Falls back to the
+      # first declared model when no explicit default is set in the preset.
+      # Used by AgentConfig#derive_media_models! to pick which model to
+      # surface when the user is on "auto" mode.
+      def default_media_model(provider_id, kind)
+        preset = PRESETS[provider_id]
+        return nil unless preset
+        explicit = preset["default_#{kind}_model"]
+        return explicit if explicit
+        media_models(provider_id, kind).first
+      end
+
+      # The set of media kinds Clacky knows about. Drives UI rendering and
+      # derivation loops — adding a new modality means listing it here plus
+      # adding the corresponding generator class.
+      def media_kinds
+        MEDIA_KINDS
+      end
+
       # Get the lite model for a provider.
       # @param provider_id [String] The provider identifier
       # @param primary_model [String, nil] The currently-selected primary model name.

data/lib/clacky/server/http_server.rb

@@ -370,6 +370,11 @@ module Clacky
           30
         elsif path.end_with?("/benchmark")
           20
+        elsif path == "/api/media/image"
+          # Image generation routes through OpenRouter (chat completions
+          # with modalities:["image"]); end-to-end latency is commonly
+          # 20-60s and can exceed 2 minutes for or-gpt-image-2 under load.
+          300
         else
           10
         end
@@ -399,6 +404,7 @@ module Clacky
         when ["PATCH",  "/api/config/settings"]  then api_update_settings(req, res)
         when ["POST",   "/api/config/models"] then api_add_model(req, res)
         when ["POST",   "/api/config/test"]   then api_test_config(req, res)
+        when ["GET",    "/api/config/media"]  then api_get_media_config(res)
         when ["GET",    "/api/providers"]     then api_list_providers(res)
         when ["GET",    "/api/onboard/status"]    then api_onboard_status(res)
         when ["GET",    "/api/browser/status"]    then api_browser_status(res)
@@ -430,6 +436,8 @@ module Clacky
         when ["POST",   "/api/upload"]            then api_upload_file(req, res)
         when ["POST",   "/api/file-action"]       then api_file_action(req, res)
         when ["GET",    "/api/local-image"]       then api_serve_local_image(req, res)
+        when ["POST",   "/api/media/image"]       then api_media_image(req, res)
+        when ["GET",    "/api/media/types"]       then api_media_types(res)
         when ["GET",    "/api/version"]           then api_get_version(res)
         when ["POST",   "/api/version/upgrade"]   then api_upgrade_version(req, res)
         when ["POST",   "/api/restart"]           then api_restart(req, res)
@@ -523,6 +531,9 @@ module Clacky
           elsif method == "DELETE" && path.match?(%r{^/api/config/models/[^/]+$})
             id = path.sub("/api/config/models/", "")
             api_delete_model(id, res)
+          elsif method == "PATCH" && path.match?(%r{^/api/config/media/(image|video|audio)$})
+            kind = path.sub("/api/config/media/", "")
+            api_update_media_config(kind, req, res)
           elsif method == "POST" && path.match?(%r{^/api/cron-tasks/[^/]+/run$})
             name = URI.decode_www_form_component(path.sub("/api/cron-tasks/", "").sub("/run", ""))
             api_run_cron_task(name, res)
@@ -697,6 +708,158 @@ module Clacky
         json_response(res, 500, { ok: false, error: e.message })
       end
 
+      # POST /api/media/image
+      # Body: { "prompt": "...", "aspect_ratio": "landscape|square|portrait",
+      #         "output_dir": "<absolute path, optional>" }
+      # Routes to the model configured with type=image in agent_config.
+      def api_media_image(req, res)
+        body = parse_json_body(req)
+        return json_response(res, 400, { error: "Invalid JSON" }) unless body
+
+        prompt = body["prompt"].to_s
+        if prompt.strip.empty?
+          return json_response(res, 422, { error: "prompt is required" })
+        end
+
+        aspect_ratio = body["aspect_ratio"].to_s
+        aspect_ratio = "landscape" if aspect_ratio.empty?
+        output_dir   = body["output_dir"].to_s
+        output_dir   = @agent_config.default_working_dir || Dir.pwd if output_dir.empty?
+
+        result = Clacky::Media::Generator.new(@agent_config).generate_image(
+          prompt: prompt,
+          aspect_ratio: aspect_ratio,
+          output_dir: output_dir
+        )
+        if result["success"]
+          log_media_usage(result, prompt: prompt)
+        end
+        status = result["success"] ? 200 : 422
+        json_response(res, status, result)
+      rescue StandardError => e
+        json_response(res, 500, { error: e.message })
+      end
+
+      private def log_media_usage(result, prompt:)
+        usage = result["usage"]
+        cost  = result["cost_usd"]
+        return if usage.nil? && cost.nil?
+
+        parts = []
+        parts << "model=#{result["model"]}"
+        parts << "provider=#{result["provider"]}"
+        if usage.is_a?(Hash)
+          parts << "prompt_tokens=#{usage["prompt_tokens"]}"
+          parts << "completion_tokens=#{usage["completion_tokens"]}"
+          parts << "cache_read=#{usage["cache_read_tokens"]}" if usage["cache_read_tokens"].to_i > 0
+          parts << "cache_write=#{usage["cache_write_tokens"]}" if usage["cache_write_tokens"].to_i > 0
+        end
+        parts << format("cost_usd=%.6f", cost.to_f) if cost
+        parts << "prompt=#{prompt[0, 60].inspect}"
+        Clacky::Logger.info("[Media] image generated #{parts.join(" ")}")
+      end
+
+      # GET /api/media/types
+      # Returns which media types are configured in agent_config.models.
+      # Used by the media-gen skill to decide whether to surface generation
+      # capabilities to the user.
+      def api_media_types(res)
+        out = {}
+        Clacky::Providers::MEDIA_KINDS.each do |t|
+          state = @agent_config.media_state(t)
+          out[t] =
+            if state["configured"]
+              {
+                configured: true,
+                model:      state["model"],
+                base_url:   state["base_url"],
+                source:     state["source"]
+              }
+            else
+              { configured: false, source: "off" }
+            end
+        end
+        json_response(res, 200, out)
+      end
+
+      # GET /api/config/media
+      # Used by the Settings UI to render the tri-state media controls.
+      # Per-kind payload mirrors AgentConfig#media_state.
+      def api_get_media_config(res)
+        out = {}
+        Clacky::Providers::MEDIA_KINDS.each do |t|
+          state = @agent_config.media_state(t)
+          entry = @agent_config.find_model_by_type(t)
+          out[t] = {
+            source:     state["source"],
+            model:      state["model"],
+            base_url:   state["base_url"],
+            api_key_masked: entry ? mask_api_key(entry["api_key"]) : nil,
+            provider:   state["provider"],
+            available:  state["available"],
+            configured: state["configured"]
+          }
+        end
+
+        # Surface what the current default model can offer, even when the
+        # user is currently in "off" — the UI uses this to render the
+        # auto-mode preview ("Auto would use X").
+        default = @agent_config.find_model_by_type("default")
+        provider_id = default && Clacky::Providers.resolve_provider(
+          base_url: default["base_url"],
+          api_key:  default["api_key"]
+        )
+        defaults = {}
+        Clacky::Providers::MEDIA_KINDS.each do |t|
+          defaults[t] = {
+            provider:  provider_id,
+            model:     provider_id ? Clacky::Providers.default_media_model(provider_id, t) : nil,
+            available: provider_id ? Clacky::Providers.media_models(provider_id, t) : []
+          }
+        end
+
+        json_response(res, 200, { media: out, default_provider: defaults })
+      end
+
+      # PATCH /api/config/media/:kind
+      # Body: { source: "off"|"auto"|"custom", model?, base_url?, api_key?,
+      #         anthropic_format? }
+      # off / auto — remove any custom entry; "auto" lets the virtual
+      # derivation in AgentConfig#find_model_by_type take over.
+      # custom — replace any existing custom entry with the supplied fields.
+      def api_update_media_config(kind, req, res)
+        body = parse_json_body(req) || {}
+        source = body["source"].to_s
+        unless %w[off auto custom].include?(source)
+          return json_response(res, 422, { error: "invalid source" })
+        end
+
+        @agent_config.models.reject! { |m| m["type"] == kind }
+
+        if source == "custom"
+          model    = body["model"].to_s.strip
+          base_url = body["base_url"].to_s.strip
+          api_key  = body["api_key"].to_s
+          if model.empty? || base_url.empty? || api_key.empty? || api_key.include?("****")
+            return json_response(res, 422, { error: "model, base_url, api_key are required" })
+          end
+
+          @agent_config.models << {
+            "id"               => SecureRandom.uuid,
+            "model"            => model,
+            "base_url"         => base_url,
+            "api_key"          => api_key,
+            "anthropic_format" => body["anthropic_format"] || false,
+            "type"             => kind
+          }
+        end
+
+        @agent_config.save
+        json_response(res, 200, { ok: true, state: @agent_config.media_state(kind) })
+      rescue => e
+        json_response(res, 422, { error: e.message })
+      end
+
       # POST /api/onboard/complete
       # Called after key setup is done (soul_setup is optional/skipped).
       # Creates the default session if none exists yet, returns it.
@@ -1614,7 +1777,7 @@ module Clacky
       # Returns current config and running status for all supported platforms.
       # POST /api/tool/browser
       # Executes a browser tool action via the shared BrowserManager daemon.
-      # Used by skill scripts (e.g. feishu_setup.rb) to reuse the server's
+      # Used by skill scripts to reuse the server's
       # existing Chrome connection without spawning a second MCP daemon.
       #
       # Request body: JSON with same params as the browser tool
@@ -3285,8 +3448,13 @@ module Clacky
             type:             m["type"]
           }
         end
-        # Filter out auto-injected models (like lite) from UI display
-        models.reject! { |m| @agent_config.models[m[:index]]["auto_injected"] }
+        # Filter out auto-injected models (lite, derived media) AND media
+        # entries (image/video/audio) — those are managed via the dedicated
+        # media-config UI, not the chat-model card list.
+        models.reject! do |m|
+          raw = @agent_config.models[m[:index]]
+          raw["auto_injected"] || Clacky::Providers::MEDIA_KINDS.include?(raw["type"].to_s)
+        end
         json_response(res, 200, {
           models: models,
           current_index: @agent_config.current_model_index,
@@ -3506,29 +3674,51 @@ module Clacky
         return json_response(res, 400, { error: "Invalid JSON" }) unless body
 
         api_key = body["api_key"].to_s
-        # If masked, use the stored key from the matching model (by index or current)
         if api_key.include?("****")
           idx = body["index"]&.to_i || @agent_config.current_model_index
           api_key = @agent_config.models.dig(idx, "api_key").to_s
         end
 
-        begin
-          model = body["model"].to_s
-          test_client = Clacky::Client.new(
-            api_key,
-            base_url:         body["base_url"].to_s,
-            model:            model,
-            anthropic_format: body["anthropic_format"] || false
-          )
-          result = test_client.test_connection(model: model)
-          if result[:success]
-            json_response(res, 200, { ok: true, message: "Connected successfully" })
-          else
-            json_response(res, 200, { ok: false, message: result[:error].to_s })
-          end
-        rescue => e
-          json_response(res, 200, { ok: false, message: e.message })
+        model            = body["model"].to_s
+        base_url         = body["base_url"].to_s
+        anthropic_format = body["anthropic_format"] || false
+
+        result, used_base_url = try_test_with_base_url(api_key, base_url, model, anthropic_format)
+
+        if result[:success] && used_base_url != base_url
+          json_response(res, 200, {
+            ok:                  true,
+            message:             "Connected (auto-corrected base_url to add /v1)",
+            effective_base_url:  used_base_url
+          })
+        elsif result[:success]
+          json_response(res, 200, { ok: true, message: "Connected successfully" })
+        else
+          json_response(res, 200, { ok: false, message: result[:error].to_s })
         end
+      rescue => e
+        json_response(res, 200, { ok: false, message: e.message })
+      end
+
+      private def try_test_with_base_url(api_key, base_url, model, anthropic_format)
+        result = run_test_connection(api_key, base_url, model, anthropic_format)
+        return [result, base_url] if result[:success]
+        return [result, base_url] unless result[:status] == 404
+        return [result, base_url] if base_url.match?(%r{/v\d+/?\z})
+
+        candidate = "#{base_url.chomp("/")}/v1"
+        retried   = run_test_connection(api_key, candidate, model, anthropic_format)
+        retried[:success] ? [retried, candidate] : [result, base_url]
+      end
+
+      private def run_test_connection(api_key, base_url, model, anthropic_format)
+        client = Clacky::Client.new(
+          api_key,
+          base_url:         base_url,
+          model:            model,
+          anthropic_format: anthropic_format
+        )
+        client.test_connection(model: model)
       end
 
       # GET /api/providers — return built-in provider presets for quick setup

data/lib/clacky/telemetry.rb

@@ -23,6 +23,11 @@ module Clacky
   #   POST /api/v1/telemetry/startup
   #   POST /api/v1/telemetry/task
   module Telemetry
+    LAUNCH_SOURCES = {
+      "installer" => "installer",
+      nil         => "cli"
+    }.freeze
+
     class << self
       # Called on every CLI startup (agent and server mode).
       # No local dedup — the server deduplicates by device_hash for unique
@@ -32,11 +37,12 @@ module Clacky
 
         brand = Clacky::BrandConfig.load
         payload = {
-          device_id:    resolve_device_id(brand),
-          version:      Clacky::VERSION,
-          os:           RbConfig::CONFIG["host_os"],
-          ruby_version: RUBY_VERSION,
-          brand:        brand.branded? ? brand.package_name : nil
+          device_id:     resolve_device_id(brand),
+          version:       Clacky::VERSION,
+          os:            RbConfig::CONFIG["host_os"],
+          ruby_version:  RUBY_VERSION,
+          brand:         brand.branded? ? brand.package_name : nil,
+          launch_source: LAUNCH_SOURCES.fetch(ENV["CLACKY_LAUNCHED_BY"], "cli")
         }.compact
 
         fire_and_forget("/api/v1/telemetry/startup", payload)

data/lib/clacky/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Clacky
-  VERSION = "1.2.8"
+  VERSION = "1.2.9"
 end