openclacky 1.3.4 → 1.3.5

data/lib/clacky/null_ui_controller.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+require_relative "ui_interface"
+
+module Clacky
+  # A UI controller that swallows every event. Used for detached/background
+  # subagents whose intermediate output must never reach a real UI stream
+  # (e.g. the WebUI chat transcript). All UIInterface methods are inherited
+  # as no-ops, so nothing this agent emits is broadcast anywhere.
+  class NullUIController
+    include Clacky::UIInterface
+  end
+end

data/lib/clacky/plain_ui_controller.rb

@@ -117,7 +117,7 @@ module Clacky
       puts_line("[warn] #{message}")
     end
 
-    def show_error(message, code: nil, top_up_url: nil)
+    def show_error(message, code: nil, top_up_url: nil, raw_message: nil)
       puts_line("[error] #{message}")
     end
 

data/lib/clacky/providers.rb

@@ -88,6 +88,30 @@ module Clacky
           "or-tts-gemini-2-5-pro"   => "Gemini 2.5 Pro TTS"
         },
         "default_audio_model" => "or-tts-gemini-2-5-flash",
+        # Speech-to-text models served by the openclacky gateway, which
+        # routes them to Vertex AI Gemini (generateContent with inline
+        # audio parts). The gateway returns transcription text.
+        "stt_models" => [
+          "or-stt-gemini-3-5-flash",
+          "or-stt-gemini-1-5-pro"
+        ],
+        "stt_model_aliases" => {
+          "or-stt-gemini-3-5-flash" => "Gemini 3.5 Flash STT",
+          "or-stt-gemini-1-5-pro"   => "Gemini 1.5 Pro STT"
+        },
+        "default_stt_model" => "or-stt-gemini-3-5-flash",
+        # Video understanding models served by the openclacky gateway, which
+        # routes video frames to Gemini (generateContent with inline image
+        # parts). The gateway returns analysis text.
+        "video_understanding_models" => [
+          "or-gemini-3-5-flash",
+          "or-gemini-3-1-pro"
+        ],
+        "video_understanding_model_aliases" => {
+          "or-gemini-3-5-flash" => "Gemini 3.5 Flash",
+          "or-gemini-3-1-pro"   => "Gemini 3.1 Pro"
+        },
+        "default_video_understanding_model" => "or-gemini-3-5-flash",
         # Default OCR sidecar — used when the primary model is text-only.
         # Candidates are derived from the provider's vision-capable models;
         # this just picks the cheap+fast default to surface in "auto" mode.
@@ -418,7 +442,7 @@ module Clacky
 
     }.freeze
 
-    MEDIA_KINDS = %w[image video audio].freeze
+    MEDIA_KINDS = %w[image video audio stt video_understanding].freeze
 
     class << self
       # Check if a provider preset exists
@@ -549,11 +573,33 @@ module Clacky
         preset&.dig("audio_model_aliases") || {}
       end
 
+      def stt_models(provider_id)
+        preset = PRESETS[provider_id]
+        preset&.dig("stt_models") || []
+      end
+
+      def stt_model_aliases(provider_id)
+        preset = PRESETS[provider_id]
+        preset&.dig("stt_model_aliases") || {}
+      end
+
+      def video_understanding_models(provider_id)
+        preset = PRESETS[provider_id]
+        preset&.dig("video_understanding_models") || []
+      end
+
+      def video_understanding_model_aliases(provider_id)
+        preset = PRESETS[provider_id]
+        preset&.dig("video_understanding_model_aliases") || {}
+      end
+
       def media_model_aliases(provider_id, kind)
         case kind.to_s
         when "image" then image_model_aliases(provider_id)
         when "video" then video_model_aliases(provider_id)
         when "audio" then audio_model_aliases(provider_id)
+        when "stt"   then stt_model_aliases(provider_id)
+        when "video_understanding" then video_understanding_model_aliases(provider_id)
         else {}
         end
       end
@@ -599,13 +645,15 @@ module Clacky
 
       # Unified entry for media model lookup by kind.
       # @param provider_id [String]
-      # @param kind [String] one of "image" / "video" / "audio"
+      # @param kind [String] one of "image" / "video" / "audio" / "stt"
       # @return [Array<String>]
       def media_models(provider_id, kind)
         case kind.to_s
         when "image" then image_models(provider_id)
         when "video" then video_models(provider_id)
         when "audio" then audio_models(provider_id)
+        when "stt"   then stt_models(provider_id)
+        when "video_understanding" then video_understanding_models(provider_id)
         else []
         end
       end

data/lib/clacky/rich_ui/rich_ui_controller.rb

@@ -331,7 +331,7 @@ module Clacky
       @shell.add_system_message("Warning: #{message}")
     end
 
-    def show_error(message)
+    def show_error(message, **)
       @shell.add_error_message(message.to_s)
     end
 

data/lib/clacky/server/channel/channel_ui_controller.rb

@@ -145,7 +145,7 @@ module Clacky
         send_text("Warning: #{message}")
       end
 
-      def show_error(message, code: nil, top_up_url: nil)
+      def show_error(message, code: nil, top_up_url: nil, raw_message: nil)
         text = "Error: #{message}"
         text += "\n#{top_up_url}" if top_up_url
         send_text(text)

data/lib/clacky/server/http_server.rb

@@ -368,6 +368,8 @@ module Clacky
             res.body                  = html
           elsif req.path.start_with?("/webui_ext/")
             self.send(:serve_webui_ext, req, res)
+          elsif req.path.start_with?("/builtin_ext/")
+            self.send(:serve_builtin_ext, req, res)
           elsif req.path.start_with?("/agent_ui/")
             self.send(:serve_agent_ui, req, res)
           elsif req.path.start_with?("/panel_ui/")
@@ -462,6 +464,10 @@ module Clacky
           600
         elsif path == "/api/media/audio/speech"
           120
+        elsif path == "/api/media/audio/transcriptions"
+          30
+        elsif path == "/api/media/video/understand"
+          60
         elsif path.start_with?("/api/backup/download") || path == "/api/backup/run"
           # Building a tar.gz of ~/.clacky (with session history) can take a while.
           120
@@ -549,6 +555,8 @@ module Clacky
         when ["POST",   "/api/media/image"]       then api_media_image(req, res)
         when ["POST",   "/api/media/video"]       then api_media_video(req, res)
         when ["POST",   "/api/media/audio/speech"] then api_media_audio_speech(req, res)
+        when ["POST",   "/api/media/audio/transcriptions"] then api_media_audio_transcriptions(req, res)
+        when ["POST",   "/api/media/video/understand"]     then api_media_video_understand(req, res)
         when ["GET",    "/api/media/types"]       then api_media_types(res)
         when ["GET",    "/api/version"]           then api_get_version(res)
         when ["POST",   "/api/version/upgrade"]   then api_upgrade_version(req, res)
@@ -677,7 +685,7 @@ module Clacky
           elsif method == "DELETE" && path.match?(%r{^/api/config/models/[^/]+$})
             id = path.sub("/api/config/models/", "")
             api_delete_model(id, res)
-          elsif method == "PATCH" && path.match?(%r{^/api/config/media/(image|video|audio)$})
+          elsif method == "PATCH" && path.match?(%r{^/api/config/media/(image|video|audio|stt|video_understanding)$})
             kind = path.sub("/api/config/media/", "")
             api_update_media_config(kind, req, res)
           elsif method == "POST" && path.match?(%r{^/api/cron-tasks/[^/]+/run$})
@@ -1033,6 +1041,9 @@ module Clacky
       # Absolute path to the user's WebUI extension directory.
       WEBUI_EXT_ROOT = File.expand_path("~/.clacky/webui_ext")
 
+      # Built-in extensions bundled with the gem (default_extensions/*/**.js).
+      BUILTIN_EXT_ROOT = File.expand_path("../default_extensions", __dir__)
+
       # Build the full <script> payload injected at {{EXT_SCRIPTS}}:
       #   1. global extensions   — ~/.clacky/webui_ext/**/*.js  (all agents)
       #   2. agent-scoped UI     — agents/<name>/webui/**/*.js   (data-agent)
@@ -1042,6 +1053,7 @@ module Clacky
       private def webui_ext_script_tags
         [
           panel_agents_script,
+          builtin_ext_script_tags,
           global_ext_script_tags,
           agent_webui_script_tags,
           official_panel_script_tags,
@@ -1093,6 +1105,21 @@ module Clacky
         dirs
       end
 
+      # Built-in extensions from default_extensions/*/*.js — loaded before user
+      # extensions so user ~/.clacky/webui_ext/ can override by ext id.
+      private def builtin_ext_script_tags
+        return "" unless Dir.exist?(BUILTIN_EXT_ROOT)
+
+        Dir.glob(File.join(BUILTIN_EXT_ROOT, "*", "**", "*.js")).sort.map do |abs|
+          rel    = abs.delete_prefix(BUILTIN_EXT_ROOT + "/")
+          ext_id = "builtin/" + rel.delete_suffix(".js")
+          src    = "/builtin_ext/#{rel}"
+          "<script>Clacky.ext._extBegin(#{ext_id.to_json}, #{nil.to_json}, #{nil.to_json})</script>" \
+            "<script src=#{src.to_json} data-ext-id=#{ext_id.to_json}></script>" \
+            "<script>Clacky.ext._extEnd()</script>"
+        end.join("\n")
+      end
+
       # Global extensions — visible for all agents (unchanged legacy behavior).
       private def global_ext_script_tags
         ext_script_block(WEBUI_EXT_ROOT, "/webui_ext")
@@ -1173,6 +1200,31 @@ module Clacky
         res.body             = File.read(abs)
       end
 
+      private def serve_builtin_ext(req, res)
+        rel = req.path.delete_prefix("/builtin_ext/")
+        abs = File.expand_path(File.join(BUILTIN_EXT_ROOT, rel))
+
+        unless abs.start_with?(BUILTIN_EXT_ROOT + File::SEPARATOR) && File.file?(abs)
+          res.status = 404
+          res.body   = "not found"
+          return
+        end
+
+        ext = File.extname(abs)
+        ctype = { ".js" => "application/javascript", ".css" => "text/css" }[ext]
+        unless ctype
+          res.status = 415
+          res.body   = "unsupported media type"
+          return
+        end
+
+        res.status           = 200
+        res["Content-Type"]  = ctype
+        res["Cache-Control"] = "no-store"
+        res["Pragma"]        = "no-cache"
+        res.body             = File.read(abs)
+      end
+
       # Serve agents/<name>/webui/<file> from built-in or user agent dir.
       # Path: /agent_ui/<name>/<rel>. User dir wins on name collision.
       private def serve_agent_ui(req, res)
@@ -1405,6 +1457,59 @@ module Clacky
         json_response(res, 500, { error: e.message })
       end
 
+      def api_media_audio_transcriptions(req, res)
+        body = parse_json_body(req)
+        return json_response(res, 400, { error: "Invalid JSON" }) unless body
+
+        audio_b64 = body["audio_base64"].to_s
+        if audio_b64.empty?
+          return json_response(res, 422, { error: "audio_base64 is required" })
+        end
+
+        mime_type = body["mime_type"].to_s
+        mime_type = "audio/webm" if mime_type.empty?
+
+        result = Clacky::Media::Generator.new(@agent_config).generate_transcription(
+          audio_base64: audio_b64,
+          mime_type: mime_type
+        )
+        if result["success"]
+          Clacky::Logger.info("[Media] stt generated model=#{result["model"]} provider=#{result["provider"]} cost_usd=#{result["cost_usd"].to_f}")
+        end
+        status = result["success"] ? 200 : 422
+        json_response(res, status, result)
+      rescue StandardError => e
+        json_response(res, 500, { error: e.message })
+      end
+
+      def api_media_video_understand(req, res)
+        body = parse_json_body(req)
+        return json_response(res, 400, { error: "Invalid JSON" }) unless body
+
+        video_b64 = body["video_base64"].to_s
+        if video_b64.empty?
+          return json_response(res, 422, { error: "video_base64 is required" })
+        end
+
+        mime_type = body["mime_type"].to_s
+        mime_type = "image/png" if mime_type.empty?
+
+        prompt = body["prompt"].to_s
+
+        result = Clacky::Media::Generator.new(@agent_config).understand_video(
+          video_base64: video_b64,
+          mime_type: mime_type,
+          prompt: prompt
+        )
+        if result["success"]
+          Clacky::Logger.info("[Media] video_understanding generated model=#{result["model"]} provider=#{result["provider"]} cost_usd=#{result["cost_usd"].to_f}")
+        end
+        status = result["success"] ? 200 : 422
+        json_response(res, status, result)
+      rescue StandardError => e
+        json_response(res, 500, { error: e.message })
+      end
+
       private def log_media_usage(result, prompt:)
         usage = result["usage"]
         cost  = result["cost_usd"]
@@ -1967,6 +2072,7 @@ module Clacky
             product_name:                  brand.product_name,
             homepage_url:                  brand.homepage_url,
             logo_url:                      brand.logo_url,
+            theme_color:                   brand.theme_color,
             test_mode:                     @brand_test,
             distribution_refresh_pending:  refresh_pending
           })
@@ -2038,7 +2144,8 @@ module Clacky
             ok:            true,
             product_name:  result[:product_name] || brand.product_name,
             user_id:       result[:user_id] || brand.license_user_id,
-            user_licensed: brand.user_licensed?
+            user_licensed: brand.user_licensed?,
+            theme_color:   brand.theme_color
           })
         else
           json_response(res, 422, { ok: false, error: result[:message] })
@@ -5820,6 +5927,22 @@ module Clacky
         Clacky::Logger.error("[interrupt] force_close_session_sockets error: #{e.class}: #{e.message}")
       end
 
+      # Run a task in a session immediately in the background, without waiting
+      # for the client to subscribe. The user bubble is persisted via
+      # display_text (Agent#run → history → replay_history), so the frontend
+      # only needs to navigate over and load history — no realtime broadcast,
+      # no subscribe-timing race. This is the stable entry point for
+      # programmatic "create session + run now" flows (spawn, extensions).
+      def run_session_task(session_id, prompt, display_message: nil)
+        return unless @registry.exist?(session_id)
+
+        agent = nil
+        @registry.with_session(session_id) { |s| agent = s[:agent] }
+        return unless agent
+
+        run_agent_task(session_id, agent) { agent.run(prompt, display_text: display_message) }
+      end
+
       # Start the pending task for a session.
       # Called when the client sends "run_task" over WS — by that point the
       # client has already subscribed, so every broadcast will be delivered.
@@ -5827,18 +5950,29 @@ module Clacky
         return unless @registry.exist?(session_id)
 
         session = @registry.get(session_id)
-        prompt      = session[:pending_task]
-        working_dir = session[:pending_working_dir]
+        prompt          = session[:pending_task]
+        working_dir     = session[:pending_working_dir]
+        display_message = session[:pending_display_message]
         return unless prompt  # nothing pending
 
         # Clear the pending fields so a re-connect doesn't re-run
-        @registry.update(session_id, pending_task: nil, pending_working_dir: nil)
+        @registry.update(session_id, pending_task: nil, pending_working_dir: nil, pending_display_message: nil)
 
         agent = nil
         @registry.with_session(session_id) { |s| agent = s[:agent] }
         return unless agent
 
-        run_agent_task(session_id, agent) { agent.run(prompt) }
+        # Surface a user message on screen before the agent starts thinking, so
+        # programmatically-submitted tasks (e.g. meeting summarization) don't
+        # appear as a thinking spinner with no preceding message. When a short
+        # display_message is provided we show that instead of the full prompt.
+        if display_message
+          web_ui = nil
+          @registry.with_session(session_id) { |s| web_ui = s[:ui] }
+          web_ui&.show_user_message(display_message, source: :web)
+        end
+
+        run_agent_task(session_id, agent) { agent.run(prompt, display_text: display_message) }
       end
 
       # Interrupt every running agent thread and persist its session state.
@@ -5921,10 +6055,11 @@ module Clacky
             top_up_url = preset && preset["website_url"]
           end
           user_message = e.respond_to?(:display_message) && e.display_message ? e.display_message : e.message
-          @registry.update(session_id, status: :error, error: user_message, error_code: code, top_up_url: top_up_url)
+          raw_message  = e.respond_to?(:raw_message) ? e.raw_message : nil
+          @registry.update(session_id, status: :error, error: user_message, error_code: code, top_up_url: top_up_url, raw_message: raw_message)
           broadcast_session_update(session_id)
-          web_ui&.show_error(user_message, code: code, top_up_url: top_up_url)
-          @session_manager.save(agent.to_session_data(status: :error, error_message: user_message))
+          web_ui&.show_error(user_message, code: code, top_up_url: top_up_url, raw_message: raw_message)
+          @session_manager.save(agent.to_session_data(status: :error, error_message: user_message, raw_message: raw_message))
         end
         @registry.with_session(session_id) { |s| s[:thread] = thread }
       end

data/lib/clacky/server/session_registry.rb

@@ -44,6 +44,7 @@ module Clacky
           error:                nil,
           error_code:           nil,
           top_up_url:           nil,
+          raw_message:          nil,
           updated_at:           nil,
           agent:                nil,
           ui:                   nil,
@@ -169,7 +170,7 @@ module Clacky
             live_name  = s[:agent]&.name
             live_name  = nil if live_name&.empty?
           live_cost_source = s[:agent]&.cost_source
-          { status: s[:status], error: s[:error], error_code: s[:error_code], top_up_url: s[:top_up_url],
+          { status: s[:status], error: s[:error], error_code: s[:error_code], top_up_url: s[:top_up_url], raw_message: s[:raw_message],
             updated_at: s[:updated_at]&.iso8601,
             model: model_info&.dig(:model), model_id: model_info&.dig(:id), name: live_name,
             total_tasks: s[:agent]&.total_tasks, total_cost: s[:agent]&.total_cost,
@@ -269,7 +270,7 @@ module Clacky
           model_info = s[:agent]&.current_model_info
           live_name  = s[:agent]&.name
           live_name  = nil if live_name&.empty?
-          { status: s[:status], error: s[:error], error_code: s[:error_code], top_up_url: s[:top_up_url],
+          { status: s[:status], error: s[:error], error_code: s[:error_code], top_up_url: s[:top_up_url], raw_message: s[:raw_message],
             updated_at: s[:updated_at]&.iso8601,
             model: model_info&.dig(:model), model_id: model_info&.dig(:id),
             name: live_name, total_tasks: s[:agent]&.total_tasks,
@@ -296,6 +297,7 @@ module Clacky
           error:         ls ? ls[:error] : nil,
           error_code:    ls&.dig(:error_code),
           top_up_url:    ls&.dig(:top_up_url),
+          raw_message:   ls&.dig(:raw_message),
           model:         ls&.dig(:model),
           model_id:      ls&.dig(:model_id),
           card_model:    ls&.dig(:card_model),

data/lib/clacky/server/web_ui_controller.rb

@@ -230,12 +230,13 @@ module Clacky
         @broadcaster.call(@session_id, event)
       end
 
-      def show_error(message, code: nil, top_up_url: nil)
+      def show_error(message, code: nil, top_up_url: nil, raw_message: nil)
         payload = { message: message }
         payload[:code] = code if code
         payload[:top_up_url] = top_up_url if top_up_url
+        payload[:raw_message] = raw_message if raw_message
         emit("error", **payload)
-        forward_to_subscribers { |sub| sub.show_error(message, code: code, top_up_url: top_up_url) }
+        forward_to_subscribers { |sub| sub.show_error(message, code: code, top_up_url: top_up_url, raw_message: raw_message) }
       end
 
       def show_success(message)

data/lib/clacky/skill_loader.rb

@@ -441,13 +441,11 @@ module Clacky
 
     # Load default skills from gem's default_skills directory
     private def load_default_skills
-      # Get the gem's lib directory
       gem_lib_dir = File.expand_path("../", __dir__)
       default_skills_dir = File.join(gem_lib_dir, "clacky", "default_skills")
 
       return unless Dir.exist?(default_skills_dir)
 
-      # Load each skill directory
       Dir.glob(File.join(default_skills_dir, "*/SKILL.md")).each do |skill_file|
         skill_dir = File.dirname(skill_file)
         skill_name = File.basename(skill_dir)
@@ -459,6 +457,20 @@ module Clacky
           @errors << "Failed to load default skill #{skill_name}: #{e.message}"
         end
       end
+
+      # Also load skills bundled inside default_extensions/*/skills/
+      ext_skills_dir = File.join(gem_lib_dir, "clacky", "default_extensions")
+      Dir.glob(File.join(ext_skills_dir, "*/skills/*/SKILL.md")).each do |skill_file|
+        skill_dir = File.dirname(skill_file)
+        skill_name = File.basename(skill_dir)
+
+        begin
+          skill = Skill.new(Pathname.new(skill_dir))
+          register_skill(skill, source: :default)
+        rescue StandardError => e
+          @errors << "Failed to load default skill #{skill_name}: #{e.message}"
+        end
+      end
     end
   end
 end

data/lib/clacky/tools/terminal/output_cleaner.rb

@@ -35,9 +35,7 @@ module Clacky
         def clean(raw)
           return "" if raw.nil? || raw.empty?
 
-          s = raw.dup
-          s.force_encoding(Encoding::UTF_8)
-          s = s.scrub("?") unless s.valid_encoding?
+          s = Clacky::Utils::Encoding.pty_to_utf8(raw)
 
           s = s.gsub(CSI_REGEX, "")
           s = s.gsub(OSC_REGEX, "")

data/lib/clacky/tools/terminal.rb

@@ -340,10 +340,6 @@ module Clacky
           project_root: cwd || Dir.pwd
         )
 
-        # PowerShell 5 on Chinese Windows emits CP936/GBK by default; force
-        # UTF-8 so our PTY (which decodes as UTF-8) doesn't see ??? bytes.
-        safe_command = force_powershell_utf8(safe_command)
-
         # Background / dedicated path — never reuse the persistent shell,
         # because these commands stay running and would occupy the slot.
         if background
@@ -1522,45 +1518,6 @@ module Clacky
         return "" if lines.empty?
         lines.last(DISPLAY_TAIL_LINES).join("\n")
       end
-
-      # PowerShell 5 on Chinese Windows defaults [Console]::OutputEncoding
-      # to CP936/GBK; our PTY decodes as UTF-8 so non-ASCII output becomes
-      # `???`. Inject UTF-8 setup into the user's PowerShell command so the
-      # shell emits UTF-8 bytes regardless of host locale.
-      POWERSHELL_PREAMBLE =
-        "[Console]::OutputEncoding=[Text.Encoding]::UTF8;"
-
-      # Only rewrites simple `powershell[.exe]` / `pwsh[.exe]` invocations.
-      # Skips -File / -EncodedCommand / commands already handling encoding /
-      # pipelines (anything risky to splice).
-      private def force_powershell_utf8(command)
-        cmd = command.to_s
-        return command unless cmd =~ /\A\s*(?:powershell(?:\.exe)?|pwsh(?:\.exe)?)\b/i
-        return command if cmd =~ /OutputEncoding/i
-        return command if cmd =~ /\s-(?:File|EncodedCommand|enc|f)\b/i
-
-        # `-Command "..."` form: pipeline / chain characters inside the
-        # quoted body are PowerShell-internal, not shell-level, so we splice
-        # safely into the quoted string.
-        if (m = cmd.match(/\A(\s*(?:powershell(?:\.exe)?|pwsh(?:\.exe)?)\s+(?:[^"'\s]+\s+)*?-(?:Command|c)\s+)(["'])(.*)\2(\s*(?:<\s*\S+\s*)?)\z/i))
-          head, quote, body, tail = m[1], m[2], m[3], m[4].to_s
-          return "#{head}#{quote}#{POWERSHELL_PREAMBLE}#{body}#{quote}#{tail}"
-        end
-
-        # Outside the quoted-Command form, refuse to splice if there's any
-        # shell-level pipe / chain — too risky to get the boundaries right.
-        return command if cmd =~ /[|&;]/
-
-        if (m = cmd.match(/\A(\s*(?:powershell(?:\.exe)?|pwsh(?:\.exe)?))(.*)\z/i))
-          exe, rest = m[1], m[2].to_s.strip
-          return command if rest.start_with?("-") && rest !~ /\A-(?:Command|c)\b/i
-          rest = rest.sub(/\A-(?:Command|c)\b\s*/i, "")
-          inner = rest.empty? ? POWERSHELL_PREAMBLE.chomp(";") : "#{POWERSHELL_PREAMBLE}#{rest}"
-          return %Q{#{exe} -Command "#{inner}"}
-        end
-
-        command
-      end
     end
   end
 end

data/lib/clacky/ui2/components/modal_component.rb

@@ -326,7 +326,7 @@ module Clacky
               print "\e[?25l"
               return :cancelled
             when "\u0015"  # Ctrl+U - clear line
-              buffer = ''
+              buffer = String.new
               cursor_pos = 0
             else
               # Regular character input