openclacky 1.2.13 → 1.2.14

data/lib/clacky/ui2/progress_handle.rb

@@ -127,6 +127,7 @@ module Clacky
         @start_time    = nil
         @ticker        = nil
         @state         = :fresh     # :fresh → :running → :closed
+        @unregistered  = false
         @metadata      = {}
         @last_chunk_at = nil
         @monitor       = Monitor.new
@@ -172,34 +173,37 @@ module Clacky
       end
 
       # Stop the ticker, render one final frame, and unregister from the
-      # owner. Idempotent — calling twice is a no-op.
+      # owner. Idempotent and crash-safe — if a previous finish was
+      # interrupted (e.g. Thread#raise(AgentInterrupted) hit between
+      # +stop_ticker+ and +unregister_progress+), a follow-up finish
+      # will still complete the unregister so the handle does not stay
+      # orphaned on the owner's progress stack.
       #
       # @param final_message [String, nil] Optional override for the last
       #   frame. If nil, the handle composes "<message>… (<elapsed>s)".
       def finish(final_message: nil)
-        Clacky::Logger.warn("[ph_debug] finish_entry", oid: object_id, state: @state, msg: @message, eid: @entry_id)
+        Clacky::Logger.warn("[ph_debug] finish_entry", oid: object_id, state: @state, unreg: @unregistered, msg: @message, eid: @entry_id)
         snapshot = @monitor.synchronize do
-          if @state != :running
-            Clacky::Logger.warn("[ph_debug] finish_noop_state", oid: object_id, state: @state)
-            return
-          end
-          @state = :closed
-          { message: final_message || @message, elapsed: elapsed_seconds }
+          return if @unregistered
+          first_close = @state == :running
+          @state = :closed if first_close
+          {
+            first_close: first_close,
+            message: final_message || @message,
+            elapsed: elapsed_seconds,
+          }
         end
 
         stop_ticker
-        # Collapse fast-finishers to a removed entry so tools that complete
-        # in under FAST_FINISH_THRESHOLD_SECONDS don't leave a permanent
-        # "Executing foo… (0s)" line. The owner interprets final_frame: nil
-        # as "remove the entry entirely".
         final_frame =
           if @quiet_on_fast_finish && snapshot[:elapsed] < FAST_FINISH_THRESHOLD_SECONDS
             nil
           else
             compose_final_frame(snapshot[:message], snapshot[:elapsed])
           end
-        Clacky::Logger.warn("[ph_debug] finish_unregister", oid: object_id, eid: @entry_id, final_frame: final_frame.to_s[0, 200])
+        Clacky::Logger.warn("[ph_debug] finish_unregister", oid: object_id, eid: @entry_id, first_close: snapshot[:first_close], final_frame: final_frame.to_s[0, 200])
         @owner.unregister_progress(self, final_frame: final_frame)
+        @monitor.synchronize { @unregistered = true }
         Clacky::Logger.warn("[ph_debug] finish_done", oid: object_id)
       end
       alias_method :cancel, :finish

data/lib/clacky/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Clacky
-  VERSION = "1.2.13"
+  VERSION = "1.2.14"
 end

data/lib/clacky/vision/resolver.rb

@@ -0,0 +1,157 @@
+# frozen_string_literal: true
+
+require "digest"
+require "base64"
+require "fileutils"
+require "json"
+require_relative "../utils/file_processor"
+
+module Clacky
+  module Vision
+    # OCR sidecar — turns image bytes into a text description by calling a
+    # vision-capable model. Used when the user's primary model is text-only
+    # (e.g. DeepSeek V4) so that uploaded images and tool screenshots still
+    # reach the conversation as useful context.
+    #
+    # Routes through Clacky::Client so we get the same OpenAI/Anthropic/
+    # Bedrock format negotiation, retry, and credit-error handling as the
+    # main agent path. Image content travels as a canonical `image_url`
+    # block (the unified internal shape understood by all three formats).
+    class Resolver
+      DEFAULT_PROMPT = <<~PROMPT.strip
+        Extract every legible text and describe the visual content of this image.
+        Output as Markdown. Preserve table layout where possible (use Markdown tables).
+        For UI screenshots, describe the layout, visible labels, and active state.
+        Be thorough but concise — the user cannot see the image and must rely on
+        your description.
+      PROMPT
+
+      MAX_TOKENS = 8192
+      CACHE_DIR  = File.join(Dir.home, ".clacky", "ocr_cache")
+      CACHE_VERSION = 1
+
+      Result = Struct.new(:status, :text, :error, keyword_init: true) do
+        def ok?;          status == :ok;          end
+        def empty?;       status == :empty;       end
+        def call_failed?; status == :call_failed; end
+        def bad_image?;   status == :bad_image;   end
+      end
+
+      def initialize(model_entry)
+        @model_entry = model_entry
+        @model       = model_entry["model"]
+        @base_url    = model_entry["base_url"]
+        @api_key     = model_entry["api_key"]
+        @anthropic   = !!model_entry["anthropic_format"]
+      end
+
+      # @return [Result] one of:
+      #   status=:ok          + text   — sidecar produced a description
+      #   status=:empty               — sidecar returned 200 but no usable text (e.g. token budget exhausted by reasoning)
+      #   status=:call_failed + error — network/parse/auth error from the sidecar
+      #   status=:bad_image           — image bytes unreadable / empty
+      def describe(image, prompt: nil)
+        prompt = prompt.to_s.strip
+        prompt = DEFAULT_PROMPT if prompt.empty?
+
+        bytes, mime = read_image(image)
+        return Result.new(status: :bad_image) if bytes.nil? || bytes.empty?
+
+        cached = cache_get(bytes, prompt)
+        return Result.new(status: :ok, text: cached) if cached
+
+        text = call_vlm(bytes, mime, prompt)
+        return Result.new(status: :empty) if text.nil? || text.strip.empty?
+
+        cache_put(bytes, prompt, text)
+        Result.new(status: :ok, text: text)
+      rescue => e
+        Clacky::Logger.warn("[Vision::Resolver] failed: #{e.class}: #{e.message}") if defined?(Clacky::Logger)
+        Result.new(status: :call_failed, error: "#{e.class}: #{e.message}")
+      end
+
+      private def read_image(image)
+        if image[:bytes]
+          [image[:bytes], image[:mime_type] || "image/png"]
+        elsif image[:data_url] || image["data_url"]
+          url = image[:data_url] || image["data_url"]
+          m = url.match(/\Adata:([^;]+);base64,(.*)\z/m)
+          return [nil, nil] unless m
+          [Base64.decode64(m[2]), m[1]]
+        elsif image[:path] || image["path"]
+          path = image[:path] || image["path"]
+          return [nil, nil] unless File.exist?(path)
+          [File.binread(path), Utils::FileProcessor.detect_mime_type(path, nil) || "image/png"]
+        else
+          [nil, nil]
+        end
+      end
+
+      private def call_vlm(bytes, mime, prompt)
+        data_url = "data:#{mime};base64,#{Base64.strict_encode64(bytes)}"
+        message = {
+          role: "user",
+          content: [
+            { type: "text", text: prompt },
+            { type: "image_url", image_url: { url: data_url } }
+          ]
+        }
+
+        client = Clacky::Client.new(
+          @api_key,
+          base_url: @base_url,
+          model: @model,
+          anthropic_format: @anthropic
+        )
+        response = client.send_messages([message], model: @model, max_tokens: MAX_TOKENS)
+        extract_text(response)
+      end
+
+      # Client#send_messages returns the raw upstream string for OpenAI/Anthropic;
+      # for Bedrock it returns the parsed text content. Normalise to String.
+      private def extract_text(response)
+        case response
+        when String then response
+        when Hash   then response[:content] || response["content"] || response.to_s
+        else response.to_s
+        end
+      end
+
+      # ── Cache ─────────────────────────────────────────────────────────────
+
+      private def cache_key(bytes, prompt)
+        sha = Digest::SHA256.hexdigest(bytes)
+        prompt_sha = Digest::SHA256.hexdigest(prompt)[0, 12]
+        "#{sha}_#{@model.gsub(/[^A-Za-z0-9_.-]/, '_')}_#{prompt_sha}"
+      end
+
+      private def cache_path(key)
+        File.join(CACHE_DIR, "#{key}.json")
+      end
+
+      private def cache_get(bytes, prompt)
+        path = cache_path(cache_key(bytes, prompt))
+        return nil unless File.exist?(path)
+        data = JSON.parse(File.read(path))
+        return nil unless data["v"] == CACHE_VERSION
+        data["text"]
+      rescue JSON::ParserError, Errno::ENOENT
+        nil
+      end
+
+      private def cache_put(bytes, prompt, text)
+        FileUtils.mkdir_p(CACHE_DIR)
+        path = cache_path(cache_key(bytes, prompt))
+        File.write(path, JSON.generate({
+          "v"     => CACHE_VERSION,
+          "model" => @model,
+          "text"  => text,
+          "ts"    => Time.now.to_i
+        }))
+      rescue => _
+        # Cache is best-effort — never fail the request because we can't write.
+        nil
+      end
+    end
+  end
+end

data/lib/clacky/web/i18n.js

@@ -505,12 +505,13 @@ const I18n = (() => {
       "settings.models.badge.default": "Default",
       "settings.models.badge.lite":    "Lite",
       "settings.media.title":         "Media Generation",
-      "settings.media.desc":          "Optional. Image / video / audio generation models.",
+      "settings.media.desc":          "Optional. Image / video / audio / OCR sidecar models.",
       "settings.media.loading":       "Loading…",
       "settings.media.error":         "Failed to load: {{msg}}",
       "settings.media.kind.image":    "Image",
       "settings.media.kind.video":    "Video",
       "settings.media.kind.audio":    "Audio",
+      "settings.media.kind.ocr":      "OCR",
       "settings.media.source.off":    "Off",
       "settings.media.source.auto":   "Auto",
       "settings.media.source.custom": "Custom",
@@ -1245,12 +1246,13 @@ const I18n = (() => {
       "settings.models.badge.default": "默认",
       "settings.models.badge.lite":    "轻量",
       "settings.media.title":         "媒体生成",
-      "settings.media.desc":          "可选。图片 / 视频 / 音频 生成模型。",
+      "settings.media.desc":          "可选。图片 / 视频 / 音频 / 图片理解（OCR）副模型。",
       "settings.media.loading":       "加载中…",
       "settings.media.error":         "加载失败：{{msg}}",
       "settings.media.kind.image":    "图片",
       "settings.media.kind.video":    "视频",
       "settings.media.kind.audio":    "音频",
+      "settings.media.kind.ocr":      "OCR",
       "settings.media.source.off":    "关闭",
       "settings.media.source.auto":   "自动",
       "settings.media.source.custom": "自定义",

data/lib/clacky/web/settings.js

@@ -1528,7 +1528,7 @@ const Settings = (() => {
   // The state object per kind:
   //   { source, configured, model, base_url, api_key_masked, provider, available }
 
-  const MEDIA_KINDS = ["image", "video", "audio"];
+  const MEDIA_KINDS = ["image", "video", "audio", "ocr"];
   let _mediaState = null;
   let _mediaDefaults = null;
   const _mediaCustomDraft = {};
@@ -1538,10 +1538,16 @@ const Settings = (() => {
     if (!container) return;
     container.innerHTML = `<div class="settings-loading">${I18n.t("settings.media.loading")}</div>`;
     try {
-      const res = await fetch("/api/config/media");
-      const data = await res.json();
-      _mediaState = data.media || {};
-      _mediaDefaults = data.default_provider || {};
+      const [mediaRes, ocrRes] = await Promise.all([
+        fetch("/api/config/media"),
+        fetch("/api/config/ocr")
+      ]);
+      const mediaData = await mediaRes.json();
+      const ocrData   = await ocrRes.json();
+      _mediaState = mediaData.media || {};
+      _mediaDefaults = mediaData.default_provider || {};
+      _mediaState["ocr"] = ocrData.ocr || { source: "off", available: [] };
+      _mediaDefaults["ocr"] = ocrData.default_provider || { available: [] };
       _renderMediaRows();
     } catch (e) {
       container.innerHTML = `<div class="settings-error">${I18n.t("settings.media.error", { msg: e.message })}</div>`;
@@ -1557,6 +1563,14 @@ const Settings = (() => {
     });
   }
 
+  function _refreshKindRows(_kind) {
+    _renderMediaRows();
+  }
+
+  async function _reloadKind(_kind) {
+    await _loadMedia();
+  }
+
   function _renderMediaRow(kind) {
     const state = (_mediaState && _mediaState[kind]) || { source: "off", available: [] };
     const def   = (_mediaDefaults && _mediaDefaults[kind]) || { available: [] };
@@ -1662,7 +1676,7 @@ const Settings = (() => {
             _setMediaResult(kind, "testing", I18n.t("settings.media.action.saving"));
             try {
               await _saveMediaConfig(kind, payload);
-              await _loadMedia();
+              await _reloadKind(kind);
             } catch (e) {
               sel.disabled = false;
               _setMediaResult(kind, "fail", e.message);
@@ -1742,7 +1756,7 @@ const Settings = (() => {
           base_url: state.base_url || "",
           api_key:  ""
         };
-        _renderMediaRows();
+        _refreshKindRows(kind);
       });
 
       const testBtn = document.createElement("button");
@@ -1818,7 +1832,7 @@ const Settings = (() => {
         const fallback = (_mediaDefaults && _mediaDefaults[kind] && _mediaDefaults[kind].model) ? "auto" : "off";
         _mediaState[kind] = { ..._mediaState[kind], source: fallback };
       }
-      _renderMediaRows();
+      _refreshKindRows(kind);
     });
 
     const saveBtn = document.createElement("button");
@@ -1838,7 +1852,7 @@ const Settings = (() => {
           api_key:  d.api_key || ""
         });
         delete _mediaCustomDraft[kind];
-        await _loadMedia();
+        await _reloadKind(kind);
       } catch (e) {
         saveBtn.disabled = false;
         cancelBtn.disabled = false;
@@ -1906,7 +1920,8 @@ const Settings = (() => {
   }
 
   async function _saveMediaConfig(kind, body) {
-    const res = await fetch(`/api/config/media/${kind}`, {
+    const url = kind === "ocr" ? `/api/config/ocr` : `/api/config/media/${kind}`;
+    const res = await fetch(url, {
       method: "PATCH",
       headers: { "Content-Type": "application/json" },
       body: JSON.stringify(body)
@@ -1920,10 +1935,14 @@ const Settings = (() => {
 
   async function _testMediaConfig(kind, { model, base_url, api_key }) {
     try {
-      const res = await fetch(`/api/config/media/test`, {
+      const url = kind === "ocr" ? `/api/config/ocr/test` : `/api/config/media/test`;
+      const payload = kind === "ocr"
+        ? { model, base_url, api_key }
+        : { kind, model, base_url, api_key };
+      const res = await fetch(url, {
         method: "POST",
         headers: { "Content-Type": "application/json" },
-        body: JSON.stringify({ kind, model, base_url, api_key })
+        body: JSON.stringify(payload)
       });
       const data = await res.json().catch(() => ({}));
       if (!res.ok) return { ok: false, message: data.error || `HTTP ${res.status}` };

data/lib/clacky.rb

@@ -128,6 +128,7 @@ require_relative "clacky/mcp/skill_provider"
 require_relative "clacky/media/base"
 require_relative "clacky/media/openai_compat"
 require_relative "clacky/media/generator"
+require_relative "clacky/vision/resolver"
 require_relative "clacky/telemetry"
 require_relative "clacky/agent"
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: openclacky
 version: !ruby/object:Gem::Version
-  version: 1.2.13
+  version: 1.2.14
 platform: ruby
 authors:
 - windy
@@ -359,6 +359,7 @@ files:
 - lib/clacky/default_parsers/pdf_parser.rb
 - lib/clacky/default_parsers/pdf_parser_ocr.py
 - lib/clacky/default_parsers/pdf_parser_plumber.py
+- lib/clacky/default_parsers/pdf_parser_vlm.py
 - lib/clacky/default_parsers/pptx_parser.rb
 - lib/clacky/default_parsers/wps_parser.rb
 - lib/clacky/default_parsers/xlsx_parser.rb
@@ -532,6 +533,7 @@ files:
 - lib/clacky/utils/trash_directory.rb
 - lib/clacky/utils/workspace_rules.rb
 - lib/clacky/version.rb
+- lib/clacky/vision/resolver.rb
 - lib/clacky/web/app.css
 - lib/clacky/web/app.js
 - lib/clacky/web/apple-touch-icon-180.png