openclacky 1.2.13 → 1.2.14

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f2e02be3208e8ffa6a857da34c1c3bff2db9cc52d075f67481e3f85d2b5fe8be
-  data.tar.gz: 6be5d6844301671cb3f3521248a7091c2978b9e4423d02b6bcc60d9ffbb60a97
+  metadata.gz: 82874a3ac7c623672bd09b5fa1be1c5dd70b1f223119a1b58b86f85417e46f1c
+  data.tar.gz: ba5f1cc02f50a0bee31e24a6ad009c265881eef8b8b9efa6f17b5bec29124414
 SHA512:
-  metadata.gz: 429dc77e88fa2f1febb7177a903c3229b2f382e4f52a02be5c3980fbaa3f64e9a81aca82a108b1e1d2f11481b768c86384e5e4d827f2ddea6f7c7067e0ef2db4
-  data.tar.gz: 2c2a3774f968f2d3f53ce1632470686b66c1d0d903617549f9cad2650a865ffd39821934591978d4df89a3c4e18eb769bbabe90b3826d557ab71fa90b4eb361b
+  metadata.gz: 5535350a83909fffe2471ab0f6505d54f9bc2436826636eacb1c8d6bbbd84e554087b31b9503d6671449789160072eb743711556f302dc42b820637b7edab83d
+  data.tar.gz: bcdec5ed7e56cfc27ee2370ae46582239fa425254e013a7577f162b28c6e2d88b821768f2e367fe2b33dd6773804740692f1f29cba0ca2d73f40d38f6b8e2243

data/.clacky/skills/gem-release/SKILL.md

@@ -177,6 +177,10 @@ Ask the user whether to use `--update-latest` before running the script.
 The script uses `set -euo pipefail` and stops on any failure. Common issues:
 
 - **Tests fail** → fix tests before re-running
+- **Web search smoke test fails (Bing)** → This often happens due to datacenter IP fingerprinting (anti-scrape blocking) returning irrelevant top-domain filler (like Mr.Bricolage). If you see "No ruby-related result from bing" during the smoke test:
+  1. Manually run `bundle exec rspec spec/integration/web_search_smoke_spec.rb --tag smoke` to verify
+  2. If it's the anti-scrape block, temporarily edit `spec/integration/web_search_smoke_spec.rb` to skip the relevance check on failure (e.g., using `skip "Bing returned anti-scrape garbage..."`)
+  3. Commit the change ("ci: skip bing smoke test relevance check on anti-scrape") and re-run the release script
 - **CI fails** → script pushes then watches CI; fix and re-push if needed
 - **gem push fails** → check RubyGems credentials (`gem signin`)
 - **gh release fails** → check `gh auth status`

data/CHANGELOG.md

@@ -5,6 +5,22 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [1.2.14] - 2026-06-08
+
+### Added
+- OCR support for scanned PDFs (optical character recognition)
+- VLM-based PDF parser for improved document understanding
+
+### Improved
+- PDF OCR processing quality
+
+### Fixed
+- PDF processing not appearing in session history
+- Stale progress indicator that wouldn't dismiss
+
+### More
+- Document Bing smoke test anti-scrape failure handling in gem-release
+
 ## [1.2.13] - 2026-06-08
 
 ### Added

data/lib/clacky/agent/session_serializer.rb

@@ -272,6 +272,7 @@ module Clacky
           # Disk files (PDF, doc, etc.): stored in display_files on the user message at send time
           disk_files  = Array(msg[:display_files]).map { |f|
             { name: f[:name] || f["name"], type: f[:type] || f["type"] || "file",
+              path: f[:path] || f["path"],
               preview_path: f[:preview_path] || f["preview_path"] }
           }
           all_files = image_files + disk_files

data/lib/clacky/agent.rb

@@ -341,19 +341,23 @@ module Clacky
         # the file_prompt builder can't emit the "not supported by model" /
         # "too large" note for downgraded images.
         downgrade_reason = f[:downgrade_reason] || f["downgrade_reason"]
+        ocr_text         = f[:ocr_text]         || f["ocr_text"]
         ref = Utils::FileProcessor.process_path(path, name: name)
         { name: ref.name, type: ref.type.to_s, path: ref.original_path,
           preview_path: ref.preview_path, parse_error: ref.parse_error, parser_path: ref.parser_path,
-          downgrade_reason: downgrade_reason }
+          downgrade_reason: downgrade_reason, ocr_text: ocr_text }
       end
 
       # Build display_files for replay: lightweight metadata so the UI can reconstruct
-      # file badges (PDF, doc, etc.) on page refresh. Images are NOT stored here — they
-      # are recovered from the image_url blocks in user_content by extract_image_files_from_content.
+      # file badges (PDF, doc, etc.) on page refresh. Vision-inlined images are NOT
+      # stored here — they recover from image_url blocks in user_content. Downgraded
+      # images (provider has no vision / too large / OCR'd) DO need path here so the
+      # UI can re-render them from the on-disk copy across session switches.
       display_files = all_disk_files.filter_map do |f|
         name = f[:name] || f["name"]
         next unless name
         { name: name, type: f[:type] || f["type"] || "file",
+          path: f[:path] || f["path"],
           preview_path: f[:preview_path] || f["preview_path"] }
       end
 
@@ -381,6 +385,7 @@ module Clacky
           parse_error      = f[:parse_error]      || f["parse_error"]
           parser_path      = f[:parser_path]      || f["parser_path"]
           downgrade_reason = f[:downgrade_reason] || f["downgrade_reason"]
+          ocr_text         = f[:ocr_text]         || f["ocr_text"]
 
           next unless name
 
@@ -396,6 +401,14 @@ module Clacky
           note = downgrade_note_for(downgrade_reason)
           lines << "Note: #{note}" if note
 
+          # OCR transcription (when an OCR sidecar successfully described
+          # an image the primary model couldn't see). Embedded inline so
+          # the LLM has the description colocated with the file entry.
+          if ocr_text && !ocr_text.strip.empty?
+            lines << "OCR description:"
+            lines << ocr_text.strip
+          end
+
           # Parser failed — instruct LLM to fix and re-run
           if preview_path.nil? && parse_error
             lines << "Parse failed: #{parse_error}"
@@ -1098,6 +1111,9 @@ module Clacky
       # base64 data in a `role:"tool"` message causes it to be JSON-encoded as
       # plain text, inflating token counts by 20-40x.  The tool result carries a
       # plain-text description for the LLM; the actual image is delivered here.
+      vision_supported = @config.current_model_supports?(:vision)
+      ocr_entry = vision_supported ? nil : @config.find_model_by_type("ocr")
+
       tool_results.each do |tr|
         inject = tr[:image_inject]
         next unless inject
@@ -1109,12 +1125,18 @@ module Clacky
 
         data_url = "data:#{mime_type};base64,#{base64_data}"
         label = path ? File.basename(path.to_s) : "image"
-        image_block = { type: "image_url", image_url: { url: data_url } }
-        image_block[:image_path] = path if path
-        image_content = [
-          { type: "text", text: "[Image: #{label}]" },
-          image_block
-        ]
+
+        image_content =
+          if vision_supported
+            image_block = { type: "image_url", image_url: { url: data_url } }
+            image_block[:image_path] = path if path
+            [{ type: "text", text: "[Image: #{label}]" }, image_block]
+          else
+            ocr_result = try_ocr(ocr_entry, data_url: data_url, name: label)
+            text = ocr_text_for_inject(label, ocr_result, ocr_entry)
+            [{ type: "text", text: text }]
+          end
+
         @history.append({
           role:             "user",
           content:          image_content,
@@ -1494,6 +1516,11 @@ module Clacky
       # the current model (no stale state on `/model` switch).
       vision_supported = @config.current_model_supports?(:vision)
 
+      # OCR sidecar — only consulted when the primary doesn't see images.
+      # When the sidecar entry has "primary"=>true, the primary itself can see,
+      # so vision_supported was already true and we never enter the OCR branch.
+      ocr_entry = vision_supported ? nil : @config.find_model_by_type("ocr")
+
       vision_images = []  # Array of { url:, name:, size_bytes:, path: }
       downgraded    = []
 
@@ -1510,8 +1537,11 @@ module Clacky
           file_ref  = Utils::FileProcessor.save_image_to_disk(body: raw, mime_type: mime, filename: name)
           reason    = downgrade_reason_for(vision_supported, byte_size, max_bytes)
           if reason
-            downgraded << { name: name, path: file_ref.original_path, type: "image",
-                            mime_type: mime, size_bytes: byte_size, downgrade_reason: reason }
+            ocr_result = (reason == :provider_no_vision) ? try_ocr(ocr_entry, data_url: data_url, name: name) : nil
+            entry = { name: name, path: file_ref.original_path, type: "image",
+                      mime_type: mime, size_bytes: byte_size, downgrade_reason: reason }
+            apply_ocr_outcome!(entry, ocr_result)
+            downgraded << entry
           else
             vision_images << { url: data_url, name: name, size_bytes: byte_size, path: file_ref.original_path }
           end
@@ -1522,8 +1552,11 @@ module Clacky
             byte_size = (b64_data.bytesize * 3) / 4
             reason    = downgrade_reason_for(vision_supported, byte_size, max_bytes)
             if reason
-              downgraded << { name: name, path: path, type: "image",
-                              mime_type: mime, size_bytes: byte_size, downgrade_reason: reason }
+              ocr_result = (reason == :provider_no_vision) ? try_ocr(ocr_entry, path: path, name: name) : nil
+              entry = { name: name, path: path, type: "image",
+                        mime_type: mime, size_bytes: byte_size, downgrade_reason: reason }
+              apply_ocr_outcome!(entry, ocr_result)
+              downgraded << entry
             else
               vision_images << { url: data_url_from_path, name: name, size_bytes: byte_size, path: path }
             end
@@ -1536,6 +1569,30 @@ module Clacky
       [vision_images, downgraded]
     end
 
+    # Best-effort OCR through the configured sidecar. Returns nil when no
+    # sidecar is configured or the call failed — caller falls back to the
+    # ":provider_no_vision" downgrade note (today's behaviour).
+    # @return [Clacky::Vision::Resolver::Result, nil]
+    #   nil — no sidecar exists or sidecar IS the primary (no point extra hop).
+    #         Caller treats this as ":provider_no_vision" (configure a sidecar).
+    #   Result — outcome from the sidecar call. status=:ok carries text;
+    #            :empty / :call_failed / :bad_image each get their own message
+    #            so the user can tell "image content unreadable" from
+    #            "sidecar misconfigured / down".
+    private def try_ocr(ocr_entry, data_url: nil, path: nil, name: nil)
+      return nil unless ocr_entry
+      return nil if ocr_entry["primary"]
+
+      image = data_url ? { data_url: data_url } : { path: path }
+
+      @ui&.show_progress("OCR...", progress_type: "thinking", phase: "active")
+      begin
+        Clacky::Vision::Resolver.new(ocr_entry).describe(image)
+      ensure
+        @ui&.show_progress(phase: "done")
+      end
+    end
+
     # Decide whether an image must be downgraded to a disk ref, and if so why.
     # Precedence: provider capability is checked first — a text-only model
     # can't use the image at any size, so there's no point re-checking size.
@@ -1554,9 +1611,61 @@ module Clacky
     private def downgrade_note_for(reason)
       case reason&.to_sym
       when :provider_no_vision
-        "The current model does not support vision input. Image content is not visible to the model; suggest switching to a vision-capable model if the user needs image analysis."
+        "The current model does not support vision input and no OCR sidecar is configured. Tell the user clearly that to analyze this image they need to either: (1) configure an OCR sidecar model in Settings → Media → OCR (any vision-capable model works as the sidecar — e.g. gemini-3-5-flash, gpt-4o-mini, claude-3-5-haiku), or (2) switch the current model to a vision-capable one. Do not attempt to guess the image content."
       when :too_large
         "Image was too large for inline delivery and has been saved to disk. Read it with a vision-capable tool/model if needed."
+      when :ocr_resolved
+        "The current model does not support vision input. The image has been transcribed by an OCR sidecar model — the description below is what the model sees in place of the raw pixels."
+      when :ocr_call_failed
+        "The current model does not support vision and the configured OCR sidecar call failed. Tell the user the sidecar (Settings → Media → OCR) errored — likely a misconfigured base_url / api_key, or the upstream is down. They can retry, fix the sidecar config, or switch to a vision-capable primary model. Do not guess the image content."
+      when :ocr_empty
+        "The current model does not support vision. The OCR sidecar responded but returned no readable text (the model produced no description — possibly the image is blank, or the model exhausted its token budget on internal reasoning). Tell the user honestly; do not guess the image content."
+      when :ocr_bad_image
+        "The current model does not support vision. The OCR sidecar could not read the image bytes (corrupt or unsupported format). Tell the user; do not guess the image content."
+      end
+    end
+
+    # Mutates `entry` in place based on the OCR Result outcome.
+    # Sets `:ocr_text` (only on :ok) and rewrites `:downgrade_reason` to one
+    # of :ocr_resolved / :ocr_call_failed / :ocr_empty / :ocr_bad_image.
+    # When ocr_result is nil (no sidecar configured) leaves the original
+    # :provider_no_vision reason untouched.
+    private def apply_ocr_outcome!(entry, ocr_result)
+      return entry unless ocr_result
+
+      case ocr_result.status
+      when :ok
+        entry[:ocr_text] = ocr_result.text
+        entry[:downgrade_reason] = :ocr_resolved
+      when :empty
+        entry[:downgrade_reason] = :ocr_empty
+      when :call_failed
+        entry[:downgrade_reason] = :ocr_call_failed
+        entry[:ocr_error] = ocr_result.error
+      when :bad_image
+        entry[:downgrade_reason] = :ocr_bad_image
+      end
+      entry
+    end
+
+    # Build the inline text block used by the image_inject path (tool screenshots,
+    # generated images, etc. that arrive as content blocks rather than as
+    # display_files entries).
+    private def ocr_text_for_inject(label, ocr_result, ocr_entry)
+      header = "[Image: #{label}]"
+      if ocr_result.nil?
+        return "#{header} The current model has no vision and no OCR sidecar is configured. Tell the user to either configure an OCR sidecar in Settings → Media → OCR, or switch to a vision-capable model, then retry. Do not guess the image content."
+      end
+
+      case ocr_result.status
+      when :ok
+        "#{header}\nOCR description (the current model cannot see images directly; this transcription was produced by sidecar #{ocr_entry["model"]}):\n#{ocr_result.text.strip}"
+      when :empty
+        "#{header} The OCR sidecar (#{ocr_entry["model"]}) returned no readable text. The image may be blank, or the sidecar exhausted its token budget on internal reasoning. Tell the user honestly; do not guess the image content."
+      when :call_failed
+        "#{header} The OCR sidecar (#{ocr_entry["model"]}) call failed: #{ocr_result.error}. Tell the user the sidecar errored (likely a misconfigured base_url / api_key in Settings → Media → OCR, or the upstream is down). They can retry, fix the sidecar, or switch to a vision-capable primary model. Do not guess the image content."
+      when :bad_image
+        "#{header} The OCR sidecar could not read the image bytes (corrupt or unsupported format). Tell the user; do not guess the image content."
       end
     end
 

data/lib/clacky/agent_config.rb

@@ -606,12 +606,16 @@ module Clacky
       }.compact
     end
 
-    # Find model by type (default or lite or media kind)
+    # Find model by type (default or lite or media kind or ocr sidecar)
     # Returns the model hash or nil if not found.
     # For media kinds (image/video/audio): explicit user-configured (custom)
     # entries win; otherwise an auto-derived virtual entry is returned
     # based on the default model's provider — mirroring how lite is
     # virtually derived via #lite_model_config_for_current.
+    # For "ocr": same custom→auto→nil pattern. Auto path first checks
+    # whether the default model itself supports vision (zero-overhead path,
+    # no sidecar needed); if not, derives from the provider's
+    # default_ocr_model.
     def find_model_by_type(type)
       kind = type.to_s
       if Clacky::Providers::MEDIA_KINDS.include?(kind)
@@ -622,16 +626,24 @@ module Clacky
         end
         return derive_media_model(kind, model_override: entry && entry["model"])
       end
+      if kind == "ocr"
+        entry = @models.find { |m| m["type"] == "ocr" }
+        return nil if entry && entry["disabled"]
+        if entry && entry["base_url"].to_s.strip != "" && entry["api_key"].to_s.strip != ""
+          return entry
+        end
+        return derive_ocr_model(model_override: entry && entry["model"])
+      end
       @models.find { |m| m["type"] == type }
     end
 
     private def derive_media_model(kind, model_override: nil)
-      default = find_model_by_type("default")
-      return nil unless default
+      anchor = current_model || find_model_by_type("default")
+      return nil unless anchor
 
       provider_id = Clacky::Providers.resolve_provider(
-        base_url: default["base_url"],
-        api_key:  default["api_key"]
+        base_url: anchor["base_url"],
+        api_key:  anchor["api_key"]
       )
       return nil unless provider_id
 
@@ -649,8 +661,8 @@ module Clacky
 
       {
         "model"         => model_name,
-        "base_url"      => default["base_url"],
-        "api_key"       => default["api_key"],
+        "base_url"      => anchor["base_url"],
+        "api_key"       => anchor["api_key"],
         "type"          => kind,
         "auto_injected" => true
       }
@@ -662,6 +674,54 @@ module Clacky
       @models.reject! { |m| m["auto_injected"] && Clacky::Providers::MEDIA_KINDS.include?(m["type"].to_s) }
     end
 
+    # Derive an OCR sidecar model entry from the default model's provider.
+    # Resolution order:
+    #   1. If the default model itself supports vision → return the default
+    #      directly (zero-overhead path; no separate sidecar call needed).
+    #   2. Otherwise look up the provider's default_ocr_model (or honour
+    #      model_override if it's a vision-capable model on that provider).
+    #   3. nil when the provider has no vision-capable lineup at all
+    #      (e.g. DeepSeek V4) — caller falls back to today's "no vision" UX.
+    private def derive_ocr_model(model_override: nil)
+      # Anchor on the model the session is *actually* running on, not the
+      # yml `type: default` marker — those diverge whenever the user
+      # switches model mid-session (e.g. opus → deepseek).
+      anchor = current_model || find_model_by_type("default")
+      return nil unless anchor
+
+      provider_id = Clacky::Providers.resolve_provider(
+        base_url: anchor["base_url"], api_key: anchor["api_key"]
+      )
+      return nil unless provider_id
+
+      if Clacky::Providers.supports?(provider_id, :vision, model_name: anchor["model"])
+        return {
+          "model"         => anchor["model"],
+          "base_url"      => anchor["base_url"],
+          "api_key"       => anchor["api_key"],
+          "type"          => "ocr",
+          "auto_injected" => true,
+          "primary"       => true
+        }
+      end
+
+      candidates = Clacky::Providers.ocr_models(provider_id)
+      model_name = if model_override && candidates.include?(model_override)
+                     model_override
+                   else
+                     Clacky::Providers.default_ocr_model(provider_id)
+                   end
+      return nil if model_name.nil? || model_name.to_s.empty?
+
+      {
+        "model"         => model_name,
+        "base_url"      => anchor["base_url"],
+        "api_key"       => anchor["api_key"],
+        "type"          => "ocr",
+        "auto_injected" => true
+      }
+    end
+
     # Returns the configured/derived media model entry for `kind`, plus a
     # hint about its source. UI uses this to render the tri-state control.
     # @param kind [String] one of "image" / "video" / "audio"
@@ -738,6 +798,63 @@ module Clacky
       }
     end
 
+    # Tri-state introspection for the OCR sidecar — mirrors #media_state shape
+    # so the Settings UI can reuse the same row component.
+    # @return [Hash{String=>Object}] keys:
+    #   "configured" — anything available (auto or custom)
+    #   "source"     — "off" | "auto" | "custom"
+    #   "primary"    — true when auto resolves to the default model itself
+    #                  (no sidecar call needed)
+    #   "model"/"base_url"/"provider"/"available"
+    def ocr_state
+      raw_entry = @models.find { |m| m["type"] == "ocr" }
+
+      default = find_model_by_type("default")
+      default_provider = default && Clacky::Providers.resolve_provider(
+        base_url: default["base_url"], api_key: default["api_key"]
+      )
+      available = default_provider ? Clacky::Providers.ocr_models(default_provider) : []
+
+      if raw_entry && raw_entry["disabled"]
+        return {
+          "configured" => false,
+          "source"     => "off",
+          "model"      => nil,
+          "base_url"   => nil,
+          "provider"   => nil,
+          "primary"    => false,
+          "available"  => available
+        }
+      end
+
+      is_custom = raw_entry &&
+                  raw_entry["base_url"].to_s.strip != "" &&
+                  raw_entry["api_key"].to_s.strip != ""
+      override_model = raw_entry && !is_custom ? raw_entry["model"] : nil
+
+      entry = if is_custom
+                raw_entry
+              else
+                derive_ocr_model(model_override: override_model)
+              end
+
+      provider_id = if entry
+                      Clacky::Providers.resolve_provider(
+                        base_url: entry["base_url"], api_key: entry["api_key"]
+                      )
+                    end
+
+      {
+        "configured" => !entry.nil?,
+        "source"     => is_custom ? "custom" : (entry ? "auto" : "off"),
+        "model"      => entry && entry["model"],
+        "base_url"   => entry && entry["base_url"],
+        "provider"   => provider_id,
+        "primary"    => !!(entry && entry["primary"]),
+        "available"  => available
+      }
+    end
+
     # Find model by composite key (model name + base_url).
     # Used when restoring a session to match its original model without relying
     # on the runtime-only id (which changes on every process restart).
@@ -1050,7 +1167,7 @@ module Clacky
     # Returns true if successful
     def set_model_type(index, type)
       return false if index < 0 || index >= @models.length
-      return false unless ["default", "lite", "image", "video", "audio", nil].include?(type)
+      return false unless ["default", "lite", "image", "video", "audio", "ocr", nil].include?(type)
 
       if type
         # Remove type from any other model that has it

data/lib/clacky/client.rb

@@ -398,7 +398,17 @@ module Clacky
     def parse_simple_openai_response(response)
       raise_error(response) unless response.status == 200
       parsed_body = safe_json_parse(response.body, context: "LLM response")
-      parsed_body["choices"].first["message"]["content"]
+      content = parsed_body.dig("choices", 0, "message", "content")
+      if content.nil?
+        snippet = response.body.to_s[0, 1200]
+        if defined?(Clacky::Logger)
+          Clacky::Logger.warn("[parse_simple_openai_response] no content. status=#{response.status} body=#{snippet}")
+        end
+        raise Clacky::Error,
+          "Upstream OpenAI-compatible response missing choices[0].message.content. " \
+          "Body snippet: #{snippet}"
+      end
+      content
     end
 
     # ── Prompt caching helpers ────────────────────────────────────────────────