openclacky 0.9.35 → 0.9.37

data/lib/clacky/server/http_server.rb

@@ -468,10 +468,19 @@ module Clacky
         # Backward-compat: ?source=<x> and ?profile=coding → type
         type ||= query["profile"].to_s.strip.then { |v| v.empty? ? nil : v }
         type ||= query["source"].to_s.strip.then  { |v| v.empty? ? nil : v }
-        # Fetch one extra to detect has_more without a separate count query
+
+        # Fetch one extra NON-PINNED row to detect has_more without a separate count query.
+        # `registry.list` always returns ALL matching pinned rows first (on the
+        # first page; `before` == nil), followed by non-pinned rows up to `limit+1`.
+        # So has_more is determined by whether the non-pinned section overflowed.
         sessions = @registry.list(limit: limit + 1, before: before, q: q, date: date, type: type)
-        has_more = sessions.size > limit
-        sessions = sessions.first(limit)
+
+        # Split pinned vs non-pinned to apply has_more only to the non-pinned tail.
+        pinned_part, non_pinned_part = sessions.partition { |s| s[:pinned] }
+        has_more = non_pinned_part.size > limit
+        non_pinned_part = non_pinned_part.first(limit)
+        sessions = pinned_part + non_pinned_part
+
         json_response(res, 200, { sessions: sessions, has_more: has_more })
       end
 
@@ -606,6 +615,52 @@ module Clacky
 
       # ── Brand API ─────────────────────────────────────────────────────────────
 
+      # Process-wide mutex guarding heartbeat trigger state.
+      # Used by #trigger_async_heartbeat! to ensure only one heartbeat Thread is
+      # in flight at a time, no matter how many concurrent /api/brand/status
+      # requests arrive from the Web UI poller.
+      BRAND_HEARTBEAT_MUTEX   = Mutex.new
+      # Tracks whether a heartbeat Thread is currently running.
+      @@brand_heartbeat_inflight = false
+
+      # Fire a heartbeat in a background Thread without blocking the caller.
+      #
+      # Contract:
+      #   * Only one heartbeat Thread may be running at any moment across the
+      #     whole process. If one is already in flight, this call is a no-op.
+      #   * The caller never waits: it returns immediately after (at most)
+      #     spawning the Thread.
+      #   * The Thread rescues everything so a network failure cannot kill the
+      #     server or leak an exception through the web stack.
+      def trigger_async_heartbeat!
+        BRAND_HEARTBEAT_MUTEX.synchronize do
+          if @@brand_heartbeat_inflight
+            Clacky::Logger.debug("[Brand] heartbeat already in flight, skipping")
+            return
+          end
+          @@brand_heartbeat_inflight = true
+        end
+
+        Thread.new do
+          Clacky::Logger.info("[Brand] async heartbeat starting...")
+          begin
+            brand  = Clacky::BrandConfig.load
+            result = brand.heartbeat!
+            if result[:success]
+              Clacky::Logger.info("[Brand] async heartbeat OK")
+            else
+              Clacky::Logger.warn("[Brand] async heartbeat failed — #{result[:message]}")
+            end
+          rescue StandardError => e
+            Clacky::Logger.warn("[Brand] async heartbeat raised: #{e.class}: #{e.message}")
+          ensure
+            BRAND_HEARTBEAT_MUTEX.synchronize do
+              @@brand_heartbeat_inflight = false
+            end
+          end
+        end
+      end
+
       # GET /api/brand/status
       # Returns whether brand activation is needed.
       # Mirrors the onboard/status pattern so the frontend can gate on it.
@@ -634,17 +689,15 @@ module Clacky
           return
         end
 
-        # Send heartbeat if interval has elapsed (once per day)
+        # Send heartbeat asynchronously if interval has elapsed (once per day).
+        #
+        # We must NOT block this HTTP response on the heartbeat call: a slow or
+        # unreachable license server would otherwise stall the Web UI's first
+        # paint for up to ~92s (2 hosts × 2 attempts × 23s timeout). The fresh
+        # expires_at / last_heartbeat will be picked up on the next /api/brand/status
+        # poll, which is sufficient for a once-per-day check.
         if brand.heartbeat_due?
-          Clacky::Logger.info("[Brand] api_brand_status: heartbeat due, sending...")
-          result = brand.heartbeat!
-          if result[:success]
-            Clacky::Logger.info("[Brand] api_brand_status: heartbeat OK")
-          else
-            Clacky::Logger.warn("[Brand] api_brand_status: heartbeat failed — #{result[:message]}")
-          end
-          # Reload after heartbeat to pick up updated expires_at / last_heartbeat
-          brand = Clacky::BrandConfig.load
+          trigger_async_heartbeat!
         else
           Clacky::Logger.debug("[Brand] api_brand_status: heartbeat not due yet")
         end
@@ -886,8 +939,12 @@ module Clacky
         key.empty? ? nil : key
       end
 
-      # Extract bearer token / query param / cookie from a WEBrick request.
-      # Priority: Authorization: Bearer > ?access_key= > Cookie clacky_access_key
+      # Extract bearer token or query param from a WEBrick request.
+      # Priority: Authorization: Bearer > ?access_key=
+      # The query string form is only used by WebSocket connections, which
+      # cannot set custom headers from the browser. All HTTP clients —
+      # including the web UI (via a fetch interceptor in auth.js) — use the
+      # Authorization header.
       private def extract_key(req)
         auth = req["Authorization"].to_s.strip
         if auth.start_with?("Bearer ")
@@ -899,10 +956,6 @@ module Clacky
         token = query["access_key"].to_s.strip
         return token unless token.empty?
 
-        req.cookies.each do |c|
-          return c.value if c.name == "clacky_access_key" && !c.value.to_s.empty?
-        end
-
         nil
       end
 
@@ -1075,7 +1128,18 @@ module Clacky
       end
 
       # Broadcast final upgrade result with appropriate log message.
+      #
+      # Defensive post-check: if `run_shell` reported failure but the gem
+      # is in fact now installed at the latest version, reverse the verdict.
+      # This guards against false negatives from the Terminal idle-poll
+      # mechanism (see: 0.9.36 upgrade failure bug).
       private def finish_upgrade(success, fallback_hint: "gem update openclacky")
+        if !success && gem_actually_upgraded?
+          Clacky::Logger.warn("[Upgrade] run_shell reported failure, but installed version matches latest — treating as success.")
+          broadcast_all(type: "upgrade_log", line: "\n(Verified: the new version is installed — reclassifying as success.)\n")
+          success = true
+        end
+
         if success
           Clacky::Logger.info("[Upgrade] Success!")
           broadcast_all(type: "upgrade_log", line: "\n✓ Upgrade successful! Please restart the server to apply the new version.\n")
@@ -1087,6 +1151,22 @@ module Clacky
         end
       end
 
+      # Check whether the latest published version of openclacky is already
+      # installed locally. Used as a post-upgrade sanity check so a flaky
+      # run_shell result doesn't mask a successful install.
+      # Returns false on any error (conservative — don't fabricate success).
+      private def gem_actually_upgraded?
+        latest = fetch_latest_version_from_rubygems_api
+        return false unless latest
+
+        out, exit_code = run_shell("gem list openclacky -i -v #{latest}", timeout: 30)
+        return false unless exit_code&.zero?
+        out.to_s.strip.downcase == "true"
+      rescue StandardError => e
+        Clacky::Logger.warn("[Upgrade] gem_actually_upgraded? error: #{e.message}")
+        false
+      end
+
       # POST /api/restart
       # Re-execs the current process so the newly installed gem version is loaded.
       # Uses the absolute script path captured at startup to avoid relative-path issues.
@@ -1198,18 +1278,11 @@ module Clacky
       # Run a shell command via the unified Terminal tool and return
       # [output, exit_code] — drop-in replacement for Open3.capture2e.
       #
-      # Uses Terminal#execute so the command inherits the user's real
-      # login shell (rbenv/mise shims, configured gem mirrors, etc.).
-      # On timeout / still-running, returns [output_so_far, nil].
-      #
-      # The command is routed through the Security layer like any other
-      # Terminal call; server-side commands (`gem ...`, `curl -fsSL ... -o ...`)
-      # pass through unchanged.
+      # Delegates to Terminal.run_sync which handles the idle-poll loop
+      # internally (see its docs for why that's needed — this wrapper used
+      # to re-implement it wrong and caused the 0.9.36 upgrade bug).
       private def run_shell(command, timeout: 120)
-        result = Clacky::Tools::Terminal.new.execute(command: command, timeout: timeout)
-        output    = result[:output].to_s
-        exit_code = result[:exit_code] # nil when the session is still running
-        [output, exit_code]
+        Clacky::Tools::Terminal.run_sync(command, timeout: timeout)
       end
 
       # ── Channel API ───────────────────────────────────────────────────────────
@@ -1828,6 +1901,7 @@ module Clacky
       def api_get_config(res)
         models = @agent_config.models.map.with_index do |m, i|
           {
+            id:               m["id"],   # Stable runtime id — use this for switching
             index:            i,
             model:            m["model"],
             base_url:         m["base_url"],
@@ -1838,12 +1912,19 @@ module Clacky
         end
         # Filter out auto-injected models (like lite) from UI display
         models.reject! { |m| @agent_config.models[m[:index]]["auto_injected"] }
-        json_response(res, 200, { models: models, current_index: @agent_config.current_model_index })
+        json_response(res, 200, {
+          models: models,
+          current_index: @agent_config.current_model_index,
+          current_id: @agent_config.current_model&.dig("id")
+        })
       end
 
       # POST /api/config — save updated model list
-      # Body: { models: [ { index, model, base_url, api_key, anthropic_format, type } ] }
-      # api_key may be masked ("sk-ab12****...5678") — keep existing key in that case
+      # Body: { models: [ { id?, index, model, base_url, api_key, anthropic_format, type } ] }
+      # - id may be present for existing models (preserved) or absent for newly added
+      #   models (a new id is generated). Ids are runtime-only and stripped before
+      #   writing to config.yml (see AgentConfig#to_yaml).
+      # - api_key may be masked ("sk-ab12****...5678") — keep existing key in that case
       def api_save_config(req, res)
         body = parse_json_body(req)
         return json_response(res, 400, { error: "Invalid JSON" }) unless body
@@ -1851,35 +1932,59 @@ module Clacky
         incoming = body["models"]
         return json_response(res, 400, { error: "models array required" }) unless incoming.is_a?(Array)
 
-        incoming.each_with_index do |m, i|
-          existing = @agent_config.models[i]
-          # Resolve api_key: if masked placeholder, keep the stored key
-          api_key = if m["api_key"].to_s.include?("****")
-                      existing&.dig("api_key")
+        # Build a quick id→existing-model lookup. Ids are the single source
+        # of identity for models across save/reload cycles — no index-based
+        # fallback (ids are stable; indexes are not). Live sessions' stored
+        # @current_model_id stays valid as long as the id is still present
+        # in the list after save.
+        existing_by_id = {}
+        @agent_config.models.each { |m| existing_by_id[m["id"]] = m if m["id"] }
+
+        new_models = incoming.map do |m|
+          # Lookup by id only. No id means a brand-new model — we mint one.
+          existing = m["id"] && existing_by_id[m["id"]]
+
+          # Resolve api_key with THREE cases (ordered, fail-safe):
+          #   1. Incoming key is the masked placeholder ("sk-ab12****...5678")
+          #      → user didn't retype; keep the stored key.
+          #   2. Incoming key is missing/blank AND we have an existing model
+          #      → the browser omitted api_key for non-edited rows; keep the
+          #        stored key. This is the critical 0.9.37 fix — without it,
+          #        saving one model silently wiped api_keys of all others,
+          #        because the frontend only ever hydrates api_key for the
+          #        row currently being edited (/api/config returns only
+          #        api_key_masked, never api_key).
+          #   3. Otherwise: user typed a new key (or this is a brand-new
+          #      model); use the incoming value.
+          incoming_key = m["api_key"].to_s
+          api_key = if incoming_key.include?("****")
+                      existing&.dig("api_key").to_s
+                    elsif incoming_key.empty? && existing
+                      existing["api_key"].to_s
                     else
-                      m["api_key"]
+                      incoming_key
                     end
 
-          if existing
-            existing["model"]            = m["model"]            if m.key?("model")
-            existing["base_url"]         = m["base_url"]         if m.key?("base_url")
-            existing["api_key"]          = api_key               if api_key
-            existing["anthropic_format"] = m["anthropic_format"] if m.key?("anthropic_format")
-            existing["type"]             = m["type"]             if m.key?("type")
-          else
-            @agent_config.add_model(
-              model:            m["model"].to_s,
-              api_key:          api_key.to_s,
-              base_url:         m["base_url"].to_s,
-              anthropic_format: m["anthropic_format"] || false,
-              type:             m["type"]
-            )
-          end
+          {
+            "id"               => (existing && existing["id"]) || SecureRandom.uuid,
+            "model"            => m["model"].to_s,
+            "base_url"         => m["base_url"].to_s,
+            "api_key"          => api_key,
+            "anthropic_format" => m["anthropic_format"] || false,
+            "type"             => m["type"]
+          }.tap { |h| h.delete("type") if h["type"].nil? || h["type"].to_s.empty? }
         end
 
-        # Remove models that are no longer present (trim to incoming length)
-        while @agent_config.models.length > incoming.length
-          @agent_config.models.pop
+        # Replace @models in place — do NOT reassign the array, because every
+        # live session holds a reference to the same array (Plan B shared
+        # models). `replace` mutates in place so all sessions see the new list.
+        @agent_config.models.replace(new_models)
+
+        # Re-anchor current_model_index to the model still holding type: default
+        if (new_default_idx = new_models.find_index { |m| m["type"] == "default" })
+          @agent_config.current_model_index = new_default_idx
+        elsif @agent_config.current_model_index >= new_models.length
+          @agent_config.current_model_index = [new_models.length - 1, 0].max
         end
 
         @agent_config.save
@@ -2004,36 +2109,38 @@ module Clacky
 
       def api_switch_session_model(session_id, req, res)
         body = parse_json_body(req)
-        new_model_name = body["model"].to_s.strip
+        model_id = body["model_id"].to_s.strip
 
-        return json_response(res, 400, { error: "model is required" }) if new_model_name.empty?
+        return json_response(res, 400, { error: "model_id is required" }) if model_id.empty?
         return json_response(res, 404, { error: "Session not found" }) unless @registry.ensure(session_id)
 
         agent = nil
         @registry.with_session(session_id) { |s| agent = s[:agent] }
-        
-        # Find the model configuration index by model name (use global config)
-        model_index = @agent_config.models.find_index { |m| m["model"] == new_model_name }
-        
-        if model_index.nil?
-          return json_response(res, 400, { error: "Model '#{new_model_name}' not found in configuration" })
+
+        # With Plan B (shared @models reference), every session's AgentConfig
+        # points at the same @models array as the global @agent_config. So
+        # resolving the model by stable id here and in agent.switch_model_by_id
+        # will always agree — no more index divergence after add/delete.
+        target_model = @agent_config.models.find { |m| m["id"] == model_id }
+        if target_model.nil?
+          return json_response(res, 400, { error: "Model not found in configuration" })
         end
-        
-        # Switch to the model by index (unified interface with CLI)
-        # This handles: config.switch_model + client rebuild + message_compressor rebuild
-        success = agent.switch_model(model_index)
-        
+
+        # Switch to the model by id (unified interface with CLI)
+        # Handles: config.switch_model_by_id + client rebuild + message_compressor rebuild
+        success = agent.switch_model_by_id(model_id)
+
         unless success
           return json_response(res, 500, { error: "Failed to switch model" })
         end
-        
+
         # Persist the change (saves to session file, NOT global config.yml)
         @session_manager.save(agent.to_session_data)
-        
+
         # Broadcast update to all clients
         broadcast_session_update(session_id)
-        
-        json_response(res, 200, { ok: true, model: new_model_name })
+
+        json_response(res, 200, { ok: true, model_id: model_id, model: target_model["model"] })
       rescue => e
         json_response(res, 500, { error: e.message })
       end
@@ -2071,16 +2178,34 @@ module Clacky
       end
 
       def api_delete_session(session_id, res)
-        if @registry.delete(session_id)
-          # Also remove the persisted session file from disk
-          @session_manager.delete(session_id)
-          # Notify connected clients the session is gone
-          broadcast(session_id, { type: "session_deleted", session_id: session_id })
-          unsubscribe_all(session_id)
-          json_response(res, 200, { ok: true })
-        else
-          json_response(res, 404, { error: "Session not found" })
+        # A session exists if it's either in the runtime registry OR on disk.
+        # Old sessions that were never restored into memory this server run
+        # (e.g. shown via "load more" in the WebUI list) are disk-only — we
+        # must still be able to delete them. Previously this endpoint only
+        # consulted @registry and returned 404 for disk-only sessions,
+        # causing the "can't delete old sessions" bug.
+        in_registry = @registry.exist?(session_id)
+        on_disk     = !@session_manager.load(session_id).nil?
+
+        unless in_registry || on_disk
+          return json_response(res, 404, { error: "Session not found" })
         end
+
+        # Registry delete is best-effort — only meaningful when the session
+        # is actually live (cancels idle timer, interrupts the agent thread).
+        # For disk-only sessions this is a no-op and returns false, which is
+        # fine and no longer blocks the disk cleanup below.
+        @registry.delete(session_id) if in_registry
+
+        # Always physically remove the persisted session file (+ chunks).
+        @session_manager.delete(session_id) if on_disk
+
+        # Notify any still-connected clients (mainly matters when the
+        # session was live, but harmless otherwise).
+        broadcast(session_id, { type: "session_deleted", session_id: session_id })
+        unsubscribe_all(session_id)
+
+        json_response(res, 200, { ok: true })
       end
 
       # Export a session bundle as a .zip download containing:

data/lib/clacky/server/session_registry.rb

@@ -143,11 +143,22 @@ module Clacky
       #            nil = no source filter (all sessions)
       #   profile: "general"|"coding"|nil
       #            nil = no agent_profile filter
-      #   limit:   max sessions to return
+      #   limit:   max sessions to return (applies to NON-PINNED only; see below)
       #   before:  ISO8601 cursor — only sessions with created_at < before
+      #             (also applies to NON-PINNED only; pinned items are a separate
+      #             logical section, they should never be paginated away)
+      #   include_pinned: when true (default), all matching pinned sessions are
+      #             always returned on the FIRST page (before == nil) regardless
+      #             of limit. Subsequent pages (before set) contain only
+      #             non-pinned sessions. This guarantees that users who pinned
+      #             an old session always see it at the top of the sidebar,
+      #             even if many newer sessions exist.
+      #
+      # Ordering of the returned array:
+      #   [ ...all_pinned_matching (newest-first), ...non_pinned (newest-first, limited) ]
       #
       # source and profile are orthogonal — either can be nil independently.
-      def list(limit: nil, before: nil, q: nil, date: nil, type: nil)
+      def list(limit: nil, before: nil, q: nil, date: nil, type: nil, include_pinned: true)
         return [] unless @session_manager
 
         live = @mutex.synchronize do
@@ -185,10 +196,26 @@ module Clacky
           }
         end
 
-        all = all.select { |s| (s[:created_at] || "") < before } if before
-        all = all.first(limit) if limit
+        # ── Split pinned vs non-pinned BEFORE applying `before`/`limit`.
+        # Pinned sessions bypass pagination entirely so an old pinned session
+        # never falls off the first page just because newer sessions exist.
+        # (Regression fix for 0.9.37: previously `all_sessions` was only
+        # sorted by created_at and `limit` cut off old pinned rows, making
+        # them invisible until the user clicked "load more".)
+        pinned, non_pinned = all.partition { |s| s[:pinned] }
+
+        # `before` cursor ONLY applies to non-pinned (paginated) sessions.
+        non_pinned = non_pinned.select { |s| (s[:created_at] || "") < before } if before
+        non_pinned = non_pinned.first(limit) if limit
+
+        # Pinned section: only included on the first page (before == nil) so
+        # "load more" responses don't re-send them. On first page, return ALL
+        # matching pinned sessions regardless of limit.
+        pinned_section = (include_pinned && before.nil?) ? pinned : []
+
+        ordered = pinned_section + non_pinned
 
-        all.map do |s|
+        ordered.map do |s|
           id = s[:session_id]
           ls = live[id]
           {

data/lib/clacky/tools/edit.rb

@@ -44,7 +44,10 @@ module Clacky
         end
 
         begin
-          content = File.read(path)
+          # Scrub invalid UTF-8 bytes at read time — otherwise editing a file
+          # that contains non-UTF-8 bytes would poison history / error messages
+          # and cause JSON.generate to fail during replay.
+          content = safe_utf8(File.read(path))
 
           # Find matching string using layered strategy (shared with preview)
           match_result = Utils::StringMatcher.find_match(content, old_string)
@@ -127,6 +130,13 @@ module Clacky
         replacements = result[:replacements] || result["replacements"] || 1
         "Modified #{replacements} occurrence#{replacements > 1 ? "s" : ""}"
       end
+
+      # Scrub invalid UTF-8 byte sequences (see file_reader.rb for rationale).
+      private def safe_utf8(str)
+        return str if str.nil?
+        return str if str.encoding == Encoding::UTF_8 && str.valid_encoding?
+        str.encode("UTF-8", invalid: :replace, undef: :replace, replace: "\u{FFFD}")
+      end
     end
   end
 end

data/lib/clacky/tools/file_reader.rb

@@ -86,8 +86,10 @@ module Clacky
             }
           end
 
-          # Read text file with optional line range
-          all_lines = File.readlines(expanded_path)
+          # Read text file with optional line range.
+          # Scrub invalid UTF-8 bytes (e.g. GBK-encoded files) so downstream
+          # JSON.generate / history persistence won't blow up later.
+          all_lines = File.readlines(expanded_path).map! { |line| safe_utf8(line) }
           total_lines = all_lines.size
 
           # Calculate start index (convert 1-indexed to 0-indexed)
@@ -313,7 +315,11 @@ module Clacky
       # List first-level directory contents (files and directories)
       private def list_directory_contents(path)
         begin
-          entries = Dir.entries(path).reject { |entry| entry == "." || entry == ".." }
+          # Scrub entry names — filenames on disk may contain non-UTF-8 bytes
+          # (e.g. GBK/Shift-JIS names on macOS/Linux) which would poison history.
+          entries = Dir.entries(path)
+                       .map { |entry| safe_utf8(entry) }
+                       .reject { |entry| entry == "." || entry == ".." }
 
           # Separate files and directories
           files = []
@@ -353,6 +359,16 @@ module Clacky
           }
         end
       end
+
+      # Scrub invalid UTF-8 byte sequences so the result survives
+      # JSON.generate (session replay, API responses).
+      # Invalid bytes are replaced with U+FFFD (�). Valid UTF-8 is
+      # returned untouched via the fast path.
+      private def safe_utf8(str)
+        return str if str.nil?
+        return str if str.encoding == Encoding::UTF_8 && str.valid_encoding?
+        str.encode("UTF-8", invalid: :replace, undef: :replace, replace: "\u{FFFD}")
+      end
     end
   end
 end

data/lib/clacky/tools/glob.rb

@@ -84,6 +84,7 @@ module Clacky
           always_ignored_dirs = Clacky::Utils::FileIgnoreHelper::ALWAYS_IGNORED_DIRS
 
           all_matches = Dir.glob(full_pattern, File::FNM_DOTMATCH)
+                           .map { |p| p.encoding == Encoding::UTF_8 && p.valid_encoding? ? p : p.encode("UTF-8", invalid: :replace, undef: :replace, replace: "\u{FFFD}") }
                            .reject { |path| File.directory?(path) }
                            .reject { |path| path.end_with?(".", "..") }
                            .reject do |path|

data/lib/clacky/tools/grep.rb

@@ -271,8 +271,10 @@ module Clacky
       def search_file(file, regex, context_lines, max_matches)
         matches = []
 
-        # Use File.foreach for memory-efficient line-by-line reading
-        File.foreach(file, chomp: true).with_index do |line, index|
+        # Use File.foreach for memory-efficient line-by-line reading.
+        # Scrub invalid UTF-8 bytes so results survive JSON encoding.
+        File.foreach(file, chomp: true).with_index do |raw_line, index|
+          line = safe_utf8(raw_line)
           # Stop if we have enough matches for this file
           break if matches.length >= max_matches
 
@@ -302,7 +304,7 @@ module Clacky
 
       # Get context lines around a match
       def get_line_context(file, match_index, context_lines)
-        lines = File.readlines(file, chomp: true)
+        lines = File.readlines(file, chomp: true).map! { |l| safe_utf8(l) }
         start_line = [0, match_index - context_lines].max
         end_line = [lines.length - 1, match_index + context_lines].min
 
@@ -325,6 +327,13 @@ module Clacky
       rescue StandardError
         nil
       end
+
+      # Scrub invalid UTF-8 byte sequences (see file_reader.rb for rationale).
+      private def safe_utf8(str)
+        return str if str.nil?
+        return str if str.encoding == Encoding::UTF_8 && str.valid_encoding?
+        str.encode("UTF-8", invalid: :replace, undef: :replace, replace: "\u{FFFD}")
+      end
     end
   end
 end