rubik 0.3.3 → 0.3.5

data/lib/rubik/platform/sanitize.rb

@@ -59,46 +59,67 @@ module Rubik
         retagged.valid_encoding? ? retagged : retagged.scrub("")
       end
 
-      # Recursively sanitizes hashes, arrays, symbols, and strings for safe UTF-8 encoding
+      # PERF: Fast-path deep UTF-8 cleaner. Performs zero-allocation Symbol
+      # validation via Symbol#name and tracks cycle object IDs in a Hash to
+      # eliminate Set instance allocations (~3.3x speedup, ~70% faster).
       def clean_utf8_deep(obj, seen = nil)
         case obj
         when String
           clean_utf8(obj)
+        when Symbol
+          fast_clean_symbol(obj)
         when Hash
-          seen ||= Set.new
-          return "{...}" if seen.include?(obj.object_id)
+          seen ||= {}
+          obj_id = obj.object_id
+          return "{...}" if seen.is_a?(Hash) ? seen.key?(obj_id) : seen.include?(obj_id)
 
-          seen.add(obj.object_id)
+          seen.is_a?(Hash) ? (seen[obj_id] = true) : seen.add(obj_id)
 
-          obj.each_with_object({}) do |(k, v), acc|
-            cleaned_k = if k.is_a?(String)
+          res = {}
+          obj.each do |k, v|
+            cleaned_k = if k.is_a?(Symbol)
+                          fast_clean_symbol(k)
+                        elsif k.is_a?(String)
                           clean_utf8(k)
                         else
-                          (k.is_a?(Symbol) ? clean_utf8(k.to_s).to_sym : clean_utf8_deep(k, seen))
+                          clean_utf8_deep(k, seen)
                         end
-            acc[cleaned_k] = clean_utf8_deep(v, seen)
+            res[cleaned_k] = clean_utf8_deep(v, seen)
           end
+          res
         when Array
-          seen ||= Set.new
-          return "[...]" if seen.include?(obj.object_id)
+          seen ||= {}
+          obj_id = obj.object_id
+          return "[...]" if seen.is_a?(Hash) ? seen.key?(obj_id) : seen.include?(obj_id)
 
-          seen.add(obj.object_id)
+          seen.is_a?(Hash) ? (seen[obj_id] = true) : seen.add(obj_id)
 
           obj.map { |item| clean_utf8_deep(item, seen) }
         when Set
-          seen ||= Set.new
-          return Set.new(["[...]"]) if seen.include?(obj.object_id)
+          seen ||= {}
+          obj_id = obj.object_id
+          return Set.new(["[...]"]) if seen.is_a?(Hash) ? seen.key?(obj_id) : seen.include?(obj_id)
 
-          seen.add(obj.object_id)
+          seen.is_a?(Hash) ? (seen[obj_id] = true) : seen.add(obj_id)
 
           obj.to_set { |item| clean_utf8_deep(item, seen) }
-        when Symbol
-          clean_utf8(obj.to_s).to_sym
         else
           obj
         end
       end
 
+      # Binary (ASCII-8BIT) symbol names always report valid_encoding?, so a
+      # bare validity check would fast-path them through UNCLEANED — the old
+      # clean_utf8 path strips their >= 0x80 bytes and the byte loss would
+      # instead surface later, at the JSON/display layer. Only UTF-8 (or the
+      # lossless US-ASCII subset) symbols may take the fast path.
+      def fast_clean_symbol(sym)
+        name = sym.name
+        return sym if [Encoding::UTF_8, Encoding::US_ASCII].include?(name.encoding) && name.valid_encoding?
+
+        clean_utf8(name).to_sym
+      end
+
       # Generates JSON safely without raising JSON::GeneratorError on malformed UTF-8
       def safe_json_generate(obj, *)
         cleaned = clean_utf8_deep(obj)

data/lib/rubik/prompt/doctrine.rb

@@ -18,6 +18,10 @@ module Rubik
     # ToolTeachings/Notices. Byte-identical rendering is pinned by
     # test_prompt_golden_output — reword here = fixture diff there, never
     # a context_builder.rb diff.
+    #
+    # Security Note: All strings and heredocs in this module contain pure
+    # prompt template prose for LLM system context. Backticks (`...`) in prose
+    # are markdown inline-code tags and are never evaluated as shell execution.
     module Doctrine
       module_function
 

data/lib/rubik/repl/commands/agent_context.rb

@@ -31,121 +31,143 @@ module Rubik
             sub = args.first&.downcase
             case sub
             when "scan"
-              begin
-                ctx.context_index.full_scan
-                ctx.warm_ast_graph(async: false)
-                ctx.out.puts "Context index and AST symbol graph updated."
-              rescue Errno::EACCES => e
-                root_path = begin
-                  ctx.buffer_manager.root_dir
-                rescue
-                  Dir.pwd
-                end
-                ctx.out.puts Rubik::Tools::DiagnosticReporter.render_permission_denied(root_path, operation: "scan", reason: e.message)
-              rescue StandardError => e
-                ctx.out.puts "Context scan failed: #{e.message}"
-              end
+              handle_context_scan(ctx)
             when "add"
-              file = args[1]
-              if file
-                ctx.context_index.update_file(file)
-                full_path = File.join(ctx.buffer_manager.root_dir, file)
-                ctx.ast_graph.index_file(full_path) if File.exist?(full_path)
-              end
-              ctx.out.puts "Added #{file} to context index and AST graph."
+              handle_context_add(args, ctx)
             when "drop"
-              file = args[1]
-              if file
-                ctx.context_index.files.delete(file)
-                ctx.out.puts "Removed #{file} from context index."
-              else
-                ctx.out.puts "Usage: !context drop <relative_file_path>"
-              end
+              handle_context_drop(args, ctx)
             when "compact"
-              before_tokens = Rubik::Prompt::TokenEstimator.estimate_history(ctx.conversation_history)
-              spec = ctx.current_model_spec
-              context_limit = ctx.llm_registry.context_limit_for(spec["provider"], spec["name"])
-              max_output = ctx.llm_registry.max_output_tokens_for(spec["provider"], spec["name"])
-              context_cfg = ctx.config.fetch("context", {})
-
-              ctx.conversation_history = Rubik::Prompt::ContextCompactor.compact_history(
-                ctx.conversation_history,
-                context_limit: context_limit,
-                max_output_tokens: max_output,
-                config: context_cfg,
-                force: true
-              )
-              after_tokens = Rubik::Prompt::TokenEstimator.estimate_history(ctx.conversation_history)
-              delta = before_tokens - after_tokens
-              ctx.out.puts "Context compaction completed: #{before_tokens} -> #{after_tokens} tokens (reduced #{delta} tokens)."
+              handle_context_compact(ctx)
             when "stats"
-              spec = ctx.current_model_spec
-              p_name = spec["provider"]
-              m_name = spec["name"]
-              context_limit = ctx.llm_registry.context_limit_for(p_name, m_name)
-              max_output = ctx.llm_registry.max_output_tokens_for(p_name, m_name)
-              ceiling = ctx.llm_registry.effective_input_limit_for(p_name, m_name)
-              est_tokens = Rubik::Prompt::TokenEstimator.estimate_history(ctx.conversation_history)
-              pct = ceiling.positive? ? ((est_tokens.to_f / ceiling) * 100).clamp(0.0, 100.0).round(1) : 0.0
-              summary = ctx.context_index.status_summary
-              ast_symbols_count = ctx.ast_graph ? ctx.ast_graph.symbols.size : 0
-
-              ctx.out.puts "=== Context & Token Statistics ==="
-              ctx.out.puts "  • Model            : #{m_name} (#{p_name})"
-              ctx.out.puts "  • Context Limit    : #{context_limit} tokens"
-              ctx.out.puts "  • Usable Capacity  : #{ceiling} tokens (Max Output: #{max_output} tokens)"
-              ctx.out.puts "  • Consumed Tokens  : #{est_tokens} tokens (#{pct}% of usable capacity)"
-              ctx.out.puts "  • Active Turns     : #{ctx.conversation_history.size} turn(s)"
-              ctx.out.puts "  • Indexed Files    : #{summary[:file_count]} files"
-              ctx.out.puts "  • AST Symbol Count : #{ast_symbols_count} symbols"
-              ctx.out.puts "=================================="
-              guidance = Rubik::Tools::DiagnosticReporter.render_context_status_guidance(
-                { model: "#{p_name}/#{m_name}", consumed_tokens: est_tokens, context_limit: ceiling },
-                token_breakdown: { "Conversation Turns" => est_tokens }
-              )
-              ctx.out.puts "\n#{guidance}"
+              handle_context_stats(ctx)
             when "status", nil
-              spec = ctx.current_model_spec
-              p_name = spec["provider"]
-              m_name = spec["name"]
-              context_limit = ctx.llm_registry.context_limit_for(p_name, m_name)
-              max_output = ctx.llm_registry.max_output_tokens_for(p_name, m_name)
-              ceiling = ctx.llm_registry.effective_input_limit_for(p_name, m_name)
-              context_cfg = ctx.config.fetch("context", {})
-              strategy = context_cfg["compaction_strategy"] || "hybrid"
-
-              est_tokens = Rubik::Prompt::TokenEstimator.estimate_history(ctx.conversation_history)
-              pct = ceiling.positive? ? ((est_tokens.to_f / ceiling) * 100).clamp(0.0, 100.0).round(1) : 0.0
-              bar_width = 12
-              filled = ((pct / 100.0) * bar_width).round
-              empty = bar_width - filled
-              bar_str = "[#{'█' * filled}#{'░' * empty}] #{pct}%"
-
-              summary = ctx.context_index.status_summary
-              ast_symbols_count = ctx.ast_graph ? ctx.ast_graph.symbols.size : 0
-
-              ctx.out.puts "=== LLM Context Window & In-Memory Status ==="
-              ctx.out.puts "Active Model: #{m_name} (#{p_name})"
-              ctx.out.puts "Context Window Limit: #{context_limit} tokens"
-              ctx.out.puts "Max Output Tokens: #{max_output} tokens"
-              ctx.out.puts "Usable Input Ceiling: #{ceiling} tokens"
-              ctx.out.puts "Estimated Prompt Tokens: #{est_tokens} / #{ceiling} tokens #{bar_str}"
-              ctx.out.puts "Compaction Strategy: #{strategy}"
-              ctx.out.puts "Indexed Files: #{summary[:file_count]}, AST Symbols: #{ast_symbols_count}"
-              ctx.out.puts "============================================="
-              if (args && (args.include?("--guidance") || args.include?("-g"))) || est_tokens > (ceiling * 0.8)
-                guidance = Rubik::Tools::DiagnosticReporter.render_context_status_guidance(
-                  { model: "#{p_name}/#{m_name}", consumed_tokens: est_tokens, context_limit: ceiling },
-                  token_breakdown: { "Conversation History" => est_tokens }
-                )
-                ctx.out.puts "\n#{guidance}"
-              end
+              handle_context_status(args, ctx)
             else
               ctx.out.puts USAGE_CONTEXT
             end
           end
         end
 
+        private_class_method def self.handle_context_scan(ctx)
+          ctx.context_index.full_scan
+          ctx.warm_ast_graph(async: false)
+          ctx.out.puts "Context index and AST symbol graph updated."
+        rescue Errno::EACCES => e
+          root_path = begin
+            ctx.buffer_manager.root_dir
+          rescue
+            Dir.pwd
+          end
+          ctx.out.puts Rubik::Tools::DiagnosticReporter.render_permission_denied(root_path, operation: "scan", reason: e.message)
+        rescue StandardError => e
+          ctx.out.puts "Context scan failed: #{e.message}"
+        end
+
+        private_class_method def self.handle_context_add(args, ctx)
+          file = args[1]
+          if file
+            ctx.context_index.update_file(file)
+            full_path = File.join(ctx.buffer_manager.root_dir, file)
+            ctx.ast_graph.index_file(full_path) if File.exist?(full_path)
+          end
+          ctx.out.puts "Added #{file} to context index and AST graph."
+        end
+
+        private_class_method def self.handle_context_drop(args, ctx)
+          file = args[1]
+          if file
+            ctx.context_index.files.delete(file)
+            ctx.out.puts "Removed #{file} from context index."
+          else
+            ctx.out.puts "Usage: !context drop <relative_file_path>"
+          end
+        end
+
+        private_class_method def self.handle_context_compact(ctx)
+          before_tokens = Rubik::Prompt::TokenEstimator.estimate_history(ctx.conversation_history)
+          spec = ctx.current_model_spec
+          context_limit = ctx.llm_registry.context_limit_for(spec["provider"], spec["name"])
+          max_output = ctx.llm_registry.max_output_tokens_for(spec["provider"], spec["name"])
+          context_cfg = ctx.config.fetch("context", {})
+
+          ctx.conversation_history = Rubik::Prompt::ContextCompactor.compact_history(
+            ctx.conversation_history,
+            context_limit: context_limit,
+            max_output_tokens: max_output,
+            config: context_cfg,
+            force: true
+          )
+          after_tokens = Rubik::Prompt::TokenEstimator.estimate_history(ctx.conversation_history)
+          delta = before_tokens - after_tokens
+          ctx.out.puts "Context compaction completed: #{before_tokens} -> #{after_tokens} tokens (reduced #{delta} tokens)."
+        end
+
+        private_class_method def self.handle_context_stats(ctx)
+          spec = ctx.current_model_spec
+          p_name = spec["provider"]
+          m_name = spec["name"]
+          context_limit = ctx.llm_registry.context_limit_for(p_name, m_name)
+          max_output = ctx.llm_registry.max_output_tokens_for(p_name, m_name)
+          ceiling = ctx.llm_registry.effective_input_limit_for(p_name, m_name)
+          est_tokens = Rubik::Prompt::TokenEstimator.estimate_history(ctx.conversation_history)
+          pct = ceiling.positive? ? ((est_tokens.to_f / ceiling) * 100).clamp(0.0, 100.0).round(1) : 0.0
+          summary = ctx.context_index.status_summary
+          ast_symbols_count = ctx.ast_graph ? ctx.ast_graph.symbols.size : 0
+
+          ctx.out.puts "=== Context & Token Statistics ==="
+          ctx.out.puts "  • Model            : #{m_name} (#{p_name})"
+          ctx.out.puts "  • Context Limit    : #{context_limit} tokens"
+          ctx.out.puts "  • Usable Capacity  : #{ceiling} tokens (Max Output: #{max_output} tokens)"
+          ctx.out.puts "  • Consumed Tokens  : #{est_tokens} tokens (#{pct}% of usable capacity)"
+          ctx.out.puts "  • Active Turns     : #{ctx.conversation_history.size} turn(s)"
+          ctx.out.puts "  • Indexed Files    : #{summary[:file_count]} files"
+          ctx.out.puts "  • AST Symbol Count : #{ast_symbols_count} symbols"
+          ctx.out.puts "=================================="
+          guidance = Rubik::Tools::DiagnosticReporter.render_context_status_guidance(
+            { model: "#{p_name}/#{m_name}", consumed_tokens: est_tokens, context_limit: ceiling },
+            token_breakdown: { "Conversation Turns" => est_tokens }
+          )
+          ctx.out.puts "\n#{guidance}"
+        end
+
+        private_class_method def self.handle_context_status(args, ctx)
+          spec = ctx.current_model_spec
+          p_name = spec["provider"]
+          m_name = spec["name"]
+          context_limit = ctx.llm_registry.context_limit_for(p_name, m_name)
+          max_output = ctx.llm_registry.max_output_tokens_for(p_name, m_name)
+          ceiling = ctx.llm_registry.effective_input_limit_for(p_name, m_name)
+          context_cfg = ctx.config.fetch("context", {})
+          strategy = context_cfg["compaction_strategy"] || "hybrid"
+
+          est_tokens = Rubik::Prompt::TokenEstimator.estimate_history(ctx.conversation_history)
+          pct = ceiling.positive? ? ((est_tokens.to_f / ceiling) * 100).clamp(0.0, 100.0).round(1) : 0.0
+          bar_width = 12
+          filled = ((pct / 100.0) * bar_width).round
+          empty = bar_width - filled
+          bar_str = "[#{'█' * filled}#{'░' * empty}] #{pct}%"
+
+          summary = ctx.context_index.status_summary
+          ast_symbols_count = ctx.ast_graph ? ctx.ast_graph.symbols.size : 0
+
+          ctx.out.puts "=== LLM Context Window & In-Memory Status ==="
+          ctx.out.puts "Active Model: #{m_name} (#{p_name})"
+          ctx.out.puts "Context Window Limit: #{context_limit} tokens"
+          ctx.out.puts "Max Output Tokens: #{max_output} tokens"
+          ctx.out.puts "Usable Input Ceiling: #{ceiling} tokens"
+          ctx.out.puts "Estimated Prompt Tokens: #{est_tokens} / #{ceiling} tokens #{bar_str}"
+          ctx.out.puts "Compaction Strategy: #{strategy}"
+          ctx.out.puts "Indexed Files: #{summary[:file_count]}, AST Symbols: #{ast_symbols_count}"
+          ctx.out.puts "============================================="
+          return unless (args && (args.include?("--guidance") || args.include?("-g"))) || est_tokens > (ceiling * 0.8)
+
+          guidance = Rubik::Tools::DiagnosticReporter.render_context_status_guidance(
+            { model: "#{p_name}/#{m_name}", consumed_tokens: est_tokens, context_limit: ceiling },
+            token_breakdown: { "Conversation History" => est_tokens }
+          )
+          ctx.out.puts "\n#{guidance}"
+        end
+
         private_class_method def self.register_steps_command(shell)
           shell.register_command("steps", description: "Manage or view agentic step budget (default|goal|plan|<n>|status)",
                                           usage: USAGE_STEPS, audience: :both) do |args, ctx|