tree_haver 2.0.0 → 3.1.0

data/lib/tree_haver/backends/rust.rb

@@ -54,14 +54,14 @@ module TreeHaver
         # @return [Hash{Symbol => Object}] capability map
         # @example
         #   TreeHaver::Backends::Rust.capabilities
-        #   # => { backend: :rust, query: true, bytes_field: true, incremental: true }
+        #   # => { backend: :rust, query: true, bytes_field: true, incremental: false }
         def capabilities
           return {} unless available?
           {
             backend: :rust,
             query: true,
             bytes_field: true,
-            incremental: true,
+            incremental: false,  # TreeStump doesn't currently expose incremental parsing to Ruby
           }
         end
       end
@@ -72,16 +72,52 @@ module TreeHaver
       # tree_stump uses a registration-based API where languages are registered
       # by name, then referenced by that name when setting parser language.
       class Language
+        include Comparable
+
         # The registered language name
         # @return [String]
         attr_reader :name
 
+        # The backend this language is for
+        # @return [Symbol]
+        attr_reader :backend
+
+        # The path this language was loaded from (if known)
+        # @return [String, nil]
+        attr_reader :path
+
         # @api private
         # @param name [String] the registered language name
-        def initialize(name)
+        # @param path [String, nil] path language was loaded from
+        def initialize(name, path: nil)
           @name = name
+          @backend = :rust
+          @path = path
+        end
+
+        # Compare languages for equality
+        #
+        # Rust languages are equal if they have the same backend and name.
+        # Name uniquely identifies a registered language in TreeStump.
+        #
+        # @param other [Object] object to compare with
+        # @return [Integer, nil] -1, 0, 1, or nil if not comparable
+        def <=>(other)
+          return unless other.is_a?(Language)
+          return unless other.backend == @backend
+
+          @name <=> other.name
         end
 
+        # Hash value for this language (for use in Sets/Hashes)
+        # @return [Integer]
+        def hash
+          [@backend, @name].hash
+        end
+
+        # Alias eql? to ==
+        alias_method :eql?, :==
+
         # Load a language from a shared library path
         #
         # @param path [String] absolute path to the language shared library
@@ -102,7 +138,7 @@ module TreeHaver
             # The name is used to derive the symbol automatically (tree_sitter_<name>)
             lang_name = name || File.basename(path, ".*").sub(/^libtree-sitter-/, "")
             ::TreeStump.register_lang(lang_name, path)
-            new(lang_name)
+            new(lang_name, path: path)
           rescue RuntimeError => e
             raise TreeHaver::NotAvailable, "Failed to load language from #{path}: #{e.message}"
           end
@@ -128,34 +164,41 @@ module TreeHaver
 
         # Set the language for this parser
         #
-        # @param lang [Language, String] the language to use (Language wrapper or name string)
+        # Note: TreeHaver::Parser unwraps language objects before calling this method.
+        # When called from TreeHaver::Parser, receives String (language name).
+        # For backward compatibility and backend tests, also handles Language wrapper.
+        #
+        # @param lang [Language, String] the language wrapper or name string
         # @return [Language, String] the language that was set
         def language=(lang)
-          # tree_stump uses set_language with a string name
+          # Extract language name (handle both wrapper and raw string)
           lang_name = lang.respond_to?(:name) ? lang.name : lang.to_s
+          # tree_stump uses set_language with a string name
           @parser.set_language(lang_name)
-          lang
+          lang # rubocop:disable Lint/Void (intentional return value)
         end
 
         # Parse source code
         #
         # @param source [String] the source code to parse
-        # @return [TreeHaver::Tree] wrapped tree
+        # @return [TreeStump::Tree] raw backend tree (wrapping happens in TreeHaver::Parser)
         def parse(source)
-          tree = @parser.parse(source)
-          TreeHaver::Tree.new(tree, source: source)
+          # Return raw tree_stump tree - TreeHaver::Parser will wrap it
+          @parser.parse(source)
         end
 
         # Parse source code with optional incremental parsing
         #
-        # @param old_tree [TreeHaver::Tree, nil] previous tree for incremental parsing
+        # Note: TreeStump does not currently expose incremental parsing to Ruby.
+        # The parse method always does a full parse, ignoring old_tree.
+        #
+        # @param old_tree [TreeHaver::Tree, nil] previous tree for incremental parsing (ignored)
         # @param source [String] the source code to parse
-        # @return [TreeHaver::Tree] wrapped tree
-        def parse_string(old_tree, source)
-          # tree_stump doesn't have parse_string, use parse instead
-          # TODO: Check if tree_stump supports incremental parsing
-          tree = @parser.parse(source)
-          TreeHaver::Tree.new(tree, source: source)
+        # @return [TreeStump::Tree] raw backend tree (wrapping happens in TreeHaver::Parser)
+        def parse_string(old_tree, source) # rubocop:disable Lint/UnusedMethodArgument
+          # TreeStump's parse method only accepts source as a single argument
+          # and internally always passes None for the old tree (no incremental parsing support)
+          @parser.parse(source)
         end
       end
     end

data/lib/tree_haver/citrus_grammar_finder.rb

@@ -0,0 +1,170 @@
+# frozen_string_literal: true
+
+module TreeHaver
+  # Utility for finding and registering Citrus grammar gems.
+  #
+  # CitrusGrammarFinder provides language-agnostic discovery of Citrus grammar
+  # gems. Given a language name and gem information, it attempts to load the
+  # grammar and register it with tree_haver.
+  #
+  # Unlike tree-sitter grammars (which are .so files), Citrus grammars are
+  # Ruby modules that respond to .parse(source). This class handles the
+  # discovery and registration of these grammars.
+  #
+  # @example Basic usage with toml-rb
+  #   finder = TreeHaver::CitrusGrammarFinder.new(
+  #     language: :toml,
+  #     gem_name: "toml-rb",
+  #     grammar_const: "TomlRB::Document"
+  #   )
+  #   finder.register! if finder.available?
+  #
+  # @example With custom require path
+  #   finder = TreeHaver::CitrusGrammarFinder.new(
+  #     language: :json,
+  #     gem_name: "json-rb",
+  #     grammar_const: "JsonRB::Grammar",
+  #     require_path: "json/rb"
+  #   )
+  #
+  # @see GrammarFinder For tree-sitter grammar discovery
+  class CitrusGrammarFinder
+    # @return [Symbol] the language identifier
+    attr_reader :language_name
+
+    # @return [String] the gem name to require
+    attr_reader :gem_name
+
+    # @return [String] the constant path to the grammar (e.g., "TomlRB::Document")
+    attr_reader :grammar_const
+
+    # @return [String, nil] custom require path (defaults to gem_name with dashes to slashes)
+    attr_reader :require_path
+
+    # Initialize a Citrus grammar finder
+    #
+    # @param language [Symbol, String] the language name (e.g., :toml, :json)
+    # @param gem_name [String] the gem name (e.g., "toml-rb")
+    # @param grammar_const [String] constant path to grammar (e.g., "TomlRB::Document")
+    # @param require_path [String, nil] custom require path (defaults to gem_name with dashes→slashes)
+    def initialize(language:, gem_name:, grammar_const:, require_path: nil)
+      @language_name = language.to_sym
+      @gem_name = gem_name
+      @grammar_const = grammar_const
+      @require_path = require_path || gem_name.tr("-", "/")
+      @load_attempted = false
+      @available = false
+      @grammar_module = nil
+    end
+
+    # Check if the Citrus grammar is available
+    #
+    # Attempts to require the gem and resolve the grammar constant.
+    # Result is cached after first call.
+    #
+    # @return [Boolean] true if grammar is available
+    def available?
+      return @available if @load_attempted
+
+      @load_attempted = true
+      begin
+        # Try to require the gem
+        require @require_path
+
+        # Try to resolve the constant
+        @grammar_module = resolve_constant(@grammar_const)
+
+        # Verify it responds to parse
+        unless @grammar_module.respond_to?(:parse)
+          warn("#{@grammar_const} doesn't respond to :parse")
+          @available = false
+          return false
+        end
+
+        @available = true
+      rescue LoadError => e
+        # Always show LoadError for debugging
+        warn("CitrusGrammarFinder: Failed to load '#{@require_path}': #{e.class}: #{e.message}")
+        @available = false
+      rescue NameError => e
+        # Always show NameError for debugging
+        warn("CitrusGrammarFinder: Failed to resolve '#{@grammar_const}': #{e.class}: #{e.message}")
+        @available = false
+      rescue => e
+        # Catch any other errors
+        warn("CitrusGrammarFinder: Unexpected error: #{e.class}: #{e.message}")
+        warn(e.backtrace.first(3).join("\n")) if ENV["TREE_HAVER_DEBUG"]
+        @available = false
+      end
+
+      @available
+    end
+
+    # Get the resolved grammar module
+    #
+    # @return [Module, nil] the grammar module if available
+    def grammar_module
+      available? # Ensure we've tried to load
+      @grammar_module
+    end
+
+    # Register this Citrus grammar with TreeHaver
+    #
+    # After registration, the language can be used via:
+    #   TreeHaver::Language.{language_name}
+    #
+    # @param raise_on_missing [Boolean] if true, raises when grammar not available
+    # @return [Boolean] true if registration succeeded
+    # @raise [NotAvailable] if grammar not available and raise_on_missing is true
+    def register!(raise_on_missing: false)
+      unless available?
+        if raise_on_missing
+          raise NotAvailable, not_found_message
+        end
+        return false
+      end
+
+      TreeHaver.register_language(
+        @language_name,
+        grammar_module: @grammar_module,
+        gem_name: @gem_name,
+      )
+      true
+    end
+
+    # Get debug information about the search
+    #
+    # @return [Hash] diagnostic information
+    def search_info
+      {
+        language: @language_name,
+        gem_name: @gem_name,
+        grammar_const: @grammar_const,
+        require_path: @require_path,
+        available: available?,
+        grammar_module: @grammar_module&.name,
+      }
+    end
+
+    # Get a human-readable error message when grammar is not found
+    #
+    # @return [String] error message with installation hints
+    def not_found_message
+      "Citrus grammar for #{@language_name} not found. " \
+        "Install #{@gem_name} gem: gem install #{@gem_name}"
+    end
+
+    private
+
+    # Resolve a constant path like "TomlRB::Document"
+    #
+    # @param const_path [String] constant path
+    # @return [Object] the constant
+    # @raise [NameError] if constant not found
+    def resolve_constant(const_path)
+      const_path.split("::").reduce(Object) do |mod, const_name|
+        mod.const_get(const_name)
+      end
+    end
+  end
+end

data/lib/tree_haver/grammar_finder.rb

@@ -137,19 +137,55 @@ module TreeHaver
     # @note Paths from ENV are validated using {PathValidator.safe_library_path?}
     #   to prevent path traversal and other attacks. Invalid ENV paths are ignored.
     #
+    # @note Setting the ENV variable to an empty string explicitly disables
+    #   this grammar. This allows fallback to alternative backends (e.g., Citrus).
+    #
     # @return [String, nil] the path to the library, or nil if not found
     # @see #find_library_path_safe For stricter validation (trusted directories only)
     def find_library_path
       # Check environment variable first (highest priority)
-      env_path = ENV[env_var_name]
-      if env_path && PathValidator.safe_library_path?(env_path) && File.exist?(env_path)
-        return env_path
+      # Use key? to distinguish between "not set" and "set to empty"
+      if ENV.key?(env_var_name)
+        env_path = ENV[env_var_name]
+
+        # Empty string means "explicitly skip this grammar"
+        # This allows users to disable tree-sitter for specific languages
+        # and fall back to alternative backends like Citrus
+        if env_path.empty?
+          @env_rejection_reason = "explicitly disabled (set to empty string)"
+          return
+        end
+
+        # Store why env path was rejected for better error messages
+        @env_rejection_reason = validate_env_path(env_path)
+        return env_path if @env_rejection_reason.nil?
       end
 
       # Search all paths (these are constructed from trusted base dirs)
       search_paths.find { |path| File.exist?(path) }
     end
 
+    # Validate an environment variable path and return reason if invalid
+    # @return [String, nil] rejection reason or nil if valid
+    def validate_env_path(path)
+      # Check for leading/trailing whitespace
+      if path != path.strip
+        return "contains leading or trailing whitespace (use #{path.strip.inspect})"
+      end
+
+      # Check if path is safe
+      unless PathValidator.safe_library_path?(path)
+        return "failed security validation (may contain path traversal or suspicious characters)"
+      end
+
+      # Check if file exists
+      unless File.exist?(path)
+        return "file does not exist"
+      end
+
+      nil # Valid!
+    end
+
     # Find the grammar library path with strict security validation
     #
     # This method only returns paths that are in trusted system directories.
@@ -165,11 +201,67 @@ module TreeHaver
       end
     end
 
-    # Check if the grammar library is available
+    # Check if the grammar library is available AND usable
     #
-    # @return [Boolean] true if the library can be found
+    # This checks:
+    # 1. The grammar library file exists
+    # 2. The tree-sitter runtime is functional (can create a parser)
+    #
+    # This prevents registering grammars when tree-sitter isn't actually usable,
+    # allowing clean fallback to alternative backends like Citrus.
+    #
+    # @return [Boolean] true if the library can be found AND tree-sitter runtime works
     def available?
-      !find_library_path.nil?
+      path = find_library_path
+      return false if path.nil?
+
+      # Check if tree-sitter runtime is actually functional
+      # This is cached at the class level since it's the same for all grammars
+      self.class.tree_sitter_runtime_usable?
+    end
+
+    # Backends that use tree-sitter (require native runtime libraries)
+    # Other backends (Citrus, Prism, Psych, etc.) don't use tree-sitter
+    TREE_SITTER_BACKENDS = [
+      TreeHaver::Backends::MRI,
+      TreeHaver::Backends::FFI,
+      TreeHaver::Backends::Rust,
+      TreeHaver::Backends::Java,
+    ].freeze
+
+    class << self
+      # Check if the tree-sitter runtime is usable
+      #
+      # Tests whether we can actually create a tree-sitter parser.
+      # Result is cached since this is expensive and won't change during runtime.
+      #
+      # @return [Boolean] true if tree-sitter runtime is functional
+      def tree_sitter_runtime_usable?
+        return @tree_sitter_runtime_usable if defined?(@tree_sitter_runtime_usable)
+
+        @tree_sitter_runtime_usable = begin
+          # Try to create a parser using the current backend
+          mod = TreeHaver.resolve_backend_module(nil)
+
+          # Only tree-sitter backends are relevant here
+          # Non-tree-sitter backends (Citrus, Prism, Psych, etc.) don't use grammar files
+          return false if mod.nil?
+          return false unless TREE_SITTER_BACKENDS.include?(mod)
+
+          # Try to instantiate a parser - this will fail if runtime isn't available
+          mod::Parser.new
+          true
+        rescue NoMethodError, FFI::NotFoundError, LoadError, NotAvailable => _e
+          false
+        end
+      end
+
+      # Reset the cached tree-sitter runtime check (for testing)
+      #
+      # @api private
+      def reset_runtime_check!
+        remove_instance_variable(:@tree_sitter_runtime_usable) if defined?(@tree_sitter_runtime_usable)
+      end
     end
 
     # Check if the grammar library is available in a trusted directory
@@ -205,15 +297,17 @@ module TreeHaver
     #
     # @return [Hash] diagnostic information
     def search_info
+      found = find_library_path # This populates @env_rejection_reason
       {
         language: @language_name,
         env_var: env_var_name,
         env_value: ENV[env_var_name],
+        env_rejection_reason: @env_rejection_reason,
         symbol: symbol_name,
         library_filename: library_filename,
         search_paths: search_paths,
-        found_path: find_library_path,
-        available: available?,
+        found_path: found,
+        available: !found.nil?,
       }
     end
 
@@ -221,9 +315,19 @@ module TreeHaver
     #
     # @return [String] error message with installation hints
     def not_found_message
-      "tree-sitter #{@language_name} grammar not found. " \
-        "Searched: #{search_paths.join(", ")}. " \
-        "Install tree-sitter-#{@language_name} or set #{env_var_name}."
+      msg = "tree-sitter #{@language_name} grammar not found."
+
+      # Check if env var is set but rejected
+      env_value = ENV[env_var_name]
+      msg += if env_value && @env_rejection_reason
+        " #{env_var_name} is set to #{env_value.inspect} but #{@env_rejection_reason}."
+      elsif env_value
+        " #{env_var_name} is set but was not used (file may have been removed)."
+      else
+        " Searched: #{search_paths.join(", ")}."
+      end
+
+      msg + " Install tree-sitter-#{@language_name} or set #{env_var_name} to a valid path."
     end
 
     private

data/lib/tree_haver/language_registry.rb

@@ -4,86 +4,93 @@ module TreeHaver
   # Thread-safe language registrations and cache for loaded Language handles
   #
   # The LanguageRegistry provides two main functions:
-  # 1. **Registrations**: Store mappings from language names to shared library paths
+  # 1. **Registrations**: Store mappings from language names to backend-specific configurations
   # 2. **Cache**: Memoize loaded Language objects to avoid repeated dlopen calls
   #
-  # All operations are thread-safe and protected by a mutex.
+  # The registry supports multiple backends for the same language, allowing runtime
+  # switching, benchmarking, and fallback scenarios.
   #
-  # @example Register and cache a language
-  #   TreeHaver::LanguageRegistry.register(:toml, path: "/path/to/lib.so", symbol: "tree_sitter_toml")
-  #   lang = TreeHaver::LanguageRegistry.fetch(["/path/to/lib.so", "tree_sitter_toml", "toml"]) do
-  #     # This block is called only if not cached
-  #     load_language_from_library(...)
-  #   end
+  # Registration structure:
+  # @registrations = {
+  #   toml: {
+  #     tree_sitter: { path: "/path/to/lib.so", symbol: "tree_sitter_toml" },
+  #     citrus: { grammar_module: TomlRB::Document, gem_name: "toml-rb" }
+  #   }
+  # }
+  #
+  # @example Register tree-sitter grammar
+  #   TreeHaver::LanguageRegistry.register(:toml, :tree_sitter,
+  #     path: "/path/to/lib.so", symbol: "tree_sitter_toml")
+  #
+  # @example Register Citrus grammar
+  #   TreeHaver::LanguageRegistry.register(:toml, :citrus,
+  #     grammar_module: TomlRB::Document, gem_name: "toml-rb")
   #
   # @api private
   module LanguageRegistry
     @mutex = Mutex.new
-    @cache = {}
-    @registrations = {}
+    @cache = {} # rubocop:disable ThreadSafety/MutableClassInstanceVariable
+    @registrations = {} # rubocop:disable ThreadSafety/MutableClassInstanceVariable
 
     module_function
 
-    # Register a language helper by name
+    # Register a language for a specific backend
     #
-    # Stores a mapping from a language name to its shared library path and
-    # optional exported symbol name. After registration, the language can be
-    # accessed via dynamic helpers on {TreeHaver::Language}.
+    # Stores backend-specific configuration for a language. Multiple backends
+    # can be registered for the same language without conflict.
     #
     # @param name [Symbol, String] language identifier (e.g., :toml, :json)
-    # @param path [String] absolute path to the language shared library
-    # @param symbol [String, nil] optional exported factory symbol (e.g., "tree_sitter_toml")
+    # @param backend_type [Symbol] backend type (:tree_sitter, :citrus, :mri, :rust, :ffi, :java)
+    # @param config [Hash] backend-specific configuration
+    # @option config [String] :path tree-sitter library path (for tree-sitter backends)
+    # @option config [String] :symbol exported symbol name (for tree-sitter backends)
+    # @option config [Module] :grammar_module Citrus grammar module (for Citrus backend)
+    # @option config [String] :gem_name gem name for error messages (for Citrus backend)
     # @return [void]
-    # @example
-    #   LanguageRegistry.register(:toml, path: "/usr/local/lib/libtree-sitter-toml.so")
-    def register(name, path:, symbol: nil)
+    # @example Register tree-sitter grammar
+    #   LanguageRegistry.register(:toml, :tree_sitter,
+    #     path: "/usr/local/lib/libtree-sitter-toml.so", symbol: "tree_sitter_toml")
+    # @example Register Citrus grammar
+    #   LanguageRegistry.register(:toml, :citrus,
+    #     grammar_module: TomlRB::Document, gem_name: "toml-rb")
+    def register(name, backend_type, **config)
       key = name.to_sym
-      @mutex.synchronize do
-        @registrations[key] = {path: path, symbol: symbol}
-      end
-      nil
-    end
+      backend_key = backend_type.to_sym
 
-    # Unregister a previously registered language helper
-    #
-    # Removes the registration entry but does not affect cached Language objects.
-    #
-    # @param name [Symbol, String] language identifier to unregister
-    # @return [void]
-    # @example
-    #   LanguageRegistry.unregister(:toml)
-    def unregister(name)
-      key = name.to_sym
       @mutex.synchronize do
-        @registrations.delete(key)
+        @registrations[key] ||= {}
+        @registrations[key][backend_key] = config.compact
       end
       nil
     end
 
-    # Fetch a registration entry
+    # Fetch registration entries for a language
     #
-    # Returns the stored path and symbol for a registered language name.
+    # Returns all backend-specific configurations for a language.
     #
     # @param name [Symbol, String] language identifier
-    # @return [Hash{Symbol => String, nil}, nil] hash with :path and :symbol keys, or nil if not registered
-    # @example
-    #   entry = LanguageRegistry.registered(:toml)
-    #   # => { path: "/usr/local/lib/libtree-sitter-toml.so", symbol: "tree_sitter_toml" }
-    def registered(name)
-      @mutex.synchronize { @registrations[name.to_sym] }
-    end
+    # @param backend_type [Symbol, nil] optional backend type to filter by
+    # @return [Hash{Symbol => Hash}, Hash, nil] all backends or specific backend config
+    # @example Get all backends
+    #   entries = LanguageRegistry.registered(:toml)
+    #   # => {
+    #   #   tree_sitter: { path: "/usr/local/lib/libtree-sitter-toml.so", symbol: "tree_sitter_toml" },
+    #   #   citrus: { grammar_module: TomlRB::Document, gem_name: "toml-rb" }
+    #   # }
+    # @example Get specific backend
+    #   entry = LanguageRegistry.registered(:toml, :citrus)
+    #   # => { grammar_module: TomlRB::Document, gem_name: "toml-rb" }
+    def registered(name, backend_type = nil)
+      @mutex.synchronize do
+        lang_config = @registrations[name.to_sym]
+        return unless lang_config
 
-    # Clear all registrations
-    #
-    # Removes all registered language mappings. Primarily intended for test cleanup.
-    # Does not clear the language cache.
-    #
-    # @return [void]
-    # @example
-    #   LanguageRegistry.clear_registrations!
-    def clear_registrations!
-      @mutex.synchronize { @registrations.clear }
-      nil
+        if backend_type
+          lang_config[backend_type.to_sym]
+        else
+          lang_config
+        end
+      end
     end
 
     # Fetch a cached language by key or compute and store it
@@ -119,21 +126,5 @@ module TreeHaver
       @mutex.synchronize { @cache.clear }
       nil
     end
-
-    # Clear everything (registrations and cache)
-    #
-    # Removes all registered languages and all cached Language objects.
-    # Useful for complete teardown in tests.
-    #
-    # @return [void]
-    # @example
-    #   LanguageRegistry.clear_all!
-    def clear_all!
-      @mutex.synchronize do
-        @registrations.clear
-        @cache.clear
-      end
-      nil
-    end
   end
 end