tree_haver 5.0.5 → 7.1.0

data/lib/tree_haver/grammar_finder.rb

@@ -1,17 +1,19 @@
 # frozen_string_literal: true
 
-require "rbconfig"
+require 'rbconfig'
 
 module TreeHaver
-  # Generic utility for finding tree-sitter grammar shared libraries.
+  # Registration-first utility for finding tree-sitter grammar shared libraries.
   #
-  # GrammarFinder provides platform-aware discovery of tree-sitter grammar
-  # libraries. Given a language name, it searches common installation paths
-  # and supports environment variable overrides.
+  # GrammarFinder resolves tree-sitter grammars in a constrained order:
+  #
+  # 1. explicit environment override
+  # 2. existing TreeHaver registration
+  # 3. explicit extra paths
+  # 4. tree_sitter_language_pack parser backend registration
   #
   # This class is designed to be used by language-specific merge gems
-  # (toml-merge, json-merge, bash-merge, etc.) without requiring TreeHaver
-  # to have knowledge of each specific language.
+  # without requiring TreeHaver to own parser- or grammar-specific policy.
   #
   # == Security Considerations
   #
@@ -29,7 +31,7 @@ module TreeHaver
   # @example Basic usage
   #   finder = TreeHaver::GrammarFinder.new(:toml)
   #   path = finder.find_library_path
-  #   # => "/usr/lib/libtree-sitter-toml.so"
+  #   # => "/usr/local/lib/libtree_sitter_toml.so"
   #
   # @example Check availability
   #   finder = TreeHaver::GrammarFinder.new(:json)
@@ -51,15 +53,6 @@ module TreeHaver
   #
   # @see PathValidator For details on security validations
   class GrammarFinder
-    # Common base directories where tree-sitter libraries are installed
-    # Platform-specific extensions are appended automatically
-    BASE_SEARCH_DIRS = [
-      "/usr/lib",
-      "/usr/lib64",
-      "/usr/local/lib",
-      "/opt/homebrew/lib",
-    ].freeze
-
     # @return [Symbol] the language identifier
     attr_reader :language_name
 
@@ -77,7 +70,7 @@ module TreeHaver
 
       if validate && !PathValidator.safe_language_name?(name_str)
         raise ArgumentError, "Invalid language name: #{language_name.inspect}. " \
-          "Language names must start with a letter and contain only lowercase letters, numbers, and underscores."
+          'Language names must start with a letter and contain only lowercase letters, numbers, and underscores.'
       end
 
       @language_name = name_str.to_sym
@@ -98,41 +91,56 @@ module TreeHaver
       "tree_sitter_#{@language_name}"
     end
 
-    # Get the library filename for the current platform
+    # Get the canonical tree-sitter-language-pack filename for the current platform
     #
-    # @return [String] the library filename (e.g., "libtree-sitter-toml.so")
+    # @return [String] the library filename (e.g., "libtree_sitter_toml.so")
     def library_filename
+      library_filenames.first
+    end
+
+    # Get all accepted library filenames for this language
+    #
+    # Accept both the tree-sitter-language-pack naming convention and the
+    # historical hyphenated form used by some standalone grammar builds.
+    #
+    # @return [Array<String>]
+    def library_filenames
       ext = platform_extension
-      "libtree-sitter-#{@language_name}#{ext}"
+      [
+        "libtree_sitter_#{@language_name}#{ext}",
+        "libtree-sitter-#{@language_name}#{ext}"
+      ]
     end
 
     # Generate the full list of search paths for this language
     #
-    # Order: ENV override, extra_paths, then common system paths
+    # Order: registered path, then explicit extra paths.
     #
     # @return [Array<String>] all paths to search
     def search_paths
       paths = []
 
-      # Extra paths provided at initialization (searched after ENV)
-      @extra_paths.each do |dir|
-        paths << File.join(dir, library_filename)
-      end
+      registration = registered_tree_sitter_registration
+      paths << registration[:path] if registration&.dig(:path)
 
-      # Common system paths with platform-appropriate extension
-      BASE_SEARCH_DIRS.each do |dir|
-        paths << File.join(dir, library_filename)
+      @extra_paths.each do |dir|
+        library_filenames.each do |filename|
+          paths << File.join(dir, filename)
+        end
       end
 
-      paths
+      paths.uniq
     end
 
     # Find the grammar library path
     #
     # Searches in order:
     # 1. Environment variable override (validated for safety)
-    # 2. Extra paths provided at initialization
-    # 3. Common system installation paths
+    # 2. Existing TreeHaver tree-sitter registration
+    # 3. Extra paths provided at initialization
+    # tree_sitter_language_pack is intentionally not exposed as a shared-library
+    # path fallback here. It is registered as a TreeHaver backend module when
+    # its parser API is available.
     #
     # @note Paths from ENV are validated using {PathValidator.safe_library_path?}
     #   to prevent path traversal and other attacks. Invalid ENV paths cause
@@ -151,18 +159,18 @@ module TreeHaver
       if ENV[env_var] || ENV.key?(env_var)
         env_path = ENV[env_var]
 
-        # :nocov: defensive - ENV.key? true with nil value is rare edge case
+        # simplecov:disable defensive - ENV.key? true with nil value is rare edge case
         if env_path.nil?
-          @env_rejection_reason = "explicitly disabled (set to nil)"
+          @env_rejection_reason = 'explicitly disabled (set to nil)'
           return
         end
-        # :nocov:
+        # simplecov:enable
 
         # Empty string means "explicitly skip this grammar"
         # This allows users to disable tree-sitter for specific languages
         # and fall back to alternative backends like Citrus
         if env_path.empty?
-          @env_rejection_reason = "explicitly disabled (set to empty string)"
+          @env_rejection_reason = 'explicitly disabled (set to empty string)'
           return
         end
 
@@ -173,35 +181,36 @@ module TreeHaver
         # to a path, that path MUST work. Don't silently fall back to auto-discovery.
         if @env_rejection_reason
           raise TreeHaver::NotAvailable,
-            "#{env_var_name} is set to #{env_path.inspect} but #{@env_rejection_reason}. " \
-              "Either fix the path, unset the variable to use auto-discovery, " \
-              "or set it to empty string to explicitly disable this grammar."
+                "#{env_var_name} is set to #{env_path.inspect} but #{@env_rejection_reason}. " \
+                  'Either fix the path, unset the variable to use auto-discovery, ' \
+                  'or set it to empty string to explicitly disable this grammar.'
         end
 
         return env_path
       end
 
-      # Search all paths (these are constructed from trusted base dirs)
-      search_paths.find { |path| File.exist?(path) }
+      registered_path = registered_tree_sitter_path
+      return registered_path if registered_path
+
+      explicit_path = explicit_search_path
+      return explicit_path if explicit_path
+
+      nil
     end
 
     # Validate an environment variable path and return reason if invalid
     # @return [String, nil] rejection reason or nil if valid
     def validate_env_path(path)
       # Check for leading/trailing whitespace
-      if path != path.strip
-        return "contains leading or trailing whitespace (use #{path.strip.inspect})"
-      end
+      return "contains leading or trailing whitespace (use #{path.strip.inspect})" if path != path.strip
 
       # Check if path is safe
       unless PathValidator.safe_library_path?(path)
-        return "failed security validation (may contain path traversal or suspicious characters)"
+        return 'failed security validation (may contain path traversal or suspicious characters)'
       end
 
       # Check if file exists
-      unless File.exist?(path)
-        return "file does not exist"
-      end
+      return 'file does not exist' unless File.exist?(path)
 
       nil # Valid!
     end
@@ -215,7 +224,6 @@ module TreeHaver
     # @return [String, nil] the path to the library, or nil if not found
     # @see PathValidator::TRUSTED_DIRECTORIES For the list of trusted directories
     def find_library_path_safe
-      # Environment variable is NOT checked in safe mode - only trusted system paths
       search_paths.find do |path|
         File.exist?(path) && PathValidator.in_trusted_directory?(path)
       end
@@ -232,6 +240,8 @@ module TreeHaver
     #
     # @return [Boolean] true if the library can be found AND tree-sitter runtime works
     def available?
+      return true if tree_sitter_language_pack_parser_available?
+
       path = find_library_path
       return false if path.nil?
 
@@ -246,7 +256,7 @@ module TreeHaver
       TreeHaver::Backends::MRI,
       TreeHaver::Backends::FFI,
       TreeHaver::Backends::Rust,
-      TreeHaver::Backends::Java,
+      TreeHaver::Backends::Java
     ].freeze
 
     class << self
@@ -273,7 +283,7 @@ module TreeHaver
             true
           end
         rescue NoMethodError, LoadError, NotAvailable => _e
-          # Note: FFI::NotFoundError inherits from LoadError, so it's caught here too
+          # NOTE: FFI::NotFoundError inherits from LoadError, so it's caught here too
           false
         end
       end
@@ -303,11 +313,20 @@ module TreeHaver
     # @return [Boolean] true if registration succeeded
     # @raise [NotAvailable] if library not found and raise_on_missing is true
     def register!(raise_on_missing: false)
+      if tree_sitter_language_pack_parser_available?
+        TreeHaver.register_language(
+          @language_name,
+          backend_module: TreeHaver::Backends::Tslp,
+          backend_type: :tslp,
+          gem_name: 'tree_sitter_language_pack'
+        )
+        return true
+      end
+
       path = find_library_path
       unless path
-        if raise_on_missing
-          raise NotAvailable, not_found_message
-        end
+        raise NotAvailable, not_found_message if raise_on_missing
+
         return false
       end
 
@@ -325,11 +344,13 @@ module TreeHaver
         env_var: env_var_name,
         env_value: ENV[env_var_name],
         env_rejection_reason: @env_rejection_reason,
+        tree_sitter_language_pack_parser_available: tree_sitter_language_pack_parser_available?,
         symbol: symbol_name,
         library_filename: library_filename,
+        library_filenames: library_filenames,
         search_paths: search_paths,
         found_path: found,
-        available: !found.nil?,
+        available: tree_sitter_language_pack_parser_available? || !found.nil?
       }
     end
 
@@ -342,32 +363,66 @@ module TreeHaver
       # Check if env var is set but rejected
       env_value = ENV[env_var_name]
       msg += if env_value && @env_rejection_reason
-        " #{env_var_name} is set to #{env_value.inspect} but #{@env_rejection_reason}."
-      elsif env_value && File.exist?(env_value) && !self.class.tree_sitter_runtime_usable?
-        " #{env_var_name} is set and file exists, but no tree-sitter runtime is available. " \
-          "Add ruby_tree_sitter, ffi, or tree_stump gem to your Gemfile."
-      elsif env_value
-        " #{env_var_name} is set but was not used (file may have been removed)."
-      else
-        " Searched: #{search_paths.join(", ")}."
-      end
-
-      msg + " Install tree-sitter-#{@language_name} or set #{env_var_name} to a valid path."
+               " #{env_var_name} is set to #{env_value.inspect} but #{@env_rejection_reason}."
+             elsif env_value && File.exist?(env_value) && !self.class.tree_sitter_runtime_usable?
+               " #{env_var_name} is set and file exists, but no tree-sitter runtime is available. " \
+                 'Add ruby_tree_sitter, ffi, or tree_stump gem to your Gemfile.'
+             else
+               " Searched: #{search_paths.join(', ')}."
+             end
+
+      msg + ' Register the grammar, install tree_sitter_language_pack with parser API support, ' \
+            "or set #{env_var_name} to a valid path."
     end
 
     private
 
+    def registered_tree_sitter_registration
+      TreeHaver::LanguageRegistry.registered(@language_name, :tree_sitter)
+    end
+
+    def registered_tree_sitter_path
+      registration = registered_tree_sitter_registration
+      path = registration&.dig(:path)
+      return unless path
+      return path if File.exist?(path)
+
+      @registered_rejection_reason = "registered path does not exist: #{path}"
+      nil
+    end
+
+    def explicit_search_path
+      search_paths.find { |path| File.exist?(path) }
+    end
+
+    def tree_sitter_language_pack_parser_available?
+      return @tree_sitter_language_pack_parser_available if defined?(@tree_sitter_language_pack_parser_available)
+
+      @tree_sitter_language_pack_parser_available = begin
+        require 'tree_sitter_language_pack' unless defined?(::TreeSitterLanguagePack)
+        TreeHaver::Backends::Tslp.available? &&
+          ::TreeSitterLanguagePack.respond_to?(:has_language) &&
+          ::TreeSitterLanguagePack.has_language(@language_name.to_s) &&
+          TreeHaver::Backends::Tslp.parser_available_for?(@language_name)
+      rescue LoadError
+        false
+      rescue StandardError => e
+        @tree_sitter_language_pack_rejection_reason = e.message
+        false
+      end
+    end
+
     # Get the platform-appropriate shared library extension
     #
     # @return [String] ".so" on Linux, ".dylib" on macOS
     def platform_extension
-      case RbConfig::CONFIG["host_os"]
+      case RbConfig::CONFIG['host_os']
       when /darwin/i
-        ".dylib"
+        '.dylib'
       when /mswin|mingw|cygwin/i
-        ".dll"
+        '.dll'
       else
-        ".so"
+        '.so'
       end
     end
   end

data/lib/tree_haver/kaitai_backend.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module TreeHaver
+  KAITAI_STRUCT_BACKEND = BackendReference.new(
+    id: 'kaitai-struct',
+    family: 'kaitai'
+  ).freeze
+
+  BackendRegistry.register(KAITAI_STRUCT_BACKEND)
+
+  module_function
+
+  def kaitai_adapter_info
+    AdapterInfo.new(
+      backend: KAITAI_STRUCT_BACKEND.id,
+      backend_ref: KAITAI_STRUCT_BACKEND,
+      supports_dialects: false,
+      supported_policies: []
+    )
+  end
+
+  def kaitai_feature_profile
+    FeatureProfile.new(
+      backend: KAITAI_STRUCT_BACKEND.id,
+      backend_ref: KAITAI_STRUCT_BACKEND,
+      supports_dialects: false,
+      supported_policies: []
+    )
+  end
+end

data/lib/tree_haver/language.rb

@@ -90,12 +90,12 @@ module TreeHaver
         if validate
           unless PathValidator.safe_library_path?(path)
             errors = PathValidator.validation_errors(path)
-            raise ArgumentError, "Unsafe library path: #{path.inspect}. Errors: #{errors.join("; ")}"
+            raise ArgumentError, "Unsafe library path: #{path.inspect}. Errors: #{errors.join('; ')}"
           end
 
           if symbol && !PathValidator.safe_symbol_name?(symbol)
             raise ArgumentError, "Unsafe symbol name: #{symbol.inspect}. " \
-              "Symbol names must be valid C identifiers."
+              'Symbol names must be valid C identifiers.'
           end
         end
 
@@ -105,19 +105,20 @@ module TreeHaver
 
         if mod.nil?
           if backend
-            raise NotAvailable, "Requested backend #{backend.inspect} is not available or does not support shared libraries"
+            raise NotAvailable,
+                  "Requested backend #{backend.inspect} is not available or does not support shared libraries"
           else
             raise NotAvailable,
-              "No native tree-sitter backend is available for loading shared libraries. " \
-                "Available native backends (MRI, Rust, FFI, Java) require platform-specific setup. " \
-                "For pure-Ruby parsing, use backend-specific Language classes directly (e.g., Prism, Psych, Citrus)."
+                  'No native tree-sitter backend is available for loading shared libraries. ' \
+                    'Available native backends (MRI, Rust, FFI, Java) require platform-specific setup. ' \
+                    'For pure-Ruby parsing, use backend-specific Language classes directly (e.g., Prism, Psych, Citrus).'
           end
         end
 
         # Backend must implement .from_library; fallback to .from_path for older impls
         # Include effective backend AND ENV vars in cache key since they affect loading
         effective_b = TreeHaver.resolve_effective_backend(backend)
-        key = [effective_b, path, symbol, name, ENV["TREE_SITTER_LANG_SYMBOL"]]
+        key = [effective_b, path, symbol, name, ENV['TREE_SITTER_LANG_SYMBOL']]
         LanguageRegistry.fetch(key) do
           if mod::Language.respond_to?(:from_library)
             mod::Language.from_library(path, symbol: symbol, name: name)
@@ -128,7 +129,7 @@ module TreeHaver
       end
       # Alias for {from_library}
       # @see from_library
-      alias_method :from_path, :from_library
+      alias from_path from_library
 
       # Dynamic helper to load a registered language by name
       #
@@ -162,40 +163,38 @@ module TreeHaver
 
         # Determine which backend type to use
         backend_type = if current_backend == Backends::Citrus
-          :citrus
-        elsif current_backend == Backends::Parslet
-          :parslet
-        else
-          :tree_sitter  # MRI, Rust, FFI, Java all use tree-sitter
-        end
+                         :citrus
+                       elsif current_backend == Backends::Parslet
+                         :parslet
+                       else
+                         :tree_sitter # MRI, Rust, FFI, Java all use tree-sitter
+                       end
 
         # Get backend-specific registration
         reg = all_backends[backend_type]
 
         # If Citrus backend is active
         if backend_type == :citrus
-          if reg && reg[:grammar_module]
-            return Backends::Citrus::Language.new(reg[:grammar_module])
-          end
+          return Backends::Citrus::Language.new(reg[:grammar_module]) if reg && reg[:grammar_module]
 
-          # Fall back to error if no Citrus grammar registered
+          # No Citrus grammar for this language — provide actionable error
           raise NotAvailable,
-            "Citrus backend is active but no Citrus grammar registered for :#{method_name}. " \
-              "Either register a Citrus grammar or use a tree-sitter backend. " \
-              "Registered backends: #{all_backends.keys.inspect}"
+                "No Citrus grammar registered for :#{method_name}. " \
+                  'This language may only be available via tree-sitter. ' \
+                  'Check that the correct backend is selected (current: citrus). ' \
+                  "Registered backends for :#{method_name}: #{all_backends.keys.inspect}"
         end
 
         # If Parslet backend is active
         if backend_type == :parslet
-          if reg && reg[:grammar_class]
-            return Backends::Parslet::Language.new(reg[:grammar_class])
-          end
+          return Backends::Parslet::Language.new(reg[:grammar_class]) if reg && reg[:grammar_class]
 
-          # Fall back to error if no Parslet grammar registered
+          # No Parslet grammar for this language — provide actionable error
           raise NotAvailable,
-            "Parslet backend is active but no Parslet grammar registered for :#{method_name}. " \
-              "Either register a Parslet grammar or use a tree-sitter backend. " \
-              "Registered backends: #{all_backends.keys.inspect}"
+                "No Parslet grammar registered for :#{method_name}. " \
+                  'This language may only be available via tree-sitter. ' \
+                  'Check that the correct backend is selected (current: parslet). ' \
+                  "Registered backends for :#{method_name}: #{all_backends.keys.inspect}"
         end
 
         # For tree-sitter backends, try to load from path
@@ -204,16 +203,16 @@ module TreeHaver
           path = kwargs[:path] || args.first || reg[:path]
           # Symbol priority: kwargs override > registration > derive from method_name
           symbol = if kwargs.key?(:symbol)
-            kwargs[:symbol]
-          elsif reg[:symbol]
-            reg[:symbol]
-          else
-            "tree_sitter_#{method_name}"
-          end
+                     kwargs[:symbol]
+                   elsif reg[:symbol]
+                     reg[:symbol]
+                   else
+                     "tree_sitter_#{method_name}"
+                   end
           # Name priority: kwargs override > derive from symbol (strip tree_sitter_ prefix)
           # Using symbol-derived name ensures ruby_tree_sitter gets the correct language name
           # e.g., "toml" not "toml_both" when symbol is "tree_sitter_toml"
-          name = kwargs[:name] || symbol&.sub(/\Atree_sitter_/, "")
+          name = kwargs[:name] || symbol&.sub(/\Atree_sitter_/, '')
 
           begin
             return from_library(path, symbol: symbol, name: name)
@@ -242,8 +241,8 @@ module TreeHaver
 
         # No appropriate registration found
         raise ArgumentError,
-          "No grammar registered for :#{method_name} compatible with #{backend_type} backend. " \
-            "Registered backends: #{all_backends.keys.inspect}"
+              "No grammar registered for :#{method_name} compatible with #{backend_type} backend. " \
+                "Registered backends: #{all_backends.keys.inspect}"
       end
 
       # @api private

data/lib/tree_haver/language_pack.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+module TreeHaver
+  TSLP_BACKEND = BackendReference.new(
+    id: 'tslp',
+    family: 'tree-sitter'
+  ).freeze
+  KREUZBERG_LANGUAGE_PACK_BACKEND = BackendReference.new(
+    id: 'kreuzberg-language-pack',
+    family: 'tree-sitter'
+  ).freeze
+
+  BackendRegistry.register(TSLP_BACKEND)
+  BackendRegistry.register(KREUZBERG_LANGUAGE_PACK_BACKEND)
+  BackendRegistry.register_availability_checker(:tslp) do
+    Backends::Tslp.available?
+  end
+  BackendRegistry.register_availability_checker(:"kreuzberg-language-pack") do
+    BackendRegistry.available?(:tslp)
+  end
+
+  module_function
+
+  def language_pack_adapter_info
+    AdapterInfo.new(
+      backend: TSLP_BACKEND.id,
+      backend_ref: TSLP_BACKEND,
+      supports_dialects: false,
+      supported_policies: []
+    )
+  end
+
+  def language_pack_feature_profile
+    FeatureProfile.new(
+      backend: TSLP_BACKEND.id,
+      backend_ref: TSLP_BACKEND,
+      supports_dialects: false,
+      supported_policies: []
+    )
+  end
+end

data/lib/tree_haver/language_registry.rb

@@ -26,7 +26,7 @@ module TreeHaver
   #
   # Registration structure:
   # ```ruby
-  # @registrations = {
+  # registrations = {
   #   toml: {
   #     tree_sitter: { path: "/path/to/lib.so", symbol: "tree_sitter_toml" },
   #     citrus: { grammar_module: TomlRB::Document, gem_name: "toml-rb" }
@@ -69,8 +69,8 @@ module TreeHaver
   # @api private
   module LanguageRegistry
     @mutex = Mutex.new
-    @cache = {} # rubocop:disable ThreadSafety/MutableClassInstanceVariable
-    @registrations = {} # rubocop:disable ThreadSafety/MutableClassInstanceVariable
+    @cache = {}
+    @registrations = {}
 
     module_function
 
@@ -108,6 +108,36 @@ module TreeHaver
       nil
     end
 
+    def with_registration(name, backend_type, **config)
+      key = name.to_sym
+      backend_key = backend_type.to_sym
+      original = nil
+      had_language = false
+      had_backend = false
+
+      @mutex.synchronize do
+        had_language = @registrations.key?(key)
+        had_backend = @registrations.fetch(key, {}).key?(backend_key)
+        original = @registrations.fetch(key, {})[backend_key]&.dup
+        @registrations[key] ||= {}
+        @registrations[key][backend_key] = config.compact
+        @cache.clear
+      end
+
+      yield
+    ensure
+      @mutex.synchronize do
+        if had_backend
+          @registrations[key][backend_key] = original
+        elsif had_language
+          @registrations[key].delete(backend_key)
+        else
+          @registrations.delete(key)
+        end
+        @cache.clear
+      end
+    end
+
     # Fetch registration entries for a language
     #
     # Returns all backend-specific configurations for a language.
@@ -153,6 +183,7 @@ module TreeHaver
     def fetch(key)
       @mutex.synchronize do
         return @cache[key] if @cache.key?(key)
+
         value = yield
         @cache[key] = value
       end