tree_haver 5.0.4 → 7.0.0

data/lib/tree_haver/language.rb

@@ -1,295 +0,0 @@
-# frozen_string_literal: true
-
-module TreeHaver
-  # Factory module for loading language grammars
-  #
-  # Language is the entry point for loading and using grammars. It provides
-  # a unified interface that works across all backends (MRI, Rust, FFI, Java, Citrus, Parslet).
-  #
-  # This is a module with only module methods (factory pattern), not a class.
-  # Backend-specific Language classes (e.g., Backends::Citrus::Language,
-  # Backends::Parslet::Language) inherit from Base::Language.
-  #
-  # For tree-sitter backends, languages are loaded from shared library files (.so/.dylib/.dll).
-  # For pure-Ruby backends (Citrus, Parslet, Prism, Psych), languages are built-in or provided by gems.
-  #
-  # == Loading Languages
-  #
-  # The primary way to load a language is via registration:
-  #
-  #   TreeHaver.register_language(:toml, path: "/path/to/libtree-sitter-toml.so")
-  #   language = TreeHaver::Language.toml
-  #
-  # For explicit loading without registration:
-  #
-  #   language = TreeHaver::Language.from_library(
-  #     "/path/to/libtree-sitter-toml.so",
-  #     symbol: "tree_sitter_toml"
-  #   )
-  #
-  # For ruby_tree_sitter compatibility:
-  #
-  #   language = TreeHaver::Language.load("toml", "/path/to/libtree-sitter-toml.so")
-  #
-  # @example Register and load a language
-  #   TreeHaver.register_language(:toml, path: "/path/to/grammar.so")
-  #   language = TreeHaver::Language.toml
-  #
-  # @see Base::Language The base class that backend Language classes inherit from
-  module Language
-    class << self
-      # Load a language grammar from a shared library (ruby_tree_sitter compatibility)
-      #
-      # This method provides API compatibility with ruby_tree_sitter which uses
-      # `Language.load(name, path)`.
-      #
-      # @param name [String] the language name (e.g., "toml")
-      # @param path [String] absolute path to the language shared library
-      # @param validate [Boolean] if true, validates the path for safety (default: true)
-      # @return [Language] loaded language handle
-      # @raise [NotAvailable] if the library cannot be loaded
-      # @raise [ArgumentError] if the path fails security validation
-      # @example
-      #   language = TreeHaver::Language.load("toml", "/usr/local/lib/libtree-sitter-toml.so")
-      def load(name, path, validate: true)
-        from_library(path, symbol: "tree_sitter_#{name}", name: name, validate: validate)
-      end
-
-      # Load a language grammar from a shared library
-      #
-      # The library must export a function that returns a pointer to a TSLanguage struct.
-      # By default, TreeHaver looks for a symbol named "tree_sitter_<name>".
-      #
-      # == Security
-      #
-      # By default, paths are validated using {PathValidator} to prevent path traversal
-      # and other attacks. Set `validate: false` to skip validation (not recommended
-      # unless you've already validated the path).
-      #
-      # @param path [String] absolute path to the language shared library (.so/.dylib/.dll)
-      # @param symbol [String, nil] name of the exported function (defaults to auto-detection)
-      # @param name [String, nil] logical name for the language (used in caching)
-      # @param validate [Boolean] if true, validates path and symbol for safety (default: true)
-      # @param backend [Symbol, String, nil] optional backend to use (overrides context/global)
-      # @return [Language] loaded language handle
-      # @raise [NotAvailable] if the library cannot be loaded or the symbol is not found
-      # @raise [ArgumentError] if path or symbol fails security validation
-      # @example
-      #   language = TreeHaver::Language.from_library(
-      #     "/usr/local/lib/libtree-sitter-toml.so",
-      #     symbol: "tree_sitter_toml",
-      #     name: "toml"
-      #   )
-      # @example With explicit backend
-      #   language = TreeHaver::Language.from_library(
-      #     "/usr/local/lib/libtree-sitter-toml.so",
-      #     symbol: "tree_sitter_toml",
-      #     backend: :ffi
-      #   )
-      def from_library(path, symbol: nil, name: nil, validate: true, backend: nil)
-        if validate
-          unless PathValidator.safe_library_path?(path)
-            errors = PathValidator.validation_errors(path)
-            raise ArgumentError, "Unsafe library path: #{path.inspect}. Errors: #{errors.join("; ")}"
-          end
-
-          if symbol && !PathValidator.safe_symbol_name?(symbol)
-            raise ArgumentError, "Unsafe symbol name: #{symbol.inspect}. " \
-              "Symbol names must be valid C identifiers."
-          end
-        end
-
-        # from_library only works with tree-sitter backends that support .so files
-        # Pure Ruby backends (Citrus, Prism, Psych, Commonmarker, Markly) don't support from_library
-        mod = TreeHaver.resolve_native_backend_module(backend)
-
-        if mod.nil?
-          if backend
-            raise NotAvailable, "Requested backend #{backend.inspect} is not available or does not support shared libraries"
-          else
-            raise NotAvailable,
-              "No native tree-sitter backend is available for loading shared libraries. " \
-                "Available native backends (MRI, Rust, FFI, Java) require platform-specific setup. " \
-                "For pure-Ruby parsing, use backend-specific Language classes directly (e.g., Prism, Psych, Citrus)."
-          end
-        end
-
-        # Backend must implement .from_library; fallback to .from_path for older impls
-        # Include effective backend AND ENV vars in cache key since they affect loading
-        effective_b = TreeHaver.resolve_effective_backend(backend)
-        key = [effective_b, path, symbol, name, ENV["TREE_SITTER_LANG_SYMBOL"]]
-        LanguageRegistry.fetch(key) do
-          if mod::Language.respond_to?(:from_library)
-            mod::Language.from_library(path, symbol: symbol, name: name)
-          else
-            mod::Language.from_path(path)
-          end
-        end
-      end
-      # Alias for {from_library}
-      # @see from_library
-      alias_method :from_path, :from_library
-
-      # Dynamic helper to load a registered language by name
-      #
-      # After registering a language with {TreeHaver.register_language},
-      # you can load it using a method call. The appropriate backend will be
-      # used based on registration and current backend.
-      #
-      # @example With tree-sitter
-      #   TreeHaver.register_language(:toml, path: "/path/to/libtree-sitter-toml.so")
-      #   language = TreeHaver::Language.toml
-      #
-      # @example With both backends
-      #   TreeHaver.register_language(:toml,
-      #     path: "/path/to/libtree-sitter-toml.so", symbol: "tree_sitter_toml")
-      #   TreeHaver.register_language(:toml,
-      #     grammar_module: TomlRB::Document)
-      #   language = TreeHaver::Language.toml  # Uses appropriate grammar for active backend
-      #
-      # @param method_name [Symbol] the registered language name
-      # @param args [Array] positional arguments
-      # @param kwargs [Hash] keyword arguments
-      # @return [Language] loaded language handle
-      # @raise [NoMethodError] if the language name is not registered
-      def method_missing(method_name, *args, **kwargs, &block)
-        # Resolve only if the language name was registered
-        all_backends = TreeHaver.registered_language(method_name)
-        return super unless all_backends
-
-        # Check current backend
-        current_backend = TreeHaver.backend_module
-
-        # Determine which backend type to use
-        backend_type = if current_backend == Backends::Citrus
-          :citrus
-        elsif current_backend == Backends::Parslet
-          :parslet
-        else
-          :tree_sitter  # MRI, Rust, FFI, Java all use tree-sitter
-        end
-
-        # Get backend-specific registration
-        reg = all_backends[backend_type]
-
-        # If Citrus backend is active
-        if backend_type == :citrus
-          if reg && reg[:grammar_module]
-            return Backends::Citrus::Language.new(reg[:grammar_module])
-          end
-
-          # Fall back to error if no Citrus grammar registered
-          raise NotAvailable,
-            "Citrus backend is active but no Citrus grammar registered for :#{method_name}. " \
-              "Either register a Citrus grammar or use a tree-sitter backend. " \
-              "Registered backends: #{all_backends.keys.inspect}"
-        end
-
-        # If Parslet backend is active
-        if backend_type == :parslet
-          if reg && reg[:grammar_class]
-            return Backends::Parslet::Language.new(reg[:grammar_class])
-          end
-
-          # Fall back to error if no Parslet grammar registered
-          raise NotAvailable,
-            "Parslet backend is active but no Parslet grammar registered for :#{method_name}. " \
-              "Either register a Parslet grammar or use a tree-sitter backend. " \
-              "Registered backends: #{all_backends.keys.inspect}"
-        end
-
-        # For tree-sitter backends, try to load from path
-        # If that fails, fall back to Citrus if available
-        if reg && reg[:path]
-          path = kwargs[:path] || args.first || reg[:path]
-          # Symbol priority: kwargs override > registration > derive from method_name
-          symbol = if kwargs.key?(:symbol)
-            kwargs[:symbol]
-          elsif reg[:symbol]
-            reg[:symbol]
-          else
-            "tree_sitter_#{method_name}"
-          end
-          # Name priority: kwargs override > derive from symbol (strip tree_sitter_ prefix)
-          # Using symbol-derived name ensures ruby_tree_sitter gets the correct language name
-          # e.g., "toml" not "toml_both" when symbol is "tree_sitter_toml"
-          name = kwargs[:name] || symbol&.sub(/\Atree_sitter_/, "")
-
-          begin
-            return from_library(path, symbol: symbol, name: name)
-          rescue NotAvailable, ArgumentError, LoadError => e
-            # Tree-sitter failed to load - check for Citrus fallback
-            # Note: FFI::NotFoundError inherits from LoadError, so it's caught here too
-            handle_tree_sitter_load_failure(e, all_backends)
-          end
-        end
-
-        # No tree-sitter path registered - check for Citrus or Parslet fallback
-        # This enables auto-fallback when tree-sitter grammar is not installed
-        # but a pure Ruby grammar (Citrus or Parslet) is available.
-        # Only fall back when backend is :auto - explicit native backend requests should fail.
-        if TreeHaver.effective_backend == :auto
-          citrus_reg = all_backends[:citrus]
-          if citrus_reg && citrus_reg[:grammar_module]
-            return Backends::Citrus::Language.new(citrus_reg[:grammar_module])
-          end
-
-          parslet_reg = all_backends[:parslet]
-          if parslet_reg && parslet_reg[:grammar_class]
-            return Backends::Parslet::Language.new(parslet_reg[:grammar_class])
-          end
-        end
-
-        # No appropriate registration found
-        raise ArgumentError,
-          "No grammar registered for :#{method_name} compatible with #{backend_type} backend. " \
-            "Registered backends: #{all_backends.keys.inspect}"
-      end
-
-      # @api private
-      def respond_to_missing?(method_name, include_private = false)
-        !!TreeHaver.registered_language(method_name) || super
-      end
-
-      private
-
-      # Handle tree-sitter load failure with optional Citrus/Parslet fallback
-      #
-      # This handles cases where:
-      # - The .so file doesn't exist or can't be loaded (NotAvailable, LoadError)
-      # - FFI can't find required symbols like ts_parser_new (FFI::NotFoundError inherits from LoadError)
-      # - Invalid arguments were provided (ArgumentError)
-      #
-      # Fallback to Citrus/Parslet ONLY happens when:
-      # - The effective backend is :auto (user didn't explicitly request a native backend)
-      # - A Citrus or Parslet grammar is registered for the language
-      #
-      # If the user explicitly requested a native backend (:mri, :rust, :ffi, :java),
-      # we should NOT silently fall back to pure Ruby - that would violate the user's intent.
-      #
-      # @param error [Exception] the original error
-      # @param all_backends [Hash] all registered backends for the language
-      # @return [Backends::Citrus::Language, Backends::Parslet::Language] if fallback available and allowed
-      # @raise [Exception] re-raises original error if no fallback or fallback not allowed
-      # @api private
-      def handle_tree_sitter_load_failure(error, all_backends)
-        # Only fall back to pure Ruby when backend is :auto
-        # If user explicitly requested a native backend, respect that choice
-        effective = TreeHaver.effective_backend
-        if effective == :auto
-          citrus_reg = all_backends[:citrus]
-          if citrus_reg && citrus_reg[:grammar_module]
-            return Backends::Citrus::Language.new(citrus_reg[:grammar_module])
-          end
-
-          parslet_reg = all_backends[:parslet]
-          if parslet_reg && parslet_reg[:grammar_class]
-            return Backends::Parslet::Language.new(parslet_reg[:grammar_class])
-          end
-        end
-        # No pure Ruby fallback allowed or available, re-raise the original error
-        raise error
-      end
-    end
-  end
-end

data/lib/tree_haver/language_registry.rb

@@ -1,190 +0,0 @@
-# frozen_string_literal: true
-
-module TreeHaver
-  # Thread-safe language registrations and cache for loaded Language handles
-  #
-  # The LanguageRegistry provides two main functions:
-  # 1. **Registrations**: Store mappings from language names to backend-specific configurations
-  # 2. **Cache**: Memoize loaded Language objects to avoid repeated dlopen calls
-  #
-  # The registry supports multiple backends for the same language, allowing runtime
-  # switching, benchmarking, and fallback scenarios.
-  #
-  # == Supported Backend Types
-  #
-  # The registry is extensible and supports any backend type. Common types include:
-  #
-  # - `:tree_sitter` - Native tree-sitter grammars (.so files)
-  # - `:citrus` - Citrus PEG parser grammars (pure Ruby)
-  # - `:prism` - Ruby's Prism parser (Ruby source only)
-  # - `:psych` - Ruby's Psych parser (YAML only)
-  # - `:commonmarker` - Commonmarker gem (Markdown)
-  # - `:markly` - Markly gem (Markdown/GFM)
-  # - `:rbs` - RBS gem (RBS type signatures) - registered externally by rbs-merge
-  #
-  # External gems can register their own backend types using the same API.
-  #
-  # Registration structure:
-  # ```ruby
-  # @registrations = {
-  #   toml: {
-  #     tree_sitter: { path: "/path/to/lib.so", symbol: "tree_sitter_toml" },
-  #     citrus: { grammar_module: TomlRB::Document, gem_name: "toml-rb" }
-  #   },
-  #   ruby: {
-  #     prism: { backend_module: TreeHaver::Backends::Prism }
-  #   },
-  #   yaml: {
-  #     psych: { backend_module: TreeHaver::Backends::Psych }
-  #   },
-  #   markdown: {
-  #     commonmarker: { backend_module: TreeHaver::Backends::Commonmarker },
-  #     markly: { backend_module: TreeHaver::Backends::Markly }
-  #   },
-  #   rbs: {
-  #     rbs: { backend_module: Rbs::Merge::Backends::RbsBackend }  # External
-  #   }
-  # }
-  # ```
-  #
-  # @example Register tree-sitter grammar
-  # ```ruby
-  #   TreeHaver::LanguageRegistry.register(:toml, :tree_sitter,
-  #     path: "/path/to/lib.so", symbol: "tree_sitter_toml")
-  # ```
-  #
-  # @example Register Citrus grammar
-  # ```ruby
-  #   TreeHaver::LanguageRegistry.register(:toml, :citrus,
-  #     grammar_module: TomlRB::Document, gem_name: "toml-rb")
-  # ```
-  #
-  # @example Register a pure Ruby backend (internal or external)
-  # ```ruby
-  #   TreeHaver::LanguageRegistry.register(:rbs, :rbs,
-  #     backend_module: Rbs::Merge::Backends::RbsBackend,
-  #     gem_name: "rbs")
-  # ```
-  #
-  # @api private
-  module LanguageRegistry
-    @mutex = Mutex.new
-    @cache = {} # rubocop:disable ThreadSafety/MutableClassInstanceVariable
-    @registrations = {} # rubocop:disable ThreadSafety/MutableClassInstanceVariable
-
-    module_function
-
-    # Register a language for a specific backend
-    #
-    # Stores backend-specific configuration for a language. Multiple backends
-    # can be registered for the same language without conflict.
-    #
-    # @param name [Symbol, String] language identifier (e.g., :toml, :json, :ruby, :yaml, :rbs)
-    # @param backend_type [Symbol] backend type (:tree_sitter, :citrus, :prism, :psych, :commonmarker, :markly, or custom)
-    # @param config [Hash] backend-specific configuration
-    # @option config [String] :path tree-sitter library path (for tree-sitter backends)
-    # @option config [String] :symbol exported symbol name (for tree-sitter backends)
-    # @option config [Module] :grammar_module Citrus grammar module (for Citrus backend)
-    # @option config [Module] :backend_module backend module with Language/Parser classes (for pure Ruby backends)
-    # @option config [String] :gem_name gem name for error messages and availability checks
-    # @return [void]
-    # @example Register tree-sitter grammar
-    #   LanguageRegistry.register(:toml, :tree_sitter,
-    #     path: "/usr/local/lib/libtree-sitter-toml.so", symbol: "tree_sitter_toml")
-    # @example Register Citrus grammar
-    #   LanguageRegistry.register(:toml, :citrus,
-    #     grammar_module: TomlRB::Document, gem_name: "toml-rb")
-    # @example Register pure Ruby backend (external gem)
-    #   LanguageRegistry.register(:rbs, :rbs,
-    #     backend_module: Rbs::Merge::Backends::RbsBackend, gem_name: "rbs")
-    def register(name, backend_type, **config)
-      key = name.to_sym
-      backend_key = backend_type.to_sym
-
-      @mutex.synchronize do
-        @registrations[key] ||= {}
-        @registrations[key][backend_key] = config.compact
-      end
-      nil
-    end
-
-    # Fetch registration entries for a language
-    #
-    # Returns all backend-specific configurations for a language.
-    #
-    # @param name [Symbol, String] language identifier
-    # @param backend_type [Symbol, nil] optional backend type to filter by
-    # @return [Hash{Symbol => Hash}, Hash, nil] all backends or specific backend config
-    # @example Get all backends
-    #   entries = LanguageRegistry.registered(:toml)
-    #   # => {
-    #   #   tree_sitter: { path: "/usr/local/lib/libtree-sitter-toml.so", symbol: "tree_sitter_toml" },
-    #   #   citrus: { grammar_module: TomlRB::Document, gem_name: "toml-rb" }
-    #   # }
-    # @example Get specific backend
-    #   entry = LanguageRegistry.registered(:toml, :citrus)
-    #   # => { grammar_module: TomlRB::Document, gem_name: "toml-rb" }
-    def registered(name, backend_type = nil)
-      @mutex.synchronize do
-        lang_config = @registrations[name.to_sym]
-        return unless lang_config
-
-        if backend_type
-          lang_config[backend_type.to_sym]
-        else
-          lang_config
-        end
-      end
-    end
-
-    # Fetch a cached language by key or compute and store it
-    #
-    # This method provides thread-safe memoization for loaded Language objects.
-    # If the key exists in the cache, the cached value is returned immediately.
-    # Otherwise, the block is called to compute the value, which is then cached.
-    #
-    # @param key [Array] cache key, typically [path, symbol, name]
-    # @yieldreturn [Object] the computed language handle (called only on cache miss)
-    # @return [Object] the cached or computed language handle
-    # @example
-    #   language = LanguageRegistry.fetch(["/path/lib.so", "symbol", "toml"]) do
-    #     expensive_language_load_operation
-    #   end
-    def fetch(key)
-      @mutex.synchronize do
-        return @cache[key] if @cache.key?(key)
-        value = yield
-        @cache[key] = value
-      end
-    end
-
-    # Clear the language cache
-    #
-    # Removes all cached Language objects. The next call to {fetch} for any key
-    # will recompute the value. Does not clear registrations.
-    #
-    # @return [void]
-    # @example
-    #   LanguageRegistry.clear_cache!
-    def clear_cache!
-      @mutex.synchronize { @cache.clear }
-      nil
-    end
-
-    # Clear all registrations and cache
-    #
-    # Removes all language registrations and cached Language objects.
-    # Primarily used in tests to reset state between test cases.
-    #
-    # @return [void]
-    # @example
-    #   LanguageRegistry.clear
-    def clear
-      @mutex.synchronize do
-        @registrations.clear
-        @cache.clear
-      end
-      nil
-    end
-  end
-end

data/lib/tree_haver/library_path_utils.rb

@@ -1,80 +0,0 @@
-# frozen_string_literal: true
-
-module TreeHaver
-  # Utility methods for deriving tree-sitter symbol and language names from library paths
-  #
-  # This module provides consistent path parsing across all backends that load
-  # tree-sitter grammar libraries from shared object files (.so/.dylib/.dll).
-  #
-  # @example
-  #   TreeHaver::LibraryPathUtils.derive_symbol_from_path("/usr/lib/libtree-sitter-toml.so")
-  #   # => "tree_sitter_toml"
-  #
-  #   TreeHaver::LibraryPathUtils.derive_language_name_from_path("/usr/lib/libtree-sitter-toml.so")
-  #   # => "toml"
-  module LibraryPathUtils
-    module_function
-
-    # Derive the tree-sitter symbol name from a library path
-    #
-    # Symbol names are the exported C function names (e.g., "tree_sitter_toml")
-    # that return a pointer to the TSLanguage struct.
-    #
-    # Handles various naming conventions:
-    # - libtree-sitter-toml.so → tree_sitter_toml
-    # - libtree_sitter_toml.so → tree_sitter_toml
-    # - tree-sitter-toml.so → tree_sitter_toml
-    # - tree_sitter_toml.so → tree_sitter_toml
-    # - toml.so → tree_sitter_toml (assumes simple language name)
-    #
-    # @param path [String, nil] path like "/usr/lib/libtree-sitter-toml.so"
-    # @return [String, nil] symbol like "tree_sitter_toml", or nil if path is nil
-    def derive_symbol_from_path(path)
-      return unless path
-
-      # Extract filename without extension: "libtree-sitter-toml" or "toml"
-      filename = File.basename(path, ".*")
-
-      # Handle multi-part extensions like .so.0.24
-      filename = filename.sub(/\.so(\.\d+)*\z/, "")
-
-      # Match patterns and normalize to tree_sitter_<lang>
-      case filename
-      when /\Alib[-_]?tree[-_]sitter[-_](.+)\z/
-        "tree_sitter_#{Regexp.last_match(1).tr("-", "_")}"
-      when /\Atree[-_]sitter[-_](.+)\z/
-        "tree_sitter_#{Regexp.last_match(1).tr("-", "_")}"
-      else
-        # Assume filename is just the language name (e.g., "toml.so" -> "tree_sitter_toml")
-        # Also strip "lib" prefix if present (e.g., "libtoml.so" -> "tree_sitter_toml")
-        lang = filename.sub(/\Alib/, "").tr("-", "_")
-        "tree_sitter_#{lang}"
-      end
-    end
-
-    # Derive the language name from a library path
-    #
-    # Language names are the short identifiers (e.g., "toml", "json", "ruby")
-    # used by some backends (like tree_stump/Rust) to register grammars.
-    #
-    # @param path [String, nil] path like "/usr/lib/libtree-sitter-toml.so"
-    # @return [String, nil] language name like "toml", or nil if path is nil
-    def derive_language_name_from_path(path)
-      symbol = derive_symbol_from_path(path)
-      return unless symbol
-
-      # Strip the "tree_sitter_" prefix to get the language name
-      symbol.sub(/\Atree_sitter_/, "")
-    end
-
-    # Derive language name from a symbol
-    #
-    # @param symbol [String, nil] symbol like "tree_sitter_toml"
-    # @return [String, nil] language name like "toml", or nil if symbol is nil
-    def derive_language_name_from_symbol(symbol)
-      return unless symbol
-
-      symbol.sub(/\Atree_sitter_/, "")
-    end
-  end
-end