diver_down 0.0.1.alpha1

data/lib/diver_down/trace/call_stack.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+module DiverDown
+  module Trace
+    # To handle call stacks obtained by TracePoint more efficiently.
+    # TracePoint also acquires calls that are not trace targets, but for dependency extraction, we want to acquire only a list of targets.
+    # In this class, push/pop is performed on all call/return, but it should always be possible to trace back to the caller of the target dependency at high speed.
+    class CallStack
+      class StackEmptyError < RuntimeError; end
+
+      # @attr_reader stack [Integer] stack size
+      attr_reader :stack_size
+
+      def initialize
+        @stack_size = 0
+        @stack = {}
+      end
+
+      # @return [Boolean]
+      def empty?
+        @stack.empty?
+      end
+
+      # @return [Array<Object>]
+      def stack
+        @stack.values
+      end
+
+      # @param context [Object, nil] User defined stack context.
+      # @return [void]
+      def push(context = nil)
+        @stack_size += 1
+        @stack[@stack_size] = context unless context.nil?
+      end
+
+      # @return [void]
+      def pop
+        raise StackEmptyError if @stack_size.zero?
+
+        @stack.delete(@stack_size) if @stack.key?(@stack_size)
+        @stack_size -= 1
+      end
+    end
+  end
+end

data/lib/diver_down/trace/ignored_method_ids.rb

@@ -0,0 +1,136 @@
+# frozen_string_literal: true
+
+module DiverDown
+  module Trace
+    class IgnoredMethodIds
+      def initialize(ignored_methods)
+        # Ignore all methods in the module
+        # Hash{ Module => Boolean }
+        @ignored_modules = {}
+
+        # Ignore all methods in the class
+        # Hash{ Module => Hash{ Symbol => Boolean } }
+        @ignored_class_method_id = Hash.new { |h, k| h[k] = {} }
+
+        # Ignore all methods in the instance
+        # Hash{ Module => Hash{ Symbol => Boolean } }
+        @ignored_instance_method_id = Hash.new { |h, k| h[k] = {} }
+
+        ignored_methods.each do |ignored_method|
+          if ignored_method.include?('.')
+            # instance method
+            class_name, method_id = ignored_method.split('.')
+            mod = DiverDown::Helper.constantize(class_name)
+            @ignored_class_method_id[mod][method_id.to_sym] = true
+          elsif ignored_method.include?('#')
+            # class method
+            class_name, method_id = ignored_method.split('#')
+            mod = DiverDown::Helper.constantize(class_name)
+            @ignored_instance_method_id[mod][method_id.to_sym] = true
+          else
+            # module
+            mod = DiverDown::Helper.constantize(ignored_method)
+            @ignored_modules[mod] = true
+          end
+        end
+      end
+
+      # @param mod [Module]
+      # @param is_class [Boolean] class is true, instance is false
+      # @param method_id [Symbol]
+      # @return [Boolean]
+      def ignored?(mod, is_class, method_id)
+        ignored_module?(mod) || ignored_method?(mod, is_class, method_id)
+      end
+
+      private
+
+      def ignored_module?(mod)
+        unless @ignored_modules.key?(mod)
+          dig_superclass(mod)
+        end
+
+        @ignored_modules.fetch(mod)
+      end
+
+      def ignored_method?(mod, is_class, method_id)
+        store = if is_class
+                  # class methods
+                  @ignored_class_method_id
+                else
+                  # instance methods
+                  @ignored_instance_method_id
+                end
+
+        begin
+          dig_superclass_method_id(store, mod, method_id) unless store[mod].key?(method_id)
+        rescue TypeError => e
+          # https://github.com/ruby/ruby/blob/f42164e03700469a7000b4f00148a8ca01d75044/object.c#L2232
+          return false if e.message == 'uninitialized class'
+
+          raise
+        end
+
+        store.fetch(mod).fetch(method_id)
+      end
+
+      def dig_superclass(mod)
+        unless DiverDown::Helper.class?(mod)
+          # NOTE: Do not lookup the ancestors if module given because of the complexity of implementation
+          @ignored_modules[mod] = false
+          return
+        end
+
+        stack = []
+        current = mod
+        ignored = nil
+
+        until current.nil?
+          if @ignored_modules.key?(current)
+            ignored = @ignored_modules.fetch(current)
+            break
+          else
+            stack.push(current)
+            current = current.superclass
+          end
+        end
+
+        # Convert nil to boolean
+        ignored = !!ignored
+
+        stack.each do
+          @ignored_modules[_1] = ignored
+        end
+      end
+
+      def dig_superclass_method_id(store, mod, method_id)
+        unless DiverDown::Helper.class?(mod)
+          # NOTE: Do not lookup the ancestors if module given because of the complexity of implementation
+          store[mod][method_id] = false
+          return
+        end
+
+        stack = []
+        current = mod
+        ignored = nil
+
+        until current.nil?
+          if store[current].key?(method_id)
+            ignored = store[current].fetch(method_id)
+            break
+          else
+            stack.push(current)
+            current = current.superclass
+          end
+        end
+
+        # Convert nil to boolean
+        ignored = !!ignored
+
+        stack.each do
+          store[_1][method_id] = ignored
+        end
+      end
+    end
+  end
+end

data/lib/diver_down/trace/module_set/array_module_set.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module DiverDown
+  module Trace
+    class ModuleSet
+      class ArrayModuleSet
+        # @param [Array<Module, String>, #each] modules
+        def initialize(modules)
+          @map = {}
+
+          modules.each do
+            mod = if DiverDown::Helper.module?(_1)
+                    _1
+                  else
+                    # constantize if it is a string
+                    DiverDown::Helper.constantize(_1)
+                  end
+
+            @map[mod] = true
+          end
+        end
+
+        # @param [Module] mod
+        # @return [Boolean, nil]
+        def include?(mod)
+          @map[mod]
+        end
+      end
+    end
+  end
+end

data/lib/diver_down/trace/module_set/const_source_location_module_set.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module DiverDown
+  module Trace
+    class ModuleSet
+      class ConstSourceLocationModuleSet
+        # @param [Array<String>, Set<String>] paths
+        def initialize(paths: [])
+          @paths = paths.to_set
+        end
+
+        # @param [Module] mod
+        # @return [Boolean]
+        def include?(mod)
+          module_name = DiverDown::Helper.normalize_module_name(mod)
+
+          path, = begin
+            Object.const_source_location(module_name)
+          rescue NameError, TypeError
+            nil
+          end
+
+          path && @paths.include?(path)
+        end
+      end
+    end
+  end
+end

data/lib/diver_down/trace/module_set.rb

@@ -0,0 +1,78 @@
+# frozen_string_literal: true
+
+module DiverDown
+  module Trace
+    # A class to quickly determine if a TracePoint is a module to be traced.
+    class ModuleSet
+      require 'diver_down/trace/module_set/array_module_set'
+      require 'diver_down/trace/module_set/const_source_location_module_set'
+
+      # @param [Array<Module, String>, Set<Module, String>, nil] modules
+      # @param [Array<String>, Set<String>, nil] include
+      def initialize(modules: nil, paths: nil)
+        @cache = {}
+        @array_module_set = DiverDown::Trace::ModuleSet::ArrayModuleSet.new(modules) unless modules.nil?
+        @const_source_location_module_set = DiverDown::Trace::ModuleSet::ConstSourceLocationModuleSet.new(paths:) unless paths.nil?
+      end
+
+      # @param [Module] mod
+      # @return [Boolean]
+      def include?(mod)
+        unless @cache.key?(mod)
+          if DiverDown::Helper.class?(mod)
+            # class
+            begin
+              dig_superclass(mod)
+            rescue TypeError => e
+              # https://github.com/ruby/ruby/blob/f42164e03700469a7000b4f00148a8ca01d75044/object.c#L2232
+              return false if e.message == 'uninitialized class'
+
+              raise
+            end
+          else
+            # module
+            @cache[mod] = !!_include?(mod)
+          end
+        end
+
+        @cache.fetch(mod)
+      end
+
+      private
+
+      def _include?(mod)
+        return true if @array_module_set.nil? && @const_source_location_module_set.nil?
+
+        @array_module_set&.include?(mod) ||
+          @const_source_location_module_set&.include?(mod)
+      end
+
+      def dig_superclass(mod)
+        stack = []
+        current = mod
+        included = nil
+
+        until current.nil?
+          if @cache.key?(current)
+            included = @cache.fetch(current)
+            break
+          else
+            stack.push(current)
+            included = _include?(current)
+
+            break if included
+
+            current = current.superclass
+          end
+        end
+
+        # Convert nil to boolean
+        included = !!included
+
+        stack.each do
+          @cache[_1] = included unless @cache.key?(_1)
+        end
+      end
+    end
+  end
+end

data/lib/diver_down/trace/redefine_ruby_methods.rb

@@ -0,0 +1,64 @@
+# frozen_string_literal: true
+
+module DiverDown
+  module Trace
+    # Tracepoint traces only ruby-lang calls because tracing c-lang calls is very slow.
+    # In this case, methods such as new will not be traceable under normal circumstances, so at a minimum, redefine them in ruby and hack them so that they can be traced.
+    #
+    # Do not use this in a production environment.
+    module RedefineRubyMethods
+      DEFAULT_METHODS = {
+        Module => {
+          singleton: %i[name],
+          instance: [],
+        },
+        Class => {
+          singleton: %i[name allocate new],
+          instance: [],
+        },
+      }.freeze
+
+      class RubyMethods < Module
+        # @param class_method_names [Array<Symbol>]
+        # @param instance_method_names [Array<Symbol>]
+        def initialize(class_method_names, instance_method_names)
+          unless class_method_names.empty?
+            def_class_methods = class_method_names.inject(+'') do |buf, method_name|
+              buf << <<~METHOD
+                def #{method_name}(...)
+                  super
+                end
+
+              METHOD
+            end
+
+            mod = Module.new
+            mod.module_eval(def_class_methods, __FILE__, __LINE__)
+            const_set(:CLASS_METHODS, mod)
+          end
+
+          unless instance_method_names.empty?
+            instance_method_names.inject(+'') do |_buf, method_name|
+              define_method(method_name) do |*_args, **_options|
+                super
+              end
+            end
+          end
+        end
+
+        # @param base [Module]
+        def prepend_features(base)
+          base.singleton_class.prepend(const_get(:CLASS_METHODS)) if const_defined?(:CLASS_METHODS)
+          super
+        end
+      end
+
+      # @param map [Hash{ Class => Hash{ singleton: Array<String>, instance: Array<String> } }]
+      def self.redefine_c_methods(map = DEFAULT_METHODS)
+        map.each do |m, method_names|
+          m.prepend(RubyMethods.new(method_names[:singleton], method_names[:instance]))
+        end
+      end
+    end
+  end
+end

data/lib/diver_down/trace/tracer/session.rb

@@ -0,0 +1,121 @@
+# frozen_string_literal: true
+
+module DiverDown
+  module Trace
+    class Tracer
+      class Session
+        class NotStarted < StandardError; end
+
+        attr_reader :definition
+
+        # @param [DiverDown::Trace::ModuleSet, nil] module_set
+        # @param [DiverDown::Trace::IgnoredMethodIds, nil] ignored_method_ids
+        # @param [Set<String>, nil] target_file_set
+        # @param [#call, nil] filter_method_id_path
+        def initialize(module_set: DiverDown::Trace::ModuleSet.new, ignored_method_ids: nil, target_file_set: nil, filter_method_id_path: nil, definition: DiverDown::Definition.new)
+          @module_set = module_set
+          @ignored_method_ids = ignored_method_ids
+          @target_file_set = target_file_set
+          @filter_method_id_path = filter_method_id_path
+          @definition = definition
+          @trace_point = build_trace_point
+        end
+
+        # @return [void]
+        def start
+          @trace_point.enable
+        end
+
+        # @return [void]
+        def stop
+          @trace_point.disable
+        end
+
+        private
+
+        def build_trace_point
+          call_stack = DiverDown::Trace::CallStack.new
+          ignored_stack_size = nil
+
+          TracePoint.new(*DiverDown::Trace::Tracer.trace_events) do |tp|
+            # Skip the trace of the library itself
+            next if tp.path&.start_with?(DiverDown::LIB_DIR)
+            next if TracePoint == tp.defined_class
+
+            case tp.event
+            when :call, :c_call
+              # puts "#{tp.method_id} #{tp.path}:#{tp.lineno}"
+              mod = DiverDown::Helper.resolve_module(tp.self)
+              source_name = DiverDown::Helper.normalize_module_name(mod) if !mod.nil? && @module_set.include?(mod)
+              already_ignored = !ignored_stack_size.nil? # If the current method_id is ignored
+              current_ignored = !@ignored_method_ids.nil? && @ignored_method_ids.ignored?(mod, DiverDown::Helper.module?(tp.self), tp.method_id)
+              pushed = false
+
+              if !source_name.nil? && !(already_ignored || current_ignored)
+                source = @definition.find_or_build_source(source_name)
+
+                # If the call stack contains a call to a module to be traced
+                # `@ignored_call_stack` is not nil means the call stack contains a call to a module to be ignored
+                unless call_stack.empty?
+                  # Add dependency to called source
+                  called_stack_context = call_stack.stack[-1]
+                  called_source = called_stack_context.source
+                  dependency = called_source.find_or_build_dependency(source_name)
+
+                  # `dependency.nil?` means source_name equals to called_source.source.
+                  # self-references are not tracked because it is not "dependency".
+                  if dependency
+                    context = DiverDown::Helper.module?(tp.self) ? 'class' : 'instance'
+                    method_id = dependency.find_or_build_method_id(name: tp.method_id, context:)
+                    method_id_path = "#{called_stack_context.caller_location.path}:#{called_stack_context.caller_location.lineno}"
+                    method_id_path = @filter_method_id_path.call(method_id_path) if @filter_method_id_path
+                    method_id.add_path(method_id_path)
+                  end
+                end
+
+                # `caller_location` is nil if it is filtered by target_files
+                caller_location = find_neast_caller_location
+
+                if caller_location
+                  pushed = true
+
+                  call_stack.push(
+                    StackContext.new(
+                      source:,
+                      method_id: tp.method_id,
+                      caller_location:
+                    )
+                  )
+                end
+              end
+
+              call_stack.push unless pushed
+
+              # If a value is already stored, it means that call stack already determined to be ignored at the shallower call stack size.
+              # Since stacks deeper than the shallowest stack size are ignored, priority is given to already stored values.
+              if !already_ignored && current_ignored
+                ignored_stack_size = call_stack.stack_size
+              end
+            when :return, :c_return
+              ignored_stack_size = nil if ignored_stack_size == call_stack.stack_size
+              call_stack.pop
+            end
+          rescue StandardError
+            tp.disable
+            raise
+          end
+        end
+
+        def find_neast_caller_location
+          return caller_locations(2, 2)[0] if @target_file_set.nil?
+
+          Thread.each_caller_location do
+            return _1 if @target_file_set.include?(_1.path)
+          end
+
+          nil
+        end
+      end
+    end
+  end
+end

data/lib/diver_down/trace/tracer.rb

@@ -0,0 +1,96 @@
+# frozen_string_literal: true
+
+module DiverDown
+  module Trace
+    class Tracer
+      StackContext = Data.define(
+        :source,
+        :method_id,
+        :caller_location
+      )
+
+      # @return [Array<Symbol>]
+      def self.trace_events
+        @trace_events || %i[call c_call return c_return]
+      end
+
+      # @param events [Array<Symbol>]
+      class << self
+        attr_writer :trace_events
+      end
+
+      # @param module_set [DiverDown::Trace::ModuleSet, Array<Module, String>]
+      # @param target_files [Array<String>, nil] if nil, trace all files
+      # @param ignored_method_ids [Array<String>]
+      # @param filter_method_id_path [#call, nil] filter method_id.path
+      # @param module_set [DiverDown::Trace::ModuleSet, nil] for optimization
+      def initialize(module_set: {}, target_files: nil, ignored_method_ids: nil, filter_method_id_path: nil)
+        if target_files && !target_files.all? { Pathname.new(_1).absolute? }
+          raise ArgumentError, "target_files must be absolute path(#{target_files})"
+        end
+
+        @module_set = if module_set.is_a?(DiverDown::Trace::ModuleSet)
+                        module_set
+                      elsif module_set.is_a?(Hash)
+                        DiverDown::Trace::ModuleSet.new(**module_set)
+                      else
+                        raise ArgumentError, <<~MSG
+                          Given invalid module_set. #{module_set}"
+
+                          Available types are:
+
+                          Hash{
+                            modules: Array<Module, String> | Set<Module, String> | nil
+                            paths: Array<String> | Set<String> | nil
+                          } | DiverDown::Trace::ModuleSet
+                        MSG
+                      end
+
+        @ignored_method_ids = if ignored_method_ids.is_a?(DiverDown::Trace::IgnoredMethodIds)
+                                ignored_method_ids
+                              elsif !ignored_method_ids.nil?
+                                DiverDown::Trace::IgnoredMethodIds.new(ignored_method_ids)
+                              end
+
+        @target_file_set = target_files&.to_set
+        @filter_method_id_path = filter_method_id_path
+      end
+
+      # Trace the call stack of the block and build the definition
+      #
+      # @param title [String]
+      # @param definition_group [String, nil]
+      #
+      # @return [DiverDown::Definition]
+      def trace(title: SecureRandom.uuid, definition_group: nil, &)
+        session = new_session(title:, definition_group:)
+        session.start
+
+        yield
+
+        session.stop
+        session.definition
+      ensure
+        # Ensure to stop the session
+        session&.stop
+      end
+
+      # @param title [String]
+      # @param definition_group [String, nil]
+      #
+      # @return [TracePoint]
+      def new_session(title: SecureRandom.uuid, definition_group: nil)
+        DiverDown::Trace::Tracer::Session.new(
+          module_set: @module_set,
+          ignored_method_ids: @ignored_method_ids,
+          target_file_set: @target_file_set,
+          filter_method_id_path: @filter_method_id_path,
+          definition: DiverDown::Definition.new(
+            title:,
+            definition_group:
+          )
+        )
+      end
+    end
+  end
+end

data/lib/diver_down/trace.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+require 'active_support/inflector'
+
+module DiverDown
+  module Trace
+    require 'diver_down/trace/tracer'
+    require 'diver_down/trace/tracer/session'
+    require 'diver_down/trace/call_stack'
+    require 'diver_down/trace/module_set'
+    require 'diver_down/trace/redefine_ruby_methods'
+    require 'diver_down/trace/ignored_method_ids'
+
+    @trace_events = %i[
+      call c_call return c_return
+    ]
+
+    # Trace only Ruby-implemented methods because tracing C-implemented methods is very slow
+    # Override Ruby only with the minimal set of methods needed to trace dependencies.
+    #
+    # @return [void]
+    def self.trace_only_ruby_world!(map = DiverDown::Trace::RedefineRubyMethods::DEFAULT_METHODS)
+      DiverDown::Trace::Tracer.trace_events = %i[call return]
+      DiverDown::Trace::RedefineRubyMethods.redefine_c_methods(map)
+    end
+  end
+end

data/lib/diver_down/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module DiverDown
+  VERSION = '0.0.1.alpha1'
+end