diver_down 0.0.1.alpha1 → 0.0.1.alpha3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: be24104efe3c4cc931f29552c5922681622e5efcc29d74edd26ea0058d68823b
-  data.tar.gz: 9813779ca1e2451ed7cb97f983e198c2f749fc2f3dc88b54db0f33813e938255
+  metadata.gz: 4a41c67665f8ed7f67dc439661e5ec64ef575663daefad90bfdedaddc62a0f5f
+  data.tar.gz: 0c4e4bbc20cc18aa0b2ce247eeede23a08e20ca0ff5105815cc1e45148e59116
 SHA512:
-  metadata.gz: 583e816d1ff590d7e42e5c02f3546e67802ea1046c5bb8183e973cf65ae1ce37b2c7e1549578b4d848383ff2a5affd1f51455ad78cd3799e73cfecbc40b1c3ec
-  data.tar.gz: 150a56d11ff08e5f0a43e5483c3d5d59a6ac94dde414af484183a4b6b265c8310b8583a10488e28b1bf3f5b6a9bd205e39e2b77b5a15666bf1b03c7958d94ca8
+  metadata.gz: 82c2e82cd6a89e009630a43052e12df11e77dcdb01adeedccf06f48a7954cf7f29f7543a0c99c79c65b51b8a82933e8d213a70a14394d4b5015f3778b1a34b80
+  data.tar.gz: 25e11165390606b135111e238027294472ac3773d7f82e4495c0a100092fd28f700c25b0be10fc49e43f68f8e818bb4a15606f1f7bad7a87ecb2f3f340caf08a

data/README.md

@@ -1,11 +1,13 @@
 # DiverDown
 
-`divertdown` is a tool that dynamically analyzes application dependencies and creates a dependency map.
-This tool was created to analyze Ruby applications for use in large-scale refactoring such as moduler monolith.
+`DiverDown` is a tool designed to dynamically analyze application dependencies and generate a comprehensive dependency map. It is particularly useful for analyzing Ruby applications, aiding significantly in large-scale refactoring projects or transitions towards a modular monolith architecture.
 
-The results of the analysis can be viewed the dependency map graph, and and since the file-by-file association can be categorized into specific groups, you can deepen your architectural consideration.
+## Features
 
-## Installation
+- **Dependency Mapping**: Analyze and generate an application dependencies.
+- **Module Categorization**: Organizes file-by-file associations into specific groups, facilitating deeper modular monolith architectural analysis and understanding.
+
+## Getting Started
 
 Add this line to your application's Gemfile:
 
@@ -27,7 +29,7 @@ Or install it yourself as:
 
 ### `DiverDown::Trace`
 
-Analyzes the processing of ruby code and outputs the analysis results as `DiverDown::Definition`.
+The `DiverDown::Trace` module analyzes the execution of Ruby code and outputs the results as `DiverDown::Definition` objects.
 
 ```ruby
 tracer = DiverDown::Trace::Tracer.new
@@ -57,8 +59,49 @@ tracer.trace(title: 'title', definition_group: 'group name') do
 end
 ```
 
-The analysis results should be output to a specific directory.
-Files saved in `.msgpack`, `.json`, or `.yaml` can be read by `DiverDown::Web`.
+**Options**
+
+When analyzing user applications, it is recommended to specify the option.
+
+|name|type|description|example|default|
+| --- | --- | --- | --- | --- |
+| `module_set` | Hash{ <br>  modules: Array<Module, String> \| Set<Module, String> \| nil,<br>  paths: Array\<String> \| Set\<String> \| nil <br>}<br>\| DiverDown::Trace::ModuleSet | Specify the class/module to be included in the analysis results.<br><br>If you know the module name:<br>`{ modules: [ModA, ModB] }`<br><br>If you know module path:<br>`{ paths: ['/path/to/rails/app/models/mod_a.rb'] }` | `{ paths: Dir["app/**/*.rb"] }` | `nil`. All class/modul are target. |
+| `caller_paths` | `Array<String> \| nil` | Specifies a list of allowed paths as caller paths. By specifying the user application path in this list of paths and excluding paths under the gem, the caller path is identified back to the user application path. | `Dir["app/**/*.rb"]` | `nil`. All paths are target. |
+| `filter_method_id_path` | `#call \| nil` | lambda to convert the caller path. | `->(path) { path.remove(Rails.root) }` | `nil`. No conversion. |
+
+**Example**
+
+```ruby
+# Your rails application paths
+application_paths = [
+  *Dir['app/**/*.rb'], 
+  *Dir['lib/**/*.rb'],
+].map { File.expand_path(_1) }
+
+ignored_application_paths = [
+  'app/models/application_record.rb',
+].map { File.expand_path(_1) }
+
+module_set = DiverDown::Trace::ModuleSet.new(modules: modules - ignored_modules)
+
+filter_method_id_path = ->(path) { path.remove("#{Rails.root}/") }
+
+tracer = DiverDown::Trace::Tracer.new(
+  caller_paths: application_paths,
+  module_set: {
+    paths: (application_paths - ignored_application_paths)
+  },
+  filter_method_id_path:
+)
+
+definition = tracer.trace do
+  # do something
+end
+```
+
+#### Output Results
+
+The analysis results are intended to be saved to a specific directory in either `.json` or `.yaml` format. These files are compatible with `DiverDown::Web`, which can read and display the results.
 
 ```ruby
 dir = 'tmp/diver_down'
@@ -67,24 +110,18 @@ definition = tracer.trace do
   # do something
 end
 
-File.binwrite(File.join(dir, "#{definition.title}.msgpack"), definition.to_msgpack)
 File.write(File.join(dir, "#{definition.title}.json"), definition.to_h.to_json)
 File.write(File.join(dir, "#{definition.title}.yaml"), definition.to_h.to_yaml)
 ```
 
-**Options**
-
-TODO
-
 ### `DiverDown::Web`
 
 View the analysis results in a browser.
 
-This gem is designed to consider large application with a modular monolithic architecture.
-Each file in the analysis can be specified to belong to a module you specify on the browser.
+This gem is specifically designed to analyze large applications with a modular monolithic architecture. It allows users to categorize each analyzed file into specified modules directly through the web interface.
 
-- `--definition-dir` specifies the directory where the analysis results are stored.
-- `--module-store-path` will store the results specifying which module each file belongs to. If not specified, the specified results are stored in tempfile.
+- `--definition-dir` Specifies the directory where the analysis results are stored.
+- `--module-store-path` Designates a path to save the results that include details on which module each file belongs to. If this option is not specified, the results will be temporarily stored in a default temporary file.
 
 ```sh
 bundle exec diver_down_web --definition-dir tmp/diver_down --module-store-path tmp/module_store.yml
@@ -103,7 +140,7 @@ open http://localhost:8080
   - Ruby: `bundle exec rspec`, `bundle exec rubocop`
   - TypeScript: `pnpm run test`, `pnpm run lint`
 
-### Development DiverDown::Web
+### Development `DiverDown::Web`
 
 If you want to develop `DiverDown::Web` locally, set up a server for development.
 
@@ -114,8 +151,6 @@ $ pnpm run dev
 
 # Start server for backend
 $ bundle install
-# DIVER_DOWN_DIR specifies the directory where the analysis results are stored.
-# DIVER_DOWN_MODULE_STORE specifies a yaml file that defines which module the file belongs to, but this file is newly created, so it works even if the file does not exist.
 $ DIVER_DOWN_DIR=/path/to/definitions_dir DIVER_DOWN_MODULE_STORE=/path/to/module_store.yml bundle exec puma
 ```
 

data/lib/diver_down/definition.rb

@@ -79,11 +79,6 @@ module DiverDown
       }
     end
 
-    # @return [String]
-    def to_msgpack
-      MessagePack.pack(to_h)
-    end
-
     # @param other [Object, DiverDown::Definition::Source]
     # @return [Boolean]
     def ==(other)

data/lib/diver_down/helper.rb

@@ -62,20 +62,5 @@ module DiverDown
     def self.constantize(str)
       ::ActiveSupport::Inflector.constantize(str)
     end
-
-    # @param hash [Hash]
-    # @return [Hash]
-    def self.deep_symbolize_keys(object)
-      case object
-      when Hash
-        object.each_with_object({}) do |(key, value), memo|
-          memo[key.to_sym] = deep_symbolize_keys(value)
-        end
-      when Array
-        object.map { deep_symbolize_keys(_1) }
-      else
-        object
-      end
-    end
   end
 end

data/lib/diver_down/trace/call_stack.rb

@@ -12,32 +12,47 @@ module DiverDown
       attr_reader :stack_size
 
       def initialize
+        @ignored_stack_size = nil
         @stack_size = 0
-        @stack = {}
+        @context_stack = {}
       end
 
       # @return [Boolean]
-      def empty?
-        @stack.empty?
+      def empty_context_stack?
+        @context_stack.empty?
+      end
+
+      # @return [Array<Integer>]
+      def context_stack_size
+        @context_stack.keys
       end
 
       # @return [Array<Object>]
-      def stack
-        @stack.values
+      def context_stack
+        @context_stack.values
       end
 
       # @param context [Object, nil] User defined stack context.
       # @return [void]
-      def push(context = nil)
+      def push(context = nil, ignored: false)
         @stack_size += 1
-        @stack[@stack_size] = context unless context.nil?
+        @context_stack[@stack_size] = context unless context.nil?
+        @ignored_stack_size ||= @stack_size if ignored
+      end
+
+      # If a stack is not already a tracing target, some conditions are unnecessary so that it can be easily ignored.
+      #
+      # @return [Boolean]
+      def ignored?
+        !@ignored_stack_size.nil?
       end
 
       # @return [void]
       def pop
         raise StackEmptyError if @stack_size.zero?
 
-        @stack.delete(@stack_size) if @stack.key?(@stack_size)
+        @context_stack.delete(@stack_size) if @context_stack.key?(@stack_size)
+        @ignored_stack_size = nil if @ignored_stack_size && @ignored_stack_size == @stack_size
         @stack_size -= 1
       end
     end

data/lib/diver_down/trace/session.rb

@@ -0,0 +1,147 @@
+# frozen_string_literal: true
+
+module DiverDown
+  module Trace
+    class Session
+      StackContext = Data.define(
+        :source,
+        :method_id,
+        :path,
+        :lineno
+      )
+
+      attr_reader :definition
+
+      # @param [DiverDown::Trace::ModuleSet, nil] module_set
+      # @param [DiverDown::Trace::IgnoredMethodIds, nil] ignored_method_ids
+      # @param [Set<String>, nil] List of paths to finish traversing when searching for a caller. If nil, all paths are finished.
+      # @param [#call, nil] filter_method_id_path
+      def initialize(module_set: DiverDown::Trace::ModuleSet.new, ignored_method_ids: nil, caller_paths: nil, filter_method_id_path: nil, definition: DiverDown::Definition.new)
+        @module_set = module_set
+        @ignored_method_ids = ignored_method_ids
+        @caller_paths = caller_paths
+        @filter_method_id_path = filter_method_id_path
+        @definition = definition
+        @trace_point = build_trace_point
+      end
+
+      # @return [void]
+      def start
+        @trace_point.enable
+      end
+
+      # @return [void]
+      def stop
+        @trace_point.disable
+      end
+
+      private
+
+      def build_trace_point
+        call_stack = DiverDown::Trace::CallStack.new
+
+        TracePoint.new(*DiverDown::Trace::Tracer.trace_events) do |tp|
+          # Skip the trace of the library itself
+          next if tp.path&.start_with?(DiverDown::LIB_DIR)
+          next if TracePoint == tp.defined_class
+
+          case tp.event
+          when :call, :c_call
+            # puts "#{tp.method_id} #{tp.path}:#{tp.lineno}"
+            if call_stack.ignored?
+              call_stack.push
+              next
+            end
+
+            mod = DiverDown::Helper.resolve_module(tp.self)
+
+            if mod.nil?
+              call_stack.push
+              next
+            end
+
+            if !@ignored_method_ids.nil? && @ignored_method_ids.ignored?(mod, DiverDown::Helper.module?(tp.self), tp.method_id)
+              # If this method is ignored, the call stack is ignored until the method returns.
+              call_stack.push(ignored: true)
+              next
+            end
+
+            source_name = DiverDown::Helper.normalize_module_name(mod) if @module_set.include?(mod)
+            pushed = false
+
+            unless source_name.nil?
+              # If the call stack contains a call to a module to be traced
+              # `@ignored_call_stack` is not nil means the call stack contains a call to a module to be ignored
+              unless call_stack.empty_context_stack?
+                # Add dependency to called source
+                called_stack_context = call_stack.context_stack[-1]
+                called_source = called_stack_context.source
+                dependency = called_source.find_or_build_dependency(source_name)
+
+                # `dependency.nil?` means source_name equals to called_source.source.
+                # self-references are not tracked because it is not "dependency".
+                if dependency
+                  context = DiverDown::Helper.module?(tp.self) ? 'class' : 'instance'
+                  method_id = dependency.find_or_build_method_id(name: tp.method_id, context:)
+                  method_id_path = "#{called_stack_context.path}:#{called_stack_context.lineno}"
+                  method_id_path = @filter_method_id_path.call(method_id_path) if @filter_method_id_path
+                  method_id.add_path(method_id_path)
+                end
+              end
+
+              # Search is a heavy process and should be terminated early.
+              # The position of the most recently found caller or the start of trace is used as the maximum value.
+              maximum_back_stack_size = if call_stack.empty_context_stack?
+                                          call_stack.stack_size
+                                        else
+                                          call_stack.stack_size - call_stack.context_stack_size[-1]
+                                        end
+
+              caller_location = find_neast_caller_location(maximum_back_stack_size)
+
+              # `caller_location` is nil if it is filtered by caller_paths
+              if caller_location
+                pushed = true
+                source = @definition.find_or_build_source(source_name)
+
+                call_stack.push(
+                  StackContext.new(
+                    source:,
+                    method_id: tp.method_id,
+                    path: caller_location.path,
+                    lineno: caller_location.lineno
+                  )
+                )
+              end
+            end
+
+            call_stack.push unless pushed
+          when :return, :c_return
+            call_stack.pop
+          end
+        rescue StandardError
+          tp.disable
+          raise
+        end
+      end
+
+      def find_neast_caller_location(stack_size)
+        excluded_frame_size = 1 # Ignore neast caller because it is stack of #find_neast_caller_location.
+        finished_frame_size = excluded_frame_size + stack_size + 1
+        frame_pos = 1
+
+        # If @caller_paths is nil, return the caller location.
+        return caller_locations(excluded_frame_size + 1, excluded_frame_size + 1)[0] if @caller_paths.nil?
+
+        Thread.each_caller_location do
+          break if finished_frame_size < frame_pos
+          return _1 if @caller_paths.include?(_1.path)
+
+          frame_pos += 1
+        end
+
+        nil
+      end
+    end
+  end
+end

data/lib/diver_down/trace/tracer.rb

@@ -3,12 +3,6 @@
 module DiverDown
   module Trace
     class Tracer
-      StackContext = Data.define(
-        :source,
-        :method_id,
-        :caller_location
-      )
-
       # @return [Array<Symbol>]
       def self.trace_events
         @trace_events || %i[call c_call return c_return]
@@ -20,13 +14,13 @@ module DiverDown
       end
 
       # @param module_set [DiverDown::Trace::ModuleSet, Array<Module, String>]
-      # @param target_files [Array<String>, nil] if nil, trace all files
+      # @param caller_paths [Array<String>, nil] if nil, trace all files
       # @param ignored_method_ids [Array<String>]
       # @param filter_method_id_path [#call, nil] filter method_id.path
       # @param module_set [DiverDown::Trace::ModuleSet, nil] for optimization
-      def initialize(module_set: {}, target_files: nil, ignored_method_ids: nil, filter_method_id_path: nil)
-        if target_files && !target_files.all? { Pathname.new(_1).absolute? }
-          raise ArgumentError, "target_files must be absolute path(#{target_files})"
+      def initialize(module_set: {}, caller_paths: nil, ignored_method_ids: nil, filter_method_id_path: nil)
+        if caller_paths && !caller_paths.all? { Pathname.new(_1).absolute? }
+          raise ArgumentError, "caller_paths must be absolute path(#{caller_paths})"
         end
 
         @module_set = if module_set.is_a?(DiverDown::Trace::ModuleSet)
@@ -52,7 +46,7 @@ module DiverDown
                                 DiverDown::Trace::IgnoredMethodIds.new(ignored_method_ids)
                               end
 
-        @target_file_set = target_files&.to_set
+        @caller_paths = caller_paths&.to_set
         @filter_method_id_path = filter_method_id_path
       end
 
@@ -80,10 +74,10 @@ module DiverDown
       #
       # @return [TracePoint]
       def new_session(title: SecureRandom.uuid, definition_group: nil)
-        DiverDown::Trace::Tracer::Session.new(
+        DiverDown::Trace::Session.new(
           module_set: @module_set,
           ignored_method_ids: @ignored_method_ids,
-          target_file_set: @target_file_set,
+          caller_paths: @caller_paths,
           filter_method_id_path: @filter_method_id_path,
           definition: DiverDown::Definition.new(
             title:,

data/lib/diver_down/trace.rb

@@ -5,7 +5,7 @@ require 'active_support/inflector'
 module DiverDown
   module Trace
     require 'diver_down/trace/tracer'
-    require 'diver_down/trace/tracer/session'
+    require 'diver_down/trace/session'
     require 'diver_down/trace/call_stack'
     require 'diver_down/trace/module_set'
     require 'diver_down/trace/redefine_ruby_methods'

data/lib/diver_down/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module DiverDown
-  VERSION = '0.0.1.alpha1'
+  VERSION = '0.0.1.alpha3'
 end

data/lib/diver_down/web/action.rb

@@ -119,8 +119,9 @@ module DiverDown
       # @param per [Integer]
       # @param title [String]
       # @param source [String]
-      def definitions(page:, per:, title:, source:)
-        definition_enumerator = DiverDown::Web::DefinitionEnumerator.new(@store, title:, source:)
+      # @param definition_group [String]
+      def definitions(page:, per:, title:, source:, definition_group:)
+        definition_enumerator = DiverDown::Web::DefinitionEnumerator.new(@store, title:, source:, definition_group:)
         definitions, pagination = paginate(definition_enumerator, page, per)
 
         json(

data/lib/diver_down/web/definition_enumerator.rb

@@ -7,10 +7,12 @@ module DiverDown
 
       # @param store [DiverDown::Definition::Store]
       # @param query [String]
-      def initialize(store, title: '', source: '')
+      # @param definition_group [String]
+      def initialize(store, title: '', source: '', definition_group: '')
         @store = store
         @title = title
         @source = source
+        @definition_group = definition_group
       end
 
       # @yield [parent_bit_id, bit_id, definition]
@@ -47,6 +49,10 @@ module DiverDown
           matched &&= definition.sources.any? { _1.source_name.include?(@source) }
         end
 
+        unless @definition_group.empty?
+          matched &&= definition.definition_group.to_s.include?(@definition_group)
+        end
+
         matched
       end
     end

data/lib/diver_down/web/definition_loader.rb

@@ -8,8 +8,6 @@ module DiverDown
         hash = case File.extname(path)
                when '.yaml', '.yml'
                  from_yaml(path)
-               when '.msgpack'
-                 from_msgpack(path)
                when '.json'
                  from_json(path)
                else
@@ -28,10 +26,6 @@ module DiverDown
       def from_yaml(path)
         YAML.load_file(path)
       end
-
-      def from_msgpack(path)
-        MessagePack.unpack(File.binread(path))
-      end
     end
   end
 end

data/lib/diver_down/web.rb

@@ -45,7 +45,8 @@ module DiverDown
           page: request.params['page']&.to_i || 1,
           per: request.params['per']&.to_i || 100,
           title: request.params['title'] || '',
-          source: request.params['source'] || ''
+          source: request.params['source'] || '',
+          definition_group: request.params['definition_group'] || ''
         )
       in ['GET', %r{\A/api/sources\.json\z}]
         action.sources

data/lib/diver_down.rb

@@ -1,6 +1,5 @@
 # frozen_string_literal: true
 
-require 'msgpack'
 require_relative 'diver_down/version'
 
 module DiverDown