diver_down 0.0.1.alpha1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (37) hide show
  1. checksums.yaml +7 -0
  2. data/CHANGELOG.md +5 -0
  3. data/LICENSE.txt +21 -0
  4. data/README.md +132 -0
  5. data/exe/diver_down_web +55 -0
  6. data/lib/diver_down/definition/dependency.rb +107 -0
  7. data/lib/diver_down/definition/method_id.rb +83 -0
  8. data/lib/diver_down/definition/source.rb +90 -0
  9. data/lib/diver_down/definition.rb +112 -0
  10. data/lib/diver_down/helper.rb +81 -0
  11. data/lib/diver_down/trace/call_stack.rb +45 -0
  12. data/lib/diver_down/trace/ignored_method_ids.rb +136 -0
  13. data/lib/diver_down/trace/module_set/array_module_set.rb +31 -0
  14. data/lib/diver_down/trace/module_set/const_source_location_module_set.rb +28 -0
  15. data/lib/diver_down/trace/module_set.rb +78 -0
  16. data/lib/diver_down/trace/redefine_ruby_methods.rb +64 -0
  17. data/lib/diver_down/trace/tracer/session.rb +121 -0
  18. data/lib/diver_down/trace/tracer.rb +96 -0
  19. data/lib/diver_down/trace.rb +27 -0
  20. data/lib/diver_down/version.rb +5 -0
  21. data/lib/diver_down/web/action.rb +344 -0
  22. data/lib/diver_down/web/bit_id.rb +41 -0
  23. data/lib/diver_down/web/definition_enumerator.rb +54 -0
  24. data/lib/diver_down/web/definition_loader.rb +37 -0
  25. data/lib/diver_down/web/definition_store.rb +89 -0
  26. data/lib/diver_down/web/definition_to_dot.rb +399 -0
  27. data/lib/diver_down/web/dev_server_middleware.rb +72 -0
  28. data/lib/diver_down/web/indented_string_io.rb +59 -0
  29. data/lib/diver_down/web/module_store.rb +59 -0
  30. data/lib/diver_down/web.rb +101 -0
  31. data/lib/diver_down-trace.rb +4 -0
  32. data/lib/diver_down-web.rb +4 -0
  33. data/lib/diver_down.rb +14 -0
  34. data/web/assets/CjLq7LhZ.css +1 -0
  35. data/web/assets/bundle.js +978 -0
  36. data/web/index.html +13 -0
  37. metadata +122 -0
@@ -0,0 +1,45 @@
1
+ # frozen_string_literal: true
2
+
3
+ module DiverDown
4
+ module Trace
5
+ # To handle call stacks obtained by TracePoint more efficiently.
6
+ # TracePoint also acquires calls that are not trace targets, but for dependency extraction, we want to acquire only a list of targets.
7
+ # In this class, push/pop is performed on all call/return, but it should always be possible to trace back to the caller of the target dependency at high speed.
8
+ class CallStack
9
+ class StackEmptyError < RuntimeError; end
10
+
11
+ # @attr_reader stack [Integer] stack size
12
+ attr_reader :stack_size
13
+
14
+ def initialize
15
+ @stack_size = 0
16
+ @stack = {}
17
+ end
18
+
19
+ # @return [Boolean]
20
+ def empty?
21
+ @stack.empty?
22
+ end
23
+
24
+ # @return [Array<Object>]
25
+ def stack
26
+ @stack.values
27
+ end
28
+
29
+ # @param context [Object, nil] User defined stack context.
30
+ # @return [void]
31
+ def push(context = nil)
32
+ @stack_size += 1
33
+ @stack[@stack_size] = context unless context.nil?
34
+ end
35
+
36
+ # @return [void]
37
+ def pop
38
+ raise StackEmptyError if @stack_size.zero?
39
+
40
+ @stack.delete(@stack_size) if @stack.key?(@stack_size)
41
+ @stack_size -= 1
42
+ end
43
+ end
44
+ end
45
+ end
@@ -0,0 +1,136 @@
1
+ # frozen_string_literal: true
2
+
3
+ module DiverDown
4
+ module Trace
5
+ class IgnoredMethodIds
6
+ def initialize(ignored_methods)
7
+ # Ignore all methods in the module
8
+ # Hash{ Module => Boolean }
9
+ @ignored_modules = {}
10
+
11
+ # Ignore all methods in the class
12
+ # Hash{ Module => Hash{ Symbol => Boolean } }
13
+ @ignored_class_method_id = Hash.new { |h, k| h[k] = {} }
14
+
15
+ # Ignore all methods in the instance
16
+ # Hash{ Module => Hash{ Symbol => Boolean } }
17
+ @ignored_instance_method_id = Hash.new { |h, k| h[k] = {} }
18
+
19
+ ignored_methods.each do |ignored_method|
20
+ if ignored_method.include?('.')
21
+ # instance method
22
+ class_name, method_id = ignored_method.split('.')
23
+ mod = DiverDown::Helper.constantize(class_name)
24
+ @ignored_class_method_id[mod][method_id.to_sym] = true
25
+ elsif ignored_method.include?('#')
26
+ # class method
27
+ class_name, method_id = ignored_method.split('#')
28
+ mod = DiverDown::Helper.constantize(class_name)
29
+ @ignored_instance_method_id[mod][method_id.to_sym] = true
30
+ else
31
+ # module
32
+ mod = DiverDown::Helper.constantize(ignored_method)
33
+ @ignored_modules[mod] = true
34
+ end
35
+ end
36
+ end
37
+
38
+ # @param mod [Module]
39
+ # @param is_class [Boolean] class is true, instance is false
40
+ # @param method_id [Symbol]
41
+ # @return [Boolean]
42
+ def ignored?(mod, is_class, method_id)
43
+ ignored_module?(mod) || ignored_method?(mod, is_class, method_id)
44
+ end
45
+
46
+ private
47
+
48
+ def ignored_module?(mod)
49
+ unless @ignored_modules.key?(mod)
50
+ dig_superclass(mod)
51
+ end
52
+
53
+ @ignored_modules.fetch(mod)
54
+ end
55
+
56
+ def ignored_method?(mod, is_class, method_id)
57
+ store = if is_class
58
+ # class methods
59
+ @ignored_class_method_id
60
+ else
61
+ # instance methods
62
+ @ignored_instance_method_id
63
+ end
64
+
65
+ begin
66
+ dig_superclass_method_id(store, mod, method_id) unless store[mod].key?(method_id)
67
+ rescue TypeError => e
68
+ # https://github.com/ruby/ruby/blob/f42164e03700469a7000b4f00148a8ca01d75044/object.c#L2232
69
+ return false if e.message == 'uninitialized class'
70
+
71
+ raise
72
+ end
73
+
74
+ store.fetch(mod).fetch(method_id)
75
+ end
76
+
77
+ def dig_superclass(mod)
78
+ unless DiverDown::Helper.class?(mod)
79
+ # NOTE: Do not lookup the ancestors if module given because of the complexity of implementation
80
+ @ignored_modules[mod] = false
81
+ return
82
+ end
83
+
84
+ stack = []
85
+ current = mod
86
+ ignored = nil
87
+
88
+ until current.nil?
89
+ if @ignored_modules.key?(current)
90
+ ignored = @ignored_modules.fetch(current)
91
+ break
92
+ else
93
+ stack.push(current)
94
+ current = current.superclass
95
+ end
96
+ end
97
+
98
+ # Convert nil to boolean
99
+ ignored = !!ignored
100
+
101
+ stack.each do
102
+ @ignored_modules[_1] = ignored
103
+ end
104
+ end
105
+
106
+ def dig_superclass_method_id(store, mod, method_id)
107
+ unless DiverDown::Helper.class?(mod)
108
+ # NOTE: Do not lookup the ancestors if module given because of the complexity of implementation
109
+ store[mod][method_id] = false
110
+ return
111
+ end
112
+
113
+ stack = []
114
+ current = mod
115
+ ignored = nil
116
+
117
+ until current.nil?
118
+ if store[current].key?(method_id)
119
+ ignored = store[current].fetch(method_id)
120
+ break
121
+ else
122
+ stack.push(current)
123
+ current = current.superclass
124
+ end
125
+ end
126
+
127
+ # Convert nil to boolean
128
+ ignored = !!ignored
129
+
130
+ stack.each do
131
+ store[_1][method_id] = ignored
132
+ end
133
+ end
134
+ end
135
+ end
136
+ end
@@ -0,0 +1,31 @@
1
+ # frozen_string_literal: true
2
+
3
+ module DiverDown
4
+ module Trace
5
+ class ModuleSet
6
+ class ArrayModuleSet
7
+ # @param [Array<Module, String>, #each] modules
8
+ def initialize(modules)
9
+ @map = {}
10
+
11
+ modules.each do
12
+ mod = if DiverDown::Helper.module?(_1)
13
+ _1
14
+ else
15
+ # constantize if it is a string
16
+ DiverDown::Helper.constantize(_1)
17
+ end
18
+
19
+ @map[mod] = true
20
+ end
21
+ end
22
+
23
+ # @param [Module] mod
24
+ # @return [Boolean, nil]
25
+ def include?(mod)
26
+ @map[mod]
27
+ end
28
+ end
29
+ end
30
+ end
31
+ end
@@ -0,0 +1,28 @@
1
+ # frozen_string_literal: true
2
+
3
+ module DiverDown
4
+ module Trace
5
+ class ModuleSet
6
+ class ConstSourceLocationModuleSet
7
+ # @param [Array<String>, Set<String>] paths
8
+ def initialize(paths: [])
9
+ @paths = paths.to_set
10
+ end
11
+
12
+ # @param [Module] mod
13
+ # @return [Boolean]
14
+ def include?(mod)
15
+ module_name = DiverDown::Helper.normalize_module_name(mod)
16
+
17
+ path, = begin
18
+ Object.const_source_location(module_name)
19
+ rescue NameError, TypeError
20
+ nil
21
+ end
22
+
23
+ path && @paths.include?(path)
24
+ end
25
+ end
26
+ end
27
+ end
28
+ end
@@ -0,0 +1,78 @@
1
+ # frozen_string_literal: true
2
+
3
+ module DiverDown
4
+ module Trace
5
+ # A class to quickly determine if a TracePoint is a module to be traced.
6
+ class ModuleSet
7
+ require 'diver_down/trace/module_set/array_module_set'
8
+ require 'diver_down/trace/module_set/const_source_location_module_set'
9
+
10
+ # @param [Array<Module, String>, Set<Module, String>, nil] modules
11
+ # @param [Array<String>, Set<String>, nil] include
12
+ def initialize(modules: nil, paths: nil)
13
+ @cache = {}
14
+ @array_module_set = DiverDown::Trace::ModuleSet::ArrayModuleSet.new(modules) unless modules.nil?
15
+ @const_source_location_module_set = DiverDown::Trace::ModuleSet::ConstSourceLocationModuleSet.new(paths:) unless paths.nil?
16
+ end
17
+
18
+ # @param [Module] mod
19
+ # @return [Boolean]
20
+ def include?(mod)
21
+ unless @cache.key?(mod)
22
+ if DiverDown::Helper.class?(mod)
23
+ # class
24
+ begin
25
+ dig_superclass(mod)
26
+ rescue TypeError => e
27
+ # https://github.com/ruby/ruby/blob/f42164e03700469a7000b4f00148a8ca01d75044/object.c#L2232
28
+ return false if e.message == 'uninitialized class'
29
+
30
+ raise
31
+ end
32
+ else
33
+ # module
34
+ @cache[mod] = !!_include?(mod)
35
+ end
36
+ end
37
+
38
+ @cache.fetch(mod)
39
+ end
40
+
41
+ private
42
+
43
+ def _include?(mod)
44
+ return true if @array_module_set.nil? && @const_source_location_module_set.nil?
45
+
46
+ @array_module_set&.include?(mod) ||
47
+ @const_source_location_module_set&.include?(mod)
48
+ end
49
+
50
+ def dig_superclass(mod)
51
+ stack = []
52
+ current = mod
53
+ included = nil
54
+
55
+ until current.nil?
56
+ if @cache.key?(current)
57
+ included = @cache.fetch(current)
58
+ break
59
+ else
60
+ stack.push(current)
61
+ included = _include?(current)
62
+
63
+ break if included
64
+
65
+ current = current.superclass
66
+ end
67
+ end
68
+
69
+ # Convert nil to boolean
70
+ included = !!included
71
+
72
+ stack.each do
73
+ @cache[_1] = included unless @cache.key?(_1)
74
+ end
75
+ end
76
+ end
77
+ end
78
+ end
@@ -0,0 +1,64 @@
1
+ # frozen_string_literal: true
2
+
3
+ module DiverDown
4
+ module Trace
5
+ # Tracepoint traces only ruby-lang calls because tracing c-lang calls is very slow.
6
+ # In this case, methods such as new will not be traceable under normal circumstances, so at a minimum, redefine them in ruby and hack them so that they can be traced.
7
+ #
8
+ # Do not use this in a production environment.
9
+ module RedefineRubyMethods
10
+ DEFAULT_METHODS = {
11
+ Module => {
12
+ singleton: %i[name],
13
+ instance: [],
14
+ },
15
+ Class => {
16
+ singleton: %i[name allocate new],
17
+ instance: [],
18
+ },
19
+ }.freeze
20
+
21
+ class RubyMethods < Module
22
+ # @param class_method_names [Array<Symbol>]
23
+ # @param instance_method_names [Array<Symbol>]
24
+ def initialize(class_method_names, instance_method_names)
25
+ unless class_method_names.empty?
26
+ def_class_methods = class_method_names.inject(+'') do |buf, method_name|
27
+ buf << <<~METHOD
28
+ def #{method_name}(...)
29
+ super
30
+ end
31
+
32
+ METHOD
33
+ end
34
+
35
+ mod = Module.new
36
+ mod.module_eval(def_class_methods, __FILE__, __LINE__)
37
+ const_set(:CLASS_METHODS, mod)
38
+ end
39
+
40
+ unless instance_method_names.empty?
41
+ instance_method_names.inject(+'') do |_buf, method_name|
42
+ define_method(method_name) do |*_args, **_options|
43
+ super
44
+ end
45
+ end
46
+ end
47
+ end
48
+
49
+ # @param base [Module]
50
+ def prepend_features(base)
51
+ base.singleton_class.prepend(const_get(:CLASS_METHODS)) if const_defined?(:CLASS_METHODS)
52
+ super
53
+ end
54
+ end
55
+
56
+ # @param map [Hash{ Class => Hash{ singleton: Array<String>, instance: Array<String> } }]
57
+ def self.redefine_c_methods(map = DEFAULT_METHODS)
58
+ map.each do |m, method_names|
59
+ m.prepend(RubyMethods.new(method_names[:singleton], method_names[:instance]))
60
+ end
61
+ end
62
+ end
63
+ end
64
+ end
@@ -0,0 +1,121 @@
1
+ # frozen_string_literal: true
2
+
3
+ module DiverDown
4
+ module Trace
5
+ class Tracer
6
+ class Session
7
+ class NotStarted < StandardError; end
8
+
9
+ attr_reader :definition
10
+
11
+ # @param [DiverDown::Trace::ModuleSet, nil] module_set
12
+ # @param [DiverDown::Trace::IgnoredMethodIds, nil] ignored_method_ids
13
+ # @param [Set<String>, nil] target_file_set
14
+ # @param [#call, nil] filter_method_id_path
15
+ def initialize(module_set: DiverDown::Trace::ModuleSet.new, ignored_method_ids: nil, target_file_set: nil, filter_method_id_path: nil, definition: DiverDown::Definition.new)
16
+ @module_set = module_set
17
+ @ignored_method_ids = ignored_method_ids
18
+ @target_file_set = target_file_set
19
+ @filter_method_id_path = filter_method_id_path
20
+ @definition = definition
21
+ @trace_point = build_trace_point
22
+ end
23
+
24
+ # @return [void]
25
+ def start
26
+ @trace_point.enable
27
+ end
28
+
29
+ # @return [void]
30
+ def stop
31
+ @trace_point.disable
32
+ end
33
+
34
+ private
35
+
36
+ def build_trace_point
37
+ call_stack = DiverDown::Trace::CallStack.new
38
+ ignored_stack_size = nil
39
+
40
+ TracePoint.new(*DiverDown::Trace::Tracer.trace_events) do |tp|
41
+ # Skip the trace of the library itself
42
+ next if tp.path&.start_with?(DiverDown::LIB_DIR)
43
+ next if TracePoint == tp.defined_class
44
+
45
+ case tp.event
46
+ when :call, :c_call
47
+ # puts "#{tp.method_id} #{tp.path}:#{tp.lineno}"
48
+ mod = DiverDown::Helper.resolve_module(tp.self)
49
+ source_name = DiverDown::Helper.normalize_module_name(mod) if !mod.nil? && @module_set.include?(mod)
50
+ already_ignored = !ignored_stack_size.nil? # If the current method_id is ignored
51
+ current_ignored = !@ignored_method_ids.nil? && @ignored_method_ids.ignored?(mod, DiverDown::Helper.module?(tp.self), tp.method_id)
52
+ pushed = false
53
+
54
+ if !source_name.nil? && !(already_ignored || current_ignored)
55
+ source = @definition.find_or_build_source(source_name)
56
+
57
+ # If the call stack contains a call to a module to be traced
58
+ # `@ignored_call_stack` is not nil means the call stack contains a call to a module to be ignored
59
+ unless call_stack.empty?
60
+ # Add dependency to called source
61
+ called_stack_context = call_stack.stack[-1]
62
+ called_source = called_stack_context.source
63
+ dependency = called_source.find_or_build_dependency(source_name)
64
+
65
+ # `dependency.nil?` means source_name equals to called_source.source.
66
+ # self-references are not tracked because it is not "dependency".
67
+ if dependency
68
+ context = DiverDown::Helper.module?(tp.self) ? 'class' : 'instance'
69
+ method_id = dependency.find_or_build_method_id(name: tp.method_id, context:)
70
+ method_id_path = "#{called_stack_context.caller_location.path}:#{called_stack_context.caller_location.lineno}"
71
+ method_id_path = @filter_method_id_path.call(method_id_path) if @filter_method_id_path
72
+ method_id.add_path(method_id_path)
73
+ end
74
+ end
75
+
76
+ # `caller_location` is nil if it is filtered by target_files
77
+ caller_location = find_neast_caller_location
78
+
79
+ if caller_location
80
+ pushed = true
81
+
82
+ call_stack.push(
83
+ StackContext.new(
84
+ source:,
85
+ method_id: tp.method_id,
86
+ caller_location:
87
+ )
88
+ )
89
+ end
90
+ end
91
+
92
+ call_stack.push unless pushed
93
+
94
+ # If a value is already stored, it means that call stack already determined to be ignored at the shallower call stack size.
95
+ # Since stacks deeper than the shallowest stack size are ignored, priority is given to already stored values.
96
+ if !already_ignored && current_ignored
97
+ ignored_stack_size = call_stack.stack_size
98
+ end
99
+ when :return, :c_return
100
+ ignored_stack_size = nil if ignored_stack_size == call_stack.stack_size
101
+ call_stack.pop
102
+ end
103
+ rescue StandardError
104
+ tp.disable
105
+ raise
106
+ end
107
+ end
108
+
109
+ def find_neast_caller_location
110
+ return caller_locations(2, 2)[0] if @target_file_set.nil?
111
+
112
+ Thread.each_caller_location do
113
+ return _1 if @target_file_set.include?(_1.path)
114
+ end
115
+
116
+ nil
117
+ end
118
+ end
119
+ end
120
+ end
121
+ end
@@ -0,0 +1,96 @@
1
+ # frozen_string_literal: true
2
+
3
+ module DiverDown
4
+ module Trace
5
+ class Tracer
6
+ StackContext = Data.define(
7
+ :source,
8
+ :method_id,
9
+ :caller_location
10
+ )
11
+
12
+ # @return [Array<Symbol>]
13
+ def self.trace_events
14
+ @trace_events || %i[call c_call return c_return]
15
+ end
16
+
17
+ # @param events [Array<Symbol>]
18
+ class << self
19
+ attr_writer :trace_events
20
+ end
21
+
22
+ # @param module_set [DiverDown::Trace::ModuleSet, Array<Module, String>]
23
+ # @param target_files [Array<String>, nil] if nil, trace all files
24
+ # @param ignored_method_ids [Array<String>]
25
+ # @param filter_method_id_path [#call, nil] filter method_id.path
26
+ # @param module_set [DiverDown::Trace::ModuleSet, nil] for optimization
27
+ def initialize(module_set: {}, target_files: nil, ignored_method_ids: nil, filter_method_id_path: nil)
28
+ if target_files && !target_files.all? { Pathname.new(_1).absolute? }
29
+ raise ArgumentError, "target_files must be absolute path(#{target_files})"
30
+ end
31
+
32
+ @module_set = if module_set.is_a?(DiverDown::Trace::ModuleSet)
33
+ module_set
34
+ elsif module_set.is_a?(Hash)
35
+ DiverDown::Trace::ModuleSet.new(**module_set)
36
+ else
37
+ raise ArgumentError, <<~MSG
38
+ Given invalid module_set. #{module_set}"
39
+
40
+ Available types are:
41
+
42
+ Hash{
43
+ modules: Array<Module, String> | Set<Module, String> | nil
44
+ paths: Array<String> | Set<String> | nil
45
+ } | DiverDown::Trace::ModuleSet
46
+ MSG
47
+ end
48
+
49
+ @ignored_method_ids = if ignored_method_ids.is_a?(DiverDown::Trace::IgnoredMethodIds)
50
+ ignored_method_ids
51
+ elsif !ignored_method_ids.nil?
52
+ DiverDown::Trace::IgnoredMethodIds.new(ignored_method_ids)
53
+ end
54
+
55
+ @target_file_set = target_files&.to_set
56
+ @filter_method_id_path = filter_method_id_path
57
+ end
58
+
59
+ # Trace the call stack of the block and build the definition
60
+ #
61
+ # @param title [String]
62
+ # @param definition_group [String, nil]
63
+ #
64
+ # @return [DiverDown::Definition]
65
+ def trace(title: SecureRandom.uuid, definition_group: nil, &)
66
+ session = new_session(title:, definition_group:)
67
+ session.start
68
+
69
+ yield
70
+
71
+ session.stop
72
+ session.definition
73
+ ensure
74
+ # Ensure to stop the session
75
+ session&.stop
76
+ end
77
+
78
+ # @param title [String]
79
+ # @param definition_group [String, nil]
80
+ #
81
+ # @return [TracePoint]
82
+ def new_session(title: SecureRandom.uuid, definition_group: nil)
83
+ DiverDown::Trace::Tracer::Session.new(
84
+ module_set: @module_set,
85
+ ignored_method_ids: @ignored_method_ids,
86
+ target_file_set: @target_file_set,
87
+ filter_method_id_path: @filter_method_id_path,
88
+ definition: DiverDown::Definition.new(
89
+ title:,
90
+ definition_group:
91
+ )
92
+ )
93
+ end
94
+ end
95
+ end
96
+ end
@@ -0,0 +1,27 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'active_support/inflector'
4
+
5
+ module DiverDown
6
+ module Trace
7
+ require 'diver_down/trace/tracer'
8
+ require 'diver_down/trace/tracer/session'
9
+ require 'diver_down/trace/call_stack'
10
+ require 'diver_down/trace/module_set'
11
+ require 'diver_down/trace/redefine_ruby_methods'
12
+ require 'diver_down/trace/ignored_method_ids'
13
+
14
+ @trace_events = %i[
15
+ call c_call return c_return
16
+ ]
17
+
18
+ # Trace only Ruby-implemented methods because tracing C-implemented methods is very slow
19
+ # Override Ruby only with the minimal set of methods needed to trace dependencies.
20
+ #
21
+ # @return [void]
22
+ def self.trace_only_ruby_world!(map = DiverDown::Trace::RedefineRubyMethods::DEFAULT_METHODS)
23
+ DiverDown::Trace::Tracer.trace_events = %i[call return]
24
+ DiverDown::Trace::RedefineRubyMethods.redefine_c_methods(map)
25
+ end
26
+ end
27
+ end
@@ -0,0 +1,5 @@
1
+ # frozen_string_literal: true
2
+
3
+ module DiverDown
4
+ VERSION = '0.0.1.alpha1'
5
+ end