appmap 0.18.1

data/lib/appmap/rspec.rb

@@ -0,0 +1,288 @@
+# frozen_string_literal: true
+
+require 'appmap'
+require 'appmap/config'
+require 'appmap/inspect'
+require 'appmap/trace/tracer'
+
+require 'active_support/inflector/transliterate'
+
+module AppMap
+  # Integration of AppMap with RSpec. When enabled with APPMAP=true, the AppMap tracer will
+  # be activated around each scenario which has the metadata key `:appmap`.
+  module RSpec
+    APPMAP_OUTPUT_DIR = 'tmp/appmap/rspec'
+
+    class Recorder
+      attr_reader :config, :features, :functions
+
+      def initialize
+        @config = AppMap::Config.load_from_file('appmap.yml')
+
+        raise "Missing AppMap configuration setting: 'name'" unless @config.name
+
+        @features = AppMap.inspect(@config)
+        @functions = @features.map(&:collect_functions).flatten
+      end
+
+      def setup
+        FileUtils.mkdir_p APPMAP_OUTPUT_DIR
+      end
+
+      # TODO: Optionally populate the 'layout' from appmap config or RSpec metadata.
+      def save(example_name, events, feature_name: nil, feature_group_name: nil)
+        require 'appmap/command/record'
+        metadata = AppMap::Command::Record.detect_metadata.tap do |m|
+          m[:name] = example_name
+          m[:app] = @config.name
+          m[:feature] = feature_name if feature_name
+          m[:feature_group] = feature_group_name if feature_group_name
+          m[:frameworks] ||= []
+          m[:frameworks] << {
+            name: 'rspec',
+            version: Gem.loaded_specs['rspec-core']&.version&.to_s
+          }
+        end
+
+        appmap = {
+          version: '1.1',
+          classMap: features,
+          metadata: metadata,
+          events: events
+        }
+        fname = sanitize_filename(example_name)
+        File.write(File.join(APPMAP_OUTPUT_DIR, "#{fname}.appmap.json"), JSON.generate(appmap))
+      end
+
+      # Cribbed from v5 version of ActiveSupport:Inflector#parameterize:
+      # https://github.com/rails/rails/blob/v5.2.4/activesupport/lib/active_support/inflector/transliterate.rb#L92
+      def sanitize_filename(fname, separator: '_')
+        # Replace accented chars with their ASCII equivalents.
+        fname = ActiveSupport::Inflector.transliterate(fname)
+
+        # Turn unwanted chars into the separator.
+        fname.gsub!(/[^a-z0-9\-_]+/i, separator)
+
+        re_sep = Regexp.escape(separator)
+        re_duplicate_separator        = /#{re_sep}{2,}/
+        re_leading_trailing_separator = /^#{re_sep}|#{re_sep}$/i
+
+        # No more than one of the separator in a row.
+        fname.gsub!(re_duplicate_separator, separator)
+
+        # Finally, Remove leading/trailing separator.
+        fname.gsub(re_leading_trailing_separator, '')
+      end
+    end
+
+    class << self
+      module FeatureAnnotations
+        def feature
+          return nil unless annotations
+
+          annotations[:feature]
+        end
+
+        def feature_group
+          return nil unless annotations
+
+          annotations[:feature_group]
+        end
+
+        def annotations
+          metadata.tap do |md|
+            description_args_hashes.each do |h|
+              md.merge! h
+            end
+          end
+        end
+
+        protected
+
+        def metadata
+          return {} unless example_obj.respond_to?(:metadata)
+
+          example_obj.metadata
+        end
+
+        def description_args_hashes
+          return [] unless example_obj.respond_to?(:metadata)
+
+          (example_obj.metadata[:description_args] || []).select { |arg| arg.is_a?(Hash) }
+        end
+      end
+
+      # ScopeExample and ScopeExampleGroup is a way to handle the weird way that RSpec
+      # stores the nested example names.
+      ScopeExample = Struct.new(:example) do
+        include FeatureAnnotations
+
+        alias_method :example_obj, :example
+
+        def description
+          example.description
+        end
+
+        def parent
+          ScopeExampleGroup.new(example.example_group)
+        end
+      end
+
+      # As you can see here, the way that RSpec stores the example description and
+      # represents the example group hierarchy is pretty weird.
+      ScopeExampleGroup = Struct.new(:example_group) do
+        include FeatureAnnotations
+
+        alias_method :example_obj, :example_group
+
+        def description_args
+          # Don't stringify any hashes that RSpec considers part of the example group description.
+          example_group.metadata[:description_args].reject { |arg| arg.is_a?(Hash) }
+        end
+
+        def description?
+          return true if example_group.respond_to?(:described_class) && example_group.described_class
+
+          return true if example_group.respond_to?(:description) && !description_args.empty?
+
+          false
+        end
+
+        def description
+          description? ? description_args.join(' ') : nil
+        end
+
+        def parent
+          # An example group always has a parent; but it might be 'self'...
+          example_group.parent != example_group ? ScopeExampleGroup.new(example_group.parent) : nil
+        end
+      end
+
+      LOG = false
+
+      def is_example_group_subclass_call?(tp)
+        # Order is important here. Checking for method_id == :subclass
+        # first will avoid calling defined_class.to_s in many cases,
+        # some of which will fail.
+        #
+        # For example, ActiveRecord in Rails 4 defines #inspect (and
+        # therefore #to_s) in such a way that it will fail if called
+        # here.
+        tp.event == :call &&
+          tp.method_id == :subclass &&
+          tp.defined_class.singleton_class? &&
+          tp.defined_class.to_s == '#<Class:RSpec::Core::ExampleGroup>'
+      end
+
+      def is_example_initialize_call?(tp)
+        tp.event == :call &&
+          tp.method_id == :initialize &&
+          tp.defined_class.to_s == 'RSpec::Core::Example'
+      end
+
+      def generate_appmaps_from_specs
+        recorder = Recorder.new
+        recorder.setup
+
+        require 'set'
+        # file:lineno at which an Example block begins
+        trace_block_start = Set.new
+        # file:lineno at which an Example block ends
+        trace_block_end = Set.new
+
+        # value: a BlockParseNode from an RSpec file
+        # key: file:lineno at which the block begins
+        rspec_blocks = {}
+
+        # value: an Example instance
+        # key: file:lineno at which the Example block ends
+        examples = {}
+
+        current_tracer = nil
+
+        TracePoint.trace(:call, :b_call, :b_return) do |tp|
+          # When a new ExampleGroup is encountered, parse the source file containing it and look
+          # for blocks that might be Examples. Index each BlockParseNode by the start file:lineno.
+          if is_example_group_subclass_call?(tp)
+            example_block = tp.binding.eval('example_group_block')
+            source_path, start_line = example_block.source_location
+            require 'appmap/rspec/parser'
+            nodes, = AppMap::RSpec::Parser.new(file_path: source_path).parse
+            nodes.each do |node|
+              start_loc = [ node.file_path, node.first_line ].join(':')
+              rspec_blocks[start_loc] = node
+            end
+          end
+
+          # When a new Example is constructed with a block, look for the BlockParseNode that starts at the block's
+          # file:lineno. If it exists, store the Example object, indexed by the file:lineno at which it ends.
+          if is_example_initialize_call?(tp)
+            example_block = tp.binding.eval('example_block')
+            if example_block
+              source_path, start_line = example_block.source_location
+              start_loc = [ source_path, start_line ].join(':')
+              if (rspec_block = rspec_blocks[start_loc])
+                end_loc = [ source_path, rspec_block.last_line ].join(':')
+                trace_block_start << start_loc.tap { |loc| puts "Start: #{loc}" if LOG }
+                trace_block_end << end_loc.tap { |loc| puts "End: #{loc}" if LOG }
+                examples[end_loc] = tp.binding.eval('self')
+              end
+            end
+          end
+
+          if %i[b_call b_return].member?(tp.event)
+            loc = [ tp.path, tp.lineno ].join(':')
+            puts loc if LOG && (trace_block_start.member?(loc) || trace_block_end.member?(loc))
+
+            # When a new block is started, check if an Example block is known to begin at that
+            # file:lineno. If it is, enable the AppMap tracer.
+            if  tp.event == :b_call && trace_block_start.member?(loc)
+              puts "Starting trace on #{loc}" if LOG
+              current_tracer = AppMap::Trace.tracers.trace(recorder.functions)
+            end
+
+            # When the tracer is enabled and a block is completed, check to see if there is an
+            # Example stored at the file:lineno. If so, finish tracing and emit the 
+            # AppMap file.
+            if current_tracer && tp.event == :b_return && trace_block_end.member?(loc)
+              puts "Ending trace on #{loc}" if LOG
+              events = []
+              AppMap::Trace.tracers.delete current_tracer
+
+              while current_tracer.event?
+                events << current_tracer.next_event.to_h
+              end
+
+              example = examples[loc]
+              description = []
+              scope = ScopeExample.new(example)
+              feature_group = feature = nil
+              while scope
+                description << scope.description
+                feature ||= scope.feature
+                feature_group ||= scope.feature_group
+                scope = scope.parent
+              end
+              description.reject! { |d| d.nil? || d == '' }
+              description.reverse!
+
+              description.each do |token|
+                token = token.gsub 'it should behave like', ''
+                token.gsub! '  ', ' '
+                token.gsub! '/', '_'
+                token.strip!
+              end
+              full_description = description.join(' ')
+
+              recorder.save full_description, events,
+                            feature_name: feature,
+                            feature_group_name: feature_group
+            end
+          end
+        end
+      end
+    end
+
+    generate_appmaps_from_specs if ENV['APPMAP'] == 'true'
+  end
+end

data/lib/appmap/trace/event_handler/rack_handler_webrick.rb

@@ -0,0 +1,65 @@
+module AppMap
+  module Trace
+    module EventHandler
+      # Use the `req` and `res` parameters on Rack::Handler::WEBrick to populate the
+      # `http_server_request` and `http_server_response` info on the trace event.
+      #
+      # See https://github.com/rack/rack/blob/b72bfc9435c118c54019efae1fedd119521b76df/lib/rack/handler/webrick.rb#L26
+      module RackHandlerWebrick
+        class Call < MethodEvent
+          attr_accessor :http_server_request
+
+          class << self
+            def build_from_tracepoint(mc = Call.new, tp, path)
+              mc.tap do |_|
+                req = value_in_binding(tp, :req)
+
+                # Don't try and grab 'parameters', because:
+                # a) They aren't needed.
+                # b) We want to avoid triggering side effects like reading the request body.
+
+                mc.http_server_request = {
+                  request_method: req.request_method,
+                  path_info: req.path_info,
+                  protocol: "HTTP/#{req.http_version}"
+                }
+
+                MethodEvent.build_from_tracepoint(mc, tp, path)
+              end
+            end
+          end
+
+          def to_h
+            super.tap do |h|
+              h[:http_server_request] = http_server_request
+            end
+          end
+        end
+
+        class Return < MethodReturnIgnoreValue
+          attr_accessor :http_server_response
+
+          class << self
+            def build_from_tracepoint(mr = Return.new, tp, path, parent_id, elapsed)
+              mr.tap do |_|
+                res = value_in_binding(tp, :res)
+
+                mr.http_server_response = {
+                  status: res.status
+                }
+
+                MethodReturnIgnoreValue.build_from_tracepoint(mr, tp, path, parent_id, elapsed)
+              end
+            end
+          end
+
+          def to_h
+            super.tap do |h|
+              h[:http_server_response] = http_server_response
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/appmap/trace/tracer.rb

@@ -0,0 +1,347 @@
+module AppMap
+  module Trace
+    MethodEventStruct =
+      Struct.new(:id, :event, :defined_class, :method_id, :path, :lineno, :static, :thread_id)
+
+    class Tracers
+      def initialize
+        @tracers = []
+      end
+
+      def empty?
+        @tracers.empty?
+      end
+
+      def trace(functions, enable: true)
+        AppMap::Trace::Tracer.new(functions).tap do |tracer|
+          @tracers << tracer
+          tracer.enable if enable
+        end
+      end
+
+      def record_event(event)
+        @tracers.each do |tracer|
+          tracer.record_event(event)
+        end
+      end
+
+      def delete(tracer)
+        return unless @tracers.member?(tracer)
+
+        @tracers.delete(tracer)
+        tracer.disable
+      end
+    end
+
+    class << self
+      def tracers
+        @tracers ||= Tracers.new
+      end
+    end
+
+    # @appmap
+    class MethodEvent < MethodEventStruct
+      LIMIT = 100
+
+      COUNTER_LOCK = Mutex.new # :nodoc:
+      @@id_counter = 0
+
+      class << self
+        # Build a new instance from a TracePoint.
+        def build_from_tracepoint(me, tp, path)
+          me.id = next_id
+          me.event = tp.event
+
+          if tp.defined_class.singleton_class?
+            me.defined_class = (tp.self.is_a?(Class) || tp.self.is_a?(Module)) ? tp.self.name : tp.self.class.name
+          else
+            me.defined_class = tp.defined_class.name
+          end
+
+          me.method_id = tp.method_id
+          me.path = path
+          me.lineno = tp.lineno
+          me.static = tp.defined_class.name.nil?
+          me.thread_id = Thread.current.object_id
+        end
+
+        # Gets the next serial id.
+        #
+        # This method is thread-safe.
+        # @appmap
+        def next_id
+          COUNTER_LOCK.synchronize do
+            @@id_counter += 1
+          end
+        end
+
+        # Gets a value, by key, from the trace point binding.
+        # If the method raises an error, it can be handled by the optional block.
+        def value_in_binding(tp, key, &block)
+          tp.binding.eval(key.to_s)
+        rescue NameError, ArgumentError
+          yield if block_given?
+        end
+
+        # Gets a display string for a value. This is not meant to be a machine deserializable value.
+        def display_string(value)
+          return nil unless value
+
+          begin
+            value_string = value.to_s
+          rescue NoMethodError
+            value_string = value.inspect
+          rescue StandardError
+            warn $!.message
+            '*Error inspecting variable*'
+          end
+
+          value_string[0...LIMIT].encode('utf-8', invalid: :replace, undef: :replace, replace: '_')
+        end
+      end
+
+      alias static? static
+    end
+
+    # @appmap
+    class MethodCall < MethodEvent
+      attr_accessor :parameters, :receiver
+
+      class << self
+        # @appmap
+        def build_from_tracepoint(mc = MethodCall.new, tp, path)
+          mc.tap do |_|
+            mc.parameters = collect_parameters(tp)
+            mc.receiver = collect_self(tp)
+            MethodEvent.build_from_tracepoint(mc, tp, path)
+          end
+        end
+
+        def collect_self(tp)
+          {
+            class: tp.self.class.name,
+            object_id: tp.self.__id__,
+            value: display_string(tp.self)
+          }
+        end
+
+        def collect_parameters(tp)
+          tp.self.method(tp.method_id).parameters.collect do |pinfo|
+            kind, key = pinfo
+            value = value_in_binding(tp, key)
+            {
+              name: key,
+              class: value.class.name,
+              object_id: value.__id__,
+              value: display_string(value),
+              kind: kind # :req, :rest, :key, :keyrest, :block
+            }
+          end
+        end
+      end
+
+      def to_h
+        super.tap do |h|
+          h[:parameters] = parameters
+          h[:receiver] = receiver
+        end
+      end
+    end
+
+    class MethodReturnIgnoreValue < MethodEvent
+      attr_accessor :parent_id, :elapsed
+
+      class << self
+        def build_from_tracepoint(mr = MethodReturnIgnoreValue.new, tp, path, parent_id, elapsed)
+          mr.tap do |_|
+            mr.parent_id = parent_id
+            mr.elapsed = elapsed
+            MethodEvent.build_from_tracepoint(mr, tp, path)
+          end
+        end
+      end
+
+      def to_h
+        super.tap do |h|
+          h[:parent_id] = parent_id
+          h[:elapsed] = elapsed
+        end
+      end
+    end
+
+    class MethodReturn < MethodReturnIgnoreValue
+      attr_accessor :return_value
+
+      class << self
+        def build_from_tracepoint(mr = MethodReturn.new, tp, path, parent_id, elapsed)
+          mr.tap do |_|
+            mr.return_value = {
+              class: tp.return_value.class.name,
+              value: display_string(tp.return_value),
+              object_id: tp.return_value.__id__
+            }
+            MethodReturnIgnoreValue.build_from_tracepoint(mr, tp, path, parent_id, elapsed)
+          end
+        end
+      end
+
+      def to_h
+        super.tap do |h|
+          h[:return_value] = return_value
+        end
+      end
+    end
+
+    # Processes a series of calls into recorded events.
+    # Each call to the handle should provide a TracePoint (or duck-typed object) as the argument.
+    # On each call, a MethodEvent is constructed according to the nature of the TracePoint, and then
+    # stored using the record_event method.
+    # @appmap
+    class TracePointHandler
+      attr_accessor :call_constructor, :return_constructor
+
+      DEFAULT_HANDLER_CLASSES = {
+        call: MethodCall,
+        return: MethodReturn
+      }.freeze
+
+      # @appmap
+      def initialize(tracer)
+        @pwd = Dir.pwd
+        @tracer = tracer
+        @call_stack = Hash.new { |h, k| h[k] = [] }
+        @handler_classes = {}
+      end
+
+      # @appmap
+      def handle(tp)
+        # Absoute paths which are within the current working directory are normalized
+        # to be relative paths.
+        path = tp.path
+        if path.index(@pwd) == 0
+          path = path[@pwd.length+1..-1]
+        end
+
+        method_event = \
+          if tp.event == :call && (function = @tracer.lookup_function(path, tp.lineno))
+            call_constructor = handler_class(function, tp.event)
+            call_constructor.build_from_tracepoint(tp, path).tap do |c|
+              @call_stack[Thread.current.object_id] << [ tp.defined_class, tp.method_id, c.id, Time.now, function ]
+            end
+          elsif (c = @call_stack[Thread.current.object_id].last) &&
+                c[0] == tp.defined_class &&
+                c[1] == tp.method_id
+            function = c[4]
+            @call_stack[Thread.current.object_id].pop
+            return_constructor = handler_class(function, tp.event)
+            return_constructor.build_from_tracepoint(tp, path, c[2], Time.now - c[3])
+          end
+
+        @tracer.record_event method_event if method_event
+
+        method_event
+      rescue
+        puts $!.message
+        puts $!.backtrace.join("\n")
+        # XXX If this exception doesn't get reraised, internal errors
+        # (e.g. a missing method on TracePoint) get silently
+        # ignored. This allows tests to pass that should fail, which
+        # is bad, but is it desirable otherwise?
+        raise
+      end
+
+      protected
+
+      # Figure out which handler class should be used for a trace event. It may be
+      # a custom handler, e.g. in case we are processing a special named function such as a
+      # web server entry point, or it may be the standard :call or :return handler.
+      def handler_class(function, event)
+        cache_key = [function.location, event]
+        cached_handler = @handler_classes[cache_key]
+        return cached_handler if cached_handler
+
+        return default_handler_class(event) unless function.handler_id
+
+        require "appmap/trace/event_handler/#{function.handler_id}"
+
+        AppMap::Trace::EventHandler
+          .const_get(function.handler_id.to_s.camelize)
+          .const_get(event.to_s.capitalize).tap do |handler|
+          @handler_classes[cache_key] = handler
+        end
+      end
+
+      def default_handler_class(event)
+        DEFAULT_HANDLER_CLASSES[event] or raise "No handler class for #{event.inspect}"
+      end
+    end
+
+    # @appmap
+    class Tracer
+      # Trace a specified set of functions.
+      #
+      # functions Array of AppMap::Feature::Function.
+      # @appmap
+      def initialize(functions)
+        @functions = functions
+
+        @functions_by_location = functions.each_with_object({}) do |m, memo|
+          path, lineno = m.location.split(':', 2)
+          memo[path] ||= {}
+          memo[path][lineno.to_i] = m
+          memo
+        end
+
+        @events_mutex = Mutex.new
+        @events = []
+      end
+
+      def enable
+        handler = TracePointHandler.new(self)
+        @trace_point = TracePoint.trace(:call, :return, &handler.method(:handle))
+      end
+
+      # Private function. Use AppMap.tracers#delete.
+      def disable # :nodoc:
+        @trace_point.disable
+      end
+
+      # Whether the indicated file path and lineno is a breakpoint on which
+      # execution should interrupted.
+      # @appmap
+      def lookup_function(path, lineno)
+        (methods_by_path = @functions_by_location[path]) && methods_by_path[lineno]
+      end
+
+      # Record a program execution event.
+      #
+      # The event should be one of the MethodEvent subclasses.
+      #
+      # This method is thread-safe.
+      # @appmap
+      def record_event(event)
+        @events_mutex.synchronize do
+          @events << event
+        end
+      end
+
+      # Whether there is an event available for processing.
+      #
+      # This method is thread-safe.
+      def event?
+        @events_mutex.synchronize do
+          !@events.empty?
+        end
+      end
+
+      # Gets the next available event, if any.
+      #
+      # This method is thread-safe.
+      def next_event
+        @events_mutex.synchronize do
+          @events.shift
+        end
+      end
+    end
+  end
+end

data/lib/appmap/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module AppMap
+  VERSION = '0.18.1'
+end