appmap 0.42.1 → 0.46.0

data/lib/appmap/command/record.rb

@@ -27,7 +27,7 @@ module AppMap
           event_thread.join
           yield AppMap::APPMAP_FORMAT_VERSION,
                 AppMap.detect_metadata,
-                AppMap.class_map(tracer.event_methods, include_source: AppMap.include_source?),
+                AppMap.class_map(tracer.event_methods),
                 events
         end
 

data/lib/appmap/config.rb

@@ -1,8 +1,32 @@
 # frozen_string_literal: true
 
+require 'appmap/handler/net_http'
+require 'appmap/handler/rails/template'
+
 module AppMap
   class Config
+    # Specifies a code +path+ to be mapped.
+    # Options:
+    #
+    # * +gem+ may indicate a gem name that "owns" the path
+    # * +package_name+ can be used to make sure that the code is required so that it can be loaded. This is generally used with
+    #   builtins, or when the path to be required is not automatically required when bundler requires the gem.
+    # * +exclude+ can be used used to exclude sub-paths. Generally not used with +gem+.
+    # * +labels+ is used to apply labels to matching code. This is really only useful when the package will be applied to
+    #   specific functions, via TargetMethods.
+    # * +shallow+ indicates shallow mapping, in which only the entrypoint to a gem is recorded.
     Package = Struct.new(:path, :gem, :package_name, :exclude, :labels, :shallow) do
+      # This is for internal use only.
+      private_methods :gem
+
+      # Specifies the class that will convert code events into event objects.
+      attr_writer :handler_class
+
+      def handler_class
+        require 'appmap/handler/function'
+        @handler_class || AppMap::Handler::Function
+      end
+
       # Indicates that only the entry points to a package will be recorded.
       # Once the code has entered a package, subsequent calls within the package will not be
       # recorded unless the code leaves the package and re-enters it.
@@ -11,25 +35,36 @@ module AppMap
       end
 
       class << self
+        # Builds a package for a path, such as `app/models` in a Rails app. Generally corresponds to a `path:` entry
+        # in appmap.yml. Also used for mapping specific methods via TargetMethods.
         def build_from_path(path, shallow: false, package_name: nil, exclude: [], labels: [])
           Package.new(path, nil, package_name, exclude, labels, shallow)
         end
 
-        def build_from_gem(gem, shallow: true, package_name: nil, exclude: [], labels: [])
-          if %w[method_source activesupport].member?(gem)
+        # Builds a package for gem. Generally corresponds to a `gem:` entry in appmap.yml. Also used when mapping
+        # a builtin.
+        def build_from_gem(gem, shallow: true, package_name: nil, exclude: [], labels: [], optional: false, force: false)
+          if !force && %w[method_source activesupport].member?(gem)
             warn "WARNING: #{gem} cannot be AppMapped because it is a dependency of the appmap gem"
             return
           end
-          Package.new(gem_path(gem), gem, package_name, exclude, labels, shallow)
+          path = gem_path(gem, optional)
+          if path
+            Package.new(path, gem, package_name, exclude, labels, shallow)
+          else
+            warn "#{gem} is not available in the bundle" if AppMap::Hook::LOG
+          end
         end
 
         private_class_method :new
 
         protected
 
-        def gem_path(gem)
-          gemspec = Gem.loaded_specs[gem] or raise "Gem #{gem.inspect} not found"
-          gemspec.gem_dir
+        def gem_path(gem, optional)
+          gemspec = Gem.loaded_specs[gem]
+          # This exception will notify a user that their appmap.yml contains non-existent gems.
+          raise "Gem #{gem.inspect} not found" unless gemspec || optional
+          gemspec ? gemspec.gem_dir : nil
         end
       end
 
@@ -42,6 +77,7 @@ module AppMap
           path: path,
           package_name: package_name,
           gem: gem,
+          handler_class: handler_class.name,
           exclude: exclude.blank? ? nil : exclude,
           labels: labels.blank? ? nil : labels,
           shallow: shallow
@@ -49,44 +85,109 @@ module AppMap
       end
     end
 
-    Hook = Struct.new(:method_names, :package) do
+    Function = Struct.new(:package, :cls, :labels, :function_names) do # :nodoc:
+      def to_h
+        {
+          package: package,
+          class: cls,
+          labels: labels,
+          functions: function_names.map(&:to_sym)
+        }.compact
+      end
     end
+    private_constant :Function
 
-    OPENSSL_PACKAGES = Package.build_from_path('openssl', package_name: 'openssl', labels: %w[security crypto])
+    class TargetMethods # :nodoc:
+      attr_reader :method_names, :package
+
+      def initialize(method_names, package)
+        @method_names = method_names
+        @package = package
+      end
+
+      def include_method?(method_name)
+        Array(method_names).include?(method_name)
+      end
+
+      def to_h
+        {
+          package: package.name,
+          method_names: method_names
+        }
+      end
+    end
+    private_constant :TargetMethods
+
+    OPENSSL_PACKAGES = ->(labels) { Package.build_from_path('openssl', package_name: 'openssl', labels: labels) }
 
     # Methods that should always be hooked, with their containing
     # package and labels that should be applied to them.
     HOOKED_METHODS = {
-      'ActiveSupport::SecurityUtils' => Hook.new(:secure_compare, Package.build_from_path('active_support', labels: %w[provider.secure_compare])),
-      'ActionView::Renderer' => Hook.new(:render, Package.build_from_path('action_view', labels: %w[mvc.view])),
-      'ActionDispatch::Cookies::CookieJar' => Hook.new(%i[[]= clear update delete recycle], Package.build_from_path('action_pack', labels: %w[provider.http.cookie])),
-      'ActionDispatch::Cookies::EncryptedCookieJar' => Hook.new(%i[[]=], Package.build_from_path('action_pack', labels: %w[provider.http.cookie crypto])),
-      'CanCan::ControllerAdditions' => Hook.new(%i[authorize! can? cannot?], Package.build_from_path('cancancan', labels: %w[provider.authorization])),
-      'CanCan::Ability' => Hook.new(%i[authorize!], Package.build_from_path('cancancan', labels: %w[provider.authorization])),
+      'ActionView::Renderer' => TargetMethods.new(:render, Package.build_from_gem('actionview', shallow: false, package_name: 'action_view', labels: %w[mvc.view], optional: true).tap do |package|
+        package.handler_class = AppMap::Handler::Rails::Template::RenderHandler if package
+      end),
+      'ActionView::Resolver' => TargetMethods.new(%i[find_all find_all_anywhere], Package.build_from_gem('actionview', shallow: false, package_name: 'action_view', labels: %w[mvc.template.resolver], optional: true).tap do |package|
+        package.handler_class = AppMap::Handler::Rails::Template::ResolverHandler if package
+      end),
+      'ActionDispatch::Request::Session' => TargetMethods.new(%i[destroy [] dig values []= clear update delete fetch merge], Package.build_from_gem('actionpack', shallow: false, package_name: 'action_dispatch', labels: %w[http.session], optional: true)),
+      'ActionDispatch::Cookies::CookieJar' => TargetMethods.new(%i[[]= clear update delete recycle], Package.build_from_gem('actionpack', shallow: false, package_name: 'action_dispatch', labels: %w[http.cookie], optional: true)),
+      'ActionDispatch::Cookies::EncryptedCookieJar' => TargetMethods.new(%i[[]=], Package.build_from_gem('actionpack', shallow: false, package_name: 'action_dispatch', labels: %w[http.cookie crypto.encrypt], optional: true)),
+      'CanCan::ControllerAdditions' => TargetMethods.new(%i[authorize! can? cannot?], Package.build_from_gem('cancancan', shallow: false, labels: %w[security.authorization], optional: true)),
+      'CanCan::Ability' => TargetMethods.new(%i[authorize!], Package.build_from_gem('cancancan', shallow: false, labels: %w[security.authorization], optional: true)),
+      'ActionController::Instrumentation' => [
+        TargetMethods.new(%i[process_action send_file send_data redirect_to], Package.build_from_gem('actionpack', shallow: false, package_name: 'action_controller', labels: %w[mvc.controller], optional: true))
+      ]
     }.freeze
 
     BUILTIN_METHODS = {
-      'OpenSSL::PKey::PKey' => Hook.new(:sign, OPENSSL_PACKAGES),
-      'OpenSSL::X509::Request' => Hook.new(%i[sign verify], OPENSSL_PACKAGES),
-      'OpenSSL::PKCS5' => Hook.new(%i[pbkdf2_hmac_sha1 pbkdf2_hmac], OPENSSL_PACKAGES),
-      'OpenSSL::Cipher' => Hook.new(%i[encrypt decrypt final], OPENSSL_PACKAGES),
-      'OpenSSL::X509::Certificate' => Hook.new(:sign, OPENSSL_PACKAGES),
-      'Net::HTTP' => Hook.new(:request, Package.build_from_path('net/http', package_name: 'net/http', labels: %w[protocol.http io])),
-      'Net::SMTP' => Hook.new(:send, Package.build_from_path('net/smtp', package_name: 'net/smtp', labels: %w[protocol.smtp protocol.email io])),
-      'Net::POP3' => Hook.new(:mails, Package.build_from_path('net/pop3', package_name: 'net/pop', labels: %w[protocol.pop protocol.email io])),
-      'Net::IMAP' => Hook.new(:send_command, Package.build_from_path('net/imap', package_name: 'net/imap', labels: %w[protocol.imap protocol.email io])),
-      'Marshal' => Hook.new(%i[dump load], Package.build_from_path('marshal', labels: %w[format.marshal provider.serialization])),
-      'Psych' => Hook.new(%i[dump dump_stream load load_stream parse parse_stream], Package.build_from_path('yaml', package_name: 'psych', labels: %w[format.yaml provider.serialization])),
-      'JSON::Ext::Parser' => Hook.new(:parse, Package.build_from_path('json', package_name: 'json', labels: %w[format.json provider.serialization])),
-      'JSON::Ext::Generator::State' => Hook.new(:generate, Package.build_from_path('json', package_name: 'json', labels: %w[format.json provider.serialization])),
+      'OpenSSL::PKey::PKey' => TargetMethods.new(:sign, OPENSSL_PACKAGES.(%w[crypto.pkey])),
+      'OpenSSL::X509::Request' => TargetMethods.new(%i[sign verify], OPENSSL_PACKAGES.(%w[crypto.x509])),
+      'OpenSSL::PKCS5' => TargetMethods.new(%i[pbkdf2_hmac_sha1 pbkdf2_hmac], OPENSSL_PACKAGES.(%w[crypto.pkcs5])),
+      'OpenSSL::Cipher' => [
+        TargetMethods.new(%i[encrypt], OPENSSL_PACKAGES.(%w[crypto.encrypt])),
+        TargetMethods.new(%i[decrypt], OPENSSL_PACKAGES.(%w[crypto.decrypt]))
+      ],
+      'ActiveSupport::Callbacks::CallbackSequence' => [
+        TargetMethods.new(:invoke_before, Package.build_from_gem('activesupport', force: true, package_name: 'active_support', labels: %w[mvc.before_action])),
+        TargetMethods.new(:invoke_after, Package.build_from_gem('activesupport', force: true, package_name: 'active_support', labels: %w[mvc.after_action])),
+      ],
+      'ActiveSupport::SecurityUtils' => TargetMethods.new(:secure_compare, Package.build_from_gem('activesupport', force: true, package_name: 'active_support/security_utils', labels: %w[crypto.secure_compare])),
+      'OpenSSL::X509::Certificate' => TargetMethods.new(:sign, OPENSSL_PACKAGES.(%w[crypto.x509])),
+      'Net::HTTP' => TargetMethods.new(:request, Package.build_from_path('net/http', package_name: 'net/http', labels: %w[protocol.http]).tap do |package|
+        package.handler_class = AppMap::Handler::NetHTTP
+      end),
+      'Net::SMTP' => TargetMethods.new(:send, Package.build_from_path('net/smtp', package_name: 'net/smtp', labels: %w[protocol.email.smtp])),
+      'Net::POP3' => TargetMethods.new(:mails, Package.build_from_path('net/pop3', package_name: 'net/pop', labels: %w[protocol.email.pop])),
+      # This is happening: Method send_command not found on Net::IMAP
+      # 'Net::IMAP' => TargetMethods.new(:send_command, Package.build_from_path('net/imap', package_name: 'net/imap', labels: %w[protocol.email.imap])),
+      # 'Marshal' => TargetMethods.new(%i[dump load], Package.build_from_path('marshal', labels: %w[format.marshal])),
+      'Psych' => TargetMethods.new(%i[dump dump_stream load load_stream parse parse_stream], Package.build_from_path('yaml', package_name: 'psych', labels: %w[format.yaml])),
+      'JSON::Ext::Parser' => TargetMethods.new(:parse, Package.build_from_path('json', package_name: 'json', labels: %w[format.json])),
+      'JSON::Ext::Generator::State' => TargetMethods.new(:generate, Package.build_from_path('json', package_name: 'json', labels: %w[format.json])),
     }.freeze
 
-    attr_reader :name, :packages, :exclude
+    attr_reader :name, :packages, :exclude, :hooked_methods, :builtin_methods
 
-    def initialize(name, packages = [], exclude = [])
+    def initialize(name, packages, exclude: [], functions: [])
       @name = name
       @packages = packages
+      @hook_paths = packages.map(&:path)
       @exclude = exclude
+      @builtin_methods = BUILTIN_METHODS
+      @functions = functions
+      @hooked_methods = HOOKED_METHODS.dup
+      functions.each do |func|
+        package_options = {}
+        package_options[:labels] = func.labels if func.labels
+        @hooked_methods[func.cls] ||= []
+        @hooked_methods[func.cls] << TargetMethods.new(func.function_names, Package.build_from_path(func.package, package_options))
+      end
+
+      @hooked_methods.each_value do |hooks|
+        Array(hooks).each do |hook|
+          @hook_paths << hook.package.path if hook.package
+        end
+      end
     end
 
     class << self
@@ -98,6 +199,16 @@ module AppMap
 
       # Loads configuration from a Hash.
       def load(config_data)
+        functions = (config_data['functions'] || []).map do |function_data|
+          package = function_data['package']
+          cls = function_data['class']
+          functions = function_data['function'] || function_data['functions']
+          raise 'AppMap class configuration should specify package, class and function(s)' unless package && cls && functions
+          functions = Array(functions).map(&:to_sym)
+          labels = function_data['label'] || function_data['labels']
+          labels = Array(labels).map(&:to_s) if labels
+          Function.new(package, cls, labels, functions)
+        end
         packages = (config_data['packages'] || []).map do |package|
           gem = package['gem']
           path = package['path']
@@ -112,7 +223,8 @@ module AppMap
             Package.build_from_path(path, exclude: package['exclude'] || [], shallow: package['shallow'])
           end
         end.compact
-        Config.new config_data['name'], packages, config_data['exclude'] || []
+        exclude = config_data['exclude'] || []
+        Config.new config_data['name'], packages, exclude: exclude, functions: functions
       end
     end
 
@@ -120,58 +232,59 @@ module AppMap
       {
         name: name,
         packages: packages.map(&:to_h),
+        functions: @functions.map(&:to_h),
         exclude: exclude
       }
     end
 
-    # package_for_method finds the Package, if any, which configures the hook
-    # for a method.
-    def package_for_method(method)
-      package_hooked_by_class(method) || package_hooked_by_source_location(method)
+    # Determines if methods defined in a file path should possibly be hooked.
+    def path_enabled?(path)
+      path = AppMap::Util.normalize_path(path)
+      @hook_paths.find { |hook_path| path.index(hook_path) == 0 }
     end
 
-    def package_hooked_by_class(method)
-      defined_class, _, method_name = ::AppMap::Hook.qualify_method_name(method)
-      return find_package(defined_class, method_name)
-    end
+    # Looks up a class and method in the config, to find the matching Package configuration.
+    # This class is only used after +path_enabled?+ has returned `true`. 
+    LookupPackage = Struct.new(:config, :cls, :method) do
+      def package
+        # Global "excludes" configuration can be used to ignore any class/method.
+        return if config.never_hook?(cls, method)
 
-    def package_hooked_by_source_location(method)
-      location = method.source_location
-      location_file, = location
-      return unless location_file
-
-      location_file = location_file[Dir.pwd.length + 1..-1] if location_file.index(Dir.pwd) == 0
-      packages.select { |pkg| pkg.path }.find do |pkg|
-        (location_file.index(pkg.path) == 0) &&
-          !pkg.exclude.find { |p| location_file.index(p) }
+        package_for_code_object || package_for_location
       end
-    end
 
-    def never_hook?(method)
-      defined_class, separator, method_name = ::AppMap::Hook.qualify_method_name(method)
-      return true if exclude.member?(defined_class) || exclude.member?([ defined_class, separator, method_name ].join)
-    end
+      # Hook a method which is specified by class and method name.
+      def package_for_code_object
+        Array(config.hooked_methods[cls.name])
+          .compact
+          .find { |hook| hook.include_method?(method.name) }
+          &.package
+      end
 
-    # always_hook? indicates a method that should always be hooked.
-    def always_hook?(defined_class, method_name)
-      !!find_package(defined_class, method_name)
-    end
+      # Hook a method which is specified by code location (i.e. path).
+      def package_for_location
+        location = method.source_location
+        location_file, = location
+        return unless location_file
 
-    # included_by_location? indicates a method whose source location matches a method definition that has been
-    # configured for inclusion.
-    def included_by_location?(method)
-      !!package_for_method(method)
+        location_file = AppMap::Util.normalize_path(location_file)
+        config
+          .packages
+          .select { |pkg| pkg.path }
+          .find do |pkg|
+            (location_file.index(pkg.path) == 0) &&
+              !pkg.exclude.find { |p| location_file.index(p) }
+          end
+      end
     end
 
-    def find_package(defined_class, method_name)
-      hook = find_hook(defined_class)
-      return nil unless hook
-
-      Array(hook.method_names).include?(method_name) ? hook.package : nil
+    def lookup_package(cls, method)
+      LookupPackage.new(self, cls, method).package
     end
 
-    def find_hook(defined_class)
-      HOOKED_METHODS[defined_class] || BUILTIN_METHODS[defined_class]
+    def never_hook?(cls, method)
+      _, separator, = ::AppMap::Hook.qualify_method_name(method)
+      return true if exclude.member?(cls.name) || exclude.member?([ cls.name, separator, method.name ].join)
     end
   end
 end

data/lib/appmap/cucumber.rb

@@ -50,7 +50,7 @@ module AppMap
         appmap['metadata'] = update_metadata(scenario, appmap['metadata'])
         scenario_filename = AppMap::Util.scenario_filename(appmap['metadata']['name'])
 
-        File.write(File.join(APPMAP_OUTPUT_DIR, scenario_filename), JSON.generate(appmap))
+        AppMap::Util.write_appmap(File.join(APPMAP_OUTPUT_DIR, scenario_filename), JSON.generate(appmap))
       end
 
       def enabled?

data/lib/appmap/event.rb

@@ -21,10 +21,10 @@ module AppMap
       LIMIT = 100
 
       class << self
-        def build_from_invocation(me, event_type)
-          me.id = AppMap::Event.next_id_counter
-          me.event = event_type
-          me.thread_id = Thread.current.object_id
+        def build_from_invocation(event_type, event:)
+          event.id = AppMap::Event.next_id_counter
+          event.event = event_type
+          event.thread_id = Thread.current.object_id
         end
 
         # Gets a display string for a value. This is not meant to be a machine deserializable value.
@@ -36,7 +36,17 @@ module AppMap
           (value_string||'')[0...LIMIT].encode('utf-8', invalid: :replace, undef: :replace, replace: '_')
         end
 
-        protected
+        def object_properties(hash_like)
+          hash = hash_like.to_h
+          hash.keys.map do |key|
+            {
+              name: key,
+              class: hash[key].class.name,
+            }
+          end
+        rescue
+          nil
+        end
 
         # Heuristic for dynamically defined class whose name can be nil
         def best_class_name(value)
@@ -79,25 +89,32 @@ module AppMap
           end
         end
       end
+
+      protected
+
+      def object_properties(hash_like)
+        self.class.object_properties(hash_like)
+      end
     end
 
     class MethodCall < MethodEvent
       attr_accessor :defined_class, :method_id, :path, :lineno, :parameters, :receiver, :static
 
       class << self
-        def build_from_invocation(mc = MethodCall.new, defined_class, method, receiver, arguments)
+        def build_from_invocation(defined_class, method, receiver, arguments, event: MethodCall.new)
+          event ||= MethodCall.new
           defined_class ||= 'Class'
-          mc.tap do
+          event.tap do
             static = receiver.is_a?(Module)
-            mc.defined_class = defined_class
-            mc.method_id = method.name.to_s
+            event.defined_class = defined_class
+            event.method_id = method.name.to_s
             if method.source_location
               path = method.source_location[0]
               path = path[Dir.pwd.length + 1..-1] if path.index(Dir.pwd) == 0
-              mc.path = path
-              mc.lineno = method.source_location[1]
+              event.path = path
+              event.lineno = method.source_location[1]
             else
-              mc.path = [ defined_class, static ? '.' : '#', method.name ].join
+              event.path = [ defined_class, static ? '.' : '#', method.name ].join
             end
 
             # Check if the method has key parameters. If there are any they'll always be last.
@@ -105,7 +122,7 @@ module AppMap
             has_key = [[:dummy], *method.parameters].last.first.to_s.start_with?('key') && arguments[-1].is_a?(Hash)
             kwargs = has_key && arguments[-1].dup || {}
 
-            mc.parameters = method.parameters.map.with_index do |method_param, idx|
+            event.parameters = method.parameters.map.with_index do |method_param, idx|
               param_type, param_name = method_param
               param_name ||= 'arg'
               value = case param_type
@@ -126,13 +143,13 @@ module AppMap
                 kind: param_type
               }
             end
-            mc.receiver = {
+            event.receiver = {
               class: best_class_name(receiver),
               object_id: receiver.__id__,
               value: display_string(receiver)
             }
-            mc.static = static
-            MethodEvent.build_from_invocation(mc, :call)
+            event.static = static
+            MethodEvent.build_from_invocation(:call, event: event)
           end
         end
       end
@@ -157,11 +174,12 @@ module AppMap
       attr_accessor :parent_id, :elapsed
 
       class << self
-        def build_from_invocation(mr = MethodReturnIgnoreValue.new, parent_id, elapsed)
-          mr.tap do |_|
-            mr.parent_id = parent_id
-            mr.elapsed = elapsed
-            MethodEvent.build_from_invocation(mr, :return)
+        def build_from_invocation(parent_id, elapsed: nil, event: MethodReturnIgnoreValue.new)
+          event ||= MethodReturnIgnoreValue.new
+          event.tap do |_|
+            event.parent_id = parent_id
+            event.elapsed = elapsed
+            MethodEvent.build_from_invocation(:return, event: event)
           end
         end
       end
@@ -169,7 +187,7 @@ module AppMap
       def to_h
         super.tap do |h|
           h[:parent_id] = parent_id
-          h[:elapsed] = elapsed
+          h[:elapsed] = elapsed if elapsed
         end
       end
     end
@@ -178,10 +196,11 @@ module AppMap
       attr_accessor :return_value, :exceptions
 
       class << self
-        def build_from_invocation(mr = MethodReturn.new, parent_id, elapsed, return_value, exception)
-          mr.tap do |_|
+        def build_from_invocation(parent_id, return_value, exception, elapsed: nil, event: MethodReturn.new)
+          event ||= MethodReturn.new
+          event.tap do |_|
             if return_value
-              mr.return_value = {
+              event.return_value = {
                 class: best_class_name(return_value),
                 value: display_string(return_value),
                 object_id: return_value.__id__
@@ -202,9 +221,9 @@ module AppMap
                 next_exception = next_exception.cause
               end
 
-              mr.exceptions = exceptions
+              event.exceptions = exceptions
             end
-            MethodReturnIgnoreValue.build_from_invocation(mr, parent_id, elapsed)
+            MethodReturnIgnoreValue.build_from_invocation(parent_id, elapsed: elapsed, event: event)
           end
         end
       end