app_perf_rpm 0.0.2

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: b34ed99c3e8e44848ad0973d852d87017529d97e
+  data.tar.gz: 21b6044514766bc7fc0df5ab6fe191e22d896cd9
+SHA512:
+  metadata.gz: 31f24ad79ef22b4a7835daec2a2abf9bdb8d5c8d3b02b48d223eba656eda148841bb12724dc95ddd479f01241e951f42a5862378444c283fa1811bec2adeca48
+  data.tar.gz: d1f1aaf3e684d319ff06378ccac8c8e0ccf9f482dd3cc309493dca89a2c7ed79479b3a40f06117e8cc9b0589cf10aebc0781f7f0fc73bba9c461f005bf9145af

data/lib/app_perf_rpm/aggregator.rb

@@ -0,0 +1,77 @@
+require 'digest/md5'
+
+module AppPerfRpm
+  class Aggregator
+    def initialize
+    end
+
+    def aggregate(traces)
+      metrics = []
+      traces = arrange_traces(traces)
+      spans = traces_to_spans(traces)
+      spans_by_time(spans).each_pair do |time, spans|
+        group_spans(spans).each_pair do |(type, layer, domain, url, controller, action), grouped_spans|
+          opts = {}
+          opts["type"] = type if type
+          opts["layer"] = layer if layer
+          opts["domain"] = domain if domain
+          opts["url"] = url if url
+          opts["controller"] = controller if controller
+          opts["action"] = action if action
+          metrics << build_metric("trace.web.request.duration", time, grouped_spans, opts)
+        end
+      end
+
+      return metrics
+    end
+
+    private
+
+    def build_metric(metric_name, time, spans, opts)
+      num_spans = spans.uniq(&:trace_id).size
+      durations = spans.inject(0.0) {|s, x| s + x.exclusive_duration }.to_f
+      hits = spans.size
+
+      tags = {
+        "traces" => num_spans,
+        "hits" => hits
+      }.merge(opts)
+
+      ["metric", time.to_f, metric_name, durations, tags]
+    end
+
+    def arrange_traces(traces)
+      traces
+        .group_by(&:trace_id)
+        .map {|span| Span.arrange(span.last.dup) }
+    end
+
+    def traces_to_spans(traces)
+      traces
+        .map {|trace| trace.to_spans}
+        .flatten
+    end
+
+    def spans_by_time(spans)
+      spans.group_by {|span|
+        AppPerfRpm.floor_time(Time.at(span.started_at), dispatch_interval)
+      }
+    end
+
+    def group_spans(spans)
+      spans
+        .group_by {|span| [
+          span.type,
+          span.layer,
+          span.domain,
+          span.url,
+          span.controller,
+          span.action
+        ]}
+    end
+
+    def dispatch_interval
+      30
+    end
+  end
+end

data/lib/app_perf_rpm/backtrace.rb

@@ -0,0 +1,92 @@
+module AppPerfRpm
+  class Backtrace
+    class << self
+      def backtrace
+        bt = Kernel.caller
+        bt = clean(bt)
+        trim_backtrace(bt)
+      end
+
+      def clean(backtrace)
+        backtrace
+          .map {|b| clean_line(b) }
+          .select {|b| b !~ %r{lib/app_perf_rpm} }
+      end
+
+      #def source_extract(_backtrace = Kernel.caller(2))
+      #  if(trace = _backtrace.first)
+      #    file, line_number = extract_file_and_line_number(trace)
+
+      #    {
+      #      code: source_fragment(file, line_number),
+      #      line_number: line_number
+      #    }
+      #  else
+      #    nil
+      #  end
+      #end
+
+      def source_extract(_backtrace = Kernel.caller(0))
+        Array(_backtrace).select {|bt| bt[/^#{::AppPerfRpm.configuration.app_root.to_s}\//] }.map do |trace|
+          file, line_number = extract_file_and_line_number(trace)
+          source_to_hash(file, line_number)
+        end
+      end
+
+      def source_to_hash(file, line_number)
+        {
+          "file" => clean_line(file),
+          "code" => source_fragment(file, line_number),
+          "line_number" => line_number
+        }
+      end
+
+      private
+
+      def clean_line(line)
+        line
+          .sub(/#{::AppPerfRpm.configuration.app_root.to_s}\//, "[APP_PATH]/")
+          .sub(gems_regexp, '\2 (\3) [GEM_PATH]/\4')
+      end
+
+      def gems_regexp
+        gems_paths = (Gem.path | [Gem.default_dir]).map { |p| Regexp.escape(p) }
+        if gems_paths
+          %r{(#{gems_paths.join('|')})/gems/([^/]+)-([\w.]+)/(.*)}
+        else
+          nil
+        end
+      end
+
+      def source_fragment(path, line)
+        return unless AppPerfRpm.configuration.app_root
+        full_path = AppPerfRpm.configuration.app_root.join(path)
+        if File.exist?(full_path)
+          File.open(full_path, "r") do |file|
+            start = [line - 3, 0].max
+            lines = file.each_line.drop(start).take(6)
+            Hash[*(start + 1..(lines.count + start)).zip(lines).flatten]
+          end
+        end
+      end
+
+      def extract_file_and_line_number(trace)
+        file, line = trace.match(/^(.+?):(\d+).*$/, &:captures) || trace
+        [file, line.to_i]
+      end
+
+      def trim_backtrace(_backtrace)
+        return _backtrace unless _backtrace.is_a?(Array)
+
+        length = _backtrace.size
+        if length > 100
+          # Trim backtraces by getting the first 180 and last 20 lines
+          trimmed = _backtrace[0, 80] + ['...[snip]...'] + _backtrace[length - 20, 20]
+        else
+          trimmed = _backtrace
+        end
+        trimmed
+      end
+     end
+  end
+end

data/lib/app_perf_rpm/configuration.rb

@@ -0,0 +1,56 @@
+require 'yaml'
+
+module AppPerfRpm
+  class Configuration
+    attr_accessor :app_root,
+                  :host,
+                  :port,
+                  :license_key,
+                  :ssl,
+                  :sample_rate,
+                  :sample_threshold,
+                  :dispatch_interval,
+                  :application_name,
+                  :instrumentation,
+                  :agent_disabled
+
+    def initialize
+      reload
+    end
+
+    def reload
+      ::AppPerfRpm.mutex.synchronize do
+        self.app_root = app_root ? Pathname.new(app_root.to_s) : nil
+        self.host ||= default_if_blank(ENV["APP_PERF_HOST"], "http://localhost:5000")
+        self.ssl ||= false
+        self.license_key ||= default_if_blank(ENV["APP_PERF_LICENSE_KEY"], nil)
+        self.application_name ||= "Default"
+        self.sample_rate ||= 10 # Percentage of request to sample
+        self.sample_threshold ||= 0 # Minimum amount of duration to sample
+        self.dispatch_interval ||= 60 # In seconds
+        self.agent_disabled ||= default_if_blank(ENV["APP_PERF_AGENT_DISABLED"], false)
+        self.instrumentation = {
+          :rack                 => { :enabled => true, :backtrace => false, :trace_middleware => true },
+          :active_record        => { :enabled => true, :backtrace => false },
+          :active_record_import => { :enabled => true, :backtrace => false },
+          :action_view          => { :enabled => true, :backtrace => false },
+          :action_controller    => { :enabled => true, :backtrace => false },
+          :emque_consuming      => { :enabled => true, :backtrace => false },
+          :redis                => { :enabled => true, :backtrace => false },
+          :sequel               => { :enabled => true, :backtrace => false },
+          :sidekiq              => { :enabled => true, :backtrace => false },
+          :sinatra              => { :enabled => true, :backtrace => false },
+          :net_http             => { :enabled => true, :backtrace => false },
+          :typhoeus             => { :enabled => true, :backtrace => false },
+          :faraday              => { :enabled => true, :backtrace => false }
+        }
+      end
+    end
+
+    private
+
+    def default_if_blank(value, default)
+      value.nil? || value.blank? ? default : value
+    end
+  end
+end

data/lib/app_perf_rpm/dispatcher.rb

@@ -0,0 +1,85 @@
+require 'net/http'
+
+module AppPerfRpm
+  class Dispatcher
+    def initialize
+      @start_time = Time.now
+      @queue = Queue.new
+      @aggregator = Aggregator.new
+    end
+
+    def add_event(event)
+      @queue << event
+    end
+
+    def configuration
+      ::AppPerfRpm.configuration
+    end
+
+    def ready?
+      Time.now > @start_time + configuration.dispatch_interval.to_f &&
+      @queue.size.to_i > 0
+    end
+
+    def reset
+      @queue.clear
+      @start_time = Time.now
+    end
+
+    def dispatch
+      begin
+        spans = drain(@queue)
+        metrics = []#@aggregator.aggregate(spans)
+
+        dispatch_events(spans.map(&:to_a) + metrics)
+      rescue => ex
+        ::AppPerfRpm.logger.error "#{ex.inspect}"
+        ::AppPerfRpm.logger.error "#{ex.backtrace.inspect}"
+      ensure
+        reset
+      end
+    end
+
+    private
+
+    def dispatch_events(data)
+      if data && data.length > 0
+        uri = URI(url)
+
+        sock = Net::HTTP.new(uri.host, uri.port)
+        sock.use_ssl = configuration.ssl
+
+        req = Net::HTTP::Post.new(uri.path, { "Content-Type" => "application/json", "Accept-Encoding" => "gzip", "User-Agent" => "gzip" })
+        req.body = compress_body(data)
+        req.content_type = "application/octet-stream"
+
+        res = sock.start do |http|
+          http.read_timeout = 30
+          http.request(req)
+        end
+        data.clear
+      end
+    end
+
+    def compress_body(data)
+      body = Oj.dump({
+        "name" => configuration.application_name,
+        "host" => AppPerfRpm.host,
+        "data" => data
+      })
+
+      compressed_body = Zlib::Deflate.deflate(body, Zlib::DEFAULT_COMPRESSION)
+      Base64.encode64(compressed_body)
+    end
+
+    def drain(queue)
+      Array.new(queue.size) { queue.pop }
+    end
+
+    def url
+      host = configuration.host
+      license_key = configuration.license_key
+      @url ||= "#{host}/api/listener/2/#{license_key}"
+    end
+  end
+end

data/lib/app_perf_rpm/instrumentation.rb

@@ -0,0 +1,21 @@
+module AppPerfRpm
+  class Instrumentation
+    class << self
+      def load
+        pattern = File.join(File.dirname(__FILE__), 'instruments', '**', '*.rb')
+        Dir.glob(pattern) do |f|
+          begin
+            require f
+          rescue => e
+            AppPerfRpm.logger.error "Error loading instrumentation file '#{f}' : #{e}"
+            AppPerfRpm.logger.error "#{e.backtrace[0..10]}"
+          end
+        end
+
+        if defined? Rails::Railtie
+          require "app_perf_rpm/railtie"
+        end
+      end
+    end
+  end
+end

data/lib/app_perf_rpm/instruments/action_controller.rb

@@ -0,0 +1,51 @@
+module AppPerfRpm
+  module Instruments
+    module ActionController
+      def process_action_with_trace(method_name, *args)
+        if ::AppPerfRpm::Tracer.tracing?
+          AppPerfRpm::Tracer.trace('actioncontroller') do |span|
+            span.controller = self.class.name
+            span.action = self.action_name
+
+            process_action_without_trace(method_name, *args)
+          end
+        else
+          process_action_without_trace(method_name, *args)
+        end
+      end
+
+      def perform_action_with_trace(*arguments)
+        if ::AppPerfRpm::Tracer.tracing?
+          AppPerfRpm::Tracer.trace('actioncontroller') do |span|
+            span.controller = @_request.path_parameters['controller']
+            span.action = @_request.path_parameters['action']
+
+            perform_action_without_trace(*arguments)
+          end
+        else
+          perform_action_without_trace(*arguments)
+        end
+      end
+    end
+  end
+end
+
+if ::AppPerfRpm.configuration.instrumentation[:action_controller][:enabled] &&
+  defined?(::ActionController)
+  AppPerfRpm.logger.info "Initializing actioncontroller tracer."
+
+  ::ActionController::Base.send(
+    :include,
+    AppPerfRpm::Instruments::ActionController
+  )
+
+  ::ActionController::Base.class_eval do
+    if ::Rails::VERSION::MAJOR > 2
+      alias_method :process_action_without_trace, :process_action
+      alias_method :process_action, :process_action_with_trace
+    else
+      alias_method :perform_action_without_trace, :perform_action
+      alias_method :perform_action, :perform_action_with_trace
+    end
+  end
+end

data/lib/app_perf_rpm/instruments/action_view.rb

@@ -0,0 +1,126 @@
+if ::AppPerfRpm.configuration.instrumentation[:action_view][:enabled] && defined?(::ActionView)
+  if defined?(Rails) && Rails::VERSION::MAJOR == 2
+    ActionView::Partials.module_eval do
+      alias :render_partial_without_trace :render_partial
+      def render_partial(options = {})
+        if ::AppPerfRpm::Tracer.tracing? && options.key?(:partial) && options[:partial].is_a?(String)
+          opts = {
+            "method" => "render_partial",
+            "name" => options[:partial]
+          }
+
+          opts["backtrace"] = ::AppPerfRpm::Backtrace.backtrace
+          opts["source"] = ::AppPerfRpm::Backtrace.source_extract
+
+          AppPerfRpm::Tracer.trace("actionview", opts) do |span|
+            span.controller = @_request.path_parameters['controller']
+            span.action = @_request.path_parameters['action']
+            span.backtrace = ::AppPerfRpm::Backtrace.backtrace
+            span.source = ::AppPerfRpm::Backtrace.source_extract
+
+            render_partial_without_trace(options)
+          end
+        else
+          render_partial_without_trace(options)
+        end
+      end
+
+      alias :render_partial_collection_without_trace :render_partial_collection
+      def render_partial_collection(options = {})
+        if ::AppPerfRpm::Tracer.tracing?
+          AppPerfRpm::Tracer.trace("actionview") do |span|
+            span.backtrace = ::AppPerfRpm::Backtrace.backtrace
+            span.source = ::AppPerfRpm::Backtrace.source_extract
+            span.options = {
+              "method" => "render_partial_collection",
+              "name" => @path
+            }
+
+            render_partial_collection_without_trace(options)
+          end
+        else
+          render_partial_collection_without_trace(options)
+        end
+      end
+    end
+  else
+    ActionView::PartialRenderer.class_eval do
+      alias :render_partial_without_trace :render_partial
+      def render_partial
+        if ::AppPerfRpm::Tracer.tracing?
+          AppPerfRpm::Tracer.trace("actionview") do |span|
+            span.backtrace = ::AppPerfRpm::Backtrace.backtrace
+            span.source = ::AppPerfRpm::Backtrace.source_extract
+            span.options = {
+              "method" => "render_partial",
+              "name" => @options[:partial]
+            }
+
+            render_partial_without_trace
+          end
+        else
+          render_partial_without_trace
+        end
+      end
+
+      alias :render_collection_without_trace :render_collection
+      def render_collection
+        if ::AppPerfRpm::Tracer.tracing?
+          AppPerfRpm::Tracer.trace("actionview") do |span|
+            span.options = {
+              "method" => "render_collection",
+              "name" => @path
+            }
+
+            render_collection_without_trace
+          end
+        else
+          render_collection_without_trace
+        end
+      end
+    end
+
+    ::ActionView::TemplateRenderer.class_eval do
+      alias render_with_layout_without_trace render_with_layout
+
+      def render_with_layout(path, locals, *args, &block)
+        if ::AppPerfRpm::Tracer.tracing?
+          layout = nil
+
+          if path
+            if method(:find_layout).arity == 3
+              # Rails 5
+              layout = find_layout(path, locals.keys, [formats.first])
+            else
+              # Rails 3, 4
+              layout = find_layout(path, locals.keys)
+            end
+
+            @path = path
+          end
+
+          AppPerfRpm::Tracer.trace("actionview") do |span|
+            if layout
+              span.options = {
+                "method" => "render_with_layout",
+                "name" => layout.identifier,
+                "path" => @path,
+                "layout" => layout
+              }
+            else
+              span.options = {
+                "method" => "render_without_layout",
+                "path" => @path
+              }
+            end
+            render_with_layout_without_trace(path, locals, *args, &block)
+          end
+        else
+          render_with_layout_without_trace(path, locals, *args, &block)
+        end
+      end
+    end
+  end
+
+  AppPerfRpm.logger.info "Initializing actionview tracer."
+end

data/lib/app_perf_rpm/instruments/active_record/adapters/mysql2.rb

@@ -0,0 +1,45 @@
+module AppPerfRpm
+  module Instruments
+    module ActiveRecord
+      module Adapters
+        module Mysql2
+          include AppPerfRpm::Utils
+
+          IGNORE_STATEMENTS = {
+            "SCHEMA" => true,
+            "EXPLAIN" => true,
+            "CACHE" => true
+          }
+
+          def ignore_trace?(name)
+            IGNORE_STATEMENTS[name.to_s] ||
+              (name && name.to_sym == :skip_logging) ||
+              name == 'ActiveRecord::SchemaMigration Load'
+          end
+
+          def execute_with_trace(sql, name = nil)
+            if ::AppPerfRpm::Tracer.tracing?
+              if ignore_trace?(name)
+                execute_without_trace(sql, name)
+              else
+                sanitized_sql = sanitize_sql(sql, :mysql2)
+
+                AppPerfRpm::Tracer.trace('activerecord') do |span|
+                  span.options ={
+                    "adapter" => "mysql2",
+                    "query" => sanitized_sql,
+                    "name" => name
+                  }
+
+                  execute_without_trace(sql, name)
+                end
+              end
+            else
+              execute_without_trace(sql, name)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/app_perf_rpm/instruments/active_record/adapters/postgresql.rb

@@ -0,0 +1,102 @@
+module AppPerfRpm
+  module Instruments
+    module ActiveRecord
+      module Adapters
+        module Postgresql
+          include AppPerfRpm::Utils
+
+          IGNORE_STATEMENTS = {
+            "SCHEMA" => true,
+            "EXPLAIN" => true,
+            "CACHE" => true
+          }
+
+          def ignore_trace?(name)
+            IGNORE_STATEMENTS[name.to_s] ||
+              (name && name.to_sym == :skip_logging) ||
+              name == 'ActiveRecord::SchemaMigration Load'
+          end
+
+          def exec_query_with_trace(sql, name = nil, binds = [])
+            if ::AppPerfRpm::Tracer.tracing?
+              if ignore_trace?(name)
+                exec_query_without_trace(sql, name, binds)
+              else
+                sanitized_sql = sanitize_sql(sql, :postgres)
+
+                AppPerfRpm::Tracer.trace('activerecord') do |span|
+                  span.options = {
+                    "adapter" => "postgresql",
+                    "query" => sanitized_sql,
+                    "name" => name
+                  }
+                  exec_query_without_trace(sql, name, binds)
+                end
+              end
+            else
+              exec_query_without_trace(sql, name, binds)
+            end
+          end
+
+          def exec_delete_with_trace(sql, name = nil, binds = [])
+            if ::AppPerfRpm::Tracer.tracing?
+              if ignore_trace?(name)
+                exec_delete_without_trace(sql, name, binds)
+              else
+                sanitized_sql = sanitize_sql(sql)
+
+                AppPerfRpm::Tracer.trace('activerecord') do |span|
+                  span.options = {
+                    "adapter" => "postgresql",
+                    "query" => sanitized_sql,
+                    "name" => name
+                  }
+                  exec_delete_without_trace(sql, name, binds)
+                end
+              end
+            else
+              exec_delete_without_trace(sql, name, binds)
+            end
+          end
+
+          def exec_insert_with_trace(sql, name = nil, binds = [], *args)
+            if ::AppPerfRpm::Tracer.tracing?
+              if ignore_trace?(name)
+                exec_insert_without_trace(sql, name, binds, *args)
+              else
+                sanitized_sql = sanitize_sql(sql, :postgres)
+
+                AppPerfRpm::Tracer.trace('activerecord') do |span|
+                  span.options = {
+                    "adapter" => "postgresql",
+                    "query" => sanitized_sql,
+                    "name" => name
+                  }
+
+                  exec_insert_without_trace(sql, name, binds, *args)
+                end
+              end
+            else
+              exec_insert_without_trace(sql, name, binds, *args)
+            end
+          end
+
+          def begin_db_transaction_with_trace
+            if ::AppPerfRpm::Tracer.tracing?
+              AppPerfRpm::Tracer.trace('activerecord') do |span|
+                span.options = {
+                  "adapter" => "postgresql",
+                  "query" => "BEGIN"
+                }
+
+                begin_db_transaction_without_trace
+              end
+            else
+              begin_db_transaction_without_trace
+            end
+          end
+        end
+      end
+    end
+  end
+end