scout_apm 1.1.0.pre1 → 1.2.0.pre1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 6241efa65950ed1d898ba3d688c0d0e442ed4871
-  data.tar.gz: 166976b4a15c5e4481bb171abba5dcf2f155e70f
+  metadata.gz: 908063c42f604e0073f59f50d2fab04966553a20
+  data.tar.gz: 0bc50d1533419d27d62ca3692472d08e764536f7
 SHA512:
-  metadata.gz: d0f7893d4df03ff5216bac6334ffa04e7e0bfa0e50e9eb14be6cb8a3d8e58b40afe140ed0691b3e324b311349119917f686f152e885dfbebbe20376c0d66c571
-  data.tar.gz: 4f6c3c86c84349989dfb406711a35791bd7d9f550b9139d67432fd7e37f512a9d879cfd7f08bbca6f8c30d9eb730fe8c469e7b7790138d87f02d0a25ce697993
+  metadata.gz: f72beb94361832f4cd64faf3e618a83bef3989232ee68deb16472508ec82ef58b2cfea0973f95225d57b68c995a0724ded878acfa424c0c30b90f4b2d3cb63c0
+  data.tar.gz: 4dca8b6565dac5f29abe8deafa353da9512a5508981098e096b7cafb47585391cd77f8cb58a1f5c8a7e04250ff339e82d9c34d726af91d195b17bd2b11325ea0

data/CHANGELOG.markdown

@@ -1,3 +1,9 @@
+# 1.2.0
+
+* Middleware tracing - Track time in the Rack middleware that Rails sets up
+* Queue Time tracking - Track how much time is spent in the load balancer
+* Major refactor of internals to allow more flexibility for future features
+
 # 1.0.0
 
 * General Availability

data/lib/scout_apm/agent/reporting.rb

@@ -6,76 +6,80 @@ module ScoutApm
         @reporter ||= ScoutApm::Reporter.new(:checkin, config, logger)
       end
 
-      # Called in the worker thread. Merges in-memory metrics w/those on disk and reports metrics
-      # to the server.
+      # The data moves through a treadmill of reporting, coordinating several Rails processes by using an external file.
+      # * During the minute it is being recorded by the instruments, it gets
+      #   recorded into the ram of each process (in the Store class).
+      # * The minute after, each process writes its own metrics to a shared LayawayFile
+      # * The minute after that, the first process to wake up pushes the combined
+      #   data to the server, and wipes it. Next processes don't have anything to do.
+      #
+      # At any given point, there is data in each of those steps, moving its way through the process
       def process_metrics
-        logger.debug "Processing metrics"
-        run_samplers
-        capacity.process
-        payload = layaway.deposit_and_deliver
+        # First we write the previous minute's data to the shared-across-process layaway file.
+        store.write_to_layaway(layaway)
 
-        metrics = payload[:metrics]
-        slow_transactions = payload[:slow_transactions]
-
-        if payload.any?
-          add_metric_ids(metrics)
-
-          logger.warn "Metric Size is at Limit, truncating" if metrics.size == ScoutApm::Store::MAX_SIZE
-
-          # for debugging, count the total number of requests
-          total_request_count = 0
-
-          metrics.each do |meta,stats|
-            if meta.metric_name =~ /\AController/
-              total_request_count += stats.call_count
-            end
-          end
-
-          metadata = {
-            :app_root => ScoutApm::Environment.instance.root.to_s,
-            :unique_id => ScoutApm::Utils::UniqueId.simple,
-            :agent_version => ScoutApm::VERSION,
-          }
-
-          logger.debug("Metrics: #{metrics}")
-          logger.debug("SlowTrans: #{slow_transactions}")
-          logger.debug("Metadata: #{metadata.inspect}")
-
-
-          payload = ScoutApm::Serializers::PayloadSerializer.serialize(metadata, metrics, slow_transactions)
-          slow_transactions_kb = Marshal.dump(slow_transactions).size/1024 # just for performance debugging
-
-          logger.info "Delivering #{metrics.length} Metrics for #{total_request_count} requests and #{slow_transactions.length} Slow Transaction Traces"
+        # Then attempt to send 2 minutes ago's data up to the server.  This
+        # only acctually occurs if this process is the first to wake up this
+        # minute.
+        report_to_server
+      end
 
-          logger.debug "Total payload [#{payload.size/1024} KB] for #{total_request_count} requests and Slow Transactions [#{slow_transactions_kb} KB] for #{slow_transactions.size} transactions of durations: #{slow_transactions.map(&:total_call_time).join(',')}."
+      # In a running app, one process will get one period ready for delivery, the others will see 0.
+      def report_to_server
+        reporting_periods = layaway.periods_ready_for_delivery
+        reporting_periods.each do |rp|
+          deliver_period(rp)
+        end
+      end
 
-          if ScoutApm::Agent.instance.config.value("report_format") == 'json'
-            headers = {'Content-Type' => 'application/json'}
-          else
-            headers = {}
-          end
+      def deliver_period(reporting_period)
+        metrics = reporting_period.metrics_payload
+        slow_transactions = reporting_period.slow_transactions_payload
+        metadata = {
+          :app_root      => ScoutApm::Environment.instance.root.to_s,
+          :unique_id     => ScoutApm::Utils::UniqueId.simple,
+          :agent_version => ScoutApm::VERSION,
+          :agent_time    => reporting_period.timestamp.to_s,
+        }
+
+        log_deliver(metrics, slow_transactions, metadata)
+
+        payload = ScoutApm::Serializers::PayloadSerializer.serialize(metadata, metrics, slow_transactions)
+        response = reporter.report(payload, headers)
+        unless response && response.is_a?(Net::HTTPSuccess)
+          logger.warn "Error on checkin to #{reporter.uri.to_s}: #{response.inspect}"
+        end
+      rescue => e
+        logger.warn "Error on checkin to #{reporter.uri.to_s}"
+        logger.info e.message
+        logger.debug e.backtrace
+      end
 
-          response = reporter.report(payload, headers)
+      def log_deliver(metrics, slow_transactions, metadata)
+        total_request_count = metrics.
+          select { |meta,stats| meta.metric_name =~ /\AController/ }.
+          inject(0) {|sum, (_, stat)| sum + stat.call_count }
 
-          if response and response.is_a?(Net::HTTPSuccess)
-            directives = ScoutApm::Serializers::DirectiveSerializer.deserialize(response.body)
+        logger.info "Delivering #{metrics.length} Metrics for #{total_request_count} requests and #{slow_transactions.length} Slow Transaction Traces"
+        logger.debug("Metrics: #{metrics.pretty_inspect}\nSlowTrans: #{slow_transactions.pretty_inspect}\nMetadata: #{metadata.inspect.pretty_inspect}")
+      end
 
-            self.metric_lookup.merge!(directives[:metric_lookup])
-            if directives[:reset]
-              logger.debug "Resetting metric_lookup."
-              self.metric_lookup = Hash.new
-            end
-            logger.debug "Metric Cache Size: #{metric_lookup.size}"
-          elsif response
-            logger.warn "Error on checkin to #{reporter.uri.to_s}: #{response.inspect}"
-          end
+      # TODO: Move this into PayloadSerializer?
+      def headers
+        if ScoutApm::Agent.instance.config.value("report_format") == 'json'
+          headers = {'Content-Type' => 'application/json'}
+        else
+          headers = {}
         end
-      rescue
-        logger.warn "Error on checkin to #{reporter.uri.to_s}"
-        logger.info $!.message
-        logger.debug $!.backtrace
       end
 
+      # def process_metrics
+      # rescue
+        # logger.warn "Error on checkin to #{reporter.uri.to_s}"
+        # logger.info $!.message
+        # logger.debug $!.backtrace
+      # end
+
       # Before reporting, lookup metric_id for each MetricMeta. This speeds up
       # reporting on the server-side.
       def add_metric_ids(metrics)
@@ -85,21 +89,7 @@ module ScoutApm
           end
         end
       end
-
-      # Called from #process_metrics, which is run via the background worker.
-      def run_samplers
-        @samplers.each do |sampler|
-          begin
-            result = sampler.run
-            store.track!(sampler.metric_name, result, {:scope => nil}) if result
-          rescue => e
-            logger.info "Error reading #{sampler.human_name}"
-            logger.debug e.message
-            logger.debug e.backtrace.join("\n")
-          end
-        end
-      end
-    end # module Reporting
+    end
     include Reporting
-  end # class Agent
-end # module ScoutApm
+  end
+end

data/lib/scout_apm/agent.rb

@@ -58,8 +58,8 @@ module ScoutApm
         return false
       end
 
-      if !environment.app_server_integration(true).found? && !options[:skip_app_server_check]
-        logger.warn "Couldn't find a supported app server. Not starting agent."
+      if app_server_missing?(options) && background_job_missing?
+        logger.warn "Couldn't find a supported app server or background job framework. Not starting agent."
         return false
       end
 
@@ -90,9 +90,15 @@ module ScoutApm
 
       return false unless preconditions_met?(options)
 
+
       @started = true
 
-      logger.info "Starting monitoring for [#{environment.application_name}]. Framework [#{environment.framework}] App Server [#{environment.app_server}]."
+      logger.info "Starting monitoring for [#{environment.application_name}]. Framework [#{environment.framework}] App Server [#{environment.app_server}] Background Job Framework [#{environment.background_job_name}]."
+
+      # We need agent initialized to do this, so do it here instead.
+      # Clean up any old data in the layaway file, allows us to change the file
+      # structure / contents without worrying.
+      layaway.verify_layaway_file_contents
 
       load_instruments if should_load_instruments?(options)
 
@@ -110,6 +116,9 @@ module ScoutApm
         start_background_worker
         handle_exit
         logger.info "Scout Agent [#{ScoutApm::VERSION}] Initialized"
+      elsif environment.background_job_integration
+        environment.background_job_integration.install
+        logger.info "Scout Agent [#{ScoutApm::VERSION}] loaded in [#{environment.background_job_name}] master process. Monitoring will start after background job framework forks its workers."
       else
         environment.app_server_integration.install
         logger.info "Scout Agent [#{ScoutApm::VERSION}] loaded in [#{environment.app_server}] master process. Monitoring will start after server forks its workers."
@@ -173,22 +182,38 @@ module ScoutApm
       logger.info "Initializing worker thread."
       @background_worker = ScoutApm::BackgroundWorker.new
       @background_worker_thread = Thread.new do
-        @background_worker.start { process_metrics }
+        @background_worker.start {
+          # First, run periodic samplers. These should run once a minute,
+          # rather than per-request. "CPU Load" and similar.
+          run_samplers
+          capacity.process
+
+          ScoutApm::Agent.instance.process_metrics
+        }
       end
     end
 
     # If we want to skip the app_server_check, then we must load it.
     def should_load_instruments?(options={})
       return true if options[:skip_app_server_check]
-      environment.app_server_integration.found?
+      environment.app_server_integration.found? || !background_job_missing?
     end
 
     # Loads the instrumention logic.
     def load_instruments
-      case environment.framework
-      when :rails       then install_instrument(ScoutApm::Instruments::ActionControllerRails2)
-      when :rails3_or_4 then install_instrument(ScoutApm::Instruments::ActionControllerRails3)
-      when :sinatra     then install_instrument(ScoutApm::Instruments::Sinatra)
+      if !background_job_missing?
+        case environment.background_job_name
+        when :delayed_job
+          install_instrument(ScoutApm::Instruments::DelayedJob)
+        end
+      else
+        case environment.framework
+        when :rails       then install_instrument(ScoutApm::Instruments::ActionControllerRails2)
+        when :rails3_or_4 then
+          install_instrument(ScoutApm::Instruments::ActionControllerRails3)
+          install_instrument(ScoutApm::Instruments::Middleware)
+        when :sinatra     then install_instrument(ScoutApm::Instruments::Sinatra)
+        end
       end
 
       install_instrument(ScoutApm::Instruments::ActiveRecord)
@@ -216,5 +241,27 @@ module ScoutApm
     def deploy_integration
       environment.deploy_integration
     end
+
+    # TODO: Extract a proper class / registery for these. They don't really belong here
+    def run_samplers
+      @samplers.each do |sampler|
+        begin
+          result = sampler.run
+          store.track_one!(sampler.metric_type, sampler.metric_name, result) if result
+        rescue => e
+          logger.info "Error reading #{sampler.human_name}"
+          logger.debug e.message
+          logger.debug e.backtrace.join("\n")
+        end
+      end
+    end
+
+    def app_server_missing?(options)
+      !environment.app_server_integration(true).found? && !options[:skip_app_server_check]
+    end
+
+    def background_job_missing?
+      environment.background_job_integration.nil?
+    end
   end
 end

data/lib/scout_apm/background_job_integrations/delayed_job.rb

@@ -0,0 +1,19 @@
+module ScoutApm
+  module BackgroundJobIntegrations
+    class DelayedJob
+      attr_reader :logger
+
+      def name
+        :delayed_job
+      end
+
+      def present?
+        defined?(::Delayed::Job) && (File.basename($0) =~ /\Adelayed_job/)
+      end
+
+      def forking?
+        false
+      end
+    end
+  end
+end

data/lib/scout_apm/background_job_integrations/sidekiq.rb

@@ -0,0 +1,60 @@
+module ScoutApm
+  module BackgroundJobIntegrations
+    class Sidekiq
+      attr_reader :logger
+
+      def name
+        :sidekiq
+      end
+
+      def present?
+        defined?(::Sidekiq) && (File.basename($0) =~ /\Asidekiq/)
+      end
+
+      def forking?
+        true
+      end
+
+      def install
+        # ScoutApm::Tracer is not available when this class is defined
+        SidekiqMiddleware.class_eval do
+          include ScoutApm::Tracer
+        end
+        ::Sidekiq.configure_server do |config|
+          config.server_middleware do |chain|
+            chain.add SidekiqMiddleware
+          end
+        end
+        require 'sidekiq/processor' # sidekiq v4 has not loaded this file by this point
+        ::Sidekiq::Processor.class_eval do
+          old = instance_method(:initialize)
+          define_method(:initialize) do |boss|
+            ScoutApm::Agent.instance.start_background_worker
+            old.bind(self).call(boss)
+          end
+        end
+      end
+    end
+
+    class SidekiqMiddleware
+      def call(worker, msg, queue)
+        msg_args = msg["args"].first
+        job_class = msg_args["job_class"]
+        latency = (Time.now.to_f - (msg['enqueued_at'] || msg['created_at'])) * 1000
+
+        ScoutApm::Agent.instance.store.track_one!("Queue", queue, 0, {:extra_metrics => {:latency => latency}})
+        req = ScoutApm::RequestManager.lookup
+        req.start_layer( ScoutApm::Layer.new("Job", job_class) )
+
+        begin
+          yield
+        rescue
+          req.error!
+          raise
+        ensure
+          req.stop_layer
+        end
+      end
+    end
+  end
+end

data/lib/scout_apm/bucket_name_splitter.rb

@@ -13,8 +13,8 @@ module ScoutApm
     end
 
     private
-    def split_metric_name(name)
-      name.to_s.split(/\//, 2)
+    def split_metric_name(metric_name)
+      metric_name.to_s.split(/\//, 2)
     end
 
     def scope_hash

data/lib/scout_apm/capacity.rb

@@ -48,7 +48,8 @@ module ScoutApm
         window = 1.0 if window <= 0.0 # prevent divide-by-zero if clock adjusted.
         capacity = time_spent / window
         ScoutApm::Agent.instance.logger.debug "Instance/Capacity: #{capacity}"
-        ScoutApm::Agent.instance.store.track!("Instance/Capacity",capacity,:scope => nil)
+        ScoutApm::Agent.instance.store.track_one!("Instance", "Capacity", capacity)
+
         @processing_start_time = process_time
       end
     end

data/lib/scout_apm/context.rb

@@ -17,11 +17,7 @@ module ScoutApm
     end
 
     def self.current
-      Thread.current[:scout_context] ||= new
-    end
-
-    def self.clear!
-      Thread.current[:scout_context] = nil
+      RequestManager.lookup.context
     end
 
     # Add context

data/lib/scout_apm/environment.rb

@@ -23,6 +23,11 @@ module ScoutApm
       ScoutApm::ServerIntegrations::Null.new(STDOUT_LOGGER), # must be last
     ]
 
+    BACKGROUND_JOB_INTEGRATIONS = [
+      ScoutApm::BackgroundJobIntegrations::Sidekiq.new,
+      ScoutApm::BackgroundJobIntegrations::DelayedJob.new
+    ]
+
     FRAMEWORK_INTEGRATIONS = [
       ScoutApm::FrameworkIntegrations::Rails2.new,
       ScoutApm::FrameworkIntegrations::Rails3Or4.new,
@@ -120,7 +125,17 @@ module ScoutApm
     # If forking, don't start worker thread in the master process. Since it's
     # started as a Thread, it won't survive the fork.
     def forking?
-      app_server_integration.forking?
+      app_server_integration.forking? || (background_job_integration && background_job_integration.forking?)
+    end
+
+    def background_job_integration
+      @background_job_integration ||= BACKGROUND_JOB_INTEGRATIONS.detect {|integration| integration.present?}
+      #### Temporary Disable
+      nil
+    end
+
+    def background_job_name
+      background_job_integration && background_job_integration.name
     end
 
     def deploy_integration

data/lib/scout_apm/instruments/action_controller_rails_2.rb

@@ -34,7 +34,7 @@ module ScoutApm
           ScoutApm::Agent.instance.logger.info "Instrumenting ActionView::Template"
           ::ActionView::Template.class_eval do
             include ::ScoutApm::Tracer
-            instrument_method :render, :metric_name => 'View/#{path[%r{^(/.*/)?(.*)$},2]}/Rendering', :scope => true
+            instrument_method :render, :type => "View", :name => '#{path[%r{^(/.*/)?(.*)$},2]}/Rendering', :scope => true
           end
         end
 
@@ -57,9 +57,19 @@ module ScoutApm
       # applied to metrics recorded during this transaction. This lets us associate ActiveRecord calls with
       # specific controller actions.
       def perform_action_with_scout_instruments(*args, &block)
-        scout_controller_action = "Controller/#{controller_path}/#{action_name}"
-        self.class.scout_apm_trace(scout_controller_action, :uri => request.request_uri, :ip => request.remote_ip) do
+        req = ScoutApm::RequestManager.lookup
+        req.annotate_request(:uri => request.fullpath)
+        req.context.add_user(:ip => request.remote_ip)
+        req.set_headers(request.headers)
+        req.start_layer( ScoutApm::Layer.new("Controller", "#{controller_path}/#{action_name}") )
+
+        begin
           perform_action_without_scout_instruments(*args, &block)
+        rescue
+          req.error!
+          raise
+        ensure
+          req.stop_layer
         end
       end
     end

data/lib/scout_apm/instruments/action_controller_rails_3.rb

@@ -20,7 +20,7 @@ module ScoutApm
         if defined?(::ActionController) && defined?(::ActionController::Metal)
           ScoutApm::Agent.instance.logger.info "Instrumenting ActionController::Metal"
           ::ActionController::Metal.class_eval do
-            include ScoutApm::Tracer
+            # include ScoutApm::Tracer
             include ScoutApm::Instruments::ActionControllerRails3Instruments
           end
         end
@@ -29,12 +29,15 @@ module ScoutApm
           ScoutApm::Agent.instance.logger.info "Instrumenting ActionView::PartialRenderer"
           ::ActionView::PartialRenderer.class_eval do
             include ScoutApm::Tracer
+
             instrument_method :render_partial,
-              :metric_name => 'View/#{@template.virtual_path rescue "Unknown Partial"}/Rendering',
+              :type => "View",
+              :name => '#{@template.virtual_path rescue "Unknown Partial"}/Rendering',
               :scope => true
 
             instrument_method :collection_with_template,
-              :metric_name => 'View/#{@template.virtual_path rescue "Unknown Collection"}/Rendering',
+              :type => "View",
+              :name => '#{@template.virtual_path rescue "Unknown Collection"}/Rendering',
               :scope => true
           end
 
@@ -42,7 +45,8 @@ module ScoutApm
           ::ActionView::TemplateRenderer.class_eval do
             include ScoutApm::Tracer
             instrument_method :render_template,
-              :metric_name => 'View/#{args[0].virtual_path rescue "Unknown"}/Rendering',
+              :type => "View",
+              :name => '#{args[0].virtual_path rescue "Unknown"}/Rendering',
               :scope => true
           end
         end
@@ -50,24 +54,20 @@ module ScoutApm
     end
 
     module ActionControllerRails3Instruments
-      # Instruments the action and tracks errors.
       def process_action(*args)
-        scout_controller_action = "Controller/#{controller_path}/#{action_name}"
+        req = ScoutApm::RequestManager.lookup
+        req.annotate_request(:uri => request.fullpath)
+        req.context.add_user(:ip => request.remote_ip)
+        req.set_headers(request.headers)
 
-        self.class.scout_apm_trace(scout_controller_action, :uri => request.fullpath, :ip => request.remote_ip) do
-          Thread::current[:scout_apm_prof] = nil
-          StackProf.start(:mode => :wall, :interval => ScoutApm::Agent.instance.config.value("stackprof_interval"))
-
-          begin
-            super
-          rescue Exception
-            ScoutApm::Agent.instance.store.track!("Errors/Request", 1, :scope => nil)
-            raise
-          ensure
-            Thread::current[:scout_apm_scope_name] = nil
-            StackProf.stop
-            Thread::current[:scout_apm_prof] = StackProf.results
-          end
+        req.start_layer( ScoutApm::Layer.new("Controller", "#{controller_path}/#{action_name}") )
+        begin
+          super
+        rescue
+          req.error!
+          raise
+        ensure
+          req.stop_layer
         end
       end
     end

data/lib/scout_apm/instruments/active_record.rb

@@ -58,12 +58,12 @@ module ScoutApm
 
       def log_with_scout_instruments(*args, &block)
         sql, name = args
-        self.class.instrument(scout_ar_metric_name(sql,name), :desc => Utils::SqlSanitizer.new(sql).to_s) do
+        self.class.instrument("ActiveRecord", scout_ar_metric_name(sql,name), :desc => Utils::SqlSanitizer.new(sql).to_s ) do
           log_without_scout_instruments(sql, name, &block)
         end
       end
 
-      def scout_ar_metric_name(sql,name)
+      def scout_ar_metric_name(sql, name)
         # sql: SELECT "places".* FROM "places"  ORDER BY "places"."position" ASC
         # name: Place Load
         if name && (parts = name.split " ") && parts.size == 2
@@ -79,16 +79,13 @@ module ScoutApm
                             operation
                           end
                         end
-          metric = "ActiveRecord/#{model}/#{metric_name}" if metric_name
-          metric = "ActiveRecord/SQL/other" if metric.nil?
+          metric = "#{model}/#{metric_name}" if metric_name
+          metric = "SQL/other" if metric.nil?
         else
-          metric = "ActiveRecord/SQL/Unknown"
+          metric = "SQL/Unknown"
         end
         metric
       end
     end # module ActiveRecordInstruments
   end
 end
-
-
-

data/lib/scout_apm/instruments/delayed_job.rb

@@ -0,0 +1,56 @@
+module ScoutApm
+  module Instruments
+    class DelayedJob
+      attr_reader :logger
+
+      def initialize(logger=ScoutApm::Agent.instance.logger)
+        @logger = logger
+        @installed = false
+      end
+
+      def installed?
+        @installed
+      end
+
+      def install
+        @installed = true
+        if defined?(::Delayed::Worker)
+          ::Delayed::Worker.class_eval do
+            include ScoutApm::Tracer
+            include ScoutApm::Instruments::DelayedJobInstruments
+            alias run_without_scout_instruments run
+            alias run run_with_scout_instruments
+          end
+        end
+      end
+    end
+
+    module DelayedJobInstruments
+      def run_with_scout_instruments(job)
+        scout_method_name = method_from_handler(job.handler)
+        queue = job.queue
+        latency = (Time.now.to_f - job.created_at.to_f) * 1000
+
+        ScoutApm::Agent.instance.store.track_one!("Queue", queue, 0, {:extra_metrics => {:latency => latency}})
+        req = ScoutApm::RequestManager.lookup
+        req.start_layer( ScoutApm::Layer.new("Job", scout_method_name) )
+
+        begin
+          run_without_scout_instruments(job)
+        rescue
+          req.error!
+          raise
+        ensure
+          req.stop_layer
+        end
+      end
+
+      def method_from_handler(handler)
+        job_handler = YAML.load(handler)
+        klass = job_handler.object.name
+        method = job_handler.method_name
+        "#{klass}##{method}"
+      end
+    end
+  end
+end