scout_apm 3.0.0.pre13 → 3.0.0.pre14

data/lib/scout_apm/middleware.rb

@@ -5,8 +5,10 @@ module ScoutApm
     def initialize(app)
       @app = app
       @attempts = 0
-      @enabled = ScoutApm::Agent.instance.apm_enabled?
-      @started = ScoutApm::Agent.instance.started? && ScoutApm::Agent.instance.background_worker_running?
+      # @enabled = ScoutApm::Agent.instance.context.apm_enabled?
+      # XXX: Figure out if this middleware should even know
+      @enabled = true
+      @started = ScoutApm::Agent.instance.context.started? && ScoutApm::Agent.instance.background_worker_running?
     end
 
     # If we get a web request in, then we know we're running in some sort of app server
@@ -21,14 +23,10 @@ module ScoutApm
 
     def attempt_to_start_agent
       @attempts += 1
-      ScoutApm::Agent.instance.start(:skip_app_server_check => true)
-      ScoutApm::Agent.instance.start_background_worker
-      @started = ScoutApm::Agent.instance.started? && ScoutApm::Agent.instance.background_worker_running?
+      ScoutApm::Agent.instance.start
+      @started = ScoutApm::Agent.instance.context.started? && ScoutApm::Agent.instance.background_worker_running?
     rescue => e
-      # Can't be sure of any logging here, so fall back to ENV var and STDOUT
-      if ENV["SCOUT_LOG_LEVEL"] == "debug"
-        STDOUT.puts "Failed to start via Middleware: #{e.message}\n\t#{e.backtrace.join("\n\t")}"
-      end
+      ScoutApm::Agent.instance.context.logger("Failed to start via Middleware: #{e.message}\n\t#{e.backtrace.join("\n\t")}")
     end
   end
 end

data/lib/scout_apm/periodic_work.rb

@@ -0,0 +1,28 @@
+module ScoutApm
+  class PeriodicWork
+    attr_reader :context
+
+    def initialize(context)
+      @context = context
+      @reporting = ScoutApm::Reporting.new(context)
+    end
+
+    # Expected to be called many times over the life of the agent
+    def run
+      ScoutApm::Debug.instance.call_periodic_hooks
+      @reporting.process_metrics
+      clean_old_percentiles
+    end
+
+    private
+
+    # XXX: Move logic into a RequestHistogramsByTime class that can keep the timeout logic in it
+    def clean_old_percentiles
+      context.
+        request_histograms_by_time.
+        keys.
+        select {|timestamp| timestamp.age_in_seconds > 60 * 10 }.
+        each {|old_timestamp| context.request_histograms_by_time.delete(old_timestamp) }
+    end
+  end
+end

data/lib/scout_apm/reporter.rb

@@ -5,18 +5,24 @@ module ScoutApm
     CA_FILE     = File.join( File.dirname(__FILE__), *%w[.. .. data cacert.pem] )
     VERIFY_MODE = OpenSSL::SSL::VERIFY_PEER | OpenSSL::SSL::VERIFY_FAIL_IF_NO_PEER_CERT
 
-    attr_reader :config
-    attr_reader :logger
     attr_reader :type
+    attr_reader :context
     attr_reader :instant_key
 
-    def initialize(type = :checkin, config=Agent.instance.config, logger=Agent.instance.logger, instant_key=nil)
-      @config = config
-      @logger = logger
+    def initialize(context, type, instant_key=nil)
+      @context = context
       @type = type
       @instant_key = instant_key
     end
 
+    def config
+      context.config
+    end
+
+    def logger
+      context.logger
+    end
+
     def report(payload, headers = {})
       hosts = determine_hosts
 
@@ -27,14 +33,14 @@ module ScoutApm
         headers.merge!(compression_headers)
 
         compress_payload_size = payload.length
-        ScoutApm::Agent.instance.logger.debug("Original Size: #{original_payload_size} Compressed Size: #{compress_payload_size}")
+        logger.debug("Original Size: #{original_payload_size} Compressed Size: #{compress_payload_size}")
       end
 
       post_payload(hosts, payload, headers)
     end
 
     def uri(host)
-      encoded_app_name = CGI.escape(Environment.instance.application_name)
+      encoded_app_name = CGI.escape(context.environment.application_name)
       key = config.value('key')
 
       case type
@@ -100,7 +106,7 @@ module ScoutApm
 
     # Headers passed up with all API requests.
     def default_http_headers
-      { "Agent-Hostname" => ScoutApm::Environment.instance.hostname,
+      { "Agent-Hostname" => context.environment.hostname,
         "Content-Type"   => "application/octet-stream",
         "Agent-Version"  => ScoutApm::VERSION,
       }

data/lib/scout_apm/reporting.rb

@@ -0,0 +1,135 @@
+# Methods related to sending metrics to scoutapp.com.
+module ScoutApm
+  class Reporting
+    attr_reader :context
+
+    def initialize(context)
+      @context = context
+    end
+
+    def logger
+      context.logger
+    end
+
+    def reporter
+      @reporter ||= ScoutApm::Reporter.new(context, :checkin)
+    end
+
+    # The data moves through a treadmill of reporting, coordinating several Rails processes by using an external file.
+    # * During the minute it is being recorded by the instruments, it gets
+    #   recorded into the ram of each process (in the Store class).
+    # * The minute after, each process writes its own metrics to a shared LayawayFile
+    # * The minute after that, the first process to wake up pushes the combined
+    #   data to the server, and wipes it. Next processes don't have anything to do.
+    #
+    # At any given point, there is data in each of those steps, moving its way through the process
+    def process_metrics
+      # Write the previous minute's data to the shared-across-process layaway file.
+      context.store.write_to_layaway(context.layaway)
+
+      # Attempt to send 2 minutes ago's data up to the server.  This
+      # only acctually occurs if this process is the first to wake up this
+      # minute.
+      report_to_server
+    end
+
+    # In a running app, one process will get the period ready for delivery, the others will see 0.
+    def report_to_server
+      period_to_report = ScoutApm::StoreReportingPeriodTimestamp.minutes_ago(2)
+
+      logger.debug("Attempting to claim #{period_to_report.to_s}")
+
+      did_write = context.layaway.with_claim(period_to_report) do |rps|
+        logger.debug("Succeeded claiming #{period_to_report.to_s}")
+
+        begin
+          merged = rps.inject { |memo, rp| memo.merge(rp) }
+          logger.debug("Merged #{rps.length} reporting periods, delivering")
+          deliver_period(merged)
+          true
+        rescue => e
+          logger.debug("Error merging reporting periods #{e.message}")
+          logger.debug("Error merging reporting periods #{e.backtrace}")
+          false
+        end
+
+      end
+
+      if !did_write
+        logger.debug("Failed to obtain claim for #{period_to_report.to_s}")
+      end
+    end
+
+    def deliver_period(reporting_period)
+      metrics = reporting_period.metrics_payload
+      slow_transactions = reporting_period.slow_transactions_payload
+      jobs = reporting_period.jobs
+      slow_jobs = reporting_period.slow_jobs_payload
+      histograms = reporting_period.histograms
+      db_query_metrics = reporting_period.db_query_metrics_payload
+
+      metadata = {
+        :app_root      => context.environment.root.to_s,
+        :unique_id     => ScoutApm::Utils::UniqueId.simple,
+        :agent_version => ScoutApm::VERSION,
+        :agent_time    => reporting_period.timestamp.to_s,
+        :agent_pid     => Process.pid,
+        :platform      => "ruby",
+      }
+
+      log_deliver(metrics, slow_transactions, metadata, slow_jobs, histograms)
+
+      payload = ScoutApm::Serializers::PayloadSerializer.serialize(metadata, metrics, slow_transactions, jobs, slow_jobs, histograms, db_query_metrics)
+      logger.debug("Sending payload w/ Headers: #{headers.inspect}")
+
+      reporter.report(payload, headers)
+    rescue => e
+      logger.warn "Error on checkin"
+      logger.info e.message
+      logger.debug e.backtrace
+    end
+
+    def log_deliver(metrics, slow_transactions, metadata, jobs_traces, histograms)
+      total_request_count = metrics.
+        select { |meta,stats| meta.metric_name =~ /\AController/ }.
+        inject(0) {|sum, (_, stat)| sum + stat.call_count }
+
+      memory = metrics.
+        find {|meta,stats| meta.metric_name =~ /\AMemory/ }
+      process_log_str = if memory
+                          "Recorded from #{memory.last.call_count} processes"
+                        else
+                          "Recorded across (unknown) processes"
+                        end
+
+      time_clause       = "[#{Time.parse(metadata[:agent_time]).strftime("%H:%M")}]"
+      metrics_clause    = "#{metrics.length} Metrics for #{total_request_count} requests"
+      slow_trans_clause = "#{slow_transactions.length} Slow Transaction Traces"
+      job_clause        = "#{jobs_traces.length} Job Traces"
+      histogram_clause  = "#{histograms.length} Histograms"
+
+      logger.info "#{time_clause} Delivering #{metrics_clause} and #{slow_trans_clause} and #{job_clause}, #{process_log_str}."
+      logger.debug("\n\nMetrics: #{metrics.pretty_inspect}\nSlowTrans: #{slow_transactions.pretty_inspect}\nMetadata: #{metadata.inspect.pretty_inspect}\n\n")
+    end
+
+    # TODO: Move this into PayloadSerializer?
+    # XXX: Remove non-json report format entirely
+    def headers
+      if ScoutApm::Agent.instance.context.config.value("report_format") == 'json'
+        headers = {'Content-Type' => 'application/json'}
+      else
+        headers = {}
+      end
+    end
+
+    # Before reporting, lookup metric_id for each MetricMeta. This speeds up
+    # reporting on the server-side.
+    def add_metric_ids(metrics)
+      metrics.each do |meta,stats|
+        if metric_id = metric_lookup[meta]
+          meta.metric_id = metric_id
+        end
+      end
+    end
+  end
+end

data/lib/scout_apm/request_manager.rb

@@ -19,13 +19,11 @@ module ScoutApm
     end
 
     # Create a new TrackedRequest object for this thread
+    # XXX: Figure out who is in charge of creating a `FakeStore` - previously was here
     def self.create
-      store = if ScoutApm::Agent.instance.apm_enabled?
-                ScoutApm::Agent.instance.store
-              else
-                ScoutApm::FakeStore.new
-              end
-      Thread.current[:scout_request] = TrackedRequest.new(store)
+      agent_context = ScoutApm::Agent.instance.context
+      store = agent_context.store
+      Thread.current[:scout_request] = TrackedRequest.new(agent_context, store)
     end
   end
 end

data/lib/scout_apm/serializers/payload_serializer.rb

@@ -3,7 +3,7 @@ module ScoutApm
   module Serializers
     class PayloadSerializer
       def self.serialize(metadata, metrics, slow_transactions, jobs, slow_jobs, histograms, db_query_metrics)
-        if ScoutApm::Agent.instance.config.value("report_format") == 'json'
+        if ScoutApm::Agent.instance.context.config.value("report_format") == 'json'
           ScoutApm::Serializers::PayloadSerializerToJson.serialize(metadata, metrics, slow_transactions, jobs, slow_jobs, histograms, db_query_metrics)
         else
           metadata = metadata.dup

data/lib/scout_apm/slow_job_policy.rb

@@ -24,8 +24,12 @@ module ScoutApm
     # has been running.
     attr_reader :last_seen
 
+    # The AgentContext we're running in
+    attr_reader :context
+
+    def initialize(context)
+      @context = context
 
-    def initialize
       zero_time = Time.now
       @last_seen = Hash.new { |h, k| h[k] = zero_time }
     end
@@ -54,7 +58,7 @@ module ScoutApm
       age = Time.now - last_seen[unique_name]
 
       # What approximate percentile was this request?
-      percentile = ScoutApm::Agent.instance.request_histograms.approximate_quantile_of_value(unique_name, total_time)
+      percentile = context.request_histograms.approximate_quantile_of_value(unique_name, total_time)
 
       return speed_points(total_time) + percentile_points(percentile) + age_points(age)
     end

data/lib/scout_apm/slow_job_record.rb

@@ -23,7 +23,7 @@ module ScoutApm
     attr_reader :git_sha
     attr_reader :truncated_metrics
 
-    def initialize(queue_name, job_name, time, total_time, exclusive_time, context, metrics, allocation_metrics, mem_delta, allocations, score, truncated_metrics)
+    def initialize(agent_context, queue_name, job_name, time, total_time, exclusive_time, context, metrics, allocation_metrics, mem_delta, allocations, score, truncated_metrics)
       @queue_name = queue_name
       @job_name = job_name
       @time = time
@@ -34,13 +34,13 @@ module ScoutApm
       @allocation_metrics = allocation_metrics
       @mem_delta = mem_delta
       @allocations = allocations
-      @seconds_since_startup = (Time.now - ScoutApm::Agent.instance.process_start_time)
-      @hostname = ScoutApm::Environment.instance.hostname
-      @git_sha = ScoutApm::Environment.instance.git_revision.sha
+      @seconds_since_startup = (Time.now - agent_context.process_start_time)
+      @hostname = agent_context.environment.hostname
+      @git_sha = agent_context.environment.git_revision.sha
       @score = score
       @truncated_metrics = truncated_metrics
 
-      ScoutApm::Agent.instance.logger.debug { "Slow Job [#{metric_name}] - Call Time: #{total_call_time} Mem Delta: #{mem_delta}"}
+      agent_context.logger.debug { "Slow Job [#{metric_name}] - Call Time: #{total_call_time} Mem Delta: #{mem_delta}"}
     end
 
     def metric_name

data/lib/scout_apm/slow_request_policy.rb

@@ -24,8 +24,12 @@ module ScoutApm
     # has been running.
     attr_reader :last_seen
 
+    # The AgentContext we're running in
+    attr_reader :context
+
+    def initialize(context)
+      @context = context
 
-    def initialize
       zero_time = Time.now
       @last_seen = Hash.new { |h, k| h[k] = zero_time }
     end
@@ -54,7 +58,7 @@ module ScoutApm
       age = Time.now - last_seen[unique_name]
 
       # What approximate percentile was this request?
-      percentile = ScoutApm::Agent.instance.request_histograms.approximate_quantile_of_value(unique_name, total_time)
+      percentile = context.request_histograms.approximate_quantile_of_value(unique_name, total_time)
 
       return speed_points(total_time) + percentile_points(percentile) + age_points(age)
     end

data/lib/scout_apm/slow_transaction.rb

@@ -19,7 +19,7 @@ module ScoutApm
 
     attr_reader :truncated_metrics # True/False that says if we had to truncate the metrics of this trace
 
-    def initialize(uri, metric_name, total_call_time, metrics, allocation_metrics, context, time, raw_stackprof, mem_delta, allocations, score, truncated_metrics)
+    def initialize(agent_context, uri, metric_name, total_call_time, metrics, allocation_metrics, context, time, raw_stackprof, mem_delta, allocations, score, truncated_metrics)
       @uri = uri
       @metric_name = metric_name
       @total_call_time = total_call_time
@@ -30,13 +30,13 @@ module ScoutApm
       @prof = []
       @mem_delta = mem_delta
       @allocations = allocations
-      @seconds_since_startup = (Time.now - ScoutApm::Agent.instance.process_start_time)
-      @hostname = ScoutApm::Environment.instance.hostname
+      @seconds_since_startup = (Time.now - agent_context.process_start_time)
+      @hostname = agent_context.environment.hostname
       @score = score
-      @git_sha = ScoutApm::Environment.instance.git_revision.sha
+      @git_sha = agent_context.environment.git_revision.sha
       @truncated_metrics = truncated_metrics
 
-      ScoutApm::Agent.instance.logger.debug { "Slow Request [#{uri}] - Call Time: #{total_call_time} Mem Delta: #{mem_delta} Score: #{score}"}
+      agent_context.logger.debug { "Slow Request [#{uri}] - Call Time: #{total_call_time} Mem Delta: #{mem_delta} Score: #{score}"}
     end
 
     # Used to remove metrics when the payload will be too large.

data/lib/scout_apm/store.rb

@@ -3,14 +3,15 @@
 # the layaway file for cross-process aggregation.
 module ScoutApm
   class Store
-    def initialize
+    def initialize(context)
+      @context = context
       @mutex = Mutex.new
-      @reporting_periods = Hash.new { |h,k| h[k] = StoreReportingPeriod.new(k) }
+      @reporting_periods = Hash.new { |h,k| h[k] = StoreReportingPeriod.new(k, @context) }
       @samplers = []
     end
 
     def current_timestamp
-      StoreReportingPeriodTimestamp.new
+      StoreReportingPeriodTimestamp.new(Time.now)
     end
 
     def current_period
@@ -84,11 +85,11 @@ module ScoutApm
     # current-minute metrics.  Useful when we are shutting down the agent
     # during a restart.
     def write_to_layaway(layaway, force=false)
-      ScoutApm::Agent.instance.logger.debug("Writing to layaway#{" (Forced)" if force}")
+      logger.debug("Writing to layaway#{" (Forced)" if force}")
 
-        @reporting_periods.select { |time, rp| force || (time.timestamp < current_timestamp.timestamp) }.
-                          each   { |time, rp| collect_samplers(rp) }.
-                          each   { |time, rp| write_reporting_period(layaway, time, rp) }
+      @reporting_periods.select { |time, rp| force || (time.timestamp < current_timestamp.timestamp) }.
+                         each   { |time, rp| collect_samplers(rp) }.
+                         each   { |time, rp| write_reporting_period(layaway, time, rp) }
     end
 
     def write_reporting_period(layaway, time, rp)
@@ -96,18 +97,18 @@ module ScoutApm
         layaway.write_reporting_period(rp)
       }
     rescue => e
-      ScoutApm::Agent.instance.logger.warn("Failed writing data to layaway file: #{e.message} / #{e.backtrace}")
+      logger.warn("Failed writing data to layaway file: #{e.message} / #{e.backtrace}")
     ensure
-      ScoutApm::Agent.instance.logger.debug("Before delete, reporting periods length: #{@reporting_periods.size}")
+      logger.debug("Before delete, reporting periods length: #{@reporting_periods.size}")
       deleted_items = @reporting_periods.delete(time)
-      ScoutApm::Agent.instance.logger.debug("After delete, reporting periods length: #{@reporting_periods.size}. Did delete #{deleted_items}")
+      logger.debug("After delete, reporting periods length: #{@reporting_periods.size}. Did delete #{deleted_items}")
     end
     private :write_reporting_period
 
     ######################################
     # Sampler support
-    def add_sampler(sampler)
-      @samplers << sampler
+    def add_sampler(sampler_klass)
+      @samplers << sampler_klass.new(@context)
     end
 
     def collect_samplers(rp)
@@ -115,12 +116,17 @@ module ScoutApm
         begin
           sampler.metrics(rp.timestamp, self)
         rescue => e
-          ScoutApm::Agent.instance.logger.info "Error reading #{sampler.human_name} for period: #{rp}"
-          ScoutApm::Agent.instance.logger.debug "#{e.message}\n\t#{e.backtrace.join("\n\t")}"
+          logger.info "Error reading #{sampler.human_name} for period: #{rp}"
+          logger.debug "#{e.message}\n\t#{e.backtrace.join("\n\t")}"
         end
       end
     end
     private :collect_samplers
+
+    def logger
+      @context.logger
+    end
+    private :logger
   end
 
   # A timestamp, normalized to the beginning of a minute. Used as a hash key to
@@ -191,7 +197,7 @@ module ScoutApm
 
     attr_reader :db_query_metric_set
 
-    def initialize(timestamp)
+    def initialize(timestamp, context)
       @timestamp = timestamp
 
       @request_traces = ScoredItemSet.new
@@ -200,7 +206,7 @@ module ScoutApm
       @histograms = []
 
       @metric_set = MetricSet.new
-      @db_query_metric_set = DbQueryMetricSet.new
+      @db_query_metric_set = DbQueryMetricSet.new(context)
 
       @jobs = Hash.new
     end