scout_apm 3.0.0.pre1 → 3.0.0.pre2

data/lib/scout_apm/scored_item_set.rb

@@ -63,6 +63,12 @@ module ScoutApm
       end
     end
 
+    # Equal to another set only if exactly the same set of items is inside
+    def eql?(other)
+      items == other.items
+    end
+
+    alias :== :eql?
 
     private
 
@@ -75,5 +81,6 @@ module ScoutApm
         items[new_item.name] = [new_item.score, new_item.call]
       end
     end
+
   end
 end

data/lib/scout_apm/serializers/histograms_serializer_to_json.rb

@@ -0,0 +1,21 @@
+
+module ScoutApm
+  module Serializers
+    class HistogramsSerializerToJson
+      attr_reader :histograms
+
+      def initialize(histograms)
+        @histograms = histograms
+      end
+
+      def as_json
+        histograms.map do |histo|
+          {
+            "name" => histo.name,
+            "histogram" => histo.histogram.as_json,
+          }
+        end
+      end
+    end
+  end
+end

data/lib/scout_apm/serializers/payload_serializer.rb

@@ -2,9 +2,9 @@
 module ScoutApm
   module Serializers
     class PayloadSerializer
-      def self.serialize(metadata, metrics, slow_transactions, jobs, slow_jobs)
+      def self.serialize(metadata, metrics, slow_transactions, jobs, slow_jobs, histograms)
         if ScoutApm::Agent.instance.config.value("report_format") == 'json'
-          ScoutApm::Serializers::PayloadSerializerToJson.serialize(metadata, metrics, slow_transactions, jobs, slow_jobs)
+          ScoutApm::Serializers::PayloadSerializerToJson.serialize(metadata, metrics, slow_transactions, jobs, slow_jobs, histograms)
         else
           metadata = metadata.dup
           metadata.default = nil
@@ -15,7 +15,13 @@ module ScoutApm
                        :metrics           => metrics,
                        :slow_transactions => slow_transactions,
                        :jobs              => jobs,
-                       :slow_jobs         => slow_jobs)
+                       :slow_jobs         => slow_jobs,
+
+                       # as_json returns a ruby object. Since it's not a simple
+                       # array, use this to maintain compatibility with json
+                       # payloads. At this point, the marshal code branch is
+                       # very rarely used anyway.
+                       :histograms        => HistogramsSerializerToJson.new(histograms).as_json)
         end
       end
 

data/lib/scout_apm/serializers/payload_serializer_to_json.rb

@@ -2,7 +2,7 @@ module ScoutApm
   module Serializers
     module PayloadSerializerToJson
       class << self
-        def serialize(metadata, metrics, slow_transactions, jobs, slow_jobs)
+        def serialize(metadata, metrics, slow_transactions, jobs, slow_jobs, histograms)
           metadata.merge!({:payload_version => 2})
 
           jsonify_hash({:metadata => metadata,
@@ -10,6 +10,7 @@ module ScoutApm
                         :slow_transactions => rearrange_the_slow_transactions(slow_transactions),
                         :jobs => JobsSerializerToJson.new(jobs).as_json,
                         :slow_jobs => SlowJobsSerializerToJson.new(slow_jobs).as_json,
+                        :histograms => HistogramsSerializerToJson.new(histograms).as_json,
                       })
         end
 

data/lib/scout_apm/serializers/slow_jobs_serializer_to_json.rb

@@ -25,6 +25,7 @@ module ScoutApm
             "metrics" => MetricsToJsonSerializer.new(job.metrics).as_json, # New style of metrics
             "allocation_metrics" => MetricsToJsonSerializer.new(job.allocation_metrics).as_json, # New style of metrics
             "context" => job.context.to_hash,
+            "truncated_metrics" => job.truncated_metrics,
 
             "score" => job.score,
           }
@@ -33,4 +34,3 @@ module ScoutApm
     end
   end
 end
-

data/lib/scout_apm/slow_job_record.rb

@@ -21,8 +21,9 @@ module ScoutApm
     attr_reader :seconds_since_startup
     attr_reader :score
     attr_reader :git_sha
+    attr_reader :truncated_metrics
 
-    def initialize(queue_name, job_name, time, total_time, exclusive_time, context, metrics, allocation_metrics, mem_delta, allocations, score)
+    def initialize(queue_name, job_name, time, total_time, exclusive_time, context, metrics, allocation_metrics, mem_delta, allocations, score, truncated_metrics)
       @queue_name = queue_name
       @job_name = job_name
       @time = time
@@ -37,6 +38,8 @@ module ScoutApm
       @hostname = ScoutApm::Environment.instance.hostname
       @git_sha = ScoutApm::Environment.instance.git_revision.sha
       @score = score
+      @truncated_metrics = truncated_metrics
+
       ScoutApm::Agent.instance.logger.debug { "Slow Job [#{metric_name}] - Call Time: #{total_call_time} Mem Delta: #{mem_delta}"}
     end
 

data/lib/scout_apm/slow_transaction.rb

@@ -17,7 +17,9 @@ module ScoutApm
     attr_accessor :seconds_since_startup # hack - we need to reset these server side.
     attr_accessor :git_sha # hack - we need to reset these server side.
 
-    def initialize(uri, metric_name, total_call_time, metrics, allocation_metrics, context, time, raw_stackprof, mem_delta, allocations, score)
+    attr_reader :truncated_metrics # True/False that says if we had to truncate the metrics of this trace
+
+    def initialize(uri, metric_name, total_call_time, metrics, allocation_metrics, context, time, raw_stackprof, mem_delta, allocations, score, truncated_metrics)
       @uri = uri
       @metric_name = metric_name
       @total_call_time = total_call_time
@@ -32,6 +34,8 @@ module ScoutApm
       @hostname = ScoutApm::Environment.instance.hostname
       @score = score
       @git_sha = ScoutApm::Environment.instance.git_revision.sha
+      @truncated_metrics = truncated_metrics
+
       ScoutApm::Agent.instance.logger.debug { "Slow Request [#{uri}] - Call Time: #{total_call_time} Mem Delta: #{mem_delta} Score: #{score}"}
     end
 
@@ -46,7 +50,19 @@ module ScoutApm
     end
 
     def as_json
-      json_attributes = [:key, :time, :total_call_time, :uri, [:context, :context_hash], :score, :prof, :mem_delta, :allocations, :seconds_since_startup, :hostname, :git_sha]
+      json_attributes = [:key,
+                         :time,
+                         :total_call_time,
+                         :uri,
+                         [:context, :context_hash],
+                         :score,
+                         :prof,
+                         :mem_delta,
+                         :allocations,
+                         :seconds_since_startup,
+                         :hostname,
+                         :git_sha,
+                         :truncated_metrics]
       ScoutApm::AttributeArranger.call(self, json_attributes)
     end
 

data/lib/scout_apm/store.rb

@@ -26,7 +26,23 @@ module ScoutApm
     # Save newly collected metrics
     def track!(metrics, options={})
       @mutex.synchronize {
-        current_period.absorb_metrics!(metrics)
+        period = if options[:timestamp]
+                   @reporting_periods[options[:timestamp]]
+                 else
+                   current_period
+                 end
+        period.absorb_metrics!(metrics)
+      }
+    end
+
+    def track_histograms!(histograms, options={})
+      @mutex.synchronize {
+        period = if options[:timestamp]
+                   @reporting_periods[options[:timestamp]]
+                 else
+                   current_period
+                 end
+        period.merge_histograms!(histograms)
       }
     end
 
@@ -67,15 +83,15 @@ module ScoutApm
     def write_to_layaway(layaway, force=false)
       ScoutApm::Agent.instance.logger.debug("Writing to layaway#{" (Forced)" if force}")
 
-      @mutex.synchronize {
-        reporting_periods.select { |time, rp| force || time.timestamp < current_timestamp.timestamp}.
+        reporting_periods.select { |time, rp| force || time.timestamp < current_timestamp.timestamp }.
+                          each   { |time, rp| collect_samplers(rp) }.
                           each   { |time, rp| write_reporting_period(layaway, time, rp) }
-      }
     end
 
     def write_reporting_period(layaway, time, rp)
-      collect_samplers(rp)
-      layaway.write_reporting_period(rp)
+      @mutex.synchronize {
+        layaway.write_reporting_period(rp)
+      }
     rescue => e
       ScoutApm::Agent.instance.logger.warn("Failed writing data to layaway file: #{e.message} / #{e.backtrace}")
     ensure
@@ -91,12 +107,10 @@ module ScoutApm
     def collect_samplers(rp)
       @samplers.each do |sampler|
         begin
-          metrics = sampler.metrics(rp.timestamp)
-          rp.absorb_metrics!(metrics)
+          sampler.metrics(rp.timestamp, self)
         rescue => e
           ScoutApm::Agent.instance.logger.info "Error reading #{sampler.human_name} for period: #{rp}"
-          ScoutApm::Agent.instance.logger.debug e.message
-          ScoutApm::Agent.instance.logger.debug e.backtrace.join("\n")
+          ScoutApm::Agent.instance.logger.debug "#{e.message}\n\t#{e.backtrace.join("\n\t")}"
         end
       end
     end
@@ -159,6 +173,9 @@ module ScoutApm
     # A ScoredItemSet holding the "best" traces for the period
     attr_reader :job_traces
 
+    # An Array of HistogramsReport
+    attr_reader :histograms
+
     # A StoreReportingPeriodTimestamp representing the time that this
     # collection of metrics is for
     attr_reader :timestamp
@@ -171,6 +188,8 @@ module ScoutApm
       @request_traces = ScoredItemSet.new
       @job_traces = ScoredItemSet.new
 
+      @histograms = []
+
       @metric_set = MetricSet.new
       @jobs = Hash.new
     end
@@ -181,7 +200,8 @@ module ScoutApm
         merge_metrics!(other.metric_set).
         merge_slow_transactions!(other.slow_transactions_payload).
         merge_jobs!(other.jobs).
-        merge_slow_jobs!(other.slow_jobs_payload)
+        merge_slow_jobs!(other.slow_jobs_payload).
+        merge_histograms!(other.histograms)
       self
     end
 
@@ -230,6 +250,17 @@ module ScoutApm
       self
     end
 
+    def merge_histograms!(new_histograms)
+      new_histograms = Array(new_histograms)
+      @histograms = (histograms + new_histograms).
+        group_by { |histo| histo.name }.
+        map { |(_, histos)|
+          histos.inject { |merged, histo| merged.combine!(histo) }
+        }
+
+      self
+    end
+
     #################################
     # Retrieve Metrics for reporting
     #################################

data/lib/scout_apm/tracked_request.rb

@@ -278,7 +278,7 @@ module ScoutApm
         # If there's an instant_key, it means we need to report this right away
         if instant?
           trace = slow_converter.call
-          ScoutApm::InstantReporting.new(trace, instant_key).call()
+          ScoutApm::InstantReporting.new(trace, instant_key).call
         end
       end
 

data/lib/scout_apm/utils/gzip_helper.rb

@@ -0,0 +1,24 @@
+module ScoutApm
+  module Utils
+    # A simple wrapper around Ruby's built-in gzip support.
+    class GzipHelper
+      DEFAULT_GZIP_LEVEL = 5
+
+      attr_reader :level
+
+      def initialize(level = DEFAULT_GZIP_LEVEL)
+        @level = level
+      end
+
+      def deflate(str)
+        strio = StringIO.new
+
+        gz = Zlib::GzipWriter.new(strio, level)
+        gz.write str
+        gz.close
+
+        strio.string
+      end
+    end
+  end
+end

data/lib/scout_apm/utils/numbers.rb

@@ -0,0 +1,14 @@
+module ScoutApm
+  module Utils
+    class Numbers
+
+      # Round a float to a certain number of decimal places
+      def self.round(number, decimals)
+        factor = 10 ** decimals
+
+        (number * factor).round / factor.to_f
+      end
+
+    end
+  end
+end

data/lib/scout_apm/version.rb

@@ -1,3 +1,3 @@
 module ScoutApm
-  VERSION = "3.0.0.pre1"
-end
+  VERSION = "3.0.0.pre2"
+end

data/test/test_helper.rb

@@ -83,3 +83,13 @@ class Minitest::Test
   DATA_FILE_PATH = "#{DATA_FILE_DIR}/scout_apm.db"
 end
 
+
+module CustomAsserts
+  def assert_false(thing)
+    assert !thing
+  end
+end
+
+class Minitest::Test
+  include CustomAsserts
+end

data/test/unit/config_test.rb

@@ -7,24 +7,24 @@ class ConfigTest < Minitest::Test
     conf = ScoutApm::Config.without_file
 
     # nil for random keys
-    assert_nil conf.value("log_file_path")
+    assert_nil conf.value('log_file_path')
 
     # but has values for defaulted keys
-    assert conf.value("host")
+    assert conf.value('host')
 
     # and still reads from ENV
     ENV['SCOUT_CONFIG_TEST_KEY'] = 'testval'
-    assert_equal 'testval', conf.value("config_test_key")
+    assert_equal 'testval', conf.value('config_test_key')
   end
 
   def test_loading_a_file
-    set_rack_env("production")
+    set_rack_env('production')
 
-    conf_file = File.expand_path("../../data/config_test_1.yml", __FILE__)
+    conf_file = File.expand_path('../../data/config_test_1.yml', __FILE__)
     conf = ScoutApm::Config.with_file(conf_file)
 
-    assert_equal "debug", conf.value('log_level')
-    assert_equal "APM Test Conf (Production)", conf.value('name')
+    assert_equal 'debug', conf.value('log_level')
+    assert_equal 'APM Test Conf (Production)', conf.value('name')
   end
 
   def test_loading_file_without_env_in_file
@@ -68,5 +68,3 @@ class ConfigTest < Minitest::Test
     assert_equal ["a"], coercion.coerce(["a"])
   end
 end
-
-

data/test/unit/histogram_test.rb

@@ -79,6 +79,20 @@ class HistogramTest < Minitest::Test
     assert combined.quantile(0) < combined.quantile(100)
   end
 
+  def test_combine_dedups_identicals
+    hist1 = ScoutApm::NumericHistogram.new(5)
+    hist2 = ScoutApm::NumericHistogram.new(5)
+    hist1.add(1)
+    hist1.add(2)
+    hist2.add(2)
+    hist2.add(3)
+
+    combined = hist1.combine!(hist2)
+    assert_equal 4, combined.total
+    assert_equal [[1, 1], [2, 2], [1, 3]],
+      combined.bins.map{|bin| [bin.count, bin.value.to_i] }
+  end
+
   def test_mean
     hist = ScoutApm::NumericHistogram.new(5)
     10.times {

data/test/unit/instruments/percentile_sampler_test.rb

@@ -0,0 +1,137 @@
+require 'test_helper'
+
+require 'scout_apm/instruments/percentile_sampler'
+
+class PercentileSamplerTest < Minitest::Test
+  PercentileSampler = ScoutApm::Instruments::PercentileSampler
+  HistogramReport = ScoutApm::Instruments::HistogramReport
+
+  attr_reader :subject
+
+  def setup
+    @subject = PercentileSampler.new(logger, histograms)
+  end
+
+
+  def test_initialize_with_logger_and_histogram_set
+    assert_equal subject.logger, logger
+    assert_equal subject.histograms, histograms
+  end
+
+  def test_implements_instrument_interface
+    assert subject.respond_to?(:human_name)
+  end
+
+  def test_percentiles_returns_one_percentile_per_endpoint_at_time
+    histograms[time].add("foo", 10)
+    histograms[time].add("bar", 15)
+    histograms[time2].add("baz", 15)
+
+    assert_equal subject.percentiles(time).length, 2
+    assert_equal subject.percentiles(time2).length, 1
+  end
+
+  def test_percentiles_clears_time_from_hash
+    histograms[time].add("foo", 10)
+    histograms[time2].add("baz", 15)
+
+    subject.percentiles(time)
+
+    assert_false histograms.key?(time)
+    assert       histograms.key?(time + 10)
+  end
+
+  def test_percentiles_returns_histogram_reports
+    histograms[time].add("foo", 10)
+
+    assert subject.percentiles(time).
+      all?{ |item| item.is_a?(HistogramReport) }
+  end
+
+  def test_percentiles_returns_correct_histogram_report
+    histograms[time].add("foo", 100)
+    histograms[time].add("foo", 200)
+    histograms[time].add("foo", 100)
+    histograms[time].add("foo", 300)
+
+    report = subject.percentiles(time).first
+    histogram = report.histogram
+
+    assert_equal "foo", report.name
+    assert_equal 4, histogram.total
+    assert_equal [[2, 100], [1, 200], [1, 300]],
+      histogram.bins.map{|bin| [bin.count, bin.value] }
+  end
+
+  def test_metrics_saves_histogram_to_store
+    store = mock
+    store.expects(:track_histograms!)
+    subject.metrics(ScoutApm::StoreReportingPeriodTimestamp.new(time), store)
+  end
+
+
+  ################################################################################
+  # HistogramReport Test
+  ################################################################################
+  def test_histogram_report_combine_refuses_to_combine_mismatched_name
+    assert_raises { HistogramReport.new("foo", histogram).combine!(HistogramReport.new("bar", histogram)) }
+  end
+
+  def test_histogram_report_merge_keeps_name
+    report1 = HistogramReport.new("foo", histogram)
+    report2 = HistogramReport.new("foo", histogram)
+    combined = report1.combine!(report2)
+
+    assert "foo", combined.name
+  end
+
+  def test_histogram_report_combine_merges_histograms
+    histogram1 = histogram
+    histogram2 = histogram
+    histogram1.add(1)
+    histogram1.add(2)
+    histogram2.add(2)
+    histogram2.add(3)
+
+    report1 = HistogramReport.new("foo", histogram1)
+    report2 = HistogramReport.new("foo", histogram2)
+    combined = report1.combine!(report2)
+
+    assert_equal 4, combined.histogram.total
+    assert_equal [[1, 1], [2, 2], [1, 3]],
+      combined.histogram.bins.map{|bin| [bin.count, bin.value.to_i] }
+  end
+
+  ################################################################################
+  # Test Helpers
+  ################################################################################
+  def logger
+    @logger ||= begin
+                  @logger_io = StringIO.new
+                  Logger.new(@logger_io)
+                end
+  end
+
+  def histograms
+    @histograms ||= begin
+                      @request_histograms_by_time = Hash.new { |hash, key|
+                        hash[key] = ScoutApm::RequestHistograms.new
+                      }
+                    end
+  end
+
+  def histogram
+    max_bins = 20
+    ScoutApm::NumericHistogram.new(max_bins)
+  end
+
+  # An arbitrary time
+  def time
+    @time ||= Time.now
+  end
+
+  def time2
+    time + 10
+  end
+end
+