scout_apm 2.0.0.pre → 2.0.0.pre2

data/lib/scout_apm/utils/sql_sanitizer.rb

@@ -75,7 +75,7 @@ module ScoutApm
       end
 
       def scrubbed(str)
-        return '' if !str.is_a?(String) || str.length > 1000 # safeguard - don't sanitize or scrub large SQL statements
+        return '' if !str.is_a?(String) || str.length > 4000 # safeguard - don't sanitize or scrub large SQL statements
         return str if !str.respond_to?(:encode) # Ruby <= 1.8 doesn't have string encoding
         return str if str.valid_encoding? # Whatever encoding it is, it is valid and we can operate on it
         ScoutApm::Agent.instance.logger.debug "Scrubbing invalid sql encoding."

data/lib/scout_apm/utils/sql_sanitizer_regex.rb

@@ -13,8 +13,8 @@ module ScoutApm
 
       MYSQL_VAR_INTERPOLATION = %r|\[\[.*\]\]\s*$|.freeze
       MYSQL_REMOVE_INTEGERS = /(?<!LIMIT )\b\d+\b/.freeze
-      MYSQL_REMOVE_SINGLE_QUOTE_STRINGS = /'(?:[^']|'')*'/.freeze
-      MYSQL_REMOVE_DOUBLE_QUOTE_STRINGS = /"(?:[^"]|"")*"/.freeze
+      MYSQL_REMOVE_SINGLE_QUOTE_STRINGS = %r{'(?:\\'|[^']|'')*'}.freeze
+      MYSQL_REMOVE_DOUBLE_QUOTE_STRINGS = %r{"(?:\\"|[^"]|"")*"}.freeze
       MYSQL_IN_CLAUSE = /IN\s+\(\?[^\)]*\)/.freeze
 
       SQLITE_VAR_INTERPOLATION = %r|\[\[.*\]\]\s*$|.freeze

data/lib/scout_apm/utils/sql_sanitizer_regex_1_8_7.rb

@@ -13,8 +13,8 @@ module ScoutApm
 
       MYSQL_VAR_INTERPOLATION = %r|\[\[.*\]\]\s*$|.freeze
       MYSQL_REMOVE_INTEGERS = /\b\d+\b/.freeze
-      MYSQL_REMOVE_SINGLE_QUOTE_STRINGS = /'(?:[^']|'')*'/.freeze
-      MYSQL_REMOVE_DOUBLE_QUOTE_STRINGS = /"(?:[^"]|"")*"/.freeze
+      MYSQL_REMOVE_SINGLE_QUOTE_STRINGS = /'(?:\\'|[^']|'')*'/.freeze
+      MYSQL_REMOVE_DOUBLE_QUOTE_STRINGS = /"(?:\\"|[^"]|"")*"/.freeze
       MYSQL_IN_CLAUSE = /IN\s+\(\?[^\)]*\)/.freeze
 
       SQLITE_VAR_INTERPOLATION = %r|\[\[.*\]\]\s*$|.freeze

data/lib/scout_apm/version.rb

@@ -1,4 +1,4 @@
 module ScoutApm
-  VERSION = "2.0.0.pre"
+  VERSION = "2.0.0.pre2"
 end
 

data/scout_apm.gemspec

@@ -25,4 +25,5 @@ Gem::Specification.new do |s|
   s.add_development_dependency "pry"
   s.add_development_dependency "m"
   s.add_development_dependency "simplecov"
+  s.add_development_dependency "rake-compiler"
 end

data/test/test_helper.rb

@@ -1,11 +1,12 @@
+# Load & Start simplecov before loading scout_apm
+require 'simplecov'
+SimpleCov.start
+
 require 'minitest/autorun'
 require 'minitest/unit'
 require 'minitest/pride'
 require 'pry'
 
-# Load & Start simplecov before loading scout_apm
-require 'simplecov'
-SimpleCov.start
 
 require 'scout_apm'
 

data/test/unit/layaway_test.rb

@@ -18,15 +18,12 @@ class LayawayTest < Minitest::Test
 
   def test_merge_reporting_period
     File.open(DATA_FILE_PATH, 'w') { |file| file.write(Marshal.dump(NEW_FORMAT)) }
-    ScoutApm::Agent.instance.start
-
-    data = ScoutApm::Layaway.new
-    t = ScoutApm::StoreReportingPeriodTimestamp.new
-    data.add_reporting_period(TIMESTAMP,ScoutApm::StoreReportingPeriod.new(TIMESTAMP))
-    assert_equal [TIMESTAMP], Marshal.load(File.read(DATA_FILE_PATH)).keys
-    # TODO - add tests to verify metrics+slow transactions are merged
+    layaway = ScoutApm::Layaway.new
+    layaway.add_reporting_period(TIMESTAMP, ScoutApm::StoreReportingPeriod.new(TIMESTAMP))
+    unmarshalled = Marshal.load(File.read(DATA_FILE_PATH))
+    assert_equal [TIMESTAMP], unmarshalled.keys
   end
 
   TIMESTAMP = ScoutApm::StoreReportingPeriodTimestamp.new(Time.parse("2015-01-01"))
   NEW_FORMAT = {TIMESTAMP => ScoutApm::StoreReportingPeriod.new(TIMESTAMP)} # Format for 1.2+ agents
-end
+end

data/test/unit/metric_set_test.rb

@@ -0,0 +1,101 @@
+require 'test_helper'
+
+require 'scout_apm/metric_set'
+
+module ScoutApm
+  class MetricSetTest < Minitest::Test
+    def setup
+      @metric_set = ScoutApm::MetricSet.new
+    end
+
+    def test_absorb_one_passthrough_metric
+      @metric_set.absorb(make_fake_stat("Controller/Foo", 1))
+
+      assert_equal 1, @metric_set.metrics.length
+      assert_equal "Controller/Foo", @metric_set.metrics.first.first.metric_name
+    end
+
+    def test_absorb_one_aggregate_metric
+      @metric_set.absorb(make_fake_stat("ActiveRecord/Foo", 1))
+
+      assert_equal 1, @metric_set.metrics.length
+      assert_equal "ActiveRecord/all", @metric_set.metrics.first.first.metric_name
+    end
+
+    def test_absorb_many_aggregate_metric
+      @metric_set.absorb(make_fake_stat("ActiveRecord/Foo", 1))
+      @metric_set.absorb(make_fake_stat("ActiveRecord/Bar", 1))
+      @metric_set.absorb(make_fake_stat("ActiveRecord/Baz", 1))
+      @metric_set.absorb(make_fake_stat("HTTP/Get", 1))
+      @metric_set.absorb(make_fake_stat("HTTP/Post", 1))
+
+      metrics = @metric_set.metrics.to_a.sort_by { |m| m.first.metric_name }
+      assert_equal 2, metrics.length
+      assert_equal "ActiveRecord/all", metrics[0][0].metric_name
+      assert_equal "HTTP/all", metrics[1][0].metric_name
+      assert_equal 3, metrics[0][1].call_count
+      assert_equal 2, metrics[1][1].call_count
+    end
+
+    def test_absorb_one_error
+      @metric_set.absorb(make_fake_stat("Errors/Controller/public/index", 1))
+
+      metrics = @metric_set.metrics.to_a.sort_by { |m| m.first.metric_name }
+      assert_equal 2, metrics.length
+      assert_equal "Errors/Controller/public/index", metrics[0].first.metric_name
+      assert_equal "Errors/Request",  metrics[1].first.metric_name
+    end
+
+    def test_absorb_many_metrics
+      @metric_set.absorb_all([
+        make_fake_stat("ActiveRecord/Foo", 1),
+        make_fake_stat("Controller/Bar", 1)
+      ])
+
+      metrics = @metric_set.metrics.to_a.sort_by { |m| m.first.metric_name }
+      assert_equal 2, metrics.length
+      assert_equal "ActiveRecord/all", metrics[0].first.metric_name
+      assert_equal "Controller/Bar",  metrics[1].first.metric_name
+    end
+
+    def test_combine
+      @other_set = ScoutApm::MetricSet.new
+
+      @metric_set.absorb_all([
+        make_fake_stat("ActiveRecord/Foo", 1),
+        make_fake_stat("Controller/Bar", 1),
+        make_fake_stat("Errors/Controller/public/index", 1),
+      ])
+
+      @other_set.absorb_all([
+        make_fake_stat("ActiveRecord/Foo", 1),
+        make_fake_stat("Controller/Bar", 1),
+        make_fake_stat("Errors/Controller/public/index", 1),
+      ])
+
+      @metric_set.combine!(@other_set)
+
+      metrics = @metric_set.metrics.to_a.sort_by { |m| m.first.metric_name }
+      assert_equal 4, metrics.length
+      assert_equal "ActiveRecord/all", metrics[0][0].metric_name
+      assert_equal "Controller/Bar",  metrics[1][0].metric_name
+      assert_equal "Errors/Controller/public/index", metrics[2][0].metric_name
+      assert_equal "Errors/Request",  metrics[3][0].metric_name
+
+      assert_equal 2, metrics[0][1].call_count
+      assert_equal 2, metrics[1][1].call_count
+      assert_equal 2, metrics[2][1].call_count
+      assert_equal 2, metrics[3][1].call_count
+    end
+
+    ############################################################
+    # Test helper functions
+    ############################################################
+    def make_fake_stat(name, count)
+      meta = MetricMeta.new(name)
+      stat = MetricStats.new
+      stat.update!(count)
+      [meta, stat]
+    end
+  end
+end

data/test/unit/scored_item_set_test.rb

@@ -0,0 +1,65 @@
+require 'test_helper'
+
+require 'scout_apm/scored_item_set'
+
+class FakeScoredItem
+  def initialize(name, score)
+    @name = name
+    @score = score
+  end
+  def name; @name; end
+  def score; @score; end
+  def call; "called_#{@score}_#{@name}"; end
+end
+
+class ScoredItemSetTest < Minitest::Test
+  def test_empty_set_always_adds_item
+    set = ScoutApm::ScoredItemSet.new
+    set << FakeScoredItem.new("users/index", 10)
+
+    assert_equal set.to_a.first, "called_10_users/index"
+    assert_equal set.count, 1
+  end
+
+  def test_repeated_additions_chooses_most_expensive
+    set = ScoutApm::ScoredItemSet.new
+
+    [ FakeScoredItem.new("users/index", 10),
+      FakeScoredItem.new("users/index", 11),
+      FakeScoredItem.new("users/index", 12)
+    ].shuffle.each { |fsi| set << fsi }
+
+    assert_equal set.to_a.first, "called_12_users/index"
+    assert_equal set.count, 1
+  end
+
+  def test_multiple_items_occupy_different_buckets
+    set = ScoutApm::ScoredItemSet.new
+
+    [ FakeScoredItem.new("users/index", 10),
+      FakeScoredItem.new("users/index", 11),
+      FakeScoredItem.new("users/show", 12),
+      FakeScoredItem.new("users/show", 10)
+    ].shuffle.each { |fsi| set << fsi }
+
+    assert_equal set.count, 2
+    assert set.to_a.include?("called_11_users/index")
+    assert set.to_a.include?("called_12_users/show")
+  end
+
+  def test_evicts_at_capacity
+    set = ScoutApm::ScoredItemSet.new(3) # Force max_size to 3
+
+    [ FakeScoredItem.new("users/index", 10),
+      FakeScoredItem.new("users/show", 11),
+      FakeScoredItem.new("posts/index", 12),
+      FakeScoredItem.new("posts/move", 13)
+    ].shuffle.each { |fsi| set << fsi }
+
+    assert_equal set.count, 3
+    assert !set.to_a.include?("called_10_users/index"), "Did not Expect to see users/index in #{set.to_a.inspect}"
+    assert set.to_a.include?("called_11_users/show"),   "Expected to see users/show in #{set.to_a.inspect}"
+    assert set.to_a.include?("called_12_posts/index"),  "Expected to see posts/index in #{set.to_a.inspect}"
+    assert set.to_a.include?("called_13_posts/move"),   "Expected to see posts/move in #{set.to_a.inspect}"
+  end
+end

data/test/unit/serializers/payload_serializer_test.rb

@@ -145,7 +145,7 @@ class PayloadSerializerTest < Minitest::Test
     context = ScoutApm::Context.new
     context.add({"this" => "that"})
     context.add_user({"hello" => "goodbye"})
-    slow_t = ScoutApm::SlowTransaction.new("http://example.com/blabla", "Buckethead/something/else", 1.23, slow_transaction_metrics, context, Time.at(1448198788), [])
+    slow_t = ScoutApm::SlowTransaction.new("http://example.com/blabla", "Buckethead/something/else", 1.23, slow_transaction_metrics, context, Time.at(1448198788), [], 10)
     payload = ScoutApm::Serializers::PayloadSerializerToJson.serialize({}, {}, [slow_t], [], [])
     formatted_slow_transactions = [
       {
@@ -158,6 +158,7 @@ class PayloadSerializerTest < Minitest::Test
         "uri" => "http://example.com/blabla",
         "context" => {"this"=>"that", "user"=>{"hello"=>"goodbye"}},
         "prof" => [],
+        "score" => 10,
         "metrics" => [
           {
             "key" => {

data/test/unit/slow_item_set_test.rb

@@ -88,6 +88,7 @@ class SlowItemSetTest < Minitest::Test
       {}, # metrics
       {}, # context
       Time.now, # end time
-      []) # stackprof
+      [], # stackprof
+      0)
   end
 end

data/test/unit/slow_request_policy_test.rb

@@ -0,0 +1,42 @@
+require 'test_helper'
+
+require 'scout_apm/slow_request_policy'
+require 'scout_apm/layer'
+
+class FakeRequest
+  def initialize(name)
+    @name = name
+    @root_layer = ScoutApm::Layer.new("Controller", name)
+    @root_layer.instance_variable_set("@stop_time", Time.now)
+  end
+  def unique_name; "Controller/foo/bar"; end
+  def root_layer; @root_layer; end
+  def set_duration(seconds)
+    @root_layer.instance_variable_set("@start_time", Time.now - seconds)
+  end
+end
+
+class SlowRequestPolicyTest < Minitest::Test
+  def test_stored_records_current_time
+    test_start = Time.now
+    policy = ScoutApm::SlowRequestPolicy.new
+    request = FakeRequest.new("users/index")
+
+    policy.stored!(request)
+    assert policy.last_seen[request.unique_name] > test_start
+  end
+
+  def test_score
+    policy = ScoutApm::SlowRequestPolicy.new
+    request = FakeRequest.new("users/index")
+
+    request.set_duration(10) # 10 seconds
+    policy.last_seen[request.unique_name] = Time.now - 120 # 2 minutes since last seen
+    agent.request_histograms.add(request.unique_name, 1)
+
+    # Actual value I have in console is 1.599
+    assert policy.score(request) > 1.5
+    assert policy.score(request) < 2.0
+
+  end
+end

data/test/unit/sql_sanitizer_test.rb

@@ -85,6 +85,12 @@ module ScoutApm
         assert_equal %q|SELECT `blogs`.* FROM `blogs` WHERE (title = ?)|, ss.to_s
       end
 
+      def test_mysql_quotes
+        sql = %q|INSERT INTO `users` VALUES ('foo', 'b\'ar')|
+        ss = SqlSanitizer.new(sql).tap{ |it| it.database_engine = :mysql }
+        assert_equal %q|INSERT INTO `users` VALUES (?, ?)|, ss.to_s
+      end
+
       def test_scrubs_invalid_encoding
         sql = "SELECT `blogs`.* FROM `blogs` WHERE (title = 'a\255c')".force_encoding('UTF-8')
         assert_equal false, sql.valid_encoding?

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: scout_apm
 version: !ruby/object:Gem::Version
-  version: 2.0.0.pre
+  version: 2.0.0.pre2
 platform: ruby
 authors:
 - Derek Haynes
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-05-10 00:00:00.000000000 Z
+date: 2016-06-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rusage
@@ -81,6 +81,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake-compiler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: Monitors Ruby apps and reports detailed metrics on performance to Scout.
 email:
 - support@scoutapp.com
@@ -121,6 +135,7 @@ files:
 - lib/scout_apm/framework_integrations/ruby.rb
 - lib/scout_apm/framework_integrations/sinatra.rb
 - lib/scout_apm/histogram.rb
+- lib/scout_apm/instant_reporting.rb
 - lib/scout_apm/instruments/.DS_Store
 - lib/scout_apm/instruments/action_controller_rails_2.rb
 - lib/scout_apm/instruments/action_controller_rails_3_rails4.rb
@@ -134,6 +149,7 @@ files:
 - lib/scout_apm/instruments/mongoid.rb
 - lib/scout_apm/instruments/moped.rb
 - lib/scout_apm/instruments/net_http.rb
+- lib/scout_apm/instruments/percentile_sampler.rb
 - lib/scout_apm/instruments/process/process_cpu.rb
 - lib/scout_apm/instruments/process/process_memory.rb
 - lib/scout_apm/instruments/rails_router.rb
@@ -160,7 +176,9 @@ files:
 - lib/scout_apm/platform_integrations/heroku.rb
 - lib/scout_apm/platform_integrations/server.rb
 - lib/scout_apm/reporter.rb
+- lib/scout_apm/request_histograms.rb
 - lib/scout_apm/request_manager.rb
+- lib/scout_apm/scored_item_set.rb
 - lib/scout_apm/serializers/app_server_load_serializer.rb
 - lib/scout_apm/serializers/deploy_serializer.rb
 - lib/scout_apm/serializers/directive_serializer.rb
@@ -207,9 +225,12 @@ files:
 - test/unit/histogram_test.rb
 - test/unit/instruments/active_record_instruments_test.rb
 - test/unit/layaway_test.rb
+- test/unit/metric_set_test.rb
+- test/unit/scored_item_set_test.rb
 - test/unit/serializers/payload_serializer_test.rb
 - test/unit/slow_item_set_test.rb
 - test/unit/slow_job_policy_test.rb
+- test/unit/slow_request_policy_test.rb
 - test/unit/sql_sanitizer_test.rb
 - test/unit/utils/active_record_metric_name_test.rb
 homepage: https://github.com/scoutapp/scout_apm_ruby
@@ -232,7 +253,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: 1.3.1
 requirements: []
 rubyforge_project: scout_apm
-rubygems_version: 2.4.6
+rubygems_version: 2.6.2
 signing_key: 
 specification_version: 4
 summary: Ruby application performance monitoring
@@ -245,8 +266,12 @@ test_files:
 - test/unit/histogram_test.rb
 - test/unit/instruments/active_record_instruments_test.rb
 - test/unit/layaway_test.rb
+- test/unit/metric_set_test.rb
+- test/unit/scored_item_set_test.rb
 - test/unit/serializers/payload_serializer_test.rb
 - test/unit/slow_item_set_test.rb
 - test/unit/slow_job_policy_test.rb
+- test/unit/slow_request_policy_test.rb
 - test/unit/sql_sanitizer_test.rb
 - test/unit/utils/active_record_metric_name_test.rb
+has_rdoc: