scout_apm 3.0.0.pre6 → 3.0.0.pre7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: a17bfe8d072cc469eafdaf6f5d9a53e29282ea6a
-  data.tar.gz: 39707f4cfacb744879297744928c038fe1a1c8aa
+  metadata.gz: 622bc9489b9add4f80ca65cd3b12e54e30eea269
+  data.tar.gz: ef00b8380eb4e163cf9b991b826287d63ba27819
 SHA512:
-  metadata.gz: e9d208abf69daf67b5ca3bf487117e58cd434b445cd43126d9ec6ba90242b2179b1f4913aff3e3d8a38d89c6e0c2940d559059509baab072b7d0aac1899a936f
-  data.tar.gz: 91e4143528147d48c5eec703002cfd783579e89fca9c48806e0ec9745a57b622f660f9a7acd5186e6fc0ac5aef0c61f3e69bcc8a1bd5e29fd6ed94c5270e90d2
+  metadata.gz: 2057aaca2029567dd1a7ce09fb8b359cc2819566e0a03ef528f91b051e00170be7fef66dfe339186bc77d030c7e738be2bc3517b8306e874ce38c662988c46a2
+  data.tar.gz: 4e45127ed587dd0e86782d357f1916de915acff6698682f2b6920448ba63245a5412549d27efe7fff1e270b8bfc0e57d36cc7273ae94f1af4f83a348b525b9aa

data/CHANGELOG.markdown

@@ -2,6 +2,11 @@
 
 * ScoutProf BETA
 
+# 2.1.18
+
+* Max layaway file threshold limit
+
+
 # 2.1.17
 
 * Additional logging around file system usage

data/lib/scout_apm.rb

@@ -134,7 +134,6 @@ require 'scout_apm/slow_job_policy'
 require 'scout_apm/job_record'
 require 'scout_apm/request_histograms'
 
-require 'scout_apm/capacity'
 require 'scout_apm/attribute_arranger'
 require 'scout_apm/git_revision'
 

data/lib/scout_apm/agent.rb

@@ -12,7 +12,6 @@ module ScoutApm
     attr_accessor :store
     attr_accessor :layaway
     attr_accessor :config
-    attr_accessor :capacity
     attr_accessor :logger
     attr_accessor :log_file # path to the log file
     attr_accessor :options # options passed to the agent when +#start+ is called.
@@ -58,7 +57,6 @@ module ScoutApm
       @layaway        = ScoutApm::Layaway.new(config, environment)
       @metric_lookup  = Hash.new
 
-      @capacity       = ScoutApm::Capacity.new
       @installed_instruments = []
     end
 

data/lib/scout_apm/layaway.rb

@@ -11,6 +11,9 @@ module ScoutApm
     # Letting it sit a bit may be useful for debugging
     STALE_AGE = 10 * 60
 
+    # Failsafe to prevent writing layaway files if for some reason they are not being cleaned up
+    MAX_FILES_LIMIT = 5000
+
     # A strftime format string for how we render timestamps in filenames.
     # Must be sortable as an integer
     TIME_FORMAT = "%Y%m%d%H%M"
@@ -45,7 +48,11 @@ module ScoutApm
       @directory = Pathname.new(found)
     end
 
-    def write_reporting_period(reporting_period)
+    def write_reporting_period(reporting_period, files_limit = MAX_FILES_LIMIT)
+      if at_layaway_file_limit?(files_limit)
+        ScoutApm::Agent.instance.logger.error("Hit layaway file limit. Not writing to layaway file")
+        return false
+      end
       filename = file_for(reporting_period.timestamp)
       layaway_file = LayawayFile.new(filename)
       layaway_file.write(reporting_period)
@@ -170,6 +177,10 @@ module ScoutApm
       end
     end
 
+    def at_layaway_file_limit?(files_limit = MAX_FILES_LIMIT)
+      all_files_for(:all).count >= files_limit
+    end
+
     def log_layaway_file_information
       files_in_temp = Dir["#{directory}/*"].count
 

data/lib/scout_apm/metric_set.rb

@@ -1,6 +1,8 @@
 module ScoutApm
   class MetricSet
-    # We can't aggregate CPU, Memory, Capacity, or Controller, so pass through these metrics directly
+    # We can't aggregate a handful of things like samplers (CPU, Memory), or
+    # Controller, and Percentiles so pass through these metrics directly
+    #
     # TODO: Figure out a way to not have this duplicate what's in Samplers, and also on server's ingest
     PASSTHROUGH_METRICS = ["CPU", "Memory", "Instance", "Controller", "SlowTransaction", "Percentile", "Job"]
 

data/lib/scout_apm/version.rb

@@ -1,3 +1,3 @@
 module ScoutApm
-  VERSION = "3.0.0.pre6"
+  VERSION = "3.0.0.pre7"
 end

data/test/test_helper.rb

@@ -16,6 +16,14 @@ Kernel.module_eval do
   def self.const_unset(const)
     self.instance_eval { remove_const(const) }
   end
+
+  def silence_warnings(&block)
+    warn_level = $VERBOSE
+    $VERBOSE = nil
+    result = block.call
+    $VERBOSE = warn_level
+    result
+  end
 end
 
 # A test helper class to create a temporary "configuration" we can control entirely purposes

data/test/unit/layaway_test.rb

@@ -21,4 +21,24 @@ class LayawayTest < Minitest::Test
 
     assert_equal Pathname.new("/tmp/scout_apm_test/tmp"), ScoutApm::Layaway.new(config, env).directory
   end
+
+  def test_layaway_file_limit_prevents_new_writes
+    FileUtils.mkdir_p '/tmp/scout_apm_test/layaway_limit'
+    config = make_fake_config("data_file" => "/tmp/scout_apm_test/layaway_limit")
+    layaway = ScoutApm::Layaway.new(config, ScoutApm::Agent.instance.environment)
+    layaway.delete_files_for(:all)
+
+    current_time = Time.now.utc
+    current_rp = ScoutApm::StoreReportingPeriod.new(current_time)
+    stale_rp = ScoutApm::StoreReportingPeriod.new(current_time - current_time.sec - 120)
+
+    # layaway.write_reporting_period returns nil on successful write
+    # It should probably be changed to return true or the number of bytes written
+    assert_nil layaway.write_reporting_period(stale_rp, 1)
+
+    # layaway.write_reporting_period returns an explicit false class on failure
+    assert layaway.write_reporting_period(current_rp, 1).is_a?(FalseClass)
+
+    layaway.delete_files_for(:all)
+  end
 end

data/test/unit/store_test.rb

@@ -20,6 +20,21 @@ class StoreTest < Minitest::Test
 
     assert_equal({}, s.reporting_periods)
   end
+
+  def test_writing_layaway_removes_stale_timestamps
+    current_time = Time.now.utc
+    current_rp = ScoutApm::StoreReportingPeriod.new(current_time)
+    stale_rp = ScoutApm::StoreReportingPeriod.new(current_time - current_time.sec - 120)
+
+    s = ScoutApm::Store.new
+    ScoutApm::Instruments::Process::ProcessMemory.new(Logger.new(StringIO.new)).metrics(stale_rp.timestamp, s)
+    ScoutApm::Instruments::Process::ProcessMemory.new(Logger.new(StringIO.new)).metrics(current_rp.timestamp, s)
+    assert_equal 2, s.reporting_periods.size
+
+    s.write_to_layaway(FakeFailingLayaway.new, true)
+
+    assert_equal({}, s.reporting_periods)
+  end
 end
 
 class StoreReportingPeriodTest < Minitest::Test

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: scout_apm
 version: !ruby/object:Gem::Version
-  version: 3.0.0.pre6
+  version: 3.0.0.pre7
 platform: ruby
 authors:
 - Derek Haynes
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-12-13 00:00:00.000000000 Z
+date: 2016-12-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: minitest
@@ -146,7 +146,6 @@ files:
 - lib/scout_apm/background_worker.rb
 - lib/scout_apm/bucket_name_splitter.rb
 - lib/scout_apm/call_set.rb
-- lib/scout_apm/capacity.rb
 - lib/scout_apm/config.rb
 - lib/scout_apm/context.rb
 - lib/scout_apm/environment.rb
@@ -293,34 +292,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: 1.3.1
 requirements: []
 rubyforge_project: scout_apm
-rubygems_version: 2.5.1
+rubygems_version: 2.2.2
 signing_key: 
 specification_version: 4
 summary: Ruby application performance monitoring
-test_files:
-- test/data/config_test_1.yml
-- test/test_helper.rb
-- test/unit/agent_test.rb
-- test/unit/background_job_integrations/sidekiq_test.rb
-- test/unit/config_test.rb
-- test/unit/context_test.rb
-- test/unit/environment_test.rb
-- test/unit/git_revision_test.rb
-- test/unit/histogram_test.rb
-- test/unit/ignored_uris_test.rb
-- test/unit/instruments/active_record_instruments_test.rb
-- test/unit/instruments/net_http_test.rb
-- test/unit/instruments/percentile_sampler_test.rb
-- test/unit/layaway_test.rb
-- test/unit/layer_children_set_test.rb
-- test/unit/limited_layer_test.rb
-- test/unit/metric_set_test.rb
-- test/unit/scored_item_set_test.rb
-- test/unit/serializers/payload_serializer_test.rb
-- test/unit/slow_job_policy_test.rb
-- test/unit/slow_request_policy_test.rb
-- test/unit/sql_sanitizer_test.rb
-- test/unit/store_test.rb
-- test/unit/utils/active_record_metric_name_test.rb
-- test/unit/utils/backtrace_parser_test.rb
-- test/unit/utils/numbers_test.rb
+test_files: []

data/lib/scout_apm/capacity.rb

@@ -1,57 +0,0 @@
-# Encapsulates logic for determining capacity utilization of the Ruby processes.
-module ScoutApm
-  class Capacity
-    attr_reader :processing_start_time, :accumulated_time, :transaction_entry_time
-
-    def initialize
-      @processing_start_time = Time.now
-      @lock ||= Mutex.new # the transaction_entry_time could be modified while processing a request or when #process is called.
-      @accumulated_time = 0.0
-    end
-
-    # Called when a transaction is traced.
-    def start_transaction!
-      @lock.synchronize do
-        @transaction_entry_time = Time.now
-      end
-    end
-
-    # Called when a transaction completes to record its time used.
-    def finish_transaction!
-      @lock.synchronize do
-        if transaction_entry_time
-          @accumulated_time += (Time.now - transaction_entry_time).to_f
-        else
-          ScoutApm::Agent.instance.logger.warn "No transaction entry time. Not recording capacity metrics for transaction."
-        end
-        @transaction_entry_time = nil
-      end
-    end
-
-    # Ran when sending metrics to server. Reports capacity usage metrics.
-    def process
-      process_time = Time.now
-      ScoutApm::Agent.instance.logger.debug "Processing capacity usage for [#{@processing_start_time}] to [#{process_time}]. Time Spent: #{@accumulated_time}."
-      @lock.synchronize do
-        time_spent = @accumulated_time
-        @accumulated_time = 0.0
-        # If a transaction is still running, capture its running time up to now and
-        # reset the +transaction_entry_time+ to now.
-        if @transaction_entry_time
-          time_spent += (process_time - @transaction_entry_time).to_f
-          ScoutApm::Agent.instance.logger.debug "A transaction is running while calculating capacity. Start time: [#{transaction_entry_time}]. Will update the entry time to [#{process_time}]."
-          @transaction_entry_time = process_time # prevent from over-counting capacity usage. update the transaction start time to now.
-        end
-        time_spent = 0.0 if time_spent < 0.0
-
-        window = (process_time - processing_start_time).to_f # time period we are evaulating capacity usage.
-        window = 1.0 if window <= 0.0 # prevent divide-by-zero if clock adjusted.
-        capacity = time_spent / window
-        ScoutApm::Agent.instance.logger.debug "Instance/Capacity: #{capacity}"
-        ScoutApm::Agent.instance.store.track_one!("Instance", "Capacity", capacity)
-
-        @processing_start_time = process_time
-      end
-    end
-  end
-end