scout_apm 2.1.17 → 2.1.18

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 66a4cffa0669e1689fd354ca6cc1334917268580
-  data.tar.gz: 07b7b35f971caaecce344a6a637a945df0c8523a
+  metadata.gz: c9fe21cd99b734190fb03448350e744c3b830b15
+  data.tar.gz: a3a2160d835f28ee99715887653540d5afc35326
 SHA512:
-  metadata.gz: 4c3f3e587c1b98fd7c679b9a40fa79970c38085dda00c7524021c1caa5d51e174ada700ee2f8401292b3497b52af99e600e8cefc2e6d88e26128791e4c06dc9f
-  data.tar.gz: 1d36607a77b765fe849e7a554db6811d3e6db22e6460862141d1ffd83d6faa20bb44fff08c52894ce2ae71fbe5ff9cbd803dc5b69138fdee62f836e6b0a0a09a
+  metadata.gz: bd2c9581ae948f73cf05f2f860971dcea15bc60e5702bca3da74cdcf4977d248cbd2c85174ef4ab7d22f979bfd036f506a4c00286c588f135ce3dac8f4bf57ce
+  data.tar.gz: 864122b7668d62d6a65b80127e70d1f87852a09b1afc5a0f8b522704b99701be4b11f57777e93b7ed18d4a3b1f5226a8362f9a63ca388fc085fdbd5ee819e197

data/CHANGELOG.markdown

@@ -1,3 +1,7 @@
+# 2.1.18
+
+* Max layaway file threshold limit
+
 # 2.1.17
 
 * Additional logging around file system usage

data/lib/scout_apm.rb

@@ -126,7 +126,6 @@ require 'scout_apm/slow_job_policy'
 require 'scout_apm/job_record'
 require 'scout_apm/request_histograms'
 
-require 'scout_apm/capacity'
 require 'scout_apm/attribute_arranger'
 require 'scout_apm/git_revision'
 

data/lib/scout_apm/agent.rb

@@ -12,7 +12,6 @@ module ScoutApm
     attr_accessor :store
     attr_accessor :layaway
     attr_accessor :config
-    attr_accessor :capacity
     attr_accessor :logger
     attr_accessor :log_file # path to the log file
     attr_accessor :options # options passed to the agent when +#start+ is called.
@@ -58,7 +57,6 @@ module ScoutApm
       @layaway        = ScoutApm::Layaway.new(config, environment)
       @metric_lookup  = Hash.new
 
-      @capacity       = ScoutApm::Capacity.new
       @installed_instruments = []
     end
 

data/lib/scout_apm/layaway.rb

@@ -11,6 +11,9 @@ module ScoutApm
     # Letting it sit a bit may be useful for debugging
     STALE_AGE = 10 * 60
 
+    # Failsafe to prevent writing layaway files if for some reason they are not being cleaned up
+    MAX_FILES_LIMIT = 5000
+
     # A strftime format string for how we render timestamps in filenames.
     # Must be sortable as an integer
     TIME_FORMAT = "%Y%m%d%H%M"
@@ -45,7 +48,11 @@ module ScoutApm
       @directory = Pathname.new(found)
     end
 
-    def write_reporting_period(reporting_period)
+    def write_reporting_period(reporting_period, files_limit = MAX_FILES_LIMIT)
+      if at_layaway_file_limit?(files_limit)
+        ScoutApm::Agent.instance.logger.error("Hit layaway file limit. Not writing to layaway file")
+        return false
+      end
       filename = file_for(reporting_period.timestamp)
       layaway_file = LayawayFile.new(filename)
       layaway_file.write(reporting_period)
@@ -170,6 +177,10 @@ module ScoutApm
       end
     end
 
+    def at_layaway_file_limit?(files_limit = MAX_FILES_LIMIT)
+      all_files_for(:all).count >= files_limit
+    end
+
     def log_layaway_file_information
       files_in_temp = Dir["#{directory}/*"].count
 

data/lib/scout_apm/metric_set.rb

@@ -1,6 +1,8 @@
 module ScoutApm
   class MetricSet
-    # We can't aggregate CPU, Memory, Capacity, or Controller, so pass through these metrics directly
+    # We can't aggregate a handful of things like samplers (CPU, Memory), or
+    # Controller, and Percentiles so pass through these metrics directly
+    #
     # TODO: Figure out a way to not have this duplicate what's in Samplers, and also on server's ingest
     PASSTHROUGH_METRICS = ["CPU", "Memory", "Instance", "Controller", "SlowTransaction", "Percentile", "Job"]
 

data/lib/scout_apm/version.rb

@@ -1,4 +1,4 @@
 module ScoutApm
-  VERSION = "2.1.17"
+  VERSION = "2.1.18"
 end
 

data/test/test_helper.rb

@@ -16,6 +16,14 @@ Kernel.module_eval do
   def self.const_unset(const)
     self.instance_eval { remove_const(const) }
   end
+
+  def silence_warnings(&block)
+    warn_level = $VERBOSE
+    $VERBOSE = nil
+    result = block.call
+    $VERBOSE = warn_level
+    result
+  end
 end
 
 # A test helper class to create a temporary "configuration" we can control entirely purposes

data/test/unit/layaway_test.rb

@@ -21,4 +21,24 @@ class LayawayTest < Minitest::Test
 
     assert_equal Pathname.new("/tmp/scout_apm_test/tmp"), ScoutApm::Layaway.new(config, env).directory
   end
+
+  def test_layaway_file_limit_prevents_new_writes
+    FileUtils.mkdir_p '/tmp/scout_apm_test/layaway_limit'
+    config = make_fake_config("data_file" => "/tmp/scout_apm_test/layaway_limit")
+    layaway = ScoutApm::Layaway.new(config, ScoutApm::Agent.instance.environment)
+    layaway.delete_files_for(:all)
+
+    current_time = Time.now.utc
+    current_rp = ScoutApm::StoreReportingPeriod.new(current_time)
+    stale_rp = ScoutApm::StoreReportingPeriod.new(current_time - current_time.sec - 120)
+
+    # layaway.write_reporting_period returns nil on successful write
+    # It should probably be changed to return true or the number of bytes written
+    assert_nil layaway.write_reporting_period(stale_rp, 1)
+
+    # layaway.write_reporting_period returns an explicit false class on failure
+    assert layaway.write_reporting_period(current_rp, 1).is_a?(FalseClass)
+
+    layaway.delete_files_for(:all)
+  end
 end

data/test/unit/store_test.rb

@@ -20,6 +20,21 @@ class StoreTest < Minitest::Test
 
     assert_equal({}, s.reporting_periods)
   end
+
+  def test_writing_layaway_removes_stale_timestamps
+    current_time = Time.now.utc
+    current_rp = ScoutApm::StoreReportingPeriod.new(current_time)
+    stale_rp = ScoutApm::StoreReportingPeriod.new(current_time - current_time.sec - 120)
+
+    s = ScoutApm::Store.new
+    ScoutApm::Instruments::Process::ProcessMemory.new(Logger.new(StringIO.new)).metrics(stale_rp.timestamp, s)
+    ScoutApm::Instruments::Process::ProcessMemory.new(Logger.new(StringIO.new)).metrics(current_rp.timestamp, s)
+    assert_equal 2, s.reporting_periods.size
+
+    s.write_to_layaway(FakeFailingLayaway.new, true)
+
+    assert_equal({}, s.reporting_periods)
+  end
 end
 
 class StoreReportingPeriodTest < Minitest::Test

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: scout_apm
 version: !ruby/object:Gem::Version
-  version: 2.1.17
+  version: 2.1.18
 platform: ruby
 authors:
 - Derek Haynes
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-12-13 00:00:00.000000000 Z
+date: 2016-12-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: minitest
@@ -142,7 +142,6 @@ files:
 - lib/scout_apm/background_worker.rb
 - lib/scout_apm/bucket_name_splitter.rb
 - lib/scout_apm/call_set.rb
-- lib/scout_apm/capacity.rb
 - lib/scout_apm/config.rb
 - lib/scout_apm/context.rb
 - lib/scout_apm/environment.rb
@@ -286,34 +285,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: scout_apm
-rubygems_version: 2.5.1
+rubygems_version: 2.2.2
 signing_key: 
 specification_version: 4
 summary: Ruby application performance monitoring
-test_files:
-- test/data/config_test_1.yml
-- test/test_helper.rb
-- test/unit/agent_test.rb
-- test/unit/background_job_integrations/sidekiq_test.rb
-- test/unit/config_test.rb
-- test/unit/context_test.rb
-- test/unit/environment_test.rb
-- test/unit/git_revision_test.rb
-- test/unit/histogram_test.rb
-- test/unit/ignored_uris_test.rb
-- test/unit/instruments/active_record_instruments_test.rb
-- test/unit/instruments/net_http_test.rb
-- test/unit/instruments/percentile_sampler_test.rb
-- test/unit/layaway_test.rb
-- test/unit/layer_children_set_test.rb
-- test/unit/limited_layer_test.rb
-- test/unit/metric_set_test.rb
-- test/unit/scored_item_set_test.rb
-- test/unit/serializers/payload_serializer_test.rb
-- test/unit/slow_job_policy_test.rb
-- test/unit/slow_request_policy_test.rb
-- test/unit/sql_sanitizer_test.rb
-- test/unit/store_test.rb
-- test/unit/utils/active_record_metric_name_test.rb
-- test/unit/utils/backtrace_parser_test.rb
-- test/unit/utils/numbers_test.rb
+test_files: []

data/lib/scout_apm/capacity.rb

@@ -1,57 +0,0 @@
-# Encapsulates logic for determining capacity utilization of the Ruby processes.
-module ScoutApm
-  class Capacity
-    attr_reader :processing_start_time, :accumulated_time, :transaction_entry_time
-
-    def initialize
-      @processing_start_time = Time.now
-      @lock ||= Mutex.new # the transaction_entry_time could be modified while processing a request or when #process is called.
-      @accumulated_time = 0.0
-    end
-
-    # Called when a transaction is traced.
-    def start_transaction!
-      @lock.synchronize do
-        @transaction_entry_time = Time.now
-      end
-    end
-
-    # Called when a transaction completes to record its time used.
-    def finish_transaction!
-      @lock.synchronize do
-        if transaction_entry_time
-          @accumulated_time += (Time.now - transaction_entry_time).to_f
-        else
-          ScoutApm::Agent.instance.logger.warn "No transaction entry time. Not recording capacity metrics for transaction."
-        end
-        @transaction_entry_time = nil
-      end
-    end
-
-    # Ran when sending metrics to server. Reports capacity usage metrics.
-    def process
-      process_time = Time.now
-      ScoutApm::Agent.instance.logger.debug "Processing capacity usage for [#{@processing_start_time}] to [#{process_time}]. Time Spent: #{@accumulated_time}."
-      @lock.synchronize do
-        time_spent = @accumulated_time
-        @accumulated_time = 0.0
-        # If a transaction is still running, capture its running time up to now and
-        # reset the +transaction_entry_time+ to now.
-        if @transaction_entry_time
-          time_spent += (process_time - @transaction_entry_time).to_f
-          ScoutApm::Agent.instance.logger.debug "A transaction is running while calculating capacity. Start time: [#{transaction_entry_time}]. Will update the entry time to [#{process_time}]."
-          @transaction_entry_time = process_time # prevent from over-counting capacity usage. update the transaction start time to now.
-        end
-        time_spent = 0.0 if time_spent < 0.0
-
-        window = (process_time - processing_start_time).to_f # time period we are evaulating capacity usage.
-        window = 1.0 if window <= 0.0 # prevent divide-by-zero if clock adjusted.
-        capacity = time_spent / window
-        ScoutApm::Agent.instance.logger.debug "Instance/Capacity: #{capacity}"
-        ScoutApm::Agent.instance.store.track_one!("Instance", "Capacity", capacity)
-
-        @processing_start_time = process_time
-      end
-    end
-  end
-end