scout_apm 3.0.0.pre5 → 3.0.0.pre6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b7390634901f7ed74144c6790bc71af190f24c8d
-  data.tar.gz: 8741c95aadfaa77c821e5543fe6e831a5d8d0b9d
+  metadata.gz: a17bfe8d072cc469eafdaf6f5d9a53e29282ea6a
+  data.tar.gz: 39707f4cfacb744879297744928c038fe1a1c8aa
 SHA512:
-  metadata.gz: ef79c08136c8238d7c284a825e85a6dfd2b2d19af8b573582b0e6ec93a23dfc00789d39555380c59d152e4344caf3f05957a43d06cf743f14a8c1609ac6a0623
-  data.tar.gz: d735090eea242c1bc02591c1068ad04aa839c776245b97e733801b54f94be98b434c1729dbf6fd3c70eabcbad917471d916fa983ddd15ceb6422088ba58ec7f7
+  metadata.gz: e9d208abf69daf67b5ca3bf487117e58cd434b445cd43126d9ec6ba90242b2179b1f4913aff3e3d8a38d89c6e0c2940d559059509baab072b7d0aac1899a936f
+  data.tar.gz: 91e4143528147d48c5eec703002cfd783579e89fca9c48806e0ec9745a57b622f660f9a7acd5186e6fc0ac5aef0c61f3e69bcc8a1bd5e29fd6ed94c5270e90d2

data/CHANGELOG.markdown

@@ -2,6 +2,10 @@
 
 * ScoutProf BETA
 
+# 2.1.17
+
+* Additional logging around file system usage
+
 # 2.1.16
 
 * Extract the name correctly for DelayedJob workers run via ActiveJob

data/lib/scout_apm/agent/reporting.rb

@@ -24,7 +24,7 @@ module ScoutApm
         report_to_server
       end
 
-      # In a running app, one process will get one period ready for delivery, the others will see 0.
+      # In a running app, one process will get the period ready for delivery, the others will see 0.
       def report_to_server
         period_to_report = ScoutApm::StoreReportingPeriodTimestamp.minutes_ago(2)
 
@@ -101,7 +101,7 @@ module ScoutApm
         histogram_clause  = "#{histograms.length} Histograms"
 
         logger.info "#{time_clause} Delivering #{metrics_clause} and #{slow_trans_clause} and #{job_clause}, #{process_log_str}."
-        # logger.debug("Metrics: #{metrics.pretty_inspect}\nSlowTrans: #{slow_transactions.pretty_inspect}\nMetadata: #{metadata.inspect.pretty_inspect}")
+        # logger.debug("\n\nMetrics: #{metrics.pretty_inspect}\nSlowTrans: #{slow_transactions.pretty_inspect}\nMetadata: #{metadata.inspect.pretty_inspect}\n\n")
       end
 
       # TODO: Move this into PayloadSerializer?

data/lib/scout_apm/agent.rb

@@ -113,6 +113,7 @@ module ScoutApm
     def start(options = {})
       @options.merge!(options)
 
+
       @config = ScoutApm::Config.with_file(@config.value("config_file"))
       layaway.config = config
 
@@ -123,6 +124,12 @@ module ScoutApm
 
       load_instruments if should_load_instruments?(options)
 
+      if !@config.any_keys_found?
+        logger.info("No configuration file loaded, and no configuration found in ENV. " +
+                    "For assistance configuring Scout, visit " +
+                    "http://help.apm.scoutapp.com/#configuration-options")
+      end
+
       return false unless preconditions_met?(options)
       @started = true
       logger.info "Starting monitoring for [#{environment.application_name}]. Framework [#{environment.framework}] App Server [#{environment.app_server}] Background Job Framework [#{environment.background_job_name}]."

data/lib/scout_apm/config.rb

@@ -5,25 +5,55 @@ require 'scout_apm/environment'
 
 # Valid Config Options:
 #
+# This list is complete, but some are old and unused, or for developers of
+# scout_apm itself. See the documentation at http://help.apm.scoutapp.com for
+# customer-focused documentation.
+#
 # application_root - override the detected directory of the application
+# compress_payload - true/false to enable gzipping of payload
 # data_file        - override the default temporary storage location. Must be a location in a writable directory
-# host             - override the default hostname detection. Default varies by environment - either system hostname, or PAAS hostname
+# dev_trace        - true or false. Enables always-on tracing in development environmen only
 # direct_host      - override the default "direct" host. The direct_host bypasses the ingestion pipeline and goes directly to the webserver, and is primarily used for features under development.
+# enable_background_jobs - true or false
+# host             - configuration used in development
+# hostname         - override the default hostname detection. Default varies by environment - either system hostname, or PAAS hostname
 # key              - the account key with Scout APM. Found in Settings in the Web UI
 # log_file_path    - either a directory or "STDOUT".
 # log_level        - DEBUG / INFO / WARN as usual
 # monitor          - true or false.  False prevents any instrumentation from starting
 # name             - override the name reported to APM. This is the name that shows in the Web UI
-# uri_reporting    - 'path' or 'full_path' default is 'full_path', which reports URL params as well as the path.
+# profile          - turn on/off scoutprof (only applicable in Gem versions including scoutprof)
+# proxy            - an http proxy
 # report_format    - 'json' or 'marshal'. Marshal is legacy and will be removed.
-# dev_trace        - true or false. Enables always-on tracing in development environmen only
-# enable_background_jobs - true or false
+# uri_reporting    - 'path' or 'full_path' default is 'full_path', which reports URL params as well as the path.
 #
 # Any of these config settings can be set with an environment variable prefixed
 # by SCOUT_ and uppercasing the key: SCOUT_LOG_LEVEL for instance.
 
 module ScoutApm
   class Config
+    KNOWN_CONFIG_OPTIONS = [
+        'application_root',
+        'compress_payload',
+        'config_file',
+        'data_file',
+        'dev_trace',
+        'direct_host',
+        'disabled_instruments',
+        'enable_background_jobs',
+        'host',
+        'hostname',
+        'ignore',
+        'key',
+        'log_level',
+        'log_file_path',
+        'monitor',
+        'name',
+        'profile',
+        'proxy',
+        'report_format',
+        'uri_reporting',
+    ]
 
     ################################################################################
     # Coersions
@@ -136,6 +166,10 @@ module ScoutApm
     end
 
     def value(key)
+      if ! KNOWN_CONFIG_OPTIONS.include?(key)
+        ScoutApm::Agent.instance.logger.debug("Requested looking up a unknown configuration key: #{key} (not a problem. Evaluate and add to config.rb)")
+      end
+
       o = @overlays.detect{ |overlay| overlay.has_key?(key) }
       raw_value = if o
                     o.value(key)
@@ -148,19 +182,24 @@ module ScoutApm
       coercion.coerce(raw_value)
     end
 
+    # Did we load anything for configuration?
+    def any_keys_found?
+      @overlays.any? { |overlay| overlay.any_keys_found? }
+    end
+
     class ConfigDefaults
       DEFAULTS = {
-        'host'                   => 'https://checkin.scoutapp.com',
+        'compress_payload'       => true,
+        'dev_trace'              => false,
         'direct_host'            => 'https://apm.scoutapp.com',
-        'log_level'              => 'info',
-        'uri_reporting'          => 'full_path',
-        'report_format'          => 'json',
         'disabled_instruments'   => [],
         'enable_background_jobs' => true,
+        'host'                   => 'https://checkin.scoutapp.com',
         'ignore'                 => [],
-        'dev_trace'              => false,
+        'log_level'              => 'info',
         'profile'                => true, # for scoutprof
-        'compress_payload'       => true,
+        'report_format'          => 'json',
+        'uri_reporting'          => 'full_path',
       }.freeze
 
       def value(key)
@@ -170,6 +209,11 @@ module ScoutApm
       def has_key?(key)
         DEFAULTS.has_key?(key)
       end
+
+      # Defaults are here, but not counted as user specified.
+      def any_keys_found?
+        false
+      end
     end
 
 
@@ -184,6 +228,10 @@ module ScoutApm
       def has_key?(*)
         true
       end
+
+      def any_keys_found?
+        false
+      end
     end
 
     class ConfigEnvironment
@@ -199,6 +247,12 @@ module ScoutApm
       def key_to_env_key(key)
         'SCOUT_' + key.upcase
       end
+
+      def any_keys_found?
+        KNOWN_CONFIG_OPTIONS.any? { |option|
+          ENV.has_key?(key_to_env_key(option))
+        }
+      end
     end
 
     # Attempts to load a configuration file, and parse it as YAML. If the file
@@ -224,6 +278,12 @@ module ScoutApm
         @settings.has_key?(key)
       end
 
+      def any_keys_found?
+        KNOWN_CONFIG_OPTIONS.any? { |option|
+          @settings.has_key?(option)
+        }
+      end
+
       private
 
       def load_file(file)
@@ -255,7 +315,7 @@ module ScoutApm
             @file_loaded = false
           end
         rescue Exception => e # Explicit `Exception` handling to catch SyntaxError and anything else that ERB or YAML may throw
-          logger.debug("Failed loading configuration file: #{e.message}. ScoutAPM will continue starting with configuration from ENV and defaults")
+          logger.info("Failed loading configuration file (#{@resolved_file_path}): #{e.message}. ScoutAPM will continue starting with configuration from ENV and defaults")
           @file_loaded = false
         end
       end

data/lib/scout_apm/layaway.rb

@@ -7,9 +7,6 @@
 #
 module ScoutApm
   class Layaway
-    # How old a file needs to be in Seconds before it gets reported.
-    REPORTING_AGE = 120
-
     # How long to let a stale file sit before deleting it.
     # Letting it sit a bit may be useful for debugging
     STALE_AGE = 10 * 60
@@ -60,7 +57,6 @@ module ScoutApm
     def with_claim(timestamp)
       coordinator_file = glob_pattern(timestamp, :coordinator)
 
-
       begin
         # This file gets deleted only by a process that successfully created and obtained the exclusive lock
         f = File.open(coordinator_file, File::RDWR | File::CREAT | File::EXCL | File::NONBLOCK)
@@ -70,25 +66,35 @@ module ScoutApm
 
       begin
         if f
-
-          ScoutApm::Agent.instance.logger.debug("Obtained Reporting Lock")
-
-          files = all_files_for(timestamp).reject{|l| l.to_s == coordinator_file.to_s }
-          rps = files.map{ |layaway| LayawayFile.new(layaway).load }.compact
-          if rps.any?
-            yield rps
-
-            delete_files_for(timestamp) # also removes the coodinator_file
-            delete_stale_files(timestamp.to_time - STALE_AGE)
-          else
-            File.unlink(coordinator_file)
-            ScoutApm::Agent.instance.logger.debug("No layaway files to report")
+          begin
+            ScoutApm::Agent.instance.logger.debug("Obtained Reporting Lock")
+
+            log_layaway_file_information
+
+            files = all_files_for(timestamp).reject{|l| l.to_s == coordinator_file.to_s }
+            rps = files.map{ |layaway| LayawayFile.new(layaway).load }.compact
+            if rps.any?
+              yield rps
+
+              ScoutApm::Agent.instance.logger.debug("Deleting the now-reported layaway files for #{timestamp.to_s}")
+              delete_files_for(timestamp) # also removes the coodinator_file
+
+              ScoutApm::Agent.instance.logger.debug("Checking for any Stale layaway files")
+              delete_stale_files(timestamp.to_time - STALE_AGE)
+            else
+              File.unlink(coordinator_file)
+              ScoutApm::Agent.instance.logger.debug("No layaway files to report")
+            end
+
+            true
+          rescue Exception => e
+            ScoutApm::Agent.instance.logger.debug("Caught an exception in with_claim, with the coordination file locked: #{e.message}, #{e.backtrace.inspect}")
+            raise
+          ensure
+            # Unlock the file when done!
+            f.flock(File::LOCK_UN | File::LOCK_NB)
+            f.close
           end
-
-          # Unlock the file when done!
-          f.flock(File::LOCK_UN | File::LOCK_NB)
-          f.close
-          true
         else
           # Didn't obtain lock, another process is reporting. Return false from this function, but otherwise no work
           false
@@ -97,7 +103,10 @@ module ScoutApm
     end
 
     def delete_files_for(timestamp)
-      all_files_for(timestamp).each { |layaway| File.unlink(layaway) }
+      all_files_for(timestamp).each { |layaway|
+        ScoutApm::Agent.instance.logger.debug("Deleting layaway file: #{layaway}")
+        File.unlink(layaway)
+      }
     end
 
     def delete_stale_files(older_than)
@@ -108,6 +117,8 @@ module ScoutApm
         select { |timestamp| timestamp.to_i < older_than.strftime(TIME_FORMAT).to_i }.
           tap  { |timestamps| ScoutApm::Agent.instance.logger.debug("Deleting stale layaway files with timestamps: #{timestamps.inspect}") }.
         map    { |timestamp| delete_files_for(timestamp) }
+    rescue => e
+      ScoutApm::Agent.instance.logger.debug("Problem deleting stale files: #{e.message}, #{e.backtrace.inspect}")
     end
 
     private
@@ -158,6 +169,20 @@ module ScoutApm
         nil
       end
     end
+
+    def log_layaway_file_information
+      files_in_temp = Dir["#{directory}/*"].count
+
+      all_filenames = all_files_for(:all)
+      count_per_timestamp = Hash[
+        all_filenames.
+        group_by {|f| timestamp_from_filename(f) }.
+        map{ |timestamp, list| [timestamp, list.length] }
+      ]
+
+
+      ScoutApm::Agent.instance.logger.debug("Total in #{directory}: #{files_in_temp}. Total Layaway Files: #{all_filenames.size}.  By Timestamp: #{count_per_timestamp.inspect}")
+    end
   end
 end
 

data/lib/scout_apm/layer_converters/converter_base.rb

@@ -88,8 +88,6 @@ module ScoutApm
         if bt.any?
           meta.backtrace = bt
           @backtraces << meta
-        else
-          ScoutApm::Agent.instance.logger.debug { "Unable to capture an app-specific backtrace for #{meta.inspect}\n#{layer.backtrace}" }
         end
       end
 
@@ -157,9 +155,11 @@ module ScoutApm
         end
       end
 
-      def make_meta_options_desc_hash(layer)
+      def make_meta_options_desc_hash(layer, max_desc_length=1000)
         if layer.desc
-          {:desc => layer.desc.to_s}
+          desc_s = layer.desc.to_s
+          trimmed_desc = desc_s[0 .. max_desc_length]
+          {:desc => trimmed_desc}
         else
           {}
         end

data/lib/scout_apm/reporter.rb

@@ -27,7 +27,6 @@ module ScoutApm
         headers.merge!(compression_headers)
 
         compress_payload_size = payload.length
-        ScoutApm::Agent.instance.logger.debug("Compressed Payload: #{payload.inspect}")
         ScoutApm::Agent.instance.logger.debug("Original Size: #{original_payload_size} Compressed Size: #{compress_payload_size}")
       end
 

data/lib/scout_apm/server_integrations/puma.rb

@@ -24,10 +24,13 @@ module ScoutApm
       end
 
       def install
-        ::Puma.cli_config.options[:before_worker_boot] << Proc.new do
+        old = ::Puma.cli_config.options[:before_worker_boot] || []
+        new = Array(old) + [Proc.new do
           logger.info "Installing Puma worker loop."
           ScoutApm::Agent.instance.start_background_worker
-        end
+        end]
+
+        ::Puma.cli_config.options[:before_worker_boot] = new
       rescue
         logger.warn "Unable to install Puma worker loop: #{$!.message}"
       end

data/lib/scout_apm/store.rb

@@ -83,7 +83,7 @@ module ScoutApm
     def write_to_layaway(layaway, force=false)
       ScoutApm::Agent.instance.logger.debug("Writing to layaway#{" (Forced)" if force}")
 
-        reporting_periods.select { |time, rp| force || time.timestamp < current_timestamp.timestamp }.
+        reporting_periods.select { |time, rp| force || (time.timestamp < current_timestamp.timestamp) }.
                           each   { |time, rp| collect_samplers(rp) }.
                           each   { |time, rp| write_reporting_period(layaway, time, rp) }
     end
@@ -95,7 +95,9 @@ module ScoutApm
     rescue => e
       ScoutApm::Agent.instance.logger.warn("Failed writing data to layaway file: #{e.message} / #{e.backtrace}")
     ensure
-      reporting_periods.delete(time)
+      ScoutApm::Agent.instance.logger.debug("Before delete, reporting periods length: #{reporting_periods.size}")
+      deleted_items = reporting_periods.delete(time)
+      ScoutApm::Agent.instance.logger.debug("After delete, reporting periods length: #{reporting_periods.size}. Did delete #{deleted_items}")
     end
 
     ######################################

data/lib/scout_apm/version.rb

@@ -1,3 +1,3 @@
 module ScoutApm
-  VERSION = "3.0.0.pre5"
+  VERSION = "3.0.0.pre6"
 end

data/test/unit/config_test.rb

@@ -67,4 +67,15 @@ class ConfigTest < Minitest::Test
     assert_equal 10, coercion.coerce(10)
     assert_equal ["a"], coercion.coerce(["a"])
   end
+
+  def test_any_keys_found
+    ENV.stubs(:has_key?).returns(nil)
+
+    conf = ScoutApm::Config.with_file("a_file_that_doesnt_exist.yml")
+    assert ! conf.any_keys_found?
+
+    ENV.stubs(:has_key?).with("SCOUT_MONITOR").returns("true")
+    conf = ScoutApm::Config.with_file("a_file_that_doesnt_exist.yml")
+    assert conf.any_keys_found?
+  end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: scout_apm
 version: !ruby/object:Gem::Version
-  version: 3.0.0.pre5
+  version: 3.0.0.pre6
 platform: ruby
 authors:
 - Derek Haynes
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-11-29 00:00:00.000000000 Z
+date: 2016-12-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: minitest
@@ -293,7 +293,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: 1.3.1
 requirements: []
 rubyforge_project: scout_apm
-rubygems_version: 2.4.8
+rubygems_version: 2.5.1
 signing_key: 
 specification_version: 4
 summary: Ruby application performance monitoring