scout_apm 1.2.3 → 1.2.4.pre

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 02dd0f4a1c88164e5406dbdb596137fc8c86f82a
-  data.tar.gz: f2eef64f0f21280c47dacb09ba81feecb321db1d
+  metadata.gz: fa64d725755deae00527598cd770bf40a773fcf8
+  data.tar.gz: 17b868c65b2b6bd7749fb14dd7164a773248a26b
 SHA512:
-  metadata.gz: fc319d3d9733ccf56548f09ddbce8e180849a6f8b2efd2e47524b0f4f68f23f6ec7d93f607409c9285d9294dffe91e025b21c2339d862a58d40589612d1f7fc0
-  data.tar.gz: e5739f804d6fa5abd2a29a4786dbb21f890fe63322a66999001b0a275cbe8aaed631d53295eabbd2d575ff1eb52aebbe1479372ae2d15343a4960977ef3f02b4
+  metadata.gz: 0ba12e4e11843ef25e3005d51c6e7e9f0d38a3db2a15f3c2e5bce014bc7ceaadf68e33f1b5443535cb938936232800755e5b20f3e21e3183d26598062fc9f018
+  data.tar.gz: eaf8da676edae1147d935594e7ffdfa070bfcbd7fcf3c7bce18c835262a9e98121ae21e5fd916bd391325f4d85f8fcc4975b6f18949b7dac54e246b8d24a9f8d

data/.gitignore

@@ -4,4 +4,5 @@
 Gemfile.lock
 pkg/*
 .rvmrc
-.idea
+.idea
+test/tmp/*

data/CHANGELOG.markdown

@@ -1,3 +1,8 @@
+# 1.2.4
+
+* Removing layaway file validation in main thread
+* Fixing :force so agent will start in tests
+
 # 1.2.3
 
 * Trimming metrics from slow requests if there are more than 10.

data/lib/scout_apm/agent.rb

@@ -47,23 +47,28 @@ module ScoutApm
     end
 
     def apm_enabled?
-      config.value('monitor') and !@options[:force]
+      config.value('monitor')
+    end
+
+    # If true, the agent will start regardless of safety checks. Currently just used for testing.
+    def force?
+      @options[:force]
     end
 
     def preconditions_met?(options={})
       if !apm_enabled?
-        logger.warn "Monitoring isn't enabled for the [#{environment.env}] environment."
-        return false
+        logger.warn "Monitoring isn't enabled for the [#{environment.env}] environment. #{force? ? 'Forcing agent to start' : 'Not starting agent'}"
+        return false unless force?
       end
 
       if !environment.application_name
-        logger.warn "An application name could not be determined. Specify the :name value in scout_apm.yml. Not starting agent."
-        return false
+        logger.warn "An application name could not be determined. Specify the :name value in scout_apm.yml. #{force? ? 'Forcing agent to start' : 'Not starting agent'}."
+        return false unless force?
       end
 
       if app_server_missing?(options) && background_job_missing?
-        logger.warn "Couldn't find a supported app server or background job framework. Not starting agent."
-        return false
+        logger.warn "Couldn't find a supported app server or background job framework. #{force? ? 'Forcing agent to start' : 'Not starting agent'}."
+        return false unless force?
       end
 
       if started?
@@ -90,18 +95,10 @@ module ScoutApm
         logger.info "Starting monitoring for [#{environment.deploy_integration.name}]]."
         return environment.deploy_integration.install
       end
-
       return false unless preconditions_met?(options)
-
       @started = true
-
       logger.info "Starting monitoring for [#{environment.application_name}]. Framework [#{environment.framework}] App Server [#{environment.app_server}] Background Job Framework [#{environment.background_job_name}]."
 
-      # We need agent initialized to do this, so do it here instead.
-      # Clean up any old data in the layaway file, allows us to change the file
-      # structure / contents without worrying.
-      layaway.verify_layaway_file_contents
-
       load_instruments if should_load_instruments?(options)
 
       @samplers = [
@@ -160,8 +157,10 @@ module ScoutApm
     # in-memory metrics would be lost and a gap would appear on restarts.
     def shutdown
       return if !started?
-      @background_worker.stop
-      @background_worker.run_once
+      if @background_worker
+        @background_worker.stop
+        @background_worker.run_once
+      end
     end
 
     def started?
@@ -175,6 +174,7 @@ module ScoutApm
     def start_background_worker?
       return true if environment.app_server == :thin
       return true if environment.app_server == :webrick
+      return true if force?
       return !environment.forking?
     end
 
@@ -264,12 +264,12 @@ module ScoutApm
       end
     end
 
-    def app_server_missing?(options)
+    def app_server_missing?(options = {})
       !environment.app_server_integration(true).found? && !options[:skip_app_server_check]
     end
 
-    def background_job_missing?
-      environment.background_job_integration.nil?
+    def background_job_missing?(options = {})
+      environment.background_job_integration.nil? && !options[:skip_background_job_check]
     end
   end
 end

data/lib/scout_apm/agent/reporting.rb

@@ -15,10 +15,10 @@ module ScoutApm
       #
       # At any given point, there is data in each of those steps, moving its way through the process
       def process_metrics
-        # First we write the previous minute's data to the shared-across-process layaway file.
+        # Write the previous minute's data to the shared-across-process layaway file.
         store.write_to_layaway(layaway)
 
-        # Then attempt to send 2 minutes ago's data up to the server.  This
+        # Attempt to send 2 minutes ago's data up to the server.  This
         # only acctually occurs if this process is the first to wake up this
         # minute.
         report_to_server
@@ -65,7 +65,7 @@ module ScoutApm
           select { |meta,stats| meta.metric_name =~ /\AController/ }.
           inject(0) {|sum, (_, stat)| sum + stat.call_count }
 
-        logger.info "Delivering #{metrics.length} Metrics for #{total_request_count} requests and #{slow_transactions.length} Slow Transaction Traces"
+        logger.info "[#{Time.parse(metadata[:agent_time]).strftime("%H:%M")}] Delivering #{metrics.length} Metrics for #{total_request_count} requests and #{slow_transactions.length} Slow Transaction Traces"
         logger.debug("Metrics: #{metrics.pretty_inspect}\nSlowTrans: #{slow_transactions.pretty_inspect}\nMetadata: #{metadata.inspect.pretty_inspect}")
       end
 

data/lib/scout_apm/layaway.rb

@@ -8,21 +8,6 @@ module ScoutApm
       @file = ScoutApm::LayawayFile.new
     end
 
-    # We're changing the format, so detect if we're loading an old formatted
-    # file, and just drop it if so. There's no important data there, since it's
-    # used mostly for just syncronizing between processes
-    def verify_layaway_file_contents
-      file.read_and_write do |existing_data|
-        existing_data ||= {}
-        if existing_data.keys.all?{|k| k.is_a? StoreReportingPeriodTimestamp } &&
-            existing_data.values.all? {|v| v.is_a? StoreReportingPeriod }
-          existing_data
-        else
-          {}
-        end
-      end
-    end
-
     def add_reporting_period(time, reporting_period)
       file.read_and_write do |existing_data|
         existing_data ||= Hash.new
@@ -37,16 +22,15 @@ module ScoutApm
     # Returns an array of ReportingPeriod objects that are ready to be pushed to the server
     def periods_ready_for_delivery
       ready_for_delivery = []
-
       file.read_and_write do |existing_data|
         existing_data ||= {}
-        ready_for_delivery = existing_data.select {|time, rp| should_send?(rp) } # Select off the values we want
+        ready_for_delivery = existing_data.to_a.select {|time, rp| should_send?(rp) } # Select off the values we want. to_a is needed for compatibility with Ruby 1.8.7.
 
         # Rewrite anything not plucked out back to the file
-        existing_data.reject {|k, v| ready_for_delivery.keys.include?(k) }
+        existing_data.reject {|k, v| ready_for_delivery.map(&:first).include?(k) }
       end
 
-      return ready_for_delivery.values
+      return ready_for_delivery.map(&:last)
     end
 
     # We just want to send anything older than X

data/lib/scout_apm/store.rb

@@ -63,7 +63,11 @@ module ScoutApm
     end
 
     def eql?(o)
-      timestamp.eql?(o.timestamp)
+      self.class == o.class && timestamp.eql?(o.timestamp)
+    end
+
+    def ==(o)
+      self.eql?(o)
     end
 
     def hash

data/lib/scout_apm/version.rb

@@ -1,4 +1,4 @@
 module ScoutApm
-  VERSION = "1.2.3"
+  VERSION = "1.2.4.pre"
 end
 

data/test/test_helper.rb

@@ -19,6 +19,12 @@ end
 class Minitest::Test
   def setup
     reopen_logger
+    ENV['SCOUT_DATA_FILE'] = DATA_FILE_PATH
+  end
+
+  def teardown
+    ScoutApm::Agent.instance.shutdown
+    File.delete(DATA_FILE_PATH) if File.exist?(DATA_FILE_PATH)
   end
 
   def set_rack_env(env)
@@ -31,4 +37,6 @@ class Minitest::Test
     @logger = Logger.new(@log_contents)
     ScoutApm::Agent.instance.instance_variable_set("@logger", @logger)
   end
+
+  DATA_FILE_PATH = File.dirname(__FILE__) + '/tmp/scout_apm.db' 
 end

data/test/unit/agent_test.rb

@@ -0,0 +1,58 @@
+require 'test_helper'
+require 'scout_apm/agent'
+require 'scout_apm/slow_transaction'
+require 'scout_apm/metric_meta'
+require 'scout_apm/metric_stats'
+require 'scout_apm/context'
+require 'scout_apm/store'
+
+class AgentTest < Minitest::Test
+
+  # Safeguard to ensure the main agent thread doesn't have any interaction with the layaway file. Contention on file locks can cause delays.
+  def test_start_with_lock_on_layaway_file
+    # setup the file, putting a lock on it.
+    File.open(DATA_FILE_PATH, "w") {}
+    f=File.open(DATA_FILE_PATH, File::RDWR | File::CREAT)
+    f.flock(File::LOCK_EX)
+
+    agent = ScoutApm::Agent.instance
+
+    no_timeout = true
+    begin
+      Timeout::timeout(3) { agent.start({:monitor => true,:force => true}) }
+    rescue Timeout::Error
+      no_timeout = false
+    ensure
+      f.flock(File::LOCK_UN)
+      f.close
+    end
+    assert no_timeout, "Agent took >= 3s to start. Possible file lock issue."
+  end
+
+  def test_reset_file_with_old_format
+    File.open(DATA_FILE_PATH, 'w') { |file| file.write(Marshal.dump(OLD_FORMAT)) }
+    begin
+      ScoutApm::Agent.instance(:force => true).process_metrics
+    rescue NoMethodError
+      # The agent will raise an exception the first time metrics are processed for scout_apm < 1.2.
+      #
+      #  NoMethodError: undefined method `values' for []:Array
+      # /Users/dlite/projects/scout_apm_ruby/lib/scout_apm/layaway.rb:46:in `periods_ready_for_delivery'
+      # /Users/dlite/projects/scout_apm_ruby/lib/scout_apm/agent/reporting.rb:31:in `report_to_server'
+      # /Users/dlite/projects/scout_apm_ruby/lib/scout_apm/agent/reporting.rb:24:in `process_metrics'
+      # /Users/dlite/projects/scout_apm_ruby/test/unit/layaway_test.rb:27:in `test_reset_file_with_old_format'
+    end
+    # Data will be fine the next go-around
+    no_error = true
+    begin
+      ScoutApm::Agent.instance(:force => true).process_metrics
+    rescue Exception => e
+      no_error = false
+    end
+    assert no_error, "Error trying to process metrics after upgrading from < 1.2 data format: #{e.message if e}"
+  end
+
+  ## TODO - adds tests to ensure other potentially long-running things don't sneak in, like HTTP calls.
+
+  OLD_FORMAT = {1452533280 => {:metrics => {}, :slow_transactions => {}} } # Pre 1.2 agents used a different file format to store data. 
+end

data/test/unit/layaway_test.rb

@@ -0,0 +1,32 @@
+require 'test_helper'
+require 'scout_apm/slow_transaction'
+require 'scout_apm/metric_meta'
+require 'scout_apm/metric_stats'
+require 'scout_apm/context'
+require 'scout_apm/store'
+
+class LayawayTest < Minitest::Test
+  def test_add_reporting_period
+    File.open(DATA_FILE_PATH, 'w') { |file| file.write(Marshal.dump(NEW_FORMAT)) }
+    ScoutApm::Agent.instance.start
+
+    data = ScoutApm::Layaway.new
+    t = ScoutApm::StoreReportingPeriodTimestamp.new
+    data.add_reporting_period(t,ScoutApm::StoreReportingPeriod.new(t))
+    assert_equal [TIMESTAMP,t], Marshal.load(File.read(DATA_FILE_PATH)).keys
+  end
+
+  def test_merge_reporting_period
+    File.open(DATA_FILE_PATH, 'w') { |file| file.write(Marshal.dump(NEW_FORMAT)) }
+    ScoutApm::Agent.instance.start
+
+    data = ScoutApm::Layaway.new
+    t = ScoutApm::StoreReportingPeriodTimestamp.new
+    data.add_reporting_period(TIMESTAMP,ScoutApm::StoreReportingPeriod.new(TIMESTAMP))
+    assert_equal [TIMESTAMP], Marshal.load(File.read(DATA_FILE_PATH)).keys
+    # TODO - add tests to verify metrics+slow transactions are merged
+  end
+
+  TIMESTAMP = ScoutApm::StoreReportingPeriodTimestamp.new(Time.parse("2015-01-01"))
+  NEW_FORMAT = {TIMESTAMP => ScoutApm::StoreReportingPeriod.new(TIMESTAMP)} # Format for 1.2+ agents
+end

data/test/unit/serializers/payload_serializer_test.rb

@@ -49,7 +49,7 @@ class PayloadSerializerTest < Minitest::Test
     metrics = {
       ScoutApm::MetricMeta.new('ActiveRecord/all').tap { |meta|
         meta.desc = "SELECT * from users where filter=?"
-        meta.extra = {user: 'cooluser'}
+        meta.extra = {:user => 'cooluser'}
         meta.metric_id = nil
         meta.scope = "Controller/apps/checkin"
       } => ScoutApm::MetricStats.new.tap { |stats|
@@ -117,7 +117,7 @@ class PayloadSerializerTest < Minitest::Test
     slow_transaction_metrics = {
       ScoutApm::MetricMeta.new('ActiveRecord/all').tap { |meta|
         meta.desc = "SELECT * from users where filter=?"
-        meta.extra = {user: 'cooluser'}
+        meta.extra = {:user => 'cooluser'}
         meta.metric_id = nil
         meta.scope = "Controller/apps/checkin"
       } => ScoutApm::MetricStats.new.tap { |stats|

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: scout_apm
 version: !ruby/object:Gem::Version
-  version: 1.2.3
+  version: 1.2.4.pre
 platform: ruby
 authors:
 - Derek Haynes
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-01-07 00:00:00.000000000 Z
+date: 2016-01-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: minitest
@@ -144,9 +144,11 @@ files:
 - scout_apm.gemspec
 - test/data/config_test_1.yml
 - test/test_helper.rb
+- test/unit/agent_test.rb
 - test/unit/config_test.rb
 - test/unit/environment_test.rb
 - test/unit/instruments/active_record_instruments_test.rb
+- test/unit/layaway_test.rb
 - test/unit/serializers/payload_serializer_test.rb
 - test/unit/sql_sanitizer_test.rb
 homepage: https://github.com/scoutapp/scout_apm_ruby
@@ -164,9 +166,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
 rubyforge_project: scout_apm
 rubygems_version: 2.4.8
@@ -176,8 +178,10 @@ summary: Ruby application performance monitoring
 test_files:
 - test/data/config_test_1.yml
 - test/test_helper.rb
+- test/unit/agent_test.rb
 - test/unit/config_test.rb
 - test/unit/environment_test.rb
 - test/unit/instruments/active_record_instruments_test.rb
+- test/unit/layaway_test.rb
 - test/unit/serializers/payload_serializer_test.rb
 - test/unit/sql_sanitizer_test.rb