scout_apm 5.0.0 → 5.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f9a94a38d6b06a02e34ce7923e8084976739cc8d0fba1adfbc606f88b0bdf4d7
-  data.tar.gz: aebe73f5379911e8691be2771b3a3fc3f5fbdd4779f2038f6a6f6e0ed38e1e20
+  metadata.gz: 81ad3b683899407ed049b7267d6d79a84d5afac011445d1078f4342c345c7ace
+  data.tar.gz: 29e141854923fa21c57ca3fcc7ca9e4cc96dbd741c36c0586e7fd2cd7a8186f8
 SHA512:
-  metadata.gz: d8850b1365747879effcfe383d0b438070069d2b625365dc8ea61db7c72cf6840cdf73675c95aea7a7688750dbcb41efec1b7320232312172bc49be9b207d421
-  data.tar.gz: f3d389aa73f8b4090627843410c610edc1c5fa88740587136dbd8277d3ddfa5360e61d0d256ae23db7cf53f20b482d3f283541c077efcd2d12032e5f3f72f4e2
+  metadata.gz: a857678f6d91c709fbbdfb30409339f37e99cd83260cec9d3808f81638245de03cf4b0721270203f0ee619cc56b06fc3baa795a10e859c3068e0e2c36a2247b2
+  data.tar.gz: 972da0997d251c5704291e60e390916a417de3036aadcada469540acd20520f4edd95b3b76da9202734755bc37c3a9a22e44916c23e74c9199d3c6a7a2447797

data/.github/workflows/test.yml

@@ -36,6 +36,9 @@ jobs:
             bundler: 1.17.3
           - ruby: 2.7
           - ruby: 3.0
+          - ruby: 3.0
+            gemfile: gems/sidekiq.gemfile
+            test_features: "sidekiq_install"
 
     env:
       BUNDLE_GEMFILE: ${{ matrix.gemfile }}

data/CHANGELOG.markdown

@@ -1,3 +1,27 @@
+# Unreleased
+
+# 5.2.0
+
+* Use Sidekiq lifecycle hooks to start Scout agent on Sidekiq start. (#449)
+
+# 5.1.1
+
+* Improvements to SqlServer scrubbing in SqlSanitizer (#422)
+
+# 5.1.0
+
+* Specify correct (MIT) license in Gemspec (#430)
+* Install HTTP::Client instruments (#420)
+* Sanitize FROM jsonb_as_recordset AS correctly in Postgres (#332)
+* Call to_h on `ActiveRecord::Base.configurations` (#434)
+* Allow loading of trusted `config/scout_apm.yml` via `YAML.unsafe_load` if available (#435)
+* Better exception handling when loading config (#436)
+* Check for nil other_metric_set in merge_external_service_metrics (#437)
+* Log `warn` in InstructionSequence only if SCOUT_LOG_LEVEL is debug (#438)
+* Check for Parser::TreeRewriter before loading AutoInstruments to avoid LoadError (#440)
+* Fall back to STDERR upon exception in build_logger (#441)
+
+
 # 5.0.0
 
 * Add External Service metrics reporting (#403)

data/gems/sidekiq.gemfile

@@ -0,0 +1,4 @@
+eval_gemfile("../Gemfile")
+
+# https://github.com/scoutapp/scout_apm_ruby/issues/449
+gem 'sidekiq', '>= 6.5.0'

data/lib/scout_apm/auto_instrument/instruction_sequence.rb

@@ -10,10 +10,10 @@ module ScoutApm
             new_code = Rails.rewrite(path)
             return self.compile(new_code, path, path)
           rescue
-            warn "Failed to apply auto-instrumentation to #{path}: #{$!}"
+            warn "Failed to apply auto-instrumentation to #{path}: #{$!}" if ENV['SCOUT_LOG_LEVEL'].to_s.downcase == "debug"
           end
         elsif Rails.ignore?(path)
-          warn "AutoInstruments are ignored for path=#{path}."
+          warn "AutoInstruments are ignored for path=#{path}." if ENV['SCOUT_LOG_LEVEL'].to_s.downcase == "debug"
         end
 
         return self.compile_file(path)

data/lib/scout_apm/background_job_integrations/sidekiq.rb

@@ -37,17 +37,11 @@ module ScoutApm
       end
 
       def install_processor
-        require 'sidekiq/processor' # sidekiq v4 has not loaded this file by this point
-
-        ::Sidekiq::Processor.class_eval do
-          def initialize_with_scout(*args)
+        ::Sidekiq.configure_server do |config|
+          config.on(:startup) do
             agent = ::ScoutApm::Agent.instance
             agent.start
-            initialize_without_scout(*args)
           end
-
-          alias_method :initialize_without_scout, :initialize
-          alias_method :initialize, :initialize_with_scout
         end
       end
     end

data/lib/scout_apm/config.rb

@@ -432,7 +432,7 @@ module ScoutApm
         begin
           raw_file = File.read(@resolved_file_path)
           erb_file = ERB.new(raw_file).result(binding)
-          parsed_yaml = YAML.load(erb_file)
+          parsed_yaml = YAML.respond_to?(:unsafe_load) ? YAML.unsafe_load(erb_file) : YAML.load(erb_file)
           file_settings = parsed_yaml[app_environment]
 
           if file_settings.is_a? Hash
@@ -443,8 +443,8 @@ module ScoutApm
             logger.info("Couldn't find configuration in #{@resolved_file_path} for environment: #{app_environment}. Configuration in ENV will still be applied.")
             @file_loaded = false
           end
-        rescue Exception => e # Explicit `Exception` handling to catch SyntaxError and anything else that ERB or YAML may throw
-          logger.info("Failed loading configuration file (#{@resolved_file_path}): #{e.message}. ScoutAPM will continue starting with configuration from ENV and defaults")
+        rescue ScoutApm::AllExceptionsExceptOnesWeMustNotRescue => e # Everything except the most important exceptions we should never interfere with
+          logger.info("Failed loading configuration file (#{@resolved_file_path}): ScoutAPM will continue starting with configuration from ENV and defaults. Exception was #{e.class}: #{e.message}#{e.backtrace.map { |bt| "\n  #{bt}" }.join('')}")
           @file_loaded = false
         end
       end

data/lib/scout_apm/exceptions.rb

@@ -0,0 +1,12 @@
+module ScoutApm
+  module AllExceptionsExceptOnesWeMustNotRescue
+    # Borrowed from https://github.com/rspec/rspec-support/blob/v3.8.0/lib/rspec/support.rb#L132-L140
+    # These exceptions are dangerous to rescue as rescuing them
+    # would interfere with things we should not interfere with.
+    AVOID_RESCUING = [NoMemoryError, SignalException, Interrupt, SystemExit]
+
+    def self.===(exception)
+      AVOID_RESCUING.none? { |ar| ar === exception }
+    end
+  end
+end

data/lib/scout_apm/framework_integrations/rails_3_or_4.rb

@@ -78,7 +78,7 @@ module ScoutApm
         end
 
         if adapter.nil?
-          adapter = ActiveRecord::Base.configurations[env]["adapter"]
+          adapter = ActiveRecord::Base.configurations.to_h[env]["adapter"]
         end
 
         return adapter

data/lib/scout_apm/instrument_manager.rb

@@ -32,6 +32,7 @@ module ScoutApm
       install_instrument(ScoutApm::Instruments::NetHttp)
       install_instrument(ScoutApm::Instruments::Typhoeus)
       install_instrument(ScoutApm::Instruments::HttpClient)
+      install_instrument(ScoutApm::Instruments::HTTP)
       install_instrument(ScoutApm::Instruments::Memcached)
       install_instrument(ScoutApm::Instruments::Redis)
       install_instrument(ScoutApm::Instruments::InfluxDB)

data/lib/scout_apm/logger.rb

@@ -69,7 +69,11 @@ module ScoutApm
     private
 
     def build_logger
-      logger_class.new(@log_destination) rescue logger_class.new
+      logger_class.new(@log_destination)
+    rescue => e
+      logger = ::Logger.new(STDERR)
+      logger.error("Error while building ScoutApm logger: #{e.message}. Falling back to STDERR")
+      logger
     end
 
     def logger_class

data/lib/scout_apm/store.rb

@@ -217,6 +217,7 @@ module ScoutApm
 
     def initialize(timestamp, context)
       @timestamp = timestamp
+      @context = context
 
       @request_traces = ScoredItemSet.new(context.config.value('max_traces'))
       @job_traces = ScoredItemSet.new(context.config.value('max_traces'))
@@ -230,6 +231,11 @@ module ScoutApm
       @jobs = Hash.new
     end
 
+    def logger
+      @context.logger
+    end
+    private :logger
+
     # Merges another StoreReportingPeriod into this one
     def merge(other)
       self.
@@ -266,7 +272,11 @@ module ScoutApm
     end
 
     def merge_external_service_metrics!(other_metric_set)
-      external_service_metric_set.combine!(other_metric_set)
+      if other_metric_set.nil?
+        logger.debug("Missing other_metric_set for merge_external_service_metrics - skipping.")
+      else
+        external_service_metric_set.combine!(other_metric_set)
+      end
       self
     end
 

data/lib/scout_apm/utils/sql_sanitizer.rb

@@ -12,11 +12,13 @@ module ScoutApm
 
       PSQL_VAR_INTERPOLATION = %r|\[\[.*\]\]\s*\z|.freeze
       PSQL_REMOVE_STRINGS = /'(?:[^']|'')*'/.freeze
+      PSQL_REMOVE_JSON_STRINGS = /:"(?:[^"]|"")*"/.freeze
       PSQL_REMOVE_INTEGERS = /(?<!LIMIT )\b\d+\b/.freeze
       PSQL_AFTER_SELECT = /(?:SELECT\s+).*?(?:WHERE|FROM\z)/im.freeze # Should be everything between a FROM and a WHERE
       PSQL_PLACEHOLDER = /\$\d+/.freeze
       PSQL_IN_CLAUSE = /IN\s+\(\?[^\)]*\)/.freeze
       PSQL_AFTER_FROM = /(?:FROM\s+).*?(?:WHERE|\z)/im.freeze # Should be everything between a FROM and a WHERE
+      PSQL_AFTER_FROM_AS = /(?:FROM\s+).*?(?:AS|\z)/im.freeze # Should be everything between a FROM and AS without WHERE
       PSQL_AFTER_JOIN = /(?:JOIN\s+).*?\z/im.freeze
       PSQL_AFTER_WHERE = /(?:WHERE\s+).*?(?:SELECT|\z)/im.freeze
       PSQL_AFTER_SET = /(?:SET\s+).*?(?:WHERE|\z)/im.freeze
@@ -32,7 +34,8 @@ module ScoutApm
       SQLITE_REMOVE_INTEGERS = /(?<!LIMIT )\b\d+\b/.freeze
 
       # => "EXEC sp_executesql N'SELECT  [users].* FROM [users] WHERE (age > 50)  ORDER BY [users].[id] ASC OFFSET 0 ROWS FETCH NEXT @0 ROWS ONLY', N'@0 int', @0 = 10"
-      SQLSERVER_EXECUTESQL = /EXEC sp_executesql N'(.*?)'.*/
+      SQLSERVER_REMOVE_EXECUTESQL = /EXEC sp_executesql (N')?/.freeze
+      SQLSERVER_REMOVE_STRINGS = /'(?:[^']|'')*'/.freeze
       SQLSERVER_REMOVE_INTEGERS = /(?<!LIMIT )\b(?<!@)\d+\b/.freeze
       SQLSERVER_IN_CLAUSE = /IN\s+\(\?[^\)]*\)/.freeze
 
@@ -65,7 +68,8 @@ module ScoutApm
       private
 
       def to_s_sqlserver
-        sql.gsub!(SQLSERVER_EXECUTESQL, '\1')
+        sql.gsub!(SQLSERVER_REMOVE_EXECUTESQL, '')
+        sql.gsub!(SQLSERVER_REMOVE_STRINGS, '?')
         sql.gsub!(SQLSERVER_REMOVE_INTEGERS, '?')
         sql.gsub!(SQLSERVER_IN_CLAUSE, 'IN (?)')
         sql
@@ -76,6 +80,7 @@ module ScoutApm
         sql.gsub!(PSQL_VAR_INTERPOLATION, '')
         # sql.gsub!(PSQL_REMOVE_STRINGS, '?')
         sql.gsub!(PSQL_AFTER_WHERE) {|c| c.gsub(PSQL_REMOVE_STRINGS, '?')}
+        sql.gsub!(PSQL_AFTER_FROM_AS) {|c| c.gsub(PSQL_REMOVE_JSON_STRINGS, ':"?"')}
         sql.gsub!(PSQL_AFTER_JOIN) {|c| c.gsub(PSQL_REMOVE_STRINGS, '?')}
         sql.gsub!(PSQL_AFTER_SET) {|c| c.gsub(PSQL_REMOVE_STRINGS, '?')}
         sql.gsub!(PSQL_REMOVE_INTEGERS, '?')

data/lib/scout_apm/version.rb

@@ -1,3 +1,3 @@
 module ScoutApm
-  VERSION = "5.0.0"
+  VERSION = "5.2.0"
 end

data/lib/scout_apm.rb

@@ -27,6 +27,7 @@ require 'rusage'
 #####################################
 require 'scout_apm/version'
 
+require 'scout_apm/exceptions'
 require 'scout_apm/debug'
 require 'scout_apm/tracked_request'
 require 'scout_apm/layer'
@@ -80,6 +81,7 @@ require 'scout_apm/histogram'
 
 require 'scout_apm/instruments/net_http'
 require 'scout_apm/instruments/http_client'
+require 'scout_apm/instruments/http'
 require 'scout_apm/instruments/typhoeus'
 require 'scout_apm/instruments/moped'
 require 'scout_apm/instruments/mongoid'
@@ -221,8 +223,12 @@ if defined?(Rails) && defined?(Rails::VERSION) && defined?(Rails::VERSION::MAJOR
         ScoutApm::Agent.instance.install
 
         if ScoutApm::Agent.instance.context.config.value("auto_instruments")
-          ScoutApm::Agent.instance.context.logger.debug("AutoInstruments is enabled.")
-          require 'scout_apm/auto_instrument'
+          if defined?(Parser::TreeRewriter)
+            ScoutApm::Agent.instance.context.logger.debug("AutoInstruments is enabled.")
+            require 'scout_apm/auto_instrument'
+          else # AutoInstruments is turned on, but we don't he the prerequisites to use it
+            ScoutApm::Agent.instance.context.logger.debug("AutoInstruments is enabled, but Parser::TreeRewriter is missing. Update 'parser' gem to >= 2.5.0.")
+          end
         else
           ScoutApm::Agent.instance.context.logger.debug("AutoInstruments is disabled.")
         end

data/scout_apm.gemspec

@@ -10,7 +10,7 @@ Gem::Specification.new do |s|
   s.homepage    = "https://github.com/scoutapp/scout_apm_ruby"
   s.summary     = "Ruby application performance monitoring"
   s.description = "Monitors Ruby apps and reports detailed metrics on performance to Scout."
-  s.license     = "Proprietary (See LICENSE.md)"
+  s.license     = "MIT"
 
   s.files         = `git ls-files`.split("\n")
   s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")

data/test/unit/background_job_integrations/sidekiq_test.rb

@@ -3,8 +3,25 @@ require 'scout_apm/request_manager'
 require 'scout_apm/background_job_integrations/sidekiq'
 
 class SidekiqTest < Minitest::Test
+  SidekiqIntegration = ScoutApm::BackgroundJobIntegrations::Sidekiq
   SidekiqMiddleware = ScoutApm::BackgroundJobIntegrations::SidekiqMiddleware
 
+  ########################################
+  # Install
+  ########################################
+  if (ENV["SCOUT_TEST_FEATURES"] || "").include?("sidekiq_install")
+    require 'sidekiq'
+
+    # Sidekiq::CLI needs to be defined in order for `Sidekiq.configure_server` to work
+    Sidekiq::CLI = nil
+
+    def test_starts_on_startup
+      ::ScoutApm::Agent.any_instance.expects(:start)
+      SidekiqIntegration.new.install
+      Sidekiq.options[:lifecycle_events][:startup].map(&:call)
+    end
+  end
+
   ########################################
   # Middleware
   ########################################

data/test/unit/sql_sanitizer_test.rb

@@ -123,27 +123,27 @@ module ScoutApm
       end
 
       def test_sqlserver_integers
-        skip "SQLServer Support requires Ruby 1.9+ For Regexes"
-
         sql = "EXEC sp_executesql N'SELECT  [users].* FROM [users] WHERE (age > 50)  ORDER BY [users].[id] ASC OFFSET 0 ROWS FETCH NEXT @0 ROWS ONLY', N'@0 int', @0 = 10"
         ss = SqlSanitizer.new(sql).tap{ |it| it.database_engine = :sqlserver }
-        assert_equal %q|SELECT  [users].* FROM [users] WHERE (age > ?)  ORDER BY [users].[id] ASC OFFSET ? ROWS FETCH NEXT @0 ROWS ONLY|, ss.to_s
+        assert_equal "SELECT  [users].* FROM [users] WHERE (age > ?)  ORDER BY [users].[id] ASC OFFSET ? ROWS FETCH NEXT @0 ROWS ONLY?@0 int', @0 = ?", ss.to_s
       end
 
       def test_sqlserver_strings
-        skip "SQLServer Support requires Ruby 1.9+ For Regexes"
+        sql = "EXEC sp_executesql N'SELECT  [users].* FROM [users] WHERE first_name = N'john' AND last_name = N'doe' ORDER BY [users].[id] ASC OFFSET 0 ROWS FETCH NEXT @0 ROWS ONLY', N'@0 int', @0 = 10"
+        ss = SqlSanitizer.new(sql).tap{ |it| it.database_engine = :sqlserver }
+        assert_equal "SELECT  [users].* FROM [users] WHERE first_name = N? AND last_name = N? ORDER BY [users].[id] ASC OFFSET ? ROWS FETCH NEXT @0 ROWS ONLY?@0 int', @0 = ?", ss.to_s
+      end
 
-        sql = "EXEC sp_executesql N'SELECT  [users].* FROM [users] WHERE [users].[email] = @0  ORDER BY [users].[id] ASC OFFSET 0 ROWS FETCH NEXT @1 ROWS ONLY', N'@0 nvarchar(4000), @1 int', @0 = N'foo', @1 = 10"
+      def test_sqlserver_strings_no_executesql
+        sql = "EXEC Authenticate @username = N'abraham.lincoln', @password = N'somepassword!', @token = NULL, @app_name = N'Central Auth Service', @log_login = true, @ip_address = N'127.0.0.1', @external_type = NULL, @external_success = NULL"
         ss = SqlSanitizer.new(sql).tap{ |it| it.database_engine = :sqlserver }
-        assert_equal %q|SELECT  [users].* FROM [users] WHERE [users].[email] = @0  ORDER BY [users].[id] ASC OFFSET ? ROWS FETCH NEXT @1 ROWS ONLY|, ss.to_s
+        assert_equal "EXEC Authenticate @username = N?, @password = N?, @token = NULL, @app_name = N?, @log_login = true, @ip_address = N?, @external_type = NULL, @external_success = NULL", ss.to_s
       end
 
       def test_sqlserver_in_clause
-        skip "SQLServer Support requires Ruby 1.9+ For Regexes"
-
         sql = "EXEC sp_executesql N'SELECT  [users].* FROM [users] WHERE (id IN (1,2,3))  ORDER BY [users].[id] ASC OFFSET 0 ROWS FETCH NEXT @0 ROWS ONLY', N'@0 int', @0 = 10"
         ss = SqlSanitizer.new(sql).tap{ |it| it.database_engine = :sqlserver }
-        assert_equal %q|SELECT  [users].* FROM [users] WHERE (id IN (?))  ORDER BY [users].[id] ASC OFFSET ? ROWS FETCH NEXT @0 ROWS ONLY|, ss.to_s
+        assert_equal "SELECT  [users].* FROM [users] WHERE (id IN (?))  ORDER BY [users].[id] ASC OFFSET ? ROWS FETCH NEXT @0 ROWS ONLY?@0 int', @0 = ?", ss.to_s
       end
 
       def test_scrubs_invalid_encoding
@@ -194,6 +194,17 @@ module ScoutApm
         WHERE (title = ?)|, ss.to_s
       end
 
+      def test_postgres_insert_select_from_jsonb_to_recordset_with_as
+        sql = %q|
+        INSERT INTO foos(foo_id, bar_id, external_id, email_address, created_at, updated_at)
+          SELECT 123, 456, external_id, email_address, NOW(), NOW()
+          FROM jsonb_to_recordset($${"items":[{"external_id":1234,"email_address":"test@domain.com"}]}$$::jsonb->'items')
+            AS t(external_id integer, email_address varchar)
+        |
+        ss = SqlSanitizer.new(sql).tap{ |it| it.database_engine = :postgres }
+        assert_equal %q|INSERT INTO foos(foo_id, bar_id, external_id, email_address, created_at, updated_at) SELECT ?, ?, external_id, email_address, NOW(), NOW() FROM jsonb_to_recordset($${"items":[{"external_id":?,"email_address":"?"}]}$$::jsonb->'items') AS t(external_id integer, email_address varchar)|, ss.to_s
+      end
+
       def assert_faster_than(target_seconds)
         t1 = ::Time.now
         yield

metadata

@@ -1,15 +1,15 @@
 --- !ruby/object:Gem::Specification
 name: scout_apm
 version: !ruby/object:Gem::Version
-  version: 5.0.0
+  version: 5.2.0
 platform: ruby
 authors:
 - Derek Haynes
 - Andre Lewis
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-11-02 00:00:00.000000000 Z
+date: 2022-06-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: minitest
@@ -237,6 +237,7 @@ files:
 - gems/rails4.gemfile
 - gems/rails5.gemfile
 - gems/rails6.gemfile
+- gems/sidekiq.gemfile
 - gems/typhoeus.gemfile
 - lib/scout_apm.rb
 - lib/scout_apm/agent.rb
@@ -280,6 +281,7 @@ files:
 - lib/scout_apm/error_service/periodic_work.rb
 - lib/scout_apm/error_service/railtie.rb
 - lib/scout_apm/error_service/sidekiq.rb
+- lib/scout_apm/exceptions.rb
 - lib/scout_apm/extensions/config.rb
 - lib/scout_apm/extensions/transaction_callback_payload.rb
 - lib/scout_apm/external_service_metric_set.rb
@@ -469,9 +471,9 @@ files:
 - test/unit/utils/scm.rb
 homepage: https://github.com/scoutapp/scout_apm_ruby
 licenses:
-- Proprietary (See LICENSE.md)
+- MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -487,8 +489,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
-signing_key: 
+rubygems_version: 3.3.7
+signing_key:
 specification_version: 4
 summary: Ruby application performance monitoring
 test_files: []