scout_apm 3.0.0.pre26 → 4.0.2

data/lib/scout_apm/utils/active_record_metric_name.rb

@@ -12,22 +12,27 @@ module ScoutApm
       # Converts an SQL string and the name (typically assigned automatically
       # by rails) into a Scout metric_name.
       #
+      # This prefers to use the ActiveRecord-provided name over parsing SQL as parsing is slower.
+      #
       # sql: SELECT "places".* FROM "places"  ORDER BY "places"."position" ASC
       # name: Place Load
       # metric_name: Place/find
       def to_s
+        return @to_s if @to_s
         parsed = parse_operation
         if parsed
-          "#{model}/#{parsed}"
+          @to_s = "#{model}/#{parsed}"
         else
-          regex_name(sql)
+          @to_s = regex_name(sql)
         end
       end
 
+      # This only returns a value if a name is provided via +initialize+.
       def model
         parts.first
       end
 
+      # This only returns a value if a name is provided via +initialize+.
       def normalized_operation
         parse_operation
       end
@@ -49,12 +54,14 @@ module ScoutApm
 
       private
 
+      # This only returns a value if a name is provided via +initialize+.
       def operation
         if parts.length >= 2
           parts[1].downcase
         end
       end
 
+      # This only returns a value if a name is provided via +initialize+.
       def parts
         name.split(" ")
       end
@@ -86,6 +93,8 @@ module ScoutApm
       NON_GREEDY_CONSUME = '.*?'
       TABLE = '(?:"|`)?(.*?)(?:"|`)?\s'
       COUNT = 'COUNT\(.*?\)'
+      BEGIN_STATEMENT = 'BEGIN'.freeze # BEGIN is a reserved keyword
+      COMMIT = 'COMMIT'.freeze
 
       SELECT_REGEX = /\A#{WHITE_SPACE}(SELECT)#{WHITE_SPACE}(#{COUNT})?#{NON_GREEDY_CONSUME}#{FROM}#{WHITE_SPACE}#{TABLE}/i.freeze
       UPDATE_REGEX = /\A#{WHITE_SPACE}(UPDATE)#{WHITE_SPACE}#{TABLE}/i.freeze
@@ -121,13 +130,17 @@ module ScoutApm
             else
               SELECT_LABEL
             end
-          "#{match[3].classify}/#{operation}"
+          "#{match[3].gsub(/\W/,'').classify}/#{operation}"
         elsif match = UPDATE_REGEX.match(sql)
           "#{match[2].classify}/#{UPDATE_LABEL}"
         elsif match = INSERT_REGEX.match(sql)
           "#{match[2].classify}/#{INSERT_LABEL}"
         elsif match = DELETE_REGEX.match(sql)
           "#{match[2].classify}/#{DELETE_LABEL}"
+        elsif sql == BEGIN_STATEMENT
+          "SQL/#{BEGIN_STATEMENT.downcase}"
+        elsif sql == COMMIT
+          "SQL/#{COMMIT.downcase}"
         else
           UNKNOWN_LABEL
         end

data/lib/scout_apm/utils/backtrace_parser.rb

@@ -11,6 +11,9 @@ module ScoutApm
 
       attr_reader :call_stack
 
+      # call_stack - an +Array+ of calls, typically generated via the +caller+ method. 
+      # Example single line: 
+      # "/Users/dlite/.rvm/rubies/ruby-2.4.5/lib/ruby/2.4.0/irb/workspace.rb:87:in `eval'"
       def initialize(call_stack, root=ScoutApm::Agent.instance.context.environment.root)
         @call_stack = call_stack
         # We can't use a constant as it'd be too early to fetch environment info

data/lib/scout_apm/utils/marshal_logging.rb

@@ -0,0 +1,90 @@
+module ScoutApm
+  module Utils
+    class Error < StandardError; end
+
+    class InstanceVar
+      attr_reader :name
+      attr_reader :obj
+
+      def initialize(name, obj, parent)
+        @name = name
+        @obj = obj
+        @parent = parent
+      end
+
+      def to_s
+        "#{@name} - #{obj.class}"
+      end
+
+      def history
+        (@parent.nil? ? [] : @parent.history) + [to_s]
+      end
+    end
+
+    class MarshalLogging
+      def initialize(base_obj)
+        @base_obj = base_obj
+      end
+
+      def dive
+        to_investigate = [InstanceVar.new('Root', @base_obj, nil)]
+        max_to_check = 10000
+        checked = 0
+
+        while (var = to_investigate.shift)
+          checked += 1 
+          if checked > max_to_check
+            return "Limiting Checks (max = #{max_to_check})"
+          end
+
+          obj = var.obj
+
+          if offending_hash?(obj)
+            return "Found undumpable object: #{var.history}"
+          end
+
+          if !dumps?(obj)
+            if obj.is_a? Hash
+              keys = obj.keys
+              keys.each do |key|
+                to_investigate.push(
+                  InstanceVar.new(key.to_s, obj[key], var)
+                )
+              end
+            elsif obj.is_a? Array
+              obj.each_with_index do |value, idx|
+                to_investigate.push(
+                  InstanceVar.new("Index #{idx}", value, var)
+                )
+              end
+            else
+              symbols = obj.instance_variables
+              if !symbols.any?
+                return "Found undumpable object: #{var.history}"
+              end
+
+              symbols.each do |sym|
+                to_investigate.push(
+                  InstanceVar.new(sym, obj.instance_variable_get(sym), var)
+                )
+              end
+            end
+          end
+        end
+
+        true
+      end
+
+      def dumps?(obj)
+        Marshal.dump(obj)
+        true
+      rescue TypeError
+        false
+      end
+
+      def offending_hash?(obj)
+        obj.is_a?(Hash) && !obj.default_proc.nil?
+      end
+    end
+  end
+end

data/lib/scout_apm/utils/sql_sanitizer.rb

@@ -34,15 +34,24 @@ module ScoutApm
         when :postgres then to_s_postgres
         when :mysql    then to_s_mysql
         when :sqlite   then to_s_sqlite
+        when :sqlserver then to_s_sqlserver
         end
       end
 
       private
 
+      def to_s_sqlserver
+        sql.gsub!(SQLSERVER_EXECUTESQL, '\1')
+        sql.gsub!(SQLSERVER_REMOVE_INTEGERS, '?')
+        sql.gsub!(SQLSERVER_IN_CLAUSE, 'IN (?)')
+        sql
+      end
+
       def to_s_postgres
         sql.gsub!(PSQL_PLACEHOLDER, '?')
         sql.gsub!(PSQL_VAR_INTERPOLATION, '')
-        sql.gsub!(PSQL_REMOVE_STRINGS, '?')
+        sql.gsub!(PSQL_AFTER_WHERE) {|c| c.gsub(PSQL_REMOVE_STRINGS, '?')}
+        sql.gsub!(PSQL_AFTER_SET) {|c| c.gsub(PSQL_REMOVE_STRINGS, '?')}
         sql.gsub!(PSQL_REMOVE_INTEGERS, '?')
         sql.gsub!(PSQL_IN_CLAUSE, 'IN (?)')
         sql.gsub!(MULTIPLE_SPACES, ' ')

data/lib/scout_apm/utils/sql_sanitizer_regex.rb

@@ -5,11 +5,13 @@ module ScoutApm
       MULTIPLE_SPACES    = %r|\s+|.freeze
       MULTIPLE_QUESTIONS = /\?(,\?)+/.freeze
 
-      PSQL_VAR_INTERPOLATION = %r|\[\[.*\]\]\s*$|.freeze
+      PSQL_VAR_INTERPOLATION = %r|\[\[.*\]\]\s*\z|.freeze
       PSQL_REMOVE_STRINGS = /'(?:[^']|'')*'/.freeze
       PSQL_REMOVE_INTEGERS = /(?<!LIMIT )\b\d+\b/.freeze
       PSQL_PLACEHOLDER = /\$\d+/.freeze
       PSQL_IN_CLAUSE = /IN\s+\(\?[^\)]*\)/.freeze
+      PSQL_AFTER_WHERE = /(?:WHERE\s+).*?(?:SELECT|\z)/im.freeze
+      PSQL_AFTER_SET = /(?:SET\s+).*?(?:WHERE|\z)/im.freeze
 
       MYSQL_VAR_INTERPOLATION = %r|\[\[.*\]\]\s*$|.freeze
       MYSQL_REMOVE_INTEGERS = /(?<!LIMIT )\b\d+\b/.freeze
@@ -20,6 +22,11 @@ module ScoutApm
       SQLITE_VAR_INTERPOLATION = %r|\[\[.*\]\]\s*$|.freeze
       SQLITE_REMOVE_STRINGS = /'(?:[^']|'')*'/.freeze
       SQLITE_REMOVE_INTEGERS = /(?<!LIMIT )\b\d+\b/.freeze
+
+      # => "EXEC sp_executesql N'SELECT  [users].* FROM [users] WHERE (age > 50)  ORDER BY [users].[id] ASC OFFSET 0 ROWS FETCH NEXT @0 ROWS ONLY', N'@0 int', @0 = 10"
+      SQLSERVER_EXECUTESQL = /EXEC sp_executesql N'(.*?)'.*/
+      SQLSERVER_REMOVE_INTEGERS = /(?<!LIMIT )\b(?<!@)\d+\b/.freeze
+      SQLSERVER_IN_CLAUSE = /IN\s+\(\?[^\)]*\)/.freeze
     end
   end
 end

data/lib/scout_apm/utils/sql_sanitizer_regex_1_8_7.rb

@@ -10,6 +10,8 @@ module ScoutApm
       PSQL_REMOVE_INTEGERS = /\b\d+\b/.freeze
       PSQL_PLACEHOLDER = /\$\d+/.freeze
       PSQL_IN_CLAUSE = /IN\s+\(\?[^\)]*\)/.freeze
+      PSQL_AFTER_WHERE = /(?:WHERE\s+).*?(?:SELECT|$)/i.freeze
+      PSQL_AFTER_SET = /(?:SET\s+).*?(?:WHERE|$)/i.freeze
 
       MYSQL_VAR_INTERPOLATION = %r|\[\[.*\]\]\s*$|.freeze
       MYSQL_REMOVE_INTEGERS = /\b\d+\b/.freeze
@@ -21,6 +23,10 @@ module ScoutApm
       SQLITE_REMOVE_STRINGS = /'(?:[^']|'')*'/.freeze
       SQLITE_REMOVE_INTEGERS = /\b\d+\b/.freeze
 
+      # This is not officially supported, but will do its best.
+      SQLSERVER_EXECUTESQL = /EXEC sp_executesql N'(.*?)'.*/
+      SQLSERVER_REMOVE_INTEGERS = /\b\d+\b/.freeze
+      SQLSERVER_IN_CLAUSE = /IN\s+\(\?[^\)]*\)/.freeze
     end
   end
 end

data/lib/scout_apm/utils/unique_id.rb

@@ -11,5 +11,32 @@ module ScoutApm
         s
       end
     end
+
+    # Represents a random ID that we can use to track a certain transaction.
+    # The `trans` prefix is only for ease of reading logs - it should not be
+    # interpreted to convey any sort of meaning.
+    class TransactionId
+      def initialize
+        @random = SecureRandom.hex(16)
+      end
+
+      def to_s
+        "trans-#{@random}"
+      end
+    end
+
+    # Represents a random ID that we can use to track a certain span. The
+    # `span` prefix is only for ease of reading logs - it should not be
+    # interpreted to convey any sort of meaning.
+    class SpanId
+      def initialize
+        @random = SecureRandom.hex(16)
+      end
+
+      def to_s
+        "span-#{@random}"
+      end
+    end
+
   end
 end

data/lib/scout_apm/version.rb

@@ -1,3 +1,3 @@
 module ScoutApm
-  VERSION = "3.0.0.pre26"
+  VERSION = "4.0.2"
 end

data/scout_apm.gemspec

@@ -19,9 +19,10 @@ Gem::Specification.new do |s|
   s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
   s.require_paths = ["lib","data"]
   s.extensions << 'ext/allocations/extconf.rb'
-  s.extensions << 'ext/stacks/extconf.rb'
   s.extensions << 'ext/rusage/extconf.rb'
 
+  s.required_ruby_version = '>= 2.1'
+
   s.add_development_dependency "minitest"
   s.add_development_dependency "mocha"
   s.add_development_dependency "pry"
@@ -29,11 +30,16 @@ Gem::Specification.new do |s|
   s.add_development_dependency "rake-compiler"
   s.add_development_dependency "addressable"
   s.add_development_dependency "activesupport"
+  s.add_runtime_dependency "parser"
+
+  # These are general development dependencies which are used in instrumentation
+  # tests. Specific versions are pulled in using specific gemfiles, e.g. 
+  # `gems/rails3.gemfile`.
+  s.add_development_dependency "activerecord"
+  s.add_development_dependency "sqlite3"
 
-  if RUBY_VERSION >= "1.9.3"
-    s.add_development_dependency "rubocop"
-    s.add_development_dependency "guard"
-    s.add_development_dependency "guard-minitest"
-    s.add_development_dependency "m"
-  end
+  s.add_development_dependency "rubocop"
+  s.add_development_dependency "guard"
+  s.add_development_dependency "guard-minitest"
+  s.add_development_dependency "m"
 end

data/test/test_helper.rb

@@ -5,7 +5,7 @@ SimpleCov.start
 require 'minitest/autorun'
 require 'minitest/unit'
 require 'minitest/pride'
-require 'mocha/mini_test'
+require 'mocha/minitest'
 require 'pry'
 
 
@@ -100,7 +100,7 @@ class Minitest::Test
   end
 
   def agent_context
-    ScoutApm::AgentContext.new
+    ScoutApm::Agent.instance.context
   end
 
   DATA_FILE_DIR = File.dirname(__FILE__) + '/tmp'

data/test/unit/agent_context_test.rb

@@ -0,0 +1,29 @@
+require "test_helper"
+
+require "scout_apm/agent_context"
+
+class AgentContextTest < Minitest::Test
+  def test_has_error_service_ignored_exceptions
+    context = ScoutApm::AgentContext.new
+    assert ScoutApm::ErrorService::IgnoredExceptions, context.ignored_exceptions.class
+  end
+
+  def test_has_error_buffer
+    context = ScoutApm::AgentContext.new
+    assert ScoutApm::ErrorService::ErrorBuffer, context.error_buffer.class
+  end
+
+
+  class TestPolicy
+    def call(req); 1; end
+    def stored!(req); end
+  end
+
+  def test_customize_slow_request_policy
+    context = ScoutApm::AgentContext.new
+    assert 4, context.slow_request_policy.policies
+
+    context.slow_request_policy.add(TestPolicy.new)
+    assert 5, context.slow_request_policy.policies
+  end
+end

data/test/unit/auto_instrument/assignments-instrumented.rb

@@ -0,0 +1,31 @@
+
+class Assignments
+  def test_op_asgn
+    foo.bar += ::ScoutApm::AutoInstrument("User.size",["ROOT/test/unit/auto_instrument/assignments.rb:4:in `test_op_asgn'"]){User.size}
+    foo.bar -= ::ScoutApm::AutoInstrument("User.size",["ROOT/test/unit/auto_instrument/assignments.rb:5:in `test_op_asgn'"]){User.size}
+  end
+
+  def nested_assignment
+    @email ||= if (email = ::ScoutApm::AutoInstrument("session[\"email\"]",["ROOT/test/unit/auto_instrument/assignments.rb:9:in `nested_assignment'"]){session["email"]}).present?
+        ::ScoutApm::AutoInstrument("User.where(email: email).first",["ROOT/test/unit/auto_instrument/assignments.rb:10:in `nested_assignment'"]){User.where(email: email).first}
+      else
+        nil
+      end
+  end
+
+  def paginate_collection(coll)
+    page = (::ScoutApm::AutoInstrument("params[:page].present?",["ROOT/test/unit/auto_instrument/assignments.rb:17:in `paginate_collection'"]){params[:page].present?} ? ::ScoutApm::AutoInstrument("params[:page].to_i",["ROOT/test/unit/auto_instrument/assignments.rb:17:in `paginate_collection'"]){params[:page].to_i} : 1)
+    per_page = (::ScoutApm::AutoInstrument("params[:per_page].present?",["ROOT/test/unit/auto_instrument/assignments.rb:18:in `paginate_collection'"]){params[:per_page].present?} ? ::ScoutApm::AutoInstrument("params[:per_page].to_i",["ROOT/test/unit/auto_instrument/assignments.rb:18:in `paginate_collection'"]){params[:per_page].to_i} : 20)
+    pagination, self.collection = ::ScoutApm::AutoInstrument("pagy(...",["ROOT/test/unit/auto_instrument/assignments.rb:19:in `paginate_collection'"]){pagy(
+      coll,
+      items: per_page,
+      page: page
+    )}
+    ::ScoutApm::AutoInstrument("headers[PAGINATION_TOTAL_HEADER] = pagination.count.to_s",["ROOT/test/unit/auto_instrument/assignments.rb:24:in `paginate_collection'"]){headers[PAGINATION_TOTAL_HEADER] = pagination.count.to_s}
+    ::ScoutApm::AutoInstrument("headers[PAGINATION_TOTAL_PAGES_HEADER] = pagination.pages.to_s",["ROOT/test/unit/auto_instrument/assignments.rb:25:in `paginate_collection'"]){headers[PAGINATION_TOTAL_PAGES_HEADER] = pagination.pages.to_s}
+    ::ScoutApm::AutoInstrument("headers[PAGINATION_PER_PAGE_HEADER] = per_page.to_s",["ROOT/test/unit/auto_instrument/assignments.rb:26:in `paginate_collection'"]){headers[PAGINATION_PER_PAGE_HEADER] = per_page.to_s}
+    ::ScoutApm::AutoInstrument("headers[PAGINATION_PAGE_HEADER] = pagination.page.to_s",["ROOT/test/unit/auto_instrument/assignments.rb:27:in `paginate_collection'"]){headers[PAGINATION_PAGE_HEADER] = pagination.page.to_s}
+    ::ScoutApm::AutoInstrument("headers[PAGINATION_NEXT_PAGE_HEADER] = pagination.next.to_s",["ROOT/test/unit/auto_instrument/assignments.rb:28:in `paginate_collection'"]){headers[PAGINATION_NEXT_PAGE_HEADER] = pagination.next.to_s}
+    ::ScoutApm::AutoInstrument("collection",["ROOT/test/unit/auto_instrument/assignments.rb:29:in `paginate_collection'"]){collection}
+  end
+end

data/test/unit/auto_instrument/assignments.rb

@@ -0,0 +1,31 @@
+
+class Assignments
+  def test_op_asgn
+    foo.bar += User.size
+    foo.bar -= User.size
+  end
+
+  def nested_assignment
+    @email ||= if (email = session["email"]).present?
+        User.where(email: email).first
+      else
+        nil
+      end
+  end
+
+  def paginate_collection(coll)
+    page = (params[:page].present? ? params[:page].to_i : 1)
+    per_page = (params[:per_page].present? ? params[:per_page].to_i : 20)
+    pagination, self.collection = pagy(
+      coll,
+      items: per_page,
+      page: page
+    )
+    headers[PAGINATION_TOTAL_HEADER] = pagination.count.to_s
+    headers[PAGINATION_TOTAL_PAGES_HEADER] = pagination.pages.to_s
+    headers[PAGINATION_PER_PAGE_HEADER] = per_page.to_s
+    headers[PAGINATION_PAGE_HEADER] = pagination.page.to_s
+    headers[PAGINATION_NEXT_PAGE_HEADER] = pagination.next.to_s
+    collection
+  end
+end

data/test/unit/auto_instrument/controller-ast.txt

@@ -0,0 +1,57 @@
+s(:class,
+  s(:const, nil, :ClientsController),
+  s(:const, nil, :ApplicationController),
+  s(:begin,
+    s(:send, nil, :before_action,
+      s(:sym, :check_authorization)),
+    s(:def, :index,
+      s(:args),
+      s(:if,
+        s(:send,
+          s(:send,
+            s(:send, nil, :params), :[],
+            s(:sym, :status)), :==,
+          s(:str, "activated")),
+        s(:ivasgn, :@clients,
+          s(:send,
+            s(:const, nil, :Client), :activated)),
+        s(:ivasgn, :@clients,
+          s(:send,
+            s(:const, nil, :Client), :inactivated)))),
+    s(:def, :create,
+      s(:args),
+      s(:begin,
+        s(:ivasgn, :@client,
+          s(:send,
+            s(:const, nil, :Client), :new,
+            s(:send,
+              s(:send, nil, :params), :[],
+              s(:sym, :client)))),
+        s(:if,
+          s(:send,
+            s(:ivar, :@client), :save),
+          s(:send, nil, :redirect_to,
+            s(:ivar, :@client)),
+          s(:send, nil, :render,
+            s(:str, "new"))))),
+    s(:def, :edit,
+      s(:args),
+      s(:begin,
+        s(:ivasgn, :@client,
+          s(:send,
+            s(:const, nil, :Client), :new,
+            s(:send,
+              s(:send, nil, :params), :[],
+              s(:sym, :client)))),
+        s(:if,
+          s(:send,
+            s(:send, nil, :request), :post?),
+          s(:block,
+            s(:send,
+              s(:ivar, :@client), :transaction),
+            s(:args),
+            s(:send,
+              s(:ivar, :@client), :update_attributes,
+              s(:send,
+                s(:send, nil, :params), :[],
+                s(:sym, :client)))), nil)))))