scout_apm 2.3.2 → 2.3.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 499e2fb1ca29ef0ca1a596acf1ef561efc9b8f05
-  data.tar.gz: 4ac01fb2789e05be922d6121eabae98b72af4198
+  metadata.gz: b48d3a3bd3b4f29128855e9cecbe5e98869704f2
+  data.tar.gz: 5a1e47f348ad25195f04d7250fc232fae4a082b6
 SHA512:
-  metadata.gz: 553499a35ca3b92571b649277feda1d2e195ccf091eefb85f2e0ca5279a36c3e8ea90a0a0073104ee9db398a2c1984d87ace59e706e76b8dda8b02e39ed67f1c
-  data.tar.gz: db4cc309f2c46c2f4a77f84c7f1c1ab66b7488f08d87310f1c18daa3ba62b0554a170c501cd407050cb89a52417b255c0269b70274c26e64d6ddc9def35c7a70
+  metadata.gz: fdee3999614a9fbcc7c45087f55cd76176708ec3b162346143524cc5edd4feb44cfb692cd5c04e3f6d8d1f7b1b186083585d5dc833399a7812c4eeb4083b097b
+  data.tar.gz: b3cbeaf5c677468b2205dddc87b35c5e7a949a81c58874493033922624dc7a7d218ed8317c221b0b8dbd39258d03d21a2589252dd3737060d04998e32995d167

data/CHANGELOG.markdown

@@ -1,3 +1,8 @@
+# 2.3.3
+
+* Capture ActiveRecord calls that generate more complex queries
+* More aggressively determine names of complex queries (to determine "User/find", "Account/create" and similar)
+
 # 2.3.2
 
 * More robust startup sequence when using `rails server` vs. directly launching an app server

data/lib/scout_apm/instruments/active_record.rb

@@ -1,6 +1,38 @@
 require 'scout_apm/utils/sql_sanitizer'
 
 module ScoutApm
+  class SqlList
+    attr_reader :sqls
+
+    def initialize(sql=nil)
+      @sqls = []
+
+      if !sql.nil?
+        push(sql)
+      end
+    end
+
+    def <<(sql)
+      push(sql)
+    end
+
+    def push(sql)
+      if !(Utils::SqlSanitizer === sql)
+        sql = Utils::SqlSanitizer.new(sql)
+      end
+      @sqls << sql
+    end
+
+    # All of this one, then all of the other.
+    def merge(other)
+      @sqls += other.sqls
+    end
+
+    def to_s
+      @sqls.map{|s| s.to_s }.join(";\n")
+    end
+  end
+
   module Instruments
     class ActiveRecord
       attr_reader :logger
@@ -115,13 +147,12 @@ module ScoutApm
         # Extract data from the arguments
         sql, name = args
         metric_name = Utils::ActiveRecordMetricName.new(sql, name)
-        desc = Utils::SqlSanitizer.new(sql)
+        desc = SqlList.new(sql)
 
         # Get current ScoutApm context
         req = ScoutApm::RequestManager.lookup
         current_layer = req.current_layer
 
-
         # If we call #log, we have a real query to run, and we've already
         # gotten through the cache gatekeeper. Since we want to only trace real
         # queries, and not repeated identical queries that just hit cache, we
@@ -136,9 +167,13 @@ module ScoutApm
           # TODO: Get rid of call .to_s, need to find this without forcing a previous run of the name logic
           if current_layer.name.to_s == Utils::ActiveRecordMetricName::DEFAULT_METRIC
             current_layer.name = metric_name
-            current_layer.desc = desc
           end
 
+          if current_layer.desc.nil?
+            current_layer.desc = SqlList.new
+          end
+          current_layer.desc.merge(desc)
+
           log_without_scout_instruments(*args, &block)
 
         # OR: Start a new layer, we didn't pick up instrumentation earlier in the stack.
@@ -215,6 +250,7 @@ module ScoutApm
         req = ScoutApm::RequestManager.lookup
         layer = ScoutApm::Layer.new("ActiveRecord", Utils::ActiveRecordMetricName::DEFAULT_METRIC)
         layer.annotate_layer(:ignorable => true)
+        layer.desc = SqlList.new
         req.start_layer(layer)
         req.ignore_children!
         begin
@@ -233,6 +269,7 @@ module ScoutApm
 
         req = ScoutApm::RequestManager.lookup
         layer = ScoutApm::Layer.new("ActiveRecord", Utils::ActiveRecordMetricName.new("", "#{model} #{operation}"))
+        layer.desc = SqlList.new
         req.start_layer(layer)
         req.ignore_children!
         begin

data/lib/scout_apm/layer_converters/converter_base.rb

@@ -143,7 +143,7 @@ module ScoutApm
         end
       end
 
-      def make_meta_options_desc_hash(layer, max_desc_length=1000)
+      def make_meta_options_desc_hash(layer, max_desc_length=32768)
         if layer.desc
           desc_s = layer.desc.to_s
           trimmed_desc = desc_s[0 .. max_desc_length]

data/lib/scout_apm/utils/active_record_metric_name.rb

@@ -1,12 +1,11 @@
 module ScoutApm
   module Utils
     class ActiveRecordMetricName
-      DEFAULT_METRIC = "SQL/Unknown"
-
       attr_reader :sql, :name
+      DEFAULT_METRIC = 'SQL/other'.freeze
 
       def initialize(sql, name)
-        @sql = sql
+        @sql = sql || ""
         @name = name.to_s
       end
 
@@ -17,13 +16,11 @@ module ScoutApm
       # name: Place Load
       # metric_name: Place/find
       def to_s
-        return DEFAULT_METRIC unless name
-        return DEFAULT_METRIC unless model && operation
-
-        if parsed = parse_operation
+        parsed = parse_operation
+        if parsed
           "#{model}/#{parsed}"
         else
-          "SQL/other"
+          regex_name(sql)
         end
       end
 
@@ -76,6 +73,67 @@ module ScoutApm
           end
         end
       end
+
+
+      ########################
+      #  Regex based naming  #
+      ########################
+      #
+      WHITE_SPACE = '\s*'
+      REGEX_OPERATION = '(SELECT|UPDATE|INSERT|DELETE)'
+      FROM = 'FROM'
+      INTO = 'INTO'
+      NON_GREEDY_CONSUME = '.*?'
+      TABLE = '(?:"|`)?(.*?)(?:"|`)?\s'
+      COUNT = 'COUNT\(.*?\)'
+
+      SELECT_REGEX = /\A#{WHITE_SPACE}(SELECT)#{WHITE_SPACE}(#{COUNT})?#{NON_GREEDY_CONSUME}#{FROM}#{WHITE_SPACE}#{TABLE}/i.freeze
+      UPDATE_REGEX = /\A#{WHITE_SPACE}(UPDATE)#{WHITE_SPACE}#{TABLE}/i.freeze
+      INSERT_REGEX = /\A#{WHITE_SPACE}(INSERT)#{WHITE_SPACE}#{INTO}#{WHITE_SPACE}#{TABLE}/i.freeze
+      DELETE_REGEX = /\A#{WHITE_SPACE}(DELETE)#{WHITE_SPACE}#{FROM}#{TABLE}/i.freeze
+
+      COUNT_LABEL = 'count'.freeze
+      SELECT_LABEL = 'find'.freeze
+      UPDATE_LABEL = 'save'.freeze
+      INSERT_LABEL = 'create'.freeze
+      DELETE_LABEL = 'destroy'.freeze
+      UNKNOWN_LABEL = 'SQL/other'.freeze
+
+      # Attempt to do some basic parsing of SQL via regexes to extract the SQL
+      # verb (select, update, etc) and the table being operated on.
+      #
+      # This is a fallback from what ActiveRecord gives us, we prefer its
+      # names. But sometimes it is giving us a no-name query, and we have to
+      # attempt to figure it out ourselves.
+      #
+      # This relies on ActiveSupport's classify method. If it's not present,
+      # just skip the attempt to rename here. This could happen in a Grape or
+      # Sinatra application that doesn't import ActiveSupport. At this point,
+      # you're already using ActiveRecord, so it's likely loaded anyway.
+      def regex_name(sql)
+        # We rely on the ActiveSupport inflections code here. Bail early if we can't use it.
+        return UNKNOWN_LABEL unless UNKNOWN_LABEL.respond_to?(:classify)
+
+        if match = SELECT_REGEX.match(sql)
+          operation =
+            if match[2]
+              COUNT_LABEL
+            else
+              SELECT_LABEL
+            end
+          "#{match[3].classify}/#{operation}"
+        elsif match = UPDATE_REGEX.match(sql)
+          "#{match[2].classify}/#{UPDATE_LABEL}"
+        elsif match = INSERT_REGEX.match(sql)
+          "#{match[2].classify}/#{INSERT_LABEL}"
+        elsif match = DELETE_REGEX.match(sql)
+          "#{match[2].classify}/#{DELETE_LABEL}"
+        else
+          UNKNOWN_LABEL
+        end
+      rescue
+        UNKNOWN_LABEL
+      end
     end
   end
 end

data/lib/scout_apm/utils/sql_sanitizer.rb

@@ -74,8 +74,10 @@ module ScoutApm
         encodings.all?{|enc| Encoding.find(enc) rescue false}
       end
 
+      MAX_SQL_LENGTH = 16384
+
       def scrubbed(str)
-        return '' if !str.is_a?(String) || str.length > 4000 # safeguard - don't sanitize or scrub large SQL statements
+        return '' if !str.is_a?(String) || str.length > MAX_SQL_LENGTH # safeguard - don't sanitize or scrub large SQL statements
         return str if !str.respond_to?(:encode) # Ruby <= 1.8 doesn't have string encoding
         return str if str.valid_encoding? # Whatever encoding it is, it is valid and we can operate on it
         ScoutApm::Agent.instance.logger.debug "Scrubbing invalid sql encoding."

data/lib/scout_apm/version.rb

@@ -1,4 +1,4 @@
 module ScoutApm
-  VERSION = "2.3.2"
+  VERSION = "2.3.3"
 end
 

data/scout_apm.gemspec

@@ -30,4 +30,5 @@ Gem::Specification.new do |s|
   s.add_development_dependency "addressable"
   s.add_development_dependency "guard"
   s.add_development_dependency "guard-minitest"
+  s.add_development_dependency "activesupport"
 end

data/test/test_helper.rb

@@ -9,6 +9,8 @@ require 'mocha/mini_test'
 require 'pry'
 
 
+require 'active_support/core_ext/string/inflections'
+
 require 'scout_apm'
 
 Kernel.module_eval do

data/test/unit/utils/active_record_metric_name_test.rb

@@ -1,6 +1,7 @@
 require 'test_helper'
 require 'scout_apm/utils/active_record_metric_name'
 
+
 class ActiveRecordMetricNameTest < Minitest::Test
   # This is a bug report from Андрей Филиппов <tmn.sun@gmail.com>
   # The code that triggered the bug was: ActiveRecord::Base.connection.execute("%some sql%", :skip_logging)
@@ -9,7 +10,7 @@ class ActiveRecordMetricNameTest < Minitest::Test
     name = :skip_logging
 
     mn = ScoutApm::Utils::ActiveRecordMetricName.new(sql, name)
-    assert_equal "SQL/Unknown", mn.to_s
+    assert_equal "User/find", mn.to_s
   end
 
   def test_postgres_column_lookup
@@ -27,7 +28,7 @@ class ActiveRecordMetricNameTest < Minitest::Test
     name = "SCHEMA"
 
     mn = ScoutApm::Utils::ActiveRecordMetricName.new(sql, name)
-    assert_equal "SQL/Unknown", mn.to_s
+    assert_equal "SQL/other", mn.to_s
   end
 
 
@@ -44,23 +45,60 @@ class ActiveRecordMetricNameTest < Minitest::Test
     name = nil
 
     mn = ScoutApm::Utils::ActiveRecordMetricName.new(sql, name)
-    assert_equal "SQL/Unknown", mn.to_s
+    assert_equal "User/find", mn.to_s
   end
 
   def test_with_sql_name
-    sql = %q|INSERT INTO "users".* VALUES (1,2,3)|
+    sql = %q|INSERT INTO "users" VALUES (1,2,3)|
     name = "SQL"
 
     mn = ScoutApm::Utils::ActiveRecordMetricName.new(sql, name)
-    assert_equal "SQL/Unknown", mn.to_s
+    assert_equal "User/create", mn.to_s
   end
 
-  # TODO: Determine if there should be a distinction between Unknown and Other.
   def test_with_custom_name
     sql = %q|SELECT "users".* FROM "users" /*application:Testapp,controller:public,action:index*/|
     name = "A whole sentance describing what's what"
 
     mn = ScoutApm::Utils::ActiveRecordMetricName.new(sql, name)
-    assert_equal "SQL/other", mn.to_s
+    assert_equal "User/find", mn.to_s
+  end
+
+
+  # Regex test cases, pass these in w/ "SQL" as the AR provided name field
+  [
+    ["LoginRecord/save",     'UPDATE "login_record" SET "updated_at" = ? WHERE "login_record"."login_record_id" = ?'],
+    ["UserAccount/save",     'UPDATE "user_account" SET "last_activity" = ? WHERE "user_account"."user_account_id" = ?'],
+    ["Membership/save",      'UPDATE "membership" SET "updated_at" = ? WHERE "membership"."membership_id" = ?'],
+    ["Membership/save",      'UPDATE "membership" SET "updated_by" = ? WHERE "membership"."membership_id" = ?'],
+    ["Project/find",         'SELECT "project".project_id FROM "project" INNER JOIN "membership" ON "project"."project_id" = "membership"."project_id" WHERE "membership"."user_account_id" = ?'],
+    ["Entity/count",         'SELECT COUNT(*) FROM "entity" WHERE "entity"."entity_id" IN (?)'],
+    ["EntityEventStep/find", 'SELECT entity_event_step.entity_id, entity_event_step.event_step_id, event_step.event_id FROM entity_event_step INNER JOIN event_step ON event_step.event_step_id = entity_event_step.event_step_id WHERE event_step.event_id in (?) AND entity_event_step.entity_id in (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)'],
+    ["CustomField/count",    'SELECT COUNT(*) FROM "custom_field" WHERE "custom_field"."project_id" = ? AND "custom_field"."enabled" = ?'],
+    ["Event/find",           'SELECT MAX("events"."updated_at") AS max_id FROM "events" LEFT OUTER JOIN "configurations" ON "configurations"."event_id" = "events"."id" WHERE (configurations.date >= ?)'],
+    ["Sponsorship/find",     '  SELECT   "sponsorships"."id" AS t0_r0, "sponsorships"."event_id" AS t0_r1, "sponsorships"."name" AS t0_r2, "sponsorships"."position" AS t0_r3, "sponsorships"."created_at" AS t0_r4, "sponsorships"."updated_at" AS t0_r5, "sponsorships"."display_name" AS t0_r6, "sponsorships"."home_page_id" AS t0_r7, "sponsors"."id" AS t1_r0, "sponsors"."name" AS t1_r1, "sponsors"."url" AS t1_r2, "sponsors"."created_at" AS t1_r3, "sponsors"."updated_at" AS t1_r4, "assets"."id" AS t2_r0, "assets"."type" AS t2_r1, "assets"."assetable_id" AS t2_r2, "assets"."assetable_type" AS t2_r3, "assets"."attachment_file_name" AS t2_r4, "assets"."attachment_content_type" AS t2_r5, "assets"."attachment_file_size" AS t2_r6, "assets"."attachment_updated_at" AS t2_r7, "assets"."created_at" AS t2_r8, "assets"."updated_at" AS t2_r9 FROM    "sponsorships" LEFT OUTER JOIN "ranks" ON "ranks"."sponsorship_id" = "sponsorships"."id" LEFT OUTER JOIN "sponsors" ON "sponsors"."id" = "ranks"."sponsor_id" LEFT OUTER JOIN "assets" ON "assets".'],
+    ["MaxmindGeoliteCountry/find", 'SELECT country, country_code FROM maxmind_geolite_country WHERE start_ip_num <= ? AND ? <= end_ip_num'],
+    ["Activity/find", 'SELECT `activities`.`id` FROM `activities` INNER JOIN `activity_attendees` ON `activity_attendees`.`activity_id` = `activities`.`id` AND `activity_attendees`.`deleted` = ? LEFT JOIN activity_recurrence_periods arp ON arp.activity_id = activities.id WHERE (activities.updated_at >= ?) AND ((activities.start_date between ? and ? and activities.all_day = ?) or (CAST(activities.start_date as Date) between ? and ? and activities.all_day = ?)) AND (activity_attendees.user_id IN (?)) AND (arp.id IS NULL)'],
+
+    # Lower Case.
+    # ["MaxmindGeoliteCountry#Select", 'select country, country_code from maxmind_geolite_country where start_ip_num <= ? and ? <= end_ip_num'],
+
+    # No FROM clause (was clipped off)
+    ["SQL/other", 'SELECT "events"."id" AS t0_r0, "events"."name" AS t0_r1, "events"."subdomain" AS t0_r2, "events"."created_at" AS t0_r3, "events"."updated_at" AS t0_r4, "events"."slug" AS t0_r5, "events"."domain" AS t0_r6, "events"."published" AS t0_r7, "events"."category" AS t0_r8, "events"."venue_id" AS t0_r9, "events"."bitly_url" AS t0_r10, "events"."shown_in_events_list" AS t0_r11, "venues"."id" AS t1_r0, "venues"."name" AS t1_r1, "venues"."capacity" AS t1_r2, "venues"."address" AS t1_r3, "venues"."parking_info" AS t1_r4, "venues"."manual_coordinates" AS t1_r5, "venues"."latitude" AS t1_r6, "venues"."longitude" AS t1_r7, "venues"."description" AS t1_r8, "venues"."created_at" AS t1_r9, "venues"."updated_at" AS t1_r10, "venue_translations"."id" AS t2_r0, "venue_translations"."venue_id" AS t2_r1, "venue_translations"."locale" AS t2_r2, "venue_translations"."created_at" AS t2_r3, "venue_translations"."updated_at" AS t2_r4, "venue_translations"."name" AS t2_r5, "venue_translations"."description" AS t2_r6'],
+
+    # Stuff we don't care about in SQL
+    ["SQL/other", 'SET SESSION statement_timeout = ?'],
+    ["SQL/other", 'SHOW TIME ZONE'],
+    ["SQL/other", 'BEGIN'],
+    ["SQL/other", 'COMMIT'],
+
+    # Empty strings, or invalid SQL
+    ["SQL/other", ''],
+    ["SQL/other", 'not sql at all!'],
+  ].each_with_index do |(expected, sql), i|
+    define_method :"test_regex_naming_#{i}" do
+      actual = ScoutApm::Utils::ActiveRecordMetricName.new(sql, "SQL").to_s
+      assert_equal expected, actual
+    end
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: scout_apm
 version: !ruby/object:Gem::Version
-  version: 2.3.2
+  version: 2.3.3
 platform: ruby
 authors:
 - Derek Haynes
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-12-13 00:00:00.000000000 Z
+date: 2017-12-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: minitest
@@ -137,6 +137,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: Monitors Ruby apps and reports detailed metrics on performance to Scout.
 email:
 - support@scoutapp.com