scout_apm 3.0.0.pre13 → 3.0.0.pre14

data/test/unit/store_test.rb

@@ -3,15 +3,17 @@ require 'test_helper'
 require 'scout_apm/store'
 
 class FakeFailingLayaway
+  attr_reader :rps_written
   def write_reporting_period(rp)
-    raise "Always fails. Sucks."
+    @rps_written ||= []
+    @rps_written << rp
   end
 end
 
 class StoreTest < Minitest::Test
   # TODO: Introduce a clock object to avoid having to use 'force'
   def test_writing_layaway_removes_timestamps
-    s = ScoutApm::Store.new
+    s = ScoutApm::Store.new(ScoutApm::AgentContext.new)
     s.track_one!("Controller", "user/show", 10)
 
     assert_equal(1, s.instance_variable_get('@reporting_periods').size)
@@ -22,13 +24,14 @@ class StoreTest < Minitest::Test
   end
 
   def test_writing_layaway_removes_stale_timestamps
+    context = ScoutApm::AgentContext.new
     current_time = Time.now.utc
-    current_rp = ScoutApm::StoreReportingPeriod.new(current_time)
-    stale_rp = ScoutApm::StoreReportingPeriod.new(current_time - current_time.sec - 120)
+    current_rp = ScoutApm::StoreReportingPeriod.new(current_time, context)
+    stale_rp = ScoutApm::StoreReportingPeriod.new(current_time - current_time.sec - 120, context)
 
-    s = ScoutApm::Store.new
-    ScoutApm::Instruments::Process::ProcessMemory.new(Logger.new(StringIO.new)).metrics(stale_rp.timestamp, s)
-    ScoutApm::Instruments::Process::ProcessMemory.new(Logger.new(StringIO.new)).metrics(current_rp.timestamp, s)
+    s = ScoutApm::Store.new(context)
+    ScoutApm::Instruments::Process::ProcessMemory.new(context).metrics(stale_rp.timestamp, s)
+    ScoutApm::Instruments::Process::ProcessMemory.new(context).metrics(current_rp.timestamp, s)
     assert_equal 2, s.instance_variable_get('@reporting_periods').size
 
     s.write_to_layaway(FakeFailingLayaway.new, true)
@@ -43,7 +46,7 @@ class StoreReportingPeriodTest < Minitest::Test
   attr_reader :subject
 
   def setup
-    @subject = ScoutApm::StoreReportingPeriod.new(ScoutApm::StoreReportingPeriodTimestamp.new)
+    @subject = ScoutApm::StoreReportingPeriod.new(ScoutApm::StoreReportingPeriodTimestamp.new(Time.now), ScoutApm::AgentContext.new)
   end
 
   # Check default values at creation time

data/test/unit/utils/active_record_metric_name_test.rb

@@ -1,6 +1,7 @@
 require 'test_helper'
 require 'scout_apm/utils/active_record_metric_name'
 
+
 class ActiveRecordMetricNameTest < Minitest::Test
   # This is a bug report from Андрей Филиппов <tmn.sun@gmail.com>
   # The code that triggered the bug was: ActiveRecord::Base.connection.execute("%some sql%", :skip_logging)
@@ -9,7 +10,7 @@ class ActiveRecordMetricNameTest < Minitest::Test
     name = :skip_logging
 
     mn = ScoutApm::Utils::ActiveRecordMetricName.new(sql, name)
-    assert_equal "SQL/Unknown", mn.to_s
+    assert_equal "User/find", mn.to_s
   end
 
   def test_postgres_column_lookup
@@ -27,7 +28,7 @@ class ActiveRecordMetricNameTest < Minitest::Test
     name = "SCHEMA"
 
     mn = ScoutApm::Utils::ActiveRecordMetricName.new(sql, name)
-    assert_equal "SQL/Unknown", mn.to_s
+    assert_equal "SQL/other", mn.to_s
   end
 
 
@@ -44,23 +45,60 @@ class ActiveRecordMetricNameTest < Minitest::Test
     name = nil
 
     mn = ScoutApm::Utils::ActiveRecordMetricName.new(sql, name)
-    assert_equal "SQL/Unknown", mn.to_s
+    assert_equal "User/find", mn.to_s
   end
 
   def test_with_sql_name
-    sql = %q|INSERT INTO "users".* VALUES (1,2,3)|
+    sql = %q|INSERT INTO "users" VALUES (1,2,3)|
     name = "SQL"
 
     mn = ScoutApm::Utils::ActiveRecordMetricName.new(sql, name)
-    assert_equal "SQL/Unknown", mn.to_s
+    assert_equal "User/create", mn.to_s
   end
 
-  # TODO: Determine if there should be a distinction between Unknown and Other.
   def test_with_custom_name
     sql = %q|SELECT "users".* FROM "users" /*application:Testapp,controller:public,action:index*/|
     name = "A whole sentance describing what's what"
 
     mn = ScoutApm::Utils::ActiveRecordMetricName.new(sql, name)
-    assert_equal "SQL/other", mn.to_s
+    assert_equal "User/find", mn.to_s
+  end
+
+
+  # Regex test cases, pass these in w/ "SQL" as the AR provided name field
+  [
+    ["LoginRecord/save",     'UPDATE "login_record" SET "updated_at" = ? WHERE "login_record"."login_record_id" = ?'],
+    ["UserAccount/save",     'UPDATE "user_account" SET "last_activity" = ? WHERE "user_account"."user_account_id" = ?'],
+    ["Membership/save",      'UPDATE "membership" SET "updated_at" = ? WHERE "membership"."membership_id" = ?'],
+    ["Membership/save",      'UPDATE "membership" SET "updated_by" = ? WHERE "membership"."membership_id" = ?'],
+    ["Project/find",         'SELECT "project".project_id FROM "project" INNER JOIN "membership" ON "project"."project_id" = "membership"."project_id" WHERE "membership"."user_account_id" = ?'],
+    ["Entity/count",         'SELECT COUNT(*) FROM "entity" WHERE "entity"."entity_id" IN (?)'],
+    ["EntityEventStep/find", 'SELECT entity_event_step.entity_id, entity_event_step.event_step_id, event_step.event_id FROM entity_event_step INNER JOIN event_step ON event_step.event_step_id = entity_event_step.event_step_id WHERE event_step.event_id in (?) AND entity_event_step.entity_id in (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)'],
+    ["CustomField/count",    'SELECT COUNT(*) FROM "custom_field" WHERE "custom_field"."project_id" = ? AND "custom_field"."enabled" = ?'],
+    ["Event/find",           'SELECT MAX("events"."updated_at") AS max_id FROM "events" LEFT OUTER JOIN "configurations" ON "configurations"."event_id" = "events"."id" WHERE (configurations.date >= ?)'],
+    ["Sponsorship/find",     '  SELECT   "sponsorships"."id" AS t0_r0, "sponsorships"."event_id" AS t0_r1, "sponsorships"."name" AS t0_r2, "sponsorships"."position" AS t0_r3, "sponsorships"."created_at" AS t0_r4, "sponsorships"."updated_at" AS t0_r5, "sponsorships"."display_name" AS t0_r6, "sponsorships"."home_page_id" AS t0_r7, "sponsors"."id" AS t1_r0, "sponsors"."name" AS t1_r1, "sponsors"."url" AS t1_r2, "sponsors"."created_at" AS t1_r3, "sponsors"."updated_at" AS t1_r4, "assets"."id" AS t2_r0, "assets"."type" AS t2_r1, "assets"."assetable_id" AS t2_r2, "assets"."assetable_type" AS t2_r3, "assets"."attachment_file_name" AS t2_r4, "assets"."attachment_content_type" AS t2_r5, "assets"."attachment_file_size" AS t2_r6, "assets"."attachment_updated_at" AS t2_r7, "assets"."created_at" AS t2_r8, "assets"."updated_at" AS t2_r9 FROM    "sponsorships" LEFT OUTER JOIN "ranks" ON "ranks"."sponsorship_id" = "sponsorships"."id" LEFT OUTER JOIN "sponsors" ON "sponsors"."id" = "ranks"."sponsor_id" LEFT OUTER JOIN "assets" ON "assets".'],
+    ["MaxmindGeoliteCountry/find", 'SELECT country, country_code FROM maxmind_geolite_country WHERE start_ip_num <= ? AND ? <= end_ip_num'],
+    ["Activity/find", 'SELECT `activities`.`id` FROM `activities` INNER JOIN `activity_attendees` ON `activity_attendees`.`activity_id` = `activities`.`id` AND `activity_attendees`.`deleted` = ? LEFT JOIN activity_recurrence_periods arp ON arp.activity_id = activities.id WHERE (activities.updated_at >= ?) AND ((activities.start_date between ? and ? and activities.all_day = ?) or (CAST(activities.start_date as Date) between ? and ? and activities.all_day = ?)) AND (activity_attendees.user_id IN (?)) AND (arp.id IS NULL)'],
+
+    # Lower Case.
+    # ["MaxmindGeoliteCountry#Select", 'select country, country_code from maxmind_geolite_country where start_ip_num <= ? and ? <= end_ip_num'],
+
+    # No FROM clause (was clipped off)
+    ["SQL/other", 'SELECT "events"."id" AS t0_r0, "events"."name" AS t0_r1, "events"."subdomain" AS t0_r2, "events"."created_at" AS t0_r3, "events"."updated_at" AS t0_r4, "events"."slug" AS t0_r5, "events"."domain" AS t0_r6, "events"."published" AS t0_r7, "events"."category" AS t0_r8, "events"."venue_id" AS t0_r9, "events"."bitly_url" AS t0_r10, "events"."shown_in_events_list" AS t0_r11, "venues"."id" AS t1_r0, "venues"."name" AS t1_r1, "venues"."capacity" AS t1_r2, "venues"."address" AS t1_r3, "venues"."parking_info" AS t1_r4, "venues"."manual_coordinates" AS t1_r5, "venues"."latitude" AS t1_r6, "venues"."longitude" AS t1_r7, "venues"."description" AS t1_r8, "venues"."created_at" AS t1_r9, "venues"."updated_at" AS t1_r10, "venue_translations"."id" AS t2_r0, "venue_translations"."venue_id" AS t2_r1, "venue_translations"."locale" AS t2_r2, "venue_translations"."created_at" AS t2_r3, "venue_translations"."updated_at" AS t2_r4, "venue_translations"."name" AS t2_r5, "venue_translations"."description" AS t2_r6'],
+
+    # Stuff we don't care about in SQL
+    ["SQL/other", 'SET SESSION statement_timeout = ?'],
+    ["SQL/other", 'SHOW TIME ZONE'],
+    ["SQL/other", 'BEGIN'],
+    ["SQL/other", 'COMMIT'],
+
+    # Empty strings, or invalid SQL
+    ["SQL/other", ''],
+    ["SQL/other", 'not sql at all!'],
+  ].each_with_index do |(expected, sql), i|
+    define_method :"test_regex_naming_#{i}" do
+      actual = ScoutApm::Utils::ActiveRecordMetricName.new(sql, "SQL").to_s
+      assert_equal expected, actual
+    end
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: scout_apm
 version: !ruby/object:Gem::Version
-  version: 3.0.0.pre13
+  version: 3.0.0.pre14
 platform: ruby
 authors:
 - Derek Haynes
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-10-20 00:00:00.000000000 Z
+date: 2017-12-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: minitest
@@ -137,6 +137,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: Monitors Ruby apps and reports detailed metrics on performance to Scout.
 email:
 - support@scoutapp.com
@@ -166,8 +180,9 @@ files:
 - ext/stacks/stacks.c
 - lib/scout_apm.rb
 - lib/scout_apm/agent.rb
-- lib/scout_apm/agent/logging.rb
-- lib/scout_apm/agent/reporting.rb
+- lib/scout_apm/agent/exit_handler.rb
+- lib/scout_apm/agent/preconditions.rb
+- lib/scout_apm/agent_context.rb
 - lib/scout_apm/app_server_load.rb
 - lib/scout_apm/attribute_arranger.rb
 - lib/scout_apm/background_job_integrations/delayed_job.rb
@@ -194,6 +209,7 @@ files:
 - lib/scout_apm/instant/assets/xmlhttp_instrumentation.html
 - lib/scout_apm/instant/middleware.rb
 - lib/scout_apm/instant_reporting.rb
+- lib/scout_apm/instrument_manager.rb
 - lib/scout_apm/instruments/.DS_Store
 - lib/scout_apm/instruments/action_controller_rails_2.rb
 - lib/scout_apm/instruments/action_controller_rails_3_rails4.rb
@@ -214,6 +230,7 @@ files:
 - lib/scout_apm/instruments/rails_router.rb
 - lib/scout_apm/instruments/redis.rb
 - lib/scout_apm/instruments/resque.rb
+- lib/scout_apm/instruments/samplers.rb
 - lib/scout_apm/instruments/sinatra.rb
 - lib/scout_apm/job_record.rb
 - lib/scout_apm/layaway.rb
@@ -233,10 +250,12 @@ files:
 - lib/scout_apm/layer_converters/slow_job_converter.rb
 - lib/scout_apm/layer_converters/slow_request_converter.rb
 - lib/scout_apm/limited_layer.rb
+- lib/scout_apm/logger.rb
 - lib/scout_apm/metric_meta.rb
 - lib/scout_apm/metric_set.rb
 - lib/scout_apm/metric_stats.rb
 - lib/scout_apm/middleware.rb
+- lib/scout_apm/periodic_work.rb
 - lib/scout_apm/platform_integrations/cloud_foundry.rb
 - lib/scout_apm/platform_integrations/heroku.rb
 - lib/scout_apm/platform_integrations/server.rb
@@ -246,6 +265,7 @@ files:
 - lib/scout_apm/remote/router.rb
 - lib/scout_apm/remote/server.rb
 - lib/scout_apm/reporter.rb
+- lib/scout_apm/reporting.rb
 - lib/scout_apm/request_histograms.rb
 - lib/scout_apm/request_manager.rb
 - lib/scout_apm/scored_item_set.rb
@@ -272,6 +292,8 @@ files:
 - lib/scout_apm/stack_item.rb
 - lib/scout_apm/store.rb
 - lib/scout_apm/synchronous_recorder.rb
+- lib/scout_apm/tasks/doctor.rb
+- lib/scout_apm/tasks/support.rb
 - lib/scout_apm/trace_compactor.rb
 - lib/scout_apm/tracer.rb
 - lib/scout_apm/tracked_request.rb
@@ -281,7 +303,6 @@ files:
 - lib/scout_apm/utils/gzip_helper.rb
 - lib/scout_apm/utils/installed_gems.rb
 - lib/scout_apm/utils/klass_helper.rb
-- lib/scout_apm/utils/null_logger.rb
 - lib/scout_apm/utils/numbers.rb
 - lib/scout_apm/utils/scm.rb
 - lib/scout_apm/utils/sql_sanitizer.rb
@@ -290,6 +311,7 @@ files:
 - lib/scout_apm/utils/time.rb
 - lib/scout_apm/utils/unique_id.rb
 - lib/scout_apm/version.rb
+- lib/tasks/doctor.rake
 - scout_apm.gemspec
 - test/data/config_test_1.yml
 - test/test_helper.rb

data/lib/scout_apm/agent/logging.rb

@@ -1,74 +0,0 @@
-# Contains methods specific to logging (initializing the log file, applying the log level, applying the log format, etc.)
-module ScoutApm
-  class Agent
-    module Logging
-      def default_log_path
-        "#{environment.root}/log"
-      end
-
-      def init_logger(opts={})
-        if opts[:force]
-          @log_file = nil
-          @logger = nil
-        end
-
-        begin
-          @log_file ||= determine_log_destination
-        rescue => e
-        end
-
-        begin
-          @logger ||= Logger.new(@log_file)
-          @logger.level = log_level
-          apply_log_format
-        rescue Exception => e
-          @logger = Logger.new(STDOUT)
-          apply_log_format
-          @logger.error "Unable to open log file for writing: #{e.message}. Falling back to STDOUT"
-        end
-        @logger
-      end
-
-      def apply_log_format
-        def logger.format_message(severity, timestamp, progname, msg)
-          # since STDOUT isn't exclusive like the scout_apm.log file, apply a prefix.
-          prefix = @logdev.dev == STDOUT ? "[Scout] " : ''
-          prefix + "[#{Utils::Time.to_s(timestamp)} #{ScoutApm::Agent.instance.environment.hostname} (#{$$})] #{severity} : #{msg}\n"
-        end
-      end
-
-      def log_level
-        case config.value('log_level').downcase
-          when "debug" then Logger::DEBUG
-          when "info" then Logger::INFO
-          when "warn" then Logger::WARN
-          when "error" then Logger::ERROR
-          when "fatal" then Logger::FATAL
-          else Logger::INFO
-        end
-      end
-
-      def determine_log_destination
-        case true
-        when wants_stdout? then STDOUT
-        when wants_stderr? then STDERR
-        else "#{log_file_path}/scout_apm.log"
-        end
-      end
-
-      def wants_stdout?
-        config.value('log_file_path').to_s.upcase == 'STDOUT' || environment.platform_integration.log_to_stdout?
-      end
-
-      def wants_stderr?
-        config.value('log_file_path').to_s.upcase == 'STDERR'
-      end
-
-      def log_file_path
-        config.value('log_file_path') || default_log_path
-      end
-    end
-    include Logging
-  end
-end
-

data/lib/scout_apm/agent/reporting.rb

@@ -1,129 +0,0 @@
-# Methods related to sending metrics to scoutapp.com.
-module ScoutApm
-  class Agent
-    module Reporting
-      def reporter
-        @reporter ||= ScoutApm::Reporter.new(:checkin, config, logger)
-      end
-
-      # The data moves through a treadmill of reporting, coordinating several Rails processes by using an external file.
-      # * During the minute it is being recorded by the instruments, it gets
-      #   recorded into the ram of each process (in the Store class).
-      # * The minute after, each process writes its own metrics to a shared LayawayFile
-      # * The minute after that, the first process to wake up pushes the combined
-      #   data to the server, and wipes it. Next processes don't have anything to do.
-      #
-      # At any given point, there is data in each of those steps, moving its way through the process
-      def process_metrics
-        # Write the previous minute's data to the shared-across-process layaway file.
-        store.write_to_layaway(layaway)
-
-        # Attempt to send 2 minutes ago's data up to the server.  This
-        # only acctually occurs if this process is the first to wake up this
-        # minute.
-        report_to_server
-      end
-
-      # In a running app, one process will get the period ready for delivery, the others will see 0.
-      def report_to_server
-        period_to_report = ScoutApm::StoreReportingPeriodTimestamp.minutes_ago(2)
-
-        logger.debug("Attempting to claim #{period_to_report.to_s}")
-
-        did_write = layaway.with_claim(period_to_report) do |rps|
-          logger.debug("Succeeded claiming #{period_to_report.to_s}")
-
-          begin
-            merged = rps.inject { |memo, rp| memo.merge(rp) }
-            logger.debug("Merged #{rps.length} reporting periods, delivering")
-            deliver_period(merged)
-            true
-          rescue => e
-            logger.debug("Error merging reporting periods #{e.message}")
-            logger.debug("Error merging reporting periods #{e.backtrace}")
-            false
-          end
-
-        end
-
-        if !did_write
-          logger.debug("Failed to obtain claim for #{period_to_report.to_s}")
-        end
-
-      end
-
-
-      def deliver_period(reporting_period)
-        metrics = reporting_period.metrics_payload
-        slow_transactions = reporting_period.slow_transactions_payload
-        jobs = reporting_period.jobs
-        slow_jobs = reporting_period.slow_jobs_payload
-        histograms = reporting_period.histograms
-        db_query_metrics = reporting_period.db_query_metrics_payload
-
-        metadata = {
-          :app_root      => ScoutApm::Environment.instance.root.to_s,
-          :unique_id     => ScoutApm::Utils::UniqueId.simple,
-          :agent_version => ScoutApm::VERSION,
-          :agent_time    => reporting_period.timestamp.to_s,
-          :agent_pid     => Process.pid,
-          :platform      => "ruby",
-        }
-
-        log_deliver(metrics, slow_transactions, metadata, slow_jobs, histograms)
-
-        payload = ScoutApm::Serializers::PayloadSerializer.serialize(metadata, metrics, slow_transactions, jobs, slow_jobs, histograms, db_query_metrics)
-        logger.debug("Sending payload w/ Headers: #{headers.inspect}")
-
-        reporter.report(payload, headers)
-      rescue => e
-        logger.warn "Error on checkin"
-        logger.info e.message
-        logger.debug e.backtrace
-      end
-
-      def log_deliver(metrics, slow_transactions, metadata, jobs_traces, histograms)
-        total_request_count = metrics.
-          select { |meta,stats| meta.metric_name =~ /\AController/ }.
-          inject(0) {|sum, (_, stat)| sum + stat.call_count }
-
-        memory = metrics.
-          find {|meta,stats| meta.metric_name =~ /\AMemory/ }
-        process_log_str = if memory
-                            "Recorded from #{memory.last.call_count} processes"
-                          else
-                            "Recorded across (unknown) processes"
-                          end
-
-        time_clause       = "[#{Time.parse(metadata[:agent_time]).strftime("%H:%M")}]"
-        metrics_clause    = "#{metrics.length} Metrics for #{total_request_count} requests"
-        slow_trans_clause = "#{slow_transactions.length} Slow Transaction Traces"
-        job_clause        = "#{jobs_traces.length} Job Traces"
-        histogram_clause  = "#{histograms.length} Histograms"
-
-        logger.info "#{time_clause} Delivering #{metrics_clause} and #{slow_trans_clause} and #{job_clause}, #{process_log_str}."
-        # logger.debug("\n\nMetrics: #{metrics.pretty_inspect}\nSlowTrans: #{slow_transactions.pretty_inspect}\nMetadata: #{metadata.inspect.pretty_inspect}\n\n")
-      end
-
-      # TODO: Move this into PayloadSerializer?
-      def headers
-        if ScoutApm::Agent.instance.config.value("report_format") == 'json'
-          headers = {'Content-Type' => 'application/json'}
-        else
-          headers = {}
-        end
-      end
-
-      # Before reporting, lookup metric_id for each MetricMeta. This speeds up
-      # reporting on the server-side.
-      def add_metric_ids(metrics)
-        metrics.each do |meta,stats|
-          if metric_id = metric_lookup[meta]
-            meta.metric_id = metric_id
-          end
-        end
-      end
-    end
-    include Reporting
-  end
-end