scout_apm 2.4.10 → 2.4.11.pre

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: efa85b14b9a37dd4f890574b3e8a47a2023c1979
-  data.tar.gz: 67257b0a6bab1a0a24f1a4d3f4cea803e7e31456
+  metadata.gz: 1686beb2285cb415ac0f73e4dfa65524b85a369d
+  data.tar.gz: f56d09891f9b899721c711f43dc48e06160073b1
 SHA512:
-  metadata.gz: 47bcb3a2071df2f585c60837fac4c860159398ac8500d036875f8387aa02bea2c1f4e8ba4790322fb56ca58c85ffef482fec49c9fdb3f4030dd3ddcdff503ee5
-  data.tar.gz: ed8061983b9e6e4f2df133ce106500401562010fa10fc670fbbcd2de9f8aeebe442d95c7b7afee6469ccb436cbdae5de29a63dd529ad274174a76ad5d0e98615
+  metadata.gz: 3af376cdc73869c73775ca02aaad1817fa735a7b2c73521fe2015f0584c5085e50f7ec94c40bc402363c691e70d98c6cfbc4d0887ff1b0cb7082efeae1bc2578
+  data.tar.gz: 6ace83f9f30fd8bfeb48875e6a58c508f2cc65f1fafd36e8fc1e6421c6556ddc096a27396dc2546ae623af9765586da976da64b6cec9654c29c71feea0927bd6

data/CHANGELOG.markdown

@@ -1,3 +1,7 @@
+# 2.4.11
+
+* Adds transaction + periodic reporting callback extension support
+
 # 2.4.10
 
 * Improve ActiveRecord instrumentation across Rails 3.2+, and adding support

data/lib/scout_apm.rb

@@ -172,6 +172,9 @@ require 'scout_apm/agent/exit_handler'
 require 'scout_apm/tasks/doctor'
 require 'scout_apm/tasks/support'
 
+require 'scout_apm/extensions/config'
+require 'scout_apm/extensions/transaction_callback_payload'
+
 if defined?(Rails) && defined?(Rails::VERSION) && defined?(Rails::VERSION::MAJOR) && Rails::VERSION::MAJOR >= 3 && defined?(Rails::Railtie)
   module ScoutApm
     class Railtie < Rails::Railtie

data/lib/scout_apm/agent_context.rb

@@ -1,5 +1,8 @@
 module ScoutApm
   class AgentContext
+
+    attr_accessor :extensions
+
     # Initially start up without attempting to load a configuration file. We
     # need to be able to lookup configuration options like "application_root"
     # which would then in turn influence where the yaml configuration file is
@@ -9,6 +12,7 @@ module ScoutApm
     def initialize()
       @logger = LoggerFactory.build_minimal_logger
       @process_start_time = Time.now
+      @extensions = ScoutApm::Extensions::Config.new(self)
     end
 
     def marshal_dump

data/lib/scout_apm/context.rb

@@ -23,6 +23,15 @@ module ScoutApm
       @extra.merge({:user => @user})
     end
 
+    def to_flat_hash
+      h = to_hash
+      user = h.delete(:user)
+      if user
+        user.each { |k,v| h["user_#{k}"] = v}
+      end
+      h
+    end
+
     def self.current
       RequestManager.lookup.context
     end

data/lib/scout_apm/extensions/config.rb

@@ -0,0 +1,87 @@
+module ScoutApm
+  module Extensions
+    # !!! Extensions are a 0.x level API and breakage is expected as the API is refined.
+    # Extensions fan out data collected by the agent to additional services.
+    class Config
+      attr_reader   :agent_context
+      attr_accessor :transaction_callbacks
+      attr_accessor :periodic_callbacks
+
+      # Adds a new callback that runs after a transaction completes.
+      # These run inline during the request and thus should add minimal overhead. 
+      # For example, a transaction callback should NOT make inline HTTP calls to outside services.
+      # +callback+ must be an object that respond to a +call(payload)+ method.
+      #
+      # Example:
+      # ScoutApm::Extensions::Config.add_transaction_callback(Proc.new { |payload| puts "Duration: #{payload.duration_ms}" })
+      #
+      # +payload+ is a +ScoutApm::Extensions::TransactionCallbackPayload+ object.
+      def self.add_transaction_callback(callback)
+        agent_context.extensions.transaction_callbacks << callback
+      end
+
+      # Adds a callback that runs when the per-minute report data is sent to Scout.
+      # These run in a background thread so external HTTP calls are OK.
+      # +callback+ must be an object that responds to a +call(reporting_period, metadata)+ method.
+      # 
+      # Example:
+      # ScoutApm::Extensions::Config.add_periodic_callback(Proc.new { |reporting_period, metadata| ... })
+      def self.add_periodic_callback(callback)
+        agent_context.extensions.periodic_callbacks << callback
+      end
+
+      def initialize(agent_context)
+        @agent_context = agent_context
+        @transaction_callbacks = []
+        @periodic_callbacks = []
+      end
+
+      # Runs each reporting period callback. 
+      # Each callback runs inside a begin/rescue block so a broken callback doesn't prevent other
+      # callbacks from executing or reporting data from being sent. 
+      def run_periodic_callbacks(reporting_period, metadata)
+        return unless periodic_callbacks.any?
+
+        periodic_callbacks.each do |callback|
+          begin
+            callback.call(reporting_period, metadata)
+          rescue => e
+            logger.warn "Error running reporting callback extension=#{callback}"
+            logger.info e.message
+            logger.debug e.backtrace
+          end
+        end
+      end
+
+      # Runs each transaction callback.
+      # Each callback runs inside a begin/rescue block so a broken callback doesn't prevent other
+      # callbacks from executing or the transaction from being recorded. 
+      def run_transaction_callbacks(converter_results, context, scope_layer)
+        # It looks like layer_finder.scope = nil when a Sidekiq job is retried
+        return unless scope_layer
+        return unless transaction_callbacks.any?
+
+        payload = ScoutApm::Extensions::TransactionCallbackPayload.new(agent_context,converter_results,context,scope_layer)
+
+        transaction_callbacks.each do |callback|
+          begin
+            callback.call(payload)
+          rescue => e
+            logger.warn "Error running transaction callback extension=#{callback}"
+            logger.info e.message
+            logger.debug e.backtrace
+          end
+        end
+      end
+
+      def self.agent_context
+        ScoutApm::Agent.instance.context
+      end
+
+      def logger
+        agent_context.logger
+      end
+
+    end
+  end
+end

data/lib/scout_apm/extensions/transaction_callback_payload.rb

@@ -0,0 +1,74 @@
+module ScoutApm
+  module Extensions
+    # A +TransactionCallbackPayload+ is passed to each Transaction callback's +call+ method. 
+    # It encapsulates the data about a specific transaction.
+    class TransactionCallbackPayload
+      # A Hash that stores the output of each layer converter by name. See the naming conventions in +TrackedRequest+.
+      attr_accessor :converter_results
+
+      def initialize(agent_context,converter_results,context,scope_layer)
+        @agent_context = agent_context
+        @converter_results = converter_results
+        @context = context
+        @scope_layer = scope_layer
+      end
+
+      # A flat hash of the context associated w/this transaction (ie user ip and another other data added to context).
+      def context
+        @context.to_flat_hash
+      end
+
+      # The total duration of the transaction
+      def duration_ms
+        @scope_layer.total_call_time*1000 # ms
+      end
+
+      # The time in queue of the transaction in ms. If not present, +nil+ is returned as this is unknown.
+      def queue_time_ms
+        # Controller logic
+        if converter_results[:queue_time] && converter_results[:queue].any?
+          converter_results[:queue_time].values.first.total_call_time*1000 # ms
+        # Job logic
+        elsif converter_results[:job]
+          stat = converter_results[:job].metric_set.metrics[ScoutApm::MetricMeta.new("Latency/all", :scope => transaction_name)]
+          stat ? stat.total_call_time*1000 : nil
+        else
+          nil
+        end
+      end
+
+      def hostname
+        @agent_context.environment.hostname
+      end
+
+      def app_name
+        @agent_context.config.value('name')
+      end
+
+      # Returns +true+ if the transaction raised an exception.
+      def error?
+        converter_results[:errors] && converter_results[:errors].any?
+      end
+
+      def transation_type
+        @scope_layer.type
+      end
+
+      def transaction_name
+        @scope_layer.legacy_metric_name
+      end
+
+      # Web/Job are more language-agnostic names for controller/job. For example, Python Django does not have controllers.
+      def transaction_type_slug
+        case transation_type
+        when 'Controller'
+          'web'
+        when 'Job'
+          'job'
+        else
+          'transaction'
+        end
+      end
+    end
+  end
+end

data/lib/scout_apm/layer_converters/allocation_metric_converter.rb

@@ -8,8 +8,10 @@ module ScoutApm
         meta = MetricMeta.new("ObjectAllocations", {:scope => scope_layer.legacy_metric_name})
         stat = MetricStats.new
         stat.update!(root_layer.total_allocations)
+        metrics = { meta => stat }
 
-        @store.track!({ meta => stat })
+        @store.track!(metrics)
+        nil # not returning anything in the layer results ... not used
       end
     end
   end

data/lib/scout_apm/layer_converters/database_converter.rb

@@ -32,6 +32,8 @@ module ScoutApm
         # only due to 1 http request)
         @db_query_metric_set.increment_transaction_count!
         @store.track_db_query_metrics!(@db_query_metric_set)
+
+        nil # not returning anything in the layer results ... not used
       end
 
       def skip_layer?(layer)

data/lib/scout_apm/layer_converters/error_converter.rb

@@ -10,8 +10,10 @@ module ScoutApm
         meta = MetricMeta.new("Errors/#{scope_layer.legacy_metric_name}", {})
         stat = MetricStats.new
         stat.update!(1)
-
-        @store.track!({ meta => stat })
+        metrics = { meta => stat }
+        
+        @store.track!(metrics)
+        metrics # this result must be returned so it can be accessed by transaction callback extensions
       end
     end
   end

data/lib/scout_apm/layer_converters/histograms.rb

@@ -8,6 +8,7 @@ module ScoutApm
           context.request_histograms_by_time[@store.current_timestamp].
             add(request.unique_name, root_layer.total_call_time)
         end
+        nil # not returning anything in the layer results ... not used
       end
     end
   end

data/lib/scout_apm/layer_converters/job_converter.rb

@@ -55,6 +55,7 @@ module ScoutApm
         )
 
         @store.track_job!(record)
+        record  # this result must be returned so it can be accessed by transaction callback extensions
       end
 
       # This isn't stored as a specific layer, so grabbing it doesn't use the

data/lib/scout_apm/layer_converters/metric_converter.rb

@@ -38,6 +38,7 @@ module ScoutApm
 
       def record!
         @store.track!(@metrics)
+        @metrics  # this result must be returned so it can be accessed by transaction callback extensions
       end
     end
   end

data/lib/scout_apm/layer_converters/request_queue_time_converter.rb

@@ -28,8 +28,10 @@ module ScoutApm
         meta = MetricMeta.new("QueueTime/Request", {:scope => scope_layer.legacy_metric_name})
         stat = MetricStats.new(true)
         stat.update!(queue_time)
-
-        @store.track!({ meta => stat })
+        metrics = { meta => stat }
+        
+        @store.track!(metrics)
+        metrics  # this result must be returned so it can be accessed by transaction callback extensions
       end
 
       private

data/lib/scout_apm/layer_converters/slow_job_converter.rb

@@ -15,6 +15,7 @@ module ScoutApm
         # Let the store know we're here, and if it wants our data, it will call
         # back into #call
         @store.track_slow_job!(self)
+        nil # not returning anything in the layer results ... not used
       end
 
       #####################

data/lib/scout_apm/layer_converters/slow_request_converter.rb

@@ -11,6 +11,7 @@ module ScoutApm
         # Let the store know we're here, and if it wants our data, it will call
         # back into #call
         @store.track_slow_transaction!(self)
+        nil # not returning anything in the layer results ... not used
       end
 
       #####################

data/lib/scout_apm/reporting.rb

@@ -48,7 +48,9 @@ module ScoutApm
         begin
           merged = rps.inject { |memo, rp| memo.merge(rp) }
           logger.debug("Merged #{rps.length} reporting periods, delivering")
-          deliver_period(merged)
+          metadata = metadata(merged)
+          deliver_period(merged,metadata)
+          context.extensions.run_periodic_callbacks(merged, metadata)
           true
         rescue => e
           logger.debug("Error merging reporting periods #{e.message}")
@@ -63,15 +65,8 @@ module ScoutApm
       end
     end
 
-    def deliver_period(reporting_period)
-      metrics = reporting_period.metrics_payload
-      slow_transactions = reporting_period.slow_transactions_payload
-      jobs = reporting_period.jobs
-      slow_jobs = reporting_period.slow_jobs_payload
-      histograms = reporting_period.histograms
-      db_query_metrics = reporting_period.db_query_metrics_payload
-
-      metadata = {
+    def metadata(reporting_period)
+      {
         :app_root      => context.environment.root.to_s,
         :unique_id     => ScoutApm::Utils::UniqueId.simple,
         :agent_version => ScoutApm::VERSION,
@@ -79,6 +74,15 @@ module ScoutApm
         :agent_pid     => Process.pid,
         :platform      => "ruby",
       }
+    end
+
+    def deliver_period(reporting_period,metadata)
+      metrics = reporting_period.metrics_payload
+      slow_transactions = reporting_period.slow_transactions_payload
+      jobs = reporting_period.jobs
+      slow_jobs = reporting_period.slow_jobs_payload
+      histograms = reporting_period.histograms
+      db_query_metrics = reporting_period.db_query_metrics_payload
 
       log_deliver(metrics, slow_transactions, metadata, slow_jobs, histograms)
 

data/lib/scout_apm/tracked_request.rb

@@ -64,7 +64,6 @@ module ScoutApm
       @mem_start = mem_usage
       @recorder = agent_context.recorder
       @real_request = false
-
       ignore_request! if @recorder.nil?
     end
 
@@ -116,6 +115,7 @@ module ScoutApm
         layer.capture_backtrace!
       end
 
+
       if finalized?
         stop_request
       end
@@ -273,27 +273,34 @@ module ScoutApm
 
       apply_name_override
 
-      converters = [
-        LayerConverters::Histograms,
-        LayerConverters::MetricConverter,
-        LayerConverters::ErrorConverter,
-        LayerConverters::AllocationMetricConverter,
-        LayerConverters::RequestQueueTimeConverter,
-        LayerConverters::JobConverter,
-        LayerConverters::DatabaseConverter,
+      # Make a constant, then call converters.dup.each so it isn't inline?
+      converters = {
+        :histograms => LayerConverters::Histograms,
+        :metrics => LayerConverters::MetricConverter,
+        :errors => LayerConverters::ErrorConverter,
+        :allocation_metrics => LayerConverters::AllocationMetricConverter,
+        :queue_time => LayerConverters::RequestQueueTimeConverter,
+        :job => LayerConverters::JobConverter,
+        :db => LayerConverters::DatabaseConverter,
 
-        LayerConverters::SlowJobConverter,
-        LayerConverters::SlowRequestConverter,
-      ]
+        :slow_job => LayerConverters::SlowJobConverter,
+        :slow_req => LayerConverters::SlowRequestConverter,
+      }
 
       walker = LayerConverters::DepthFirstWalker.new(self.root_layer)
-      converters = converters.map do |klass|
+      converter_instances = converters.inject({}) do |memo, (slug, klass)|
         instance = klass.new(@agent_context, self, layer_finder, @store)
         instance.register_hooks(walker)
-        instance
+        memo[slug] = instance
+        memo
       end
       walker.walk
-      converters.each {|i| i.record! }
+      converter_results = converter_instances.inject({}) do |memo, (slug,i)| 
+        memo[slug] = i.record!
+        memo
+      end
+
+      @agent_context.extensions.run_transaction_callbacks(converter_results,context,layer_finder.scope)
 
       # If there's an instant_key, it means we need to report this right away
       if web? && instant?

data/lib/scout_apm/version.rb

@@ -1,4 +1,4 @@
 module ScoutApm
-  VERSION = "2.4.10"
+  VERSION = "2.4.11.pre"
 end
 

data/test/unit/extensions/periodic_callbacks_test.rb

@@ -0,0 +1,58 @@
+require 'test_helper'
+
+class PeriodicCallbacksTest < Minitest::Test
+
+  # We don't have a test that ensures we actually report data to the server, so we can't be 100% sure this doesn't break
+  # reporting.
+  def test_broken_callback_does_not_throw_exception
+    ScoutApm::Extensions::Config.add_periodic_callback(BrokenCallback.new)
+    # Runs via agent context as calling +add_periodic_callback+ initializing the context + extension config.
+    ScoutApm::Agent.instance.context.extensions.run_periodic_callbacks(reporting_period,metadata)
+  end
+
+  def test_callback_runs
+    Thread.current[:periodic_callback_output] = nil
+    ScoutApm::Extensions::Config.add_periodic_callback(PeriodicCallback.new)
+    ScoutApm::Agent.instance.context.extensions.run_periodic_callbacks(reporting_period,metadata)
+    assert Thread.current[:periodic_callback_output]
+  end
+
+  def run_proc_callback
+    Thread.current[:proc_periodic] = nil
+    ScoutApm::Extensions::Config.add_periodic_callback(Proc.new { |reporting_period, metadata| Thread.current[:proc_periodic] = Time.at(metadata[:agent_time].to_i) })
+    ScoutApm::Agent.instance.context.extensions.run_periodic_callbacks(reporting_period,metadata)
+    assert Thread.current[:proc_periodic]
+  end
+
+  # Doesn't respond to +call+.
+  class BrokenCallback
+  end
+
+  # Sets a Thread local so we can verify that the callback ran.
+  class PeriodicCallback
+    def call(reporting_period,metadata)
+      Thread.current[:periodic_callback_output] = true
+    end
+  end
+
+  private
+
+  def reporting_period
+    rp = ScoutApm::StoreReportingPeriod.new(ScoutApm::AgentContext.new, Time.at(metadata[:agent_time].to_i))
+    rp.absorb_metrics!(metrics)
+  end
+
+  def metrics
+    meta = ScoutApm::MetricMeta.new("Controller/users/index")
+    stats = ScoutApm::MetricStats.new
+    stats.update!(0.1)
+    {
+      meta => stats
+    }
+  end
+
+  def metadata
+    {:app_root=>"/srv/rails_app", :unique_id=>"ID", :agent_version=>"2.4.10", :agent_time=>"1523287920", :agent_pid=>21581, :platform=>"ruby"}
+  end
+
+end

data/test/unit/extensions/transaction_callbacks_test.rb

@@ -0,0 +1,58 @@
+require 'test_helper'
+
+class TransactionCallbacksTest < Minitest::Test
+
+  def setup
+    # Another test could set the recorder to +FakeRecorder+, which does not call +TrackedRequest#record!+.
+    ScoutApm::Agent.instance.context.recorder = ScoutApm::RecorderFactory.build(ScoutApm::Agent.instance.context)
+  end
+
+  # This is more of an integration test to ensure that we don't break TrackedRequest.
+  def test_broken_callback_does_not_break_tracked_request
+    ScoutApm::Extensions::Config.add_transaction_callback(BrokenCallback.new)
+
+    controller_layer = ScoutApm::Layer.new("Controller", "users/index")
+    # why doesn't this run? check if callbacks are configured
+    tr = ScoutApm::TrackedRequest.new(ScoutApm::Agent.instance.context, ScoutApm::FakeStore.new)
+    tr.start_layer(controller_layer)
+    tr.stop_layer
+  end
+
+  def test_callback_runs
+    Thread.current[:transaction_callback_output] = nil
+    ScoutApm::Extensions::Config.add_transaction_callback(TransactionCallback.new)
+
+    controller_layer = ScoutApm::Layer.new("Controller", "users/index")
+
+    tr = ScoutApm::TrackedRequest.new(ScoutApm::Agent.instance.context, ScoutApm::FakeStore.new)
+    tr.start_layer(controller_layer)
+    tr.stop_layer
+
+    assert Thread.current[:transaction_callback_output]
+  end
+
+  def test_run_proc_callback
+    Thread.current[:proc_transaction_duration] = nil
+    ScoutApm::Extensions::Config.add_transaction_callback(Proc.new { |payload| Thread.current[:proc_transaction_duration] = payload.duration_ms })
+
+    controller_layer = ScoutApm::Layer.new("Controller", "users/index")
+
+    tr = ScoutApm::TrackedRequest.new(ScoutApm::Agent.instance.context, ScoutApm::FakeStore.new)
+    tr.start_layer(controller_layer)
+    tr.stop_layer
+
+    assert Thread.current[:proc_transaction_duration]
+  end
+
+  # Doesn't respond to +call+.
+  class BrokenCallback
+  end
+
+  # Sets a Thread local so we can verify that the callback ran.
+  class TransactionCallback
+    def call(payload)
+      Thread.current[:transaction_callback_output] = true
+    end
+  end
+
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: scout_apm
 version: !ruby/object:Gem::Version
-  version: 2.4.10
+  version: 2.4.11.pre
 platform: ruby
 authors:
 - Derek Haynes
@@ -209,6 +209,8 @@ files:
 - lib/scout_apm/db_query_metric_stats.rb
 - lib/scout_apm/debug.rb
 - lib/scout_apm/environment.rb
+- lib/scout_apm/extensions/config.rb
+- lib/scout_apm/extensions/transaction_callback_payload.rb
 - lib/scout_apm/fake_store.rb
 - lib/scout_apm/framework_integrations/rails_2.rb
 - lib/scout_apm/framework_integrations/rails_3_or_4.rb
@@ -332,6 +334,8 @@ files:
 - test/unit/db_query_metric_set_test.rb
 - test/unit/db_query_metric_stats_test.rb
 - test/unit/environment_test.rb
+- test/unit/extensions/periodic_callbacks_test.rb
+- test/unit/extensions/transaction_callbacks_test.rb
 - test/unit/fake_store_test.rb
 - test/unit/git_revision_test.rb
 - test/unit/histogram_test.rb
@@ -379,12 +383,12 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
 rubyforge_project: scout_apm
-rubygems_version: 2.4.5.2
+rubygems_version: 2.4.6
 signing_key: 
 specification_version: 4
 summary: Ruby application performance monitoring
@@ -398,6 +402,8 @@ test_files:
 - test/unit/db_query_metric_set_test.rb
 - test/unit/db_query_metric_stats_test.rb
 - test/unit/environment_test.rb
+- test/unit/extensions/periodic_callbacks_test.rb
+- test/unit/extensions/transaction_callbacks_test.rb
 - test/unit/fake_store_test.rb
 - test/unit/git_revision_test.rb
 - test/unit/histogram_test.rb
@@ -429,3 +435,4 @@ test_files:
 - test/unit/utils/backtrace_parser_test.rb
 - test/unit/utils/numbers_test.rb
 - test/unit/utils/scm.rb
+has_rdoc: