scout_apm 1.1.0.pre1 → 1.2.0.pre1

data/lib/scout_apm/tracer.rb

@@ -1,11 +1,11 @@
-# Contains the methods that instrument blocks of code.
+# Provides helpers to wrap sections of code in instrumentation
 #
-# When a code block is wrapped inside #instrument(metric_name):
-# * The #instrument method pushes a StackItem onto Store#stack
-# * When a code block is finished, #instrument pops the last item off the stack and verifies it's the StackItem
-#   we created earlier.
-# * Once verified, the metrics for the recording session are merged into the in-memory Store#metric_hash. The current scope
-#   is also set for the metric (if Thread::current[:scout_apm_scope_name] isn't nil).
+# The manual approach is to wrap your code in a call like:
+#     `instrument("View", "users/index") do ... end`
+#
+# The monkey-patching approach does this for you:
+#     `instrument_method(:my_render, :type => "View", :name => "users/index)`
+
 module ScoutApm
   module Tracer
     def self.included(klass)
@@ -13,92 +13,67 @@ module ScoutApm
     end
 
     module ClassMethods
-
-      # Use to trace a method call, possibly reporting slow transaction traces to Scout.
-      # Options:
-      # * uri - the request uri
-      # * ip - the remote ip of the user. This is merged into the User context.
-      def scout_apm_trace(metric_name, options = {}, &block)
-        # TODO - wrap a lot of this into a Trace class, store that as a Thread var.
-        ScoutApm::Agent.instance.store.reset_transaction!
-        ScoutApm::Context.current.add_user(:ip => options[:ip]) if options[:ip]
-        ScoutApm::Agent.instance.capacity.start_transaction!
-        e = nil
-        instrument(metric_name, options) do
-          Thread::current[:scout_apm_scope_name] = metric_name
-          begin
-            yield
-          rescue Exception => e
-            ScoutApm::Agent.instance.store.track!("Errors/#{metric_name}",1, :scope => nil)
-          end
-          Thread::current[:scout_apm_scope_name] = nil
-        end
-        ScoutApm::Agent.instance.capacity.finish_transaction!
-        # The context is cleared after instrumentation (rather than before) as tracing controller-actions doesn't occur until the controller-action is called.
-        # It does not trace before filters, which is a likely spot to add context. This means that any context applied during before_filters would be cleared.
-        ScoutApm::Context.clear!
-        raise e if e
-      end
-
+      # Type: the Layer type - "View" or similar
+      # Name: specific name - "users/_gravatar"
+      # A Block: The code to be instrumented
+      #
       # Options:
-      # * :scope - If specified, sets the sub-scope for the metric. We allow additional scope level. This is used
-      # * uri - the request uri
       # * :ignore_children - will not instrument any method calls beneath this call. Example use case: InfluxDB uses Net::HTTP, which is instrumented. However, we can provide more specific data if we know we're doing an influx call, so we'd rather just instrument the Influx call and ignore Net::HTTP.
-      # when rendering the transaction tree in the UI.
-      def instrument(metric_name, options={}, &block)
-        # don't instrument if (1) NOT inside a transaction and (2) NOT a Controller metric.
-        if !Thread::current[:scout_apm_scope_name] and metric_name !~ /\AController\//
-          return yield
-        elsif Thread::current[:scout_ignore_children]
-          return yield
-        end
-
-        if options.delete(:scope)
-          Thread::current[:scout_apm_sub_scope] = metric_name
-        end
-
-        if options[:ignore_children]
-          Thread::current[:scout_ignore_children] = true
-        end
-        stack_item = ScoutApm::Agent.instance.store.record(metric_name)
+      #   when rendering the transaction tree in the UI.
+      # * :desc - Additional capture, SQL, or HTTP url or similar
+      # * :scope - set to true if you want to make this layer a subscope
+      def instrument(type, name, options={}) # Takes a block
+        layer = ScoutApm::Layer.new(type, name)
+        layer.desc = options[:desc] if options[:desc]
+        layer.subscopable!          if options[:scope]
+
+        req = ScoutApm::RequestManager.lookup
+        req.start_layer(layer)
+        req.ignore_children! if options[:ignore_children]
 
         begin
           yield
         ensure
-          Thread::current[:scout_apm_sub_scope] = nil if Thread::current[:scout_apm_sub_scope] == metric_name
-          if options[:ignore_children]
-            Thread::current[:scout_ignore_children] = nil
-          end
-
-          ScoutApm::Agent.instance.store.stop_recording(stack_item,options)
+          req.stop_layer
+          req.acknowledge_children! if options[:ignore_children]
         end
       end
 
+      # Wraps a method in a call to #instrument via aggressive monkey patching.
+      #
+      # Options:
+      # type - "View" or "ActiveRecord" and similar
+      # name - "users/show", "App#find"
       def instrument_method(method, options = {})
         ScoutApm::Agent.instance.logger.info "Instrumenting #{method}"
-        metric_name = options[:metric_name] || default_metric_name(method)
-        return if !instrumentable?(method) or instrumented?(method,metric_name)
+        type = options[:type] || "Custom"
+        name = options[:name] || "#{self.name}/#{method.to_s}"
+
+        return if !instrumentable?(method) or instrumented?(method, type, name)
 
-        class_eval(instrumented_method_string(
-          method,
-          {:metric_name => metric_name, :scope => options[:scope] }),
+        class_eval(
+          instrumented_method_string(method, type, name, {:scope => options[:scope] }),
           __FILE__, __LINE__
         )
 
-        alias_method _uninstrumented_method_name(method, metric_name), method
-        alias_method method, _instrumented_method_name(method, metric_name)
+        alias_method _uninstrumented_method_name(method, type, name), method
+        alias_method method, _instrumented_method_name(method, type, name)
       end
 
       private
 
-      def instrumented_method_string(method, options)
+      def instrumented_method_string(method, type, name, options={})
         klass = (self === Module) ? "self" : "self.class"
-        method_str = "def #{_instrumented_method_name(method, options[:metric_name])}(*args, &block)
-          result = #{klass}.instrument(\"#{options[:metric_name]}\",{:scope => #{options[:scope] || false}}) do
-            #{_uninstrumented_method_name(method, options[:metric_name])}(*args, &block)
+        method_str = <<-EOF
+        def #{_instrumented_method_name(method, type, name)}(*args, &block)
+          #{klass}.instrument( "#{type}",
+                               "#{name}",
+                               {:scope => #{options[:scope] || false}}
+                             ) do
+            #{_uninstrumented_method_name(method, type, name)}(*args, &block)
           end
-          result
-        end"
+        end
+        EOF
 
         method_str
       end
@@ -111,26 +86,22 @@ module ScoutApm
       end
 
       # +True+ if the method is already instrumented.
-      def instrumented?(method,metric_name)
-        instrumented = method_defined?(_instrumented_method_name(method, metric_name))
-        ScoutApm::Agent.instance.logger.warn "The method [#{self.name}##{method}] has already been instrumented" if instrumented
+      def instrumented?(method, type, name)
+        instrumented = method_defined?(_instrumented_method_name(method, type, name))
+        ScoutApm::Agent.instance.logger.warn("The method [#{self.name}##{method}] has already been instrumented") if instrumented
         instrumented
       end
 
-      def default_metric_name(method)
-        "Custom/#{self.name}/#{method.to_s}"
-      end
-
       # given a method and a metric, this method returns the
       # untraced alias of the method name
-      def _uninstrumented_method_name(method, metric_name)
-        "#{_sanitize_name(method)}_without_scout_instrument_#{_sanitize_name(metric_name)}"
+      def _uninstrumented_method_name(method, type, name)
+        "#{_sanitize_name(method)}_without_scout_instrument_#{_sanitize_name(name)}"
       end
 
       # given a method and a metric, this method returns the traced
       # alias of the method name
-      def _instrumented_method_name(method, metric_name)
-        "#{_sanitize_name(method)}_with_scout_instrument_#{_sanitize_name(metric_name)}"
+      def _instrumented_method_name(method, type, name)
+        "#{_sanitize_name(method)}_with_scout_instrument_#{_sanitize_name(name)}"
       end
 
       # Method names like +any?+ or +replace!+ contain a trailing character that would break when

data/lib/scout_apm/tracked_request.rb

@@ -0,0 +1,168 @@
+# A TrackedRequest is a stack of layers, where completed layers (go into, then
+# come out of a layer) are forgotten as they finish. Layers are attached to
+# their children as the process goes, building a tree structure within the
+# layer objects. When the last layer is finished (hence the whole request is
+# finished) it hands the root layer off to be recorded.
+
+module ScoutApm
+  class TrackedRequest
+    # Context is application defined extra information.  (ie, which user, what
+    # is their email/ip, what plan are they on, what locale are they using,
+    # etc) See documentation for examples on how to set this from a
+    # before_filter
+    attr_reader :context
+
+    # The first layer registered with this request. All other layers will be
+    # children of this layer.
+    attr_reader :root_layer
+
+    # As we go through a request, instrumentation can mark more general data into the Request
+    # Known Keys:
+    #   :uri - the full URI requested by the user
+    attr_reader :annotations
+
+    # Nil until the request is finalized, at which point it will hold the
+    # entire raw stackprof output for this request
+    attr_reader :stackprof
+
+    # Headers as recorded by rails
+    # Can be nil if we never reach a Rails Controller
+    attr_reader :headers
+
+    def initialize
+      @layers = []
+      @annotations = {}
+      @ignoring_children = false
+      @context = Context.new
+      @root_layer = nil
+      @stackprof = nil
+      @error = false
+    end
+
+    def start_layer(layer)
+      start_request(layer) unless @layer
+
+      # ScoutApm::Agent.instance.logger.info("Starting Layer: #{layer.to_s}")
+      @layers[-1].add_child(layer) if @layers.any?
+      @layers.push(layer)
+    end
+
+    def stop_layer
+      layer = @layers.pop
+      layer.record_stop_time!
+
+      # Do this here, rather than in the layer because we need this caller. Maybe able to move it?
+      if layer.total_exclusive_time > ScoutApm::SlowTransaction::BACKTRACE_THRESHOLD
+        layer.store_backtrace(ScoutApm::SlowTransaction.backtrace_parser(caller))
+      end
+
+      if finalized?
+        stop_request
+      end
+    end
+
+    ###################################
+    # Request Lifecycle
+    ###################################
+
+    # Are we finished with this request?
+    # We're done if we have no layers left after popping one off
+    def finalized?
+      @layers.none?
+    end
+
+    # Run at the beginning of the whole request
+    #
+    # * Capture the first layer as the root_layer
+    # * Start Stackprof
+    def start_request(layer)
+      @root_layer = layer unless @root_layer # capture root layer
+      StackProf.start(:mode => :wall, :interval => ScoutApm::Agent.instance.config.value("stackprof_interval"))
+    end
+
+    # Run at the end of the whole request
+    #
+    # * Collect stackprof info
+    # * Send the request off to be stored
+    def stop_request
+      StackProf.stop
+      @stackprof = StackProf.results
+
+      record!
+    end
+
+    ###################################
+    # Annotations
+    ###################################
+
+    # As we learn things about this request, we can add data here.
+    # For instance, when we know where Rails routed this request to, we can store that scope info.
+    # Or as soon as we know which URI it was directed at, we can store that.
+    #
+    # This data is internal to ScoutApm, to add custom information, use the Context api.
+    def annotate_request(hsh)
+      @annotations.merge!(hsh)
+    end
+
+    # This request had an exception.  Mark it down as an error
+    def error!
+      @error = true
+    end
+
+    def error?
+      @error
+    end
+
+    def set_headers(headers)
+      @headers = headers
+    end
+
+    ###################################
+    # Persist the Request
+    ###################################
+
+    # Convert this request to the appropriate structure, then report it into
+    # the peristent Store object
+    def record!
+      @recorded = true
+
+      metrics = LayerMetricConverter.new(self).call
+      ScoutApm::Agent.instance.store.track!(metrics)
+
+      slow = LayerSlowTransactionConverter.new(self).call
+      ScoutApm::Agent.instance.store.track_slow_transaction!(slow)
+
+      error_metrics = LayerErrorConverter.new(self).call
+      ScoutApm::Agent.instance.store.track!(error_metrics)
+
+      queue_time_metrics = RequestQueueTime.new(self).call
+      ScoutApm::Agent.instance.store.track!(queue_time_metrics)
+    end
+
+    # Have we already persisted this request?
+    # Used to know when we should just create a new one (don't attempt to add
+    # data to an already-recorded request). See RequestManager
+    def recorded?
+      @recorded
+    end
+
+    ###################################
+    # Ignoring Children
+    ###################################
+
+    # Enable this when you would otherwise double track something interesting.
+    # This came up when we implemented InfluxDB instrumentation, which is more
+    # specific, and useful than the fact that InfluxDB happens to use Net::HTTP
+    # internally
+    #
+    # When enabled, new layers won't be added to the current Request.
+
+    def ignore_children!
+      @ignoring_children = true
+    end
+
+    def acknowledge_children!
+      @ignoring_children = false
+    end
+  end
+end

data/lib/scout_apm/version.rb

@@ -1,4 +1,4 @@
 module ScoutApm
-  VERSION = "1.1.0.pre1"
+  VERSION = "1.2.0.pre1"
 end
 

data/lib/scout_apm.rb

@@ -10,8 +10,9 @@ require 'net/http'
 require 'openssl'
 require 'set'
 require 'socket'
-require 'yaml'
 require 'thread'
+require 'time'
+require 'yaml'
 
 #####################################
 # Gem Requires
@@ -27,6 +28,12 @@ end
 #####################################
 require 'scout_apm/version'
 
+require 'scout_apm/tracked_request'
+require 'scout_apm/layer'
+require 'scout_apm/request_manager'
+require 'scout_apm/layer_converter'
+require 'scout_apm/request_queue_time'
+
 require 'scout_apm/server_integrations/passenger'
 require 'scout_apm/server_integrations/puma'
 require 'scout_apm/server_integrations/rainbows'
@@ -35,6 +42,9 @@ require 'scout_apm/server_integrations/unicorn'
 require 'scout_apm/server_integrations/webrick'
 require 'scout_apm/server_integrations/null'
 
+require 'scout_apm/background_job_integrations/sidekiq'
+require 'scout_apm/background_job_integrations/delayed_job'
+
 require 'scout_apm/framework_integrations/rails_2'
 require 'scout_apm/framework_integrations/rails_3_or_4'
 require 'scout_apm/framework_integrations/sinatra'
@@ -50,9 +60,11 @@ require 'scout_apm/deploy_integrations/capistrano_3'
 require 'scout_apm/instruments/net_http'
 require 'scout_apm/instruments/moped'
 require 'scout_apm/instruments/mongoid'
+require 'scout_apm/instruments/delayed_job'
 require 'scout_apm/instruments/active_record'
 require 'scout_apm/instruments/action_controller_rails_2'
 require 'scout_apm/instruments/action_controller_rails_3'
+require 'scout_apm/instruments/middleware'
 require 'scout_apm/instruments/sinatra'
 require 'scout_apm/instruments/process/process_cpu'
 require 'scout_apm/instruments/process/process_memory'
@@ -98,13 +110,14 @@ if defined?(Rails) && defined?(Rails::VERSION) && defined?(Rails::VERSION::MAJOR
   module ScoutApm
     class Railtie < Rails::Railtie
       initializer "scout_apm.start" do |app|
+        # attempt to start on first-request if not otherwise started, which is
+        # a good catch-all for Webrick, and Passenger and similar, where we
+        # can't detect the running app server until actual requests come in.
+        app.middleware.use ScoutApm::Middleware
+
         # Attempt to start right away, this will work best for preloading apps, Unicorn & Puma & similar
         ScoutApm::Agent.instance.start
 
-        # And attempt to start on first-request, which is a good catch-all for
-        # Webrick, and Passenger and similar, where we can't detect the running app server
-        # until actual requests come in.
-        app.middleware.use ScoutApm::Middleware
       end
     end
   end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: scout_apm
 version: !ruby/object:Gem::Version
-  version: 1.1.0.pre1
+  version: 1.2.0.pre1
 platform: ruby
 authors:
 - Derek Haynes
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-12-02 00:00:00.000000000 Z
+date: 2015-12-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: minitest
@@ -72,6 +72,8 @@ files:
 - lib/scout_apm/agent/reporting.rb
 - lib/scout_apm/app_server_load.rb
 - lib/scout_apm/attribute_arranger.rb
+- lib/scout_apm/background_job_integrations/delayed_job.rb
+- lib/scout_apm/background_job_integrations/sidekiq.rb
 - lib/scout_apm/background_worker.rb
 - lib/scout_apm/bucket_name_splitter.rb
 - lib/scout_apm/capacity.rb
@@ -89,6 +91,8 @@ files:
 - lib/scout_apm/instruments/action_controller_rails_2.rb
 - lib/scout_apm/instruments/action_controller_rails_3.rb
 - lib/scout_apm/instruments/active_record.rb
+- lib/scout_apm/instruments/delayed_job.rb
+- lib/scout_apm/instruments/middleware.rb
 - lib/scout_apm/instruments/mongoid.rb
 - lib/scout_apm/instruments/moped.rb
 - lib/scout_apm/instruments/net_http.rb
@@ -97,6 +101,8 @@ files:
 - lib/scout_apm/instruments/sinatra.rb
 - lib/scout_apm/layaway.rb
 - lib/scout_apm/layaway_file.rb
+- lib/scout_apm/layer.rb
+- lib/scout_apm/layer_converter.rb
 - lib/scout_apm/metric_meta.rb
 - lib/scout_apm/metric_stats.rb
 - lib/scout_apm/middleware.rb
@@ -104,6 +110,8 @@ files:
 - lib/scout_apm/platform_integrations/heroku.rb
 - lib/scout_apm/platform_integrations/server.rb
 - lib/scout_apm/reporter.rb
+- lib/scout_apm/request_manager.rb
+- lib/scout_apm/request_queue_time.rb
 - lib/scout_apm/serializers/app_server_load_serializer.rb
 - lib/scout_apm/serializers/deploy_serializer.rb
 - lib/scout_apm/serializers/directive_serializer.rb
@@ -121,6 +129,7 @@ files:
 - lib/scout_apm/stackprof_tree_collapser.rb
 - lib/scout_apm/store.rb
 - lib/scout_apm/tracer.rb
+- lib/scout_apm/tracked_request.rb
 - lib/scout_apm/utils/fake_stack_prof.rb
 - lib/scout_apm/utils/installed_gems.rb
 - lib/scout_apm/utils/null_logger.rb