traceview 3.2.1-java → 3.3.0-java

data/lib/traceview.rb

@@ -57,6 +57,8 @@ begin
   # it will load all of the Ruby instrumentation
   require 'traceview/ruby'
   require 'oboe/backward_compatibility'
+
+  require 'traceview/test' if ENV['TRACEVIEW_GEM_TEST']
 rescue => e
   $stderr.puts "[traceview/error] Problem loading: #{e.inspect}"
   $stderr.puts e.backtrace

data/lib/traceview/api/layerinit.rb

@@ -11,11 +11,13 @@ module TraceView
       # layer.
       #
       def report_init(layer = 'rack')
-        # Don't send __Init in development or test
-        return if %w(development test).include? ENV['RACK_ENV']
-
-        # Don't send __Init if the c-extension hasn't loaded
-        return unless TraceView.loaded
+        # Don't send __Init in development, test or if the gem
+        # isn't fully loaded (e.g. missing c-extension)
+        if %w(development test).include? ENV['RACK_ENV'] ||
+                          ENV.key?('TRACEVIEW_GEM_TEST') ||
+                          !TraceView.loaded
+          return
+        end
 
         platform_info = TraceView::Util.build_init_report
 

data/lib/traceview/api/logging.rb

@@ -1,6 +1,13 @@
 # Copyright (c) 2013 AppNeta, Inc.
 # All rights reserved.
 
+# Make sure Set is loaded if possible.
+begin
+  require 'set'
+rescue LoadError
+  class Set; end
+end
+
 module TraceView
   module API
     ##
@@ -105,7 +112,7 @@ module TraceView
           # sample rate and sample source
           opts[:SampleRate]        = TraceView.sample_rate
           opts[:SampleSource]      = TraceView.sample_source
-          opts[:TraceOrigin] = :always_sampled
+          opts[:TraceOrigin]       = :always_sampled
 
           log_event(layer, 'entry', TraceView::Context.startTrace, opts)
         end

data/lib/traceview/config.rb

@@ -14,8 +14,8 @@ module TraceView
     @@instrumentation = [:action_controller, :action_view, :active_record,
                          :cassandra, :curb, :dalli, :em_http_request, :excon, :faraday,
                          :grape, :httpclient, :nethttp, :memcached, :memcache, :mongo,
-                         :moped, :rack, :redis, :resque, :rest_client, :sequel,
-                         :typhoeus]
+                         :moped, :rack, :redis, :resque, :rest_client, :sequel, :sidekiqclient,
+                         :sidekiqworker, :typhoeus]
 
     # Subgrouping of instrumentation
     @@http_clients = [:curb, :excon, :em_http_request, :faraday, :httpclient, :nethttp, :rest_client, :typhoeus]
@@ -60,6 +60,8 @@ module TraceView
       TraceView::Config[:resque][:collect_backtraces] = true
       TraceView::Config[:rest_client][:collect_backtraces] = false
       TraceView::Config[:sequel][:collect_backtraces] = true
+      TraceView::Config[:sidekiqclient][:collect_backtraces] = false
+      TraceView::Config[:sidekiqworker][:collect_backtraces] = false
       TraceView::Config[:typhoeus][:collect_backtraces] = false
 
       # Special instrument specific flags
@@ -214,6 +216,9 @@ module TraceView
         @@config[key.to_sym] = value.to_i
         TraceView.set_sample_rate(value) if TraceView.loaded
 
+      elsif key == :action_blacklist
+        TraceView.logger.warn "[traceview/deprecation] :action_blacklist will be deprecated in a future version."
+
       elsif key == :include_url_query_params
         # Obey the global flag and update all of the per instrumentation
         # <tt>:log_args</tt> values.

data/lib/traceview/inst/sidekiq-client.rb

@@ -0,0 +1,54 @@
+module TraceView
+  class SidekiqClient
+    def collect_kvs(args)
+      begin
+        # Attempt to collect up pertinent info.  If we hit something unexpected,
+        # keep calm and instrument on.
+
+        report_kvs = {}
+        worker_class, msg, queue, _ = args
+
+        report_kvs[:Spec]      = :msgclient
+        report_kvs[:Flavor]    = :sidekiq
+        report_kvs[:Queue]     = queue
+        report_kvs[:Retry]     = msg['retry']
+        report_kvs[:JobName]   = worker_class
+        report_kvs[:MsgID]     = msg['jid']
+        report_kvs[:Args]      = msg['args'].to_s[0..1024] if TV::Config[:sidekiqclient][:log_args]
+        report_kvs[:Backtrace] = TV::API.backtrace         if TV::Config[:sidekiqclient][:collect_backtraces]
+      rescue => e
+        TraceView.logger.warn "[traceview/sidekiq] Non-fatal error capturing KVs: #{e.message}"
+      end
+      report_kvs
+    end
+
+    def call(*args)
+      # args: 0: worker_class, 1: msg, 2: queue, 3: redis_pool
+
+      result = nil
+      report_kvs = collect_kvs(args)
+
+      TraceView::API.log_entry('sidekiq-client', report_kvs)
+      args[1]['X-Trace'] = TraceView::Context.toString if TraceView.tracing?
+
+      result = yield
+
+      report_kvs = { :JobID => result["jid"] }
+      result
+    rescue => e
+      TraceView::API.log_exception('sidekiq-client', e, report_kvs)
+      raise
+    ensure
+      TraceView::API.log_exit('sidekiq-client', report_kvs)
+    end
+  end
+end
+
+if defined?(::Sidekiq) && RUBY_VERSION >= '2.0' && TraceView::Config[:sidekiqclient][:enabled]
+  ::Sidekiq.configure_client do |config|
+    config.client_middleware do |chain|
+      ::TraceView.logger.info '[traceview/loading] Adding Sidekiq client middleware' if TraceView::Config[:verbose]
+      chain.add ::TraceView::SidekiqClient
+    end
+  end
+end

data/lib/traceview/inst/sidekiq-worker.rb

@@ -0,0 +1,61 @@
+module TraceView
+  class SidekiqWorker
+    def collect_kvs(args)
+      begin
+        # Attempt to collect up pertinent info.  If we hit something unexpected,
+        # keep calm and instrument on.
+        report_kvs = {}
+        worker, msg, queue = args
+
+        # Background Job Spec KVs
+        report_kvs[:Spec] = :job
+        report_kvs[:Flavor]    = :sidekiq
+        report_kvs[:Queue]     = queue
+        report_kvs[:Retry]     = msg['retry']
+        report_kvs[:JobName]   = worker.class.to_s
+        report_kvs[:MsgID]     = msg['jid']
+        report_kvs[:Args]      = msg['args'].to_s[0..1024] if TV::Config[:sidekiqworker][:log_args]
+        report_kvs['Backtrace'] = TV::API.backtrace        if TV::Config[:sidekiqworker][:collect_backtraces]
+
+        # Webserver Spec KVs
+        report_kvs['HTTP-Host'] = Socket.gethostname
+        report_kvs[:Controller] = "Sidekiq_#{queue}"
+        report_kvs[:Action] = msg['class']
+        report_kvs[:URL] = "/sidekiq/#{queue}/#{msg['class'].to_s}"
+      rescue => e
+        TraceView.logger.warn "[traceview/sidekiq] Non-fatal error capturing KVs: #{e.message}"
+      end
+      report_kvs
+    end
+
+    def call(*args)
+      # args: 0: worker, 1: msg, 2: queue
+      result = nil
+      report_kvs = collect_kvs(args)
+
+      # Continue the trace from the enqueue side?
+      incoming_context = nil
+      if args[1].is_a?(Hash) && TraceView::XTrace.valid?(args[1]['X-Trace'])
+        incoming_context = args[1]['X-Trace']
+        report_kvs[:Async] = true
+      end
+
+      result = TraceView::API.start_trace('sidekiq-worker', incoming_context, report_kvs) do
+        yield
+      end
+
+      result[0]
+    end
+  end
+end
+
+if defined?(::Sidekiq) && RUBY_VERSION >= '2.0' && TraceView::Config[:sidekiqworker][:enabled]
+  ::TraceView.logger.info '[traceview/loading] Instrumenting sidekiq' if TraceView::Config[:verbose]
+
+  ::Sidekiq.configure_server do |config|
+    config.server_middleware do |chain|
+      ::TraceView.logger.info '[traceview/loading] Adding Sidekiq worker middleware' if TraceView::Config[:verbose]
+      chain.add ::TraceView::SidekiqWorker
+    end
+  end
+end

data/lib/traceview/test.rb

@@ -0,0 +1,20 @@
+# Copyright (c) 2015 AppNeta, Inc.
+# All rights reserved.
+
+module TraceView
+  module Test
+    class << self
+      def load_extras
+        # If we're using the libraries gemfile (with sidekiq and resque)
+        if File.basename(ENV['BUNDLE_GEMFILE']) =~ /libraries/
+          # Load all of the test workers
+          pattern = File.join(File.dirname(__FILE__), '../../test/jobs/', '*.rb')
+          Dir.glob(pattern) do |f|
+            TV.logger.debug "Loading test job file: #{f}"
+            require f
+          end
+        end
+      end
+    end
+  end
+end

data/lib/traceview/util.rb

@@ -215,6 +215,7 @@ module TraceView
           platform_info['Ruby.Redis.Version']      = "Redis-#{::Redis::VERSION}"           if defined?(::Redis::VERSION)
           platform_info['Ruby.Resque.Version']     = "Resque-#{::Resque::VERSION}"         if defined?(::Resque::VERSION)
           platform_info['Ruby.RestClient.Version'] = "RestClient-#{::RestClient::VERSION}" if defined?(::RestClient::VERSION)
+          platform_info['Ruby.Sidekiq.Version']    = "Sidekiq-#{::Sidekiq::VERSION}"       if defined?(::Sidekiq::VERSION)
           platform_info['Ruby.Typhoeus.Version']   = "Typhoeus-#{::Typhoeus::VERSION}"     if defined?(::Typhoeus::VERSION)
 
           # Special case since the Mongo 1.x driver doesn't embed the version number in the gem directly

data/lib/traceview/version.rb

@@ -7,8 +7,8 @@ module TraceView
   # traceview.gemspec during gem build process
   module Version
     MAJOR = 3
-    MINOR = 2
-    PATCH = 1
+    MINOR = 3
+    PATCH = 0
     BUILD = nil
 
     STRING = [MAJOR, MINOR, PATCH, BUILD].compact.join('.')

data/test/instrumentation/curb_test.rb

@@ -1,7 +1,7 @@
 # Copyright (c) 2015 AppNeta, Inc.
 # All rights reserved.
 
-if RUBY_VERSION > '1.8.7'
+if RUBY_VERSION > '1.8.7' && !defined?(JRUBY_VERSION)
 
   require 'minitest_helper'
   require 'traceview/inst/rack'

data/test/instrumentation/sidekiq-client_test.rb

@@ -0,0 +1,131 @@
+# Copyright (c) 2015 AppNeta, Inc.
+# All rights reserved.
+
+if RUBY_VERSION >= '2.0'
+  require 'minitest_helper'
+  require 'sidekiq'
+  require_relative "../jobs/remote_call_worker_job"
+  require_relative "../jobs/db_worker_job"
+  require_relative "../jobs/error_worker_job"
+
+  class SidekiqClientTest < Minitest::Test
+    def setup
+      clear_all_traces
+      @collect_backtraces = TraceView::Config[:sidekiqclient][:collect_backtraces]
+      @log_args = TraceView::Config[:sidekiqclient][:log_args]
+    end
+
+    def teardown
+      TraceView::Config[:sidekiqclient][:collect_backtraces] = @collect_backtraces
+      TraceView::Config[:sidekiqclient][:log_args] = @log_args
+    end
+
+    def test_enqueue
+      # Queue up a job to be run
+      jid, _ = ::TraceView::API.start_trace(:enqueue_test) do
+        Sidekiq::Client.push('queue' => 'critical', 'class' => ::RemoteCallWorkerJob, 'args' => [1, 2, 3], 'retry' => false)
+      end
+
+      # Allow the job to be run
+      sleep 5
+
+      traces = get_all_traces
+      assert_equal 23, traces.count, "Trace count"
+      valid_edges?(traces)
+
+      assert_equal 'sidekiq-client',       traces[1]['Layer']
+      assert_equal 'entry',                traces[1]['Label']
+
+      assert_equal 'msgclient',            traces[1]['Spec']
+      assert_equal 'sidekiq',              traces[1]['Flavor']
+      assert_equal 'critical',             traces[1]['Queue']
+      assert_equal jid,                    traces[1]['MsgID']
+      assert_equal '[1, 2, 3]',            traces[1]['Args']
+      assert_equal "RemoteCallWorkerJob",  traces[1]['JobName']
+      assert_equal 'false',                traces[1]['Retry']
+      assert_equal false,                  traces[1].key?('Backtrace')
+
+      assert_equal 'sidekiq-client',       traces[2]['Layer']
+      assert_equal 'exit',                 traces[2]['Label']
+    end
+
+    def test_collect_backtraces_default_value
+      assert_equal TV::Config[:sidekiqclient][:collect_backtraces], false, "default backtrace collection"
+    end
+
+    def test_log_args_default_value
+      assert_equal TV::Config[:sidekiqclient][:log_args], true, "log_args default "
+    end
+
+    def test_obey_collect_backtraces_when_false
+      TraceView::Config[:sidekiqclient][:collect_backtraces] = false
+
+      # Queue up a job to be run
+      ::TraceView::API.start_trace(:enqueue_test) do
+        Sidekiq::Client.push('queue' => 'critical', 'class' => ::RemoteCallWorkerJob, 'args' => [1, 2, 3], 'retry' => false)
+      end
+
+      # Allow the job to be run
+      sleep 5
+
+      traces = get_all_traces
+      assert_equal 23, traces.count, "Trace count"
+      valid_edges?(traces)
+      assert_equal 'sidekiq-client',   traces[1]['Layer']
+      assert_equal false,              traces[1].key?('Backtrace')
+    end
+
+    def test_obey_collect_backtraces_when_true
+      TraceView::Config[:sidekiqclient][:collect_backtraces] = true
+
+      # Queue up a job to be run
+      ::TraceView::API.start_trace(:enqueue_test) do
+        Sidekiq::Client.push('queue' => 'critical', 'class' => ::RemoteCallWorkerJob, 'args' => [1, 2, 3], 'retry' => false)
+      end
+
+      # Allow the job to be run
+      sleep 5
+
+      traces = get_all_traces
+      assert_equal 23, traces.count, "Trace count"
+      valid_edges?(traces)
+      assert_equal 'sidekiq-client',   traces[1]['Layer']
+      assert_equal true,               traces[1].key?('Backtrace')
+    end
+
+    def test_obey_log_args_when_false
+      TraceView::Config[:sidekiqclient][:log_args] = false
+
+      # Queue up a job to be run
+      ::TraceView::API.start_trace(:enqueue_test) do
+        Sidekiq::Client.push('queue' => 'critical', 'class' => ::RemoteCallWorkerJob, 'args' => [1, 2, 3], 'retry' => false)
+      end
+
+      # Allow the job to be run
+      sleep 5
+
+      traces = get_all_traces
+      assert_equal 23, traces.count, "Trace count"
+      valid_edges?(traces)
+      assert_equal false, traces[1].key?('Args')
+    end
+
+    def test_obey_log_args_when_true
+      TraceView::Config[:sidekiqclient][:log_args] = true
+
+      # Queue up a job to be run
+      ::TraceView::API.start_trace(:enqueue_test) do
+        Sidekiq::Client.push('queue' => 'critical', 'class' => ::RemoteCallWorkerJob, 'args' => [1, 2, 3], 'retry' => false)
+      end
+
+      # Allow the job to be run
+      sleep 5
+
+      traces = get_all_traces
+      assert_equal 23, traces.count, "Trace count"
+      valid_edges?(traces)
+      assert_equal true,         traces[1].key?('Args')
+      assert_equal '[1, 2, 3]',  traces[1]['Args']
+    end
+  end
+end

data/test/instrumentation/sidekiq-worker_test.rb

@@ -0,0 +1,180 @@
+# Copyright (c) 2015 AppNeta, Inc.
+# All rights reserved.
+
+if RUBY_VERSION >= '2.0'
+  require 'minitest_helper'
+  require 'sidekiq'
+  require_relative "../jobs/remote_call_worker_job"
+  require_relative "../jobs/db_worker_job"
+  require_relative "../jobs/error_worker_job"
+
+  class SidekiqWorkerTest < Minitest::Test
+    def setup
+      clear_all_traces
+      @collect_backtraces = TraceView::Config[:sidekiqworker][:collect_backtraces]
+      @log_args = TraceView::Config[:sidekiqworker][:log_args]
+    end
+
+    def teardown
+      TraceView::Config[:sidekiqworker][:collect_backtraces] = @collect_backtraces
+      TraceView::Config[:sidekiqworker][:log_args] = @log_args
+    end
+
+    def test_reports_version_init
+      init_kvs = ::TraceView::Util.build_init_report
+      assert init_kvs.key?('Ruby.Sidekiq.Version')
+      assert_equal "Sidekiq-#{::Sidekiq::VERSION}", init_kvs['Ruby.Sidekiq.Version']
+    end
+
+    def test_job_run
+      # Queue up a job to be run
+      jid = Sidekiq::Client.push('queue' => 'critical', 'class' => RemoteCallWorkerJob, 'args' => [1, 2, 3], 'retry' => false)
+
+      # Allow the job to be run
+      sleep 5
+
+      traces = get_all_traces
+      assert_equal 17, traces.count, "Trace count"
+      validate_outer_layers(traces, "sidekiq-worker")
+      valid_edges?(traces)
+
+      # Validate entry layer KVs
+      assert_equal 'always_sampled', traces[0]['TraceOrigin'],       "is always_sampled"
+      assert_equal true,             traces[0].key?('SampleRate'),   "has SampleRate KV"
+      assert_equal true,             traces[0].key?('SampleSource'), "has SampleSource KV"
+
+      # Validate Webserver Spec KVs
+      assert_equal Socket.gethostname,    traces[0]['HTTP-Host']
+      assert_equal "Sidekiq_critical",    traces[0]['Controller']
+      assert_equal "RemoteCallWorkerJob", traces[0]['Action']
+      assert_equal "/sidekiq/critical/RemoteCallWorkerJob", traces[0]['URL']
+
+      # Validate Job Spec KVs
+      assert_equal "job",                 traces[0]['Spec']
+      assert_equal 'sidekiq',             traces[0]['Flavor']
+      assert_equal "RemoteCallWorkerJob", traces[0]['JobName']
+      assert_equal jid,                   traces[0]['MsgID']
+      assert_equal "critical",            traces[0]['Queue']
+      assert_equal "[1, 2, 3]",           traces[0]['Args']
+      assert_equal "false",               traces[0]['Retry']
+
+      assert_equal false,                 traces[0].key?('Backtrace')
+      assert_equal "net-http",            traces[4]['Layer']
+      assert_equal "entry",               traces[4]['Label']
+      assert_equal "memcache",            traces[15]['Layer']
+    end
+
+    def test_jobs_with_errors
+      # Queue up a job to be run
+      jid = Sidekiq::Client.push('queue' => 'critical', 'class' => ErrorWorkerJob, 'args' => [1, 2, 3], 'retry' => false)
+
+      # Allow the job to be run
+      sleep 5
+
+      traces = get_all_traces
+      assert_equal 3, traces.count, "Trace count"
+      validate_outer_layers(traces, "sidekiq-worker")
+      valid_edges?(traces)
+
+      # Validate Webserver Spec KVs
+      assert_equal Socket.gethostname,                  traces[0]['HTTP-Host']
+      assert_equal "Sidekiq_critical",                  traces[0]['Controller']
+      assert_equal "ErrorWorkerJob",                    traces[0]['Action']
+      assert_equal "/sidekiq/critical/ErrorWorkerJob",  traces[0]['URL']
+
+      # Validate Job Spec KVs
+      assert_equal "job",             traces[0]['Spec']
+      assert_equal 'sidekiq',         traces[0]['Flavor']
+      assert_equal "ErrorWorkerJob",  traces[0]['JobName']
+      assert_equal jid,               traces[0]['MsgID']
+      assert_equal "critical",        traces[0]['Queue']
+      assert_equal "[1, 2, 3]",       traces[0]['Args']
+      assert_equal "false",           traces[0]['Retry']
+
+      assert_equal traces[1]['Layer'], 'sidekiq-worker'
+      assert_equal traces[1]['Label'], 'error'
+      assert_equal traces[1]['ErrorClass'], "RuntimeError"
+      assert traces[1].key?('ErrorMsg')
+      assert traces[1].key?('Backtrace')
+    end
+
+    def test_collect_backtraces_default_value
+      assert_equal TV::Config[:sidekiqworker][:collect_backtraces], false, "default backtrace collection"
+    end
+
+    def test_log_args_default_value
+      assert_equal TV::Config[:sidekiqworker][:log_args], true, "log_args default "
+    end
+
+    def test_obey_collect_backtraces_when_false
+      TraceView::Config[:sidekiqworker][:collect_backtraces] = false
+
+      # Queue up a job to be run
+      Sidekiq::Client.push('queue' => 'critical', 'class' => ::RemoteCallWorkerJob, 'args' => [1, 2, 3], 'retry' => false)
+
+      # Allow the job to be run
+      sleep 5
+
+      traces = get_all_traces
+      assert_equal 17, traces.count, "Trace count"
+      valid_edges?(traces)
+      assert_equal 'sidekiq-worker',   traces[0]['Layer']
+      assert_equal false,              traces[0].key?('Backtrace')
+    end
+
+    def test_obey_collect_backtraces_when_true
+      # FIXME: This can't be tested with the current Sidekiq minitest integration (e.g. already booted
+      # sidekiq in a different process)
+      skip
+
+      TraceView::Config[:sidekiqworker][:collect_backtraces] = true
+
+      # Queue up a job to be run
+      Sidekiq::Client.push('queue' => 'critical', 'class' => ::RemoteCallWorkerJob, 'args' => [1, 2, 3], 'retry' => false)
+
+      # Allow the job to be run
+      sleep 5
+
+      traces = get_all_traces
+      assert_equal 17, traces.count, "Trace count"
+      valid_edges?(traces)
+      assert_equal 'sidekiq-worker',   traces[0]['Layer']
+      assert_equal true,               traces[0].key?('Backtrace')
+    end
+
+    def test_obey_log_args_when_false
+      # FIXME: This can't be tested with the current Sidekiq minitest integration (e.g. already booted
+      # sidekiq in a different process)
+      skip
+
+      TraceView::Config[:sidekiqworker][:log_args] = false
+
+      # Queue up a job to be run
+      Sidekiq::Client.push('queue' => 'critical', 'class' => ::RemoteCallWorkerJob, 'args' => [1, 2, 3], 'retry' => false)
+
+      # Allow the job to be run
+      sleep 5
+
+      traces = get_all_traces
+      assert_equal 17, traces.count, "Trace count"
+      valid_edges?(traces)
+      assert_equal false, traces[0].key?('Args')
+    end
+
+    def test_obey_log_args_when_true
+      TraceView::Config[:sidekiqworker][:log_args] = true
+
+      # Queue up a job to be run
+      Sidekiq::Client.push('queue' => 'critical', 'class' => ::RemoteCallWorkerJob, 'args' => [1, 2, 3], 'retry' => false)
+
+      # Allow the job to be run
+      sleep 5
+
+      traces = get_all_traces
+      assert_equal 17, traces.count, "Trace count"
+      valid_edges?(traces)
+      assert_equal true,         traces[0].key?('Args')
+      assert_equal '[1, 2, 3]',  traces[0]['Args']
+    end
+  end
+end