oboe 2.4.0.1 → 2.5.0.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 33bba2d752d8947638ef9b22ad51e4b5c947e8a8
-  data.tar.gz: b4f96bd122d67e99bf6647203b51097c49181e8c
+  metadata.gz: 812e394b380905601c8d1fbae241aaedce6231b4
+  data.tar.gz: 317bb57275d9eb6b23054d9d2e51dd0c8fbd64d6
 SHA512:
-  metadata.gz: 78319d63f067e929ac9e293be2770b9c820bf853ee713f301169a5d905a69a1d0edb8218288d6fa65cf9de749f9487642ad826974faae49fc06af8c5ca118bb1
-  data.tar.gz: a985bb58d63612bc8c93304ae3c3a5ea25e48a27c4a1856a945d3d919a4a3e556f68959d6fd2d0e8062246abe2d2ac51ba7cf3aa409442e0401e75122ab66cea
+  metadata.gz: af3e85508ec28a8a56e05ebed1f1ee6a90c8475ca258c9b9127ee81c8d33226331f4604f856426cb1aca24b90592f7d0ec22e2c6da0df4d47e3e30a0e96f4294
+  data.tar.gz: a8744c90e5d216672d774b094deadf1d1ccf031a25de3f262984c10d83593fb5908fdb05527671ee7a53f99b5719c9238f65cea6965c197861815cf482970d86

data/.travis.yml

@@ -1,5 +1,6 @@
 language: ruby
 rvm:
+  - 2.1.0
   - 2.0.0
   - 1.9.3
   - 1.9.2
@@ -29,4 +30,5 @@ services:
   - mongodb
   - memcached
   - cassandra
+  - redis
 

data/{CHANGELOG → CHANGELOG.md}

@@ -1,3 +1,12 @@
+# oboe 2.5.0.7 (02/13/14)
+
+* Added new Redis redis-rb gem (>= 3.0.0) instrumentation
+* Fix a SampleSource bitmask high bit issue
+* Expanded __Init reports
+* Fix Ruby standalone returning nil X-Trace headers (1B000000...)
+* Test against Ruby 2.1.0 on TravisCI
+* Fix errant Oboe::Config warning
+
 # oboe 2.4.0.1 (01/12/13)
 
 * Report SampleRate & SampleSource per updated SWIG API

data/Gemfile

@@ -15,7 +15,7 @@ end
 group :development do
   gem 'ruby-debug',   :platform => :mri_18
   gem 'ruby-debug19', :platform => :mri_19, :require => 'ruby-debug'
-  gem 'debugger',     :platform => :mri_20 
+  gem 'byebug',       :platform => :mri_20 
   gem 'perftools.rb', :platform => :mri,    :require => 'perftools'
   gem 'pry'
 end
@@ -29,4 +29,5 @@ gem 'mongo'
 gem 'bson_ext' # For Mongo, Yours Truly
 gem 'moped' unless (RUBY_VERSION =~ /^1.8/) == 0
 gem 'resque'
+gem 'redis'
 

data/README.md

@@ -76,9 +76,16 @@ The oboe gem has the ability to instrument any arbitrary Ruby application or scr
     Bundler.require
     
     require 'oboe'
+
+    # Tracing mode can be 'never', 'through' (to follow upstream) or 'always'
+    Oboe::Config[:tracing_mode] = 'always'
+
+    # Number of requests to trace out of each million
+    Oboe::Config[:sample_rate] = 1000000
+
     Oboe::Ruby.initialize
 
-From here, you can use the Tracing API to instrument areas of code (see below).
+From here, you can use the Tracing API to instrument areas of code using `Oboe::API.start_trace` (see below).  If you prefer to instead dive directly into code, take a look at [this example](https://gist.github.com/pglombardo/8550713) of an instrumented Ruby script.
 
 ## Other
 
@@ -88,13 +95,15 @@ You can send deploy notifications to TraceView and have the events show up on yo
 
 ## The Tracing API
 
-You can instrument any arbitrary block of code using the following pattern:
+You can instrument any arbitrary block of code using `Oboe::API.trace`:
 
     # layer_name will show up in the TraceView app dashboard
     layer_name = 'subsystemX'
 
-    # report_kvs are a set of information Key/Value pairs that are sent to TraceView along with the performance metrics.
-    # These KV pairs can report on request, environment or client specific information.
+    # report_kvs are a set of information Key/Value pairs that are sent to
+    # TraceView dashboard along with the performance metrics.  These KV 
+    # pairs are used to report request, environment and/or client specific
+    # information.
 
     report_kvs = {}
     report_kvs[:mykey] = @client.id
@@ -103,7 +112,13 @@ You can instrument any arbitrary block of code using the following pattern:
       # the block of code to be traced
     end
 
-Find more details in the TraceView [documentation portal](https://support.tv.appneta.com/support/solutions/articles/86395-ruby-instrumentation-public-api).
+`Oboe::API.trace` is used within the context of a request.  It will follow the upstream state of the request being traced.  i.e. the block of code will only be traced when the parent request is being traced.
+
+This tracing state of a request can also be queried by using `Oboe.tracing?`.
+
+If you need to instrument code outside the context of a request (such as a cron job, background job or an arbitrary ruby script), use `Oboe::API.start_trace` instead which will initiate new traces based on configuration and probability (based on the sample rate).
+
+Find more details in the [RubyDoc page](http://rdoc.info/gems/oboe/Oboe/API/Tracing) or in [this example](https://gist.github.com/pglombardo/8550713) on how to use the Tracing API in an independent Ruby script.
 
 ## Tracing Methods
 

data/lib/base.rb

@@ -14,13 +14,13 @@ OBOE_SAMPLE_RATE_SOURCE_DEFAULT_MISCONFIGURED  = 5
 OBOE_SAMPLE_RATE_SOURCE_OBOE_DEFAULT           = 6
 
 # Masks for bitwise ops
-ZERO_MASK = 0b00000000000000000000000000
+ZERO_MASK = 0b0000000000000000000000000000
 
-SAMPLE_RATE_MASK   = 0b00111111111111111111111111
-SAMPLE_SOURCE_MASK = 0b11000000000000000000000000
+SAMPLE_RATE_MASK   = 0b0000111111111111111111111111
+SAMPLE_SOURCE_MASK = 0b1111000000000000000000000000
 
-ZERO_SAMPLE_RATE_MASK   = 0b11000000000000000000000000
-ZERO_SAMPLE_SOURCE_MASK = 0b00111111111111111111111111
+ZERO_SAMPLE_RATE_MASK   = 0b1111000000000000000000000000
+ZERO_SAMPLE_SOURCE_MASK = 0b0000111111111111111111111111
 
 module OboeBase
   attr_accessor :reporter

data/lib/joboe_metal.rb

@@ -81,7 +81,11 @@ module Oboe_metal < MetalBase
           Oboe.reporter = Oboe::UdpReporter.new(Oboe::Config[:reporter_host])
         end
 
-        Oboe::API.report_init('rack') unless ["development", "test"].include? ENV['RACK_ENV']
+        # Only report __Init from here if we are not instrumenting a framework.
+        # Otherwise, frameworks will handle reporting __Init after full initialization
+        unless defined?(::Rails) or defined?(::Sinatra) or defined?(::Padrino)
+          Oboe::API.report_init('rack') unless ["development", "test"].include? ENV['RACK_ENV']
+        end
       
       rescue Exception => e
         $stderr.puts e.message

data/lib/oboe.rb

@@ -21,10 +21,13 @@ begin
       end
     rescue LoadError
       Oboe.loaded = false
-      $stderr.puts "=============================================================="
-      $stderr.puts "Missing TraceView libraries.  Tracing disabled."
-      $stderr.puts "See: https://support.tv.appneta.com/solution/articles/137973" 
-      $stderr.puts "=============================================================="
+
+      unless $ENV['RAILS_GROUP'] == 'assets'
+        $stderr.puts "=============================================================="
+        $stderr.puts "Missing TraceView libraries.  Tracing disabled."
+        $stderr.puts "See: https://support.tv.appneta.com/solution/articles/137973" 
+        $stderr.puts "=============================================================="
+      end
     end
   end
  

data/lib/oboe/api/layerinit.rb

@@ -28,6 +28,19 @@ module Oboe
           elsif defined?(::Sinatra)
             platform_info['Ruby.Sinatra.Version'] = "Sinatra-#{::Sinatra::VERSION}"
           end
+          
+          # Report the instrumented libraries 
+          platform_info['Ruby.Cassandra.Version'] = "Cassandra-#{::Cassandra.VERSION}" if defined?(::Cassandra)
+          platform_info['Ruby.Dalli.Version']     = "Dalli-#{::Dalli::VERSION}"        if defined?(::Dalli)
+          platform_info['Ruby.MemCache.Version']  = "MemCache-#{::MemCache::VERSION}"  if defined?(::MemCache)
+          platform_info['Ruby.Moped.Version']     = "Moped-#{::Moped::VERSION}"        if defined?(::Moped)
+          platform_info['Ruby.Redis.Version']     = "Redis-#{::Redis::VERSION}"        if defined?(::Redis)
+          platform_info['Ruby.Resque.Version']    = "Resque-#{::Resque::VERSION}"      if defined?(::Resque)
+         
+          # Special case since the Mongo 1.x driver doesn't embed the version number in the gem directly
+          if ::Gem.loaded_specs.has_key?('mongo')
+            platform_info['Ruby.Mongo.Version']     = "Mongo-#{::Gem.loaded_specs['mongo'].version.to_s}" 
+          end
 
           # Report the server in use (if possible)
           if defined?(::Unicorn)
@@ -44,10 +57,20 @@ module Oboe
             platform_info['Ruby.AppContainer.Version'] = "Mongrel2-#{::Mongrel2::VERSION}"
           elsif defined?(::Trinidad)
             platform_info['Ruby.AppContainer.Version'] = "Trinidad-#{::Trinidad::VERSION}"
+          elsif defined?(::WEBrick)
+            platform_info['Ruby.AppContainer.Version'] = "WEBrick-#{::WEBrick::VERSION}"
           else
             platform_info['Ruby.AppContainer.Version'] = "Unknown"
           end
-        rescue
+          
+          # If we couldn't load the c extension correctly, report the error to the dashboard.
+          unless Oboe.loaded
+            platform_info['Error'] = "Missing TraceView libraries.  Tracing disabled."
+          end
+        
+        rescue StandardError => e
+          Oboe.logger.debug "Error in layerinit: #{e.message}"
+          Oboe.logger.debug e.backtrace
         end
 
         start_trace(layer, nil, platform_info) { }

data/lib/oboe/config.rb

@@ -12,7 +12,7 @@ module Oboe
     @@config = {}
 
     @@instrumentation = [ :cassandra, :dalli, :nethttp, :memcached, :memcache, :mongo,
-                          :moped, :rack, :resque, :action_controller, :action_view, 
+                          :moped, :rack, :redis, :resque, :action_controller, :action_view, 
                           :active_record ]
 
     ##
@@ -42,6 +42,7 @@ module Oboe
       Oboe::Config[:mongo][:collect_backtraces] = true
       Oboe::Config[:moped][:collect_backtraces] = true
       Oboe::Config[:nethttp][:collect_backtraces] = true
+      Oboe::Config[:redis][:collect_backtraces] = false
       Oboe::Config[:resque][:collect_backtraces] = true
 
       # Special instrument specific flags
@@ -56,6 +57,9 @@ module Oboe
       # Setup an empty host blacklist (see: Oboe::API::Util.blacklisted?)
       @@config[:blacklist] = []
 
+      # Access Key is empty until loaded from config file or env var
+      @@config[:access_key] = ""
+
       # The oboe Ruby client has the ability to sanitize query literals
       # from SQL statements.  By default this is disabled.  Enable to
       # avoid collecting and reporting query literals to TraceView.

data/lib/oboe/frameworks/rails.rb

@@ -110,6 +110,7 @@ if defined?(::Rails)
           Oboe::Loading.load_access_key
           Oboe::Inst.load_instrumentation
           Oboe::Rails.load_instrumentation
+          Oboe::API.report_init('rack') unless ["development", "test"].include? ENV['RACK_ENV']
         end
       end
     end
@@ -125,6 +126,7 @@ if defined?(::Rails)
       Oboe::Inst.load_instrumentation
       Oboe::Rails.load_instrumentation
       Oboe::Rails.include_helpers
+      Oboe::API.report_init('rack') unless ["development", "test"].include? ENV['RACK_ENV']
     end
   end
 end

data/lib/oboe/inst/rack.rb

@@ -71,7 +71,7 @@ module Oboe
       xtrace = e.instance_variable_get(:@xtrace)
       raise
     ensure
-      result[1]['X-Trace'] = xtrace if xtrace
+      result[1]['X-Trace'] = xtrace if Oboe::XTrace.valid?(xtrace)
       return result
     end
   end

data/lib/oboe/inst/redis.rb

@@ -0,0 +1,273 @@
+# Copyright (c) 2014 AppNeta, Inc.
+# All rights reserved.
+
+module Oboe
+  module Inst
+    module Redis
+      module Client
+        # The operations listed in this constant skip collecting KVKey
+        NO_KEY_OPS = [ :keys, :randomkey, :scan, :sdiff, :sdiffstore, :sinter, 
+                       :sinterstore, :smove, :sunion, :sunionstore, :zinterstore,
+                       :zunionstore, :publish, :select, :eval, :evalsha, :script ]
+
+        # Instead of a giant switch statement, we use a hash constant to map out what
+        # KVs need to be collected for each of the many many Redis operations
+        # Hash formatting by undiagnosed OCD
+        KV_COLLECT_MAP = {
+          :brpoplpush  => { :destination  => 2 }, :rpoplpush   => { :destination  => 2 },
+          :sdiffstore  => { :destination  => 1 }, :sinterstore => { :destination  => 1 },
+          :sunionstore => { :destination  => 1 }, :zinterstore => { :destination  => 1 },
+          :zunionstore => { :destination  => 1 }, :publish     => { :channel      => 1 }, 
+          :incrby      => { :increment    => 2 }, :incrbyfloat => { :increment    => 2 },
+          :pexpire     => { :milliseconds => 2 }, :pexpireat   => { :milliseconds => 2 },
+          :expireat    => { :timestamp    => 2 }, :decrby      => { :decrement    => 2 },
+          :psetex      => { :ttl     => 2 },      :restore  => { :ttl     => 2 },
+          :setex       => { :ttl     => 2 },      :setnx    => { :ttl     => 2 },
+          :move        => { :db      => 2 },      :select   => { :db      => 1 },
+          :lindex      => { :index   => 2 },      :getset   => { :value   => 2 },
+          :keys        => { :pattern => 1 },      :expire   => { :seconds => 2 },
+          :rename      => { :newkey  => 2 },      :renamenx => { :newkey  => 2 },
+          :getbit      => { :offset  => 2 },      :setbit   => { :offset  => 2 },
+          :setrange    => { :offset  => 2 },      :evalsha  => { :sha     => 1 },   
+          :getrange    => { :start => 2, :end       => 3 },
+          :zrange      => { :start => 2, :end       => 3 },
+          :bitcount    => { :start => 2, :stop      => 3 },
+          :lrange      => { :start => 2, :stop      => 3 },
+          :zrevrange   => { :start => 2, :stop      => 3 },
+          :hincrby     => { :field => 2, :increment => 3 },
+          :smove           => { :source    => 1, :destination => 2 },
+          :bitop           => { :operation => 1, :destkey     => 2 },
+          :hincrbyfloat    => { :field     => 2, :increment   => 3 },
+          :zremrangebyrank => { :start     => 2, :stop        => 3 },
+        }
+            
+        # The following operations don't require any special handling. For these,
+        # we only collect KVKey and KVOp
+        #
+        # :append, :blpop, :brpop, :decr, :del, :dump, :exists, 
+        # :hgetall, :hkeys, :hlen, :hvals, :hmset, :incr, :linsert, 
+        # :llen, :lpop, :lpush, :lpushx, :lrem, :lset, :ltrim, 
+        # :persist, :pttl, :hscan, :rpop, :rpush, :rpushx, :sadd, 
+        # :scard, :sismember, :smembers, :strlen, :sort, :spop, 
+        # :srandmember, :srem, :sscan, :ttl, :type, :zadd, :zcard, 
+        # :zcount, :zincrby, :zrangebyscore, :zrank, :zrem, 
+        # :zremrangebyscore, :zrevrank, :zrevrangebyscore, :zscore
+        #
+        # For the operations in NO_KEY_OPS (above) we only collect
+        # KVOp (no KVKey)
+
+        def self.included(klass)
+          # We wrap two of the Redis methods to instrument
+          # operations
+          ::Oboe::Util.method_alias(klass, :call, ::Redis::Client)
+          ::Oboe::Util.method_alias(klass, :call_pipeline, ::Redis::Client)
+        end
+
+        # Given any Redis operation command array, this method
+        # extracts the Key/Values to report to the TraceView
+        # dashboard.
+        #
+        # @param command [Array] the Redis operation array
+        # @param r [Return] the return value from the operation
+        # @return [Hash] the Key/Values to report
+        def extract_trace_details(command, r)
+          kvs = {}
+          op = command.first
+
+          begin
+            kvs[:KVOp] = command[0]
+            kvs[:RemoteHost] = @options[:host]
+
+            unless NO_KEY_OPS.include? op or (command[1].is_a?(Array) and command[1].count > 1)
+              if command[1].is_a?(Array)
+                kvs[:KVKey] = command[1].first
+              else
+                kvs[:KVKey] = command[1]
+              end
+            end
+
+            if KV_COLLECT_MAP[op]
+              # Extract KVs from command for this op
+              KV_COLLECT_MAP[op].each { |k, v|
+                kvs[k] = command[v]
+              }
+            else
+              # This case statement handle special cases not handled
+              # by KV_COLLECT_MAP
+              case op
+              when :set
+                if command.count > 3
+                  options = command[3]
+                  kvs[:ex] = options[:ex] if options.has_key?(:ex)
+                  kvs[:px] = options[:px] if options.has_key?(:px)
+                  kvs[:nx] = options[:nx] if options.has_key?(:nx)
+                  kvs[:xx] = options[:xx] if options.has_key?(:xx)
+                end
+              
+              when :get
+                kvs[:KVHit] = r.nil? ? 0 : 1
+              
+              when :hdel, :hexists, :hget, :hset, :hsetnx
+                kvs[:field] = command[2] unless command[2].is_a?(Array)
+                if op == :hget
+                  kvs[:KVHit] = r.nil? ? 0 : 1
+                end
+
+              when :eval
+                if command[1].length > 1024
+                  kvs[:Script] = command[1][0..1023] + "(...snip...)"
+                else
+                  kvs[:Script] = command[1]
+                end
+              
+              when :script
+                kvs[:subcommand] = command[1]
+                kvs[:Backtrace] = Oboe::API.backtrace if Oboe::Config[:redis][:collect_backtraces]
+                if command[1] == "load"
+                  if command[1].length > 1024
+                    kvs[:Script] = command[2][0..1023] + "(...snip...)"
+                  else
+                    kvs[:Script] = command[2]
+                  end
+                elsif command[1] == :exists
+                  if command[2].is_a?(Array)
+                    kvs[:KVKey] = command[2].inspect
+                  else
+                    kvs[:KVKey] = command[2]
+                  end
+                end
+
+              when :mget
+                if command[1].is_a?(Array)
+                  kvs[:KVKeyCount] = command[1].count 
+                else
+                  kvs[:KVKeyCount] = command.count - 1
+                end 
+                values = r.select{ |i| i }
+                kvs[:KVHitCount] = values.count
+              
+              when :hmget
+                kvs[:KVKeyCount] = command.count - 2
+                values = r.select{ |i| i }
+                kvs[:KVHitCount] = values.count
+             
+              when :mset, :msetnx
+                if command[1].is_a?(Array)
+                  kvs[:KVKeyCount] = command[1].count / 2
+                else
+                  kvs[:KVKeyCount] = (command.count - 1) / 2
+                end
+              end # case op
+            end # if KV_COLLECT_MAP[op]
+            
+          rescue StandardError => e
+            Oboe.logger.debug "Error collecting redis KVs: #{e.message}"
+            Oboe.logger.debug e.backtrace.join("\n")
+          end
+
+          kvs
+        end
+
+        # Extracts the Key/Values to report from a pipelined
+        # call to the TraceView dashboard.
+        #
+        # @param pipeline [Redis::Pipeline] the Redis pipeline instance
+        # @return [Hash] the Key/Values to report
+        def extract_pipeline_details(pipeline)
+          kvs = {}
+
+          begin
+            kvs[:RemoteHost] = @options[:host]
+            kvs[:Backtrace] = Oboe::API.backtrace if Oboe::Config[:redis][:collect_backtraces]
+
+            command_count = pipeline.commands.count
+            kvs[:KVOpCount] = command_count
+
+            if pipeline.commands.first == :multi
+              kvs[:KVOp] = :multi
+            else
+              kvs[:KVOp] = :pipeline
+            end
+           
+            # Report pipelined operations  if the number
+            # of ops is reasonable
+            if command_count < 12
+              ops = []
+              pipeline.commands.each do |c|
+                ops << c.first
+              end
+              kvs[:KVOps] = ops.join(", ")
+            end
+          rescue StandardError => e
+            Oboe.logger.debug "[oboe/debug] Error extracting pipelined commands: #{e.message}"
+            Oboe.logger.debug e.backtrace
+          end
+          kvs
+        end
+
+        #
+        # The wrapper method for Redis::Client.call.  Here
+        # (when tracing) we capture KVs to report and pass 
+        # the call along
+        #
+        def call_with_oboe(command, &block)
+          if Oboe.tracing?
+            ::Oboe::API.log_entry('redis', {})
+
+            begin
+              r = call_without_oboe(command, &block)
+              report_kvs = extract_trace_details(command, r)
+              r
+            rescue StandardError => e
+              ::Oboe::API.log_exception('redis', e)
+              raise
+            ensure
+              ::Oboe::API.log_exit('redis', report_kvs)
+            end
+
+          else
+            call_without_oboe(command, &block)
+          end
+        end
+       
+        #
+        # The wrapper method for Redis::Client.call_pipeline.  Here
+        # (when tracing) we capture KVs to report and pass the call along
+        #
+        def call_pipeline_with_oboe(pipeline)
+          if Oboe.tracing?
+            report_kvs = {}
+
+            # Fall back to the raw tracing API so we can pass KVs
+            # back on exit (a limitation of the Oboe::API.trace 
+            # block method)  This removes the need for an info
+            # event to send additonal KVs
+            ::Oboe::API.log_entry('redis', {})
+
+            report_kvs = extract_pipeline_details(pipeline)
+
+            begin
+              call_pipeline_without_oboe(pipeline)
+            rescue StandardError => e
+              ::Oboe::API.log_exception('redis', e)
+              raise
+            ensure
+              ::Oboe::API.log_exit('redis', report_kvs)
+            end
+          else
+            call_pipeline_without_oboe(pipeline)
+          end
+        end
+
+      end
+    end
+  end
+end
+
+if Oboe::Config[:redis][:enabled] 
+  if defined?(::Redis) and Gem::Version.new(::Redis::VERSION) >= Gem::Version.new('3.0.0')
+    Oboe.logger.info "[oboe/loading] Instrumenting redis" if Oboe::Config[:verbose]
+    ::Oboe::Util.send_include(::Redis::Client, ::Oboe::Inst::Redis::Client)
+  end
+end
+