oboe 2.7.16.1-java → 2.7.17.1-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 11e535361d6ddf6ec6a8dca58b742b50f42ba114
-  data.tar.gz: ef8bc3bc820153012eba2de3fdbd3eb66c36f90c
+  metadata.gz: f5af7c88e442b978109bfd9e26216d22ba96ebbc
+  data.tar.gz: 87967943c1a3c45db5e9576a9dd629eb4207254c
 SHA512:
-  metadata.gz: 0c7b74535912fea2bb7e7a431b1c8e6800fb042f46b43201ee39e8abd202944783cb4a8cd0edf70bb17d578959a4f6e51b214528071306946def97ef771e0d8e
-  data.tar.gz: dde645f8bdc7320d53cfabe2b2fda1deb2dacd5d107f4e81017e688a2402b4f065b1a63823c555ef29d1e8114a10b1744f040087bc301b595d05c2eda127282c
+  metadata.gz: f534a642ed23073afa600719331797cfe7981d007918d18b172776e0a31783e260dc288bc3d7d8b507abae3abc51356f6581aa170af0cfa763bef5190fae9f8a
+  data.tar.gz: 2babaaa796835cc02dade64dd032ce6f154dbb491e25c54b7f34da1acaf604a06ccf9bac8a27eb8fa6ea0b97daf17999000ebf494e15046ae18c4c377e76f116

data/CHANGELOG.md

@@ -4,6 +4,18 @@ https://github.com/appneta/oboe-ruby/releases
 
 Dates in this file are in the format MM/DD/YYYY.
 
+# oboe 2.7.17.1
+
+This patch release includes:
+
+* New config option to optionally not report URL query parameters: #116
+* New HTTPClient instrumentation: #115
+
+Pushed to Rubygems:
+
+https://rubygems.org/gems/oboe/versions/2.7.17.1
+https://rubygems.org/gems/oboe/versions/2.7.17.1-java
+
 # oboe 2.7.16.1
 
 This patch release includes:

data/Gemfile

@@ -3,6 +3,7 @@ source 'https://rubygems.org'
 group :development, :test do
   gem 'minitest', "5.5.1"
   gem 'minitest-reporters'
+  gem 'minitest-debugger', :require => false
   gem 'rack-test'
   gem 'puma'
   if RUBY_VERSION < '1.9.3'
@@ -38,6 +39,7 @@ gem 'mongo'
 gem 'resque'
 gem 'redis'
 gem 'faraday'
+gem 'httpclient'
 gem 'excon'
 gem 'typhoeus'
 gem 'sequel'

data/lib/oboe/config.rb

@@ -13,9 +13,13 @@ module Oboe
 
     @@instrumentation = [:action_controller, :action_view, :active_record,
                          :cassandra, :dalli, :em_http_request, :excon, :faraday,
-                         :grape, :nethttp, :memcached, :memcache, :mongo,
+                         :grape, :httpclient, :nethttp, :memcached, :memcache, :mongo,
                          :moped, :rack, :redis, :resque, :rest_client, :sequel,
                          :typhoeus]
+
+    # Subgrouping of instrumentation
+    @@http_clients = [:excon, :faraday, :httpclient, :nethttp, :rest_client, :typhoeus]
+
     ##
     # Return the raw nested hash.
     #
@@ -41,10 +45,11 @@ module Oboe
       Oboe::Config[:action_view][:collect_backtraces] = true
       Oboe::Config[:cassandra][:collect_backtraces] = true
       Oboe::Config[:dalli][:collect_backtraces] = false
-      Oboe::Config[:faraday][:collect_backtraces] = false
-      Oboe::Config[:grape][:collect_backtraces] = true
       Oboe::Config[:em_http_request][:collect_backtraces] = false
       Oboe::Config[:excon][:collect_backtraces] = true
+      Oboe::Config[:faraday][:collect_backtraces] = false
+      Oboe::Config[:grape][:collect_backtraces] = true
+      Oboe::Config[:httpclient][:collect_backtraces] = true
       Oboe::Config[:memcache][:collect_backtraces] = false
       Oboe::Config[:memcached][:collect_backtraces] = false
       Oboe::Config[:mongo][:collect_backtraces] = true
@@ -71,6 +76,33 @@ module Oboe
       # Access Key is empty until loaded from config file or env var
       @@config[:access_key] = ''
 
+      # Logging of outgoing HTTP query args
+      #
+      # This optionally disables the logging of query args of outgoing
+      # HTTP clients such as Net::HTTP, excon, typhoeus and others.
+      #
+      # This flag is global to all HTTP client instrumentation.
+      #
+      # To configure this on a per instrumentation basis, set this
+      # option to true and instead disable the instrumenstation specific
+      # option <tt>log_args</tt>:
+      #
+      #   Oboe::Config[:nethttp][:log_args] = false
+      #   Oboe::Config[:excon][:log_args] = false
+      #   Oboe::Config[:typhoeus][:log_args] = true
+      #
+      @@config[:include_url_query_params] = true
+
+      # Logging of incoming HTTP query args
+      #
+      # This optionally disables the logging of incoming URL request
+      # query args.
+      #
+      # This flag is global and currently only affects the Rack
+      # instrumentation which reports incoming request URLs and
+      # query args by default.
+      @@config[:include_remote_url_params] = true
+
       # The oboe Ruby client has the ability to sanitize query literals
       # from SQL statements.  By default this is disabled.  Enable to
       # avoid collecting and reporting query literals to TraceView.
@@ -110,6 +142,7 @@ module Oboe
       # report all raised exception regardless.
       @@config[:report_rescued_errors] = false
 
+      # Environment support for OpenShift.
       if ENV.key?('OPENSHIFT_TRACEVIEW_TLYZER_IP')
         # We're running on OpenShift
         @@config[:tracing_mode] = 'always'
@@ -145,9 +178,8 @@ module Oboe
       if key == :sampling_rate
         Oboe.logger.warn 'sampling_rate is not a supported setting for Oboe::Config.  ' \
                          'Please use :sample_rate.'
-      end
 
-      if key == :sample_rate
+      elsif key == :sample_rate
         unless value.is_a?(Integer) || value.is_a?(Float)
           fail 'oboe :sample_rate must be a number between 1 and 1000000 (1m)'
         end
@@ -159,8 +191,19 @@ module Oboe
 
         # Assure value is an integer
         @@config[key.to_sym] = value.to_i
-
         Oboe.set_sample_rate(value) if Oboe.loaded
+
+      elsif key == :include_url_query_params
+        # Obey the global flag and update all of the per instrumentation
+        # <tt>:log_args</tt> values.
+        @@http_clients.each do |i|
+          @@config[i][:log_args] = value
+        end
+
+      elsif key == :include_remote_url_params
+        # Obey the global flag and update all of the per instrumentation
+        # <tt>:log_args</tt> values.
+        @@config[:rack][:log_args] = value
       end
 
       # Update liboboe if updating :tracing_mode
@@ -169,18 +212,24 @@ module Oboe
       end
     end
 
-    def self.instrumentation_list
-      @@instrumentation
-    end
-
     def self.method_missing(sym, *args)
+      class_var_name = "@@#{sym}"
+
       if sym.to_s =~ /(.+)=$/
         self[$1] = args.first
       else
-        unless @@config.key?(sym)
-          Oboe.logger.warn "[oboe/warn] Unknown method call on Oboe::Config: #{sym}"
+        # Try part of the @@config hash first
+        if @@config.key?(sym)
+          self[sym]
+
+        # Then try as a class variable
+        elsif self.class_variable_defined?(class_var_name.to_sym)
+          self.class_eval(class_var_name)
+
+        # Congrats - You've won a brand new nil...
+        else
+          nil
         end
-        self[sym]
       end
     end
   end

data/lib/oboe/inst/excon.rb

@@ -15,7 +15,8 @@ module Oboe
         kvs['RemoteProtocol'] = ::Oboe::Util.upcase(@data[:scheme])
         kvs['RemoteHost'] = @data[:host]
 
-        if @data[:query] && @data[:query].length
+        # Conditionally log query args
+        if Oboe::Config[:excon][:log_args] && (@data[:query] && @data[:query].length)
           kvs['ServiceArg'] = @data[:path] + '?' + @data[:query]
         else
           kvs['ServiceArg'] = @data[:path]

data/lib/oboe/inst/http.rb

@@ -29,7 +29,14 @@ if Oboe::Config[:nethttp][:enabled]
           opts['IsService'] = 1
           opts['RemoteProtocol'] = use_ssl? ? 'HTTPS' : 'HTTP'
           opts['RemoteHost'] = addr_port
-          opts['ServiceArg'] = req.path
+
+          # Conditionally log query params
+          if Oboe::Config[:nethttp][:log_args]
+            opts['ServiceArg'] = req.path
+          else
+            opts['ServiceArg'] = req.path.split('?').first
+          end
+
           opts['HTTPMethod'] = req.method
           opts['Blacklisted'] = true if blacklisted
           opts['Backtrace'] = Oboe::API.backtrace if Oboe::Config[:nethttp][:collect_backtraces]

data/lib/oboe/inst/httpclient.rb

@@ -0,0 +1,173 @@
+# Copyright (c) 2015 AppNeta, Inc.
+# All rights reserved.
+
+module Oboe
+  module Inst
+    module HTTPClient
+      def self.included(klass)
+        ::Oboe::Util.method_alias(klass, :do_request, ::HTTPClient)
+        ::Oboe::Util.method_alias(klass, :do_request_async, ::HTTPClient)
+        ::Oboe::Util.method_alias(klass, :do_get_stream, ::HTTPClient)
+      end
+
+      def oboe_collect(method, uri, query = nil)
+        kvs = {}
+        kvs['IsService'] = 1
+
+        # Conditionally log URL query params
+        # Because of the hook points, the query arg can come in under <tt>query</tt>
+        # or as a part of <tt>uri</tt> (not both).  Here we handle both cases.
+        if Oboe::Config[:httpclient][:log_args]
+          if query
+            kvs['RemoteURL'] = uri.to_s + '?' + Oboe::Util.to_query(query)
+          else
+            kvs['RemoteURL'] = uri.to_s
+          end
+        else
+          kvs['RemoteURL'] = uri.to_s.split('?').first
+        end
+
+        kvs['HTTPMethod'] = ::Oboe::Util.upcase(method)
+        kvs['Backtrace'] = Oboe::API.backtrace if Oboe::Config[:httpclient][:collect_backtraces]
+        kvs
+      rescue => e
+        Oboe.logger.debug "[oboe/debug] Error capturing httpclient KVs: #{e.message}"
+        Oboe.logger.debug e.backtrace.join('\n') if ::Oboe::Config[:verbose]
+      end
+
+      def do_request_with_oboe(method, uri, query, body, header, &block)
+        # If we're not tracing, just do a fast return.
+        if !Oboe.tracing?
+          return request_without_oboe(method, uri, query, body, header, &block)
+        end
+
+        begin
+          response_context = nil
+
+          # Avoid cross host tracing for blacklisted domains
+          blacklisted = Oboe::API.blacklisted?(uri.hostname)
+
+          kvs = oboe_collect(method, uri, query)
+          kvs['Blacklisted'] = true if blacklisted
+
+          Oboe::API.log_entry('httpclient', kvs)
+          kvs.clear
+
+          req_context = Oboe::Context.toString()
+
+          # Be aware of various ways to call/use httpclient
+          if header.is_a?(Array)
+            header.push ["X-Trace", req_context]
+          elsif header.is_a?(Hash)
+            header['X-Trace'] = req_context unless blacklisted
+          end
+
+          # The core httpclient call
+          response = do_request_without_oboe(method, uri, query, body, header, &block)
+
+          response_context = response.headers['X-Trace']
+          kvs['HTTPStatus'] = response.status_code
+
+          # If we get a redirect, report the location header
+          if ((300..308).to_a.include? response.status.to_i) && response.headers.key?("Location")
+            kvs["Location"] = response.headers["Location"]
+          end
+
+          if response_context && !blacklisted
+            Oboe::XTrace.continue_service_context(req_context, response_context)
+          end
+
+          response
+        rescue => e
+          Oboe::API.log_exception('httpclient', e)
+          raise e
+        ensure
+          Oboe::API.log_exit('httpclient', kvs)
+        end
+      end
+
+      def do_request_async_with_oboe(method, uri, query, body, header)
+        if Oboe.tracing?
+          # Since async is done by calling Thread.new { .. }, we somehow
+          # have to pass the tracing context into that new thread.  Here
+          # we stowaway the context in the request headers to be picked up
+          # (and removed from req headers) in do_get_stream.
+          if header.is_a?(Array)
+            header.push ["oboe.context", Oboe::Context.toString]
+          elsif header.is_a?(Hash)
+            header['oboe.context'] = Oboe::Context.toString
+          end
+        end
+
+        do_request_async_without_oboe(method, uri, query, body, header)
+      end
+
+      def do_get_stream_with_oboe(req, proxy, conn)
+        unless req.headers.key?("oboe.context")
+          return do_get_stream_without_oboe(req, proxy, conn)
+        end
+
+        # Pickup context and delete the headers stowaway
+        Oboe::Context.fromString req.headers["oboe.context"]
+        req.header.delete "oboe.context"
+
+        begin
+          response = nil
+          response_context = nil
+          uri = req.http_header.request_uri
+          method = req.http_header.request_method
+
+          # Avoid cross host tracing for blacklisted domains
+          blacklisted = Oboe::API.blacklisted?(uri.hostname)
+
+          kvs = oboe_collect(method, uri)
+          kvs['Blacklisted'] = true if blacklisted
+          kvs['Async'] = 1
+
+          Oboe::API.log_entry('httpclient', kvs)
+          kvs.clear
+
+          req_context = Oboe::Context.toString()
+          req.header.add('X-Trace', req_context)
+
+          # The core httpclient call
+          result = do_get_stream_without_oboe(req, proxy, conn)
+
+          # Older HTTPClient < 2.6.0 returns HTTPClient::Connection
+          if result.is_a?(::HTTP::Message)
+            response = result
+          else
+            response = conn.pop
+          end
+
+          response_context = response.headers['X-Trace']
+          kvs['HTTPStatus'] = response.status_code
+
+          # If we get a redirect, report the location header
+          if ((300..308).to_a.include? response.status.to_i) && response.headers.key?("Location")
+            kvs["Location"] = response.headers["Location"]
+          end
+
+          if response_context && !blacklisted
+            Oboe::XTrace.continue_service_context(req_context, response_context)
+          end
+
+          # Older HTTPClient < 2.6.0 returns HTTPClient::Connection
+          conn.push response if result.is_a?(::HTTPClient::Connection)
+          result
+        rescue => e
+          Oboe::API.log_exception('httpclient', e)
+          raise e
+        ensure
+          # Oboe::API.log_exit('httpclient', kvs.merge('Async' => 1))
+          Oboe::API.log_exit('httpclient', kvs)
+        end
+      end
+    end
+  end
+end
+
+if Oboe::Config[:httpclient][:enabled] && defined?(::HTTPClient)
+  ::Oboe.logger.info '[oboe/loading] Instrumenting httpclient' if Oboe::Config[:verbose]
+  ::Oboe::Util.send_include(::HTTPClient, ::Oboe::Inst::HTTPClient)
+end

data/lib/oboe/inst/rack.rb

@@ -19,7 +19,6 @@ module Oboe
         report_kvs['Port']             = req.port
         report_kvs['Proto']            = req.scheme
         report_kvs['Query-String']     = URI.unescape(req.query_string) unless req.query_string.empty?
-        report_kvs[:URL]               = URI.unescape(req.path)
         report_kvs[:Method]            = req.request_method
         report_kvs['AJAX']             = true if req.xhr?
         report_kvs['ClientIP']         = req.ip
@@ -62,9 +61,13 @@ module Oboe
       end
 
       req = ::Rack::Request.new(env)
-
       report_kvs = {}
-      report_kvs[:URL] = URI.unescape(req.path)
+
+      if Oboe::Config[:rack][:log_args]
+        report_kvs[:URL] = URI.unescape(req.fullpath)
+      else
+        report_kvs[:URL] = URI.unescape(req.path)
+      end
 
       # Check for and validate X-Trace request header to pick up tracing context
       xtrace = env.is_a?(Hash) ? env['HTTP_X_TRACE'] : nil

data/lib/oboe/inst/typhoeus.rb

@@ -28,15 +28,19 @@ module Oboe
         end
 
         kvs = {}
+        kvs['IsService'] = 1
         kvs[:HTTPStatus] = response.code
         kvs['Backtrace'] = Oboe::API.backtrace if Oboe::Config[:typhoeus][:collect_backtraces]
 
         uri = URI(response.effective_url)
-        kvs['IsService'] = 1
-        kvs['RemoteProtocol'] = ::Oboe::Util.upcase(uri.scheme)
-        kvs['RemoteHost'] = uri.host
-        kvs['RemotePort'] = uri.port ? uri.port : 80
-        kvs['ServiceArg'] = uri.path
+
+        # Conditionally log query params
+        if Oboe::Config[:typhoeus][:log_args]
+          kvs['RemoteURL'] = uri.to_s
+        else
+          kvs['RemoteURL'] = uri.to_s.split('?').first
+        end
+
         kvs['HTTPMethod'] = ::Oboe::Util.upcase(options[:method])
         kvs['Blacklisted'] = true if blacklisted
 

data/lib/oboe/util.rb

@@ -138,6 +138,25 @@ module Oboe
         end
       end
 
+
+      ##
+      # to_query
+      #
+      # Used under Ruby 1.8.7 to convert a hash into a URL
+      # query.  A backport of Hash#to_query.
+      #
+      def to_query(h)
+        return "" unless h.is_a?(Hash)
+
+        # If called from a newer Ruby, use the builtin.
+        return h.to_query if RUBY_VERSION >= '1.9.3'
+
+        result = []
+
+        h.each { |k, v| result.push (k.to_s + '=' + v.to_s) }
+        return result.sort.join('&')
+      end
+
       ##
       #  build_report
       #
@@ -174,6 +193,7 @@ module Oboe
           platform_info['Ruby.Dalli.Version']      = "Dalli-#{::Dalli::VERSION}"           if defined?(::Dalli)
           platform_info['Ruby.Excon.Version']      = "Excon-#{::Excon::VERSION}"           if defined?(::Excon::VERSION)
           platform_info['Ruby.Faraday.Version']    = "Faraday-#{::Faraday::VERSION}"       if defined?(::Faraday)
+          platform_info['Ruby.HTTPClient.Version'] = "HTTPClient-#{::HTTPClient::VERSION}" if defined?(::HTTPClient::VERSION)
           platform_info['Ruby.MemCache.Version']   = "MemCache-#{::MemCache::VERSION}"     if defined?(::MemCache)
           platform_info['Ruby.Moped.Version']      = "Moped-#{::Moped::VERSION}"           if defined?(::Moped)
           platform_info['Ruby.Redis.Version']      = "Redis-#{::Redis::VERSION}"           if defined?(::Redis)