traceview 3.0.0

data/lib/traceview/inst/rest-client.rb

@@ -0,0 +1,38 @@
+# Copyright (c) 2015 AppNeta, Inc.
+# All rights reserved.
+
+module TraceView
+  module Inst
+    module RestClientRequest
+      def self.included(klass)
+        ::TraceView::Util.method_alias(klass, :execute, ::RestClient::Request)
+      end
+
+      ##
+      # execute_with_traceview
+      #
+      # The wrapper method for RestClient::Request.execute
+      #
+      def execute_with_traceview & block
+        kvs = {}
+        kvs['Backtrace'] = TraceView::API.backtrace if TraceView::Config[:rest_client][:collect_backtraces]
+        TraceView::API.log_entry("rest-client", kvs)
+
+        # The core rest-client call
+        execute_without_traceview(&block)
+      rescue => e
+        TraceView::API.log_exception('rest-client', e)
+        raise e
+      ensure
+        TraceView::API.log_exit("rest-client")
+      end
+    end
+  end
+end
+
+if TraceView::Config[:rest_client][:enabled]
+  if defined?(::RestClient)
+    TraceView.logger.info '[traceview/loading] Instrumenting rest-client' if TraceView::Config[:verbose]
+    ::TraceView::Util.send_include(::RestClient::Request, ::TraceView::Inst::RestClientRequest)
+  end
+end

data/lib/traceview/inst/sequel.rb

@@ -0,0 +1,162 @@
+# Copyright (c) 2015 AppNeta, Inc.
+# All rights reserved.
+
+module TraceView
+  module Inst
+    module Sequel
+      def extract_trace_details(sql, opts)
+        kvs = {}
+
+        if TraceView::Config[:sanitize_sql]
+          # Sanitize SQL and don't report binds
+          if sql.is_a?(Symbol)
+            kvs[:Query] = sql
+          else
+            kvs[:Query] = sql.gsub(/('[\s\S][^\']*\'|\d*\.\d*)/, '?')
+          end
+        else
+          # Report raw SQL and any binds if they exist
+          kvs[:Query] = sql.to_s
+          kvs[:QueryArgs] = opts[:arguments] if opts.is_a?(Hash) and opts.key?(:arguments)
+        end
+        kvs[:IsPreparedStatement] = true if sql.is_a?(Symbol)
+
+        kvs[:Backtrace] = TraceView::API.backtrace if TraceView::Config[:sequel][:collect_backtraces]
+
+        if ::Sequel::VERSION < '3.41.0' && !(self.class.to_s =~ /Dataset$/)
+          db_opts = @opts
+        elsif @pool
+          db_opts = @pool.db.opts
+        else
+          db_opts = @db.opts
+        end
+
+        kvs[:Database]   = db_opts[:database]
+        kvs[:RemoteHost] = db_opts[:host]
+        kvs[:RemotePort] = db_opts[:port] if db_opts.key?(:port)
+        kvs[:Flavor]     = db_opts[:adapter]
+      rescue => e
+        TraceView.logger.debug "[traceview/debug Error capturing Sequel KVs: #{e.message}" if TraceView::Config[:verbose]
+      ensure
+        return kvs
+      end
+    end
+
+    module SequelDatabase
+      def self.included(klass)
+        ::TraceView::Util.method_alias(klass, :run, ::Sequel::Database)
+        ::TraceView::Util.method_alias(klass, :execute_ddl, ::Sequel::Database)
+        ::TraceView::Util.method_alias(klass, :execute_dui, ::Sequel::Database)
+        ::TraceView::Util.method_alias(klass, :execute_insert, ::Sequel::Database)
+      end
+
+      def run_with_traceview(sql, opts=::Sequel::OPTS)
+        kvs = extract_trace_details(sql, opts)
+
+        TraceView::API.log_entry('sequel', kvs)
+
+        run_without_traceview(sql, opts)
+      rescue => e
+        TraceView::API.log_exception('sequel', e)
+        raise e
+      ensure
+        TraceView::API.log_exit('sequel')
+      end
+
+      def exec_with_traceview(method, sql, opts=::Sequel::OPTS, &block)
+        kvs = extract_trace_details(sql, opts)
+
+        TraceView::API.log_entry('sequel', kvs)
+
+        send(method, sql, opts, &block)
+      rescue => e
+        TraceView::API.log_exception('sequel', e)
+        raise e
+      ensure
+        TraceView::API.log_exit('sequel')
+      end
+
+      def execute_ddl_with_traceview(sql, opts=::Sequel::OPTS, &block)
+        # If we're already tracing a sequel operation, then this call likely came
+        # from Sequel::Dataset.  In this case, just pass it on.
+        return execute_ddl_without_traceview(sql, opts, &block) if TraceView.tracing_layer?('sequel')
+
+        exec_with_traceview(:execute_ddl_without_traceview, sql, opts, &block)
+      end
+
+      def execute_dui_with_traceview(sql, opts=::Sequel::OPTS, &block)
+        # If we're already tracing a sequel operation, then this call likely came
+        # from Sequel::Dataset.  In this case, just pass it on.
+        return execute_dui_without_traceview(sql, opts, &block) if TraceView.tracing_layer?('sequel')
+
+        exec_with_traceview(:execute_dui_without_traceview, sql, opts, &block)
+      end
+
+      def execute_insert_with_traceview(sql, opts=::Sequel::OPTS, &block)
+        # If we're already tracing a sequel operation, then this call likely came
+        # from Sequel::Dataset.  In this case, just pass it on.
+        return execute_insert_without_traceview(sql, opts, &block) if TraceView.tracing_layer?('sequel')
+
+        exec_with_traceview(:execute_insert_without_traceview, sql, opts, &block)
+      end
+    end # module SequelDatabase
+
+    module SequelDataset
+
+      def self.included(klass)
+        ::TraceView::Util.method_alias(klass, :execute, ::Sequel::Dataset)
+        ::TraceView::Util.method_alias(klass, :execute_ddl, ::Sequel::Dataset)
+        ::TraceView::Util.method_alias(klass, :execute_dui, ::Sequel::Dataset)
+        ::TraceView::Util.method_alias(klass, :execute_insert, ::Sequel::Dataset)
+      end
+
+      def exec_with_traceview(method, sql, opts=::Sequel::OPTS, &block)
+        kvs = extract_trace_details(sql, opts)
+
+        TraceView::API.log_entry('sequel', kvs)
+
+        send(method, sql, opts, &block)
+      rescue => e
+        TraceView::API.log_exception('sequel', e)
+        raise e
+      ensure
+        TraceView::API.log_exit('sequel')
+      end
+
+      def execute_with_traceview(sql, opts=::Sequel::OPTS, &block)
+        exec_with_traceview(:execute_without_traceview, sql, opts, &block)
+      end
+
+      def execute_ddl_with_traceview(sql, opts=::Sequel::OPTS, &block)
+        exec_with_traceview(:execute_ddl_without_traceview, sql, opts, &block)
+      end
+
+      def execute_dui_with_traceview(sql, opts=::Sequel::OPTS, &block)
+        exec_with_traceview(:execute_dui_without_traceview, sql, opts, &block)
+      end
+
+      def execute_insert_with_traceview(sql, opts=::Sequel::OPTS, &block)
+        exec_with_traceview(:execute_insert_without_traceview, sql, opts, &block)
+      end
+
+    end # module SequelDataset
+  end # module Inst
+end # module TraceView
+
+if TraceView::Config[:sequel][:enabled]
+  if defined?(::Sequel) && ::Sequel::VERSION < "4.0.0"
+    # For versions before 4.0.0, Sequel::OPTS wasn't defined.
+    # Define it as an empty hash for backwards compatibility.
+    module ::Sequel
+      OPTS = {}
+    end
+  end
+
+  if defined?(::Sequel)
+    TraceView.logger.info '[traceview/loading] Instrumenting sequel' if TraceView::Config[:verbose]
+    ::TraceView::Util.send_include(::Sequel::Database, ::TraceView::Inst::Sequel)
+    ::TraceView::Util.send_include(::Sequel::Database, ::TraceView::Inst::SequelDatabase)
+    ::TraceView::Util.send_include(::Sequel::Dataset, ::TraceView::Inst::Sequel)
+    ::TraceView::Util.send_include(::Sequel::Dataset, ::TraceView::Inst::SequelDataset)
+  end
+end

data/lib/traceview/inst/typhoeus.rb

@@ -0,0 +1,102 @@
+# Copyright (c) 2015 AppNeta, Inc.
+# All rights reserved.
+
+module TraceView
+  module Inst
+    module TyphoeusRequestOps
+
+      def self.included(klass)
+        ::TraceView::Util.method_alias(klass, :run, ::Typhoeus::Request::Operations)
+      end
+
+      def run_with_traceview
+        return run_without_traceview unless TraceView.tracing?
+
+        TraceView::API.log_entry('typhoeus')
+
+        # Prepare X-Trace header handling
+        blacklisted = TraceView::API.blacklisted?(url)
+        context = TraceView::Context.toString
+        task_id = TraceView::XTrace.task_id(context)
+        options[:headers]['X-Trace'] = context unless blacklisted
+
+        response = run_without_traceview
+
+        if response.code == 0
+          TraceView::API.log('typhoeus', 'error', { :ErrorClass => response.return_code,
+                                               :ErrorMsg => response.return_message })
+        end
+
+        kvs = {}
+        kvs['IsService'] = 1
+        kvs[:HTTPStatus] = response.code
+        kvs['Backtrace'] = TraceView::API.backtrace if TraceView::Config[:typhoeus][:collect_backtraces]
+
+        uri = URI(response.effective_url)
+
+        # Conditionally log query params
+        if TraceView::Config[:typhoeus][:log_args]
+          kvs['RemoteURL'] = uri.to_s
+        else
+          kvs['RemoteURL'] = uri.to_s.split('?').first
+        end
+
+        kvs['HTTPMethod'] = ::TraceView::Util.upcase(options[:method])
+        kvs['Blacklisted'] = true if blacklisted
+
+        # Re-attach net::http edge unless it's blacklisted or if we don't have a
+        # valid X-Trace header
+        unless blacklisted
+          xtrace = response.headers['X-Trace']
+
+          if xtrace && TraceView::XTrace.valid?(xtrace) && TraceView.tracing?
+
+            # Assure that we received back a valid X-Trace with the same task_id
+            if task_id == TraceView::XTrace.task_id(xtrace)
+              TraceView::Context.fromString(xtrace)
+            else
+              TraceView.logger.debug "Mismatched returned X-Trace ID: #{xtrace}"
+            end
+          end
+        end
+
+        TraceView::API.log('typhoeus', 'info', kvs)
+        response
+      rescue => e
+        TraceView::API.log_exception('typhoeus', e)
+        raise e
+      ensure
+        TraceView::API.log_exit('typhoeus')
+      end
+    end
+
+    module TyphoeusHydraRunnable
+      def self.included(klass)
+        ::TraceView::Util.method_alias(klass, :run, ::Typhoeus::Hydra)
+      end
+
+      def run_with_traceview
+        kvs = {}
+
+        kvs[:queued_requests] = queued_requests.count
+        kvs[:max_concurrency] = max_concurrency
+
+        # FIXME: Until we figure out a strategy to deal with libcurl internal
+        # threading and Ethon's use of easy handles, here we just do a simple
+        # trace of the hydra run.
+        TraceView::API.trace("typhoeus_hydra", kvs) do
+          run_without_traceview
+        end
+      end
+    end
+
+  end
+end
+
+if TraceView::Config[:typhoeus][:enabled]
+  if defined?(::Typhoeus)
+    TraceView.logger.info '[traceview/loading] Instrumenting typhoeus' if TraceView::Config[:verbose]
+    ::TraceView::Util.send_include(::Typhoeus::Request::Operations, ::TraceView::Inst::TyphoeusRequestOps)
+    ::TraceView::Util.send_include(::Typhoeus::Hydra, ::TraceView::Inst::TyphoeusHydraRunnable)
+  end
+end

data/lib/traceview/instrumentation.rb

@@ -0,0 +1,21 @@
+# Copyright (c) 2013 AppNeta, Inc.
+# All rights reserved.
+
+module TraceView
+  ##
+  # The Inst module holds all of the instrumentation extensions for various
+  # libraries suchs as Redis, Dalli and Resque.
+  module Inst
+    def self.load_instrumentation
+      # Load the general instrumentation
+      pattern = File.join(File.dirname(__FILE__), 'inst', '*.rb')
+      Dir.glob(pattern) do |f|
+        begin
+          require f
+        rescue => e
+          TraceView.logger.error "[traceview/loading] Error loading instrumentation file '#{f}' : #{e}"
+        end
+      end
+    end
+  end
+end

data/lib/traceview/loading.rb

@@ -0,0 +1,94 @@
+# Copyright (c) 2013 AppNeta, Inc.
+# All rights reserved.
+
+require 'digest/sha1'
+
+module TraceView
+  module Util
+    ##
+    # This module is used solely for RUM ID calculation
+    #
+    module Base64URL
+      module_function
+
+      def encode(bin)
+        c = [bin].pack('m0').gsub(/\=+\Z/, '').tr('+/', '-_').rstrip
+        m = c.size % 4
+        c += '=' * (4 - m) if m != 0
+        c
+      end
+
+      def decode(bin)
+        m = bin.size % 4
+        bin += '=' * (4 - m) if m != 0
+        bin.tr('-_', '+/').unpack('m0').first
+      end
+    end
+  end
+
+  ##
+  # This module houses all of the loading functionality for the traceview gem.
+  #
+  # Note that this does not necessarily _have_ to include initialization routines
+  # (although it can).
+  #
+  # Actual initialization is often separated out as it can be dependent on on the state
+  # of the stack boot process.  e.g. code requiring that initializers, frameworks or
+  # instrumented libraries are already loaded...
+  #
+  module Loading
+    ##
+    # Load the TraceView access key (either from system configuration file
+    # or environment variable) and calculate internal RUM ID
+    #
+    def self.load_access_key
+      if ENV.key?('TRACEVIEW_CUUID')
+        # Preferably get access key from environment (e.g. Heroku)
+        TraceView::Config[:access_key] = ENV['TRACEVIEW_CUUID']
+        TraceView::Config[:rum_id] = TraceView::Util::Base64URL.encode(Digest::SHA1.digest('RUM' + TraceView::Config[:access_key]))
+      else
+        # ..else read from system-wide configuration file
+        if TraceView::Config.access_key.empty?
+          config_file = '/etc/tracelytics.conf'
+          return unless File.exist?(config_file)
+
+          File.open(config_file).each do |line|
+            if line =~ /^tracelyzer.access_key=/ || line =~ /^access_key/
+              bits = line.split(/=/)
+              TraceView::Config[:access_key] = bits[1].strip
+              TraceView::Config[:rum_id] = TraceView::Util::Base64URL.encode(Digest::SHA1.digest('RUM' + TraceView::Config[:access_key]))
+              break
+            end
+          end
+        end
+      end
+    rescue StandardError => e
+      TraceView.logger.error "Trouble obtaining access_key and rum_id: #{e.inspect}"
+    end
+
+    ##
+    # Load the traceview tracing API
+    #
+    def self.require_api
+      pattern = File.join(File.dirname(__FILE__), 'api', '*.rb')
+      Dir.glob(pattern) do |f|
+        require f
+      end
+      require 'traceview/api'
+
+      begin
+        TraceView::API.extend_with_tracing
+      rescue LoadError => e
+        TraceView.logger.fatal "[traceview/error] Couldn't load api: #{e.message}"
+      end
+    end
+  end
+end
+
+TraceView::Loading.require_api
+
+# Auto-start the Reporter unless we running Unicorn on Heroku
+# In that case, we start the reporters after fork
+unless TraceView.heroku? && TraceView.forking_webserver?
+  TraceView::Reporter.start if TraceView.loaded
+end

data/lib/traceview/logger.rb

@@ -0,0 +1,41 @@
+# Copyright (c) 2013 AppNeta, Inc.
+# All rights reserved.
+
+require 'logger'
+
+module TraceView
+  class << self
+    attr_accessor :logger
+  end
+
+  class Logger
+    # Fatal message
+    def fatal(string, exception = nil)
+      TraceView.logger.fatal(string) if TraceView.logger
+    end
+
+    # Error message
+    def error(msg, exception = nil)
+      TraceView.logger.error(string) if TraceView.logger
+    end
+
+    # Warn message
+    def warn(msg, exception = nil)
+      TraceView.logger.warn(string) if TraceView.logger
+    end
+
+    # Info message
+    def info(msg, exception = nil)
+      TraceView.logger.info(string) if TraceView.logger
+    end
+
+    # Debug message
+    def debug(msg, exception = nil)
+      TraceView.logger.debug(string) if TraceView.logger
+    end
+
+  end
+end
+
+TraceView.logger = Logger.new(STDERR)
+