skylight 1.7.2 → 2.0.0.beta1

data/lib/skylight/helpers.rb

@@ -145,10 +145,14 @@ module Skylight
               title:       #{title.inspect},
               description: #{desc.inspect})
 
+            meta = {}
             begin
               send(:before_instrument_#{name}, *args, &blk)
+            rescue Exception => e
+              meta[:exception_object] = e
+              raise e
             ensure
-              Skylight.done(span) if span
+              Skylight.done(span, meta) if span
             end
           end
         RUBY

data/lib/skylight/instrumenter.rb

@@ -1,255 +1,24 @@
-require 'thread'
-require 'strscan'
-require 'skylight/api'
-
 module Skylight
-  # @api private
-  class Instrumenter
-    KEY  = :__skylight_current_trace
-    LOCK = Mutex.new
-
-    TOO_MANY_UNIQUES = "<too many unique descriptions>"
-
-    include Util::Logging
-
-    class TraceInfo
-      def current
-        Thread.current[KEY]
-      end
-
-      def current=(trace)
-        Thread.current[KEY] = trace
-      end
-    end
-
-    def self.instance
-      @instance
-    end
-
-    # Do start
-    # @param [Config] config The config
-    def self.start!(config = nil)
-      return @instance if @instance
-
-      LOCK.synchronize do
-        return @instance if @instance
-        @instance = new(config).start!
-      end
-    rescue => e
-      message = sprintf("[SKYLIGHT] [#{Skylight::VERSION}] Unable to start Instrumenter; msg=%s; class=%s", e.message, e.class)
-      if config && config.respond_to?(:logger)
-        config.logger.warn message
-      else
-        warn message
-      end
-      false
-    end
-
-    def self.stop!
-      LOCK.synchronize do
-        return unless @instance
-        # This is only really helpful for getting specs to pass.
-        @instance.current_trace = nil
-
-        @instance.shutdown
-        @instance = nil
-      end
-    end
-
-    at_exit do
-      stop!
-    end
-
-    attr_reader :config, :gc, :trace_info
-
-    def self.new(config)
-      config ||= {}
-      config = Config.load(config) unless config.is_a?(Config)
-      config.validate!
-
-      inst = native_new(config.to_native_env)
-      inst.send(:initialize, config)
-      inst
-    end
-
-    def initialize(config)
-      @gc = config.gc
-      @config = config
-      @subscriber = Subscriber.new(config, self)
-
-      @trace_info = @config[:trace_info] || TraceInfo.new
-    end
-
-    def current_trace
-      @trace_info.current
+  class Instrumenter < Core::Instrumenter
+    def self.trace_class
+      Trace
     end
 
-    def current_trace=(trace)
-      @trace_info.current = trace
-    end
-
-    def start!
+    def check_install!
       # Warn if there was an error installing Skylight.
-      # We do this here since we can't report these issues via Gem install without stopping install entirely.
-      Skylight.check_install_errors(config)
-
-      unless Skylight.native?
-        Skylight.warn_skylight_native_missing(config)
-        return
-      end
-
-      t { "starting instrumenter" }
-
-      unless config.validate_with_server
-        log_error "invalid config"
-        return
-      end
-
-      t { "starting native instrumenter" }
-      unless native_start
-        warn "failed to start instrumenter"
-        return
-      end
-
-      config.gc.enable
-      @subscriber.register!
-
-      self
-
-    rescue Exception => e
-      log_error "failed to start instrumenter; msg=%s; config=%s", e.message, config.inspect
-      t { e.backtrace.join("\n") }
-      nil
-    end
 
-    def shutdown
-      @subscriber.unregister!
-      native_stop
-    end
-
-    def trace(endpoint, cat, title=nil, desc=nil)
-      # If a trace is already in progress, continue with that one
-      if trace = @trace_info.current
-        return yield(trace) if block_given?
-        return trace
-      end
-
-      begin
-        trace = Trace.new(self, endpoint, Util::Clock.nanos, cat, title, desc)
-      rescue Exception => e
-        log_error e.message
-        t { e.backtrace.join("\n") }
-        return
-      end
-
-      @trace_info.current = trace
-      return trace unless block_given?
-
-      begin
-        yield trace
-
-      ensure
-        @trace_info.current = nil
-        t { "submitting trace" }
-        trace.submit
-      end
-    end
-
-    def disable
-      @disabled = true
-      yield
-    ensure
-      @disabled = false
-    end
-
-    def disabled?
-      @disabled
-    end
-
-    @scanner = StringScanner.new('')
-    def self.match?(string, regex)
-      @scanner.string = string
-      @scanner.match?(regex)
-    end
-
-    def match?(string, regex)
-      self.class.match?(string, regex)
-    end
-
-    def done(span)
-      return unless trace = @trace_info.current
-      trace.done(span)
-    end
-
-    def broken!
-      return unless trace = @trace_info.current
-      trace.broken!
-    end
-
-    def instrument(cat, title=nil, desc=nil)
-      raise ArgumentError, 'cat is required' unless cat
-
-      unless trace = @trace_info.current
-        return yield if block_given?
-        return
-      end
-
-      cat = cat.to_s
-
-      unless match?(cat, CATEGORY_REGEX)
-        warn "invalid skylight instrumentation category; value=%s", cat
-        return yield if block_given?
-        return
+      if defined?(Skylight.check_install_errors)
+        Skylight.check_install_errors(config)
       end
 
-      cat = "other.#{cat}" unless match?(cat, TIER_REGEX)
-
-      unless sp = trace.instrument(cat, title, desc)
-        return yield if block_given?
+      if !Skylight.native? && defined?(Skylight.warn_skylight_native_missing)
+        Skylight.warn_skylight_native_missing(config)
         return
       end
-
-      return sp unless block_given?
-
-      begin
-        yield sp
-      ensure
-        trace.done(sp)
-      end
     end
 
-    def limited_description(description)
-      endpoint = @trace_info.current.endpoint
-
-      if description
-        if native_track_desc(endpoint, description)
-          description
-        else
-          TOO_MANY_UNIQUES
-        end
-      end
+    def process_sql(sql)
+      Skylight.lex_sql(sql, config[:use_old_sql_lexer])
     end
-
-    def process(trace)
-      t { fmt "processing trace" }
-
-      if ignore?(trace)
-        t { fmt "ignoring trace" }
-        return false
-      end
-
-      begin
-        native_submit_trace(trace)
-        true
-      rescue => e
-        warn "failed to submit trace to worker; err=%s", e
-        false
-      end
-    end
-
-    def ignore?(trace)
-      config.ignored_endpoints.include?(trace.endpoint.sub(%r{<sk-segment>.+</sk-segment>}, ''))
-    end
-
   end
 end

data/lib/skylight/middleware.rb

@@ -1,92 +1,4 @@
 module Skylight
-  # @api private
-  class Middleware
-
-    class BodyProxy
-      def initialize(body, &block)
-        @body, @block, @closed = body, block, false
-      end
-
-      def respond_to?(*args)
-        return false if args.first.to_s =~ /^to_ary$/
-        super or @body.respond_to?(*args)
-      end
-
-      def close
-        return if @closed
-        @closed = true
-        begin
-          @body.close if @body.respond_to? :close
-        ensure
-          @block.call
-        end
-      end
-
-      def closed?
-        @closed
-      end
-
-      # N.B. This method is a special case to address the bug described by
-      # https://github.com/rack/rack/issues/434.
-      # We are applying this special case for #each only. Future bugs of this
-      # class will be handled by requesting users to patch their ruby
-      # implementation, to save adding too many methods in this class.
-      def each(*args, &block)
-        @body.each(*args, &block)
-      end
-
-      def method_missing(*args, &block)
-        super if args.first.to_s =~ /^to_ary$/
-        @body.__send__(*args, &block)
-      end
-    end
-
-    def self.with_after_close(resp, &block)
-      # Responses should be arrays but in some situations they aren't
-      #   e.g. https://github.com/ruby-grape/grape/issues/1041
-      # The safest approach seems to be to rely on implicit destructuring
-      #   since that is currently what Rack::Lint does.
-      # See also https://github.com/rack/rack/issues/1239
-      status, headers, body = resp
-
-      [status, headers, BodyProxy.new(body, &block)]
-    end
-
-    include Util::Logging
-
-    # For Util::Logging
-    attr_reader :config
-
-    def initialize(app, opts={})
-      @app = app
-      @config = opts[:config]
-    end
-
-    def call(env)
-      if Skylight.tracing?
-        error "Already instrumenting. Make sure the Middleware hasn't been added more than once."
-      end
-
-      if env["REQUEST_METHOD"] == "HEAD"
-        t { "middleware skipping HEAD" }
-        @app.call(env)
-      else
-        begin
-          t { "middleware beginning trace" }
-          trace = Skylight.trace "Rack", 'app.rack.request'
-          resp = @app.call(env)
-
-          if trace
-            Middleware.with_after_close(resp) { trace.submit }
-          else
-            resp
-          end
-        rescue Exception
-          t { "middleware exception: #{trace}"}
-          trace.submit if trace
-          raise
-        end
-      end
-    end
+  class Middleware < Core::Middleware
   end
 end

data/lib/skylight/native.rb

@@ -1,4 +1,4 @@
-require 'skylight/util/platform'
+require 'skylight/core/util/platform'
 
 module Skylight
   # @api private
@@ -10,14 +10,14 @@ module Skylight
   end
 
   def self.libskylight_path
-    ENV['SKYLIGHT_LIB_PATH'] || File.expand_path("../native/#{Util::Platform.tuple}", __FILE__)
+    ENV['SKYLIGHT_LIB_PATH'] || File.expand_path("../native/#{Core::Util::Platform.tuple}", __FILE__)
   end
 
   skylight_required = ENV.key?("SKYLIGHT_REQUIRED") && ENV['SKYLIGHT_REQUIRED'] !~ /^false$/i
 
   begin
     unless ENV.key?("SKYLIGHT_DISABLE_AGENT") && ENV['SKYLIGHT_DISABLE_AGENT'] !~ /^false$/i
-      lib = "#{libskylight_path}/libskylight.#{Util::Platform.libext}"
+      lib = "#{libskylight_path}/libskylight.#{Core::Util::Platform.libext}"
 
       if File.exist?(lib)
         # First attempt to require the native extension
@@ -39,7 +39,9 @@ module Skylight
     raise if skylight_required
   end
 
-  unless Skylight.native?
+  if Skylight.native?
+    Skylight::Core::Util::Clock.use_native!
+  else
     class Instrumenter
       def self.native_new(*args)
         allocate
@@ -54,7 +56,7 @@ module Skylight
 
     if File.exist?(install_log) && File.read(install_log) =~ /ERROR/
       config.alert_logger.error \
-          "[SKYLIGHT] [#{Skylight::VERSION}] The Skylight native extension failed to install. " \
+          "[SKYLIGHT] [#{Skylight::Core::VERSION}] The Skylight native extension failed to install. " \
           "Please check #{install_log} and notify support@skylight.io. " \
           "The missing extension will not affect the functioning of your application."
     end
@@ -63,7 +65,7 @@ module Skylight
   # @api private
   def self.warn_skylight_native_missing(config)
     config.alert_logger.error \
-      "[SKYLIGHT] [#{Skylight::VERSION}] The Skylight native extension for " \
+      "[SKYLIGHT] [#{Skylight::Core::VERSION}] The Skylight native extension for " \
       "your platform wasn't found. Supported operating systems are " \
       "Linux 2.6.18+ and Mac OS X 10.8+. The missing extension will not " \
       "affect the functioning of your application. If you are on a " \

data/lib/skylight/native_ext_fetcher.rb

@@ -0,0 +1,251 @@
+require 'uri'
+require 'logger'
+require 'net/http'
+require 'fileutils'
+require 'digest/sha2'
+require 'skylight/util/ssl'
+require 'skylight/core/util/proxy'
+
+# Used from extconf.rb
+module Skylight
+  # Utility class for fetching the native extension from a URL
+  class NativeExtFetcher
+    BASE_URL = "https://s3.amazonaws.com/skylight-agent-packages/skylight-native"
+    MAX_REDIRECTS = 5
+    MAX_RETRIES = 3
+
+    include FileUtils
+
+    class FetchError < StandardError; end
+
+    # Creates a new fetcher and fetches
+    # @param opts [Hash]
+    def self.fetch(opts = {})
+      fetcher = new(
+        opts[:source] || BASE_URL,
+        opts[:target],
+        opts[:version],
+        opts[:checksum],
+        opts[:arch],
+        opts[:required],
+        opts[:platform],
+        opts[:logger] || Logger.new(STDOUT))
+
+      fetcher.fetch
+    end
+
+    # @param source [String] the base url to download from
+    # @param target [String] file to download as
+    # @param version [String] version to download
+    # @param checksum [String] checksum of the archive
+    # @param arch [String] platform architecture, e.g. `linux-x86_64`
+    # @param required [Boolean] whether the download is required to be successful
+    # @param platform
+    # @param log [Logger]
+    def initialize(source, target, version, checksum, arch, required, platform, log)
+      raise "source required" unless source
+      raise "target required" unless target
+      raise "checksum required" unless checksum
+      raise "arch required" unless arch
+
+      @source = source
+      @target = target
+      @version = version
+      @checksum = checksum
+      @required = required
+      @platform = platform
+      @arch = arch
+      @log = log
+    end
+
+    # Fetch the native extension, verify, inflate, and save (if applicable)
+    #
+    # @return [String] the inflated archive
+    def fetch
+      log "fetching native ext; curr-platform=#{@platform}; " \
+        "requested-arch=#{@arch}; version=#{@version}"
+
+      tar_gz = "#{@target}/#{basename}"
+
+      unless sha2 = fetch_native_ext(source_uri, tar_gz, MAX_RETRIES, MAX_REDIRECTS)
+        maybe_raise "could not fetch native extension"
+        return
+      end
+
+      unless verify_checksum(sha2)
+        maybe_raise "could not verify checksum"
+        return
+      end
+
+      Dir.chdir File.dirname(tar_gz) do
+        system "tar xzvf #{tar_gz}"
+      end
+
+      true
+    ensure
+      rm_f tar_gz if tar_gz
+    end
+
+    def fetch_native_ext(uri, out, attempts, redirects)
+      redirects.times do |i|
+        # Ensure the location is available
+        mkdir_p File.dirname(out)
+        rm_f out
+
+        remaining_attempts = attempts
+
+        log "attempting to fetch from remote; uri=#{uri}"
+
+        begin
+          host, port, use_ssl, path = deconstruct_uri(uri)
+
+          File.open out, 'w' do |f|
+            res, extra = http_get(host, port, use_ssl, path, f)
+
+            case res
+            when :success
+              log "successfully downloaded native ext; out=#{out}"
+              return extra
+            when :redirect
+              log "fetching native ext; uri=#{uri}; redirected=#{res}"
+              uri = extra
+
+              next
+            end
+          end
+        rescue => e
+          remaining_attempts -= 1
+
+          error "failed to fetch native extension; uri=#{uri}; msg=#{e.message}; remaining-attempts=#{remaining_attempts}", e
+
+          if remaining_attempts > 0
+            sleep 2
+            retry
+          end
+
+          return
+        end
+      end
+
+      log "exceeded max redirects"
+      return
+    end
+
+    # Get with `Net::HTTP`
+    #
+    # @param host [String] host for `Net::HTTP` request
+    # @param port [String,Integer] port for `Net::HTTP` request
+    # @param use_ssl [Boolean] whether SSL should be used for this request
+    # @param path [String] the path to request
+    # @param out [IO]
+    #
+    # If `ENV['HTTP_PROXY']` is set, it will be used as a proxy for this request.
+    def http_get(host, port, use_ssl, path, out)
+      if http_proxy = Core::Util::Proxy.detect_url(ENV)
+        log "connecting with proxy: #{http_proxy}"
+        uri = URI.parse(http_proxy)
+        p_host, p_port = uri.host, uri.port
+        p_user, p_pass = uri.userinfo.split(/:/) if uri.userinfo
+      end
+
+      opts = {}
+      opts[:use_ssl] = use_ssl
+
+      if use_ssl
+        opts[:ca_file] = Util::SSL.ca_cert_file_or_default
+      end
+
+      Net::HTTP.start(host, port, p_host, p_port, p_user, p_pass, use_ssl: use_ssl) do |http|
+        http.request_get path do |resp|
+          case resp
+          when Net::HTTPSuccess
+            digest = Digest::SHA2.new
+
+            resp.read_body do |chunk|
+              digest << chunk
+              out.write chunk
+            end
+
+            return [ :success, digest.hexdigest ]
+          when Net::HTTPRedirection
+            unless location = resp['location']
+              raise "received redirect but no location"
+            end
+
+            return [ :redirect, location ]
+          else
+            raise "received HTTP status code #{resp.code}"
+          end
+        end
+      end
+    end
+
+    # Verify the checksum of the archive
+    #
+    # @param actual [String]
+    # @return [Boolean] whether the checksum matches
+    def verify_checksum(actual)
+      unless @checksum == actual
+        log "checksum mismatch; expected=#{@checksum}; actual=#{actual}"
+        return false
+      end
+
+      true
+    rescue Exception => e
+      error "failed to read skylight agent archive; e=#{e.message}"
+      false
+    end
+
+    def basename
+      "skylight_#{@arch}.tar.gz"
+    end
+
+    # The url that will be fetched
+    #
+    # @return String
+    def source_uri
+      "#{@source}/#{@version}/#{basename}"
+    end
+
+    # Split the uri string into its component parts
+    #
+    # @param uri [String] the uri
+    # @return [Array<String>] the host, port, scheme, and request_uri
+    def deconstruct_uri(uri)
+      uri = URI(uri)
+      [ uri.host, uri.port, uri.scheme == 'https', uri.request_uri ]
+    end
+
+    # Log an error and raise if `required` is `true`
+    #
+    # @param err [String]
+    # @return [void]
+    def maybe_raise(err)
+      error err
+
+      if @required
+        raise err
+      end
+    end
+
+    # Log an `info` to the `logger`
+    #
+    # @param msg [String]
+    # @return [void]
+    def log(msg)
+      msg = "[SKYLIGHT] #{msg}"
+      @log.info msg
+    end
+
+    # Log an `error` to the `logger`
+    #
+    # @param msg [String]
+    # @param e [Exception] the exception associated with the error
+    # @return [void]
+    def error(msg, e=nil)
+      msg = "[SKYLIGHT] #{msg}"
+      msg << "\n#{e.backtrace.join("\n")}" if e
+      @log.error msg
+    end
+  end
+end