skylight 0.3.21 → 0.4.0.alpha1

data/lib/skylight/util.rb

@@ -1,18 +1,16 @@
 module Skylight
   # @api private
   module Util
-    # Already defined by the native extension so we can't autoload
+    # Used from the main lib
+    require 'skylight/util/allocation_free'
     require 'skylight/util/clock'
+    require 'skylight/util/hostname'
+    require 'skylight/util/logging'
+    require 'skylight/util/ssl'
 
-    autoload :AllocationFree, 'skylight/util/allocation_free'
-    autoload :Conversions,    'skylight/util/conversions'
-    autoload :Gzip,           'skylight/util/gzip'
-    autoload :HTTP,           'skylight/util/http'
-    autoload :Inflector,      'skylight/util/inflector'
-    autoload :Logging,        'skylight/util/logging'
-    autoload :Queue,          'skylight/util/queue'
-    autoload :Task,           'skylight/util/task'
-    autoload :UniformSample,  'skylight/util/uniform_sample'
-    autoload :NativeExtFetcher, 'skylight/util/native_ext_fetcher'
+    # Used from the CLI
+    autoload :Gzip,      'skylight/util/gzip'
+    autoload :HTTP,      'skylight/util/http'
+    autoload :Inflector, 'skylight/util/inflector'
   end
-end
+end

data/lib/skylight/util/hostname.rb

@@ -0,0 +1,17 @@
+require 'socket'
+require 'securerandom'
+
+module Skylight
+  module Util
+    module Hostname
+      def self.default_hostname
+        if hostname = Socket.gethostname
+          hostname.strip!
+          hostname = nil if hostname == ''
+        end
+
+        hostname || "gen-#{SecureRandom.uuid}"
+      end
+    end
+  end
+end

data/lib/skylight/util/http.rb

@@ -1,7 +1,9 @@
+require 'uri'
 require 'json'
 require 'openssl'
 require 'net/http'
 require 'net/https'
+require 'skylight/util/ssl'
 
 module Skylight
   module Util
@@ -15,7 +17,6 @@ module Skylight
       AUTHORIZATION    = 'authorization'.freeze
       DEFLATE          = 'deflate'.freeze
       GZIP             = 'gzip'.freeze
-      DEFAULT_CA_FILE  = File.expand_path('../../data/cacert.pem', __FILE__)
 
       include Logging
 
@@ -30,44 +31,30 @@ module Skylight
       class StartError < StandardError; end
       class ReadResponseError < StandardError; end
 
-      def initialize(config, service = :report, opts = {})
+      def initialize(config, service = :auth, opts = {})
         @config = config
-        @ssl  = config["#{service}.ssl"]
-        @host = config["#{service}.host"]
-        @port = config["#{service}.port"]
-
-        @proxy_addr = config["#{service}.proxy_addr"]
-        @proxy_port = config["#{service}.proxy_port"]
-        @proxy_user = config["#{service}.proxy_user"]
-        @proxy_pass = config["#{service}.proxy_pass"]
-
-        @timeout = opts[:timeout] || 15
-
-        unless @proxy_addr
-          if http_proxy = ENV['HTTP_PROXY'] || ENV['http_proxy']
-            uri = URI.parse(http_proxy)
-            @proxy_addr, @proxy_port = uri.host, uri.port
-            @proxy_user, @proxy_pass = (uri.userinfo || '').split(/:/)
-          end
+
+        unless url = config["#{service}_url"]
+          raise ArgumentError, "no URL specified"
         end
 
-        @deflate = config["#{service}.deflate"]
-        @authentication = config[:'authentication']
-      end
+        url = URI.parse(url)
 
-      def self.detect_ca_cert_file!
-        @ca_cert_file = false
-        if defined?(OpenSSL::X509::DEFAULT_CERT_FILE)
-          if OpenSSL::X509::DEFAULT_CERT_FILE
-            @ca_cert_file = File.exist?(OpenSSL::X509::DEFAULT_CERT_FILE)
-          end
+        @ssl  = url.scheme == 'https'
+        @host = url.host
+        @port = url.port
+
+        if proxy_url = config[:proxy_url]
+          proxy_url = URI.parse(proxy_url)
+          @proxy_addr, @proxy_port = proxy_url.host, proxy_url.port
+          @proxy_user, @proxy_pass = (proxy_url.userinfo || '').split(/:/)
         end
-      end
 
-      detect_ca_cert_file!
+        @open_timeout = get_timeout(:connect, config, service, opts)
+        @read_timeout = get_timeout(:read, config, service, opts)
 
-      def self.ca_cert_file?
-        @ca_cert_file
+        @deflate = config["#{service}_http_deflate"]
+        @authentication = config[:'authentication']
       end
 
       def get(endpoint, hdrs = {})
@@ -87,6 +74,11 @@ module Skylight
 
     private
 
+      def get_timeout(type, config, service, opts)
+        config.duration_ms("#{service}_http_#{type}_timeout") ||
+          opts[:timeout] || 15
+      end
+
       def build_request(type, endpoint, hdrs, length=nil)
         headers = {}
 
@@ -131,14 +123,19 @@ module Skylight
           req.body = body
         end
 
-        http = Net::HTTP.new(@host, @port, @proxy_addr, @proxy_port, @proxy_user, @proxy_pass)
+        http = Net::HTTP.new(@host, @port,
+          @proxy_addr, @proxy_port, @proxy_user, @proxy_pass)
 
-        http.open_timeout = @timeout
-        http.read_timeout = @timeout
+        http.open_timeout = @open_timeout
+        http.read_timeout = @read_timeout
 
         if @ssl
           http.use_ssl = true
-          http.ca_file = DEFAULT_CA_FILE unless HTTP.ca_cert_file?
+
+          unless SSL.ca_cert_file?
+            http.ca_file = SSL.ca_cert_file_or_default
+          end
+
           http.verify_mode = OpenSSL::SSL::VERIFY_PEER
         end
 

data/lib/skylight/util/logging.rb

@@ -2,8 +2,27 @@ require 'logger'
 
 module Skylight
   module Util
+    # Log both to the specified logger and STDOUT
+    class AlertLogger
+      def initialize(logger)
+        @logger = logger
+      end
+
+      def write(*args)
+        STDERR.write *args
+        @logger.<<(*args)
+      end
+
+      def close
+      end
+    end
+
     module Logging
-      if ENV[TRACE_ENV_KEY]
+      def self.trace?
+        ENV[TRACE_ENV_KEY]
+      end
+
+      if trace?
         def trace(msg, *args)
           log :debug, msg, *args
         end

data/lib/skylight/util/multi_io.rb

@@ -0,0 +1,21 @@
+# Util allowing proxying writes to multiple location
+# Used from extconf
+module Skylight
+  module Util
+    class MultiIO
+
+      def initialize(*targets)
+         @targets = targets
+      end
+
+      def write(*args)
+        @targets.each {|t| t.write(*args)}
+      end
+
+      def close
+        @targets.each(&:close)
+      end
+
+    end
+  end
+end

data/lib/skylight/util/native_ext_fetcher.rb

@@ -1,20 +1,20 @@
 require 'uri'
 require 'logger'
-require 'zlib'
 require 'net/http'
+require 'fileutils'
 require 'digest/sha2'
+require 'skylight/util/ssl'
 
-# Must require 'rubygems/platform' vs. just requiring 'rubygems' to avoid a
-# stack overflow bug on ruby 1.9.2.
-require 'rubygems/platform'
-
+# Used from extconf.rb
 module Skylight
   module Util
     class NativeExtFetcher
-      BASE_URL = "https://github.com/skylightio/skylight-rust/releases/download"
+      BASE_URL = "https://s3.amazonaws.com/skylight-agent-packages/skylight-native"
       MAX_REDIRECTS = 5
       MAX_RETRIES = 3
 
+      include FileUtils
+
       class FetchError < StandardError; end
 
       def self.fetch(opts = {})
@@ -25,13 +25,15 @@ module Skylight
           opts[:checksum],
           opts[:arch],
           opts[:required],
+          opts[:platform],
           opts[:logger] || Logger.new(STDOUT))
 
         fetcher.fetch
       end
 
-      def initialize(source, target, version, checksum, arch, required, log)
+      def initialize(source, target, version, checksum, arch, required, platform, log)
         raise "source required" unless source
+        raise "target required" unless target
         raise "checksum required" unless checksum
         raise "arch required" unless arch
 
@@ -40,61 +42,42 @@ module Skylight
         @version = version
         @checksum = checksum
         @required = required
+        @platform = platform
         @arch = arch
         @log = log
       end
 
       def fetch
-        log "fetching native ext; curr-platform=#{Gem::Platform.local.to_s}; " \
+        log "fetching native ext; curr-platform=#{@platform}; " \
           "requested-arch=#{@arch}; version=#{@version}"
 
-        unless gziped = fetch_native_ext(source_uri, MAX_RETRIES, MAX_REDIRECTS)
+        tar_gz = "#{@target}/#{basename}"
+
+        unless sha2 = fetch_native_ext(source_uri, tar_gz, MAX_RETRIES, MAX_REDIRECTS)
           maybe_raise "could not fetch native extension"
           return
         end
 
-        unless verify_checksum(gziped)
+        unless verify_checksum(sha2)
           maybe_raise "could not verify checksum"
           return
         end
 
-        archive = inflate(gziped)
-
-        if @target
-          File.open @target, 'w' do |f|
-            f.write(archive)
-          end
+        Dir.chdir File.dirname(tar_gz) do
+          system "tar xzvf #{tar_gz}"
         end
 
-        archive
-      end
-
-      def http_get(host, port, use_ssl, path)
-        if http_proxy = ENV['HTTP_PROXY'] || ENV['http_proxy']
-          log "connecting with proxy: #{http_proxy}"
-          uri = URI.parse(http_proxy)
-          p_host, p_port = uri.host, uri.port
-          p_user, p_pass = uri.userinfo.split(/:/) if uri.userinfo
-        end
-
-        Net::HTTP.start(host, port, p_host, p_port, p_user, p_pass, use_ssl: use_ssl) do |http|
-          case response = http.get(path)
-          when Net::HTTPSuccess
-            return [ :success, response.body ]
-          when Net::HTTPRedirection
-            unless location = response['location']
-              raise "received redirect but no location"
-            end
-
-            return [ :redirect, location ]
-          else
-            raise "received HTTP status code #{response.code}"
-          end
-        end
+        true
+      ensure
+        rm_f tar_gz if tar_gz
       end
 
-      def fetch_native_ext(uri, attempts, redirects)
+      def fetch_native_ext(uri, out, attempts, redirects)
         redirects.times do |i|
+          # Ensure the location is available
+          mkdir_p File.dirname(out)
+          rm_f out
+
           remaining_attempts = attempts
 
           log "attempting to fetch from remote; uri=#{uri}"
@@ -102,26 +85,20 @@ module Skylight
           begin
             host, port, use_ssl, path = deconstruct_uri(uri)
 
-            status, body = http_get(host, port, use_ssl, path)
+            File.open out, 'w' do |f|
+              res, extra = http_get(host, port, use_ssl, path, f)
 
-            case status
-            when :success
-              if body
-                log "successfully downloaded native ext; body=#{body.bytesize}bytes"
+              case res
+              when :success
+                log "successfully downloaded native ext; out=#{out}"
+                return extra
               else
-                log "response did not contain a body"
-              end
-
-              return body
-            when :redirect
-              log "fetching native ext; uri=#{uri}; redirected=#{body}"
-              uri = body
+                log "fetching native ext; uri=#{uri}; redirected=#{res}"
+                uri = res
 
-              next
-            else
-              raise "received unknown return; status=#{status}; body=#{body}"
+                next
+              end
             end
-
           rescue => e
             remaining_attempts -= 1
 
@@ -140,27 +117,64 @@ module Skylight
         return
       end
 
-      def verify_checksum(archive)
-        expected = @checksum
-        actual = Digest::SHA2.hexdigest(archive)
+      def http_get(host, port, use_ssl, path, out)
+        if http_proxy = ENV['HTTP_PROXY'] || ENV['http_proxy']
+          log "connecting with proxy: #{http_proxy}"
+          uri = URI.parse(http_proxy)
+          p_host, p_port = uri.host, uri.port
+          p_user, p_pass = uri.userinfo.split(/:/) if uri.userinfo
+        end
+
+        opts = {}
+        opts[:use_ssl] = use_ssl
+
+        if use_ssl
+          opts[:ca_file] = SSL.ca_cert_file_or_default
+        end
+
+        Net::HTTP.start(host, port, p_host, p_port, p_user, p_pass, use_ssl: use_ssl) do |http|
+          http.request_get path do |resp|
+            case resp
+            when Net::HTTPSuccess
+              digest = Digest::SHA2.new
+
+              resp.read_body do |chunk|
+                digest << chunk
+                out.write chunk
+              end
+
+              return [ :success, digest.hexdigest ]
+            when Net::HTTPRedirection
+              unless location = resp['location']
+                raise "received redirect but no location"
+              end
+
+              return [ :redirect, location ]
+            else
+              raise "received HTTP status code #{resp.code}"
+            end
+          end
+        end
+      end
 
-        unless expected == actual
-          log "checksum mismatch; expected=#{expected}; actual=#{actual}"
+      def verify_checksum(actual)
+        unless @checksum == actual
+          log "checksum mismatch; expected=#{@checksum}; actual=#{actual}"
           return false
         end
 
         true
+      rescue Exception => e
+        error "failed to read skylight agent archive; e=#{e.message}"
+        false
       end
 
-      def inflate(archive)
-        inflater = Zlib::Inflate.new(32 + Zlib::MAX_WBITS)
-        inflated = inflater.inflate(archive)
-        inflater.close
-        inflated
+      def basename
+        "skylight_#{@arch}.tar.gz"
       end
 
       def source_uri
-        "#{@source}/#{@version}/libskylight.#{@version}.#{@arch}.a.gz"
+        "#{@source}/#{@version}/#{basename}"
       end
 
       def deconstruct_uri(uri)

data/lib/skylight/util/platform.rb

@@ -0,0 +1,67 @@
+require 'rbconfig'
+
+# Used from extconf and to load libskylight
+module Skylight
+  module Util
+    module Platform
+      # Normalize the platform OS
+      OS = case os = RbConfig::CONFIG['host_os'].downcase
+      when /linux/
+        "linux"
+      when /darwin/
+        "darwin"
+      when /freebsd/
+        "freebsd"
+      when /netbsd/
+        "netbsd"
+      when /openbsd/
+        "openbsd"
+      when /sunos|solaris/
+        "solaris"
+      when /mingw|mswin/
+        "windows"
+      else
+        os
+      end
+
+      # Normalize the platform CPU
+      ARCH = case cpu = RbConfig::CONFIG['host_cpu'].downcase
+      when /amd64|x86_64/
+        "x86_64"
+      when /i?86|x86|i86pc/
+        "i386"
+      when /ppc|powerpc/
+        "powerpc"
+      when /^arm/
+        "arm"
+      else
+        cpu
+      end
+
+      LIBEXT = case OS
+      when /darwin/
+        'dylib'
+      when /linux|bsd|solaris/
+        'so'
+      when /windows|cygwin/
+        'dll'
+      else
+        'so'
+      end
+
+      TUPLE = "#{ARCH}-#{OS}"
+
+      def self.tuple
+        TUPLE
+      end
+
+      def self.libext
+        LIBEXT
+      end
+
+      def self.dlext
+        RbConfig::CONFIG['DLEXT']
+      end
+    end
+  end
+end