skylight 0.3.21 → 0.4.0.alpha1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (58) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +0 -4
  3. data/ext/extconf.rb +92 -47
  4. data/ext/libskylight.yml +4 -4
  5. data/ext/skylight_native.c +248 -286
  6. data/lib/skylight.rb +19 -114
  7. data/lib/skylight/api.rb +1 -1
  8. data/lib/skylight/config.rb +176 -146
  9. data/lib/skylight/data/cacert.pem +717 -719
  10. data/lib/skylight/formatters/http.rb +1 -1
  11. data/lib/skylight/instrumenter.rb +28 -35
  12. data/lib/skylight/native.rb +58 -72
  13. data/lib/skylight/normalizers.rb +0 -1
  14. data/lib/skylight/normalizers/active_record/sql.rb +0 -4
  15. data/lib/skylight/probes/excon/middleware.rb +3 -1
  16. data/lib/skylight/probes/net_http.rb +3 -1
  17. data/lib/skylight/subscriber.rb +0 -4
  18. data/lib/skylight/trace.rb +189 -0
  19. data/lib/skylight/util.rb +10 -12
  20. data/lib/skylight/util/hostname.rb +17 -0
  21. data/lib/skylight/util/http.rb +33 -36
  22. data/lib/skylight/util/logging.rb +20 -1
  23. data/lib/skylight/util/multi_io.rb +21 -0
  24. data/lib/skylight/util/native_ext_fetcher.rb +83 -69
  25. data/lib/skylight/util/platform.rb +67 -0
  26. data/lib/skylight/util/ssl.rb +50 -0
  27. data/lib/skylight/version.rb +1 -1
  28. metadata +9 -34
  29. data/ext/rust_support/ruby.h +0 -93
  30. data/ext/skylight.h +0 -85
  31. data/ext/skylight.map +0 -4
  32. data/ext/test/extconf.rb +0 -18
  33. data/ext/test/skylight_native_test.c +0 -82
  34. data/ext/test/skylight_test.h +0 -20
  35. data/lib/skylight/formatters.rb +0 -6
  36. data/lib/skylight/messages.rb +0 -21
  37. data/lib/skylight/messages/error.rb +0 -15
  38. data/lib/skylight/messages/hello.rb +0 -13
  39. data/lib/skylight/messages/trace.rb +0 -179
  40. data/lib/skylight/messages/trace_envelope.rb +0 -19
  41. data/lib/skylight/metrics.rb +0 -9
  42. data/lib/skylight/metrics/ewma.rb +0 -69
  43. data/lib/skylight/metrics/meter.rb +0 -58
  44. data/lib/skylight/metrics/process_cpu_gauge.rb +0 -65
  45. data/lib/skylight/metrics/process_mem_gauge.rb +0 -34
  46. data/lib/skylight/util/conversions.rb +0 -9
  47. data/lib/skylight/util/queue.rb +0 -96
  48. data/lib/skylight/util/task.rb +0 -172
  49. data/lib/skylight/util/uniform_sample.rb +0 -63
  50. data/lib/skylight/worker.rb +0 -19
  51. data/lib/skylight/worker/builder.rb +0 -73
  52. data/lib/skylight/worker/collector.rb +0 -274
  53. data/lib/skylight/worker/connection.rb +0 -87
  54. data/lib/skylight/worker/connection_set.rb +0 -56
  55. data/lib/skylight/worker/embedded.rb +0 -24
  56. data/lib/skylight/worker/metrics_reporter.rb +0 -104
  57. data/lib/skylight/worker/server.rb +0 -336
  58. data/lib/skylight/worker/standalone.rb +0 -421
@@ -1,18 +1,16 @@
1
1
  module Skylight
2
2
  # @api private
3
3
  module Util
4
- # Already defined by the native extension so we can't autoload
4
+ # Used from the main lib
5
+ require 'skylight/util/allocation_free'
5
6
  require 'skylight/util/clock'
7
+ require 'skylight/util/hostname'
8
+ require 'skylight/util/logging'
9
+ require 'skylight/util/ssl'
6
10
 
7
- autoload :AllocationFree, 'skylight/util/allocation_free'
8
- autoload :Conversions, 'skylight/util/conversions'
9
- autoload :Gzip, 'skylight/util/gzip'
10
- autoload :HTTP, 'skylight/util/http'
11
- autoload :Inflector, 'skylight/util/inflector'
12
- autoload :Logging, 'skylight/util/logging'
13
- autoload :Queue, 'skylight/util/queue'
14
- autoload :Task, 'skylight/util/task'
15
- autoload :UniformSample, 'skylight/util/uniform_sample'
16
- autoload :NativeExtFetcher, 'skylight/util/native_ext_fetcher'
11
+ # Used from the CLI
12
+ autoload :Gzip, 'skylight/util/gzip'
13
+ autoload :HTTP, 'skylight/util/http'
14
+ autoload :Inflector, 'skylight/util/inflector'
17
15
  end
18
- end
16
+ end
@@ -0,0 +1,17 @@
1
+ require 'socket'
2
+ require 'securerandom'
3
+
4
+ module Skylight
5
+ module Util
6
+ module Hostname
7
+ def self.default_hostname
8
+ if hostname = Socket.gethostname
9
+ hostname.strip!
10
+ hostname = nil if hostname == ''
11
+ end
12
+
13
+ hostname || "gen-#{SecureRandom.uuid}"
14
+ end
15
+ end
16
+ end
17
+ end
@@ -1,7 +1,9 @@
1
+ require 'uri'
1
2
  require 'json'
2
3
  require 'openssl'
3
4
  require 'net/http'
4
5
  require 'net/https'
6
+ require 'skylight/util/ssl'
5
7
 
6
8
  module Skylight
7
9
  module Util
@@ -15,7 +17,6 @@ module Skylight
15
17
  AUTHORIZATION = 'authorization'.freeze
16
18
  DEFLATE = 'deflate'.freeze
17
19
  GZIP = 'gzip'.freeze
18
- DEFAULT_CA_FILE = File.expand_path('../../data/cacert.pem', __FILE__)
19
20
 
20
21
  include Logging
21
22
 
@@ -30,44 +31,30 @@ module Skylight
30
31
  class StartError < StandardError; end
31
32
  class ReadResponseError < StandardError; end
32
33
 
33
- def initialize(config, service = :report, opts = {})
34
+ def initialize(config, service = :auth, opts = {})
34
35
  @config = config
35
- @ssl = config["#{service}.ssl"]
36
- @host = config["#{service}.host"]
37
- @port = config["#{service}.port"]
38
-
39
- @proxy_addr = config["#{service}.proxy_addr"]
40
- @proxy_port = config["#{service}.proxy_port"]
41
- @proxy_user = config["#{service}.proxy_user"]
42
- @proxy_pass = config["#{service}.proxy_pass"]
43
-
44
- @timeout = opts[:timeout] || 15
45
-
46
- unless @proxy_addr
47
- if http_proxy = ENV['HTTP_PROXY'] || ENV['http_proxy']
48
- uri = URI.parse(http_proxy)
49
- @proxy_addr, @proxy_port = uri.host, uri.port
50
- @proxy_user, @proxy_pass = (uri.userinfo || '').split(/:/)
51
- end
36
+
37
+ unless url = config["#{service}_url"]
38
+ raise ArgumentError, "no URL specified"
52
39
  end
53
40
 
54
- @deflate = config["#{service}.deflate"]
55
- @authentication = config[:'authentication']
56
- end
41
+ url = URI.parse(url)
57
42
 
58
- def self.detect_ca_cert_file!
59
- @ca_cert_file = false
60
- if defined?(OpenSSL::X509::DEFAULT_CERT_FILE)
61
- if OpenSSL::X509::DEFAULT_CERT_FILE
62
- @ca_cert_file = File.exist?(OpenSSL::X509::DEFAULT_CERT_FILE)
63
- end
43
+ @ssl = url.scheme == 'https'
44
+ @host = url.host
45
+ @port = url.port
46
+
47
+ if proxy_url = config[:proxy_url]
48
+ proxy_url = URI.parse(proxy_url)
49
+ @proxy_addr, @proxy_port = proxy_url.host, proxy_url.port
50
+ @proxy_user, @proxy_pass = (proxy_url.userinfo || '').split(/:/)
64
51
  end
65
- end
66
52
 
67
- detect_ca_cert_file!
53
+ @open_timeout = get_timeout(:connect, config, service, opts)
54
+ @read_timeout = get_timeout(:read, config, service, opts)
68
55
 
69
- def self.ca_cert_file?
70
- @ca_cert_file
56
+ @deflate = config["#{service}_http_deflate"]
57
+ @authentication = config[:'authentication']
71
58
  end
72
59
 
73
60
  def get(endpoint, hdrs = {})
@@ -87,6 +74,11 @@ module Skylight
87
74
 
88
75
  private
89
76
 
77
+ def get_timeout(type, config, service, opts)
78
+ config.duration_ms("#{service}_http_#{type}_timeout") ||
79
+ opts[:timeout] || 15
80
+ end
81
+
90
82
  def build_request(type, endpoint, hdrs, length=nil)
91
83
  headers = {}
92
84
 
@@ -131,14 +123,19 @@ module Skylight
131
123
  req.body = body
132
124
  end
133
125
 
134
- http = Net::HTTP.new(@host, @port, @proxy_addr, @proxy_port, @proxy_user, @proxy_pass)
126
+ http = Net::HTTP.new(@host, @port,
127
+ @proxy_addr, @proxy_port, @proxy_user, @proxy_pass)
135
128
 
136
- http.open_timeout = @timeout
137
- http.read_timeout = @timeout
129
+ http.open_timeout = @open_timeout
130
+ http.read_timeout = @read_timeout
138
131
 
139
132
  if @ssl
140
133
  http.use_ssl = true
141
- http.ca_file = DEFAULT_CA_FILE unless HTTP.ca_cert_file?
134
+
135
+ unless SSL.ca_cert_file?
136
+ http.ca_file = SSL.ca_cert_file_or_default
137
+ end
138
+
142
139
  http.verify_mode = OpenSSL::SSL::VERIFY_PEER
143
140
  end
144
141
 
@@ -2,8 +2,27 @@ require 'logger'
2
2
 
3
3
  module Skylight
4
4
  module Util
5
+ # Log both to the specified logger and STDOUT
6
+ class AlertLogger
7
+ def initialize(logger)
8
+ @logger = logger
9
+ end
10
+
11
+ def write(*args)
12
+ STDERR.write *args
13
+ @logger.<<(*args)
14
+ end
15
+
16
+ def close
17
+ end
18
+ end
19
+
5
20
  module Logging
6
- if ENV[TRACE_ENV_KEY]
21
+ def self.trace?
22
+ ENV[TRACE_ENV_KEY]
23
+ end
24
+
25
+ if trace?
7
26
  def trace(msg, *args)
8
27
  log :debug, msg, *args
9
28
  end
@@ -0,0 +1,21 @@
1
+ # Util allowing proxying writes to multiple location
2
+ # Used from extconf
3
+ module Skylight
4
+ module Util
5
+ class MultiIO
6
+
7
+ def initialize(*targets)
8
+ @targets = targets
9
+ end
10
+
11
+ def write(*args)
12
+ @targets.each {|t| t.write(*args)}
13
+ end
14
+
15
+ def close
16
+ @targets.each(&:close)
17
+ end
18
+
19
+ end
20
+ end
21
+ end
@@ -1,20 +1,20 @@
1
1
  require 'uri'
2
2
  require 'logger'
3
- require 'zlib'
4
3
  require 'net/http'
4
+ require 'fileutils'
5
5
  require 'digest/sha2'
6
+ require 'skylight/util/ssl'
6
7
 
7
- # Must require 'rubygems/platform' vs. just requiring 'rubygems' to avoid a
8
- # stack overflow bug on ruby 1.9.2.
9
- require 'rubygems/platform'
10
-
8
+ # Used from extconf.rb
11
9
  module Skylight
12
10
  module Util
13
11
  class NativeExtFetcher
14
- BASE_URL = "https://github.com/skylightio/skylight-rust/releases/download"
12
+ BASE_URL = "https://s3.amazonaws.com/skylight-agent-packages/skylight-native"
15
13
  MAX_REDIRECTS = 5
16
14
  MAX_RETRIES = 3
17
15
 
16
+ include FileUtils
17
+
18
18
  class FetchError < StandardError; end
19
19
 
20
20
  def self.fetch(opts = {})
@@ -25,13 +25,15 @@ module Skylight
25
25
  opts[:checksum],
26
26
  opts[:arch],
27
27
  opts[:required],
28
+ opts[:platform],
28
29
  opts[:logger] || Logger.new(STDOUT))
29
30
 
30
31
  fetcher.fetch
31
32
  end
32
33
 
33
- def initialize(source, target, version, checksum, arch, required, log)
34
+ def initialize(source, target, version, checksum, arch, required, platform, log)
34
35
  raise "source required" unless source
36
+ raise "target required" unless target
35
37
  raise "checksum required" unless checksum
36
38
  raise "arch required" unless arch
37
39
 
@@ -40,61 +42,42 @@ module Skylight
40
42
  @version = version
41
43
  @checksum = checksum
42
44
  @required = required
45
+ @platform = platform
43
46
  @arch = arch
44
47
  @log = log
45
48
  end
46
49
 
47
50
  def fetch
48
- log "fetching native ext; curr-platform=#{Gem::Platform.local.to_s}; " \
51
+ log "fetching native ext; curr-platform=#{@platform}; " \
49
52
  "requested-arch=#{@arch}; version=#{@version}"
50
53
 
51
- unless gziped = fetch_native_ext(source_uri, MAX_RETRIES, MAX_REDIRECTS)
54
+ tar_gz = "#{@target}/#{basename}"
55
+
56
+ unless sha2 = fetch_native_ext(source_uri, tar_gz, MAX_RETRIES, MAX_REDIRECTS)
52
57
  maybe_raise "could not fetch native extension"
53
58
  return
54
59
  end
55
60
 
56
- unless verify_checksum(gziped)
61
+ unless verify_checksum(sha2)
57
62
  maybe_raise "could not verify checksum"
58
63
  return
59
64
  end
60
65
 
61
- archive = inflate(gziped)
62
-
63
- if @target
64
- File.open @target, 'w' do |f|
65
- f.write(archive)
66
- end
66
+ Dir.chdir File.dirname(tar_gz) do
67
+ system "tar xzvf #{tar_gz}"
67
68
  end
68
69
 
69
- archive
70
- end
71
-
72
- def http_get(host, port, use_ssl, path)
73
- if http_proxy = ENV['HTTP_PROXY'] || ENV['http_proxy']
74
- log "connecting with proxy: #{http_proxy}"
75
- uri = URI.parse(http_proxy)
76
- p_host, p_port = uri.host, uri.port
77
- p_user, p_pass = uri.userinfo.split(/:/) if uri.userinfo
78
- end
79
-
80
- Net::HTTP.start(host, port, p_host, p_port, p_user, p_pass, use_ssl: use_ssl) do |http|
81
- case response = http.get(path)
82
- when Net::HTTPSuccess
83
- return [ :success, response.body ]
84
- when Net::HTTPRedirection
85
- unless location = response['location']
86
- raise "received redirect but no location"
87
- end
88
-
89
- return [ :redirect, location ]
90
- else
91
- raise "received HTTP status code #{response.code}"
92
- end
93
- end
70
+ true
71
+ ensure
72
+ rm_f tar_gz if tar_gz
94
73
  end
95
74
 
96
- def fetch_native_ext(uri, attempts, redirects)
75
+ def fetch_native_ext(uri, out, attempts, redirects)
97
76
  redirects.times do |i|
77
+ # Ensure the location is available
78
+ mkdir_p File.dirname(out)
79
+ rm_f out
80
+
98
81
  remaining_attempts = attempts
99
82
 
100
83
  log "attempting to fetch from remote; uri=#{uri}"
@@ -102,26 +85,20 @@ module Skylight
102
85
  begin
103
86
  host, port, use_ssl, path = deconstruct_uri(uri)
104
87
 
105
- status, body = http_get(host, port, use_ssl, path)
88
+ File.open out, 'w' do |f|
89
+ res, extra = http_get(host, port, use_ssl, path, f)
106
90
 
107
- case status
108
- when :success
109
- if body
110
- log "successfully downloaded native ext; body=#{body.bytesize}bytes"
91
+ case res
92
+ when :success
93
+ log "successfully downloaded native ext; out=#{out}"
94
+ return extra
111
95
  else
112
- log "response did not contain a body"
113
- end
114
-
115
- return body
116
- when :redirect
117
- log "fetching native ext; uri=#{uri}; redirected=#{body}"
118
- uri = body
96
+ log "fetching native ext; uri=#{uri}; redirected=#{res}"
97
+ uri = res
119
98
 
120
- next
121
- else
122
- raise "received unknown return; status=#{status}; body=#{body}"
99
+ next
100
+ end
123
101
  end
124
-
125
102
  rescue => e
126
103
  remaining_attempts -= 1
127
104
 
@@ -140,27 +117,64 @@ module Skylight
140
117
  return
141
118
  end
142
119
 
143
- def verify_checksum(archive)
144
- expected = @checksum
145
- actual = Digest::SHA2.hexdigest(archive)
120
+ def http_get(host, port, use_ssl, path, out)
121
+ if http_proxy = ENV['HTTP_PROXY'] || ENV['http_proxy']
122
+ log "connecting with proxy: #{http_proxy}"
123
+ uri = URI.parse(http_proxy)
124
+ p_host, p_port = uri.host, uri.port
125
+ p_user, p_pass = uri.userinfo.split(/:/) if uri.userinfo
126
+ end
127
+
128
+ opts = {}
129
+ opts[:use_ssl] = use_ssl
130
+
131
+ if use_ssl
132
+ opts[:ca_file] = SSL.ca_cert_file_or_default
133
+ end
134
+
135
+ Net::HTTP.start(host, port, p_host, p_port, p_user, p_pass, use_ssl: use_ssl) do |http|
136
+ http.request_get path do |resp|
137
+ case resp
138
+ when Net::HTTPSuccess
139
+ digest = Digest::SHA2.new
140
+
141
+ resp.read_body do |chunk|
142
+ digest << chunk
143
+ out.write chunk
144
+ end
145
+
146
+ return [ :success, digest.hexdigest ]
147
+ when Net::HTTPRedirection
148
+ unless location = resp['location']
149
+ raise "received redirect but no location"
150
+ end
151
+
152
+ return [ :redirect, location ]
153
+ else
154
+ raise "received HTTP status code #{resp.code}"
155
+ end
156
+ end
157
+ end
158
+ end
146
159
 
147
- unless expected == actual
148
- log "checksum mismatch; expected=#{expected}; actual=#{actual}"
160
+ def verify_checksum(actual)
161
+ unless @checksum == actual
162
+ log "checksum mismatch; expected=#{@checksum}; actual=#{actual}"
149
163
  return false
150
164
  end
151
165
 
152
166
  true
167
+ rescue Exception => e
168
+ error "failed to read skylight agent archive; e=#{e.message}"
169
+ false
153
170
  end
154
171
 
155
- def inflate(archive)
156
- inflater = Zlib::Inflate.new(32 + Zlib::MAX_WBITS)
157
- inflated = inflater.inflate(archive)
158
- inflater.close
159
- inflated
172
+ def basename
173
+ "skylight_#{@arch}.tar.gz"
160
174
  end
161
175
 
162
176
  def source_uri
163
- "#{@source}/#{@version}/libskylight.#{@version}.#{@arch}.a.gz"
177
+ "#{@source}/#{@version}/#{basename}"
164
178
  end
165
179
 
166
180
  def deconstruct_uri(uri)
@@ -0,0 +1,67 @@
1
+ require 'rbconfig'
2
+
3
+ # Used from extconf and to load libskylight
4
+ module Skylight
5
+ module Util
6
+ module Platform
7
+ # Normalize the platform OS
8
+ OS = case os = RbConfig::CONFIG['host_os'].downcase
9
+ when /linux/
10
+ "linux"
11
+ when /darwin/
12
+ "darwin"
13
+ when /freebsd/
14
+ "freebsd"
15
+ when /netbsd/
16
+ "netbsd"
17
+ when /openbsd/
18
+ "openbsd"
19
+ when /sunos|solaris/
20
+ "solaris"
21
+ when /mingw|mswin/
22
+ "windows"
23
+ else
24
+ os
25
+ end
26
+
27
+ # Normalize the platform CPU
28
+ ARCH = case cpu = RbConfig::CONFIG['host_cpu'].downcase
29
+ when /amd64|x86_64/
30
+ "x86_64"
31
+ when /i?86|x86|i86pc/
32
+ "i386"
33
+ when /ppc|powerpc/
34
+ "powerpc"
35
+ when /^arm/
36
+ "arm"
37
+ else
38
+ cpu
39
+ end
40
+
41
+ LIBEXT = case OS
42
+ when /darwin/
43
+ 'dylib'
44
+ when /linux|bsd|solaris/
45
+ 'so'
46
+ when /windows|cygwin/
47
+ 'dll'
48
+ else
49
+ 'so'
50
+ end
51
+
52
+ TUPLE = "#{ARCH}-#{OS}"
53
+
54
+ def self.tuple
55
+ TUPLE
56
+ end
57
+
58
+ def self.libext
59
+ LIBEXT
60
+ end
61
+
62
+ def self.dlext
63
+ RbConfig::CONFIG['DLEXT']
64
+ end
65
+ end
66
+ end
67
+ end