ddtrace 0.34.1 → 0.36.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (113) hide show
  1. checksums.yaml +5 -5
  2. data/.circleci/config.yml +58 -9
  3. data/.circleci/images/primary/Dockerfile-jruby-9.2 +77 -0
  4. data/.rubocop.yml +4 -0
  5. data/Appraisals +9 -7
  6. data/CHANGELOG.md +89 -3
  7. data/Rakefile +11 -2
  8. data/ddtrace.gemspec +5 -3
  9. data/docker-compose.yml +35 -0
  10. data/docs/DevelopmentGuide.md +1 -1
  11. data/docs/GettingStarted.md +89 -36
  12. data/lib/ddtrace.rb +1 -1
  13. data/lib/ddtrace/buffer.rb +9 -9
  14. data/lib/ddtrace/chunker.rb +34 -0
  15. data/lib/ddtrace/configuration.rb +28 -5
  16. data/lib/ddtrace/configuration/base.rb +1 -1
  17. data/lib/ddtrace/configuration/components.rb +154 -0
  18. data/lib/ddtrace/configuration/options.rb +1 -1
  19. data/lib/ddtrace/configuration/settings.rb +131 -63
  20. data/lib/ddtrace/context.rb +6 -6
  21. data/lib/ddtrace/context_flush.rb +1 -1
  22. data/lib/ddtrace/contrib/action_cable/instrumentation.rb +1 -1
  23. data/lib/ddtrace/contrib/action_pack/action_controller/instrumentation.rb +2 -2
  24. data/lib/ddtrace/contrib/action_view/events/render_partial.rb +1 -1
  25. data/lib/ddtrace/contrib/action_view/events/render_template.rb +1 -1
  26. data/lib/ddtrace/contrib/action_view/instrumentation/partial_renderer.rb +1 -1
  27. data/lib/ddtrace/contrib/action_view/instrumentation/template_renderer.rb +2 -2
  28. data/lib/ddtrace/contrib/action_view/patcher.rb +1 -1
  29. data/lib/ddtrace/contrib/active_record/events/instantiation.rb +1 -1
  30. data/lib/ddtrace/contrib/active_record/events/sql.rb +1 -1
  31. data/lib/ddtrace/contrib/active_support/cache/instrumentation.rb +2 -2
  32. data/lib/ddtrace/contrib/active_support/notifications/subscription.rb +2 -2
  33. data/lib/ddtrace/contrib/analytics.rb +1 -1
  34. data/lib/ddtrace/contrib/configuration/settings.rb +1 -1
  35. data/lib/ddtrace/contrib/dalli/patcher.rb +1 -1
  36. data/lib/ddtrace/contrib/dalli/quantize.rb +1 -1
  37. data/lib/ddtrace/contrib/elasticsearch/patcher.rb +1 -1
  38. data/lib/ddtrace/contrib/excon/middleware.rb +2 -2
  39. data/lib/ddtrace/contrib/extensions.rb +29 -5
  40. data/lib/ddtrace/contrib/faraday/patcher.rb +1 -1
  41. data/lib/ddtrace/contrib/grape/endpoint.rb +5 -5
  42. data/lib/ddtrace/contrib/grape/patcher.rb +1 -1
  43. data/lib/ddtrace/contrib/grpc/datadog_interceptor/client.rb +1 -1
  44. data/lib/ddtrace/contrib/grpc/datadog_interceptor/server.rb +2 -2
  45. data/lib/ddtrace/contrib/grpc/patcher.rb +1 -1
  46. data/lib/ddtrace/contrib/http/circuit_breaker.rb +8 -32
  47. data/lib/ddtrace/contrib/http/instrumentation.rb +2 -2
  48. data/lib/ddtrace/contrib/mongodb/subscribers.rb +2 -2
  49. data/lib/ddtrace/contrib/patchable.rb +1 -1
  50. data/lib/ddtrace/contrib/patcher.rb +3 -3
  51. data/lib/ddtrace/contrib/presto/instrumentation.rb +3 -3
  52. data/lib/ddtrace/contrib/presto/patcher.rb +1 -1
  53. data/lib/ddtrace/contrib/rack/middlewares.rb +2 -2
  54. data/lib/ddtrace/contrib/rack/patcher.rb +2 -2
  55. data/lib/ddtrace/contrib/rack/request_queue.rb +1 -1
  56. data/lib/ddtrace/contrib/rails/configuration/settings.rb +14 -0
  57. data/lib/ddtrace/contrib/rails/framework.rb +54 -48
  58. data/lib/ddtrace/contrib/rails/integration.rb +1 -1
  59. data/lib/ddtrace/contrib/rake/instrumentation.rb +2 -2
  60. data/lib/ddtrace/contrib/redis/quantize.rb +1 -1
  61. data/lib/ddtrace/contrib/resque/resque_job.rb +2 -2
  62. data/lib/ddtrace/contrib/sidekiq/tracing.rb +1 -1
  63. data/lib/ddtrace/contrib/sinatra/env.rb +20 -0
  64. data/lib/ddtrace/contrib/sinatra/ext.rb +6 -0
  65. data/lib/ddtrace/contrib/sinatra/patcher.rb +1 -0
  66. data/lib/ddtrace/contrib/sinatra/tracer.rb +98 -35
  67. data/lib/ddtrace/contrib/sinatra/tracer_middleware.rb +16 -13
  68. data/lib/ddtrace/correlation.rb +9 -6
  69. data/lib/ddtrace/diagnostics/health.rb +2 -6
  70. data/lib/ddtrace/encoding.rb +13 -39
  71. data/lib/ddtrace/event.rb +1 -1
  72. data/lib/ddtrace/ext/correlation.rb +1 -0
  73. data/lib/ddtrace/ext/diagnostics.rb +2 -0
  74. data/lib/ddtrace/ext/environment.rb +1 -0
  75. data/lib/ddtrace/ext/forced_tracing.rb +1 -1
  76. data/lib/ddtrace/logger.rb +3 -44
  77. data/lib/ddtrace/metrics.rb +5 -5
  78. data/lib/ddtrace/monkey.rb +1 -1
  79. data/lib/ddtrace/opentracer/global_tracer.rb +1 -1
  80. data/lib/ddtrace/pin.rb +18 -17
  81. data/lib/ddtrace/pipeline.rb +1 -1
  82. data/lib/ddtrace/propagation/http_propagator.rb +2 -2
  83. data/lib/ddtrace/runtime/cgroup.rb +1 -1
  84. data/lib/ddtrace/runtime/container.rb +1 -1
  85. data/lib/ddtrace/runtime/metrics.rb +5 -2
  86. data/lib/ddtrace/sampler.rb +2 -2
  87. data/lib/ddtrace/sampling/rule.rb +1 -1
  88. data/lib/ddtrace/sampling/rule_sampler.rb +1 -1
  89. data/lib/ddtrace/span.rb +4 -4
  90. data/lib/ddtrace/sync_writer.rb +3 -8
  91. data/lib/ddtrace/tracer.rb +26 -31
  92. data/lib/ddtrace/transport/http.rb +1 -1
  93. data/lib/ddtrace/transport/http/api/instance.rb +4 -0
  94. data/lib/ddtrace/transport/http/builder.rb +3 -5
  95. data/lib/ddtrace/transport/http/client.rb +7 -64
  96. data/lib/ddtrace/transport/http/response.rb +1 -1
  97. data/lib/ddtrace/transport/http/statistics.rb +1 -1
  98. data/lib/ddtrace/transport/http/traces.rb +10 -7
  99. data/lib/ddtrace/transport/io.rb +1 -1
  100. data/lib/ddtrace/transport/io/client.rb +2 -2
  101. data/lib/ddtrace/transport/io/response.rb +3 -1
  102. data/lib/ddtrace/transport/io/traces.rb +50 -3
  103. data/lib/ddtrace/transport/parcel.rb +0 -4
  104. data/lib/ddtrace/transport/statistics.rb +2 -2
  105. data/lib/ddtrace/transport/traces.rb +160 -10
  106. data/lib/ddtrace/utils.rb +1 -1
  107. data/lib/ddtrace/version.rb +2 -2
  108. data/lib/ddtrace/workers.rb +5 -13
  109. data/lib/ddtrace/workers/async.rb +2 -2
  110. data/lib/ddtrace/workers/runtime_metrics.rb +47 -0
  111. data/lib/ddtrace/workers/trace_writer.rb +199 -0
  112. data/lib/ddtrace/writer.rb +20 -27
  113. metadata +22 -32
@@ -18,7 +18,7 @@ module Datadog
18
18
  def default(options = {})
19
19
  new(
20
20
  options.fetch(:out, STDOUT),
21
- options.fetch(:encoder, Encoding::JSONEncoder::V2)
21
+ options.fetch(:encoder, Encoding::JSONEncoder)
22
22
  )
23
23
  end
24
24
  end
@@ -37,9 +37,9 @@ module Datadog
37
37
 
38
38
  # Log error
39
39
  if stats.consecutive_errors > 0
40
- Datadog::Logger.log.debug(message)
40
+ Datadog.logger.debug(message)
41
41
  else
42
- Datadog::Logger.log.error(message)
42
+ Datadog.logger.error(message)
43
43
  end
44
44
 
45
45
  # Update statistics
@@ -6,12 +6,14 @@ module Datadog
6
6
  # Response from HTTP transport for traces
7
7
  class Response
8
8
  include Transport::Response
9
+ include Transport::Traces::Response
9
10
 
10
11
  attr_reader \
11
12
  :result
12
13
 
13
- def initialize(result)
14
+ def initialize(result, trace_count = 1)
14
15
  @result = result
16
+ @trace_count = trace_count
15
17
  end
16
18
 
17
19
  def ok?
@@ -10,16 +10,15 @@ module Datadog
10
10
  module Traces
11
11
  # Response from HTTP transport for traces
12
12
  class Response < IO::Response
13
- include Transport::Traces::Response
14
13
  end
15
14
 
16
15
  # Extensions for HTTP client
17
16
  module Client
18
17
  def send_traces(traces)
19
18
  # Build a request
20
- req = Transport::Traces::Request.new(traces)
19
+ req = Transport::Traces::Request.new(Parcel.new(traces))
21
20
 
22
- send_request(req) do |out, request|
21
+ [send_request(req) do |out, request|
23
22
  # Encode trace data
24
23
  data = encode_data(encoder, request)
25
24
 
@@ -32,7 +31,55 @@ module Datadog
32
31
 
33
32
  # Generate response
34
33
  Traces::Response.new(result)
34
+ end]
35
+ end
36
+ end
37
+
38
+ # Encoder for IO-specific trace encoding
39
+ # API compliant when used with {JSONEncoder}.
40
+ module Encoder
41
+ ENCODED_IDS = [
42
+ :trace_id,
43
+ :span_id,
44
+ :parent_id
45
+ ].freeze
46
+
47
+ # Encodes a list of traces
48
+ def encode_traces(encoder, traces)
49
+ trace_hashes = traces.map do |trace|
50
+ encode_trace(trace)
35
51
  end
52
+
53
+ # Wrap traces & encode them
54
+ encoder.encode(traces: trace_hashes)
55
+ end
56
+
57
+ private
58
+
59
+ def encode_trace(trace)
60
+ # Convert each trace to hash
61
+ trace.map(&:to_hash).tap do |spans|
62
+ # Convert IDs to hexadecimal
63
+ spans.each do |span|
64
+ ENCODED_IDS.each do |id|
65
+ span[id] = span[id].to_s(16) if span.key?(id)
66
+ end
67
+ end
68
+ end
69
+ end
70
+ end
71
+
72
+ # Transfer object for list of traces
73
+ class Parcel
74
+ include Transport::Parcel
75
+ include Encoder
76
+
77
+ def count
78
+ data.length
79
+ end
80
+
81
+ def encode_with(encoder)
82
+ encode_traces(encoder, data)
36
83
  end
37
84
  end
38
85
 
@@ -8,10 +8,6 @@ module Datadog
8
8
  def initialize(data)
9
9
  @data = data
10
10
  end
11
-
12
- def encode_with(encoder)
13
- encoder.encode(data)
14
- end
15
11
  end
16
12
  end
17
13
  end
@@ -20,7 +20,7 @@ module Datadog
20
20
  end
21
21
 
22
22
  # Send health metrics
23
- Diagnostics::Health.metrics.send_metrics(
23
+ Datadog.health_metrics.send_metrics(
24
24
  metrics_for_response(response).values
25
25
  )
26
26
  end
@@ -37,7 +37,7 @@ module Datadog
37
37
  stats.consecutive_errors += 1
38
38
 
39
39
  # Send health metrics
40
- Diagnostics::Health.metrics.send_metrics(
40
+ Datadog.health_metrics.send_metrics(
41
41
  metrics_for_exception(exception).values
42
42
  )
43
43
  end
@@ -1,32 +1,182 @@
1
1
  require 'ddtrace/transport/parcel'
2
2
  require 'ddtrace/transport/request'
3
+ require 'ddtrace/chunker'
3
4
 
4
5
  module Datadog
5
6
  module Transport
6
7
  module Traces
7
- # Data transfer object for trace data
8
- class Parcel
8
+ # Data transfer object for encoded traces
9
+ class EncodedParcel
9
10
  include Transport::Parcel
10
11
 
11
- def count
12
- data.length
12
+ attr_reader :trace_count
13
+
14
+ def initialize(data, trace_count)
15
+ super(data)
16
+ @trace_count = trace_count
13
17
  end
14
18
 
15
- def encode_with(encoder)
16
- encoder.encode_traces(data)
19
+ def count
20
+ data.length
17
21
  end
18
22
  end
19
23
 
20
24
  # Traces request
21
25
  class Request < Transport::Request
22
- def initialize(traces)
23
- super(Parcel.new(traces))
24
- end
25
26
  end
26
27
 
27
28
  # Traces response
28
29
  module Response
29
- attr_reader :service_rates
30
+ attr_reader :service_rates, :trace_count
31
+ end
32
+
33
+ # Traces chunker
34
+ class Chunker
35
+ # Trace agent limit payload size of 10 MiB (since agent v5.11.0):
36
+ # https://github.com/DataDog/datadog-agent/blob/6.14.1/pkg/trace/api/api.go#L46
37
+ #
38
+ # We set the value to a conservative 5 MiB, in case network speed is slow.
39
+ DEFAULT_MAX_PAYLOAD_SIZE = 5 * 1024 * 1024
40
+
41
+ attr_reader :encoder, :max_size
42
+
43
+ #
44
+ # Single traces larger than +max_size+ will be discarded.
45
+ #
46
+ # @param encoder [Datadog::Encoding::Encoder]
47
+ # @param max_size [String] maximum acceptable payload size
48
+ def initialize(encoder, max_size: DEFAULT_MAX_PAYLOAD_SIZE)
49
+ @encoder = encoder
50
+ @max_size = max_size
51
+ end
52
+
53
+ # Encodes a list of traces in chunks.
54
+ # Before serializing, all traces are normalized. Trace nesting is not changed.
55
+ #
56
+ # @param traces [Enumerable<Trace>] list of traces
57
+ # @return [Enumerable[Array[Bytes,Integer]]] list of encoded chunks: each containing a byte array and
58
+ # number of traces
59
+ def encode_in_chunks(traces)
60
+ encoded_traces = traces.map { |t| encode_one(t) }.reject(&:nil?)
61
+
62
+ Datadog::Chunker.chunk_by_size(encoded_traces, max_size).map do |chunk|
63
+ [encoder.join(chunk), chunk.size]
64
+ end
65
+ end
66
+
67
+ private
68
+
69
+ def encode_one(trace)
70
+ encoded = Encoder.encode_trace(encoder, trace)
71
+
72
+ if encoded.size > max_size
73
+ # This single trace is too large, we can't flush it
74
+ Datadog.logger.debug { "Dropping trace. Payload too large: '#{trace.map(&:to_hash)}'" }
75
+ Datadog.health_metrics.transport_trace_too_large(1)
76
+
77
+ return nil
78
+ end
79
+
80
+ encoded
81
+ end
82
+ end
83
+
84
+ # Encodes traces using {Datadog::Encoding::Encoder} instances.
85
+ module Encoder
86
+ module_function
87
+
88
+ def encode_trace(encoder, trace)
89
+ encoder.encode(trace.map(&:to_hash))
90
+ end
91
+ end
92
+
93
+ # Sends traces based on transport API configuration.
94
+ #
95
+ # This class initializes the HTTP client, breaks down large
96
+ # batches of traces into smaller chunks and handles
97
+ # API version downgrade handshake.
98
+ class Transport
99
+ attr_reader :client, :apis, :default_api, :current_api_id
100
+
101
+ def initialize(apis, default_api)
102
+ @apis = apis
103
+ @default_api = default_api
104
+
105
+ change_api!(default_api)
106
+ end
107
+
108
+ def send_traces(traces)
109
+ encoder = current_api.encoder
110
+ chunker = Datadog::Transport::Traces::Chunker.new(encoder)
111
+
112
+ responses = chunker.encode_in_chunks(traces.lazy).map do |encoded_traces, trace_count|
113
+ request = Request.new(EncodedParcel.new(encoded_traces, trace_count))
114
+
115
+ client.send_payload(request).tap do |response|
116
+ if downgrade?(response)
117
+ downgrade!
118
+ return send_traces(traces)
119
+ end
120
+ end
121
+ end.force
122
+
123
+ Datadog.health_metrics.transport_chunked(responses.size)
124
+
125
+ responses
126
+ end
127
+
128
+ def stats
129
+ @client.stats
130
+ end
131
+
132
+ def current_api
133
+ apis[@current_api_id]
134
+ end
135
+
136
+ private
137
+
138
+ def downgrade?(response)
139
+ return false unless apis.fallbacks.key?(@current_api_id)
140
+ response.not_found? || response.unsupported?
141
+ end
142
+
143
+ def downgrade!
144
+ downgrade_api_id = apis.fallbacks[@current_api_id]
145
+ raise NoDowngradeAvailableError, @current_api_id if downgrade_api_id.nil?
146
+ change_api!(downgrade_api_id)
147
+ end
148
+
149
+ def change_api!(api_id)
150
+ raise UnknownApiVersionError, api_id unless apis.key?(api_id)
151
+ @current_api_id = api_id
152
+ @client = HTTP::Client.new(current_api)
153
+ end
154
+
155
+ # Raised when configured with an unknown API version
156
+ class UnknownApiVersionError < StandardError
157
+ attr_reader :version
158
+
159
+ def initialize(version)
160
+ @version = version
161
+ end
162
+
163
+ def message
164
+ "No matching transport API for version #{version}!"
165
+ end
166
+ end
167
+
168
+ # Raised when configured with an unknown API version
169
+ class NoDowngradeAvailableError < StandardError
170
+ attr_reader :version
171
+
172
+ def initialize(version)
173
+ @version = version
174
+ end
175
+
176
+ def message
177
+ "No downgrade from transport API version #{version} is available!"
178
+ end
179
+ end
30
180
  end
31
181
  end
32
182
  end
@@ -57,7 +57,7 @@ module Datadog
57
57
  str.encode(::Encoding::UTF_8)
58
58
  end
59
59
  rescue => e
60
- Logger.log.debug("Error encoding string in UTF-8: #{e}")
60
+ Datadog.logger.debug("Error encoding string in UTF-8: #{e}")
61
61
 
62
62
  options.fetch(:placeholder, STRING_PLACEHOLDER)
63
63
  end
@@ -1,8 +1,8 @@
1
1
  module Datadog
2
2
  module VERSION
3
3
  MAJOR = 0
4
- MINOR = 34
5
- PATCH = 1
4
+ MINOR = 36
5
+ PATCH = 0
6
6
  PRE = nil
7
7
 
8
8
  STRING = [MAJOR, MINOR, PATCH, PRE].compact.join('.')
@@ -1,5 +1,8 @@
1
1
  require 'time'
2
2
 
3
+ require 'ddtrace/workers/trace_writer'
4
+ require 'ddtrace/workers/runtime_metrics'
5
+
3
6
  require 'ddtrace/buffer'
4
7
  require 'ddtrace/runtime/metrics'
5
8
 
@@ -25,7 +28,6 @@ module Datadog
25
28
 
26
29
  # Callbacks
27
30
  @trace_task = options[:on_trace]
28
- @runtime_metrics_task = options[:on_runtime_metrics]
29
31
 
30
32
  # Intervals
31
33
  interval = options.fetch(:interval, DEFAULT_FLUSH_INTERVAL)
@@ -55,26 +57,18 @@ module Datadog
55
57
  # ensures that the thread will not die because of an exception.
56
58
  # TODO[manu]: findout the reason and reschedule the send if it's not
57
59
  # a fatal exception
58
- Datadog::Logger.log.error(
60
+ Datadog.logger.error(
59
61
  "Error during traces flush: dropped #{traces.length} items. Cause: #{e} Location: #{e.backtrace.first}"
60
62
  )
61
63
  end
62
64
  end
63
65
 
64
- def callback_runtime_metrics
65
- @runtime_metrics_task.call unless @runtime_metrics_task.nil?
66
- rescue StandardError => e
67
- Datadog::Logger.log.error(
68
- "Error during runtime metrics flush. Cause: #{e} Location: #{e.backtrace.first}"
69
- )
70
- end
71
-
72
66
  # Start the timer execution.
73
67
  def start
74
68
  @mutex.synchronize do
75
69
  return if @run
76
70
  @run = true
77
- Logger.log.debug("Starting thread in the process: #{Process.pid}")
71
+ Datadog.logger.debug("Starting thread in the process: #{Process.pid}")
78
72
  @worker = Thread.new { perform }
79
73
  end
80
74
  end
@@ -112,8 +106,6 @@ module Datadog
112
106
  loop do
113
107
  @back_off = flush_data ? @flush_interval : [@back_off * BACK_OFF_RATIO, BACK_OFF_MAX].min
114
108
 
115
- callback_runtime_metrics
116
-
117
109
  @mutex.synchronize do
118
110
  return if !@run && @trace_buffer.empty?
119
111
  @shutdown.wait(@mutex, @back_off) if @run # do not wait when shutting down
@@ -121,7 +121,7 @@ module Datadog
121
121
  @run_async = true
122
122
  @pid = Process.pid
123
123
  @error = nil
124
- Logger.log.debug("Starting thread in the process: #{Process.pid}")
124
+ Datadog.logger.debug("Starting thread in the process: #{Process.pid}")
125
125
 
126
126
  @worker = ::Thread.new do
127
127
  begin
@@ -129,7 +129,7 @@ module Datadog
129
129
  # rubocop:disable Lint/RescueException
130
130
  rescue Exception => e
131
131
  @error = e
132
- Logger.log.debug("Worker thread error. Cause #{e.message} Location: #{e.backtrace.first}")
132
+ Datadog.logger.debug("Worker thread error. Cause #{e.message} Location: #{e.backtrace.first}")
133
133
  raise
134
134
  end
135
135
  end
@@ -0,0 +1,47 @@
1
+ require 'forwardable'
2
+
3
+ require 'ddtrace/runtime/metrics'
4
+
5
+ require 'ddtrace/worker'
6
+ require 'ddtrace/workers/polling'
7
+
8
+ module Datadog
9
+ module Workers
10
+ # Emits runtime metrics asynchronously on a timed loop
11
+ class RuntimeMetrics < Worker
12
+ extend Forwardable
13
+ include Workers::Polling
14
+
15
+ attr_reader \
16
+ :metrics
17
+
18
+ def initialize(options = {})
19
+ @metrics = options.fetch(:metrics, Runtime::Metrics.new)
20
+
21
+ # Workers::Async::Thread settings
22
+ self.fork_policy = options.fetch(:fork_policy, Workers::Async::Thread::FORK_POLICY_STOP)
23
+
24
+ # Workers::IntervalLoop settings
25
+ self.interval = options[:interval] if options.key?(:interval)
26
+ self.back_off_ratio = options[:back_off_ratio] if options.key?(:back_off_ratio)
27
+ self.back_off_max = options[:back_off_max] if options.key?(:back_off_max)
28
+
29
+ self.enabled = options.fetch(:enabled, false)
30
+ end
31
+
32
+ def perform
33
+ metrics.flush
34
+ true
35
+ end
36
+
37
+ def associate_with_span(*args)
38
+ # Start the worker
39
+ metrics.associate_with_span(*args).tap { perform }
40
+ end
41
+
42
+ def_delegators \
43
+ :metrics,
44
+ :register_service
45
+ end
46
+ end
47
+ end