datadog 2.21.0 → 2.23.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (205) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +106 -2
  3. data/ext/LIBDATADOG_DEVELOPMENT.md +3 -0
  4. data/ext/datadog_profiling_native_extension/collectors_discrete_dynamic_sampler.c +1 -1
  5. data/ext/datadog_profiling_native_extension/collectors_stack.c +4 -0
  6. data/ext/datadog_profiling_native_extension/datadog_ruby_common.h +1 -1
  7. data/ext/datadog_profiling_native_extension/extconf.rb +6 -4
  8. data/ext/datadog_profiling_native_extension/heap_recorder.c +1 -1
  9. data/ext/libdatadog_api/datadog_ruby_common.h +1 -1
  10. data/ext/libdatadog_api/ddsketch.c +106 -0
  11. data/ext/libdatadog_api/feature_flags.c +554 -0
  12. data/ext/libdatadog_api/feature_flags.h +5 -0
  13. data/ext/libdatadog_api/init.c +5 -0
  14. data/ext/libdatadog_api/library_config.c +34 -25
  15. data/ext/libdatadog_api/process_discovery.c +19 -13
  16. data/ext/libdatadog_extconf_helpers.rb +1 -1
  17. data/lib/datadog/appsec/api_security/endpoint_collection/grape_route_serializer.rb +26 -0
  18. data/lib/datadog/appsec/api_security/endpoint_collection/rails_collector.rb +59 -0
  19. data/lib/datadog/appsec/api_security/endpoint_collection/rails_route_serializer.rb +29 -0
  20. data/lib/datadog/appsec/api_security/endpoint_collection/sinatra_route_serializer.rb +26 -0
  21. data/lib/datadog/appsec/api_security/endpoint_collection.rb +10 -0
  22. data/lib/datadog/appsec/api_security/route_extractor.rb +23 -6
  23. data/lib/datadog/appsec/api_security/sampler.rb +7 -4
  24. data/lib/datadog/appsec/assets/blocked.html +8 -0
  25. data/lib/datadog/appsec/assets/blocked.json +1 -1
  26. data/lib/datadog/appsec/assets/blocked.text +3 -1
  27. data/lib/datadog/appsec/assets/waf_rules/README.md +30 -36
  28. data/lib/datadog/appsec/assets/waf_rules/recommended.json +359 -4
  29. data/lib/datadog/appsec/assets/waf_rules/strict.json +43 -2
  30. data/lib/datadog/appsec/assets.rb +1 -1
  31. data/lib/datadog/appsec/compressed_json.rb +1 -1
  32. data/lib/datadog/appsec/configuration/settings.rb +9 -0
  33. data/lib/datadog/appsec/contrib/active_record/instrumentation.rb +3 -1
  34. data/lib/datadog/appsec/contrib/excon/ssrf_detection_middleware.rb +3 -2
  35. data/lib/datadog/appsec/contrib/faraday/ssrf_detection_middleware.rb +3 -1
  36. data/lib/datadog/appsec/contrib/graphql/gateway/watcher.rb +3 -1
  37. data/lib/datadog/appsec/contrib/rack/gateway/watcher.rb +9 -4
  38. data/lib/datadog/appsec/contrib/rack/request_middleware.rb +5 -1
  39. data/lib/datadog/appsec/contrib/rails/gateway/watcher.rb +7 -2
  40. data/lib/datadog/appsec/contrib/rails/patcher.rb +30 -0
  41. data/lib/datadog/appsec/contrib/rest_client/request_ssrf_detection_patch.rb +3 -1
  42. data/lib/datadog/appsec/contrib/sinatra/gateway/watcher.rb +10 -4
  43. data/lib/datadog/appsec/event.rb +12 -14
  44. data/lib/datadog/appsec/metrics/collector.rb +19 -3
  45. data/lib/datadog/appsec/metrics/telemetry_exporter.rb +2 -1
  46. data/lib/datadog/appsec/monitor/gateway/watcher.rb +4 -4
  47. data/lib/datadog/appsec/remote.rb +29 -13
  48. data/lib/datadog/appsec/response.rb +18 -4
  49. data/lib/datadog/appsec/security_engine/result.rb +28 -9
  50. data/lib/datadog/appsec/security_engine/runner.rb +17 -7
  51. data/lib/datadog/appsec/security_event.rb +5 -7
  52. data/lib/datadog/core/configuration/components.rb +44 -9
  53. data/lib/datadog/core/configuration/config_helper.rb +1 -1
  54. data/lib/datadog/core/configuration/settings.rb +14 -0
  55. data/lib/datadog/core/configuration/stable_config.rb +10 -0
  56. data/lib/datadog/core/configuration/supported_configurations.rb +330 -299
  57. data/lib/datadog/core/configuration.rb +1 -1
  58. data/lib/datadog/core/ddsketch.rb +19 -0
  59. data/lib/datadog/core/environment/ext.rb +6 -0
  60. data/lib/datadog/core/environment/process.rb +79 -0
  61. data/lib/datadog/core/environment/yjit.rb +2 -1
  62. data/lib/datadog/core/feature_flags.rb +61 -0
  63. data/lib/datadog/core/pin.rb +4 -8
  64. data/lib/datadog/core/process_discovery.rb +4 -2
  65. data/lib/datadog/core/remote/client/capabilities.rb +7 -0
  66. data/lib/datadog/core/remote/component.rb +4 -6
  67. data/lib/datadog/core/remote/transport/config.rb +2 -10
  68. data/lib/datadog/core/remote/transport/http/config.rb +9 -9
  69. data/lib/datadog/core/remote/transport/http/negotiation.rb +17 -8
  70. data/lib/datadog/core/remote/transport/http.rb +2 -0
  71. data/lib/datadog/core/remote/transport/negotiation.rb +2 -18
  72. data/lib/datadog/core/remote/worker.rb +25 -37
  73. data/lib/datadog/core/tag_builder.rb +0 -4
  74. data/lib/datadog/core/tag_normalizer.rb +84 -0
  75. data/lib/datadog/core/telemetry/component.rb +18 -3
  76. data/lib/datadog/core/telemetry/emitter.rb +6 -6
  77. data/lib/datadog/core/telemetry/event/app_endpoints_loaded.rb +30 -0
  78. data/lib/datadog/core/telemetry/event/app_started.rb +52 -49
  79. data/lib/datadog/core/telemetry/event/synth_app_client_configuration_change.rb +1 -1
  80. data/lib/datadog/core/telemetry/event.rb +1 -0
  81. data/lib/datadog/core/telemetry/logger.rb +2 -2
  82. data/lib/datadog/core/telemetry/logging.rb +2 -8
  83. data/lib/datadog/core/telemetry/transport/http/telemetry.rb +5 -6
  84. data/lib/datadog/core/telemetry/transport/telemetry.rb +1 -2
  85. data/lib/datadog/core/transport/http/client.rb +69 -0
  86. data/lib/datadog/core/transport/response.rb +4 -1
  87. data/lib/datadog/core/utils/array.rb +29 -0
  88. data/lib/datadog/{appsec/api_security → core/utils}/lru_cache.rb +10 -21
  89. data/lib/datadog/core/utils/network.rb +22 -1
  90. data/lib/datadog/core/utils/only_once_successful.rb +6 -2
  91. data/lib/datadog/core/utils.rb +2 -0
  92. data/lib/datadog/data_streams/configuration/settings.rb +49 -0
  93. data/lib/datadog/data_streams/configuration.rb +11 -0
  94. data/lib/datadog/data_streams/ext.rb +11 -0
  95. data/lib/datadog/data_streams/extensions.rb +16 -0
  96. data/lib/datadog/data_streams/pathway_context.rb +169 -0
  97. data/lib/datadog/data_streams/processor.rb +509 -0
  98. data/lib/datadog/data_streams/transport/http/api.rb +33 -0
  99. data/lib/datadog/data_streams/transport/http/client.rb +21 -0
  100. data/lib/datadog/data_streams/transport/http/stats.rb +87 -0
  101. data/lib/datadog/data_streams/transport/http.rb +41 -0
  102. data/lib/datadog/data_streams/transport/stats.rb +60 -0
  103. data/lib/datadog/data_streams.rb +100 -0
  104. data/lib/datadog/di/boot.rb +1 -0
  105. data/lib/datadog/di/component.rb +14 -16
  106. data/lib/datadog/di/context.rb +70 -0
  107. data/lib/datadog/di/el/compiler.rb +164 -0
  108. data/lib/datadog/di/el/evaluator.rb +159 -0
  109. data/lib/datadog/di/el/expression.rb +42 -0
  110. data/lib/datadog/di/el.rb +5 -0
  111. data/lib/datadog/di/error.rb +29 -0
  112. data/lib/datadog/di/instrumenter.rb +163 -48
  113. data/lib/datadog/di/probe.rb +55 -15
  114. data/lib/datadog/di/probe_builder.rb +39 -1
  115. data/lib/datadog/di/probe_manager.rb +13 -4
  116. data/lib/datadog/di/probe_notification_builder.rb +105 -67
  117. data/lib/datadog/di/proc_responder.rb +32 -0
  118. data/lib/datadog/di/serializer.rb +151 -7
  119. data/lib/datadog/di/transport/diagnostics.rb +2 -2
  120. data/lib/datadog/di/transport/http/diagnostics.rb +2 -4
  121. data/lib/datadog/di/transport/http/input.rb +2 -4
  122. data/lib/datadog/di/transport/http.rb +6 -2
  123. data/lib/datadog/di/transport/input.rb +64 -4
  124. data/lib/datadog/open_feature/component.rb +60 -0
  125. data/lib/datadog/open_feature/configuration.rb +27 -0
  126. data/lib/datadog/open_feature/evaluation_engine.rb +69 -0
  127. data/lib/datadog/open_feature/exposures/batch_builder.rb +32 -0
  128. data/lib/datadog/open_feature/exposures/buffer.rb +43 -0
  129. data/lib/datadog/open_feature/exposures/deduplicator.rb +30 -0
  130. data/lib/datadog/open_feature/exposures/event.rb +60 -0
  131. data/lib/datadog/open_feature/exposures/reporter.rb +40 -0
  132. data/lib/datadog/open_feature/exposures/worker.rb +116 -0
  133. data/lib/datadog/open_feature/ext.rb +14 -0
  134. data/lib/datadog/open_feature/native_evaluator.rb +38 -0
  135. data/lib/datadog/open_feature/noop_evaluator.rb +26 -0
  136. data/lib/datadog/open_feature/provider.rb +141 -0
  137. data/lib/datadog/open_feature/remote.rb +74 -0
  138. data/lib/datadog/open_feature/resolution_details.rb +35 -0
  139. data/lib/datadog/open_feature/transport.rb +72 -0
  140. data/lib/datadog/open_feature.rb +19 -0
  141. data/lib/datadog/opentelemetry/configuration/settings.rb +159 -0
  142. data/lib/datadog/opentelemetry/metrics.rb +110 -0
  143. data/lib/datadog/opentelemetry/sdk/configurator.rb +25 -1
  144. data/lib/datadog/opentelemetry/sdk/metrics_exporter.rb +38 -0
  145. data/lib/datadog/opentelemetry.rb +3 -0
  146. data/lib/datadog/profiling/collectors/code_provenance.rb +15 -6
  147. data/lib/datadog/profiling/collectors/cpu_and_wall_time_worker.rb +1 -1
  148. data/lib/datadog/profiling/collectors/idle_sampling_helper.rb +1 -1
  149. data/lib/datadog/profiling/profiler.rb +4 -0
  150. data/lib/datadog/profiling/tag_builder.rb +36 -3
  151. data/lib/datadog/profiling.rb +1 -2
  152. data/lib/datadog/single_step_instrument.rb +1 -1
  153. data/lib/datadog/tracing/component.rb +6 -17
  154. data/lib/datadog/tracing/configuration/dynamic.rb +2 -2
  155. data/lib/datadog/tracing/configuration/ext.rb +9 -0
  156. data/lib/datadog/tracing/configuration/settings.rb +77 -3
  157. data/lib/datadog/tracing/contrib/action_pack/action_controller/instrumentation.rb +4 -4
  158. data/lib/datadog/tracing/contrib/action_pack/utils.rb +1 -2
  159. data/lib/datadog/tracing/contrib/active_job/log_injection.rb +21 -7
  160. data/lib/datadog/tracing/contrib/active_job/patcher.rb +5 -1
  161. data/lib/datadog/tracing/contrib/aws/instrumentation.rb +4 -2
  162. data/lib/datadog/tracing/contrib/component.rb +2 -2
  163. data/lib/datadog/tracing/contrib/ethon/easy_patch.rb +4 -1
  164. data/lib/datadog/tracing/contrib/excon/configuration/settings.rb +11 -3
  165. data/lib/datadog/tracing/contrib/faraday/configuration/settings.rb +11 -7
  166. data/lib/datadog/tracing/contrib/grape/configuration/settings.rb +7 -3
  167. data/lib/datadog/tracing/contrib/graphql/configuration/settings.rb +7 -0
  168. data/lib/datadog/tracing/contrib/graphql/ext.rb +1 -0
  169. data/lib/datadog/tracing/contrib/graphql/unified_trace.rb +74 -44
  170. data/lib/datadog/tracing/contrib/http/configuration/settings.rb +11 -3
  171. data/lib/datadog/tracing/contrib/httpclient/configuration/settings.rb +11 -3
  172. data/lib/datadog/tracing/contrib/httprb/configuration/settings.rb +11 -3
  173. data/lib/datadog/tracing/contrib/kafka/instrumentation/consumer.rb +66 -0
  174. data/lib/datadog/tracing/contrib/kafka/instrumentation/producer.rb +66 -0
  175. data/lib/datadog/tracing/contrib/kafka/patcher.rb +14 -0
  176. data/lib/datadog/tracing/contrib/karafka/framework.rb +30 -0
  177. data/lib/datadog/tracing/contrib/karafka/monitor.rb +11 -0
  178. data/lib/datadog/tracing/contrib/karafka/patcher.rb +32 -0
  179. data/lib/datadog/tracing/contrib/rack/middlewares.rb +59 -27
  180. data/lib/datadog/tracing/contrib/rack/route_inference.rb +53 -0
  181. data/lib/datadog/tracing/contrib/rails/middlewares.rb +2 -2
  182. data/lib/datadog/tracing/contrib/rest_client/request_patch.rb +4 -1
  183. data/lib/datadog/tracing/contrib/roda/instrumentation.rb +3 -1
  184. data/lib/datadog/tracing/contrib/sinatra/tracer_middleware.rb +3 -1
  185. data/lib/datadog/tracing/contrib/status_range_matcher.rb +7 -0
  186. data/lib/datadog/tracing/contrib/waterdrop/configuration/settings.rb +27 -0
  187. data/lib/datadog/tracing/contrib/waterdrop/distributed/propagation.rb +48 -0
  188. data/lib/datadog/tracing/contrib/waterdrop/ext.rb +17 -0
  189. data/lib/datadog/tracing/contrib/waterdrop/integration.rb +43 -0
  190. data/lib/datadog/tracing/contrib/waterdrop/middleware.rb +46 -0
  191. data/lib/datadog/tracing/contrib/waterdrop/patcher.rb +46 -0
  192. data/lib/datadog/tracing/contrib/waterdrop/producer.rb +50 -0
  193. data/lib/datadog/tracing/contrib/waterdrop.rb +37 -0
  194. data/lib/datadog/tracing/contrib.rb +1 -0
  195. data/lib/datadog/tracing/metadata/ext.rb +9 -1
  196. data/lib/datadog/tracing/transport/http/client.rb +12 -26
  197. data/lib/datadog/tracing/transport/trace_formatter.rb +11 -0
  198. data/lib/datadog/tracing/transport/traces.rb +3 -5
  199. data/lib/datadog/version.rb +2 -2
  200. data/lib/datadog.rb +2 -0
  201. metadata +92 -16
  202. data/ext/libdatadog_api/macos_development.md +0 -26
  203. data/lib/datadog/core/remote/transport/http/client.rb +0 -49
  204. data/lib/datadog/core/telemetry/transport/http/client.rb +0 -49
  205. data/lib/datadog/di/transport/http/client.rb +0 -47
@@ -11,11 +11,45 @@ module Datadog
11
11
  # which is required to use features such as API Catalog.
12
12
  # DEV-3.0: This tracer should be the default one in the next major version.
13
13
  module UnifiedTrace
14
+ include ::GraphQL::Tracing::PlatformTrace
15
+
14
16
  def initialize(*args, **kwargs)
15
17
  @has_prepare_span = respond_to?(:prepare_span)
18
+
19
+ # Cache configuration values to avoid repeated lookups
20
+ config = Datadog.configuration.tracing[:graphql]
21
+ @service_name = config[:service_name]
22
+ @analytics_enabled = config[:analytics_enabled]
23
+ @analytics_sample_rate = config[:analytics_sample_rate]
24
+ @error_extensions_config = config[:error_extensions]
25
+
26
+ load_error_event_attributes(config[:error_tracking])
27
+
16
28
  super
17
29
  end
18
30
 
31
+ def load_error_event_attributes(error_tracking)
32
+ if error_tracking
33
+ @event_name = Tracing::Metadata::Ext::Errors::EVENT_NAME
34
+ @message_key = Tracing::Metadata::Ext::Errors::ATTRIBUTE_MESSAGE
35
+ @type_key = Tracing::Metadata::Ext::Errors::ATTRIBUTE_TYPE
36
+ @stacktrace_key = Tracing::Metadata::Ext::Errors::ATTRIBUTE_STACKTRACE
37
+ @locations_key = 'graphql.error.locations'
38
+ @path_key = 'graphql.error.path'
39
+ @extensions_key = 'graphql.error.extensions.'
40
+ else
41
+ @event_name = Ext::EVENT_QUERY_ERROR
42
+ @message_key = 'message'
43
+ @type_key = 'type'
44
+ @stacktrace_key = 'stacktrace'
45
+ @locations_key = 'locations'
46
+ @path_key = 'path'
47
+ @extensions_key = 'extensions.'
48
+ end
49
+ end
50
+
51
+ private :load_error_event_attributes
52
+
19
53
  def lex(*args, query_string:, **kwargs)
20
54
  trace(proc { super }, 'lex', query_string, query_string: query_string)
21
55
  end
@@ -56,7 +90,7 @@ module Datadog
56
90
  span.set_tag(Tracing::Metadata::Ext::TAG_KIND, Tracing::Metadata::Ext::SpanKind::TAG_SERVER)
57
91
 
58
92
  span.set_tag('graphql.source', query.query_string)
59
- span.set_tag('graphql.operation.type', query.selected_operation.operation_type)
93
+ span.set_tag('graphql.operation.type', query.selected_operation&.operation_type)
60
94
  if query.selected_operation_name
61
95
  span.set_tag(
62
96
  'graphql.operation.name',
@@ -130,8 +164,6 @@ module Datadog
130
164
  resolve_type_span(proc { super }, 'resolve_type_lazy', **kwargs)
131
165
  end
132
166
 
133
- include ::GraphQL::Tracing::PlatformTrace
134
-
135
167
  def platform_field_key(field, *args, **kwargs)
136
168
  field.path
137
169
  end
@@ -144,6 +176,26 @@ module Datadog
144
176
  "#{type.graphql_name}.resolve_type"
145
177
  end
146
178
 
179
+ # Serialize error's `locations` array as an array of Strings, given
180
+ # Span Events do not support hashes nested inside arrays.
181
+ #
182
+ # Here's an example in which `locations`:
183
+ # [
184
+ # {"line" => 3, "column" => 10},
185
+ # {"line" => 7, "column" => 8},
186
+ # ]
187
+ # is serialized as:
188
+ # ["3:10", "7:8"]
189
+ def self.serialize_error_locations(locations)
190
+ # locations are only provided by the `graphql` library when the error can
191
+ # be associated to a particular point in the query.
192
+ return [] if locations.nil?
193
+
194
+ locations.map do |location|
195
+ "#{location["line"]}:#{location["column"]}"
196
+ end
197
+ end
198
+
147
199
  private
148
200
 
149
201
  # Traces the given callable with the given trace key, resource, and kwargs.
@@ -156,16 +208,14 @@ module Datadog
156
208
  # @param kwargs [Hash] the arguments to pass to `prepare_span`
157
209
  # @yield [Span] the block to run before the trace, same as the `before` parameter
158
210
  def trace(callable, trace_key, resource, before = nil, after = nil, **kwargs, &before_block)
159
- config = Datadog.configuration.tracing[:graphql]
160
-
161
211
  Tracing.trace(
162
212
  "graphql.#{trace_key}",
163
213
  type: 'graphql',
164
214
  resource: resource,
165
- service: config[:service_name]
215
+ service: @service_name
166
216
  ) do |span|
167
- if Contrib::Analytics.enabled?(config[:analytics_enabled])
168
- Contrib::Analytics.set_sample_rate(span, config[:analytics_sample_rate])
217
+ if Contrib::Analytics.enabled?(@analytics_enabled)
218
+ Contrib::Analytics.set_sample_rate(span, @analytics_sample_rate)
169
219
  end
170
220
 
171
221
  # A sanity check for us.
@@ -198,34 +248,29 @@ module Datadog
198
248
  end
199
249
 
200
250
  def operation_resource(operation)
201
- if operation.name
251
+ if operation&.name
202
252
  "#{operation.operation_type} #{operation.name}"
203
253
  else
204
- "anonymous"
254
+ 'anonymous'
205
255
  end
206
256
  end
207
257
 
208
258
  # Create a Span Event for each error that occurs at query level.
209
- #
210
- # These are represented in the Datadog App as special GraphQL errors,
211
- # given their event name `dd.graphql.query.error`.
212
259
  def add_query_error_events(span, errors)
213
- capture_extensions = Datadog.configuration.tracing[:graphql][:error_extensions]
214
260
  errors.each do |error|
215
- extensions = if !capture_extensions.empty? && (extensions = error.extensions)
261
+ attributes = if !@error_extensions_config.empty? && (extensions = error.extensions)
216
262
  # Capture extensions, ensuring all values are primitives
217
263
  extensions.each_with_object({}) do |(key, value), hash|
218
- next unless capture_extensions.include?(key.to_s)
264
+ next unless @error_extensions_config.include?(key.to_s)
219
265
 
220
266
  value = case value
221
267
  when TrueClass, FalseClass, Integer, Float
222
268
  value
223
269
  else
224
- # Stringify anything that is not a boolean or a number
225
270
  value.to_s
226
271
  end
227
272
 
228
- hash["extensions.#{key}"] = value
273
+ hash[@extensions_key + key.to_s] = value
229
274
  end
230
275
  else
231
276
  {}
@@ -235,36 +280,21 @@ module Datadog
235
280
  # This is an unwritten contract in the `graphql` library.
236
281
  # See for an example: https://github.com/rmosolgo/graphql-ruby/blob/0afa241775e5a113863766cce126214dee093464/lib/graphql/execution_error.rb#L32
237
282
  graphql_error = error.to_h
238
- error = Core::Error.build_from(error)
239
-
240
- span.span_events << Datadog::Tracing::SpanEvent.new(
241
- Ext::EVENT_QUERY_ERROR,
242
- attributes: extensions.merge!(
243
- message: graphql_error['message'],
244
- type: error.type,
245
- stacktrace: error.backtrace,
246
- locations: serialize_error_locations(graphql_error['locations']),
247
- path: graphql_error['path'],
283
+ parsed_error = Core::Error.build_from(error)
284
+
285
+ span.span_events << SpanEvent.new(
286
+ @event_name,
287
+ attributes: attributes.merge!(
288
+ @type_key => parsed_error.type,
289
+ @stacktrace_key => parsed_error.backtrace,
290
+ @message_key => graphql_error['message'],
291
+ @locations_key =>
292
+ Datadog::Tracing::Contrib::GraphQL::UnifiedTrace.serialize_error_locations(graphql_error['locations']),
293
+ @path_key => graphql_error['path'],
248
294
  )
249
295
  )
250
296
  end
251
297
  end
252
-
253
- # Serialize error's `locations` array as an array of Strings, given
254
- # Span Events do not support hashes nested inside arrays.
255
- #
256
- # Here's an example in which `locations`:
257
- # [
258
- # {"line" => 3, "column" => 10},
259
- # {"line" => 7, "column" => 8},
260
- # ]
261
- # is serialized as:
262
- # ["3:10", "7:8"]
263
- def serialize_error_locations(locations)
264
- locations.map do |location|
265
- "#{location["line"]}:#{location["column"]}"
266
- end
267
- end
268
298
  end
269
299
  end
270
300
  end
@@ -46,9 +46,17 @@ module Datadog
46
46
 
47
47
  option :error_status_codes do |o|
48
48
  o.env Ext::ENV_ERROR_STATUS_CODES
49
- o.default 400...600
50
- o.setter do |v|
51
- Tracing::Contrib::StatusRangeMatcher.new(v) if v
49
+ o.setter do |value|
50
+ if value.nil?
51
+ # Fallback to global config, which is defaulted to client (400..499) + server (500..599)
52
+ # DEV-3.0: `http` is a client library, this should fall back to `http_error_statuses.client` only.
53
+ # We cannot change it without causing a breaking change.
54
+ client_global_error_statuses = Datadog.configuration.tracing.http_error_statuses.client
55
+ server_global_error_statuses = Datadog.configuration.tracing.http_error_statuses.server
56
+ client_global_error_statuses + server_global_error_statuses
57
+ else
58
+ Tracing::Contrib::StatusRangeMatcher.new(value)
59
+ end
52
60
  end
53
61
  o.env_parser do |v|
54
62
  Tracing::Contrib::StatusRangeEnvParser.call(v) if v
@@ -45,9 +45,17 @@ module Datadog
45
45
 
46
46
  option :error_status_codes do |o|
47
47
  o.env Ext::ENV_ERROR_STATUS_CODES
48
- o.default 400...600
49
- o.setter do |v|
50
- Tracing::Contrib::StatusRangeMatcher.new(v) if v
48
+ o.setter do |value|
49
+ if value.nil?
50
+ # Fallback to global config, which is defaulted to client (400..499) + server (500..599)
51
+ # DEV-3.0: `httpclient` is a client library, this should fall back to `http_error_statuses.client` only.
52
+ # We cannot change it without causing a breaking change.
53
+ client_global_error_statuses = Datadog.configuration.tracing.http_error_statuses.client
54
+ server_global_error_statuses = Datadog.configuration.tracing.http_error_statuses.server
55
+ client_global_error_statuses + server_global_error_statuses
56
+ else
57
+ Tracing::Contrib::StatusRangeMatcher.new(value)
58
+ end
51
59
  end
52
60
  o.env_parser do |v|
53
61
  Tracing::Contrib::StatusRangeEnvParser.call(v) if v
@@ -45,9 +45,17 @@ module Datadog
45
45
 
46
46
  option :error_status_codes do |o|
47
47
  o.env Ext::ENV_ERROR_STATUS_CODES
48
- o.default 400...600
49
- o.setter do |v|
50
- Tracing::Contrib::StatusRangeMatcher.new(v) if v
48
+ o.setter do |value|
49
+ if value.nil?
50
+ # Fallback to global config, which is defaulted to client (400..499) + server (500..599)
51
+ # DEV-3.0: `httprb` is a client library, this should fall back to `http_error_statuses.client` only.
52
+ # We cannot change it without causing a breaking change.
53
+ client_global_error_statuses = Datadog.configuration.tracing.http_error_statuses.client
54
+ server_global_error_statuses = Datadog.configuration.tracing.http_error_statuses.server
55
+ client_global_error_statuses + server_global_error_statuses
56
+ else
57
+ Tracing::Contrib::StatusRangeMatcher.new(value)
58
+ end
51
59
  end
52
60
  o.env_parser do |v|
53
61
  Tracing::Contrib::StatusRangeEnvParser.call(v) if v
@@ -0,0 +1,66 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Datadog
4
+ module Tracing
5
+ module Contrib
6
+ module Kafka
7
+ module Instrumentation
8
+ # Instrumentation for Kafka::Consumer
9
+ module Consumer
10
+ def self.prepended(base)
11
+ base.prepend(InstanceMethods)
12
+ end
13
+
14
+ # Instance methods for consumer instrumentation
15
+ module InstanceMethods
16
+ def each_message(**kwargs, &block)
17
+ return super unless Datadog::DataStreams.enabled?
18
+
19
+ wrapped_block = proc do |message|
20
+ Datadog.logger.debug { "Kafka each_message: DSM enabled for topic #{message.topic}" }
21
+
22
+ begin
23
+ headers = message.headers || {}
24
+ Datadog::DataStreams.set_consume_checkpoint(
25
+ type: 'kafka',
26
+ source: message.topic,
27
+ auto_instrumentation: true
28
+ ) { |key| headers[key] }
29
+ rescue => e
30
+ Datadog.logger.debug("Error setting DSM checkpoint: #{e.class}: #{e}")
31
+ end
32
+
33
+ yield(message) if block
34
+ end
35
+
36
+ super(**kwargs, &wrapped_block)
37
+ end
38
+
39
+ def each_batch(**kwargs, &block)
40
+ return super unless Datadog::DataStreams.enabled?
41
+
42
+ wrapped_block = proc do |batch|
43
+ Datadog.logger.debug { "Kafka each_batch: DSM enabled for topic #{batch.topic}" }
44
+
45
+ begin
46
+ Datadog::DataStreams.set_consume_checkpoint(
47
+ type: 'kafka',
48
+ source: batch.topic,
49
+ auto_instrumentation: true
50
+ )
51
+ rescue => e
52
+ Datadog.logger.debug("Error setting DSM checkpoint: #{e.class}: #{e}")
53
+ end
54
+
55
+ yield(batch) if block
56
+ end
57
+
58
+ super(**kwargs, &wrapped_block)
59
+ end
60
+ end
61
+ end
62
+ end
63
+ end
64
+ end
65
+ end
66
+ end
@@ -0,0 +1,66 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Datadog
4
+ module Tracing
5
+ module Contrib
6
+ module Kafka
7
+ module Instrumentation
8
+ # Instrumentation for Kafka::Producer
9
+ module Producer
10
+ def self.prepended(base)
11
+ base.prepend(InstanceMethods)
12
+ end
13
+
14
+ module InstanceMethods
15
+ def deliver_messages(**kwargs)
16
+ if Datadog::DataStreams.enabled?
17
+ begin
18
+ pending_messages = instance_variable_get(:@pending_message_queue)
19
+
20
+ if pending_messages && !pending_messages.empty?
21
+ pending_messages.each do |message|
22
+ message.headers ||= {}
23
+ Datadog::DataStreams.set_produce_checkpoint(
24
+ type: 'kafka',
25
+ destination: message.topic,
26
+ auto_instrumentation: true
27
+ ) do |key, value|
28
+ message.headers[key] = value
29
+ end
30
+ end
31
+ end
32
+ rescue => e
33
+ Datadog.logger.debug("Error setting DSM checkpoint: #{e.class}: #{e}")
34
+ end
35
+ end
36
+
37
+ super
38
+ end
39
+
40
+ def send_messages(messages, **kwargs)
41
+ if Datadog::DataStreams.enabled?
42
+ begin
43
+ messages.each do |message|
44
+ message[:headers] ||= {}
45
+ Datadog::DataStreams.set_produce_checkpoint(
46
+ type: 'kafka',
47
+ destination: message[:topic],
48
+ auto_instrumentation: true
49
+ ) do |key, value|
50
+ message[:headers][key] = value
51
+ end
52
+ end
53
+ rescue => e
54
+ Datadog.logger.debug("Error setting DSM checkpoint: #{e.class}: #{e}")
55
+ end
56
+ end
57
+
58
+ super
59
+ end
60
+ end
61
+ end
62
+ end
63
+ end
64
+ end
65
+ end
66
+ end
@@ -21,6 +21,20 @@ module Datadog
21
21
  def patch
22
22
  # Subscribe to Kafka events
23
23
  Events.subscribe!
24
+
25
+ # Apply monkey patches for additional instrumentation (e.g., DSM)
26
+ patch_producer if defined?(::Kafka::Producer)
27
+ patch_consumer if defined?(::Kafka::Consumer)
28
+ end
29
+
30
+ def patch_producer
31
+ require_relative 'instrumentation/producer'
32
+ ::Kafka::Producer.prepend(Instrumentation::Producer)
33
+ end
34
+
35
+ def patch_consumer
36
+ require_relative 'instrumentation/consumer'
37
+ ::Kafka::Consumer.prepend(Instrumentation::Consumer)
24
38
  end
25
39
  end
26
40
  end
@@ -0,0 +1,30 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Datadog
4
+ module Tracing
5
+ module Contrib
6
+ module Karafka
7
+ # Karafka framework code, used to essentially:
8
+ # - handle configuration entries which are specific to Datadog tracing
9
+ # - instrument parts of the framework when needed
10
+ module Framework
11
+ def self.setup
12
+ Datadog.configure do |datadog_config|
13
+ karafka_config = datadog_config.tracing[:karafka]
14
+ activate_waterdrop!(datadog_config, karafka_config)
15
+ end
16
+ end
17
+
18
+ # Apply relevant configuration from Karafka to WaterDrop
19
+ def self.activate_waterdrop!(datadog_config, karafka_config)
20
+ datadog_config.tracing.instrument(
21
+ :waterdrop,
22
+ service_name: karafka_config[:service_name],
23
+ distributed_tracing: karafka_config[:distributed_tracing],
24
+ )
25
+ end
26
+ end
27
+ end
28
+ end
29
+ end
30
+ end
@@ -47,6 +47,17 @@ module Datadog
47
47
  span.set_tag(Ext::TAG_CONSUMER, consumer)
48
48
  span.set_tag(Contrib::Ext::Messaging::TAG_DESTINATION, job.executor.topic.name)
49
49
  span.set_tag(Contrib::Ext::Messaging::TAG_SYSTEM, Ext::TAG_SYSTEM)
50
+
51
+ # DSM: Track consumer offset stats for batch processing
52
+ if Datadog.configuration.data_streams.enabled
53
+ job.messages.each do |message|
54
+ Datadog::DataStreams.track_kafka_consume(
55
+ job.executor.topic.name,
56
+ job.executor.partition,
57
+ message.metadata.offset
58
+ )
59
+ end
60
+ end
50
61
  end
51
62
 
52
63
  super
@@ -35,6 +35,24 @@ module Datadog
35
35
  Datadog::Tracing.continue_trace!(trace_digest) if trace_digest
36
36
  end
37
37
 
38
+ if Datadog::DataStreams.enabled?
39
+ begin
40
+ headers = if message.metadata.respond_to?(:raw_headers)
41
+ message.metadata.raw_headers
42
+ else
43
+ message.metadata.headers
44
+ end
45
+
46
+ Datadog::DataStreams.set_consume_checkpoint(
47
+ type: 'kafka',
48
+ source: message.topic,
49
+ auto_instrumentation: true
50
+ ) { |key| headers[key] }
51
+ rescue => e
52
+ Datadog.logger.debug("Error setting DSM checkpoint: #{e.class}: #{e}")
53
+ end
54
+ end
55
+
38
56
  Tracing.trace(Ext::SPAN_MESSAGE_CONSUME) do |span|
39
57
  span.set_tag(Ext::TAG_OFFSET, message.metadata.offset)
40
58
  span.set_tag(Contrib::Ext::Messaging::TAG_DESTINATION, message.topic)
@@ -48,6 +66,18 @@ module Datadog
48
66
  end
49
67
  end
50
68
 
69
+ module AppPatch
70
+ ONLY_ONCE_PER_APP = Hash.new { |h, key| h[key] = Core::Utils::OnlyOnce.new }
71
+
72
+ def initialized!
73
+ ONLY_ONCE_PER_APP[self].run do
74
+ # Activate tracing on components related to Karafka (e.g. WaterDrop)
75
+ Contrib::Karafka::Framework.setup
76
+ end
77
+ super
78
+ end
79
+ end
80
+
51
81
  # Patcher enables patching of 'karafka' module.
52
82
  module Patcher
53
83
  include Contrib::Patcher
@@ -60,9 +90,11 @@ module Datadog
60
90
 
61
91
  def patch
62
92
  require_relative 'monitor'
93
+ require_relative 'framework'
63
94
 
64
95
  ::Karafka::Instrumentation::Monitor.prepend(Monitor)
65
96
  ::Karafka::Messages::Messages.prepend(MessagesPatch)
97
+ ::Karafka::App.singleton_class.prepend(AppPatch)
66
98
  end
67
99
  end
68
100
  end
@@ -13,6 +13,7 @@ require_relative 'ext'
13
13
  require_relative 'header_collection'
14
14
  require_relative 'header_tagging'
15
15
  require_relative 'request_queue'
16
+ require_relative 'route_inference'
16
17
  require_relative 'trace_proxy_middleware'
17
18
 
18
19
  module Datadog
@@ -138,28 +139,7 @@ module Datadog
138
139
  request_span.set_tag(Tracing::Metadata::Ext::TAG_OPERATION, Ext::TAG_OPERATION_REQUEST)
139
140
  request_span.set_tag(Tracing::Metadata::Ext::TAG_KIND, Tracing::Metadata::Ext::SpanKind::TAG_SERVER)
140
141
 
141
- if status != 404 && (last_route = trace.get_tag(Tracing::Metadata::Ext::HTTP::TAG_ROUTE))
142
- last_script_name = trace.get_tag(Tracing::Metadata::Ext::HTTP::TAG_ROUTE_PATH) || ''
143
-
144
- # If the last_script_name is empty but the env['SCRIPT_NAME'] is NOT empty
145
- # then the current rack request was not routed and must be accounted for
146
- # which only happens in pure nested rack requests i.e /rack/rack/hello/world
147
- #
148
- # To account for the unaccounted nested rack requests of /rack/hello/world,
149
- # we use 'PATH_INFO knowing that rack cannot have named parameters
150
- if last_script_name == '' && env['SCRIPT_NAME'] && env['SCRIPT_NAME'] != ''
151
- last_script_name = last_route
152
- last_route = env['PATH_INFO']
153
- end
154
-
155
- # Clear the route and route path tags from the request trace to avoid possibility of misplacement
156
- trace.clear_tag(Tracing::Metadata::Ext::HTTP::TAG_ROUTE)
157
- trace.clear_tag(Tracing::Metadata::Ext::HTTP::TAG_ROUTE_PATH)
158
-
159
- # Ensure tags are placed in rack.request span as desired
160
- request_span.set_tag(Tracing::Metadata::Ext::HTTP::TAG_ROUTE, last_script_name + last_route)
161
- request_span.clear_tag(Tracing::Metadata::Ext::HTTP::TAG_ROUTE_PATH)
162
- end
142
+ set_route_and_endpoint_tags(trace: trace, request_span: request_span, status: status, env: env)
163
143
 
164
144
  # Set analytics sample rate
165
145
  if Contrib::Analytics.enabled?(configuration[:analytics_enabled])
@@ -218,16 +198,14 @@ module Datadog
218
198
  request_span.set_tag(Tracing::Metadata::Ext::HTTP::TAG_USER_AGENT, user_agent)
219
199
  end
220
200
 
221
- if request_span.get_tag(Tracing::Metadata::Ext::HTTP::TAG_ROUTE).nil? && status != 404
222
- request_span.set_tag(Tracing::Metadata::Ext::HTTP::TAG_ROUTE, env['PATH_INFO'])
223
- end
224
-
225
201
  HeaderTagging.tag_request_headers(request_span, request_header_collection, configuration)
226
202
  HeaderTagging.tag_response_headers(request_span, headers, configuration) if headers
227
203
 
228
204
  # detect if the status code is a 5xx and flag the request span as an error
229
205
  # unless it has been already set by the underlying framework
230
- request_span.status = 1 if status.to_s.start_with?('5') && request_span.status.zero?
206
+ if request_span.status.zero? && Datadog.configuration.tracing.http_error_statuses.server.include?(status)
207
+ request_span.status = Tracing::Metadata::Ext::Errors::STATUS
208
+ end
231
209
  end
232
210
  # rubocop:enable Metrics/AbcSize
233
211
  # rubocop:enable Metrics/CyclomaticComplexity
@@ -240,6 +218,60 @@ module Datadog
240
218
  Datadog.configuration.tracing[:rack]
241
219
  end
242
220
 
221
+ # rubocop:disable Metrics/AbcSize
222
+ # rubocop:disable Metrics/MethodLength
223
+ # rubocop:disable Metrics/CyclomaticComplexity
224
+ # rubocop:disable Metrics/PerceivedComplexity
225
+ def set_route_and_endpoint_tags(trace:, request_span:, status:, env:)
226
+ return if status == 404
227
+
228
+ if (last_route = trace.get_tag(Tracing::Metadata::Ext::HTTP::TAG_ROUTE))
229
+ last_script_name = trace.get_tag(Tracing::Metadata::Ext::HTTP::TAG_ROUTE_PATH) || ''
230
+
231
+ # This happens when processing requests to a nested rack application,
232
+ # when parent app is instrumented, and the nested app is not instrumented
233
+ #
234
+ # When resource_renaming.always_simplified_endpoint is set to true,
235
+ # we infer the route from the full request path.
236
+ if last_script_name == '' && env['SCRIPT_NAME'] != '' &&
237
+ !Datadog.configuration.tracing.resource_renaming.always_simplified_endpoint &&
238
+ (inferred_route = RouteInference.infer(env['PATH_INFO']))
239
+ set_endpoint_tag(request_span, last_route + inferred_route)
240
+ end
241
+
242
+ # Clear the route and route path tags from the request trace to avoid possibility of misplacement
243
+ trace.clear_tag(Tracing::Metadata::Ext::HTTP::TAG_ROUTE)
244
+ trace.clear_tag(Tracing::Metadata::Ext::HTTP::TAG_ROUTE_PATH)
245
+
246
+ # Ensure tags are placed in rack.request span as desired
247
+ request_span.set_tag(Tracing::Metadata::Ext::HTTP::TAG_ROUTE, last_script_name + last_route)
248
+ request_span.clear_tag(Tracing::Metadata::Ext::HTTP::TAG_ROUTE_PATH)
249
+ end
250
+
251
+ if Datadog.configuration.tracing.resource_renaming.always_simplified_endpoint ||
252
+ request_span.get_tag(Tracing::Metadata::Ext::HTTP::TAG_ROUTE).nil?
253
+ if (inferred_route = RouteInference.read_or_infer(env))
254
+ set_endpoint_tag(request_span, inferred_route) if inferred_route
255
+ end
256
+ elsif !request_span.get_tag(Tracing::Metadata::Ext::HTTP::TAG_ENDPOINT)
257
+ set_endpoint_tag(request_span, request_span.get_tag(Tracing::Metadata::Ext::HTTP::TAG_ROUTE))
258
+ end
259
+ end
260
+ # rubocop:enable Metrics/AbcSize
261
+ # rubocop:enable Metrics/MethodLength
262
+ # rubocop:enable Metrics/CyclomaticComplexity
263
+ # rubocop:enable Metrics/PerceivedComplexity
264
+
265
+ def set_endpoint_tag(request_span, value)
266
+ # In the first iteration, http.endpoint must be reported in 2 cases:
267
+ # 1. resource renaming is enabled
268
+ # 2. AppSec is enabled and resource renaming is disabled (by default, not explicitly)
269
+ if Datadog.configuration.tracing.resource_renaming.enabled ||
270
+ Datadog.configuration.appsec.enabled && Datadog.configuration.tracing.resource_renaming.options[:enabled].default_precedence?
271
+ request_span.set_tag(Tracing::Metadata::Ext::HTTP::TAG_ENDPOINT, value)
272
+ end
273
+ end
274
+
243
275
  # rubocop:disable Metrics/AbcSize
244
276
  # rubocop:disable Metrics/MethodLength
245
277
  def parse_url(env, original_env)