ddtrace 1.4.1 → 1.6.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (155) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +144 -1
  3. data/LICENSE-3rdparty.csv +1 -0
  4. data/ext/ddtrace_profiling_loader/ddtrace_profiling_loader.c +9 -2
  5. data/ext/ddtrace_profiling_loader/extconf.rb +17 -0
  6. data/ext/ddtrace_profiling_native_extension/NativeExtensionDesign.md +38 -2
  7. data/ext/ddtrace_profiling_native_extension/clock_id.h +1 -0
  8. data/ext/ddtrace_profiling_native_extension/clock_id_from_pthread.c +1 -0
  9. data/ext/ddtrace_profiling_native_extension/collectors_cpu_and_wall_time.c +517 -42
  10. data/ext/ddtrace_profiling_native_extension/collectors_cpu_and_wall_time.h +3 -0
  11. data/ext/ddtrace_profiling_native_extension/collectors_cpu_and_wall_time_worker.c +208 -30
  12. data/ext/ddtrace_profiling_native_extension/collectors_stack.c +156 -46
  13. data/ext/ddtrace_profiling_native_extension/collectors_stack.h +11 -2
  14. data/ext/ddtrace_profiling_native_extension/extconf.rb +11 -1
  15. data/ext/ddtrace_profiling_native_extension/http_transport.c +83 -64
  16. data/ext/ddtrace_profiling_native_extension/libdatadog_helpers.h +4 -4
  17. data/ext/ddtrace_profiling_native_extension/native_extension_helpers.rb +3 -4
  18. data/ext/ddtrace_profiling_native_extension/private_vm_api_access.c +59 -0
  19. data/ext/ddtrace_profiling_native_extension/private_vm_api_access.h +3 -0
  20. data/ext/ddtrace_profiling_native_extension/profiling.c +10 -0
  21. data/ext/ddtrace_profiling_native_extension/ruby_helpers.c +0 -1
  22. data/ext/ddtrace_profiling_native_extension/ruby_helpers.h +4 -2
  23. data/ext/ddtrace_profiling_native_extension/stack_recorder.c +45 -29
  24. data/ext/ddtrace_profiling_native_extension/stack_recorder.h +7 -7
  25. data/lib/datadog/appsec/assets/waf_rules/recommended.json +1169 -275
  26. data/lib/datadog/appsec/assets/waf_rules/risky.json +78 -78
  27. data/lib/datadog/appsec/assets/waf_rules/strict.json +278 -88
  28. data/lib/datadog/appsec/configuration/settings.rb +0 -2
  29. data/lib/datadog/appsec/contrib/rack/gateway/watcher.rb +25 -20
  30. data/lib/datadog/appsec/contrib/rack/reactive/request.rb +11 -11
  31. data/lib/datadog/appsec/contrib/rack/reactive/request_body.rb +11 -11
  32. data/lib/datadog/appsec/contrib/rack/reactive/response.rb +11 -11
  33. data/lib/datadog/appsec/contrib/rack/request.rb +3 -0
  34. data/lib/datadog/appsec/contrib/rack/request_middleware.rb +46 -19
  35. data/lib/datadog/appsec/contrib/rails/gateway/watcher.rb +7 -6
  36. data/lib/datadog/appsec/contrib/rails/integration.rb +1 -1
  37. data/lib/datadog/appsec/contrib/rails/reactive/action.rb +11 -11
  38. data/lib/datadog/appsec/contrib/rails/request.rb +3 -0
  39. data/lib/datadog/appsec/contrib/sinatra/gateway/watcher.rb +14 -12
  40. data/lib/datadog/appsec/contrib/sinatra/reactive/routed.rb +11 -11
  41. data/lib/datadog/appsec/event.rb +6 -10
  42. data/lib/datadog/appsec/instrumentation/gateway.rb +16 -2
  43. data/lib/datadog/appsec/processor.rb +18 -2
  44. data/lib/datadog/ci/ext/environment.rb +16 -4
  45. data/lib/datadog/core/configuration/agent_settings_resolver.rb +0 -3
  46. data/lib/datadog/core/configuration/components.rb +28 -16
  47. data/lib/datadog/core/configuration/settings.rb +127 -8
  48. data/lib/datadog/core/configuration.rb +1 -1
  49. data/lib/datadog/core/diagnostics/environment_logger.rb +5 -1
  50. data/lib/datadog/core/header_collection.rb +41 -0
  51. data/lib/datadog/core/telemetry/collector.rb +0 -2
  52. data/lib/datadog/core/utils/compression.rb +5 -1
  53. data/lib/datadog/core/workers/async.rb +0 -2
  54. data/lib/datadog/core.rb +0 -54
  55. data/lib/datadog/opentracer/tracer.rb +4 -6
  56. data/lib/datadog/profiling/collectors/cpu_and_wall_time.rb +12 -2
  57. data/lib/datadog/profiling/collectors/cpu_and_wall_time_worker.rb +5 -3
  58. data/lib/datadog/profiling/collectors/old_stack.rb +1 -1
  59. data/lib/datadog/profiling/exporter.rb +2 -4
  60. data/lib/datadog/profiling/http_transport.rb +1 -1
  61. data/lib/datadog/profiling.rb +1 -1
  62. data/lib/datadog/tracing/client_ip.rb +164 -0
  63. data/lib/datadog/tracing/configuration/ext.rb +14 -0
  64. data/lib/datadog/tracing/contrib/aws/instrumentation.rb +2 -0
  65. data/lib/datadog/tracing/contrib/aws/services.rb +0 -2
  66. data/lib/datadog/tracing/contrib/dalli/ext.rb +1 -0
  67. data/lib/datadog/tracing/contrib/dalli/instrumentation.rb +4 -0
  68. data/lib/datadog/tracing/contrib/elasticsearch/ext.rb +2 -0
  69. data/lib/datadog/tracing/contrib/elasticsearch/patcher.rb +3 -0
  70. data/lib/datadog/tracing/contrib/ethon/easy_patch.rb +2 -2
  71. data/lib/datadog/tracing/contrib/ethon/multi_patch.rb +2 -0
  72. data/lib/datadog/tracing/contrib/excon/middleware.rb +2 -0
  73. data/lib/datadog/tracing/contrib/ext.rb +25 -0
  74. data/lib/datadog/tracing/contrib/faraday/middleware.rb +3 -2
  75. data/lib/datadog/tracing/contrib/grape/endpoint.rb +0 -2
  76. data/lib/datadog/tracing/contrib/graphql/configuration/settings.rb +1 -1
  77. data/lib/datadog/tracing/contrib/grpc/datadog_interceptor/client.rb +5 -0
  78. data/lib/datadog/tracing/contrib/grpc/datadog_interceptor/server.rb +7 -1
  79. data/lib/datadog/tracing/contrib/grpc/ext.rb +2 -0
  80. data/lib/datadog/tracing/contrib/hanami/action_tracer.rb +47 -0
  81. data/lib/datadog/tracing/contrib/hanami/configuration/settings.rb +22 -0
  82. data/lib/datadog/tracing/contrib/hanami/ext.rb +24 -0
  83. data/lib/datadog/tracing/contrib/hanami/integration.rb +44 -0
  84. data/lib/datadog/tracing/contrib/hanami/patcher.rb +33 -0
  85. data/lib/datadog/tracing/contrib/hanami/plugin.rb +23 -0
  86. data/lib/datadog/tracing/contrib/hanami/renderer_policy_tracing.rb +41 -0
  87. data/lib/datadog/tracing/contrib/hanami/router_tracing.rb +44 -0
  88. data/lib/datadog/tracing/contrib/http/instrumentation.rb +2 -0
  89. data/lib/datadog/tracing/contrib/httpclient/instrumentation.rb +2 -0
  90. data/lib/datadog/tracing/contrib/httprb/instrumentation.rb +2 -0
  91. data/lib/datadog/tracing/contrib/mongodb/ext.rb +7 -0
  92. data/lib/datadog/tracing/contrib/mongodb/subscribers.rb +4 -0
  93. data/lib/datadog/tracing/contrib/mysql2/configuration/settings.rb +12 -0
  94. data/lib/datadog/tracing/contrib/mysql2/ext.rb +1 -0
  95. data/lib/datadog/tracing/contrib/mysql2/instrumentation.rb +16 -0
  96. data/lib/datadog/tracing/contrib/pg/configuration/settings.rb +12 -0
  97. data/lib/datadog/tracing/contrib/pg/ext.rb +2 -1
  98. data/lib/datadog/tracing/contrib/pg/instrumentation.rb +38 -21
  99. data/lib/datadog/tracing/contrib/propagation/sql_comment/comment.rb +43 -0
  100. data/lib/datadog/tracing/contrib/propagation/sql_comment/ext.rb +32 -0
  101. data/lib/datadog/tracing/contrib/propagation/sql_comment/mode.rb +28 -0
  102. data/lib/datadog/tracing/contrib/propagation/sql_comment.rb +49 -0
  103. data/lib/datadog/tracing/contrib/rack/header_collection.rb +35 -0
  104. data/lib/datadog/tracing/contrib/rack/middlewares.rb +105 -43
  105. data/lib/datadog/tracing/contrib/redis/ext.rb +2 -0
  106. data/lib/datadog/tracing/contrib/redis/instrumentation.rb +4 -2
  107. data/lib/datadog/tracing/contrib/redis/integration.rb +2 -1
  108. data/lib/datadog/tracing/contrib/redis/patcher.rb +40 -0
  109. data/lib/datadog/tracing/contrib/redis/tags.rb +5 -0
  110. data/lib/datadog/tracing/contrib/rest_client/request_patch.rb +2 -0
  111. data/lib/datadog/tracing/contrib/sinatra/env.rb +12 -23
  112. data/lib/datadog/tracing/contrib/sinatra/ext.rb +7 -3
  113. data/lib/datadog/tracing/contrib/sinatra/patcher.rb +2 -2
  114. data/lib/datadog/tracing/contrib/sinatra/tracer.rb +8 -80
  115. data/lib/datadog/tracing/contrib/sinatra/tracer_middleware.rb +14 -9
  116. data/lib/datadog/tracing/contrib/utils/quantization/http.rb +92 -10
  117. data/lib/datadog/tracing/contrib.rb +1 -0
  118. data/lib/datadog/tracing/distributed/datadog_tags_codec.rb +84 -0
  119. data/lib/datadog/tracing/distributed/headers/datadog.rb +122 -30
  120. data/lib/datadog/tracing/distributed/headers/ext.rb +2 -0
  121. data/lib/datadog/tracing/flush.rb +57 -35
  122. data/lib/datadog/tracing/metadata/ext.rb +11 -9
  123. data/lib/datadog/tracing/metadata/tagging.rb +9 -0
  124. data/lib/datadog/tracing/propagation/http.rb +9 -1
  125. data/lib/datadog/tracing/sampling/ext.rb +31 -0
  126. data/lib/datadog/tracing/sampling/priority_sampler.rb +46 -4
  127. data/lib/datadog/tracing/sampling/rate_by_key_sampler.rb +8 -9
  128. data/lib/datadog/tracing/sampling/rate_by_service_sampler.rb +29 -5
  129. data/lib/datadog/tracing/sampling/rate_limiter.rb +3 -0
  130. data/lib/datadog/tracing/sampling/rate_sampler.rb +20 -3
  131. data/lib/datadog/tracing/sampling/rule_sampler.rb +4 -3
  132. data/lib/datadog/tracing/sampling/span/ext.rb +25 -0
  133. data/lib/datadog/tracing/sampling/span/matcher.rb +9 -0
  134. data/lib/datadog/tracing/sampling/span/rule.rb +82 -0
  135. data/lib/datadog/tracing/sampling/span/rule_parser.rb +104 -0
  136. data/lib/datadog/tracing/sampling/span/sampler.rb +75 -0
  137. data/lib/datadog/tracing/span_operation.rb +0 -2
  138. data/lib/datadog/tracing/trace_digest.rb +3 -0
  139. data/lib/datadog/tracing/trace_operation.rb +32 -3
  140. data/lib/datadog/tracing/trace_segment.rb +7 -2
  141. data/lib/datadog/tracing/tracer.rb +34 -6
  142. data/lib/datadog/tracing/writer.rb +7 -0
  143. data/lib/ddtrace/transport/trace_formatter.rb +7 -0
  144. data/lib/ddtrace/transport/traces.rb +3 -1
  145. data/lib/ddtrace/version.rb +1 -1
  146. metadata +36 -18
  147. data/lib/datadog/profiling/old_ext.rb +0 -42
  148. data/lib/datadog/profiling/transport/http/api/endpoint.rb +0 -85
  149. data/lib/datadog/profiling/transport/http/api/instance.rb +0 -38
  150. data/lib/datadog/profiling/transport/http/api/spec.rb +0 -42
  151. data/lib/datadog/profiling/transport/http/api.rb +0 -45
  152. data/lib/datadog/profiling/transport/http/builder.rb +0 -30
  153. data/lib/datadog/profiling/transport/http/client.rb +0 -37
  154. data/lib/datadog/profiling/transport/http/response.rb +0 -21
  155. data/lib/datadog/profiling/transport/http.rb +0 -118
@@ -17,69 +17,12 @@ module Datadog
17
17
  # Datadog::Tracing::Contrib::Sinatra::Tracer is a Sinatra extension which traces
18
18
  # requests.
19
19
  module Tracer
20
- def route(verb, action, *)
21
- # Keep track of the route name when the app is instantiated for an
22
- # incoming request.
23
- condition do
24
- # If the option to prepend script names is enabled, then
25
- # prepend the script name from the request onto the action.
26
- #
27
- # DEV: env['sinatra.route'] already exists with very similar information,
28
- # DEV: but doesn't account for our `resource_script_names` logic.
29
- #
30
- @datadog_route = if Datadog.configuration.tracing[:sinatra][:resource_script_names]
31
- "#{request.script_name}#{action}"
32
- else
33
- action
34
- end
35
- end
36
-
37
- super
38
- end
39
-
40
20
  def self.registered(app)
41
21
  app.use TracerMiddleware, app_instance: app
42
-
43
- app.after do
44
- next unless Tracing.enabled?
45
-
46
- span = Sinatra::Env.datadog_span(env, app)
47
-
48
- # TODO: `route` should *only* be populated if @datadog_route is defined.
49
- # TODO: If @datadog_route is not defined, then this Sinatra app is not responsible
50
- # TODO: for handling this request.
51
- # TODO:
52
- # TODO: This change would be BREAKING for any Sinatra app (classic or modular),
53
- # TODO: as it affects the `resource` value for requests not handled by the Sinatra app.
54
- # TODO: Currently we use "#{method} #{path}" in such aces, but `path` is the raw,
55
- # TODO: high-cardinality HTTP path, and can contain PII.
56
- # TODO:
57
- # TODO: The value we should use as the `resource` when the Sinatra app is not
58
- # TODO: responsible for the request is a tricky subject.
59
- # TODO: The best option is a value that clearly communicates that this app did not
60
- # TODO: handle this request. It's important to keep in mind that an unhandled request
61
- # TODO: by this Sinatra app might still be handled by another Rack middleware (which can
62
- # TODO: be a Sinatra app itself) or it might just 404 if not handled at all.
63
- # TODO:
64
- # TODO: A possible value for `resource` could set a high level description, e.g.
65
- # TODO: `request.request_method`, given we don't have the response object available yet.
66
- route = if defined?(@datadog_route)
67
- @datadog_route
68
- else
69
- # Fallback in case no routes have matched
70
- request.path
71
- end
72
-
73
- span.resource = "#{request.request_method} #{route}"
74
- span.set_tag(Ext::TAG_ROUTE_PATH, route)
75
- end
76
22
  end
77
23
 
78
24
  # Method overrides for Sinatra::Base
79
25
  module Base
80
- MISSING_REQUEST_SPAN_ONLY_ONCE = Core::Utils::OnlyOnce.new
81
- private_constant :MISSING_REQUEST_SPAN_ONLY_ONCE
82
-
83
26
  def render(engine, data, *)
84
27
  return super unless Tracing.enabled?
85
28
 
@@ -102,19 +45,21 @@ module Datadog
102
45
 
103
46
  # Invoked when a matching route is found.
104
47
  # This method yields directly to user code.
105
- # rubocop:disable Metrics/MethodLength
106
48
  def route_eval
107
49
  configuration = Datadog.configuration.tracing[:sinatra]
108
50
  return super unless Tracing.enabled?
109
51
 
52
+ datadog_route = Sinatra::Env.route_path(env)
53
+
110
54
  Tracing.trace(
111
55
  Ext::SPAN_ROUTE,
112
56
  service: configuration[:service_name],
113
57
  span_type: Tracing::Metadata::Ext::HTTP::TYPE_INBOUND,
114
- resource: "#{request.request_method} #{@datadog_route}",
58
+ resource: "#{request.request_method} #{datadog_route}",
115
59
  ) do |span, trace|
116
60
  span.set_tag(Ext::TAG_APP_NAME, settings.name || settings.superclass.name)
117
- span.set_tag(Ext::TAG_ROUTE_PATH, @datadog_route)
61
+ span.set_tag(Ext::TAG_ROUTE_PATH, datadog_route)
62
+
118
63
  if request.script_name && !request.script_name.empty?
119
64
  span.set_tag(Ext::TAG_SCRIPT_NAME, request.script_name)
120
65
  end
@@ -124,32 +69,15 @@ module Datadog
124
69
 
125
70
  trace.resource = span.resource
126
71
 
127
- sinatra_request_span =
128
- if self.class <= ::Sinatra::Application # Classic style (top-level) application
129
- Sinatra::Env.datadog_span(env, ::Sinatra::Application)
130
- else
131
- Sinatra::Env.datadog_span(env, self.class)
132
- end
133
- if sinatra_request_span
134
- sinatra_request_span.resource = span.resource
135
- else
136
- MISSING_REQUEST_SPAN_ONLY_ONCE.run do
137
- Datadog.logger.warn do
138
- 'Sinatra integration is misconfigured, reported traces will be missing request metadata ' \
139
- 'such as path and HTTP status code. ' \
140
- 'Did you forget to add `register Datadog::Tracing::Contrib::Sinatra::Tracer` to your ' \
141
- '`Sinatra::Base` subclass? ' \
142
- 'See <https://docs.datadoghq.com/tracing/setup_overview/setup/ruby/#sinatra> for more details.'
143
- end
144
- end
145
- end
72
+ sinatra_request_span = Sinatra::Env.datadog_span(env)
73
+
74
+ sinatra_request_span.resource = span.resource
146
75
 
147
76
  Contrib::Analytics.set_measured(span)
148
77
 
149
78
  super
150
79
  end
151
80
  end
152
- # rubocop:enable Metrics/MethodLength
153
81
  end
154
82
  end
155
83
  end
@@ -25,10 +25,12 @@ module Datadog
25
25
  def call(env)
26
26
  # Set the trace context (e.g. distributed tracing)
27
27
  if configuration[:distributed_tracing] && Tracing.active_trace.nil?
28
- original_trace = Propagation::HTTP.extract(env)
28
+ original_trace = Tracing::Propagation::HTTP.extract(env)
29
29
  Tracing.continue_trace!(original_trace)
30
30
  end
31
31
 
32
+ return @app.call(env) if Sinatra::Env.datadog_span(env)
33
+
32
34
  Tracing.trace(
33
35
  Ext::SPAN_REQUEST,
34
36
  service: configuration[:service_name],
@@ -39,7 +41,7 @@ module Datadog
39
41
  # the nil signals that there's no good one yet and is also seen by profiler, when sampling the resource
40
42
  span.resource = nil
41
43
 
42
- Sinatra::Env.set_datadog_span(env, @app_instance, span)
44
+ Sinatra::Env.set_datadog_span(env, span)
43
45
 
44
46
  response = @app.call(env)
45
47
  ensure
@@ -51,14 +53,18 @@ module Datadog
51
53
  span.set_tag(Tracing::Metadata::Ext::TAG_OPERATION, Ext::TAG_OPERATION_REQUEST)
52
54
 
53
55
  request = ::Sinatra::Request.new(env)
56
+
54
57
  span.set_tag(Tracing::Metadata::Ext::HTTP::TAG_URL, request.path)
55
58
  span.set_tag(Tracing::Metadata::Ext::HTTP::TAG_METHOD, request.request_method)
59
+
60
+ datadog_route = Sinatra::Env.route_path(env)
61
+
62
+ span.set_tag(Ext::TAG_ROUTE_PATH, datadog_route) if datadog_route
63
+
56
64
  if request.script_name && !request.script_name.empty?
57
65
  span.set_tag(Ext::TAG_SCRIPT_NAME, request.script_name)
58
66
  end
59
67
 
60
- span.set_tag(Ext::TAG_APP_NAME, @app_instance.settings.name)
61
-
62
68
  # If this app handled the request, then Contrib::Sinatra::Tracer OR Contrib::Sinatra::Base set the
63
69
  # resource; if no resource was set, let's use a fallback
64
70
  span.resource = env['REQUEST_METHOD'] if span.resource.nil?
@@ -79,7 +85,10 @@ module Datadog
79
85
  end
80
86
 
81
87
  if (headers = response[1])
82
- Sinatra::Headers.response_header_tags(headers, configuration[:headers][:response]).each do |name, value|
88
+ Sinatra::Headers.response_header_tags(
89
+ headers,
90
+ configuration[:headers][:response]
91
+ ).each do |name, value|
83
92
  span.set_tag(name, value) if span.get_tag(name).nil?
84
93
  end
85
94
  end
@@ -111,10 +120,6 @@ module Datadog
111
120
  def configuration
112
121
  Datadog.configuration.tracing[:sinatra]
113
122
  end
114
-
115
- def header_to_rack_header(name)
116
- "HTTP_#{name.to_s.upcase.gsub(/[-\s]/, '_')}"
117
- end
118
123
  end
119
124
  end
120
125
  end
@@ -14,12 +14,28 @@ module Datadog
14
14
 
15
15
  PLACEHOLDER = '?'.freeze
16
16
 
17
+ # taken from Ruby https://github.com/ruby/uri/blob/ffbab83de6d8748c9454414e02db5317609166eb/lib/uri/rfc3986_parser.rb
18
+ # but adjusted to parse only <scheme>://<host>:<port>/ components
19
+ # and stop there, since we don't care about the path, query string,
20
+ # and fragment components
21
+ RFC3986_URL_BASE = /\A(?<URI>(?<scheme>[A-Za-z][+\-.0-9A-Za-z]*):(?<hier-part>\/\/(?<authority>(?:(?<userinfo>(?:%\h\h|[!$&-.0-;=A-Z_a-z~])*)@)?(?<host>(?<IP-literal>\[(?:(?<IPv6address>(?:\h{1,4}:){6}(?<ls32>\h{1,4}:\h{1,4}|(?<IPv4address>(?<dec-octet>[1-9]\d|1\d{2}|2[0-4]\d|25[0-5]|\d)\.\g<dec-octet>\.\g<dec-octet>\.\g<dec-octet>))|::(?:\h{1,4}:){5}\g<ls32>|\h{1,4}?::(?:\h{1,4}:){4}\g<ls32>|(?:(?:\h{1,4}:)?\h{1,4})?::(?:\h{1,4}:){3}\g<ls32>|(?:(?:\h{1,4}:){,2}\h{1,4})?::(?:\h{1,4}:){2}\g<ls32>|(?:(?:\h{1,4}:){,3}\h{1,4})?::\h{1,4}:\g<ls32>|(?:(?:\h{1,4}:){,4}\h{1,4})?::\g<ls32>|(?:(?:\h{1,4}:){,5}\h{1,4})?::\h{1,4}|(?:(?:\h{1,4}:){,6}\h{1,4})?::)|(?<IPvFuture>v\h+\.[!$&-.0-;=A-Z_a-z~]+))\])|\g<IPv4address>|(?<reg-name>(?:%\h\h|[!$&-.0-9;=A-Z_a-z~])*))(?::(?<port>\d*))?)))(?:\/|\z)/.freeze # rubocop:disable Style/RegexpLiteral, Layout/LineLength
22
+
17
23
  module_function
18
24
 
19
25
  def url(url, options = {})
20
26
  url!(url, options)
21
27
  rescue StandardError
22
- options[:placeholder] || PLACEHOLDER
28
+ placeholder = options[:placeholder] || PLACEHOLDER
29
+
30
+ options[:base] == :exclude ? placeholder : "#{base_url(url)}/#{placeholder}"
31
+ end
32
+
33
+ def base_url(url, options = {})
34
+ if (m = RFC3986_URL_BASE.match(url))
35
+ m[1]
36
+ else
37
+ ''
38
+ end
23
39
  end
24
40
 
25
41
  def url!(url, options = {})
@@ -32,8 +48,14 @@ module Datadog
32
48
  uri.query = (!query.nil? && query.empty? ? nil : query)
33
49
  end
34
50
 
35
- # Remove any URI framents
51
+ # Remove any URI fragments
36
52
  uri.fragment = nil unless options[:fragment] == :show
53
+
54
+ if options[:base] == :exclude
55
+ uri.host = nil
56
+ uri.port = nil
57
+ uri.scheme = nil
58
+ end
37
59
  end.to_s
38
60
  end
39
61
 
@@ -45,22 +67,26 @@ module Datadog
45
67
 
46
68
  def query!(query, options = {})
47
69
  options ||= {}
48
- options[:show] = options[:show] || []
70
+ options[:obfuscate] = {} if options[:obfuscate] == :internal
71
+ options[:show] = options[:show] || (options[:obfuscate] ? :all : [])
49
72
  options[:exclude] = options[:exclude] || []
50
73
 
51
74
  # Short circuit if query string is meant to exclude everything
52
75
  # or if the query string is meant to include everything
53
76
  return '' if options[:exclude] == :all
54
- return query if options[:show] == :all
55
77
 
56
- collect_query(query, uniq: true) do |key, value|
57
- if options[:exclude].include?(key)
58
- [nil, nil]
59
- else
60
- value = options[:show].include?(key) ? value : nil
61
- [key, value]
78
+ unless options[:show] == :all && !(options[:obfuscate] && options[:exclude])
79
+ query = collect_query(query, uniq: true) do |key, value|
80
+ if options[:exclude].include?(key)
81
+ [nil, nil]
82
+ else
83
+ value = options[:show] == :all || options[:show].include?(key) ? value : nil
84
+ [key, value]
85
+ end
62
86
  end
63
87
  end
88
+
89
+ options[:obfuscate] ? obfuscate_query(query, options[:obfuscate]) : query
64
90
  end
65
91
 
66
92
  # Iterate over each key value pair, yielding to the block given.
@@ -91,6 +117,62 @@ module Datadog
91
117
  end
92
118
 
93
119
  private_class_method :collect_query
120
+
121
+ # Scans over the query string and obfuscates sensitive data by
122
+ # replacing matches with an opaque value
123
+ def obfuscate_query(query, options = {})
124
+ options[:regex] = nil if options[:regex] == :internal
125
+ re = options[:regex] || OBFUSCATOR_REGEX
126
+ with = options[:with] || OBFUSCATOR_WITH
127
+
128
+ query.gsub(re, with)
129
+ end
130
+
131
+ private_class_method :obfuscate_query
132
+
133
+ OBFUSCATOR_WITH = '<redacted>'.freeze
134
+
135
+ # rubocop:disable Layout/LineLength
136
+ OBFUSCATOR_REGEX = %r{
137
+ (?: # JSON-ish leading quote
138
+ (?:"|%22)?
139
+ )
140
+ (?: # common keys
141
+ (?:old_?|new_?)?p(?:ass)?w(?:or)?d(?:1|2)? # pw, password variants
142
+ |pass(?:_?phrase)? # pass, passphrase variants
143
+ |secret
144
+ |(?: # key, key_id variants
145
+ api_?
146
+ |private_?
147
+ |public_?
148
+ |access_?
149
+ |secret_?
150
+ )key(?:_?id)?
151
+ |token
152
+ |consumer_?(?:id|key|secret)
153
+ |sign(?:ed|ature)?
154
+ |auth(?:entication|orization)?
155
+ )
156
+ (?:
157
+ # '=' query string separator, plus value til next '&' separator
158
+ (?:\s|%20)*(?:=|%3D)[^&]+
159
+ # JSON-ish '": "somevalue"', key being handled with case above, without the opening '"'
160
+ |(?:"|%22) # closing '"' at end of key
161
+ (?:\s|%20)*(?::|%3A)(?:\s|%20)* # ':' key-value spearator, with surrounding spaces
162
+ (?:"|%22) # opening '"' at start of value
163
+ (?:%2[^2]|%[^2]|[^"%])+ # value
164
+ (?:"|%22) # closing '"' at end of value
165
+ )
166
+ |(?: # other common secret values
167
+ bearer(?:\s|%20)+[a-z0-9._\-]+
168
+ |token(?::|%3A)[a-z0-9]{13}
169
+ |gh[opsu]_[0-9a-zA-Z]{36}
170
+ |ey[I-L](?:[\w=-]|%3D)+\.ey[I-L](?:[\w=-]|%3D)+(?:\.(?:[\w.+/=-]|%3D|%2F|%2B)+)?
171
+ |-{5}BEGIN(?:[a-z\s]|%20)+PRIVATE(?:\s|%20)KEY-{5}[^\-]+-{5}END(?:[a-z\s]|%20)+PRIVATE(?:\s|%20)KEY(?:-{5})?(?:\n|%0A)?
172
+ |(?:ssh-(?:rsa|dss)|ecdsa-[a-z0-9]+-[a-z0-9]+)(?:\s|%20)*(?:[a-z0-9/.+]|%2F|%5C|%2B){100,}(?:=|%3D)*(?:(?:\s+)[a-z0-9._-]+)?
173
+ )
174
+ }ix.freeze
175
+ # rubocop:enable Layout/LineLength
94
176
  end
95
177
  end
96
178
  end
@@ -49,6 +49,7 @@ require_relative 'contrib/faraday/integration'
49
49
  require_relative 'contrib/grape/integration'
50
50
  require_relative 'contrib/graphql/integration'
51
51
  require_relative 'contrib/grpc/integration'
52
+ require_relative 'contrib/hanami/integration'
52
53
  require_relative 'contrib/http/integration'
53
54
  require_relative 'contrib/httpclient/integration'
54
55
  require_relative 'contrib/httprb/integration'
@@ -0,0 +1,84 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Datadog
4
+ module Tracing
5
+ module Distributed
6
+ # Encodes and decodes distributed 'x-datadog-tags' tags for transport
7
+ # to and from external processes.
8
+ module DatadogTagsCodec
9
+ # Backport `Regexp::match?` because it is measurably the most performant
10
+ # way to check if a string matches a regular expression.
11
+ module RefineRegexp
12
+ unless Regexp.method_defined?(:match?)
13
+ refine ::Regexp do
14
+ def match?(*args)
15
+ !match(*args).nil?
16
+ end
17
+ end
18
+ end
19
+ end
20
+ using RefineRegexp
21
+
22
+ # ASCII characters 32-126, except `,`, `=`, and ` `. At least one character.
23
+ VALID_KEY_CHARS = /\A(?:(?![,= ])[\u0020-\u007E])+\Z/.freeze
24
+ # ASCII characters 32-126, except `,`. At least one character.
25
+ VALID_VALUE_CHARS = /\A(?:(?!,)[\u0020-\u007E])+\Z/.freeze
26
+
27
+ # Serializes a {Hash<String,String>} into a `x-datadog-tags`-compatible
28
+ # String.
29
+ #
30
+ # @param tags [Hash<String,String>] trace tag hash
31
+ # @return [String] serialized tags hash
32
+ # @raise [EncodingError] if tags cannot be serialized to the `x-datadog-tags` format
33
+ def self.encode(tags)
34
+ begin
35
+ tags.map do |raw_key, raw_value|
36
+ key = raw_key.to_s
37
+ value = raw_value.to_s
38
+
39
+ raise EncodingError, "Invalid key `#{key}` for value `#{value}`" unless VALID_KEY_CHARS.match?(key)
40
+ raise EncodingError, "Invalid value `#{value}` for key `#{key}`" unless VALID_VALUE_CHARS.match?(value)
41
+
42
+ "#{key}=#{value.strip}"
43
+ end.join(',')
44
+ rescue => e
45
+ raise EncodingError, "Error encoding tags `#{tags}`: `#{e}`"
46
+ end
47
+ end
48
+
49
+ # Deserializes a `x-datadog-tags`-formatted String into a {Hash<String,String>}.
50
+ #
51
+ # @param string [String] tags as serialized by {#encode}
52
+ # @return [Hash<String,String>] decoded input as a hash of strings
53
+ # @raise [DecodingError] if string does not conform to the `x-datadog-tags` format
54
+ def self.decode(string)
55
+ result = Hash[string.split(',').map do |raw_tag|
56
+ raw_tag.split('=', 2).tap do |raw_key, raw_value|
57
+ key = raw_key.to_s
58
+ value = raw_value.to_s
59
+
60
+ raise DecodingError, "Invalid key: #{key}" unless VALID_KEY_CHARS.match?(key)
61
+ raise DecodingError, "Invalid value: #{value}" unless VALID_VALUE_CHARS.match?(value)
62
+
63
+ value.strip!
64
+ end
65
+ end]
66
+
67
+ raise DecodingError, "Invalid empty tags: #{string}" if result.empty? && !string.empty?
68
+
69
+ result
70
+ end
71
+
72
+ # An error occurred during distributed tags encoding.
73
+ # See {#message} for more information.
74
+ class EncodingError < StandardError
75
+ end
76
+
77
+ # An error occurred during distributed tags decoding.
78
+ # See {#message} for more information.
79
+ class DecodingError < StandardError
80
+ end
81
+ end
82
+ end
83
+ end
84
+ end
@@ -1,8 +1,11 @@
1
+ # frozen_string_literal: true
1
2
  # typed: true
2
3
 
3
4
  require_relative 'parser'
4
5
  require_relative 'ext'
6
+ require_relative '../../metadata/ext'
5
7
  require_relative '../../trace_digest'
8
+ require_relative '../datadog_tags_codec'
6
9
 
7
10
  module Datadog
8
11
  module Tracing
@@ -10,40 +13,129 @@ module Datadog
10
13
  module Headers
11
14
  # Datadog provides helpers to inject or extract headers for Datadog style headers
12
15
  module Datadog
13
- include Ext
16
+ class << self
17
+ include Ext
14
18
 
15
- def self.inject!(digest, env)
16
- return if digest.nil?
19
+ def inject!(digest, env)
20
+ return if digest.nil?
17
21
 
18
- env[HTTP_HEADER_TRACE_ID] = digest.trace_id.to_s
19
- env[HTTP_HEADER_PARENT_ID] = digest.span_id.to_s
20
- env[HTTP_HEADER_SAMPLING_PRIORITY] = digest.trace_sampling_priority.to_s if digest.trace_sampling_priority
21
- env[HTTP_HEADER_ORIGIN] = digest.trace_origin.to_s unless digest.trace_origin.nil?
22
+ env[HTTP_HEADER_TRACE_ID] = digest.trace_id.to_s
23
+ env[HTTP_HEADER_PARENT_ID] = digest.span_id.to_s
24
+ env[HTTP_HEADER_SAMPLING_PRIORITY] = digest.trace_sampling_priority.to_s if digest.trace_sampling_priority
25
+ env[HTTP_HEADER_ORIGIN] = digest.trace_origin.to_s unless digest.trace_origin.nil?
22
26
 
23
- env
24
- end
27
+ inject_tags(digest, env)
28
+
29
+ env
30
+ end
31
+
32
+ def extract(env)
33
+ # Extract values from headers
34
+ headers = Parser.new(env)
35
+ trace_id = headers.id(HTTP_HEADER_TRACE_ID)
36
+ parent_id = headers.id(HTTP_HEADER_PARENT_ID)
37
+ origin = headers.header(HTTP_HEADER_ORIGIN)
38
+ sampling_priority = headers.number(HTTP_HEADER_SAMPLING_PRIORITY)
39
+
40
+ # Return early if this propagation is not valid
41
+ # DEV: To be valid we need to have a trace id and a parent id
42
+ # or when it is a synthetics trace, just the trace id.
43
+ # DEV: `Parser#id` will not return 0
44
+ return unless (trace_id && parent_id) || (origin && trace_id)
45
+
46
+ trace_distributed_tags = extract_tags(headers)
47
+
48
+ # Return new trace headers
49
+ TraceDigest.new(
50
+ span_id: parent_id,
51
+ trace_id: trace_id,
52
+ trace_origin: origin,
53
+ trace_sampling_priority: sampling_priority,
54
+ trace_distributed_tags: trace_distributed_tags,
55
+ )
56
+ end
57
+
58
+ private
59
+
60
+ # Export trace distributed tags through the `x-datadog-tags` header.
61
+ #
62
+ # DEV: This method accesses global state (the active trace) to record its error state as a trace tag.
63
+ # DEV: This means errors cannot be reported if there's not active span.
64
+ # DEV: Ideally, we'd have a dedicated error reporting stream for all of ddtrace.
65
+ # DEV: The same comment applies to the {.extract_tags}.
66
+ def inject_tags(digest, env)
67
+ return if digest.trace_distributed_tags.nil? || digest.trace_distributed_tags.empty?
68
+
69
+ if ::Datadog.configuration.tracing.x_datadog_tags_max_length <= 0
70
+ active_trace = Tracing.active_trace
71
+ active_trace.set_tag('_dd.propagation_error', 'disabled') if active_trace
72
+ return
73
+ end
74
+
75
+ encoded_tags = DatadogTagsCodec.encode(digest.trace_distributed_tags)
76
+
77
+ if encoded_tags.size > ::Datadog.configuration.tracing.x_datadog_tags_max_length
78
+ active_trace = Tracing.active_trace
79
+ active_trace.set_tag('_dd.propagation_error', 'inject_max_size') if active_trace
80
+
81
+ ::Datadog.logger.warn(
82
+ "Failed to inject x-datadog-tags: tags are too large (size:#{encoded_tags.size} " \
83
+ "limit:#{::Datadog.configuration.tracing.x_datadog_tags_max_length}). This limit can be configured " \
84
+ 'through the DD_TRACE_X_DATADOG_TAGS_MAX_LENGTH environment variable.'
85
+ )
86
+ return
87
+ end
88
+
89
+ env[HTTP_HEADER_TAGS] = encoded_tags
90
+ rescue => e
91
+ active_trace = Tracing.active_trace
92
+ active_trace.set_tag('_dd.propagation_error', 'encoding_error') if active_trace
93
+ ::Datadog.logger.warn(
94
+ "Failed to inject x-datadog-tags: #{e.class.name} #{e.message} at #{Array(e.backtrace).first}"
95
+ )
96
+ end
97
+
98
+ # Import `x-datadog-tags` header tags as trace distributed tags.
99
+ # Only tags that have the `_dd.p.` prefix are processed.
100
+ def extract_tags(headers)
101
+ tags_header = headers.header(HTTP_HEADER_TAGS)
102
+ return unless tags_header
103
+
104
+ if ::Datadog.configuration.tracing.x_datadog_tags_max_length <= 0
105
+ active_trace = Tracing.active_trace
106
+ active_trace.set_tag('_dd.propagation_error', 'disabled') if active_trace
107
+ return
108
+ end
109
+
110
+ if tags_header.size > ::Datadog.configuration.tracing.x_datadog_tags_max_length
111
+ active_trace = Tracing.active_trace
112
+ active_trace.set_tag('_dd.propagation_error', 'extract_max_size') if active_trace
113
+
114
+ ::Datadog.logger.warn(
115
+ "Failed to extract x-datadog-tags: tags are too large (size:#{tags_header.size} " \
116
+ "limit:#{::Datadog.configuration.tracing.x_datadog_tags_max_length}). This limit can be configured " \
117
+ 'through the DD_TRACE_X_DATADOG_TAGS_MAX_LENGTH environment variable.'
118
+ )
119
+ return
120
+ end
121
+
122
+ tags = DatadogTagsCodec.decode(tags_header)
123
+ # Only extract keys with the expected Datadog prefix
124
+ tags.select! do |key, _|
125
+ key.start_with?(Tracing::Metadata::Ext::Distributed::TAGS_PREFIX) && key != EXCLUDED_TAG
126
+ end
127
+ tags
128
+ rescue => e
129
+ active_trace = Tracing.active_trace
130
+ active_trace.set_tag('_dd.propagation_error', 'decoding_error') if active_trace
131
+ ::Datadog.logger.warn(
132
+ "Failed to extract x-datadog-tags: #{e.class.name} #{e.message} at #{Array(e.backtrace).first}"
133
+ )
134
+ end
25
135
 
26
- def self.extract(env)
27
- # Extract values from headers
28
- headers = Parser.new(env)
29
- trace_id = headers.id(HTTP_HEADER_TRACE_ID)
30
- parent_id = headers.id(HTTP_HEADER_PARENT_ID)
31
- origin = headers.header(HTTP_HEADER_ORIGIN)
32
- sampling_priority = headers.number(HTTP_HEADER_SAMPLING_PRIORITY)
33
-
34
- # Return early if this propagation is not valid
35
- # DEV: To be valid we need to have a trace id and a parent id
36
- # or when it is a synthetics trace, just the trace id.
37
- # DEV: `Parser#id` will not return 0
38
- return unless (trace_id && parent_id) || (origin && trace_id)
39
-
40
- # Return new trace headers
41
- TraceDigest.new(
42
- span_id: parent_id,
43
- trace_id: trace_id,
44
- trace_origin: origin,
45
- trace_sampling_priority: sampling_priority
46
- )
136
+ # We want to exclude tags that we don't want to propagate downstream.
137
+ EXCLUDED_TAG = '_dd.p.upstream_services'
138
+ private_constant :EXCLUDED_TAG
47
139
  end
48
140
  end
49
141
  end
@@ -12,6 +12,8 @@ module Datadog
12
12
  HTTP_HEADER_PARENT_ID = 'x-datadog-parent-id'.freeze
13
13
  HTTP_HEADER_SAMPLING_PRIORITY = 'x-datadog-sampling-priority'.freeze
14
14
  HTTP_HEADER_ORIGIN = 'x-datadog-origin'.freeze
15
+ # Distributed trace-level tags
16
+ HTTP_HEADER_TAGS = 'x-datadog-tags'.freeze
15
17
 
16
18
  # B3 headers used for distributed tracing
17
19
  B3_HEADER_TRACE_ID = 'x-b3-traceid'.freeze