ddtrace 1.4.1 → 1.6.1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (155) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +144 -1
  3. data/LICENSE-3rdparty.csv +1 -0
  4. data/ext/ddtrace_profiling_loader/ddtrace_profiling_loader.c +9 -2
  5. data/ext/ddtrace_profiling_loader/extconf.rb +17 -0
  6. data/ext/ddtrace_profiling_native_extension/NativeExtensionDesign.md +38 -2
  7. data/ext/ddtrace_profiling_native_extension/clock_id.h +1 -0
  8. data/ext/ddtrace_profiling_native_extension/clock_id_from_pthread.c +1 -0
  9. data/ext/ddtrace_profiling_native_extension/collectors_cpu_and_wall_time.c +517 -42
  10. data/ext/ddtrace_profiling_native_extension/collectors_cpu_and_wall_time.h +3 -0
  11. data/ext/ddtrace_profiling_native_extension/collectors_cpu_and_wall_time_worker.c +208 -30
  12. data/ext/ddtrace_profiling_native_extension/collectors_stack.c +156 -46
  13. data/ext/ddtrace_profiling_native_extension/collectors_stack.h +11 -2
  14. data/ext/ddtrace_profiling_native_extension/extconf.rb +11 -1
  15. data/ext/ddtrace_profiling_native_extension/http_transport.c +83 -64
  16. data/ext/ddtrace_profiling_native_extension/libdatadog_helpers.h +4 -4
  17. data/ext/ddtrace_profiling_native_extension/native_extension_helpers.rb +3 -4
  18. data/ext/ddtrace_profiling_native_extension/private_vm_api_access.c +59 -0
  19. data/ext/ddtrace_profiling_native_extension/private_vm_api_access.h +3 -0
  20. data/ext/ddtrace_profiling_native_extension/profiling.c +10 -0
  21. data/ext/ddtrace_profiling_native_extension/ruby_helpers.c +0 -1
  22. data/ext/ddtrace_profiling_native_extension/ruby_helpers.h +4 -2
  23. data/ext/ddtrace_profiling_native_extension/stack_recorder.c +45 -29
  24. data/ext/ddtrace_profiling_native_extension/stack_recorder.h +7 -7
  25. data/lib/datadog/appsec/assets/waf_rules/recommended.json +1169 -275
  26. data/lib/datadog/appsec/assets/waf_rules/risky.json +78 -78
  27. data/lib/datadog/appsec/assets/waf_rules/strict.json +278 -88
  28. data/lib/datadog/appsec/configuration/settings.rb +0 -2
  29. data/lib/datadog/appsec/contrib/rack/gateway/watcher.rb +25 -20
  30. data/lib/datadog/appsec/contrib/rack/reactive/request.rb +11 -11
  31. data/lib/datadog/appsec/contrib/rack/reactive/request_body.rb +11 -11
  32. data/lib/datadog/appsec/contrib/rack/reactive/response.rb +11 -11
  33. data/lib/datadog/appsec/contrib/rack/request.rb +3 -0
  34. data/lib/datadog/appsec/contrib/rack/request_middleware.rb +46 -19
  35. data/lib/datadog/appsec/contrib/rails/gateway/watcher.rb +7 -6
  36. data/lib/datadog/appsec/contrib/rails/integration.rb +1 -1
  37. data/lib/datadog/appsec/contrib/rails/reactive/action.rb +11 -11
  38. data/lib/datadog/appsec/contrib/rails/request.rb +3 -0
  39. data/lib/datadog/appsec/contrib/sinatra/gateway/watcher.rb +14 -12
  40. data/lib/datadog/appsec/contrib/sinatra/reactive/routed.rb +11 -11
  41. data/lib/datadog/appsec/event.rb +6 -10
  42. data/lib/datadog/appsec/instrumentation/gateway.rb +16 -2
  43. data/lib/datadog/appsec/processor.rb +18 -2
  44. data/lib/datadog/ci/ext/environment.rb +16 -4
  45. data/lib/datadog/core/configuration/agent_settings_resolver.rb +0 -3
  46. data/lib/datadog/core/configuration/components.rb +28 -16
  47. data/lib/datadog/core/configuration/settings.rb +127 -8
  48. data/lib/datadog/core/configuration.rb +1 -1
  49. data/lib/datadog/core/diagnostics/environment_logger.rb +5 -1
  50. data/lib/datadog/core/header_collection.rb +41 -0
  51. data/lib/datadog/core/telemetry/collector.rb +0 -2
  52. data/lib/datadog/core/utils/compression.rb +5 -1
  53. data/lib/datadog/core/workers/async.rb +0 -2
  54. data/lib/datadog/core.rb +0 -54
  55. data/lib/datadog/opentracer/tracer.rb +4 -6
  56. data/lib/datadog/profiling/collectors/cpu_and_wall_time.rb +12 -2
  57. data/lib/datadog/profiling/collectors/cpu_and_wall_time_worker.rb +5 -3
  58. data/lib/datadog/profiling/collectors/old_stack.rb +1 -1
  59. data/lib/datadog/profiling/exporter.rb +2 -4
  60. data/lib/datadog/profiling/http_transport.rb +1 -1
  61. data/lib/datadog/profiling.rb +1 -1
  62. data/lib/datadog/tracing/client_ip.rb +164 -0
  63. data/lib/datadog/tracing/configuration/ext.rb +14 -0
  64. data/lib/datadog/tracing/contrib/aws/instrumentation.rb +2 -0
  65. data/lib/datadog/tracing/contrib/aws/services.rb +0 -2
  66. data/lib/datadog/tracing/contrib/dalli/ext.rb +1 -0
  67. data/lib/datadog/tracing/contrib/dalli/instrumentation.rb +4 -0
  68. data/lib/datadog/tracing/contrib/elasticsearch/ext.rb +2 -0
  69. data/lib/datadog/tracing/contrib/elasticsearch/patcher.rb +3 -0
  70. data/lib/datadog/tracing/contrib/ethon/easy_patch.rb +2 -2
  71. data/lib/datadog/tracing/contrib/ethon/multi_patch.rb +2 -0
  72. data/lib/datadog/tracing/contrib/excon/middleware.rb +2 -0
  73. data/lib/datadog/tracing/contrib/ext.rb +25 -0
  74. data/lib/datadog/tracing/contrib/faraday/middleware.rb +3 -2
  75. data/lib/datadog/tracing/contrib/grape/endpoint.rb +0 -2
  76. data/lib/datadog/tracing/contrib/graphql/configuration/settings.rb +1 -1
  77. data/lib/datadog/tracing/contrib/grpc/datadog_interceptor/client.rb +5 -0
  78. data/lib/datadog/tracing/contrib/grpc/datadog_interceptor/server.rb +7 -1
  79. data/lib/datadog/tracing/contrib/grpc/ext.rb +2 -0
  80. data/lib/datadog/tracing/contrib/hanami/action_tracer.rb +47 -0
  81. data/lib/datadog/tracing/contrib/hanami/configuration/settings.rb +22 -0
  82. data/lib/datadog/tracing/contrib/hanami/ext.rb +24 -0
  83. data/lib/datadog/tracing/contrib/hanami/integration.rb +44 -0
  84. data/lib/datadog/tracing/contrib/hanami/patcher.rb +33 -0
  85. data/lib/datadog/tracing/contrib/hanami/plugin.rb +23 -0
  86. data/lib/datadog/tracing/contrib/hanami/renderer_policy_tracing.rb +41 -0
  87. data/lib/datadog/tracing/contrib/hanami/router_tracing.rb +44 -0
  88. data/lib/datadog/tracing/contrib/http/instrumentation.rb +2 -0
  89. data/lib/datadog/tracing/contrib/httpclient/instrumentation.rb +2 -0
  90. data/lib/datadog/tracing/contrib/httprb/instrumentation.rb +2 -0
  91. data/lib/datadog/tracing/contrib/mongodb/ext.rb +7 -0
  92. data/lib/datadog/tracing/contrib/mongodb/subscribers.rb +4 -0
  93. data/lib/datadog/tracing/contrib/mysql2/configuration/settings.rb +12 -0
  94. data/lib/datadog/tracing/contrib/mysql2/ext.rb +1 -0
  95. data/lib/datadog/tracing/contrib/mysql2/instrumentation.rb +16 -0
  96. data/lib/datadog/tracing/contrib/pg/configuration/settings.rb +12 -0
  97. data/lib/datadog/tracing/contrib/pg/ext.rb +2 -1
  98. data/lib/datadog/tracing/contrib/pg/instrumentation.rb +38 -21
  99. data/lib/datadog/tracing/contrib/propagation/sql_comment/comment.rb +43 -0
  100. data/lib/datadog/tracing/contrib/propagation/sql_comment/ext.rb +32 -0
  101. data/lib/datadog/tracing/contrib/propagation/sql_comment/mode.rb +28 -0
  102. data/lib/datadog/tracing/contrib/propagation/sql_comment.rb +49 -0
  103. data/lib/datadog/tracing/contrib/rack/header_collection.rb +35 -0
  104. data/lib/datadog/tracing/contrib/rack/middlewares.rb +105 -43
  105. data/lib/datadog/tracing/contrib/redis/ext.rb +2 -0
  106. data/lib/datadog/tracing/contrib/redis/instrumentation.rb +4 -2
  107. data/lib/datadog/tracing/contrib/redis/integration.rb +2 -1
  108. data/lib/datadog/tracing/contrib/redis/patcher.rb +40 -0
  109. data/lib/datadog/tracing/contrib/redis/tags.rb +5 -0
  110. data/lib/datadog/tracing/contrib/rest_client/request_patch.rb +2 -0
  111. data/lib/datadog/tracing/contrib/sinatra/env.rb +12 -23
  112. data/lib/datadog/tracing/contrib/sinatra/ext.rb +7 -3
  113. data/lib/datadog/tracing/contrib/sinatra/patcher.rb +2 -2
  114. data/lib/datadog/tracing/contrib/sinatra/tracer.rb +8 -80
  115. data/lib/datadog/tracing/contrib/sinatra/tracer_middleware.rb +14 -9
  116. data/lib/datadog/tracing/contrib/utils/quantization/http.rb +92 -10
  117. data/lib/datadog/tracing/contrib.rb +1 -0
  118. data/lib/datadog/tracing/distributed/datadog_tags_codec.rb +84 -0
  119. data/lib/datadog/tracing/distributed/headers/datadog.rb +122 -30
  120. data/lib/datadog/tracing/distributed/headers/ext.rb +2 -0
  121. data/lib/datadog/tracing/flush.rb +57 -35
  122. data/lib/datadog/tracing/metadata/ext.rb +11 -9
  123. data/lib/datadog/tracing/metadata/tagging.rb +9 -0
  124. data/lib/datadog/tracing/propagation/http.rb +9 -1
  125. data/lib/datadog/tracing/sampling/ext.rb +31 -0
  126. data/lib/datadog/tracing/sampling/priority_sampler.rb +46 -4
  127. data/lib/datadog/tracing/sampling/rate_by_key_sampler.rb +8 -9
  128. data/lib/datadog/tracing/sampling/rate_by_service_sampler.rb +29 -5
  129. data/lib/datadog/tracing/sampling/rate_limiter.rb +3 -0
  130. data/lib/datadog/tracing/sampling/rate_sampler.rb +20 -3
  131. data/lib/datadog/tracing/sampling/rule_sampler.rb +4 -3
  132. data/lib/datadog/tracing/sampling/span/ext.rb +25 -0
  133. data/lib/datadog/tracing/sampling/span/matcher.rb +9 -0
  134. data/lib/datadog/tracing/sampling/span/rule.rb +82 -0
  135. data/lib/datadog/tracing/sampling/span/rule_parser.rb +104 -0
  136. data/lib/datadog/tracing/sampling/span/sampler.rb +75 -0
  137. data/lib/datadog/tracing/span_operation.rb +0 -2
  138. data/lib/datadog/tracing/trace_digest.rb +3 -0
  139. data/lib/datadog/tracing/trace_operation.rb +32 -3
  140. data/lib/datadog/tracing/trace_segment.rb +7 -2
  141. data/lib/datadog/tracing/tracer.rb +34 -6
  142. data/lib/datadog/tracing/writer.rb +7 -0
  143. data/lib/ddtrace/transport/trace_formatter.rb +7 -0
  144. data/lib/ddtrace/transport/traces.rb +3 -1
  145. data/lib/ddtrace/version.rb +1 -1
  146. metadata +36 -18
  147. data/lib/datadog/profiling/old_ext.rb +0 -42
  148. data/lib/datadog/profiling/transport/http/api/endpoint.rb +0 -85
  149. data/lib/datadog/profiling/transport/http/api/instance.rb +0 -38
  150. data/lib/datadog/profiling/transport/http/api/spec.rb +0 -42
  151. data/lib/datadog/profiling/transport/http/api.rb +0 -45
  152. data/lib/datadog/profiling/transport/http/builder.rb +0 -30
  153. data/lib/datadog/profiling/transport/http/client.rb +0 -37
  154. data/lib/datadog/profiling/transport/http/response.rb +0 -21
  155. data/lib/datadog/profiling/transport/http.rb +0 -118
@@ -17,69 +17,12 @@ module Datadog
17
17
  # Datadog::Tracing::Contrib::Sinatra::Tracer is a Sinatra extension which traces
18
18
  # requests.
19
19
  module Tracer
20
- def route(verb, action, *)
21
- # Keep track of the route name when the app is instantiated for an
22
- # incoming request.
23
- condition do
24
- # If the option to prepend script names is enabled, then
25
- # prepend the script name from the request onto the action.
26
- #
27
- # DEV: env['sinatra.route'] already exists with very similar information,
28
- # DEV: but doesn't account for our `resource_script_names` logic.
29
- #
30
- @datadog_route = if Datadog.configuration.tracing[:sinatra][:resource_script_names]
31
- "#{request.script_name}#{action}"
32
- else
33
- action
34
- end
35
- end
36
-
37
- super
38
- end
39
-
40
20
  def self.registered(app)
41
21
  app.use TracerMiddleware, app_instance: app
42
-
43
- app.after do
44
- next unless Tracing.enabled?
45
-
46
- span = Sinatra::Env.datadog_span(env, app)
47
-
48
- # TODO: `route` should *only* be populated if @datadog_route is defined.
49
- # TODO: If @datadog_route is not defined, then this Sinatra app is not responsible
50
- # TODO: for handling this request.
51
- # TODO:
52
- # TODO: This change would be BREAKING for any Sinatra app (classic or modular),
53
- # TODO: as it affects the `resource` value for requests not handled by the Sinatra app.
54
- # TODO: Currently we use "#{method} #{path}" in such aces, but `path` is the raw,
55
- # TODO: high-cardinality HTTP path, and can contain PII.
56
- # TODO:
57
- # TODO: The value we should use as the `resource` when the Sinatra app is not
58
- # TODO: responsible for the request is a tricky subject.
59
- # TODO: The best option is a value that clearly communicates that this app did not
60
- # TODO: handle this request. It's important to keep in mind that an unhandled request
61
- # TODO: by this Sinatra app might still be handled by another Rack middleware (which can
62
- # TODO: be a Sinatra app itself) or it might just 404 if not handled at all.
63
- # TODO:
64
- # TODO: A possible value for `resource` could set a high level description, e.g.
65
- # TODO: `request.request_method`, given we don't have the response object available yet.
66
- route = if defined?(@datadog_route)
67
- @datadog_route
68
- else
69
- # Fallback in case no routes have matched
70
- request.path
71
- end
72
-
73
- span.resource = "#{request.request_method} #{route}"
74
- span.set_tag(Ext::TAG_ROUTE_PATH, route)
75
- end
76
22
  end
77
23
 
78
24
  # Method overrides for Sinatra::Base
79
25
  module Base
80
- MISSING_REQUEST_SPAN_ONLY_ONCE = Core::Utils::OnlyOnce.new
81
- private_constant :MISSING_REQUEST_SPAN_ONLY_ONCE
82
-
83
26
  def render(engine, data, *)
84
27
  return super unless Tracing.enabled?
85
28
 
@@ -102,19 +45,21 @@ module Datadog
102
45
 
103
46
  # Invoked when a matching route is found.
104
47
  # This method yields directly to user code.
105
- # rubocop:disable Metrics/MethodLength
106
48
  def route_eval
107
49
  configuration = Datadog.configuration.tracing[:sinatra]
108
50
  return super unless Tracing.enabled?
109
51
 
52
+ datadog_route = Sinatra::Env.route_path(env)
53
+
110
54
  Tracing.trace(
111
55
  Ext::SPAN_ROUTE,
112
56
  service: configuration[:service_name],
113
57
  span_type: Tracing::Metadata::Ext::HTTP::TYPE_INBOUND,
114
- resource: "#{request.request_method} #{@datadog_route}",
58
+ resource: "#{request.request_method} #{datadog_route}",
115
59
  ) do |span, trace|
116
60
  span.set_tag(Ext::TAG_APP_NAME, settings.name || settings.superclass.name)
117
- span.set_tag(Ext::TAG_ROUTE_PATH, @datadog_route)
61
+ span.set_tag(Ext::TAG_ROUTE_PATH, datadog_route)
62
+
118
63
  if request.script_name && !request.script_name.empty?
119
64
  span.set_tag(Ext::TAG_SCRIPT_NAME, request.script_name)
120
65
  end
@@ -124,32 +69,15 @@ module Datadog
124
69
 
125
70
  trace.resource = span.resource
126
71
 
127
- sinatra_request_span =
128
- if self.class <= ::Sinatra::Application # Classic style (top-level) application
129
- Sinatra::Env.datadog_span(env, ::Sinatra::Application)
130
- else
131
- Sinatra::Env.datadog_span(env, self.class)
132
- end
133
- if sinatra_request_span
134
- sinatra_request_span.resource = span.resource
135
- else
136
- MISSING_REQUEST_SPAN_ONLY_ONCE.run do
137
- Datadog.logger.warn do
138
- 'Sinatra integration is misconfigured, reported traces will be missing request metadata ' \
139
- 'such as path and HTTP status code. ' \
140
- 'Did you forget to add `register Datadog::Tracing::Contrib::Sinatra::Tracer` to your ' \
141
- '`Sinatra::Base` subclass? ' \
142
- 'See <https://docs.datadoghq.com/tracing/setup_overview/setup/ruby/#sinatra> for more details.'
143
- end
144
- end
145
- end
72
+ sinatra_request_span = Sinatra::Env.datadog_span(env)
73
+
74
+ sinatra_request_span.resource = span.resource
146
75
 
147
76
  Contrib::Analytics.set_measured(span)
148
77
 
149
78
  super
150
79
  end
151
80
  end
152
- # rubocop:enable Metrics/MethodLength
153
81
  end
154
82
  end
155
83
  end
@@ -25,10 +25,12 @@ module Datadog
25
25
  def call(env)
26
26
  # Set the trace context (e.g. distributed tracing)
27
27
  if configuration[:distributed_tracing] && Tracing.active_trace.nil?
28
- original_trace = Propagation::HTTP.extract(env)
28
+ original_trace = Tracing::Propagation::HTTP.extract(env)
29
29
  Tracing.continue_trace!(original_trace)
30
30
  end
31
31
 
32
+ return @app.call(env) if Sinatra::Env.datadog_span(env)
33
+
32
34
  Tracing.trace(
33
35
  Ext::SPAN_REQUEST,
34
36
  service: configuration[:service_name],
@@ -39,7 +41,7 @@ module Datadog
39
41
  # the nil signals that there's no good one yet and is also seen by profiler, when sampling the resource
40
42
  span.resource = nil
41
43
 
42
- Sinatra::Env.set_datadog_span(env, @app_instance, span)
44
+ Sinatra::Env.set_datadog_span(env, span)
43
45
 
44
46
  response = @app.call(env)
45
47
  ensure
@@ -51,14 +53,18 @@ module Datadog
51
53
  span.set_tag(Tracing::Metadata::Ext::TAG_OPERATION, Ext::TAG_OPERATION_REQUEST)
52
54
 
53
55
  request = ::Sinatra::Request.new(env)
56
+
54
57
  span.set_tag(Tracing::Metadata::Ext::HTTP::TAG_URL, request.path)
55
58
  span.set_tag(Tracing::Metadata::Ext::HTTP::TAG_METHOD, request.request_method)
59
+
60
+ datadog_route = Sinatra::Env.route_path(env)
61
+
62
+ span.set_tag(Ext::TAG_ROUTE_PATH, datadog_route) if datadog_route
63
+
56
64
  if request.script_name && !request.script_name.empty?
57
65
  span.set_tag(Ext::TAG_SCRIPT_NAME, request.script_name)
58
66
  end
59
67
 
60
- span.set_tag(Ext::TAG_APP_NAME, @app_instance.settings.name)
61
-
62
68
  # If this app handled the request, then Contrib::Sinatra::Tracer OR Contrib::Sinatra::Base set the
63
69
  # resource; if no resource was set, let's use a fallback
64
70
  span.resource = env['REQUEST_METHOD'] if span.resource.nil?
@@ -79,7 +85,10 @@ module Datadog
79
85
  end
80
86
 
81
87
  if (headers = response[1])
82
- Sinatra::Headers.response_header_tags(headers, configuration[:headers][:response]).each do |name, value|
88
+ Sinatra::Headers.response_header_tags(
89
+ headers,
90
+ configuration[:headers][:response]
91
+ ).each do |name, value|
83
92
  span.set_tag(name, value) if span.get_tag(name).nil?
84
93
  end
85
94
  end
@@ -111,10 +120,6 @@ module Datadog
111
120
  def configuration
112
121
  Datadog.configuration.tracing[:sinatra]
113
122
  end
114
-
115
- def header_to_rack_header(name)
116
- "HTTP_#{name.to_s.upcase.gsub(/[-\s]/, '_')}"
117
- end
118
123
  end
119
124
  end
120
125
  end
@@ -14,12 +14,28 @@ module Datadog
14
14
 
15
15
  PLACEHOLDER = '?'.freeze
16
16
 
17
+ # taken from Ruby https://github.com/ruby/uri/blob/ffbab83de6d8748c9454414e02db5317609166eb/lib/uri/rfc3986_parser.rb
18
+ # but adjusted to parse only <scheme>://<host>:<port>/ components
19
+ # and stop there, since we don't care about the path, query string,
20
+ # and fragment components
21
+ RFC3986_URL_BASE = /\A(?<URI>(?<scheme>[A-Za-z][+\-.0-9A-Za-z]*):(?<hier-part>\/\/(?<authority>(?:(?<userinfo>(?:%\h\h|[!$&-.0-;=A-Z_a-z~])*)@)?(?<host>(?<IP-literal>\[(?:(?<IPv6address>(?:\h{1,4}:){6}(?<ls32>\h{1,4}:\h{1,4}|(?<IPv4address>(?<dec-octet>[1-9]\d|1\d{2}|2[0-4]\d|25[0-5]|\d)\.\g<dec-octet>\.\g<dec-octet>\.\g<dec-octet>))|::(?:\h{1,4}:){5}\g<ls32>|\h{1,4}?::(?:\h{1,4}:){4}\g<ls32>|(?:(?:\h{1,4}:)?\h{1,4})?::(?:\h{1,4}:){3}\g<ls32>|(?:(?:\h{1,4}:){,2}\h{1,4})?::(?:\h{1,4}:){2}\g<ls32>|(?:(?:\h{1,4}:){,3}\h{1,4})?::\h{1,4}:\g<ls32>|(?:(?:\h{1,4}:){,4}\h{1,4})?::\g<ls32>|(?:(?:\h{1,4}:){,5}\h{1,4})?::\h{1,4}|(?:(?:\h{1,4}:){,6}\h{1,4})?::)|(?<IPvFuture>v\h+\.[!$&-.0-;=A-Z_a-z~]+))\])|\g<IPv4address>|(?<reg-name>(?:%\h\h|[!$&-.0-9;=A-Z_a-z~])*))(?::(?<port>\d*))?)))(?:\/|\z)/.freeze # rubocop:disable Style/RegexpLiteral, Layout/LineLength
22
+
17
23
  module_function
18
24
 
19
25
  def url(url, options = {})
20
26
  url!(url, options)
21
27
  rescue StandardError
22
- options[:placeholder] || PLACEHOLDER
28
+ placeholder = options[:placeholder] || PLACEHOLDER
29
+
30
+ options[:base] == :exclude ? placeholder : "#{base_url(url)}/#{placeholder}"
31
+ end
32
+
33
+ def base_url(url, options = {})
34
+ if (m = RFC3986_URL_BASE.match(url))
35
+ m[1]
36
+ else
37
+ ''
38
+ end
23
39
  end
24
40
 
25
41
  def url!(url, options = {})
@@ -32,8 +48,14 @@ module Datadog
32
48
  uri.query = (!query.nil? && query.empty? ? nil : query)
33
49
  end
34
50
 
35
- # Remove any URI framents
51
+ # Remove any URI fragments
36
52
  uri.fragment = nil unless options[:fragment] == :show
53
+
54
+ if options[:base] == :exclude
55
+ uri.host = nil
56
+ uri.port = nil
57
+ uri.scheme = nil
58
+ end
37
59
  end.to_s
38
60
  end
39
61
 
@@ -45,22 +67,26 @@ module Datadog
45
67
 
46
68
  def query!(query, options = {})
47
69
  options ||= {}
48
- options[:show] = options[:show] || []
70
+ options[:obfuscate] = {} if options[:obfuscate] == :internal
71
+ options[:show] = options[:show] || (options[:obfuscate] ? :all : [])
49
72
  options[:exclude] = options[:exclude] || []
50
73
 
51
74
  # Short circuit if query string is meant to exclude everything
52
75
  # or if the query string is meant to include everything
53
76
  return '' if options[:exclude] == :all
54
- return query if options[:show] == :all
55
77
 
56
- collect_query(query, uniq: true) do |key, value|
57
- if options[:exclude].include?(key)
58
- [nil, nil]
59
- else
60
- value = options[:show].include?(key) ? value : nil
61
- [key, value]
78
+ unless options[:show] == :all && !(options[:obfuscate] && options[:exclude])
79
+ query = collect_query(query, uniq: true) do |key, value|
80
+ if options[:exclude].include?(key)
81
+ [nil, nil]
82
+ else
83
+ value = options[:show] == :all || options[:show].include?(key) ? value : nil
84
+ [key, value]
85
+ end
62
86
  end
63
87
  end
88
+
89
+ options[:obfuscate] ? obfuscate_query(query, options[:obfuscate]) : query
64
90
  end
65
91
 
66
92
  # Iterate over each key value pair, yielding to the block given.
@@ -91,6 +117,62 @@ module Datadog
91
117
  end
92
118
 
93
119
  private_class_method :collect_query
120
+
121
+ # Scans over the query string and obfuscates sensitive data by
122
+ # replacing matches with an opaque value
123
+ def obfuscate_query(query, options = {})
124
+ options[:regex] = nil if options[:regex] == :internal
125
+ re = options[:regex] || OBFUSCATOR_REGEX
126
+ with = options[:with] || OBFUSCATOR_WITH
127
+
128
+ query.gsub(re, with)
129
+ end
130
+
131
+ private_class_method :obfuscate_query
132
+
133
+ OBFUSCATOR_WITH = '<redacted>'.freeze
134
+
135
+ # rubocop:disable Layout/LineLength
136
+ OBFUSCATOR_REGEX = %r{
137
+ (?: # JSON-ish leading quote
138
+ (?:"|%22)?
139
+ )
140
+ (?: # common keys
141
+ (?:old_?|new_?)?p(?:ass)?w(?:or)?d(?:1|2)? # pw, password variants
142
+ |pass(?:_?phrase)? # pass, passphrase variants
143
+ |secret
144
+ |(?: # key, key_id variants
145
+ api_?
146
+ |private_?
147
+ |public_?
148
+ |access_?
149
+ |secret_?
150
+ )key(?:_?id)?
151
+ |token
152
+ |consumer_?(?:id|key|secret)
153
+ |sign(?:ed|ature)?
154
+ |auth(?:entication|orization)?
155
+ )
156
+ (?:
157
+ # '=' query string separator, plus value til next '&' separator
158
+ (?:\s|%20)*(?:=|%3D)[^&]+
159
+ # JSON-ish '": "somevalue"', key being handled with case above, without the opening '"'
160
+ |(?:"|%22) # closing '"' at end of key
161
+ (?:\s|%20)*(?::|%3A)(?:\s|%20)* # ':' key-value spearator, with surrounding spaces
162
+ (?:"|%22) # opening '"' at start of value
163
+ (?:%2[^2]|%[^2]|[^"%])+ # value
164
+ (?:"|%22) # closing '"' at end of value
165
+ )
166
+ |(?: # other common secret values
167
+ bearer(?:\s|%20)+[a-z0-9._\-]+
168
+ |token(?::|%3A)[a-z0-9]{13}
169
+ |gh[opsu]_[0-9a-zA-Z]{36}
170
+ |ey[I-L](?:[\w=-]|%3D)+\.ey[I-L](?:[\w=-]|%3D)+(?:\.(?:[\w.+/=-]|%3D|%2F|%2B)+)?
171
+ |-{5}BEGIN(?:[a-z\s]|%20)+PRIVATE(?:\s|%20)KEY-{5}[^\-]+-{5}END(?:[a-z\s]|%20)+PRIVATE(?:\s|%20)KEY(?:-{5})?(?:\n|%0A)?
172
+ |(?:ssh-(?:rsa|dss)|ecdsa-[a-z0-9]+-[a-z0-9]+)(?:\s|%20)*(?:[a-z0-9/.+]|%2F|%5C|%2B){100,}(?:=|%3D)*(?:(?:\s+)[a-z0-9._-]+)?
173
+ )
174
+ }ix.freeze
175
+ # rubocop:enable Layout/LineLength
94
176
  end
95
177
  end
96
178
  end
@@ -49,6 +49,7 @@ require_relative 'contrib/faraday/integration'
49
49
  require_relative 'contrib/grape/integration'
50
50
  require_relative 'contrib/graphql/integration'
51
51
  require_relative 'contrib/grpc/integration'
52
+ require_relative 'contrib/hanami/integration'
52
53
  require_relative 'contrib/http/integration'
53
54
  require_relative 'contrib/httpclient/integration'
54
55
  require_relative 'contrib/httprb/integration'
@@ -0,0 +1,84 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Datadog
4
+ module Tracing
5
+ module Distributed
6
+ # Encodes and decodes distributed 'x-datadog-tags' tags for transport
7
+ # to and from external processes.
8
+ module DatadogTagsCodec
9
+ # Backport `Regexp::match?` because it is measurably the most performant
10
+ # way to check if a string matches a regular expression.
11
+ module RefineRegexp
12
+ unless Regexp.method_defined?(:match?)
13
+ refine ::Regexp do
14
+ def match?(*args)
15
+ !match(*args).nil?
16
+ end
17
+ end
18
+ end
19
+ end
20
+ using RefineRegexp
21
+
22
+ # ASCII characters 32-126, except `,`, `=`, and ` `. At least one character.
23
+ VALID_KEY_CHARS = /\A(?:(?![,= ])[\u0020-\u007E])+\Z/.freeze
24
+ # ASCII characters 32-126, except `,`. At least one character.
25
+ VALID_VALUE_CHARS = /\A(?:(?!,)[\u0020-\u007E])+\Z/.freeze
26
+
27
+ # Serializes a {Hash<String,String>} into a `x-datadog-tags`-compatible
28
+ # String.
29
+ #
30
+ # @param tags [Hash<String,String>] trace tag hash
31
+ # @return [String] serialized tags hash
32
+ # @raise [EncodingError] if tags cannot be serialized to the `x-datadog-tags` format
33
+ def self.encode(tags)
34
+ begin
35
+ tags.map do |raw_key, raw_value|
36
+ key = raw_key.to_s
37
+ value = raw_value.to_s
38
+
39
+ raise EncodingError, "Invalid key `#{key}` for value `#{value}`" unless VALID_KEY_CHARS.match?(key)
40
+ raise EncodingError, "Invalid value `#{value}` for key `#{key}`" unless VALID_VALUE_CHARS.match?(value)
41
+
42
+ "#{key}=#{value.strip}"
43
+ end.join(',')
44
+ rescue => e
45
+ raise EncodingError, "Error encoding tags `#{tags}`: `#{e}`"
46
+ end
47
+ end
48
+
49
+ # Deserializes a `x-datadog-tags`-formatted String into a {Hash<String,String>}.
50
+ #
51
+ # @param string [String] tags as serialized by {#encode}
52
+ # @return [Hash<String,String>] decoded input as a hash of strings
53
+ # @raise [DecodingError] if string does not conform to the `x-datadog-tags` format
54
+ def self.decode(string)
55
+ result = Hash[string.split(',').map do |raw_tag|
56
+ raw_tag.split('=', 2).tap do |raw_key, raw_value|
57
+ key = raw_key.to_s
58
+ value = raw_value.to_s
59
+
60
+ raise DecodingError, "Invalid key: #{key}" unless VALID_KEY_CHARS.match?(key)
61
+ raise DecodingError, "Invalid value: #{value}" unless VALID_VALUE_CHARS.match?(value)
62
+
63
+ value.strip!
64
+ end
65
+ end]
66
+
67
+ raise DecodingError, "Invalid empty tags: #{string}" if result.empty? && !string.empty?
68
+
69
+ result
70
+ end
71
+
72
+ # An error occurred during distributed tags encoding.
73
+ # See {#message} for more information.
74
+ class EncodingError < StandardError
75
+ end
76
+
77
+ # An error occurred during distributed tags decoding.
78
+ # See {#message} for more information.
79
+ class DecodingError < StandardError
80
+ end
81
+ end
82
+ end
83
+ end
84
+ end
@@ -1,8 +1,11 @@
1
+ # frozen_string_literal: true
1
2
  # typed: true
2
3
 
3
4
  require_relative 'parser'
4
5
  require_relative 'ext'
6
+ require_relative '../../metadata/ext'
5
7
  require_relative '../../trace_digest'
8
+ require_relative '../datadog_tags_codec'
6
9
 
7
10
  module Datadog
8
11
  module Tracing
@@ -10,40 +13,129 @@ module Datadog
10
13
  module Headers
11
14
  # Datadog provides helpers to inject or extract headers for Datadog style headers
12
15
  module Datadog
13
- include Ext
16
+ class << self
17
+ include Ext
14
18
 
15
- def self.inject!(digest, env)
16
- return if digest.nil?
19
+ def inject!(digest, env)
20
+ return if digest.nil?
17
21
 
18
- env[HTTP_HEADER_TRACE_ID] = digest.trace_id.to_s
19
- env[HTTP_HEADER_PARENT_ID] = digest.span_id.to_s
20
- env[HTTP_HEADER_SAMPLING_PRIORITY] = digest.trace_sampling_priority.to_s if digest.trace_sampling_priority
21
- env[HTTP_HEADER_ORIGIN] = digest.trace_origin.to_s unless digest.trace_origin.nil?
22
+ env[HTTP_HEADER_TRACE_ID] = digest.trace_id.to_s
23
+ env[HTTP_HEADER_PARENT_ID] = digest.span_id.to_s
24
+ env[HTTP_HEADER_SAMPLING_PRIORITY] = digest.trace_sampling_priority.to_s if digest.trace_sampling_priority
25
+ env[HTTP_HEADER_ORIGIN] = digest.trace_origin.to_s unless digest.trace_origin.nil?
22
26
 
23
- env
24
- end
27
+ inject_tags(digest, env)
28
+
29
+ env
30
+ end
31
+
32
+ def extract(env)
33
+ # Extract values from headers
34
+ headers = Parser.new(env)
35
+ trace_id = headers.id(HTTP_HEADER_TRACE_ID)
36
+ parent_id = headers.id(HTTP_HEADER_PARENT_ID)
37
+ origin = headers.header(HTTP_HEADER_ORIGIN)
38
+ sampling_priority = headers.number(HTTP_HEADER_SAMPLING_PRIORITY)
39
+
40
+ # Return early if this propagation is not valid
41
+ # DEV: To be valid we need to have a trace id and a parent id
42
+ # or when it is a synthetics trace, just the trace id.
43
+ # DEV: `Parser#id` will not return 0
44
+ return unless (trace_id && parent_id) || (origin && trace_id)
45
+
46
+ trace_distributed_tags = extract_tags(headers)
47
+
48
+ # Return new trace headers
49
+ TraceDigest.new(
50
+ span_id: parent_id,
51
+ trace_id: trace_id,
52
+ trace_origin: origin,
53
+ trace_sampling_priority: sampling_priority,
54
+ trace_distributed_tags: trace_distributed_tags,
55
+ )
56
+ end
57
+
58
+ private
59
+
60
+ # Export trace distributed tags through the `x-datadog-tags` header.
61
+ #
62
+ # DEV: This method accesses global state (the active trace) to record its error state as a trace tag.
63
+ # DEV: This means errors cannot be reported if there's not active span.
64
+ # DEV: Ideally, we'd have a dedicated error reporting stream for all of ddtrace.
65
+ # DEV: The same comment applies to the {.extract_tags}.
66
+ def inject_tags(digest, env)
67
+ return if digest.trace_distributed_tags.nil? || digest.trace_distributed_tags.empty?
68
+
69
+ if ::Datadog.configuration.tracing.x_datadog_tags_max_length <= 0
70
+ active_trace = Tracing.active_trace
71
+ active_trace.set_tag('_dd.propagation_error', 'disabled') if active_trace
72
+ return
73
+ end
74
+
75
+ encoded_tags = DatadogTagsCodec.encode(digest.trace_distributed_tags)
76
+
77
+ if encoded_tags.size > ::Datadog.configuration.tracing.x_datadog_tags_max_length
78
+ active_trace = Tracing.active_trace
79
+ active_trace.set_tag('_dd.propagation_error', 'inject_max_size') if active_trace
80
+
81
+ ::Datadog.logger.warn(
82
+ "Failed to inject x-datadog-tags: tags are too large (size:#{encoded_tags.size} " \
83
+ "limit:#{::Datadog.configuration.tracing.x_datadog_tags_max_length}). This limit can be configured " \
84
+ 'through the DD_TRACE_X_DATADOG_TAGS_MAX_LENGTH environment variable.'
85
+ )
86
+ return
87
+ end
88
+
89
+ env[HTTP_HEADER_TAGS] = encoded_tags
90
+ rescue => e
91
+ active_trace = Tracing.active_trace
92
+ active_trace.set_tag('_dd.propagation_error', 'encoding_error') if active_trace
93
+ ::Datadog.logger.warn(
94
+ "Failed to inject x-datadog-tags: #{e.class.name} #{e.message} at #{Array(e.backtrace).first}"
95
+ )
96
+ end
97
+
98
+ # Import `x-datadog-tags` header tags as trace distributed tags.
99
+ # Only tags that have the `_dd.p.` prefix are processed.
100
+ def extract_tags(headers)
101
+ tags_header = headers.header(HTTP_HEADER_TAGS)
102
+ return unless tags_header
103
+
104
+ if ::Datadog.configuration.tracing.x_datadog_tags_max_length <= 0
105
+ active_trace = Tracing.active_trace
106
+ active_trace.set_tag('_dd.propagation_error', 'disabled') if active_trace
107
+ return
108
+ end
109
+
110
+ if tags_header.size > ::Datadog.configuration.tracing.x_datadog_tags_max_length
111
+ active_trace = Tracing.active_trace
112
+ active_trace.set_tag('_dd.propagation_error', 'extract_max_size') if active_trace
113
+
114
+ ::Datadog.logger.warn(
115
+ "Failed to extract x-datadog-tags: tags are too large (size:#{tags_header.size} " \
116
+ "limit:#{::Datadog.configuration.tracing.x_datadog_tags_max_length}). This limit can be configured " \
117
+ 'through the DD_TRACE_X_DATADOG_TAGS_MAX_LENGTH environment variable.'
118
+ )
119
+ return
120
+ end
121
+
122
+ tags = DatadogTagsCodec.decode(tags_header)
123
+ # Only extract keys with the expected Datadog prefix
124
+ tags.select! do |key, _|
125
+ key.start_with?(Tracing::Metadata::Ext::Distributed::TAGS_PREFIX) && key != EXCLUDED_TAG
126
+ end
127
+ tags
128
+ rescue => e
129
+ active_trace = Tracing.active_trace
130
+ active_trace.set_tag('_dd.propagation_error', 'decoding_error') if active_trace
131
+ ::Datadog.logger.warn(
132
+ "Failed to extract x-datadog-tags: #{e.class.name} #{e.message} at #{Array(e.backtrace).first}"
133
+ )
134
+ end
25
135
 
26
- def self.extract(env)
27
- # Extract values from headers
28
- headers = Parser.new(env)
29
- trace_id = headers.id(HTTP_HEADER_TRACE_ID)
30
- parent_id = headers.id(HTTP_HEADER_PARENT_ID)
31
- origin = headers.header(HTTP_HEADER_ORIGIN)
32
- sampling_priority = headers.number(HTTP_HEADER_SAMPLING_PRIORITY)
33
-
34
- # Return early if this propagation is not valid
35
- # DEV: To be valid we need to have a trace id and a parent id
36
- # or when it is a synthetics trace, just the trace id.
37
- # DEV: `Parser#id` will not return 0
38
- return unless (trace_id && parent_id) || (origin && trace_id)
39
-
40
- # Return new trace headers
41
- TraceDigest.new(
42
- span_id: parent_id,
43
- trace_id: trace_id,
44
- trace_origin: origin,
45
- trace_sampling_priority: sampling_priority
46
- )
136
+ # We want to exclude tags that we don't want to propagate downstream.
137
+ EXCLUDED_TAG = '_dd.p.upstream_services'
138
+ private_constant :EXCLUDED_TAG
47
139
  end
48
140
  end
49
141
  end
@@ -12,6 +12,8 @@ module Datadog
12
12
  HTTP_HEADER_PARENT_ID = 'x-datadog-parent-id'.freeze
13
13
  HTTP_HEADER_SAMPLING_PRIORITY = 'x-datadog-sampling-priority'.freeze
14
14
  HTTP_HEADER_ORIGIN = 'x-datadog-origin'.freeze
15
+ # Distributed trace-level tags
16
+ HTTP_HEADER_TAGS = 'x-datadog-tags'.freeze
15
17
 
16
18
  # B3 headers used for distributed tracing
17
19
  B3_HEADER_TRACE_ID = 'x-b3-traceid'.freeze