ddtrace 1.14.0 → 1.15.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (270) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +178 -2
  3. data/ext/ddtrace_profiling_native_extension/NativeExtensionDesign.md +3 -5
  4. data/ext/ddtrace_profiling_native_extension/clock_id.h +0 -3
  5. data/ext/ddtrace_profiling_native_extension/clock_id_from_pthread.c +0 -22
  6. data/ext/ddtrace_profiling_native_extension/clock_id_noop.c +0 -1
  7. data/ext/ddtrace_profiling_native_extension/collectors_cpu_and_wall_time_worker.c +41 -6
  8. data/ext/ddtrace_profiling_native_extension/collectors_idle_sampling_helper.c +3 -0
  9. data/ext/ddtrace_profiling_native_extension/collectors_stack.c +76 -24
  10. data/ext/ddtrace_profiling_native_extension/collectors_stack.h +1 -1
  11. data/ext/ddtrace_profiling_native_extension/collectors_thread_context.c +207 -32
  12. data/ext/ddtrace_profiling_native_extension/collectors_thread_context.h +1 -1
  13. data/ext/ddtrace_profiling_native_extension/extconf.rb +8 -2
  14. data/ext/ddtrace_profiling_native_extension/http_transport.c +26 -10
  15. data/ext/ddtrace_profiling_native_extension/libdatadog_helpers.c +42 -0
  16. data/ext/ddtrace_profiling_native_extension/libdatadog_helpers.h +6 -0
  17. data/ext/ddtrace_profiling_native_extension/native_extension_helpers.rb +1 -16
  18. data/ext/ddtrace_profiling_native_extension/pid_controller.c +57 -0
  19. data/ext/ddtrace_profiling_native_extension/pid_controller.h +45 -0
  20. data/ext/ddtrace_profiling_native_extension/private_vm_api_access.c +17 -12
  21. data/ext/ddtrace_profiling_native_extension/profiling.c +0 -2
  22. data/ext/ddtrace_profiling_native_extension/stack_recorder.c +74 -37
  23. data/ext/ddtrace_profiling_native_extension/stack_recorder.h +13 -3
  24. data/lib/datadog/appsec/assets/waf_rules/processors.json +92 -0
  25. data/lib/datadog/appsec/assets/waf_rules/recommended.json +698 -75
  26. data/lib/datadog/appsec/assets/waf_rules/scanners.json +114 -0
  27. data/lib/datadog/appsec/assets/waf_rules/strict.json +98 -8
  28. data/lib/datadog/appsec/assets.rb +8 -0
  29. data/lib/datadog/appsec/component.rb +9 -2
  30. data/lib/datadog/appsec/configuration/settings.rb +61 -2
  31. data/lib/datadog/appsec/contrib/rack/gateway/watcher.rb +8 -6
  32. data/lib/datadog/appsec/contrib/rack/reactive/request.rb +2 -7
  33. data/lib/datadog/appsec/contrib/rack/reactive/request_body.rb +2 -5
  34. data/lib/datadog/appsec/contrib/rack/reactive/response.rb +2 -5
  35. data/lib/datadog/appsec/contrib/rack/request_body_middleware.rb +3 -2
  36. data/lib/datadog/appsec/contrib/rack/request_middleware.rb +23 -9
  37. data/lib/datadog/appsec/contrib/rails/gateway/watcher.rb +3 -2
  38. data/lib/datadog/appsec/contrib/rails/patcher.rb +9 -3
  39. data/lib/datadog/appsec/contrib/rails/reactive/action.rb +2 -5
  40. data/lib/datadog/appsec/contrib/sinatra/gateway/watcher.rb +6 -4
  41. data/lib/datadog/appsec/contrib/sinatra/patcher.rb +13 -7
  42. data/lib/datadog/appsec/contrib/sinatra/reactive/routed.rb +2 -5
  43. data/lib/datadog/appsec/event.rb +106 -50
  44. data/lib/datadog/appsec/monitor/gateway/watcher.rb +3 -3
  45. data/lib/datadog/appsec/monitor/reactive/set_user.rb +2 -5
  46. data/lib/datadog/appsec/processor/actions.rb +49 -0
  47. data/lib/datadog/appsec/processor/rule_merger.rb +22 -2
  48. data/lib/datadog/appsec/processor.rb +34 -6
  49. data/lib/datadog/appsec/remote.rb +4 -1
  50. data/lib/datadog/appsec/response.rb +82 -4
  51. data/lib/datadog/appsec/sample_rate.rb +21 -0
  52. data/lib/datadog/appsec.rb +2 -2
  53. data/lib/datadog/core/configuration/agent_settings_resolver.rb +29 -24
  54. data/lib/datadog/core/configuration/base.rb +1 -11
  55. data/lib/datadog/core/configuration/components.rb +7 -2
  56. data/lib/datadog/core/configuration/ext.rb +21 -0
  57. data/lib/datadog/core/configuration/option.rb +2 -4
  58. data/lib/datadog/core/configuration/option_definition.rb +17 -41
  59. data/lib/datadog/core/configuration/options.rb +5 -5
  60. data/lib/datadog/core/configuration/settings.rb +47 -45
  61. data/lib/datadog/core/environment/execution.rb +47 -9
  62. data/lib/datadog/core/environment/variable_helpers.rb +0 -69
  63. data/lib/datadog/core/error.rb +1 -0
  64. data/lib/datadog/core/git/ext.rb +2 -0
  65. data/lib/datadog/core/remote/client/capabilities.rb +1 -1
  66. data/lib/datadog/core/remote/component.rb +2 -2
  67. data/lib/datadog/core/remote/negotiation.rb +2 -2
  68. data/lib/datadog/core/remote/transport/config.rb +60 -0
  69. data/lib/datadog/core/remote/transport/http/api/instance.rb +39 -0
  70. data/lib/datadog/core/remote/transport/http/api/spec.rb +21 -0
  71. data/lib/datadog/core/remote/transport/http/api.rb +58 -0
  72. data/lib/datadog/core/remote/transport/http/builder.rb +219 -0
  73. data/lib/datadog/core/remote/transport/http/client.rb +48 -0
  74. data/lib/datadog/core/remote/transport/http/config.rb +280 -0
  75. data/lib/datadog/core/remote/transport/http/negotiation.rb +146 -0
  76. data/lib/datadog/core/remote/transport/http.rb +179 -0
  77. data/lib/datadog/core/{transport → remote/transport}/negotiation.rb +25 -23
  78. data/lib/datadog/core/telemetry/collector.rb +3 -2
  79. data/lib/datadog/core/telemetry/http/transport.rb +2 -1
  80. data/lib/datadog/core/transport/ext.rb +47 -0
  81. data/lib/datadog/core/transport/http/adapters/net.rb +168 -0
  82. data/lib/datadog/core/transport/http/adapters/registry.rb +29 -0
  83. data/lib/datadog/core/transport/http/adapters/test.rb +89 -0
  84. data/lib/datadog/core/transport/http/adapters/unix_socket.rb +83 -0
  85. data/lib/datadog/core/transport/http/api/endpoint.rb +31 -0
  86. data/lib/datadog/core/transport/http/api/fallbacks.rb +26 -0
  87. data/lib/datadog/core/transport/http/api/map.rb +18 -0
  88. data/lib/datadog/core/transport/http/env.rb +62 -0
  89. data/lib/datadog/core/transport/http/response.rb +60 -0
  90. data/lib/datadog/core/transport/parcel.rb +22 -0
  91. data/lib/datadog/core/transport/request.rb +17 -0
  92. data/lib/datadog/core/transport/response.rb +64 -0
  93. data/lib/datadog/core/workers/polling.rb +2 -2
  94. data/lib/datadog/opentelemetry/api/context.rb +10 -3
  95. data/lib/datadog/opentelemetry/sdk/propagator.rb +2 -1
  96. data/lib/datadog/opentelemetry/sdk/span_processor.rb +14 -2
  97. data/lib/datadog/opentelemetry/sdk/trace/span.rb +68 -0
  98. data/lib/datadog/opentelemetry/trace.rb +58 -0
  99. data/lib/datadog/opentelemetry.rb +1 -0
  100. data/lib/datadog/opentracer.rb +9 -0
  101. data/lib/datadog/profiling/collectors/cpu_and_wall_time_worker.rb +12 -18
  102. data/lib/datadog/profiling/collectors/idle_sampling_helper.rb +1 -1
  103. data/lib/datadog/profiling/collectors/thread_context.rb +9 -1
  104. data/lib/datadog/profiling/component.rb +24 -99
  105. data/lib/datadog/profiling/ext.rb +0 -12
  106. data/lib/datadog/profiling/flush.rb +0 -3
  107. data/lib/datadog/profiling/http_transport.rb +6 -3
  108. data/lib/datadog/profiling/native_extension.rb +0 -21
  109. data/lib/datadog/profiling/profiler.rb +11 -12
  110. data/lib/datadog/profiling.rb +8 -81
  111. data/lib/datadog/tracing/component.rb +10 -4
  112. data/lib/datadog/tracing/configuration/agent_settings_resolver.rb +13 -0
  113. data/lib/datadog/tracing/configuration/ext.rb +4 -2
  114. data/lib/datadog/tracing/configuration/settings.rb +14 -7
  115. data/lib/datadog/tracing/contrib/action_pack/configuration/settings.rb +1 -1
  116. data/lib/datadog/tracing/contrib/active_job/configuration/settings.rb +1 -1
  117. data/lib/datadog/tracing/contrib/active_record/events/sql.rb +4 -0
  118. data/lib/datadog/tracing/contrib/active_support/cache/instrumentation.rb +104 -197
  119. data/lib/datadog/tracing/contrib/active_support/cache/patcher.rb +3 -0
  120. data/lib/datadog/tracing/contrib/aws/instrumentation.rb +7 -0
  121. data/lib/datadog/tracing/contrib/configuration/settings.rb +1 -1
  122. data/lib/datadog/tracing/contrib/dalli/configuration/settings.rb +6 -0
  123. data/lib/datadog/tracing/contrib/dalli/ext.rb +7 -0
  124. data/lib/datadog/tracing/contrib/dalli/instrumentation.rb +9 -2
  125. data/lib/datadog/tracing/contrib/delayed_job/configuration/settings.rb +1 -1
  126. data/lib/datadog/tracing/contrib/elasticsearch/patcher.rb +5 -0
  127. data/lib/datadog/tracing/contrib/ethon/easy_patch.rb +5 -0
  128. data/lib/datadog/tracing/contrib/ethon/multi_patch.rb +8 -0
  129. data/lib/datadog/tracing/contrib/excon/middleware.rb +5 -0
  130. data/lib/datadog/tracing/contrib/ext.rb +3 -0
  131. data/lib/datadog/tracing/contrib/faraday/configuration/settings.rb +1 -1
  132. data/lib/datadog/tracing/contrib/faraday/middleware.rb +5 -0
  133. data/lib/datadog/tracing/contrib/grpc/configuration/settings.rb +21 -1
  134. data/lib/datadog/tracing/contrib/grpc/datadog_interceptor/client.rb +11 -1
  135. data/lib/datadog/tracing/contrib/grpc/datadog_interceptor/server.rb +18 -0
  136. data/lib/datadog/tracing/contrib/grpc/datadog_interceptor.rb +0 -4
  137. data/lib/datadog/tracing/contrib/http/circuit_breaker.rb +3 -3
  138. data/lib/datadog/tracing/contrib/http/instrumentation.rb +5 -0
  139. data/lib/datadog/tracing/contrib/httpclient/instrumentation.rb +5 -0
  140. data/lib/datadog/tracing/contrib/httprb/instrumentation.rb +5 -0
  141. data/lib/datadog/tracing/contrib/mongodb/subscribers.rb +7 -0
  142. data/lib/datadog/tracing/contrib/mysql2/instrumentation.rb +13 -3
  143. data/lib/datadog/tracing/contrib/opensearch/integration.rb +2 -2
  144. data/lib/datadog/tracing/contrib/opensearch/patcher.rb +7 -0
  145. data/lib/datadog/tracing/contrib/pg/instrumentation.rb +5 -0
  146. data/lib/datadog/tracing/contrib/presto/instrumentation.rb +5 -0
  147. data/lib/datadog/tracing/contrib/propagation/sql_comment.rb +1 -1
  148. data/lib/datadog/tracing/contrib/que/configuration/settings.rb +1 -1
  149. data/lib/datadog/tracing/contrib/racecar/event.rb +5 -0
  150. data/lib/datadog/tracing/contrib/rack/header_tagging.rb +14 -4
  151. data/lib/datadog/tracing/contrib/rails/configuration/settings.rb +4 -4
  152. data/lib/datadog/tracing/contrib/rake/configuration/settings.rb +1 -1
  153. data/lib/datadog/tracing/contrib/redis/tags.rb +5 -0
  154. data/lib/datadog/tracing/contrib/resque/configuration/settings.rb +1 -1
  155. data/lib/datadog/tracing/contrib/rest_client/request_patch.rb +5 -0
  156. data/lib/datadog/tracing/contrib/sequel/utils.rb +5 -0
  157. data/lib/datadog/tracing/contrib/shoryuken/configuration/settings.rb +1 -1
  158. data/lib/datadog/tracing/contrib/sidekiq/configuration/settings.rb +1 -1
  159. data/lib/datadog/tracing/contrib/sneakers/configuration/settings.rb +1 -1
  160. data/lib/datadog/tracing/contrib/utils/quantization/http.rb +2 -2
  161. data/lib/datadog/tracing/distributed/propagation.rb +13 -33
  162. data/lib/datadog/tracing/metadata/tagging.rb +3 -3
  163. data/lib/datadog/tracing/sync_writer.rb +3 -3
  164. data/lib/datadog/tracing/tracer.rb +2 -0
  165. data/lib/datadog/{core → tracing}/transport/http/api/instance.rb +1 -1
  166. data/lib/datadog/{core → tracing}/transport/http/api/spec.rb +1 -1
  167. data/lib/datadog/tracing/transport/http/api.rb +43 -0
  168. data/lib/datadog/{core → tracing}/transport/http/builder.rb +13 -68
  169. data/lib/datadog/tracing/transport/http/client.rb +57 -0
  170. data/lib/datadog/tracing/transport/http/statistics.rb +47 -0
  171. data/lib/datadog/tracing/transport/http/traces.rb +152 -0
  172. data/lib/datadog/tracing/transport/http.rb +124 -0
  173. data/lib/datadog/tracing/transport/io/client.rb +89 -0
  174. data/lib/datadog/tracing/transport/io/response.rb +27 -0
  175. data/lib/datadog/tracing/transport/io/traces.rb +101 -0
  176. data/lib/datadog/tracing/transport/io.rb +30 -0
  177. data/lib/datadog/tracing/transport/serializable_trace.rb +126 -0
  178. data/lib/datadog/tracing/transport/statistics.rb +77 -0
  179. data/lib/datadog/tracing/transport/trace_formatter.rb +209 -0
  180. data/lib/datadog/tracing/transport/traces.rb +224 -0
  181. data/lib/datadog/tracing/workers/trace_writer.rb +5 -3
  182. data/lib/datadog/tracing/workers.rb +3 -2
  183. data/lib/datadog/tracing/writer.rb +5 -2
  184. data/lib/ddtrace/transport/ext.rb +17 -15
  185. data/lib/ddtrace/version.rb +1 -1
  186. data/lib/ddtrace.rb +1 -1
  187. metadata +72 -96
  188. data/lib/datadog/ci/configuration/components.rb +0 -32
  189. data/lib/datadog/ci/configuration/settings.rb +0 -51
  190. data/lib/datadog/ci/contrib/cucumber/configuration/settings.rb +0 -35
  191. data/lib/datadog/ci/contrib/cucumber/ext.rb +0 -22
  192. data/lib/datadog/ci/contrib/cucumber/formatter.rb +0 -94
  193. data/lib/datadog/ci/contrib/cucumber/instrumentation.rb +0 -28
  194. data/lib/datadog/ci/contrib/cucumber/integration.rb +0 -47
  195. data/lib/datadog/ci/contrib/cucumber/patcher.rb +0 -27
  196. data/lib/datadog/ci/contrib/minitest/configuration/settings.rb +0 -35
  197. data/lib/datadog/ci/contrib/minitest/ext.rb +0 -21
  198. data/lib/datadog/ci/contrib/minitest/integration.rb +0 -49
  199. data/lib/datadog/ci/contrib/minitest/patcher.rb +0 -27
  200. data/lib/datadog/ci/contrib/minitest/test_helper.rb +0 -68
  201. data/lib/datadog/ci/contrib/rspec/configuration/settings.rb +0 -35
  202. data/lib/datadog/ci/contrib/rspec/example.rb +0 -68
  203. data/lib/datadog/ci/contrib/rspec/ext.rb +0 -21
  204. data/lib/datadog/ci/contrib/rspec/integration.rb +0 -48
  205. data/lib/datadog/ci/contrib/rspec/patcher.rb +0 -27
  206. data/lib/datadog/ci/ext/app_types.rb +0 -9
  207. data/lib/datadog/ci/ext/environment.rb +0 -575
  208. data/lib/datadog/ci/ext/settings.rb +0 -10
  209. data/lib/datadog/ci/ext/test.rb +0 -35
  210. data/lib/datadog/ci/extensions.rb +0 -19
  211. data/lib/datadog/ci/flush.rb +0 -38
  212. data/lib/datadog/ci/test.rb +0 -81
  213. data/lib/datadog/ci.rb +0 -21
  214. data/lib/datadog/core/configuration/dependency_resolver.rb +0 -28
  215. data/lib/datadog/core/configuration/option_definition_set.rb +0 -22
  216. data/lib/datadog/core/configuration/option_set.rb +0 -10
  217. data/lib/datadog/core/transport/config.rb +0 -58
  218. data/lib/datadog/core/transport/http/api.rb +0 -57
  219. data/lib/datadog/core/transport/http/client.rb +0 -45
  220. data/lib/datadog/core/transport/http/config.rb +0 -278
  221. data/lib/datadog/core/transport/http/negotiation.rb +0 -144
  222. data/lib/datadog/core/transport/http.rb +0 -169
  223. data/lib/datadog/core/utils/object_set.rb +0 -43
  224. data/lib/datadog/core/utils/string_table.rb +0 -47
  225. data/lib/datadog/profiling/backtrace_location.rb +0 -34
  226. data/lib/datadog/profiling/buffer.rb +0 -43
  227. data/lib/datadog/profiling/collectors/old_stack.rb +0 -301
  228. data/lib/datadog/profiling/encoding/profile.rb +0 -41
  229. data/lib/datadog/profiling/event.rb +0 -15
  230. data/lib/datadog/profiling/events/stack.rb +0 -82
  231. data/lib/datadog/profiling/old_recorder.rb +0 -107
  232. data/lib/datadog/profiling/pprof/builder.rb +0 -125
  233. data/lib/datadog/profiling/pprof/converter.rb +0 -102
  234. data/lib/datadog/profiling/pprof/message_set.rb +0 -16
  235. data/lib/datadog/profiling/pprof/payload.rb +0 -20
  236. data/lib/datadog/profiling/pprof/pprof.proto +0 -212
  237. data/lib/datadog/profiling/pprof/pprof_pb.rb +0 -81
  238. data/lib/datadog/profiling/pprof/stack_sample.rb +0 -139
  239. data/lib/datadog/profiling/pprof/string_table.rb +0 -12
  240. data/lib/datadog/profiling/pprof/template.rb +0 -118
  241. data/lib/datadog/profiling/trace_identifiers/ddtrace.rb +0 -43
  242. data/lib/datadog/profiling/trace_identifiers/helper.rb +0 -45
  243. data/lib/ddtrace/transport/http/adapters/net.rb +0 -168
  244. data/lib/ddtrace/transport/http/adapters/registry.rb +0 -27
  245. data/lib/ddtrace/transport/http/adapters/test.rb +0 -85
  246. data/lib/ddtrace/transport/http/adapters/unix_socket.rb +0 -77
  247. data/lib/ddtrace/transport/http/api/endpoint.rb +0 -29
  248. data/lib/ddtrace/transport/http/api/fallbacks.rb +0 -24
  249. data/lib/ddtrace/transport/http/api/instance.rb +0 -35
  250. data/lib/ddtrace/transport/http/api/map.rb +0 -16
  251. data/lib/ddtrace/transport/http/api/spec.rb +0 -17
  252. data/lib/ddtrace/transport/http/api.rb +0 -39
  253. data/lib/ddtrace/transport/http/builder.rb +0 -176
  254. data/lib/ddtrace/transport/http/client.rb +0 -52
  255. data/lib/ddtrace/transport/http/env.rb +0 -58
  256. data/lib/ddtrace/transport/http/response.rb +0 -58
  257. data/lib/ddtrace/transport/http/statistics.rb +0 -43
  258. data/lib/ddtrace/transport/http/traces.rb +0 -144
  259. data/lib/ddtrace/transport/http.rb +0 -117
  260. data/lib/ddtrace/transport/io/client.rb +0 -85
  261. data/lib/ddtrace/transport/io/response.rb +0 -25
  262. data/lib/ddtrace/transport/io/traces.rb +0 -99
  263. data/lib/ddtrace/transport/io.rb +0 -28
  264. data/lib/ddtrace/transport/parcel.rb +0 -20
  265. data/lib/ddtrace/transport/request.rb +0 -15
  266. data/lib/ddtrace/transport/response.rb +0 -60
  267. data/lib/ddtrace/transport/serializable_trace.rb +0 -122
  268. data/lib/ddtrace/transport/statistics.rb +0 -75
  269. data/lib/ddtrace/transport/trace_formatter.rb +0 -207
  270. data/lib/ddtrace/transport/traces.rb +0 -216
@@ -1,6 +1,7 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  require_relative '../../core/utils/duration'
4
+ require_relative '../sample_rate'
4
5
 
5
6
  module Datadog
6
7
  module AppSec
@@ -25,7 +26,7 @@ module Datadog
25
26
  add_settings!(base)
26
27
  end
27
28
 
28
- # rubocop:disable Metrics/AbcSize,Metrics/MethodLength,Metrics/BlockLength
29
+ # rubocop:disable Metrics/AbcSize,Metrics/MethodLength,Metrics/BlockLength,Metrics/CyclomaticComplexity,Metrics/PerceivedComplexity
29
30
  def self.add_settings!(base)
30
31
  base.class_eval do
31
32
  settings :appsec do
@@ -95,6 +96,46 @@ module Datadog
95
96
  o.default DEFAULT_OBFUSCATOR_VALUE_REGEX
96
97
  end
97
98
 
99
+ settings :block do
100
+ settings :templates do
101
+ option :html do |o|
102
+ o.env 'DD_APPSEC_HTTP_BLOCKED_TEMPLATE_HTML'
103
+ o.type :string, nilable: true
104
+ o.setter do |value|
105
+ if value
106
+ raise(ArgumentError, "appsec.templates.html: file not found: #{value}") unless File.exist?(value)
107
+
108
+ File.open(value, 'rb', &:read) || ''
109
+ end
110
+ end
111
+ end
112
+
113
+ option :json do |o|
114
+ o.env 'DD_APPSEC_HTTP_BLOCKED_TEMPLATE_JSON'
115
+ o.type :string, nilable: true
116
+ o.setter do |value|
117
+ if value
118
+ raise(ArgumentError, "appsec.templates.json: file not found: #{value}") unless File.exist?(value)
119
+
120
+ File.open(value, 'rb', &:read) || ''
121
+ end
122
+ end
123
+ end
124
+
125
+ option :text do |o|
126
+ o.env 'DD_APPSEC_HTTP_BLOCKED_TEMPLATE_TEXT'
127
+ o.type :string, nilable: true
128
+ o.setter do |value|
129
+ if value
130
+ raise(ArgumentError, "appsec.templates.text: file not found: #{value}") unless File.exist?(value)
131
+
132
+ File.open(value, 'rb', &:read) || ''
133
+ end
134
+ end
135
+ end
136
+ end
137
+ end
138
+
98
139
  settings :track_user_events do
99
140
  option :enabled do |o|
100
141
  o.default true
@@ -129,10 +170,28 @@ module Datadog
129
170
  end
130
171
  end
131
172
  end
173
+
174
+ settings :api_security do
175
+ option :enabled do |o|
176
+ o.type :bool
177
+ o.env 'DD_EXPERIMENTAL_API_SECURITY_ENABLED'
178
+ o.default false
179
+ end
180
+
181
+ option :sample_rate do |o|
182
+ o.type :float
183
+ o.env 'DD_API_SECURITY_REQUEST_SAMPLE_RATE'
184
+ o.default 0.1
185
+ o.setter do |value|
186
+ value = 1 if value > 1
187
+ SampleRate.new(value)
188
+ end
189
+ end
190
+ end
132
191
  end
133
192
  end
134
193
  end
135
- # rubocop:enable Metrics/AbcSize,Metrics/MethodLength,Metrics/BlockLength
194
+ # rubocop:enable Metrics/AbcSize,Metrics/MethodLength,Metrics/BlockLength,Metrics/CyclomaticComplexity,Metrics/PerceivedComplexity
136
195
  end
137
196
  end
138
197
  end
@@ -28,7 +28,7 @@ module Datadog
28
28
  scope = gateway_request.env[Datadog::AppSec::Ext::SCOPE_KEY]
29
29
 
30
30
  AppSec::Reactive::Operation.new('rack.request') do |op|
31
- Rack::Reactive::Request.subscribe(op, scope.processor_context) do |result, _block|
31
+ Rack::Reactive::Request.subscribe(op, scope.processor_context) do |result|
32
32
  if result.status == :match
33
33
  # TODO: should this hash be an Event instance instead?
34
34
  event = {
@@ -48,7 +48,7 @@ module Datadog
48
48
  end
49
49
  end
50
50
 
51
- _result, block = Rack::Reactive::Request.publish(op, gateway_request)
51
+ block = Rack::Reactive::Request.publish(op, gateway_request)
52
52
  end
53
53
 
54
54
  next [nil, [[:block, event]]] if block
@@ -67,11 +67,12 @@ module Datadog
67
67
  def watch_response(gateway = Instrumentation.gateway)
68
68
  gateway.watch('rack.response', :appsec) do |stack, gateway_response|
69
69
  block = false
70
+
70
71
  event = nil
71
72
  scope = gateway_response.scope
72
73
 
73
74
  AppSec::Reactive::Operation.new('rack.response') do |op|
74
- Rack::Reactive::Response.subscribe(op, scope.processor_context) do |result, _block|
75
+ Rack::Reactive::Response.subscribe(op, scope.processor_context) do |result|
75
76
  if result.status == :match
76
77
  # TODO: should this hash be an Event instance instead?
77
78
  event = {
@@ -91,7 +92,7 @@ module Datadog
91
92
  end
92
93
  end
93
94
 
94
- _result, block = Rack::Reactive::Response.publish(op, gateway_response)
95
+ block = Rack::Reactive::Response.publish(op, gateway_response)
95
96
  end
96
97
 
97
98
  next [nil, [[:block, event]]] if block
@@ -110,11 +111,12 @@ module Datadog
110
111
  def watch_request_body(gateway = Instrumentation.gateway)
111
112
  gateway.watch('rack.request.body', :appsec) do |stack, gateway_request|
112
113
  block = false
114
+
113
115
  event = nil
114
116
  scope = gateway_request.env[Datadog::AppSec::Ext::SCOPE_KEY]
115
117
 
116
118
  AppSec::Reactive::Operation.new('rack.request.body') do |op|
117
- Rack::Reactive::RequestBody.subscribe(op, scope.processor_context) do |result, _block|
119
+ Rack::Reactive::RequestBody.subscribe(op, scope.processor_context) do |result|
118
120
  if result.status == :match
119
121
  # TODO: should this hash be an Event instance instead?
120
122
  event = {
@@ -134,7 +136,7 @@ module Datadog
134
136
  end
135
137
  end
136
138
 
137
- _result, block = Rack::Reactive::RequestBody.publish(op, gateway_request)
139
+ block = Rack::Reactive::RequestBody.publish(op, gateway_request)
138
140
  end
139
141
 
140
142
  next [nil, [[:block, event]]] if block
@@ -30,7 +30,6 @@ module Datadog
30
30
  end
31
31
  end
32
32
 
33
- # rubocop:disable Metrics/MethodLength
34
33
  def self.subscribe(op, waf_context)
35
34
  op.subscribe(*ADDRESSES) do |*values|
36
35
  Datadog.logger.debug { "reacted to #{ADDRESSES.inspect}: #{values.inspect}" }
@@ -61,11 +60,8 @@ module Datadog
61
60
  when :match
62
61
  Datadog.logger.debug { "WAF: #{result.inspect}" }
63
62
 
64
- block = result.actions.include?('block')
65
-
66
- yield [result, block]
67
-
68
- throw(:block, [result, true]) if block
63
+ yield result
64
+ throw(:block, true) unless result.actions.empty?
69
65
  when :ok
70
66
  Datadog.logger.debug { "WAF OK: #{result.inspect}" }
71
67
  when :invalid_call
@@ -76,7 +72,6 @@ module Datadog
76
72
  Datadog.logger.debug { "WAF UNKNOWN: #{result.status.inspect} #{result.inspect}" }
77
73
  end
78
74
  end
79
- # rubocop:enable Metrics/MethodLength
80
75
  end
81
76
  end
82
77
  end
@@ -39,11 +39,8 @@ module Datadog
39
39
  when :match
40
40
  Datadog.logger.debug { "WAF: #{result.inspect}" }
41
41
 
42
- block = result.actions.include?('block')
43
-
44
- yield [result, block]
45
-
46
- throw(:block, [result, true]) if block
42
+ yield result
43
+ throw(:block, true) unless result.actions.empty?
47
44
  when :ok
48
45
  Datadog.logger.debug { "WAF OK: #{result.inspect}" }
49
46
  when :invalid_call
@@ -45,11 +45,8 @@ module Datadog
45
45
  when :match
46
46
  Datadog.logger.debug { "WAF: #{result.inspect}" }
47
47
 
48
- block = result.actions.include?('block')
49
-
50
- yield [result, block]
51
-
52
- throw(:block, [result, true]) if block
48
+ yield result
49
+ throw(:block, true) unless result.actions.empty?
53
50
  when :ok
54
51
  Datadog.logger.debug { "WAF OK: #{result.inspect}" }
55
52
  when :invalid_call
@@ -30,8 +30,9 @@ module Datadog
30
30
  @app.call(env)
31
31
  end
32
32
 
33
- if request_response && request_response.any? { |action, _event| action == :block }
34
- request_return = AppSec::Response.negotiate(env).to_rack
33
+ if request_response
34
+ blocked_event = request_response.find { |action, _event| action == :block }
35
+ request_return = AppSec::Response.negotiate(env, blocked_event.last[:actions]).to_rack if blocked_event
35
36
  end
36
37
 
37
38
  request_return
@@ -61,8 +61,9 @@ module Datadog
61
61
  end
62
62
  end
63
63
 
64
- if request_response && request_response.any? { |action, _event| action == :block }
65
- request_return = AppSec::Response.negotiate(env).to_rack
64
+ if request_response
65
+ blocked_event = request_response.find { |action, _options| action == :block }
66
+ request_return = AppSec::Response.negotiate(env, blocked_event.last[:actions]).to_rack if blocked_event
66
67
  end
67
68
 
68
69
  gateway_response = Gateway::Response.new(
@@ -74,6 +75,16 @@ module Datadog
74
75
 
75
76
  _response_return, response_response = Instrumentation.gateway.push('rack.response', gateway_response)
76
77
 
78
+ result = scope.processor_context.extract_schema
79
+
80
+ if result
81
+ scope.processor_context.events << {
82
+ trace: scope.trace,
83
+ span: scope.service_entry_span,
84
+ waf_result: result,
85
+ }
86
+ end
87
+
77
88
  scope.processor_context.events.each do |e|
78
89
  e[:response] ||= gateway_response
79
90
  e[:request] ||= gateway_request
@@ -81,8 +92,9 @@ module Datadog
81
92
 
82
93
  AppSec::Event.record(scope.service_entry_span, *scope.processor_context.events)
83
94
 
84
- if response_response && response_response.any? { |action, _event| action == :block }
85
- request_return = AppSec::Response.negotiate(env).to_rack
95
+ if response_response
96
+ blocked_event = response_response.find { |action, _options| action == :block }
97
+ request_return = AppSec::Response.negotiate(env, blocked_event.last[:actions]).to_rack if blocked_event
86
98
  end
87
99
 
88
100
  request_return
@@ -137,17 +149,19 @@ module Datadog
137
149
  )
138
150
  end
139
151
 
140
- if processor.ruleset_info
141
- span.set_tag('_dd.appsec.event_rules.version', processor.ruleset_info[:version])
152
+ if processor.diagnostics
153
+ diagnostics = processor.diagnostics
154
+
155
+ span.set_tag('_dd.appsec.event_rules.version', diagnostics['ruleset_version'])
142
156
 
143
157
  unless @oneshot_tags_sent
144
158
  # Small race condition, but it's inoccuous: worst case the tags
145
159
  # are sent a couple of times more than expected
146
160
  @oneshot_tags_sent = true
147
161
 
148
- span.set_tag('_dd.appsec.event_rules.loaded', processor.ruleset_info[:loaded].to_f)
149
- span.set_tag('_dd.appsec.event_rules.error_count', processor.ruleset_info[:failed].to_f)
150
- span.set_tag('_dd.appsec.event_rules.errors', JSON.dump(processor.ruleset_info[:errors]))
162
+ span.set_tag('_dd.appsec.event_rules.loaded', diagnostics['rules']['loaded'].size.to_f)
163
+ span.set_tag('_dd.appsec.event_rules.error_count', diagnostics['rules']['failed'].size.to_f)
164
+ span.set_tag('_dd.appsec.event_rules.errors', JSON.dump(diagnostics['rules']['errors']))
151
165
  span.set_tag('_dd.appsec.event_rules.addresses', JSON.dump(processor.addresses))
152
166
 
153
167
  # Ensure these tags reach the backend
@@ -20,11 +20,12 @@ module Datadog
20
20
  def watch_request_action(gateway = Instrumentation.gateway)
21
21
  gateway.watch('rails.request.action', :appsec) do |stack, gateway_request|
22
22
  block = false
23
+
23
24
  event = nil
24
25
  scope = gateway_request.env[Datadog::AppSec::Ext::SCOPE_KEY]
25
26
 
26
27
  AppSec::Reactive::Operation.new('rails.request.action') do |op|
27
- Rails::Reactive::Action.subscribe(op, scope.processor_context) do |result, _block|
28
+ Rails::Reactive::Action.subscribe(op, scope.processor_context) do |result|
28
29
  if result.status == :match
29
30
  # TODO: should this hash be an Event instance instead?
30
31
  event = {
@@ -44,7 +45,7 @@ module Datadog
44
45
  end
45
46
  end
46
47
 
47
- _result, block = Rails::Reactive::Action.publish(op, gateway_request)
48
+ block = Rails::Reactive::Action.publish(op, gateway_request)
48
49
  end
49
50
 
50
51
  next [nil, [[:block, event]]] if block
@@ -83,9 +83,15 @@ module Datadog
83
83
  super
84
84
  end
85
85
 
86
- if request_response && request_response.any? { |action, _event| action == :block }
87
- @_response = AppSec::Response.negotiate(env).to_action_dispatch_response
88
- request_return = @_response.body
86
+ if request_response
87
+ blocked_event = request_response.find { |action, _options| action == :block }
88
+ if blocked_event
89
+ @_response = AppSec::Response.negotiate(
90
+ env,
91
+ blocked_event.last[:actions]
92
+ ).to_action_dispatch_response
93
+ request_return = @_response.body
94
+ end
89
95
  end
90
96
 
91
97
  request_return
@@ -45,11 +45,8 @@ module Datadog
45
45
  when :match
46
46
  Datadog.logger.debug { "WAF: #{result.inspect}" }
47
47
 
48
- block = result.actions.include?('block')
49
-
50
- yield [result, block]
51
-
52
- throw(:block, [result, true]) if block
48
+ yield result
49
+ throw(:block, true) unless result.actions.empty?
53
50
  when :ok
54
51
  Datadog.logger.debug { "WAF OK: #{result.inspect}" }
55
52
  when :invalid_call
@@ -22,11 +22,12 @@ module Datadog
22
22
  def watch_request_dispatch(gateway = Instrumentation.gateway)
23
23
  gateway.watch('sinatra.request.dispatch', :appsec) do |stack, gateway_request|
24
24
  block = false
25
+
25
26
  event = nil
26
27
  scope = gateway_request.env[Datadog::AppSec::Ext::SCOPE_KEY]
27
28
 
28
29
  AppSec::Reactive::Operation.new('sinatra.request.dispatch') do |op|
29
- Rack::Reactive::RequestBody.subscribe(op, scope.processor_context) do |result, _block|
30
+ Rack::Reactive::RequestBody.subscribe(op, scope.processor_context) do |result|
30
31
  if result.status == :match
31
32
  # TODO: should this hash be an Event instance instead?
32
33
  event = {
@@ -46,7 +47,7 @@ module Datadog
46
47
  end
47
48
  end
48
49
 
49
- _result, block = Rack::Reactive::RequestBody.publish(op, gateway_request)
50
+ block = Rack::Reactive::RequestBody.publish(op, gateway_request)
50
51
  end
51
52
 
52
53
  next [nil, [[:block, event]]] if block
@@ -65,11 +66,12 @@ module Datadog
65
66
  def watch_request_routed(gateway = Instrumentation.gateway)
66
67
  gateway.watch('sinatra.request.routed', :appsec) do |stack, (gateway_request, gateway_route_params)|
67
68
  block = false
69
+
68
70
  event = nil
69
71
  scope = gateway_request.env[Datadog::AppSec::Ext::SCOPE_KEY]
70
72
 
71
73
  AppSec::Reactive::Operation.new('sinatra.request.routed') do |op|
72
- Sinatra::Reactive::Routed.subscribe(op, scope.processor_context) do |result, _block|
74
+ Sinatra::Reactive::Routed.subscribe(op, scope.processor_context) do |result|
73
75
  if result.status == :match
74
76
  # TODO: should this hash be an Event instance instead?
75
77
  event = {
@@ -89,7 +91,7 @@ module Datadog
89
91
  end
90
92
  end
91
93
 
92
- _result, block = Sinatra::Reactive::Routed.publish(op, [gateway_request, gateway_route_params])
94
+ block = Sinatra::Reactive::Routed.publish(op, [gateway_request, gateway_route_params])
93
95
  end
94
96
 
95
97
  next [nil, [[:block, event]]] if block
@@ -65,9 +65,12 @@ module Datadog
65
65
  catch(Datadog::AppSec::Contrib::Sinatra::Ext::ROUTE_INTERRUPT) { super }
66
66
  end
67
67
 
68
- if request_response && request_response.any? { |action, _event| action == :block }
69
- self.response = AppSec::Response.negotiate(env).to_sinatra_response
70
- request_return = nil
68
+ if request_response
69
+ blocked_event = request_response.find { |action, _options| action == :block }
70
+ if blocked_event
71
+ self.response = AppSec::Response.negotiate(env, blocked_event.last[:actions]).to_sinatra_response
72
+ request_return = nil
73
+ end
71
74
  end
72
75
 
73
76
  request_return
@@ -103,11 +106,14 @@ module Datadog
103
106
  [gateway_request, gateway_route_params]
104
107
  )
105
108
 
106
- if request_response && request_response.any? { |action, _event| action == :block }
107
- self.response = AppSec::Response.negotiate(env).to_sinatra_response
109
+ if request_response
110
+ blocked_event = request_response.find { |action, _options| action == :block }
111
+ if blocked_event
112
+ self.response = AppSec::Response.negotiate(env, blocked_event.last[:actions]).to_sinatra_response
108
113
 
109
- # interrupt request and return response to dispatch! for consistency
110
- throw(Datadog::AppSec::Contrib::Sinatra::Ext::ROUTE_INTERRUPT, response)
114
+ # interrupt request and return response to dispatch! for consistency
115
+ throw(Datadog::AppSec::Contrib::Sinatra::Ext::ROUTE_INTERRUPT, response)
116
+ end
111
117
  end
112
118
 
113
119
  yield(*args)
@@ -40,11 +40,8 @@ module Datadog
40
40
  when :match
41
41
  Datadog.logger.debug { "WAF: #{result.inspect}" }
42
42
 
43
- block = result.actions.include?('block')
44
-
45
- yield [result, block]
46
-
47
- throw(:block, [result, true]) if block
43
+ yield result
44
+ throw(:block, true) unless result.actions.empty?
48
45
  when :ok
49
46
  Datadog.logger.debug { "WAF OK: #{result.inspect}" }
50
47
  when :invalid_call
@@ -1,4 +1,6 @@
1
1
  require 'json'
2
+ require 'zlib'
3
+ require 'base64'
2
4
 
3
5
  require_relative 'rate_limiter'
4
6
 
@@ -34,78 +36,132 @@ module Datadog
34
36
  Content-Language
35
37
  ].map!(&:downcase).freeze
36
38
 
39
+ MAX_ENCODED_SCHEMA_SIZE = 25000
40
+ # For more information about this number
41
+ # please check https://github.com/DataDog/dd-trace-rb/pull/3177#issuecomment-1747221082
42
+ MIN_SCHEMA_SIZE_FOR_COMPRESSION = 260
43
+
37
44
  # Record events for a trace
38
45
  #
39
46
  # This is expected to be called only once per trace for the rate limiter
40
47
  # to properly apply
41
- def self.record(span, *events)
42
- # ensure rate limiter is called only when there are events to record
43
- return if events.empty? || span.nil?
48
+ class << self
49
+ def record(span, *events)
50
+ # ensure rate limiter is called only when there are events to record
51
+ return if events.empty? || span.nil?
44
52
 
45
- Datadog::AppSec::RateLimiter.limit(:traces) do
46
- record_via_span(span, *events)
53
+ Datadog::AppSec::RateLimiter.limit(:traces) do
54
+ record_via_span(span, *events)
55
+ end
47
56
  end
48
- end
49
57
 
50
- def self.record_via_span(span, *events)
51
- events.group_by { |e| e[:trace] }.each do |trace, event_group|
52
- unless trace
53
- Datadog.logger.debug { "{ error: 'no trace: cannot record', event_group: #{event_group.inspect}}" }
54
- next
55
- end
58
+ def record_via_span(span, *events)
59
+ events.group_by { |e| e[:trace] }.each do |trace, event_group|
60
+ unless trace
61
+ Datadog.logger.debug { "{ error: 'no trace: cannot record', event_group: #{event_group.inspect}}" }
62
+ next
63
+ end
64
+
65
+ trace.keep!
66
+ trace.set_tag(
67
+ Datadog::Tracing::Metadata::Ext::Distributed::TAG_DECISION_MAKER,
68
+ Datadog::Tracing::Sampling::Ext::Decision::ASM
69
+ )
56
70
 
57
- trace.keep!
58
- trace.set_tag(
59
- Datadog::Tracing::Metadata::Ext::Distributed::TAG_DECISION_MAKER,
60
- Datadog::Tracing::Sampling::Ext::Decision::ASM
61
- )
62
-
63
- # prepare and gather tags to apply
64
- service_entry_tags = build_service_entry_tags(event_group)
65
- # complex types are unsupported, we need to serialize to a string
66
- triggers = service_entry_tags.delete('_dd.appsec.triggers')
67
- span.set_tag('_dd.appsec.json', JSON.dump({ triggers: triggers }))
68
-
69
- # apply tags to service entry span
70
- service_entry_tags.each do |key, value|
71
- span.set_tag(key, value)
71
+ # prepare and gather tags to apply
72
+ service_entry_tags = build_service_entry_tags(event_group)
73
+
74
+ # apply tags to service entry span
75
+ service_entry_tags.each do |key, value|
76
+ span.set_tag(key, value)
77
+ end
72
78
  end
73
79
  end
74
- end
75
80
 
76
- def self.build_service_entry_tags(event_group)
77
- event_group.each_with_object({}) do |event, tags|
78
- # TODO: assume HTTP request context for now
81
+ # rubocop:disable Metrics/MethodLength
82
+ def build_service_entry_tags(event_group)
83
+ waf_events = []
84
+ entry_tags = event_group.each_with_object({ '_dd.origin' => 'appsec' }) do |event, tags|
85
+ # TODO: assume HTTP request context for now
86
+ if (request = event[:request])
87
+ request.headers.each do |header, value|
88
+ tags["http.request.headers.#{header}"] = value if ALLOWED_REQUEST_HEADERS.include?(header.downcase)
89
+ end
90
+
91
+ tags['http.host'] = request.host
92
+ tags['http.useragent'] = request.user_agent
93
+ tags['network.client.ip'] = request.remote_addr
94
+ end
79
95
 
80
- if (request = event[:request])
81
- request_headers = request.headers.select do |k, _|
82
- ALLOWED_REQUEST_HEADERS.include?(k.downcase)
96
+ if (response = event[:response])
97
+ response.headers.each do |header, value|
98
+ tags["http.response.headers.#{header}"] = value if ALLOWED_RESPONSE_HEADERS.include?(header.downcase)
99
+ end
83
100
  end
84
101
 
85
- request_headers.each do |header, value|
86
- tags["http.request.headers.#{header}"] = value
102
+ waf_result = event[:waf_result]
103
+ # accumulate triggers
104
+ waf_events += waf_result.events
105
+
106
+ waf_result.derivatives.each do |key, value|
107
+ parsed_value = json_parse(value)
108
+ next unless parsed_value
109
+
110
+ parsed_value_size = parsed_value.size
111
+
112
+ schema_value = if parsed_value_size >= MIN_SCHEMA_SIZE_FOR_COMPRESSION
113
+ compressed_and_base64_encoded(parsed_value)
114
+ else
115
+ parsed_value
116
+ end
117
+ next unless schema_value
118
+
119
+ if schema_value.size >= MAX_ENCODED_SCHEMA_SIZE
120
+ Datadog.logger.debug do
121
+ "Schema key: #{key} exceeds the max size value. It will not be included as part of the span tags"
122
+ end
123
+ next
124
+ end
125
+
126
+ tags[key] = schema_value
87
127
  end
88
128
 
89
- tags['http.host'] = request.host
90
- tags['http.useragent'] = request.user_agent
91
- tags['network.client.ip'] = request.remote_addr
129
+ tags
92
130
  end
93
131
 
94
- if (response = event[:response])
95
- response_headers = response.headers.select do |k, _|
96
- ALLOWED_RESPONSE_HEADERS.include?(k.downcase)
97
- end
132
+ appsec_events = json_parse({ triggers: waf_events })
133
+ entry_tags['_dd.appsec.json'] = appsec_events if appsec_events
134
+ entry_tags
135
+ end
136
+ # rubocop:enable Metrics/MethodLength
98
137
 
99
- response_headers.each do |header, value|
100
- tags["http.response.headers.#{header}"] = value
101
- end
138
+ private
139
+
140
+ def compressed_and_base64_encoded(value)
141
+ Base64.encode64(gzip(value))
142
+ rescue TypeError => e
143
+ Datadog.logger.debug do
144
+ "Failed to compress and encode value when populating AppSec::Event. Error: #{e.message}"
102
145
  end
146
+ nil
147
+ end
103
148
 
104
- tags['_dd.origin'] = 'appsec'
149
+ def json_parse(value)
150
+ JSON.dump(value)
151
+ rescue ArgumentError => e
152
+ Datadog.logger.debug do
153
+ "Failed to parse value to JSON when populating AppSec::Event. Error: #{e.message}"
154
+ end
155
+ nil
156
+ end
105
157
 
106
- # accumulate triggers
107
- tags['_dd.appsec.triggers'] ||= []
108
- tags['_dd.appsec.triggers'] += event[:waf_result].data
158
+ def gzip(value)
159
+ sio = StringIO.new
160
+ # For an in depth comparison of Zlib options check https://github.com/DataDog/dd-trace-rb/pull/3177#issuecomment-1747215473
161
+ gz = Zlib::GzipWriter.new(sio, Zlib::BEST_SPEED, Zlib::DEFAULT_STRATEGY)
162
+ gz.write(value)
163
+ gz.close
164
+ sio.string
109
165
  end
110
166
  end
111
167
  end
@@ -24,7 +24,7 @@ module Datadog
24
24
  scope = Datadog::AppSec.active_scope
25
25
 
26
26
  AppSec::Reactive::Operation.new('identity.set_user') do |op|
27
- Monitor::Reactive::SetUser.subscribe(op, scope.processor_context) do |result, _block|
27
+ Monitor::Reactive::SetUser.subscribe(op, scope.processor_context) do |result|
28
28
  if result.status == :match
29
29
  # TODO: should this hash be an Event instance instead?
30
30
  event = {
@@ -44,10 +44,10 @@ module Datadog
44
44
  end
45
45
  end
46
46
 
47
- _result, block = Monitor::Reactive::SetUser.publish(op, user)
47
+ block = Monitor::Reactive::SetUser.publish(op, user)
48
48
  end
49
49
 
50
- throw(Datadog::AppSec::Ext::INTERRUPT, [nil, [:block, event]]) if block
50
+ throw(Datadog::AppSec::Ext::INTERRUPT, [nil, [[:block, event]]]) if block
51
51
 
52
52
  ret, res = stack.call(user)
53
53