datadog 2.7.1 → 2.18.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (441) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +353 -1
  3. data/ext/datadog_profiling_native_extension/clock_id.h +2 -2
  4. data/ext/datadog_profiling_native_extension/collectors_cpu_and_wall_time_worker.c +78 -102
  5. data/ext/datadog_profiling_native_extension/collectors_discrete_dynamic_sampler.c +1 -1
  6. data/ext/datadog_profiling_native_extension/collectors_discrete_dynamic_sampler.h +1 -1
  7. data/ext/datadog_profiling_native_extension/collectors_idle_sampling_helper.c +16 -16
  8. data/ext/datadog_profiling_native_extension/collectors_stack.c +235 -57
  9. data/ext/datadog_profiling_native_extension/collectors_stack.h +21 -5
  10. data/ext/datadog_profiling_native_extension/collectors_thread_context.c +376 -156
  11. data/ext/datadog_profiling_native_extension/collectors_thread_context.h +1 -0
  12. data/ext/datadog_profiling_native_extension/datadog_ruby_common.c +1 -4
  13. data/ext/datadog_profiling_native_extension/datadog_ruby_common.h +10 -0
  14. data/ext/datadog_profiling_native_extension/encoded_profile.c +79 -0
  15. data/ext/datadog_profiling_native_extension/encoded_profile.h +8 -0
  16. data/ext/datadog_profiling_native_extension/extconf.rb +14 -8
  17. data/ext/datadog_profiling_native_extension/gvl_profiling_helper.c +2 -0
  18. data/ext/datadog_profiling_native_extension/gvl_profiling_helper.h +0 -8
  19. data/ext/datadog_profiling_native_extension/heap_recorder.c +295 -532
  20. data/ext/datadog_profiling_native_extension/heap_recorder.h +6 -8
  21. data/ext/datadog_profiling_native_extension/http_transport.c +64 -98
  22. data/ext/datadog_profiling_native_extension/libdatadog_helpers.c +22 -0
  23. data/ext/datadog_profiling_native_extension/libdatadog_helpers.h +8 -5
  24. data/ext/datadog_profiling_native_extension/private_vm_api_access.c +69 -1
  25. data/ext/datadog_profiling_native_extension/private_vm_api_access.h +16 -4
  26. data/ext/datadog_profiling_native_extension/profiling.c +19 -8
  27. data/ext/datadog_profiling_native_extension/ruby_helpers.c +9 -21
  28. data/ext/datadog_profiling_native_extension/ruby_helpers.h +2 -10
  29. data/ext/datadog_profiling_native_extension/stack_recorder.c +231 -181
  30. data/ext/datadog_profiling_native_extension/stack_recorder.h +2 -2
  31. data/ext/datadog_profiling_native_extension/time_helpers.h +1 -1
  32. data/ext/datadog_profiling_native_extension/unsafe_api_calls_check.c +47 -0
  33. data/ext/datadog_profiling_native_extension/unsafe_api_calls_check.h +31 -0
  34. data/ext/libdatadog_api/crashtracker.c +17 -15
  35. data/ext/libdatadog_api/crashtracker.h +5 -0
  36. data/ext/libdatadog_api/datadog_ruby_common.c +1 -4
  37. data/ext/libdatadog_api/datadog_ruby_common.h +10 -0
  38. data/ext/libdatadog_api/extconf.rb +2 -2
  39. data/ext/libdatadog_api/init.c +15 -0
  40. data/ext/libdatadog_api/library_config.c +164 -0
  41. data/ext/libdatadog_api/library_config.h +25 -0
  42. data/ext/libdatadog_api/macos_development.md +3 -3
  43. data/ext/libdatadog_api/process_discovery.c +112 -0
  44. data/ext/libdatadog_api/process_discovery.h +5 -0
  45. data/ext/libdatadog_extconf_helpers.rb +2 -2
  46. data/lib/datadog/appsec/actions_handler/serializable_backtrace.rb +89 -0
  47. data/lib/datadog/appsec/actions_handler.rb +49 -0
  48. data/lib/datadog/appsec/anonymizer.rb +16 -0
  49. data/lib/datadog/appsec/api_security/lru_cache.rb +56 -0
  50. data/lib/datadog/appsec/api_security/route_extractor.rb +65 -0
  51. data/lib/datadog/appsec/api_security/sampler.rb +59 -0
  52. data/lib/datadog/appsec/api_security.rb +23 -0
  53. data/lib/datadog/appsec/assets/waf_rules/README.md +50 -5
  54. data/lib/datadog/appsec/assets/waf_rules/recommended.json +623 -253
  55. data/lib/datadog/appsec/assets/waf_rules/strict.json +69 -107
  56. data/lib/datadog/appsec/autoload.rb +1 -1
  57. data/lib/datadog/appsec/component.rb +49 -65
  58. data/lib/datadog/appsec/compressed_json.rb +40 -0
  59. data/lib/datadog/appsec/configuration/settings.rb +212 -27
  60. data/lib/datadog/appsec/context.rb +74 -0
  61. data/lib/datadog/appsec/contrib/active_record/instrumentation.rb +92 -0
  62. data/lib/datadog/appsec/contrib/active_record/integration.rb +41 -0
  63. data/lib/datadog/appsec/contrib/active_record/patcher.rb +101 -0
  64. data/lib/datadog/appsec/contrib/auto_instrument.rb +1 -1
  65. data/lib/datadog/appsec/contrib/devise/configuration.rb +52 -0
  66. data/lib/datadog/appsec/contrib/devise/data_extractor.rb +78 -0
  67. data/lib/datadog/appsec/contrib/devise/ext.rb +22 -0
  68. data/lib/datadog/appsec/contrib/devise/integration.rb +1 -2
  69. data/lib/datadog/appsec/contrib/devise/patcher.rb +33 -25
  70. data/lib/datadog/appsec/contrib/devise/patches/signin_tracking_patch.rb +102 -0
  71. data/lib/datadog/appsec/contrib/devise/patches/signup_tracking_patch.rb +69 -0
  72. data/lib/datadog/appsec/contrib/devise/{patcher/rememberable_patch.rb → patches/skip_signin_tracking_patch.rb} +3 -3
  73. data/lib/datadog/appsec/contrib/devise/tracking_middleware.rb +106 -0
  74. data/lib/datadog/appsec/contrib/excon/integration.rb +41 -0
  75. data/lib/datadog/appsec/contrib/excon/patcher.rb +28 -0
  76. data/lib/datadog/appsec/contrib/excon/ssrf_detection_middleware.rb +42 -0
  77. data/lib/datadog/appsec/contrib/faraday/connection_patch.rb +22 -0
  78. data/lib/datadog/appsec/contrib/faraday/integration.rb +42 -0
  79. data/lib/datadog/appsec/contrib/faraday/patcher.rb +53 -0
  80. data/lib/datadog/appsec/contrib/faraday/rack_builder_patch.rb +22 -0
  81. data/lib/datadog/appsec/contrib/faraday/ssrf_detection_middleware.rb +41 -0
  82. data/lib/datadog/appsec/contrib/graphql/appsec_trace.rb +1 -7
  83. data/lib/datadog/appsec/contrib/graphql/gateway/watcher.rb +17 -30
  84. data/lib/datadog/appsec/contrib/graphql/integration.rb +1 -1
  85. data/lib/datadog/appsec/contrib/graphql/patcher.rb +0 -3
  86. data/lib/datadog/appsec/contrib/rack/ext.rb +34 -0
  87. data/lib/datadog/appsec/contrib/rack/gateway/response.rb +3 -3
  88. data/lib/datadog/appsec/contrib/rack/gateway/watcher.rb +78 -98
  89. data/lib/datadog/appsec/contrib/rack/integration.rb +1 -1
  90. data/lib/datadog/appsec/contrib/rack/patcher.rb +0 -3
  91. data/lib/datadog/appsec/contrib/rack/request_body_middleware.rb +10 -11
  92. data/lib/datadog/appsec/contrib/rack/request_middleware.rb +73 -78
  93. data/lib/datadog/appsec/contrib/rails/gateway/watcher.rb +16 -33
  94. data/lib/datadog/appsec/contrib/rails/integration.rb +1 -1
  95. data/lib/datadog/appsec/contrib/rails/patcher.rb +25 -38
  96. data/lib/datadog/appsec/contrib/rest_client/integration.rb +45 -0
  97. data/lib/datadog/appsec/contrib/rest_client/patcher.rb +28 -0
  98. data/lib/datadog/appsec/contrib/rest_client/request_ssrf_detection_patch.rb +38 -0
  99. data/lib/datadog/appsec/contrib/sinatra/gateway/watcher.rb +31 -68
  100. data/lib/datadog/appsec/contrib/sinatra/integration.rb +1 -1
  101. data/lib/datadog/appsec/contrib/sinatra/patcher.rb +5 -31
  102. data/lib/datadog/appsec/event.rb +96 -135
  103. data/lib/datadog/appsec/ext.rb +12 -3
  104. data/lib/datadog/appsec/instrumentation/gateway/argument.rb +7 -2
  105. data/lib/datadog/appsec/instrumentation/gateway/middleware.rb +24 -0
  106. data/lib/datadog/appsec/instrumentation/gateway.rb +17 -22
  107. data/lib/datadog/appsec/metrics/collector.rb +38 -0
  108. data/lib/datadog/appsec/metrics/exporter.rb +35 -0
  109. data/lib/datadog/appsec/metrics/telemetry.rb +23 -0
  110. data/lib/datadog/appsec/metrics.rb +13 -0
  111. data/lib/datadog/appsec/monitor/gateway/watcher.rb +52 -32
  112. data/lib/datadog/appsec/processor/rule_loader.rb +30 -36
  113. data/lib/datadog/appsec/remote.rb +31 -57
  114. data/lib/datadog/appsec/response.rb +19 -85
  115. data/lib/datadog/appsec/security_engine/engine.rb +194 -0
  116. data/lib/datadog/appsec/security_engine/result.rb +67 -0
  117. data/lib/datadog/appsec/security_engine/runner.rb +87 -0
  118. data/lib/datadog/appsec/security_engine.rb +9 -0
  119. data/lib/datadog/appsec/security_event.rb +39 -0
  120. data/lib/datadog/appsec/utils.rb +0 -2
  121. data/lib/datadog/appsec.rb +22 -12
  122. data/lib/datadog/auto_instrument.rb +3 -0
  123. data/lib/datadog/core/buffer/random.rb +18 -2
  124. data/lib/datadog/core/configuration/agent_settings.rb +52 -0
  125. data/lib/datadog/core/configuration/agent_settings_resolver.rb +4 -18
  126. data/lib/datadog/core/configuration/agentless_settings_resolver.rb +176 -0
  127. data/lib/datadog/core/configuration/components.rb +74 -32
  128. data/lib/datadog/core/configuration/components_state.rb +23 -0
  129. data/lib/datadog/core/configuration/ext.rb +5 -1
  130. data/lib/datadog/core/configuration/option.rb +81 -45
  131. data/lib/datadog/core/configuration/option_definition.rb +6 -4
  132. data/lib/datadog/core/configuration/options.rb +3 -3
  133. data/lib/datadog/core/configuration/settings.rb +121 -50
  134. data/lib/datadog/core/configuration/stable_config.rb +22 -0
  135. data/lib/datadog/core/configuration.rb +43 -11
  136. data/lib/datadog/{tracing → core}/contrib/rails/utils.rb +1 -3
  137. data/lib/datadog/core/crashtracking/component.rb +4 -13
  138. data/lib/datadog/core/crashtracking/tag_builder.rb +4 -22
  139. data/lib/datadog/core/diagnostics/environment_logger.rb +1 -1
  140. data/lib/datadog/core/encoding.rb +17 -1
  141. data/lib/datadog/core/environment/agent_info.rb +78 -0
  142. data/lib/datadog/core/environment/cgroup.rb +10 -12
  143. data/lib/datadog/core/environment/container.rb +38 -40
  144. data/lib/datadog/core/environment/ext.rb +6 -6
  145. data/lib/datadog/core/environment/git.rb +1 -0
  146. data/lib/datadog/core/environment/identity.rb +3 -3
  147. data/lib/datadog/core/environment/platform.rb +3 -3
  148. data/lib/datadog/core/environment/variable_helpers.rb +1 -1
  149. data/lib/datadog/core/error.rb +11 -9
  150. data/lib/datadog/core/logger.rb +2 -2
  151. data/lib/datadog/core/metrics/client.rb +27 -27
  152. data/lib/datadog/core/metrics/logging.rb +5 -5
  153. data/lib/datadog/core/process_discovery/tracer_memfd.rb +15 -0
  154. data/lib/datadog/core/process_discovery.rb +36 -0
  155. data/lib/datadog/core/rate_limiter.rb +4 -2
  156. data/lib/datadog/core/remote/client/capabilities.rb +6 -0
  157. data/lib/datadog/core/remote/client.rb +107 -92
  158. data/lib/datadog/core/remote/component.rb +18 -19
  159. data/lib/datadog/core/remote/configuration/digest.rb +7 -7
  160. data/lib/datadog/core/remote/configuration/path.rb +1 -1
  161. data/lib/datadog/core/remote/configuration/repository.rb +14 -1
  162. data/lib/datadog/core/remote/negotiation.rb +9 -9
  163. data/lib/datadog/core/remote/transport/config.rb +4 -3
  164. data/lib/datadog/core/remote/transport/http/api.rb +13 -18
  165. data/lib/datadog/core/remote/transport/http/client.rb +5 -4
  166. data/lib/datadog/core/remote/transport/http/config.rb +27 -55
  167. data/lib/datadog/core/remote/transport/http/negotiation.rb +8 -51
  168. data/lib/datadog/core/remote/transport/http.rb +25 -94
  169. data/lib/datadog/core/remote/transport/negotiation.rb +17 -4
  170. data/lib/datadog/core/remote/worker.rb +10 -7
  171. data/lib/datadog/core/runtime/metrics.rb +12 -5
  172. data/lib/datadog/core/tag_builder.rb +56 -0
  173. data/lib/datadog/core/telemetry/component.rb +84 -49
  174. data/lib/datadog/core/telemetry/emitter.rb +23 -11
  175. data/lib/datadog/core/telemetry/event/app_client_configuration_change.rb +66 -0
  176. data/lib/datadog/core/telemetry/event/app_closing.rb +18 -0
  177. data/lib/datadog/core/telemetry/event/app_dependencies_loaded.rb +33 -0
  178. data/lib/datadog/core/telemetry/event/app_heartbeat.rb +18 -0
  179. data/lib/datadog/core/telemetry/event/app_integrations_change.rb +58 -0
  180. data/lib/datadog/core/telemetry/event/app_started.rb +269 -0
  181. data/lib/datadog/core/telemetry/event/base.rb +40 -0
  182. data/lib/datadog/core/telemetry/event/distributions.rb +18 -0
  183. data/lib/datadog/core/telemetry/event/generate_metrics.rb +43 -0
  184. data/lib/datadog/core/telemetry/event/log.rb +76 -0
  185. data/lib/datadog/core/telemetry/event/message_batch.rb +42 -0
  186. data/lib/datadog/core/telemetry/event/synth_app_client_configuration_change.rb +43 -0
  187. data/lib/datadog/core/telemetry/event.rb +17 -383
  188. data/lib/datadog/core/telemetry/ext.rb +1 -0
  189. data/lib/datadog/core/telemetry/http/adapters/net.rb +12 -97
  190. data/lib/datadog/core/telemetry/logger.rb +5 -4
  191. data/lib/datadog/core/telemetry/logging.rb +12 -6
  192. data/lib/datadog/core/telemetry/metric.rb +28 -6
  193. data/lib/datadog/core/telemetry/request.rb +4 -4
  194. data/lib/datadog/core/telemetry/transport/http/api.rb +43 -0
  195. data/lib/datadog/core/telemetry/transport/http/client.rb +49 -0
  196. data/lib/datadog/core/telemetry/transport/http/telemetry.rb +92 -0
  197. data/lib/datadog/core/telemetry/transport/http.rb +63 -0
  198. data/lib/datadog/core/telemetry/transport/telemetry.rb +51 -0
  199. data/lib/datadog/core/telemetry/worker.rb +128 -25
  200. data/lib/datadog/core/transport/http/adapters/net.rb +17 -2
  201. data/lib/datadog/core/transport/http/adapters/test.rb +2 -1
  202. data/lib/datadog/core/transport/http/adapters/unix_socket.rb +1 -1
  203. data/lib/datadog/{tracing → core}/transport/http/api/instance.rb +18 -1
  204. data/lib/datadog/core/transport/http/api/spec.rb +36 -0
  205. data/lib/datadog/{tracing → core}/transport/http/builder.rb +53 -31
  206. data/lib/datadog/core/transport/http/env.rb +8 -0
  207. data/lib/datadog/core/transport/http.rb +75 -0
  208. data/lib/datadog/core/transport/response.rb +4 -0
  209. data/lib/datadog/core/utils/at_fork_monkey_patch.rb +6 -6
  210. data/lib/datadog/core/utils/duration.rb +32 -32
  211. data/lib/datadog/core/utils/forking.rb +2 -2
  212. data/lib/datadog/core/utils/network.rb +6 -6
  213. data/lib/datadog/core/utils/only_once_successful.rb +16 -5
  214. data/lib/datadog/core/utils/time.rb +20 -0
  215. data/lib/datadog/core/utils/truncation.rb +21 -0
  216. data/lib/datadog/core/utils.rb +7 -0
  217. data/lib/datadog/core/vendor/multipart-post/multipart/post/composite_read_io.rb +1 -1
  218. data/lib/datadog/core/vendor/multipart-post/multipart/post/multipartable.rb +8 -8
  219. data/lib/datadog/core/vendor/multipart-post/multipart/post/parts.rb +7 -7
  220. data/lib/datadog/core/worker.rb +1 -1
  221. data/lib/datadog/core/workers/async.rb +29 -12
  222. data/lib/datadog/core/workers/interval_loop.rb +12 -1
  223. data/lib/datadog/core/workers/runtime_metrics.rb +2 -2
  224. data/lib/datadog/core.rb +8 -0
  225. data/lib/datadog/di/base.rb +115 -0
  226. data/lib/datadog/di/boot.rb +34 -0
  227. data/lib/datadog/di/code_tracker.rb +26 -15
  228. data/lib/datadog/di/component.rb +23 -14
  229. data/lib/datadog/di/configuration/settings.rb +25 -1
  230. data/lib/datadog/di/contrib/active_record.rb +1 -0
  231. data/lib/datadog/di/contrib/railtie.rb +15 -0
  232. data/lib/datadog/di/contrib.rb +28 -0
  233. data/lib/datadog/di/error.rb +5 -0
  234. data/lib/datadog/di/instrumenter.rb +162 -21
  235. data/lib/datadog/di/logger.rb +30 -0
  236. data/lib/datadog/di/preload.rb +18 -0
  237. data/lib/datadog/di/probe.rb +14 -7
  238. data/lib/datadog/di/probe_builder.rb +1 -0
  239. data/lib/datadog/di/probe_manager.rb +11 -5
  240. data/lib/datadog/di/probe_notification_builder.rb +54 -38
  241. data/lib/datadog/di/probe_notifier_worker.rb +60 -26
  242. data/lib/datadog/di/redactor.rb +0 -1
  243. data/lib/datadog/di/remote.rb +147 -0
  244. data/lib/datadog/di/serializer.rb +19 -8
  245. data/lib/datadog/di/transport/diagnostics.rb +62 -0
  246. data/lib/datadog/di/transport/http/api.rb +42 -0
  247. data/lib/datadog/di/transport/http/client.rb +47 -0
  248. data/lib/datadog/di/transport/http/diagnostics.rb +65 -0
  249. data/lib/datadog/di/transport/http/input.rb +77 -0
  250. data/lib/datadog/di/transport/http.rb +57 -0
  251. data/lib/datadog/di/transport/input.rb +70 -0
  252. data/lib/datadog/di/utils.rb +103 -0
  253. data/lib/datadog/di.rb +14 -76
  254. data/lib/datadog/error_tracking/collector.rb +87 -0
  255. data/lib/datadog/error_tracking/component.rb +167 -0
  256. data/lib/datadog/error_tracking/configuration/settings.rb +63 -0
  257. data/lib/datadog/error_tracking/configuration.rb +11 -0
  258. data/lib/datadog/error_tracking/ext.rb +18 -0
  259. data/lib/datadog/error_tracking/extensions.rb +16 -0
  260. data/lib/datadog/error_tracking/filters.rb +77 -0
  261. data/lib/datadog/error_tracking.rb +18 -0
  262. data/lib/datadog/kit/appsec/events.rb +15 -3
  263. data/lib/datadog/kit/identity.rb +9 -5
  264. data/lib/datadog/opentelemetry/api/baggage.rb +90 -0
  265. data/lib/datadog/opentelemetry/api/baggage.rbs +26 -0
  266. data/lib/datadog/opentelemetry/api/context.rb +16 -2
  267. data/lib/datadog/opentelemetry/sdk/trace/span.rb +1 -1
  268. data/lib/datadog/opentelemetry.rb +2 -1
  269. data/lib/datadog/profiling/collectors/code_provenance.rb +18 -9
  270. data/lib/datadog/profiling/collectors/cpu_and_wall_time_worker.rb +4 -0
  271. data/lib/datadog/profiling/collectors/idle_sampling_helper.rb +1 -0
  272. data/lib/datadog/profiling/collectors/info.rb +3 -0
  273. data/lib/datadog/profiling/collectors/thread_context.rb +17 -2
  274. data/lib/datadog/profiling/component.rb +64 -82
  275. data/lib/datadog/profiling/encoded_profile.rb +11 -0
  276. data/lib/datadog/profiling/exporter.rb +3 -4
  277. data/lib/datadog/profiling/ext.rb +0 -14
  278. data/lib/datadog/profiling/flush.rb +5 -8
  279. data/lib/datadog/profiling/http_transport.rb +8 -87
  280. data/lib/datadog/profiling/load_native_extension.rb +1 -33
  281. data/lib/datadog/profiling/profiler.rb +2 -0
  282. data/lib/datadog/profiling/scheduler.rb +10 -2
  283. data/lib/datadog/profiling/stack_recorder.rb +9 -9
  284. data/lib/datadog/profiling/tag_builder.rb +5 -41
  285. data/lib/datadog/profiling/tasks/setup.rb +2 -0
  286. data/lib/datadog/profiling.rb +6 -2
  287. data/lib/datadog/tracing/analytics.rb +1 -1
  288. data/lib/datadog/tracing/component.rb +16 -12
  289. data/lib/datadog/tracing/configuration/ext.rb +8 -1
  290. data/lib/datadog/tracing/configuration/settings.rb +22 -10
  291. data/lib/datadog/tracing/context_provider.rb +1 -1
  292. data/lib/datadog/tracing/contrib/action_cable/integration.rb +5 -2
  293. data/lib/datadog/tracing/contrib/action_mailer/integration.rb +6 -2
  294. data/lib/datadog/tracing/contrib/action_pack/action_controller/instrumentation.rb +15 -0
  295. data/lib/datadog/tracing/contrib/action_pack/action_dispatch/instrumentation.rb +19 -12
  296. data/lib/datadog/tracing/contrib/action_pack/ext.rb +2 -0
  297. data/lib/datadog/tracing/contrib/action_pack/integration.rb +5 -2
  298. data/lib/datadog/tracing/contrib/action_view/integration.rb +5 -2
  299. data/lib/datadog/tracing/contrib/active_job/integration.rb +5 -2
  300. data/lib/datadog/tracing/contrib/active_record/integration.rb +7 -3
  301. data/lib/datadog/tracing/contrib/active_support/cache/events/cache.rb +7 -2
  302. data/lib/datadog/tracing/contrib/active_support/cache/instrumentation.rb +36 -1
  303. data/lib/datadog/tracing/contrib/active_support/cache/patcher.rb +4 -0
  304. data/lib/datadog/tracing/contrib/active_support/cache/redis.rb +14 -4
  305. data/lib/datadog/tracing/contrib/active_support/configuration/settings.rb +10 -0
  306. data/lib/datadog/tracing/contrib/active_support/integration.rb +5 -2
  307. data/lib/datadog/tracing/contrib/auto_instrument.rb +2 -2
  308. data/lib/datadog/tracing/contrib/aws/instrumentation.rb +10 -0
  309. data/lib/datadog/tracing/contrib/aws/integration.rb +3 -0
  310. data/lib/datadog/tracing/contrib/aws/parsed_context.rb +5 -1
  311. data/lib/datadog/tracing/contrib/concurrent_ruby/integration.rb +3 -0
  312. data/lib/datadog/tracing/contrib/configuration/settings.rb +1 -1
  313. data/lib/datadog/tracing/contrib/elasticsearch/configuration/settings.rb +4 -0
  314. data/lib/datadog/tracing/contrib/elasticsearch/patcher.rb +6 -1
  315. data/lib/datadog/tracing/contrib/ethon/easy_patch.rb +4 -5
  316. data/lib/datadog/tracing/contrib/excon/middleware.rb +5 -3
  317. data/lib/datadog/tracing/contrib/ext.rb +1 -0
  318. data/lib/datadog/tracing/contrib/extensions.rb +29 -3
  319. data/lib/datadog/tracing/contrib/faraday/middleware.rb +5 -3
  320. data/lib/datadog/tracing/contrib/graphql/configuration/error_extension_env_parser.rb +21 -0
  321. data/lib/datadog/tracing/contrib/graphql/configuration/settings.rb +11 -0
  322. data/lib/datadog/tracing/contrib/graphql/ext.rb +5 -0
  323. data/lib/datadog/tracing/contrib/graphql/unified_trace.rb +102 -11
  324. data/lib/datadog/tracing/contrib/grpc/datadog_interceptor/client.rb +7 -1
  325. data/lib/datadog/tracing/contrib/grpc/distributed/propagation.rb +3 -0
  326. data/lib/datadog/tracing/contrib/http/circuit_breaker.rb +0 -15
  327. data/lib/datadog/tracing/contrib/http/distributed/propagation.rb +4 -1
  328. data/lib/datadog/tracing/contrib/http/instrumentation.rb +6 -10
  329. data/lib/datadog/tracing/contrib/http/integration.rb +3 -0
  330. data/lib/datadog/tracing/contrib/httpclient/instrumentation.rb +6 -16
  331. data/lib/datadog/tracing/contrib/httprb/instrumentation.rb +7 -15
  332. data/lib/datadog/tracing/contrib/httprb/integration.rb +3 -0
  333. data/lib/datadog/tracing/contrib/kafka/integration.rb +3 -0
  334. data/lib/datadog/tracing/contrib/karafka/configuration/settings.rb +27 -0
  335. data/lib/datadog/tracing/contrib/karafka/distributed/propagation.rb +48 -0
  336. data/lib/datadog/tracing/contrib/karafka/ext.rb +27 -0
  337. data/lib/datadog/tracing/contrib/karafka/integration.rb +45 -0
  338. data/lib/datadog/tracing/contrib/karafka/monitor.rb +66 -0
  339. data/lib/datadog/tracing/contrib/karafka/patcher.rb +71 -0
  340. data/lib/datadog/tracing/contrib/karafka.rb +37 -0
  341. data/lib/datadog/tracing/contrib/lograge/patcher.rb +4 -2
  342. data/lib/datadog/tracing/contrib/mongodb/configuration/settings.rb +8 -0
  343. data/lib/datadog/tracing/contrib/mongodb/ext.rb +1 -0
  344. data/lib/datadog/tracing/contrib/mongodb/integration.rb +3 -0
  345. data/lib/datadog/tracing/contrib/mongodb/subscribers.rb +18 -1
  346. data/lib/datadog/tracing/contrib/opensearch/configuration/settings.rb +17 -0
  347. data/lib/datadog/tracing/contrib/opensearch/ext.rb +9 -0
  348. data/lib/datadog/tracing/contrib/opensearch/integration.rb +3 -0
  349. data/lib/datadog/tracing/contrib/opensearch/patcher.rb +5 -1
  350. data/lib/datadog/tracing/contrib/patcher.rb +5 -2
  351. data/lib/datadog/tracing/contrib/presto/integration.rb +3 -0
  352. data/lib/datadog/tracing/contrib/rack/header_collection.rb +11 -1
  353. data/lib/datadog/tracing/contrib/rack/integration.rb +2 -2
  354. data/lib/datadog/tracing/contrib/rack/middlewares.rb +1 -1
  355. data/lib/datadog/tracing/contrib/rack/request_queue.rb +1 -1
  356. data/lib/datadog/tracing/contrib/rails/framework.rb +2 -2
  357. data/lib/datadog/tracing/contrib/rails/patcher.rb +1 -1
  358. data/lib/datadog/tracing/contrib/rest_client/integration.rb +3 -0
  359. data/lib/datadog/tracing/contrib/rest_client/request_patch.rb +5 -3
  360. data/lib/datadog/tracing/contrib/sidekiq/client_tracer.rb +6 -1
  361. data/lib/datadog/tracing/contrib/sidekiq/distributed/propagation.rb +3 -0
  362. data/lib/datadog/tracing/contrib/sidekiq/ext.rb +1 -0
  363. data/lib/datadog/tracing/contrib/sidekiq/server_tracer.rb +5 -2
  364. data/lib/datadog/tracing/contrib/span_attribute_schema.rb +6 -1
  365. data/lib/datadog/tracing/contrib/support.rb +28 -0
  366. data/lib/datadog/tracing/contrib.rb +1 -0
  367. data/lib/datadog/tracing/correlation.rb +9 -2
  368. data/lib/datadog/tracing/distributed/b3_multi.rb +1 -1
  369. data/lib/datadog/tracing/distributed/b3_single.rb +1 -1
  370. data/lib/datadog/tracing/distributed/baggage.rb +131 -0
  371. data/lib/datadog/tracing/distributed/datadog.rb +4 -2
  372. data/lib/datadog/tracing/distributed/propagation.rb +25 -4
  373. data/lib/datadog/tracing/distributed/propagation_policy.rb +42 -0
  374. data/lib/datadog/tracing/metadata/errors.rb +4 -4
  375. data/lib/datadog/tracing/metadata/ext.rb +5 -0
  376. data/lib/datadog/tracing/metadata/metastruct.rb +36 -0
  377. data/lib/datadog/tracing/metadata/metastruct_tagging.rb +42 -0
  378. data/lib/datadog/tracing/metadata.rb +2 -0
  379. data/lib/datadog/tracing/sampling/rate_sampler.rb +2 -1
  380. data/lib/datadog/tracing/sampling/span/rule.rb +0 -1
  381. data/lib/datadog/tracing/span.rb +22 -5
  382. data/lib/datadog/tracing/span_event.rb +124 -4
  383. data/lib/datadog/tracing/span_operation.rb +52 -16
  384. data/lib/datadog/tracing/sync_writer.rb +10 -6
  385. data/lib/datadog/tracing/trace_digest.rb +9 -2
  386. data/lib/datadog/tracing/trace_operation.rb +55 -27
  387. data/lib/datadog/tracing/trace_segment.rb +6 -4
  388. data/lib/datadog/tracing/tracer.rb +66 -14
  389. data/lib/datadog/tracing/transport/http/api.rb +5 -4
  390. data/lib/datadog/tracing/transport/http/client.rb +5 -4
  391. data/lib/datadog/tracing/transport/http/traces.rb +13 -44
  392. data/lib/datadog/tracing/transport/http.rb +13 -70
  393. data/lib/datadog/tracing/transport/serializable_trace.rb +31 -7
  394. data/lib/datadog/tracing/transport/trace_formatter.rb +7 -0
  395. data/lib/datadog/tracing/transport/traces.rb +47 -13
  396. data/lib/datadog/tracing/utils.rb +1 -1
  397. data/lib/datadog/tracing/workers/trace_writer.rb +8 -5
  398. data/lib/datadog/tracing/workers.rb +5 -4
  399. data/lib/datadog/tracing/writer.rb +10 -6
  400. data/lib/datadog/tracing.rb +16 -3
  401. data/lib/datadog/version.rb +2 -2
  402. data/lib/datadog.rb +2 -0
  403. metadata +149 -54
  404. data/ext/datadog_profiling_loader/datadog_profiling_loader.c +0 -142
  405. data/ext/datadog_profiling_loader/extconf.rb +0 -60
  406. data/lib/datadog/appsec/assets/waf_rules/processors.json +0 -92
  407. data/lib/datadog/appsec/assets/waf_rules/scanners.json +0 -114
  408. data/lib/datadog/appsec/contrib/devise/event.rb +0 -57
  409. data/lib/datadog/appsec/contrib/devise/patcher/authenticatable_patch.rb +0 -77
  410. data/lib/datadog/appsec/contrib/devise/patcher/registration_controller_patch.rb +0 -54
  411. data/lib/datadog/appsec/contrib/devise/resource.rb +0 -35
  412. data/lib/datadog/appsec/contrib/devise/tracking.rb +0 -57
  413. data/lib/datadog/appsec/contrib/graphql/reactive/multiplex.rb +0 -46
  414. data/lib/datadog/appsec/contrib/patcher.rb +0 -12
  415. data/lib/datadog/appsec/contrib/rack/reactive/request.rb +0 -69
  416. data/lib/datadog/appsec/contrib/rack/reactive/request_body.rb +0 -47
  417. data/lib/datadog/appsec/contrib/rack/reactive/response.rb +0 -53
  418. data/lib/datadog/appsec/contrib/rails/reactive/action.rb +0 -53
  419. data/lib/datadog/appsec/contrib/sinatra/ext.rb +0 -14
  420. data/lib/datadog/appsec/contrib/sinatra/reactive/routed.rb +0 -48
  421. data/lib/datadog/appsec/monitor/reactive/set_user.rb +0 -45
  422. data/lib/datadog/appsec/processor/actions.rb +0 -49
  423. data/lib/datadog/appsec/processor/context.rb +0 -107
  424. data/lib/datadog/appsec/processor/rule_merger.rb +0 -170
  425. data/lib/datadog/appsec/processor.rb +0 -106
  426. data/lib/datadog/appsec/reactive/address_hash.rb +0 -22
  427. data/lib/datadog/appsec/reactive/engine.rb +0 -47
  428. data/lib/datadog/appsec/reactive/operation.rb +0 -68
  429. data/lib/datadog/appsec/reactive/subscriber.rb +0 -19
  430. data/lib/datadog/appsec/scope.rb +0 -58
  431. data/lib/datadog/appsec/utils/trace_operation.rb +0 -15
  432. data/lib/datadog/core/crashtracking/agent_base_url.rb +0 -21
  433. data/lib/datadog/core/remote/transport/http/api/instance.rb +0 -39
  434. data/lib/datadog/core/remote/transport/http/api/spec.rb +0 -21
  435. data/lib/datadog/core/remote/transport/http/builder.rb +0 -219
  436. data/lib/datadog/core/telemetry/http/env.rb +0 -20
  437. data/lib/datadog/core/telemetry/http/ext.rb +0 -28
  438. data/lib/datadog/core/telemetry/http/response.rb +0 -70
  439. data/lib/datadog/core/telemetry/http/transport.rb +0 -90
  440. data/lib/datadog/di/transport.rb +0 -81
  441. data/lib/datadog/tracing/transport/http/api/spec.rb +0 -19
@@ -1,8 +1,8 @@
1
1
  # frozen_string_literal: true
2
2
 
3
+ require_relative '../../event'
4
+ require_relative '../../security_event'
3
5
  require_relative '../../instrumentation/gateway'
4
- require_relative '../../reactive/operation'
5
- require_relative '../reactive/set_user'
6
6
 
7
7
  module Datadog
8
8
  module AppSec
@@ -10,51 +10,71 @@ module Datadog
10
10
  module Gateway
11
11
  # Watcher for Apssec internal events
12
12
  module Watcher
13
+ ARBITRARY_VALUE = 'invalid'
14
+ EVENT_LOGIN_SUCCESS = 'users.login.success'
15
+ EVENT_LOGIN_FAILURE = 'users.login.failure'
16
+ WATCHED_LOGIN_EVENTS = [EVENT_LOGIN_SUCCESS, EVENT_LOGIN_FAILURE].freeze
17
+
13
18
  class << self
14
19
  def watch
15
20
  gateway = Instrumentation.gateway
16
21
 
17
22
  watch_user_id(gateway)
23
+ watch_user_login(gateway)
18
24
  end
19
25
 
20
26
  def watch_user_id(gateway = Instrumentation.gateway)
21
27
  gateway.watch('identity.set_user', :appsec) do |stack, user|
22
- block = false
23
- event = nil
24
- scope = Datadog::AppSec.active_scope
25
-
26
- AppSec::Reactive::Operation.new('identity.set_user') do |op|
27
- Monitor::Reactive::SetUser.subscribe(op, scope.processor_context) do |result|
28
- if result.status == :match
29
- # TODO: should this hash be an Event instance instead?
30
- event = {
31
- waf_result: result,
32
- trace: scope.trace,
33
- span: scope.service_entry_span,
34
- user: user,
35
- actions: result.actions
36
- }
37
-
38
- # We want to keep the trace in case of security event
39
- scope.trace.keep! if scope.trace
40
- Datadog::AppSec::Event.tag_and_keep!(scope, result)
41
- scope.processor_context.events << event
42
- end
43
- end
44
-
45
- block = Monitor::Reactive::SetUser.publish(op, user)
28
+ context = AppSec.active_context
29
+
30
+ if user.id.nil? && user.login.nil? && user.session_id.nil?
31
+ Datadog.logger.debug { 'AppSec: skipping WAF check because no user information was provided' }
32
+ next stack.call(user)
33
+ end
34
+
35
+ persistent_data = {}
36
+ persistent_data['usr.id'] = user.id if user.id
37
+ persistent_data['usr.login'] = user.login if user.login
38
+ persistent_data['usr.session_id'] = user.session_id if user.session_id
39
+
40
+ result = context.run_waf(persistent_data, {}, Datadog.configuration.appsec.waf_timeout)
41
+
42
+ if result.match? || result.derivatives.any?
43
+ context.events.push(
44
+ AppSec::SecurityEvent.new(result, trace: context.trace, span: context.span)
45
+ )
46
+ end
47
+
48
+ if result.match?
49
+ AppSec::Event.tag_and_keep!(context, result)
50
+ AppSec::ActionsHandler.handle(result.actions)
46
51
  end
47
52
 
48
- throw(Datadog::AppSec::Ext::INTERRUPT, [nil, [[:block, event]]]) if block
53
+ stack.call(user)
54
+ end
55
+ end
56
+
57
+ def watch_user_login(gateway = Instrumentation.gateway)
58
+ gateway.watch('appsec.events.user_lifecycle', :appsec) do |stack, kind|
59
+ context = AppSec.active_context
49
60
 
50
- ret, res = stack.call(user)
61
+ next stack.call(kind) unless WATCHED_LOGIN_EVENTS.include?(kind)
62
+
63
+ persistent_data = {"server.business_logic.#{kind}" => ARBITRARY_VALUE}
64
+ result = context.run_waf(persistent_data, {}, Datadog.configuration.appsec.waf_timeout)
65
+
66
+ if result.match? || result.derivatives.any?
67
+ context.events.push(
68
+ AppSec::SecurityEvent.new(result, trace: context.trace, span: context.span)
69
+ )
70
+ end
51
71
 
52
- if event
53
- res ||= []
54
- res << [:monitor, event]
72
+ if result.match?
73
+ AppSec::Event.tag_and_keep!(context, result)
74
+ AppSec::ActionsHandler.handle(result.actions)
55
75
  end
56
76
 
57
- [ret, res]
77
+ stack.call(kind)
58
78
  end
59
79
  end
60
80
  end
@@ -10,50 +10,47 @@ module Datadog
10
10
  module RuleLoader
11
11
  class << self
12
12
  def load_rules(ruleset:, telemetry:)
13
- begin
14
- case ruleset
15
- when :recommended, :strict
16
- JSON.parse(Datadog::AppSec::Assets.waf_rules(ruleset))
17
- when :risky
18
- Datadog.logger.warn(
19
- 'The :risky Application Security Management ruleset has been deprecated and no longer available.'\
20
- 'The `:recommended` ruleset will be used instead.'\
21
- 'Please remove the `appsec.ruleset = :risky` setting from your Datadog.configure block.'
22
- )
23
- JSON.parse(Datadog::AppSec::Assets.waf_rules(:recommended))
24
- when String
25
- JSON.parse(File.read(File.expand_path(ruleset)))
26
- when File, StringIO
27
- JSON.parse(ruleset.read || '').tap { ruleset.rewind }
28
- when Hash
29
- ruleset
30
- else
31
- raise ArgumentError, "unsupported value for ruleset setting: #{ruleset.inspect}"
32
- end
33
- rescue StandardError => e
34
- Datadog.logger.error do
35
- "libddwaf ruleset failed to load, ruleset: #{ruleset.inspect} error: #{e.inspect}"
36
- end
13
+ case ruleset
14
+ when :recommended, :strict
15
+ JSON.parse(Datadog::AppSec::Assets.waf_rules(ruleset))
16
+ when :risky
17
+ Datadog.logger.warn(
18
+ 'The :risky Application Security Management ruleset has been deprecated and no longer available.' \
19
+ 'The `:recommended` ruleset will be used instead.' \
20
+ 'Please remove the `appsec.ruleset = :risky` setting from your Datadog.configure block.'
21
+ )
22
+ JSON.parse(Datadog::AppSec::Assets.waf_rules(:recommended))
23
+ when String
24
+ JSON.parse(File.read(File.expand_path(ruleset)))
25
+ when File, StringIO
26
+ JSON.parse(ruleset.read || '').tap { ruleset.rewind }
27
+ when Hash
28
+ ruleset
29
+ else
30
+ raise ArgumentError, "unsupported value for ruleset setting: #{ruleset.inspect}"
31
+ end
32
+ rescue => e
33
+ Datadog.logger.error do
34
+ "libddwaf ruleset failed to load, ruleset: #{ruleset.inspect} error: #{e.inspect}"
35
+ end
37
36
 
38
- telemetry.report(e, description: 'libddwaf ruleset failed to load')
37
+ telemetry.report(e, description: 'libddwaf ruleset failed to load')
39
38
 
40
- nil
41
- end
39
+ raise e
42
40
  end
43
41
 
44
42
  def load_data(ip_denylist: [], user_id_denylist: [])
45
43
  data = []
46
- data << [denylist_data('blocked_ips', ip_denylist)] if ip_denylist.any?
47
- data << [denylist_data('blocked_users', user_id_denylist)] if user_id_denylist.any?
44
+ data << denylist_data('blocked_ips', ip_denylist) if ip_denylist.any?
45
+ data << denylist_data('blocked_users', user_id_denylist) if user_id_denylist.any?
48
46
 
49
47
  data
50
48
  end
51
49
 
52
50
  def load_exclusions(ip_passlist: [])
53
- exclusions = []
54
- exclusions << [passlist_exclusions(ip_passlist)] if ip_passlist.any?
51
+ return [] if ip_passlist.empty?
55
52
 
56
- exclusions
53
+ passlist_exclusions(ip_passlist)
57
54
  end
58
55
 
59
56
  private
@@ -62,7 +59,7 @@ module Datadog
62
59
  {
63
60
  'id' => id,
64
61
  'type' => 'data_with_expiration',
65
- 'data' => denylist.map { |v| { 'value' => v.to_s, 'expiration' => 2**63 } }
62
+ 'data' => denylist.map { |v| {'value' => v.to_s, 'expiration' => 2**63} }
66
63
  }
67
64
  end
68
65
 
@@ -74,9 +71,6 @@ module Datadog
74
71
  when Hash
75
72
  pass = ip_passlist[:pass]
76
73
  monitor = ip_passlist[:monitor]
77
- else
78
- pass = []
79
- monitor = []
80
74
  end
81
75
 
82
76
  exclusions = []
@@ -1,7 +1,6 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  require_relative '../core/remote/dispatcher'
4
- require_relative 'processor/rule_merger'
5
4
  require_relative 'processor/rule_loader'
6
5
 
7
6
  module Datadog
@@ -9,20 +8,23 @@ module Datadog
9
8
  # Remote
10
9
  module Remote
11
10
  class ReadError < StandardError; end
11
+
12
12
  class NoRulesError < StandardError; end
13
13
 
14
14
  class << self
15
- CAP_ASM_RESERVED_1 = 1 << 0 # RESERVED
16
- CAP_ASM_ACTIVATION = 1 << 1 # Remote activation via ASM_FEATURES product
17
- CAP_ASM_IP_BLOCKING = 1 << 2 # accept IP blocking data from ASM_DATA product
18
- CAP_ASM_DD_RULES = 1 << 3 # read ASM rules from ASM_DD product
19
- CAP_ASM_EXCLUSIONS = 1 << 4 # exclusion filters (passlist) via ASM product
20
- CAP_ASM_REQUEST_BLOCKING = 1 << 5 # can block on request info
21
- CAP_ASM_RESPONSE_BLOCKING = 1 << 6 # can block on response info
22
- CAP_ASM_USER_BLOCKING = 1 << 7 # accept user blocking data from ASM_DATA product
23
- CAP_ASM_CUSTOM_RULES = 1 << 8 # accept custom rules
24
- CAP_ASM_CUSTOM_BLOCKING_RESPONSE = 1 << 9 # supports custom http code or redirect sa blocking response
25
- CAP_ASM_TRUSTED_IPS = 1 << 10 # supports trusted ip
15
+ CAP_ASM_RESERVED_1 = 1 << 0 # RESERVED
16
+ CAP_ASM_ACTIVATION = 1 << 1 # Remote activation via ASM_FEATURES product
17
+ CAP_ASM_IP_BLOCKING = 1 << 2 # accept IP blocking data from ASM_DATA product
18
+ CAP_ASM_DD_RULES = 1 << 3 # read ASM rules from ASM_DD product
19
+ CAP_ASM_EXCLUSIONS = 1 << 4 # exclusion filters (passlist) via ASM product
20
+ CAP_ASM_REQUEST_BLOCKING = 1 << 5 # can block on request info
21
+ CAP_ASM_RESPONSE_BLOCKING = 1 << 6 # can block on response info
22
+ CAP_ASM_USER_BLOCKING = 1 << 7 # accept user blocking data from ASM_DATA product
23
+ CAP_ASM_CUSTOM_RULES = 1 << 8 # accept custom rules
24
+ CAP_ASM_CUSTOM_BLOCKING_RESPONSE = 1 << 9 # supports custom http code or redirect sa blocking response
25
+ CAP_ASM_TRUSTED_IPS = 1 << 10 # supports trusted ip
26
+ CAP_ASM_RASP_SSRF = 1 << 23 # support for server-side request forgery exploit prevention rules
27
+ CAP_ASM_RASP_SQLI = 1 << 21 # support for SQL injection exploit prevention rules
26
28
 
27
29
  # TODO: we need to dynamically add CAP_ASM_ACTIVATION once we support it
28
30
  ASM_CAPABILITIES = [
@@ -35,6 +37,8 @@ module Datadog
35
37
  CAP_ASM_CUSTOM_RULES,
36
38
  CAP_ASM_CUSTOM_BLOCKING_RESPONSE,
37
39
  CAP_ASM_TRUSTED_IPS,
40
+ CAP_ASM_RASP_SSRF,
41
+ CAP_ASM_RASP_SQLI,
38
42
  ].freeze
39
43
 
40
44
  ASM_PRODUCTS = [
@@ -52,65 +56,35 @@ module Datadog
52
56
  remote_features_enabled? ? ASM_PRODUCTS : []
53
57
  end
54
58
 
55
- # rubocop:disable Metrics/MethodLength
56
59
  def receivers(telemetry)
57
60
  return [] unless remote_features_enabled?
58
61
 
59
62
  matcher = Core::Remote::Dispatcher::Matcher::Product.new(ASM_PRODUCTS)
60
63
  receiver = Core::Remote::Dispatcher::Receiver.new(matcher) do |repository, changes|
61
- changes.each do |change|
62
- Datadog.logger.debug { "remote config change: '#{change.path}'" }
63
- end
64
+ next unless AppSec.security_engine
64
65
 
65
- rules = []
66
- custom_rules = []
67
- data = []
68
- overrides = []
69
- exclusions = []
70
- actions = []
71
-
72
- repository.contents.each do |content|
73
- parsed_content = parse_content(content)
74
-
75
- case content.path.product
76
- when 'ASM_DD'
77
- rules << parsed_content
78
- when 'ASM_DATA'
79
- data << parsed_content['rules_data'] if parsed_content['rules_data']
80
- when 'ASM'
81
- overrides << parsed_content['rules_override'] if parsed_content['rules_override']
82
- exclusions << parsed_content['exclusions'] if parsed_content['exclusions']
83
- custom_rules << parsed_content['custom_rules'] if parsed_content['custom_rules']
84
- actions.concat(parsed_content['actions']) if parsed_content['actions']
85
- end
86
- end
87
-
88
- if rules.empty?
89
- settings_rules = AppSec::Processor::RuleLoader.load_rules(
90
- telemetry: telemetry,
91
- ruleset: Datadog.configuration.appsec.ruleset
92
- )
66
+ changes.each do |change|
67
+ content = repository[change.path]
68
+ next unless content || change.type == :delete
93
69
 
94
- raise NoRulesError, 'no default rules available' unless settings_rules
70
+ case change.type
71
+ when :insert, :update
72
+ AppSec.security_engine.add_or_update_config(parse_content(content), path: change.path.to_s) # steep:ignore
95
73
 
96
- rules = [settings_rules]
74
+ content.applied # steep:ignore
75
+ when :delete
76
+ AppSec.security_engine.remove_config_at_path(change.path.to_s) # steep:ignore
77
+ end
97
78
  end
98
79
 
99
- ruleset = AppSec::Processor::RuleMerger.merge(
100
- rules: rules,
101
- data: data,
102
- overrides: overrides,
103
- exclusions: exclusions,
104
- custom_rules: custom_rules,
105
- telemetry: telemetry
106
- )
107
-
108
- Datadog::AppSec.reconfigure(ruleset: ruleset, actions: actions, telemetry: telemetry)
80
+ # This is subject to change - we need to remove the reconfiguration mutex
81
+ # and track usages of each WAF handle instead, so that we know when an old
82
+ # WAF handle can be finalized.
83
+ AppSec.reconfigure!
109
84
  end
110
85
 
111
86
  [receiver]
112
87
  end
113
- # rubocop:enable Metrics/MethodLength
114
88
 
115
89
  private
116
90
 
@@ -19,104 +19,38 @@ module Datadog
19
19
  [status, headers, body]
20
20
  end
21
21
 
22
- def to_sinatra_response
23
- ::Sinatra::Response.new(body, status, headers)
24
- end
25
-
26
- def to_action_dispatch_response
27
- ::ActionDispatch::Response.new(status, headers, body)
28
- end
29
-
30
22
  class << self
31
- def negotiate(env, actions)
32
- # @type var configured_response: Response?
33
- configured_response = nil
34
- actions.each do |action|
35
- # Need to use next to make steep happy :(
36
- # I rather use break to stop the execution
37
- next if configured_response
38
-
39
- action_configuration = AppSec::Processor::Actions.fetch_configuration(action)
40
- next unless action_configuration
41
-
42
- configured_response = case action_configuration['type']
43
- when 'block_request'
44
- block_response(env, action_configuration['parameters'])
45
- when 'redirect_request'
46
- redirect_response(env, action_configuration['parameters'])
47
- end
48
- end
23
+ def from_interrupt_params(interrupt_params, http_accept_header)
24
+ return redirect_response(interrupt_params) if interrupt_params['location']
49
25
 
50
- configured_response || default_response(env)
51
- end
52
-
53
- def graphql_response(gateway_multiplex)
54
- multiplex_return = []
55
- gateway_multiplex.queries.each do |query|
56
- # This method is only called in places where GraphQL-Ruby is already required
57
- query_result = ::GraphQL::Query::Result.new(
58
- query: query,
59
- values: JSON.parse(content('application/json'))
60
- )
61
- multiplex_return << query_result
62
- end
63
-
64
- multiplex_return
26
+ block_response(interrupt_params, http_accept_header)
65
27
  end
66
28
 
67
29
  private
68
30
 
69
- def default_response(env)
70
- content_type = content_type(env)
71
-
72
- body = []
73
- body << content(content_type)
31
+ def block_response(interrupt_params, http_accept_header)
32
+ content_type = case interrupt_params['type']
33
+ when nil, 'auto' then content_type(http_accept_header)
34
+ else FORMAT_TO_CONTENT_TYPE.fetch(interrupt_params['type'], DEFAULT_CONTENT_TYPE)
35
+ end
74
36
 
75
37
  Response.new(
76
- status: 403,
77
- headers: { 'Content-Type' => content_type },
78
- body: body,
38
+ status: interrupt_params['status_code']&.to_i || 403,
39
+ headers: {'Content-Type' => content_type},
40
+ body: [content(content_type)],
79
41
  )
80
42
  end
81
43
 
82
- def block_response(env, options)
83
- content_type = if options['type'] == 'auto'
84
- content_type(env)
85
- else
86
- FORMAT_TO_CONTENT_TYPE[options['type']]
87
- end
88
-
89
- body = []
90
- body << content(content_type)
44
+ def redirect_response(interrupt_params)
45
+ status_code = interrupt_params['status_code'].to_i
91
46
 
92
47
  Response.new(
93
- status: options['status_code'] || 403,
94
- headers: { 'Content-Type' => content_type },
95
- body: body,
48
+ status: ((status_code >= 300 && status_code < 400) ? status_code : 303),
49
+ headers: {'Location' => interrupt_params.fetch('location')},
50
+ body: [],
96
51
  )
97
52
  end
98
53
 
99
- def redirect_response(env, options)
100
- if options['location'] && !options['location'].empty?
101
- content_type = content_type(env)
102
-
103
- status = options['status_code'] >= 300 && options['status_code'] < 400 ? options['status_code'] : 303
104
-
105
- headers = {
106
- 'Content-Type' => content_type,
107
- 'Location' => options['location']
108
- }
109
-
110
- Response.new(
111
- status: status,
112
- headers: headers,
113
- body: [],
114
- )
115
- else
116
- default_response(env)
117
- end
118
- end
119
-
120
54
  CONTENT_TYPE_TO_FORMAT = {
121
55
  'application/json' => :json,
122
56
  'text/html' => :html,
@@ -130,10 +64,10 @@ module Datadog
130
64
 
131
65
  DEFAULT_CONTENT_TYPE = 'application/json'
132
66
 
133
- def content_type(env)
134
- return DEFAULT_CONTENT_TYPE unless env.key?('HTTP_ACCEPT')
67
+ def content_type(http_accept_header)
68
+ return DEFAULT_CONTENT_TYPE if http_accept_header.nil?
135
69
 
136
- accept_types = env['HTTP_ACCEPT'].split(',').map(&:strip)
70
+ accept_types = http_accept_header.split(',').map(&:strip)
137
71
 
138
72
  accepted = accept_types.map { |m| Utils::HTTP::MediaRange.new(m) }.sort!.reverse!
139
73
 
@@ -0,0 +1,194 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Datadog
4
+ module AppSec
5
+ module SecurityEngine
6
+ # SecurityEngine::Engine creates WAF builder and manages its configuration.
7
+ # It also rebuilds WAF handle from the WAF builder when configuration changes.
8
+ class Engine
9
+ DEFAULT_RULES_CONFIG_PATH = 'ASM_DD/default'
10
+ TELEMETRY_ACTIONS = %w[init update].freeze
11
+ DIAGNOSTICS_CONFIG_KEYS = %w[
12
+ rules
13
+ custom_rules
14
+ exclusions
15
+ actions
16
+ processors
17
+ scanners
18
+ rules_override
19
+ rules_data
20
+ exclusion_data
21
+ ].freeze
22
+
23
+ attr_reader :waf_addresses, :ruleset_version
24
+
25
+ def initialize(appsec_settings:, telemetry:)
26
+ @default_ruleset = appsec_settings.ruleset
27
+
28
+ # NOTE: replace appsec_settings argument with default_ruleset when removing these deprecated settings
29
+ @default_ip_denylist = appsec_settings.ip_denylist
30
+ @default_user_id_denylist = appsec_settings.user_id_denylist
31
+ @default_ip_passlist = appsec_settings.ip_passlist
32
+
33
+ @waf_builder = WAF::HandleBuilder.new(
34
+ obfuscator: {
35
+ key_regex: appsec_settings.obfuscator_key_regex,
36
+ value_regex: appsec_settings.obfuscator_value_regex
37
+ }
38
+ )
39
+
40
+ diagnostics = load_default_config(telemetry: telemetry)
41
+ report_configuration_diagnostics(diagnostics, action: 'init', telemetry: telemetry)
42
+
43
+ @waf_handle = @waf_builder.build_handle
44
+ @waf_addresses = @waf_handle.known_addresses
45
+ rescue WAF::Error => e
46
+ error_message = "AppSec security engine failed to initialize"
47
+
48
+ Datadog.logger.error("#{error_message}, error #{e.inspect}")
49
+ telemetry.report(e, description: error_message)
50
+
51
+ raise e
52
+ end
53
+
54
+ def finalize!
55
+ @waf_handle&.finalize!
56
+ @waf_builder&.finalize!
57
+
58
+ @waf_addresses = []
59
+ @ruleset_version = nil
60
+ end
61
+
62
+ def new_runner
63
+ SecurityEngine::Runner.new(@waf_handle.build_context)
64
+ end
65
+
66
+ def add_or_update_config(config, path:)
67
+ @is_ruleset_update = path.include?('ASM_DD')
68
+
69
+ # default config has to be removed when adding an ASM_DD config
70
+ remove_config_at_path(DEFAULT_RULES_CONFIG_PATH) if @is_ruleset_update
71
+
72
+ diagnostics = @waf_builder.add_or_update_config(config, path: path)
73
+ @ruleset_version = diagnostics['ruleset_version'] if diagnostics.key?('ruleset_version')
74
+ report_configuration_diagnostics(diagnostics, action: 'update', telemetry: AppSec.telemetry)
75
+
76
+ # we need to load default config if diagnostics contains top-level error for rules or processors
77
+ if @is_ruleset_update &&
78
+ (diagnostics.key?('error') ||
79
+ diagnostics.dig('rules', 'error') ||
80
+ diagnostics.dig('processors', 'errors'))
81
+ diagnostics = load_default_config(telemetry: AppSec.telemetry)
82
+ report_configuration_diagnostics(diagnostics, action: 'update', telemetry: AppSec.telemetry)
83
+ end
84
+
85
+ diagnostics
86
+ rescue WAF::Error => e
87
+ error_message = "libddwaf builder failed to add or update config at path: #{path}"
88
+
89
+ Datadog.logger.debug("#{error_message}, error: #{e.inspect}")
90
+ AppSec.telemetry.report(e, description: error_message)
91
+ end
92
+
93
+ def remove_config_at_path(path)
94
+ result = @waf_builder.remove_config_at_path(path)
95
+
96
+ if result && path != DEFAULT_RULES_CONFIG_PATH && path.include?('ASM_DD')
97
+ diagnostics = load_default_config(telemetry: AppSec.telemetry)
98
+ report_configuration_diagnostics(diagnostics, action: 'update', telemetry: AppSec.telemetry)
99
+ end
100
+
101
+ result
102
+ rescue WAF::Error => e
103
+ error_message = "libddwaf handle builder failed to remove config at path: #{path}"
104
+
105
+ Datadog.logger.error("#{error_message}, error: #{e.inspect}")
106
+ AppSec.telemetry.report(e, description: error_message)
107
+ end
108
+
109
+ def reconfigure!
110
+ old_waf_handle = @waf_handle
111
+
112
+ @waf_handle = @waf_builder.build_handle
113
+ @waf_addresses = @waf_handle.known_addresses
114
+
115
+ old_waf_handle&.finalize!
116
+ rescue WAF::Error => e
117
+ error_message = "AppSec security engine failed to reconfigure"
118
+
119
+ Datadog.logger.error("#{error_message}, error #{e.inspect}")
120
+ AppSec.telemetry.report(e, description: error_message)
121
+
122
+ if old_waf_handle
123
+ Datadog.logger.warn("Reverting to the previous configuration")
124
+
125
+ @waf_handle = old_waf_handle
126
+ @waf_addresses = old_waf_handle.known_addresses
127
+ end
128
+ end
129
+
130
+ private
131
+
132
+ def load_default_config(telemetry:)
133
+ config = AppSec::Processor::RuleLoader.load_rules(telemetry: telemetry, ruleset: @default_ruleset)
134
+
135
+ # deprecated - ip and user id denylists should be configured via RC
136
+ config['rules_data'] ||= AppSec::Processor::RuleLoader.load_data(
137
+ ip_denylist: @default_ip_denylist,
138
+ user_id_denylist: @default_user_id_denylist
139
+ )
140
+
141
+ # deprecated - ip passlist should be configured via RC
142
+ config['exclusions'] ||= AppSec::Processor::RuleLoader.load_exclusions(ip_passlist: @default_ip_passlist)
143
+
144
+ diagnostics = @waf_builder.add_or_update_config(config, path: DEFAULT_RULES_CONFIG_PATH)
145
+ @ruleset_version = diagnostics['ruleset_version']
146
+
147
+ diagnostics
148
+ end
149
+
150
+ def report_configuration_diagnostics(diagnostics, action:, telemetry:)
151
+ raise ArgumentError, 'action must be one of TELEMETRY_ACTIONS' unless TELEMETRY_ACTIONS.include?(action)
152
+
153
+ common_tags = {
154
+ waf_version: Datadog::AppSec::WAF::VERSION::BASE_STRING,
155
+ event_rules_version: diagnostics.fetch('ruleset_version', @ruleset_version).to_s,
156
+ action: action
157
+ }
158
+
159
+ if diagnostics['error']
160
+ telemetry.inc(
161
+ Ext::TELEMETRY_METRICS_NAMESPACE, 'waf.config_errors', 1,
162
+ tags: common_tags.merge(scope: 'top-level')
163
+ )
164
+
165
+ telemetry.error(diagnostics['error'])
166
+ end
167
+
168
+ diagnostics.each do |config_key, config_diagnostics|
169
+ next unless DIAGNOSTICS_CONFIG_KEYS.include?(config_key)
170
+ next if !config_diagnostics.key?('error') && config_diagnostics.fetch('errors', []).empty?
171
+
172
+ if config_diagnostics['error']
173
+ telemetry.error(config_diagnostics['error'])
174
+
175
+ telemetry.inc(
176
+ Ext::TELEMETRY_METRICS_NAMESPACE, 'waf.config_errors', 1,
177
+ tags: common_tags.merge(config_key: config_key, scope: 'top-level')
178
+ )
179
+ elsif config_diagnostics['errors']
180
+ config_diagnostics['errors'].each do |error, config_ids|
181
+ telemetry.error("#{error}: [#{config_ids.join(",")}]")
182
+ end
183
+
184
+ telemetry.inc(
185
+ Ext::TELEMETRY_METRICS_NAMESPACE, 'waf.config_errors', config_diagnostics['errors'].count,
186
+ tags: common_tags.merge(config_key: config_key, scope: 'item')
187
+ )
188
+ end
189
+ end
190
+ end
191
+ end
192
+ end
193
+ end
194
+ end