ddtrace 1.13.1 → 1.15.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +207 -2
- data/ext/ddtrace_profiling_native_extension/NativeExtensionDesign.md +3 -5
- data/ext/ddtrace_profiling_native_extension/clock_id.h +0 -3
- data/ext/ddtrace_profiling_native_extension/clock_id_from_pthread.c +0 -22
- data/ext/ddtrace_profiling_native_extension/clock_id_noop.c +0 -1
- data/ext/ddtrace_profiling_native_extension/collectors_cpu_and_wall_time_worker.c +41 -6
- data/ext/ddtrace_profiling_native_extension/collectors_idle_sampling_helper.c +3 -0
- data/ext/ddtrace_profiling_native_extension/collectors_stack.c +76 -24
- data/ext/ddtrace_profiling_native_extension/collectors_stack.h +1 -1
- data/ext/ddtrace_profiling_native_extension/collectors_thread_context.c +207 -32
- data/ext/ddtrace_profiling_native_extension/collectors_thread_context.h +1 -1
- data/ext/ddtrace_profiling_native_extension/extconf.rb +8 -2
- data/ext/ddtrace_profiling_native_extension/http_transport.c +26 -10
- data/ext/ddtrace_profiling_native_extension/libdatadog_helpers.c +42 -0
- data/ext/ddtrace_profiling_native_extension/libdatadog_helpers.h +6 -0
- data/ext/ddtrace_profiling_native_extension/native_extension_helpers.rb +1 -1
- data/ext/ddtrace_profiling_native_extension/pid_controller.c +57 -0
- data/ext/ddtrace_profiling_native_extension/pid_controller.h +45 -0
- data/ext/ddtrace_profiling_native_extension/private_vm_api_access.c +17 -12
- data/ext/ddtrace_profiling_native_extension/profiling.c +0 -2
- data/ext/ddtrace_profiling_native_extension/stack_recorder.c +74 -37
- data/ext/ddtrace_profiling_native_extension/stack_recorder.h +13 -3
- data/lib/datadog/appsec/assets/waf_rules/processors.json +92 -0
- data/lib/datadog/appsec/assets/waf_rules/recommended.json +698 -75
- data/lib/datadog/appsec/assets/waf_rules/scanners.json +114 -0
- data/lib/datadog/appsec/assets/waf_rules/strict.json +98 -8
- data/lib/datadog/appsec/assets.rb +8 -0
- data/lib/datadog/appsec/component.rb +9 -2
- data/lib/datadog/appsec/configuration/settings.rb +68 -3
- data/lib/datadog/appsec/contrib/rack/gateway/watcher.rb +8 -6
- data/lib/datadog/appsec/contrib/rack/reactive/request.rb +2 -7
- data/lib/datadog/appsec/contrib/rack/reactive/request_body.rb +2 -5
- data/lib/datadog/appsec/contrib/rack/reactive/response.rb +2 -5
- data/lib/datadog/appsec/contrib/rack/request_body_middleware.rb +3 -2
- data/lib/datadog/appsec/contrib/rack/request_middleware.rb +23 -9
- data/lib/datadog/appsec/contrib/rails/gateway/watcher.rb +3 -2
- data/lib/datadog/appsec/contrib/rails/patcher.rb +9 -3
- data/lib/datadog/appsec/contrib/rails/reactive/action.rb +2 -5
- data/lib/datadog/appsec/contrib/sinatra/gateway/watcher.rb +6 -4
- data/lib/datadog/appsec/contrib/sinatra/patcher.rb +13 -7
- data/lib/datadog/appsec/contrib/sinatra/reactive/routed.rb +2 -5
- data/lib/datadog/appsec/event.rb +106 -50
- data/lib/datadog/appsec/monitor/gateway/watcher.rb +3 -3
- data/lib/datadog/appsec/monitor/reactive/set_user.rb +2 -5
- data/lib/datadog/appsec/processor/actions.rb +49 -0
- data/lib/datadog/appsec/processor/rule_merger.rb +22 -2
- data/lib/datadog/appsec/processor.rb +34 -6
- data/lib/datadog/appsec/remote.rb +4 -1
- data/lib/datadog/appsec/response.rb +82 -4
- data/lib/datadog/appsec/sample_rate.rb +21 -0
- data/lib/datadog/appsec.rb +2 -2
- data/lib/datadog/core/configuration/agent_settings_resolver.rb +29 -24
- data/lib/datadog/core/configuration/base.rb +1 -11
- data/lib/datadog/core/configuration/components.rb +9 -2
- data/lib/datadog/core/configuration/ext.rb +21 -0
- data/lib/datadog/core/configuration/option.rb +2 -4
- data/lib/datadog/core/configuration/option_definition.rb +17 -41
- data/lib/datadog/core/configuration/options.rb +5 -5
- data/lib/datadog/core/configuration/settings.rb +47 -45
- data/lib/datadog/core/diagnostics/environment_logger.rb +130 -234
- data/lib/datadog/core/environment/execution.rb +50 -2
- data/lib/datadog/core/environment/variable_helpers.rb +0 -69
- data/lib/datadog/core/error.rb +1 -0
- data/lib/datadog/core/git/ext.rb +2 -0
- data/lib/datadog/core/remote/client/capabilities.rb +1 -1
- data/lib/datadog/core/remote/component.rb +2 -2
- data/lib/datadog/core/remote/negotiation.rb +2 -2
- data/lib/datadog/core/remote/transport/config.rb +60 -0
- data/lib/datadog/core/remote/transport/http/api/instance.rb +39 -0
- data/lib/datadog/core/remote/transport/http/api/spec.rb +21 -0
- data/lib/datadog/core/remote/transport/http/api.rb +58 -0
- data/lib/datadog/core/remote/transport/http/builder.rb +219 -0
- data/lib/datadog/core/remote/transport/http/client.rb +48 -0
- data/lib/datadog/core/remote/transport/http/config.rb +280 -0
- data/lib/datadog/core/remote/transport/http/negotiation.rb +146 -0
- data/lib/datadog/core/remote/transport/http.rb +179 -0
- data/lib/datadog/core/{transport → remote/transport}/negotiation.rb +25 -23
- data/lib/datadog/core/telemetry/collector.rb +13 -4
- data/lib/datadog/core/telemetry/http/transport.rb +2 -1
- data/lib/datadog/core/transport/ext.rb +47 -0
- data/lib/datadog/core/transport/http/adapters/net.rb +168 -0
- data/lib/datadog/core/transport/http/adapters/registry.rb +29 -0
- data/lib/datadog/core/transport/http/adapters/test.rb +89 -0
- data/lib/datadog/core/transport/http/adapters/unix_socket.rb +83 -0
- data/lib/datadog/core/transport/http/api/endpoint.rb +31 -0
- data/lib/datadog/core/transport/http/api/fallbacks.rb +26 -0
- data/lib/datadog/core/transport/http/api/map.rb +18 -0
- data/lib/datadog/core/transport/http/env.rb +62 -0
- data/lib/datadog/core/transport/http/response.rb +60 -0
- data/lib/datadog/core/transport/parcel.rb +22 -0
- data/lib/datadog/core/transport/request.rb +17 -0
- data/lib/datadog/core/transport/response.rb +64 -0
- data/lib/datadog/core/workers/polling.rb +2 -2
- data/lib/datadog/opentelemetry/api/context.rb +10 -3
- data/lib/datadog/opentelemetry/sdk/propagator.rb +2 -1
- data/lib/datadog/opentelemetry/sdk/span_processor.rb +14 -2
- data/lib/datadog/opentelemetry/sdk/trace/span.rb +68 -0
- data/lib/datadog/opentelemetry/trace.rb +58 -0
- data/lib/datadog/opentelemetry.rb +1 -0
- data/lib/datadog/opentracer.rb +9 -0
- data/lib/datadog/profiling/collectors/cpu_and_wall_time_worker.rb +12 -18
- data/lib/datadog/profiling/collectors/idle_sampling_helper.rb +1 -1
- data/lib/datadog/profiling/collectors/thread_context.rb +9 -1
- data/lib/datadog/profiling/component.rb +34 -99
- data/lib/datadog/profiling/diagnostics/environment_logger.rb +39 -0
- data/lib/datadog/profiling/exporter.rb +4 -4
- data/lib/datadog/profiling/ext.rb +0 -12
- data/lib/datadog/profiling/flush.rb +2 -7
- data/lib/datadog/profiling/http_transport.rb +6 -3
- data/lib/datadog/profiling/native_extension.rb +0 -21
- data/lib/datadog/profiling/profiler.rb +11 -12
- data/lib/datadog/profiling.rb +9 -81
- data/lib/datadog/tracing/component.rb +12 -5
- data/lib/datadog/tracing/configuration/agent_settings_resolver.rb +13 -0
- data/lib/datadog/tracing/configuration/ext.rb +4 -2
- data/lib/datadog/tracing/configuration/settings.rb +14 -7
- data/lib/datadog/tracing/contrib/action_pack/configuration/settings.rb +1 -1
- data/lib/datadog/tracing/contrib/active_job/configuration/settings.rb +1 -1
- data/lib/datadog/tracing/contrib/active_record/configuration/resolver.rb +18 -11
- data/lib/datadog/tracing/contrib/active_record/events/sql.rb +4 -0
- data/lib/datadog/tracing/contrib/active_record/utils.rb +1 -1
- data/lib/datadog/tracing/contrib/active_support/cache/instrumentation.rb +104 -197
- data/lib/datadog/tracing/contrib/active_support/cache/patcher.rb +3 -0
- data/lib/datadog/tracing/contrib/aws/instrumentation.rb +7 -0
- data/lib/datadog/tracing/contrib/configuration/settings.rb +1 -1
- data/lib/datadog/tracing/contrib/dalli/configuration/settings.rb +6 -0
- data/lib/datadog/tracing/contrib/dalli/ext.rb +7 -0
- data/lib/datadog/tracing/contrib/dalli/instrumentation.rb +9 -2
- data/lib/datadog/tracing/contrib/delayed_job/configuration/settings.rb +1 -1
- data/lib/datadog/tracing/contrib/elasticsearch/patcher.rb +104 -102
- data/lib/datadog/tracing/contrib/ethon/easy_patch.rb +5 -0
- data/lib/datadog/tracing/contrib/ethon/multi_patch.rb +8 -0
- data/lib/datadog/tracing/contrib/excon/middleware.rb +5 -0
- data/lib/datadog/tracing/contrib/ext.rb +3 -0
- data/lib/datadog/tracing/contrib/faraday/configuration/settings.rb +1 -1
- data/lib/datadog/tracing/contrib/faraday/middleware.rb +5 -0
- data/lib/datadog/tracing/contrib/grpc/configuration/settings.rb +21 -1
- data/lib/datadog/tracing/contrib/grpc/datadog_interceptor/client.rb +11 -1
- data/lib/datadog/tracing/contrib/grpc/datadog_interceptor/server.rb +18 -0
- data/lib/datadog/tracing/contrib/grpc/datadog_interceptor.rb +0 -4
- data/lib/datadog/tracing/contrib/http/circuit_breaker.rb +3 -3
- data/lib/datadog/tracing/contrib/http/instrumentation.rb +5 -0
- data/lib/datadog/tracing/contrib/httpclient/instrumentation.rb +5 -0
- data/lib/datadog/tracing/contrib/httprb/instrumentation.rb +5 -0
- data/lib/datadog/tracing/contrib/lograge/instrumentation.rb +1 -17
- data/lib/datadog/tracing/contrib/mongodb/subscribers.rb +7 -0
- data/lib/datadog/tracing/contrib/mysql2/instrumentation.rb +13 -3
- data/lib/datadog/tracing/contrib/opensearch/integration.rb +2 -2
- data/lib/datadog/tracing/contrib/opensearch/patcher.rb +7 -0
- data/lib/datadog/tracing/contrib/pg/instrumentation.rb +5 -0
- data/lib/datadog/tracing/contrib/presto/instrumentation.rb +5 -0
- data/lib/datadog/tracing/contrib/propagation/sql_comment.rb +1 -1
- data/lib/datadog/tracing/contrib/que/configuration/settings.rb +1 -1
- data/lib/datadog/tracing/contrib/racecar/event.rb +5 -0
- data/lib/datadog/tracing/contrib/rack/header_tagging.rb +14 -4
- data/lib/datadog/tracing/contrib/rails/configuration/settings.rb +4 -4
- data/lib/datadog/tracing/contrib/rake/configuration/settings.rb +1 -1
- data/lib/datadog/tracing/contrib/redis/tags.rb +5 -0
- data/lib/datadog/tracing/contrib/resque/configuration/settings.rb +1 -1
- data/lib/datadog/tracing/contrib/rest_client/request_patch.rb +5 -0
- data/lib/datadog/tracing/contrib/semantic_logger/instrumentation.rb +3 -20
- data/lib/datadog/tracing/contrib/sequel/utils.rb +5 -0
- data/lib/datadog/tracing/contrib/shoryuken/configuration/settings.rb +1 -1
- data/lib/datadog/tracing/contrib/sidekiq/configuration/settings.rb +1 -1
- data/lib/datadog/tracing/contrib/sneakers/configuration/settings.rb +1 -1
- data/lib/datadog/tracing/contrib/utils/quantization/http.rb +11 -11
- data/lib/datadog/tracing/correlation.rb +20 -0
- data/lib/datadog/tracing/diagnostics/environment_logger.rb +159 -0
- data/lib/datadog/tracing/distributed/propagation.rb +13 -33
- data/lib/datadog/tracing/metadata/tagging.rb +3 -3
- data/lib/datadog/tracing/sync_writer.rb +3 -3
- data/lib/datadog/tracing/tracer.rb +2 -0
- data/lib/datadog/{core → tracing}/transport/http/api/instance.rb +1 -1
- data/lib/datadog/{core → tracing}/transport/http/api/spec.rb +1 -1
- data/lib/datadog/tracing/transport/http/api.rb +43 -0
- data/lib/datadog/{core → tracing}/transport/http/builder.rb +13 -68
- data/lib/datadog/tracing/transport/http/client.rb +57 -0
- data/lib/datadog/tracing/transport/http/statistics.rb +47 -0
- data/lib/datadog/tracing/transport/http/traces.rb +152 -0
- data/lib/datadog/tracing/transport/http.rb +124 -0
- data/lib/datadog/tracing/transport/io/client.rb +89 -0
- data/lib/datadog/tracing/transport/io/response.rb +27 -0
- data/lib/datadog/tracing/transport/io/traces.rb +101 -0
- data/lib/datadog/tracing/transport/io.rb +30 -0
- data/lib/datadog/tracing/transport/serializable_trace.rb +126 -0
- data/lib/datadog/tracing/transport/statistics.rb +77 -0
- data/lib/datadog/tracing/transport/trace_formatter.rb +209 -0
- data/lib/datadog/tracing/transport/traces.rb +224 -0
- data/lib/datadog/tracing/workers/trace_writer.rb +6 -4
- data/lib/datadog/tracing/workers.rb +3 -2
- data/lib/datadog/tracing/writer.rb +5 -2
- data/lib/ddtrace/transport/ext.rb +17 -15
- data/lib/ddtrace/version.rb +2 -2
- data/lib/ddtrace.rb +1 -1
- metadata +74 -96
- data/lib/datadog/ci/configuration/components.rb +0 -32
- data/lib/datadog/ci/configuration/settings.rb +0 -51
- data/lib/datadog/ci/contrib/cucumber/configuration/settings.rb +0 -35
- data/lib/datadog/ci/contrib/cucumber/ext.rb +0 -22
- data/lib/datadog/ci/contrib/cucumber/formatter.rb +0 -94
- data/lib/datadog/ci/contrib/cucumber/instrumentation.rb +0 -28
- data/lib/datadog/ci/contrib/cucumber/integration.rb +0 -47
- data/lib/datadog/ci/contrib/cucumber/patcher.rb +0 -27
- data/lib/datadog/ci/contrib/minitest/configuration/settings.rb +0 -35
- data/lib/datadog/ci/contrib/minitest/ext.rb +0 -21
- data/lib/datadog/ci/contrib/minitest/integration.rb +0 -49
- data/lib/datadog/ci/contrib/minitest/patcher.rb +0 -27
- data/lib/datadog/ci/contrib/minitest/test_helper.rb +0 -68
- data/lib/datadog/ci/contrib/rspec/configuration/settings.rb +0 -35
- data/lib/datadog/ci/contrib/rspec/example.rb +0 -68
- data/lib/datadog/ci/contrib/rspec/ext.rb +0 -21
- data/lib/datadog/ci/contrib/rspec/integration.rb +0 -48
- data/lib/datadog/ci/contrib/rspec/patcher.rb +0 -27
- data/lib/datadog/ci/ext/app_types.rb +0 -9
- data/lib/datadog/ci/ext/environment.rb +0 -575
- data/lib/datadog/ci/ext/settings.rb +0 -10
- data/lib/datadog/ci/ext/test.rb +0 -35
- data/lib/datadog/ci/extensions.rb +0 -19
- data/lib/datadog/ci/flush.rb +0 -38
- data/lib/datadog/ci/test.rb +0 -81
- data/lib/datadog/ci.rb +0 -21
- data/lib/datadog/core/configuration/dependency_resolver.rb +0 -28
- data/lib/datadog/core/configuration/option_definition_set.rb +0 -22
- data/lib/datadog/core/configuration/option_set.rb +0 -10
- data/lib/datadog/core/transport/config.rb +0 -58
- data/lib/datadog/core/transport/http/api.rb +0 -57
- data/lib/datadog/core/transport/http/client.rb +0 -45
- data/lib/datadog/core/transport/http/config.rb +0 -278
- data/lib/datadog/core/transport/http/negotiation.rb +0 -144
- data/lib/datadog/core/transport/http.rb +0 -169
- data/lib/datadog/core/utils/object_set.rb +0 -43
- data/lib/datadog/core/utils/string_table.rb +0 -47
- data/lib/datadog/profiling/backtrace_location.rb +0 -34
- data/lib/datadog/profiling/buffer.rb +0 -43
- data/lib/datadog/profiling/collectors/old_stack.rb +0 -301
- data/lib/datadog/profiling/encoding/profile.rb +0 -41
- data/lib/datadog/profiling/event.rb +0 -15
- data/lib/datadog/profiling/events/stack.rb +0 -82
- data/lib/datadog/profiling/old_recorder.rb +0 -107
- data/lib/datadog/profiling/pprof/builder.rb +0 -125
- data/lib/datadog/profiling/pprof/converter.rb +0 -102
- data/lib/datadog/profiling/pprof/message_set.rb +0 -16
- data/lib/datadog/profiling/pprof/payload.rb +0 -20
- data/lib/datadog/profiling/pprof/pprof.proto +0 -212
- data/lib/datadog/profiling/pprof/pprof_pb.rb +0 -81
- data/lib/datadog/profiling/pprof/stack_sample.rb +0 -139
- data/lib/datadog/profiling/pprof/string_table.rb +0 -12
- data/lib/datadog/profiling/pprof/template.rb +0 -118
- data/lib/datadog/profiling/trace_identifiers/ddtrace.rb +0 -43
- data/lib/datadog/profiling/trace_identifiers/helper.rb +0 -45
- data/lib/ddtrace/transport/http/adapters/net.rb +0 -168
- data/lib/ddtrace/transport/http/adapters/registry.rb +0 -27
- data/lib/ddtrace/transport/http/adapters/test.rb +0 -85
- data/lib/ddtrace/transport/http/adapters/unix_socket.rb +0 -77
- data/lib/ddtrace/transport/http/api/endpoint.rb +0 -29
- data/lib/ddtrace/transport/http/api/fallbacks.rb +0 -24
- data/lib/ddtrace/transport/http/api/instance.rb +0 -35
- data/lib/ddtrace/transport/http/api/map.rb +0 -16
- data/lib/ddtrace/transport/http/api/spec.rb +0 -17
- data/lib/ddtrace/transport/http/api.rb +0 -39
- data/lib/ddtrace/transport/http/builder.rb +0 -176
- data/lib/ddtrace/transport/http/client.rb +0 -52
- data/lib/ddtrace/transport/http/env.rb +0 -58
- data/lib/ddtrace/transport/http/response.rb +0 -58
- data/lib/ddtrace/transport/http/statistics.rb +0 -43
- data/lib/ddtrace/transport/http/traces.rb +0 -144
- data/lib/ddtrace/transport/http.rb +0 -117
- data/lib/ddtrace/transport/io/client.rb +0 -85
- data/lib/ddtrace/transport/io/response.rb +0 -25
- data/lib/ddtrace/transport/io/traces.rb +0 -99
- data/lib/ddtrace/transport/io.rb +0 -28
- data/lib/ddtrace/transport/parcel.rb +0 -20
- data/lib/ddtrace/transport/request.rb +0 -15
- data/lib/ddtrace/transport/response.rb +0 -60
- data/lib/ddtrace/transport/serializable_trace.rb +0 -122
- data/lib/ddtrace/transport/statistics.rb +0 -75
- data/lib/ddtrace/transport/trace_formatter.rb +0 -207
- data/lib/ddtrace/transport/traces.rb +0 -216
data/lib/datadog/appsec/event.rb
CHANGED
@@ -1,4 +1,6 @@
|
|
1
1
|
require 'json'
|
2
|
+
require 'zlib'
|
3
|
+
require 'base64'
|
2
4
|
|
3
5
|
require_relative 'rate_limiter'
|
4
6
|
|
@@ -34,78 +36,132 @@ module Datadog
|
|
34
36
|
Content-Language
|
35
37
|
].map!(&:downcase).freeze
|
36
38
|
|
39
|
+
MAX_ENCODED_SCHEMA_SIZE = 25000
|
40
|
+
# For more information about this number
|
41
|
+
# please check https://github.com/DataDog/dd-trace-rb/pull/3177#issuecomment-1747221082
|
42
|
+
MIN_SCHEMA_SIZE_FOR_COMPRESSION = 260
|
43
|
+
|
37
44
|
# Record events for a trace
|
38
45
|
#
|
39
46
|
# This is expected to be called only once per trace for the rate limiter
|
40
47
|
# to properly apply
|
41
|
-
|
42
|
-
|
43
|
-
|
48
|
+
class << self
|
49
|
+
def record(span, *events)
|
50
|
+
# ensure rate limiter is called only when there are events to record
|
51
|
+
return if events.empty? || span.nil?
|
44
52
|
|
45
|
-
|
46
|
-
|
53
|
+
Datadog::AppSec::RateLimiter.limit(:traces) do
|
54
|
+
record_via_span(span, *events)
|
55
|
+
end
|
47
56
|
end
|
48
|
-
end
|
49
57
|
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
58
|
+
def record_via_span(span, *events)
|
59
|
+
events.group_by { |e| e[:trace] }.each do |trace, event_group|
|
60
|
+
unless trace
|
61
|
+
Datadog.logger.debug { "{ error: 'no trace: cannot record', event_group: #{event_group.inspect}}" }
|
62
|
+
next
|
63
|
+
end
|
64
|
+
|
65
|
+
trace.keep!
|
66
|
+
trace.set_tag(
|
67
|
+
Datadog::Tracing::Metadata::Ext::Distributed::TAG_DECISION_MAKER,
|
68
|
+
Datadog::Tracing::Sampling::Ext::Decision::ASM
|
69
|
+
)
|
56
70
|
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
service_entry_tags = build_service_entry_tags(event_group)
|
65
|
-
# complex types are unsupported, we need to serialize to a string
|
66
|
-
triggers = service_entry_tags.delete('_dd.appsec.triggers')
|
67
|
-
span.set_tag('_dd.appsec.json', JSON.dump({ triggers: triggers }))
|
68
|
-
|
69
|
-
# apply tags to service entry span
|
70
|
-
service_entry_tags.each do |key, value|
|
71
|
-
span.set_tag(key, value)
|
71
|
+
# prepare and gather tags to apply
|
72
|
+
service_entry_tags = build_service_entry_tags(event_group)
|
73
|
+
|
74
|
+
# apply tags to service entry span
|
75
|
+
service_entry_tags.each do |key, value|
|
76
|
+
span.set_tag(key, value)
|
77
|
+
end
|
72
78
|
end
|
73
79
|
end
|
74
|
-
end
|
75
80
|
|
76
|
-
|
77
|
-
event_group
|
78
|
-
|
81
|
+
# rubocop:disable Metrics/MethodLength
|
82
|
+
def build_service_entry_tags(event_group)
|
83
|
+
waf_events = []
|
84
|
+
entry_tags = event_group.each_with_object({ '_dd.origin' => 'appsec' }) do |event, tags|
|
85
|
+
# TODO: assume HTTP request context for now
|
86
|
+
if (request = event[:request])
|
87
|
+
request.headers.each do |header, value|
|
88
|
+
tags["http.request.headers.#{header}"] = value if ALLOWED_REQUEST_HEADERS.include?(header.downcase)
|
89
|
+
end
|
90
|
+
|
91
|
+
tags['http.host'] = request.host
|
92
|
+
tags['http.useragent'] = request.user_agent
|
93
|
+
tags['network.client.ip'] = request.remote_addr
|
94
|
+
end
|
79
95
|
|
80
|
-
|
81
|
-
|
82
|
-
|
96
|
+
if (response = event[:response])
|
97
|
+
response.headers.each do |header, value|
|
98
|
+
tags["http.response.headers.#{header}"] = value if ALLOWED_RESPONSE_HEADERS.include?(header.downcase)
|
99
|
+
end
|
83
100
|
end
|
84
101
|
|
85
|
-
|
86
|
-
|
102
|
+
waf_result = event[:waf_result]
|
103
|
+
# accumulate triggers
|
104
|
+
waf_events += waf_result.events
|
105
|
+
|
106
|
+
waf_result.derivatives.each do |key, value|
|
107
|
+
parsed_value = json_parse(value)
|
108
|
+
next unless parsed_value
|
109
|
+
|
110
|
+
parsed_value_size = parsed_value.size
|
111
|
+
|
112
|
+
schema_value = if parsed_value_size >= MIN_SCHEMA_SIZE_FOR_COMPRESSION
|
113
|
+
compressed_and_base64_encoded(parsed_value)
|
114
|
+
else
|
115
|
+
parsed_value
|
116
|
+
end
|
117
|
+
next unless schema_value
|
118
|
+
|
119
|
+
if schema_value.size >= MAX_ENCODED_SCHEMA_SIZE
|
120
|
+
Datadog.logger.debug do
|
121
|
+
"Schema key: #{key} exceeds the max size value. It will not be included as part of the span tags"
|
122
|
+
end
|
123
|
+
next
|
124
|
+
end
|
125
|
+
|
126
|
+
tags[key] = schema_value
|
87
127
|
end
|
88
128
|
|
89
|
-
tags
|
90
|
-
tags['http.useragent'] = request.user_agent
|
91
|
-
tags['network.client.ip'] = request.remote_addr
|
129
|
+
tags
|
92
130
|
end
|
93
131
|
|
94
|
-
|
95
|
-
|
96
|
-
|
97
|
-
|
132
|
+
appsec_events = json_parse({ triggers: waf_events })
|
133
|
+
entry_tags['_dd.appsec.json'] = appsec_events if appsec_events
|
134
|
+
entry_tags
|
135
|
+
end
|
136
|
+
# rubocop:enable Metrics/MethodLength
|
98
137
|
|
99
|
-
|
100
|
-
|
101
|
-
|
138
|
+
private
|
139
|
+
|
140
|
+
def compressed_and_base64_encoded(value)
|
141
|
+
Base64.encode64(gzip(value))
|
142
|
+
rescue TypeError => e
|
143
|
+
Datadog.logger.debug do
|
144
|
+
"Failed to compress and encode value when populating AppSec::Event. Error: #{e.message}"
|
102
145
|
end
|
146
|
+
nil
|
147
|
+
end
|
103
148
|
|
104
|
-
|
149
|
+
def json_parse(value)
|
150
|
+
JSON.dump(value)
|
151
|
+
rescue ArgumentError => e
|
152
|
+
Datadog.logger.debug do
|
153
|
+
"Failed to parse value to JSON when populating AppSec::Event. Error: #{e.message}"
|
154
|
+
end
|
155
|
+
nil
|
156
|
+
end
|
105
157
|
|
106
|
-
|
107
|
-
|
108
|
-
|
158
|
+
def gzip(value)
|
159
|
+
sio = StringIO.new
|
160
|
+
# For an in depth comparison of Zlib options check https://github.com/DataDog/dd-trace-rb/pull/3177#issuecomment-1747215473
|
161
|
+
gz = Zlib::GzipWriter.new(sio, Zlib::BEST_SPEED, Zlib::DEFAULT_STRATEGY)
|
162
|
+
gz.write(value)
|
163
|
+
gz.close
|
164
|
+
sio.string
|
109
165
|
end
|
110
166
|
end
|
111
167
|
end
|
@@ -24,7 +24,7 @@ module Datadog
|
|
24
24
|
scope = Datadog::AppSec.active_scope
|
25
25
|
|
26
26
|
AppSec::Reactive::Operation.new('identity.set_user') do |op|
|
27
|
-
Monitor::Reactive::SetUser.subscribe(op, scope.processor_context) do |result
|
27
|
+
Monitor::Reactive::SetUser.subscribe(op, scope.processor_context) do |result|
|
28
28
|
if result.status == :match
|
29
29
|
# TODO: should this hash be an Event instance instead?
|
30
30
|
event = {
|
@@ -44,10 +44,10 @@ module Datadog
|
|
44
44
|
end
|
45
45
|
end
|
46
46
|
|
47
|
-
|
47
|
+
block = Monitor::Reactive::SetUser.publish(op, user)
|
48
48
|
end
|
49
49
|
|
50
|
-
throw(Datadog::AppSec::Ext::INTERRUPT, [nil, [:block, event]]) if block
|
50
|
+
throw(Datadog::AppSec::Ext::INTERRUPT, [nil, [[:block, event]]]) if block
|
51
51
|
|
52
52
|
ret, res = stack.call(user)
|
53
53
|
|
@@ -38,11 +38,8 @@ module Datadog
|
|
38
38
|
when :match
|
39
39
|
Datadog.logger.debug { "WAF: #{result.inspect}" }
|
40
40
|
|
41
|
-
|
42
|
-
|
43
|
-
yield [result, block]
|
44
|
-
|
45
|
-
throw(:block, [result, true]) if block
|
41
|
+
yield result
|
42
|
+
throw(:block, true) unless result.actions.empty?
|
46
43
|
when :ok
|
47
44
|
Datadog.logger.debug { "WAF OK: #{result.inspect}" }
|
48
45
|
when :invalid_call
|
@@ -0,0 +1,49 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Datadog
|
4
|
+
module AppSec
|
5
|
+
class Processor
|
6
|
+
# Actions store the actions information in memory
|
7
|
+
# Also, takes care of merging when RC send new information
|
8
|
+
module Actions
|
9
|
+
class << self
|
10
|
+
def actions
|
11
|
+
@actions ||= []
|
12
|
+
end
|
13
|
+
|
14
|
+
def fecth_configuration(action)
|
15
|
+
actions.find { |action_configuration| action_configuration['id'] == action }
|
16
|
+
end
|
17
|
+
|
18
|
+
def merge(actions_to_merge)
|
19
|
+
return if actions_to_merge.empty?
|
20
|
+
|
21
|
+
if actions.empty?
|
22
|
+
@actions = actions_to_merge
|
23
|
+
else
|
24
|
+
merged_actions = []
|
25
|
+
actions_dup = actions.dup
|
26
|
+
|
27
|
+
actions_to_merge.each do |new_action|
|
28
|
+
existing_action = actions_dup.find { |action| new_action['id'] == action['id'] }
|
29
|
+
|
30
|
+
# the old action is discard and the new kept
|
31
|
+
actions_dup.delete(existing_action) if existing_action
|
32
|
+
merged_actions << new_action
|
33
|
+
end
|
34
|
+
|
35
|
+
@actions = merged_actions.concat(actions_dup)
|
36
|
+
end
|
37
|
+
end
|
38
|
+
|
39
|
+
private
|
40
|
+
|
41
|
+
# Used in tests
|
42
|
+
def reset
|
43
|
+
@actions = []
|
44
|
+
end
|
45
|
+
end
|
46
|
+
end
|
47
|
+
end
|
48
|
+
end
|
49
|
+
end
|
@@ -1,5 +1,7 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
+
require_relative '../assets'
|
4
|
+
|
3
5
|
module Datadog
|
4
6
|
module AppSec
|
5
7
|
class Processor
|
@@ -16,8 +18,25 @@ module Datadog
|
|
16
18
|
end
|
17
19
|
end
|
18
20
|
|
21
|
+
DEFAULT_WAF_PROCESSORS = begin
|
22
|
+
JSON.parse(Datadog::AppSec::Assets.waf_processors)
|
23
|
+
rescue StandardError => e
|
24
|
+
Datadog.logger.error { "libddwaf rulemerger failed to parse default waf processors. Error: #{e.inspect}" }
|
25
|
+
[]
|
26
|
+
end
|
27
|
+
|
28
|
+
DEFAULT_WAF_SCANNERS = begin
|
29
|
+
JSON.parse(Datadog::AppSec::Assets.waf_scanners)
|
30
|
+
rescue StandardError => e
|
31
|
+
Datadog.logger.error { "libddwaf rulemerger failed to parse default waf scanners. Error: #{e.inspect}" }
|
32
|
+
[]
|
33
|
+
end
|
34
|
+
|
19
35
|
class << self
|
20
|
-
def merge(
|
36
|
+
def merge(
|
37
|
+
rules:, data: [], overrides: [], exclusions: [], custom_rules: [],
|
38
|
+
processors: DEFAULT_WAF_PROCESSORS, scanners: DEFAULT_WAF_SCANNERS
|
39
|
+
)
|
21
40
|
combined_rules = combine_rules(rules)
|
22
41
|
|
23
42
|
combined_data = combine_data(data) if data.any?
|
@@ -29,7 +48,8 @@ module Datadog
|
|
29
48
|
combined_rules['rules_override'] = combined_overrides if combined_overrides
|
30
49
|
combined_rules['exclusions'] = combined_exclusions if combined_exclusions
|
31
50
|
combined_rules['custom_rules'] = combined_custom_rules if combined_custom_rules
|
32
|
-
|
51
|
+
combined_rules['processors'] = processors
|
52
|
+
combined_rules['scanners'] = scanners
|
33
53
|
combined_rules
|
34
54
|
end
|
35
55
|
|
@@ -22,8 +22,15 @@ module Datadog
|
|
22
22
|
|
23
23
|
start_ns = Core::Utils::Time.get_time(:nanosecond)
|
24
24
|
|
25
|
-
|
26
|
-
|
25
|
+
input.reject! do |_, v|
|
26
|
+
case v
|
27
|
+
when TrueClass, FalseClass
|
28
|
+
false
|
29
|
+
else
|
30
|
+
v.nil? ? true : v.empty?
|
31
|
+
end
|
32
|
+
end
|
33
|
+
|
27
34
|
_code, res = @context.run(input, timeout)
|
28
35
|
|
29
36
|
stop_ns = Core::Utils::Time.get_time(:nanosecond)
|
@@ -38,15 +45,36 @@ module Datadog
|
|
38
45
|
@run_mutex.unlock
|
39
46
|
end
|
40
47
|
|
48
|
+
def extract_schema
|
49
|
+
return unless extract_schema?
|
50
|
+
|
51
|
+
input = {
|
52
|
+
'waf.context.processor' => {
|
53
|
+
'extract-schema' => true
|
54
|
+
}
|
55
|
+
}
|
56
|
+
|
57
|
+
_code, res = @context.run(input, WAF::LibDDWAF::DDWAF_RUN_TIMEOUT)
|
58
|
+
|
59
|
+
res
|
60
|
+
end
|
61
|
+
|
41
62
|
def finalize
|
42
63
|
@context.finalize
|
43
64
|
end
|
65
|
+
|
66
|
+
private
|
67
|
+
|
68
|
+
def extract_schema?
|
69
|
+
Datadog.configuration.appsec.api_security.enabled &&
|
70
|
+
Datadog.configuration.appsec.api_security.sample_rate.sample?
|
71
|
+
end
|
44
72
|
end
|
45
73
|
|
46
|
-
attr_reader :
|
74
|
+
attr_reader :diagnostics, :addresses
|
47
75
|
|
48
76
|
def initialize(ruleset:)
|
49
|
-
@
|
77
|
+
@diagnostics = nil
|
50
78
|
@addresses = []
|
51
79
|
settings = Datadog.configuration.appsec
|
52
80
|
|
@@ -83,7 +111,7 @@ module Datadog
|
|
83
111
|
}
|
84
112
|
|
85
113
|
@handle = Datadog::AppSec::WAF::Handle.new(ruleset, obfuscator: obfuscator_config)
|
86
|
-
@
|
114
|
+
@diagnostics = @handle.diagnostics
|
87
115
|
@addresses = @handle.required_addresses
|
88
116
|
|
89
117
|
true
|
@@ -92,7 +120,7 @@ module Datadog
|
|
92
120
|
"libddwaf failed to initialize, error: #{e.inspect}"
|
93
121
|
end
|
94
122
|
|
95
|
-
@
|
123
|
+
@diagnostics = e.diagnostics if e.diagnostics
|
96
124
|
|
97
125
|
false
|
98
126
|
rescue StandardError => e
|
@@ -31,6 +31,7 @@ module Datadog
|
|
31
31
|
CAP_ASM_RESPONSE_BLOCKING,
|
32
32
|
CAP_ASM_DD_RULES,
|
33
33
|
CAP_ASM_CUSTOM_RULES,
|
34
|
+
CAP_ASM_CUSTOM_BLOCKING_RESPONSE,
|
34
35
|
].freeze
|
35
36
|
|
36
37
|
ASM_PRODUCTS = [
|
@@ -63,6 +64,7 @@ module Datadog
|
|
63
64
|
data = []
|
64
65
|
overrides = []
|
65
66
|
exclusions = []
|
67
|
+
actions = []
|
66
68
|
|
67
69
|
repository.contents.each do |content|
|
68
70
|
parsed_content = parse_content(content)
|
@@ -76,6 +78,7 @@ module Datadog
|
|
76
78
|
overrides << parsed_content['rules_override'] if parsed_content['rules_override']
|
77
79
|
exclusions << parsed_content['exclusions'] if parsed_content['exclusions']
|
78
80
|
custom_rules << parsed_content['custom_rules'] if parsed_content['custom_rules']
|
81
|
+
actions.concat(parsed_content['actions']) if parsed_content['actions']
|
79
82
|
end
|
80
83
|
end
|
81
84
|
|
@@ -95,7 +98,7 @@ module Datadog
|
|
95
98
|
custom_rules: custom_rules,
|
96
99
|
)
|
97
100
|
|
98
|
-
Datadog::AppSec.reconfigure(ruleset: ruleset)
|
101
|
+
Datadog::AppSec.reconfigure(ruleset: ruleset, actions: actions)
|
99
102
|
end
|
100
103
|
|
101
104
|
[receiver]
|
@@ -28,19 +28,80 @@ module Datadog
|
|
28
28
|
end
|
29
29
|
|
30
30
|
class << self
|
31
|
-
def negotiate(env)
|
31
|
+
def negotiate(env, actions)
|
32
|
+
# @type var configured_response: Response?
|
33
|
+
configured_response = nil
|
34
|
+
actions.each do |action|
|
35
|
+
# Need to use next to make steep happy :(
|
36
|
+
# I rather use break to stop the execution
|
37
|
+
next if configured_response
|
38
|
+
|
39
|
+
action_configuration = AppSec::Processor::Actions.fecth_configuration(action)
|
40
|
+
next unless action_configuration
|
41
|
+
|
42
|
+
configured_response = case action_configuration['type']
|
43
|
+
when 'block_request'
|
44
|
+
block_response(env, action_configuration['parameters'])
|
45
|
+
when 'redirect_request'
|
46
|
+
redirect_response(env, action_configuration['parameters'])
|
47
|
+
end
|
48
|
+
end
|
49
|
+
|
50
|
+
configured_response || default_response(env)
|
51
|
+
end
|
52
|
+
|
53
|
+
private
|
54
|
+
|
55
|
+
def default_response(env)
|
32
56
|
content_type = content_type(env)
|
33
57
|
|
34
|
-
|
58
|
+
body = []
|
59
|
+
body << content(content_type)
|
35
60
|
|
36
61
|
Response.new(
|
37
62
|
status: 403,
|
38
63
|
headers: { 'Content-Type' => content_type },
|
39
|
-
body:
|
64
|
+
body: body,
|
40
65
|
)
|
41
66
|
end
|
42
67
|
|
43
|
-
|
68
|
+
def block_response(env, options)
|
69
|
+
content_type = if options['type'] == 'auto'
|
70
|
+
content_type(env)
|
71
|
+
else
|
72
|
+
FORMAT_TO_CONTENT_TYPE[options['type']]
|
73
|
+
end
|
74
|
+
|
75
|
+
body = []
|
76
|
+
body << content(content_type)
|
77
|
+
|
78
|
+
Response.new(
|
79
|
+
status: options['status_code'] || 403,
|
80
|
+
headers: { 'Content-Type' => content_type },
|
81
|
+
body: body,
|
82
|
+
)
|
83
|
+
end
|
84
|
+
|
85
|
+
def redirect_response(env, options)
|
86
|
+
if options['location'] && !options['location'].empty?
|
87
|
+
content_type = content_type(env)
|
88
|
+
|
89
|
+
status = options['status_code'] >= 300 && options['status_code'] < 400 ? options['status_code'] : 303
|
90
|
+
|
91
|
+
headers = {
|
92
|
+
'Content-Type' => content_type,
|
93
|
+
'Location' => options['location']
|
94
|
+
}
|
95
|
+
|
96
|
+
Response.new(
|
97
|
+
status: status,
|
98
|
+
headers: headers,
|
99
|
+
body: [],
|
100
|
+
)
|
101
|
+
else
|
102
|
+
default_response(env)
|
103
|
+
end
|
104
|
+
end
|
44
105
|
|
45
106
|
CONTENT_TYPE_TO_FORMAT = {
|
46
107
|
'application/json' => :json,
|
@@ -48,6 +109,11 @@ module Datadog
|
|
48
109
|
'text/plain' => :text,
|
49
110
|
}.freeze
|
50
111
|
|
112
|
+
FORMAT_TO_CONTENT_TYPE = {
|
113
|
+
'json' => 'application/json',
|
114
|
+
'html' => 'text/html',
|
115
|
+
}.freeze
|
116
|
+
|
51
117
|
DEFAULT_CONTENT_TYPE = 'application/json'
|
52
118
|
|
53
119
|
def content_type(env)
|
@@ -67,6 +133,18 @@ module Datadog
|
|
67
133
|
rescue Datadog::AppSec::Utils::HTTP::MediaRange::ParseError
|
68
134
|
DEFAULT_CONTENT_TYPE
|
69
135
|
end
|
136
|
+
|
137
|
+
def content(content_type)
|
138
|
+
content_format = CONTENT_TYPE_TO_FORMAT[content_type]
|
139
|
+
|
140
|
+
using_default = Datadog.configuration.appsec.block.templates.using_default?(content_format)
|
141
|
+
|
142
|
+
if using_default
|
143
|
+
Datadog::AppSec::Assets.blocked(format: content_format)
|
144
|
+
else
|
145
|
+
Datadog.configuration.appsec.block.templates.send(content_format)
|
146
|
+
end
|
147
|
+
end
|
70
148
|
end
|
71
149
|
end
|
72
150
|
end
|
@@ -0,0 +1,21 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Datadog
|
4
|
+
module AppSec
|
5
|
+
# SampleRate basic sample rate
|
6
|
+
class SampleRate
|
7
|
+
attr_reader :rate
|
8
|
+
|
9
|
+
def initialize(rate)
|
10
|
+
@rate = rate
|
11
|
+
end
|
12
|
+
|
13
|
+
def sample?
|
14
|
+
return false if rate <= 0
|
15
|
+
return true if rate >= 1
|
16
|
+
|
17
|
+
Kernel.rand < rate
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
data/lib/datadog/appsec.rb
CHANGED
@@ -23,12 +23,12 @@ module Datadog
|
|
23
23
|
appsec_component.processor if appsec_component
|
24
24
|
end
|
25
25
|
|
26
|
-
def reconfigure(ruleset:)
|
26
|
+
def reconfigure(ruleset:, actions:)
|
27
27
|
appsec_component = components.appsec
|
28
28
|
|
29
29
|
return unless appsec_component
|
30
30
|
|
31
|
-
appsec_component.reconfigure(ruleset: ruleset)
|
31
|
+
appsec_component.reconfigure(ruleset: ruleset, actions: actions)
|
32
32
|
end
|
33
33
|
|
34
34
|
def reconfigure_lock(&block)
|