datadog 2.16.0 → 2.18.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +72 -1
- data/ext/datadog_profiling_native_extension/collectors_cpu_and_wall_time_worker.c +12 -46
- data/ext/datadog_profiling_native_extension/collectors_stack.c +227 -49
- data/ext/datadog_profiling_native_extension/collectors_stack.h +19 -3
- data/ext/datadog_profiling_native_extension/collectors_thread_context.c +63 -12
- data/ext/datadog_profiling_native_extension/collectors_thread_context.h +1 -0
- data/ext/datadog_profiling_native_extension/encoded_profile.c +22 -12
- data/ext/datadog_profiling_native_extension/encoded_profile.h +1 -0
- data/ext/datadog_profiling_native_extension/extconf.rb +7 -0
- data/ext/datadog_profiling_native_extension/heap_recorder.c +239 -363
- data/ext/datadog_profiling_native_extension/heap_recorder.h +4 -6
- data/ext/datadog_profiling_native_extension/http_transport.c +45 -72
- data/ext/datadog_profiling_native_extension/libdatadog_helpers.c +22 -0
- data/ext/datadog_profiling_native_extension/libdatadog_helpers.h +8 -5
- data/ext/datadog_profiling_native_extension/private_vm_api_access.c +1 -0
- data/ext/datadog_profiling_native_extension/private_vm_api_access.h +6 -3
- data/ext/datadog_profiling_native_extension/ruby_helpers.c +1 -13
- data/ext/datadog_profiling_native_extension/ruby_helpers.h +2 -10
- data/ext/datadog_profiling_native_extension/stack_recorder.c +156 -60
- data/ext/libdatadog_api/crashtracker.c +10 -3
- data/ext/libdatadog_api/extconf.rb +2 -2
- data/ext/libdatadog_api/library_config.c +54 -12
- data/ext/libdatadog_api/library_config.h +6 -0
- data/ext/libdatadog_api/macos_development.md +3 -3
- data/ext/libdatadog_api/process_discovery.c +2 -7
- data/ext/libdatadog_extconf_helpers.rb +2 -2
- data/lib/datadog/appsec/api_security/lru_cache.rb +56 -0
- data/lib/datadog/appsec/api_security/route_extractor.rb +65 -0
- data/lib/datadog/appsec/api_security/sampler.rb +59 -0
- data/lib/datadog/appsec/api_security.rb +23 -0
- data/lib/datadog/appsec/assets/waf_rules/recommended.json +257 -85
- data/lib/datadog/appsec/assets/waf_rules/strict.json +10 -78
- data/lib/datadog/appsec/component.rb +30 -54
- data/lib/datadog/appsec/configuration/settings.rb +60 -2
- data/lib/datadog/appsec/context.rb +6 -6
- data/lib/datadog/appsec/contrib/devise/tracking_middleware.rb +1 -1
- data/lib/datadog/appsec/contrib/rack/request_middleware.rb +27 -16
- data/lib/datadog/appsec/processor/rule_loader.rb +5 -6
- data/lib/datadog/appsec/remote.rb +15 -55
- data/lib/datadog/appsec/security_engine/engine.rb +194 -0
- data/lib/datadog/appsec/security_engine/runner.rb +10 -11
- data/lib/datadog/appsec.rb +4 -7
- data/lib/datadog/core/buffer/random.rb +18 -2
- data/lib/datadog/core/configuration/agent_settings.rb +52 -0
- data/lib/datadog/core/configuration/agent_settings_resolver.rb +4 -46
- data/lib/datadog/core/configuration/components.rb +31 -24
- data/lib/datadog/core/configuration/components_state.rb +23 -0
- data/lib/datadog/core/configuration/option.rb +27 -27
- data/lib/datadog/core/configuration/option_definition.rb +4 -4
- data/lib/datadog/core/configuration/options.rb +1 -1
- data/lib/datadog/core/configuration/settings.rb +32 -20
- data/lib/datadog/core/configuration/stable_config.rb +1 -2
- data/lib/datadog/core/configuration.rb +16 -16
- data/lib/datadog/core/crashtracking/component.rb +2 -1
- data/lib/datadog/core/crashtracking/tag_builder.rb +4 -22
- data/lib/datadog/core/encoding.rb +1 -1
- data/lib/datadog/core/environment/cgroup.rb +10 -12
- data/lib/datadog/core/environment/container.rb +38 -40
- data/lib/datadog/core/environment/ext.rb +6 -6
- data/lib/datadog/core/environment/identity.rb +3 -3
- data/lib/datadog/core/environment/platform.rb +3 -3
- data/lib/datadog/core/error.rb +11 -9
- data/lib/datadog/core/logger.rb +2 -2
- data/lib/datadog/core/metrics/client.rb +12 -14
- data/lib/datadog/core/metrics/logging.rb +5 -5
- data/lib/datadog/core/process_discovery/tracer_memfd.rb +15 -0
- data/lib/datadog/core/process_discovery.rb +5 -1
- data/lib/datadog/core/rate_limiter.rb +4 -2
- data/lib/datadog/core/remote/client.rb +32 -31
- data/lib/datadog/core/remote/component.rb +3 -3
- data/lib/datadog/core/remote/configuration/digest.rb +7 -7
- data/lib/datadog/core/remote/configuration/path.rb +1 -1
- data/lib/datadog/core/remote/configuration/repository.rb +12 -0
- data/lib/datadog/core/remote/transport/http/client.rb +1 -1
- data/lib/datadog/core/remote/transport/http/config.rb +21 -5
- data/lib/datadog/core/remote/transport/http/negotiation.rb +1 -1
- data/lib/datadog/core/runtime/metrics.rb +3 -3
- data/lib/datadog/core/tag_builder.rb +56 -0
- data/lib/datadog/core/telemetry/component.rb +39 -24
- data/lib/datadog/core/telemetry/emitter.rb +7 -1
- data/lib/datadog/core/telemetry/event/app_client_configuration_change.rb +66 -0
- data/lib/datadog/core/telemetry/event/app_closing.rb +18 -0
- data/lib/datadog/core/telemetry/event/app_dependencies_loaded.rb +33 -0
- data/lib/datadog/core/telemetry/event/app_heartbeat.rb +18 -0
- data/lib/datadog/core/telemetry/event/app_integrations_change.rb +58 -0
- data/lib/datadog/core/telemetry/event/app_started.rb +269 -0
- data/lib/datadog/core/telemetry/event/base.rb +40 -0
- data/lib/datadog/core/telemetry/event/distributions.rb +18 -0
- data/lib/datadog/core/telemetry/event/generate_metrics.rb +43 -0
- data/lib/datadog/core/telemetry/event/log.rb +76 -0
- data/lib/datadog/core/telemetry/event/message_batch.rb +42 -0
- data/lib/datadog/core/telemetry/event/synth_app_client_configuration_change.rb +43 -0
- data/lib/datadog/core/telemetry/event.rb +17 -475
- data/lib/datadog/core/telemetry/logger.rb +5 -4
- data/lib/datadog/core/telemetry/logging.rb +11 -5
- data/lib/datadog/core/telemetry/metric.rb +3 -3
- data/lib/datadog/core/telemetry/transport/http/telemetry.rb +2 -2
- data/lib/datadog/core/telemetry/transport/telemetry.rb +0 -1
- data/lib/datadog/core/telemetry/worker.rb +48 -27
- data/lib/datadog/core/transport/http/adapters/net.rb +17 -2
- data/lib/datadog/core/transport/http/adapters/test.rb +2 -1
- data/lib/datadog/core/transport/http/builder.rb +14 -14
- data/lib/datadog/core/transport/http/env.rb +8 -0
- data/lib/datadog/core/utils/at_fork_monkey_patch.rb +6 -6
- data/lib/datadog/core/utils/duration.rb +32 -32
- data/lib/datadog/core/utils/forking.rb +2 -2
- data/lib/datadog/core/utils/network.rb +6 -6
- data/lib/datadog/core/utils/only_once_successful.rb +16 -5
- data/lib/datadog/core/utils/time.rb +10 -2
- data/lib/datadog/core/utils/truncation.rb +21 -0
- data/lib/datadog/core/utils.rb +7 -0
- data/lib/datadog/core/vendor/multipart-post/multipart/post/composite_read_io.rb +1 -1
- data/lib/datadog/core/vendor/multipart-post/multipart/post/multipartable.rb +8 -8
- data/lib/datadog/core/vendor/multipart-post/multipart/post/parts.rb +7 -7
- data/lib/datadog/core/worker.rb +1 -1
- data/lib/datadog/core/workers/async.rb +9 -10
- data/lib/datadog/di/instrumenter.rb +52 -2
- data/lib/datadog/di/probe_notification_builder.rb +31 -41
- data/lib/datadog/di/probe_notifier_worker.rb +9 -1
- data/lib/datadog/di/serializer.rb +6 -2
- data/lib/datadog/di/transport/http/input.rb +10 -0
- data/lib/datadog/di/transport/input.rb +10 -2
- data/lib/datadog/error_tracking/component.rb +2 -2
- data/lib/datadog/profiling/collectors/code_provenance.rb +18 -9
- data/lib/datadog/profiling/collectors/cpu_and_wall_time_worker.rb +4 -0
- data/lib/datadog/profiling/collectors/idle_sampling_helper.rb +1 -0
- data/lib/datadog/profiling/collectors/thread_context.rb +16 -1
- data/lib/datadog/profiling/component.rb +7 -9
- data/lib/datadog/profiling/ext.rb +0 -13
- data/lib/datadog/profiling/flush.rb +1 -1
- data/lib/datadog/profiling/http_transport.rb +3 -8
- data/lib/datadog/profiling/profiler.rb +2 -0
- data/lib/datadog/profiling/scheduler.rb +10 -2
- data/lib/datadog/profiling/stack_recorder.rb +5 -5
- data/lib/datadog/profiling/tag_builder.rb +5 -41
- data/lib/datadog/profiling/tasks/setup.rb +2 -0
- data/lib/datadog/tracing/contrib/action_pack/action_controller/instrumentation.rb +15 -0
- data/lib/datadog/tracing/contrib/action_pack/action_dispatch/instrumentation.rb +19 -12
- data/lib/datadog/tracing/contrib/action_pack/ext.rb +2 -0
- data/lib/datadog/tracing/contrib/active_support/cache/events/cache.rb +4 -1
- data/lib/datadog/tracing/contrib/active_support/cache/instrumentation.rb +33 -0
- data/lib/datadog/tracing/contrib/active_support/cache/patcher.rb +4 -0
- data/lib/datadog/tracing/contrib/active_support/cache/redis.rb +2 -4
- data/lib/datadog/tracing/contrib/aws/instrumentation.rb +10 -0
- data/lib/datadog/tracing/contrib/aws/parsed_context.rb +5 -1
- data/lib/datadog/tracing/contrib/http/instrumentation.rb +1 -5
- data/lib/datadog/tracing/contrib/httpclient/instrumentation.rb +1 -5
- data/lib/datadog/tracing/contrib/httprb/instrumentation.rb +1 -5
- data/lib/datadog/tracing/contrib/lograge/patcher.rb +4 -2
- data/lib/datadog/tracing/contrib/patcher.rb +5 -2
- data/lib/datadog/tracing/contrib/sidekiq/ext.rb +1 -0
- data/lib/datadog/tracing/contrib/sidekiq/server_tracer.rb +5 -2
- data/lib/datadog/tracing/contrib/support.rb +28 -0
- data/lib/datadog/tracing/metadata/errors.rb +4 -4
- data/lib/datadog/tracing/sync_writer.rb +1 -1
- data/lib/datadog/tracing/trace_operation.rb +12 -4
- data/lib/datadog/tracing/tracer.rb +6 -2
- data/lib/datadog/version.rb +1 -1
- metadata +31 -12
- data/lib/datadog/appsec/assets/waf_rules/processors.json +0 -321
- data/lib/datadog/appsec/assets/waf_rules/scanners.json +0 -1023
- data/lib/datadog/appsec/processor/rule_merger.rb +0 -171
- data/lib/datadog/appsec/processor.rb +0 -107
@@ -0,0 +1,194 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Datadog
|
4
|
+
module AppSec
|
5
|
+
module SecurityEngine
|
6
|
+
# SecurityEngine::Engine creates WAF builder and manages its configuration.
|
7
|
+
# It also rebuilds WAF handle from the WAF builder when configuration changes.
|
8
|
+
class Engine
|
9
|
+
DEFAULT_RULES_CONFIG_PATH = 'ASM_DD/default'
|
10
|
+
TELEMETRY_ACTIONS = %w[init update].freeze
|
11
|
+
DIAGNOSTICS_CONFIG_KEYS = %w[
|
12
|
+
rules
|
13
|
+
custom_rules
|
14
|
+
exclusions
|
15
|
+
actions
|
16
|
+
processors
|
17
|
+
scanners
|
18
|
+
rules_override
|
19
|
+
rules_data
|
20
|
+
exclusion_data
|
21
|
+
].freeze
|
22
|
+
|
23
|
+
attr_reader :waf_addresses, :ruleset_version
|
24
|
+
|
25
|
+
def initialize(appsec_settings:, telemetry:)
|
26
|
+
@default_ruleset = appsec_settings.ruleset
|
27
|
+
|
28
|
+
# NOTE: replace appsec_settings argument with default_ruleset when removing these deprecated settings
|
29
|
+
@default_ip_denylist = appsec_settings.ip_denylist
|
30
|
+
@default_user_id_denylist = appsec_settings.user_id_denylist
|
31
|
+
@default_ip_passlist = appsec_settings.ip_passlist
|
32
|
+
|
33
|
+
@waf_builder = WAF::HandleBuilder.new(
|
34
|
+
obfuscator: {
|
35
|
+
key_regex: appsec_settings.obfuscator_key_regex,
|
36
|
+
value_regex: appsec_settings.obfuscator_value_regex
|
37
|
+
}
|
38
|
+
)
|
39
|
+
|
40
|
+
diagnostics = load_default_config(telemetry: telemetry)
|
41
|
+
report_configuration_diagnostics(diagnostics, action: 'init', telemetry: telemetry)
|
42
|
+
|
43
|
+
@waf_handle = @waf_builder.build_handle
|
44
|
+
@waf_addresses = @waf_handle.known_addresses
|
45
|
+
rescue WAF::Error => e
|
46
|
+
error_message = "AppSec security engine failed to initialize"
|
47
|
+
|
48
|
+
Datadog.logger.error("#{error_message}, error #{e.inspect}")
|
49
|
+
telemetry.report(e, description: error_message)
|
50
|
+
|
51
|
+
raise e
|
52
|
+
end
|
53
|
+
|
54
|
+
def finalize!
|
55
|
+
@waf_handle&.finalize!
|
56
|
+
@waf_builder&.finalize!
|
57
|
+
|
58
|
+
@waf_addresses = []
|
59
|
+
@ruleset_version = nil
|
60
|
+
end
|
61
|
+
|
62
|
+
def new_runner
|
63
|
+
SecurityEngine::Runner.new(@waf_handle.build_context)
|
64
|
+
end
|
65
|
+
|
66
|
+
def add_or_update_config(config, path:)
|
67
|
+
@is_ruleset_update = path.include?('ASM_DD')
|
68
|
+
|
69
|
+
# default config has to be removed when adding an ASM_DD config
|
70
|
+
remove_config_at_path(DEFAULT_RULES_CONFIG_PATH) if @is_ruleset_update
|
71
|
+
|
72
|
+
diagnostics = @waf_builder.add_or_update_config(config, path: path)
|
73
|
+
@ruleset_version = diagnostics['ruleset_version'] if diagnostics.key?('ruleset_version')
|
74
|
+
report_configuration_diagnostics(diagnostics, action: 'update', telemetry: AppSec.telemetry)
|
75
|
+
|
76
|
+
# we need to load default config if diagnostics contains top-level error for rules or processors
|
77
|
+
if @is_ruleset_update &&
|
78
|
+
(diagnostics.key?('error') ||
|
79
|
+
diagnostics.dig('rules', 'error') ||
|
80
|
+
diagnostics.dig('processors', 'errors'))
|
81
|
+
diagnostics = load_default_config(telemetry: AppSec.telemetry)
|
82
|
+
report_configuration_diagnostics(diagnostics, action: 'update', telemetry: AppSec.telemetry)
|
83
|
+
end
|
84
|
+
|
85
|
+
diagnostics
|
86
|
+
rescue WAF::Error => e
|
87
|
+
error_message = "libddwaf builder failed to add or update config at path: #{path}"
|
88
|
+
|
89
|
+
Datadog.logger.debug("#{error_message}, error: #{e.inspect}")
|
90
|
+
AppSec.telemetry.report(e, description: error_message)
|
91
|
+
end
|
92
|
+
|
93
|
+
def remove_config_at_path(path)
|
94
|
+
result = @waf_builder.remove_config_at_path(path)
|
95
|
+
|
96
|
+
if result && path != DEFAULT_RULES_CONFIG_PATH && path.include?('ASM_DD')
|
97
|
+
diagnostics = load_default_config(telemetry: AppSec.telemetry)
|
98
|
+
report_configuration_diagnostics(diagnostics, action: 'update', telemetry: AppSec.telemetry)
|
99
|
+
end
|
100
|
+
|
101
|
+
result
|
102
|
+
rescue WAF::Error => e
|
103
|
+
error_message = "libddwaf handle builder failed to remove config at path: #{path}"
|
104
|
+
|
105
|
+
Datadog.logger.error("#{error_message}, error: #{e.inspect}")
|
106
|
+
AppSec.telemetry.report(e, description: error_message)
|
107
|
+
end
|
108
|
+
|
109
|
+
def reconfigure!
|
110
|
+
old_waf_handle = @waf_handle
|
111
|
+
|
112
|
+
@waf_handle = @waf_builder.build_handle
|
113
|
+
@waf_addresses = @waf_handle.known_addresses
|
114
|
+
|
115
|
+
old_waf_handle&.finalize!
|
116
|
+
rescue WAF::Error => e
|
117
|
+
error_message = "AppSec security engine failed to reconfigure"
|
118
|
+
|
119
|
+
Datadog.logger.error("#{error_message}, error #{e.inspect}")
|
120
|
+
AppSec.telemetry.report(e, description: error_message)
|
121
|
+
|
122
|
+
if old_waf_handle
|
123
|
+
Datadog.logger.warn("Reverting to the previous configuration")
|
124
|
+
|
125
|
+
@waf_handle = old_waf_handle
|
126
|
+
@waf_addresses = old_waf_handle.known_addresses
|
127
|
+
end
|
128
|
+
end
|
129
|
+
|
130
|
+
private
|
131
|
+
|
132
|
+
def load_default_config(telemetry:)
|
133
|
+
config = AppSec::Processor::RuleLoader.load_rules(telemetry: telemetry, ruleset: @default_ruleset)
|
134
|
+
|
135
|
+
# deprecated - ip and user id denylists should be configured via RC
|
136
|
+
config['rules_data'] ||= AppSec::Processor::RuleLoader.load_data(
|
137
|
+
ip_denylist: @default_ip_denylist,
|
138
|
+
user_id_denylist: @default_user_id_denylist
|
139
|
+
)
|
140
|
+
|
141
|
+
# deprecated - ip passlist should be configured via RC
|
142
|
+
config['exclusions'] ||= AppSec::Processor::RuleLoader.load_exclusions(ip_passlist: @default_ip_passlist)
|
143
|
+
|
144
|
+
diagnostics = @waf_builder.add_or_update_config(config, path: DEFAULT_RULES_CONFIG_PATH)
|
145
|
+
@ruleset_version = diagnostics['ruleset_version']
|
146
|
+
|
147
|
+
diagnostics
|
148
|
+
end
|
149
|
+
|
150
|
+
def report_configuration_diagnostics(diagnostics, action:, telemetry:)
|
151
|
+
raise ArgumentError, 'action must be one of TELEMETRY_ACTIONS' unless TELEMETRY_ACTIONS.include?(action)
|
152
|
+
|
153
|
+
common_tags = {
|
154
|
+
waf_version: Datadog::AppSec::WAF::VERSION::BASE_STRING,
|
155
|
+
event_rules_version: diagnostics.fetch('ruleset_version', @ruleset_version).to_s,
|
156
|
+
action: action
|
157
|
+
}
|
158
|
+
|
159
|
+
if diagnostics['error']
|
160
|
+
telemetry.inc(
|
161
|
+
Ext::TELEMETRY_METRICS_NAMESPACE, 'waf.config_errors', 1,
|
162
|
+
tags: common_tags.merge(scope: 'top-level')
|
163
|
+
)
|
164
|
+
|
165
|
+
telemetry.error(diagnostics['error'])
|
166
|
+
end
|
167
|
+
|
168
|
+
diagnostics.each do |config_key, config_diagnostics|
|
169
|
+
next unless DIAGNOSTICS_CONFIG_KEYS.include?(config_key)
|
170
|
+
next if !config_diagnostics.key?('error') && config_diagnostics.fetch('errors', []).empty?
|
171
|
+
|
172
|
+
if config_diagnostics['error']
|
173
|
+
telemetry.error(config_diagnostics['error'])
|
174
|
+
|
175
|
+
telemetry.inc(
|
176
|
+
Ext::TELEMETRY_METRICS_NAMESPACE, 'waf.config_errors', 1,
|
177
|
+
tags: common_tags.merge(config_key: config_key, scope: 'top-level')
|
178
|
+
)
|
179
|
+
elsif config_diagnostics['errors']
|
180
|
+
config_diagnostics['errors'].each do |error, config_ids|
|
181
|
+
telemetry.error("#{error}: [#{config_ids.join(",")}]")
|
182
|
+
end
|
183
|
+
|
184
|
+
telemetry.inc(
|
185
|
+
Ext::TELEMETRY_METRICS_NAMESPACE, 'waf.config_errors', config_diagnostics['errors'].count,
|
186
|
+
tags: common_tags.merge(config_key: config_key, scope: 'item')
|
187
|
+
)
|
188
|
+
end
|
189
|
+
end
|
190
|
+
end
|
191
|
+
end
|
192
|
+
end
|
193
|
+
end
|
194
|
+
end
|
@@ -9,10 +9,9 @@ module Datadog
|
|
9
9
|
class Runner
|
10
10
|
SUCCESSFUL_EXECUTION_CODES = [:ok, :match].freeze
|
11
11
|
|
12
|
-
def initialize(
|
12
|
+
def initialize(waf_context)
|
13
13
|
@mutex = Mutex.new
|
14
|
-
@
|
15
|
-
@telemetry = telemetry
|
14
|
+
@waf_context = waf_context
|
16
15
|
|
17
16
|
@debug_tag = "libddwaf:#{WAF::VERSION::STRING} method:ddwaf_run"
|
18
17
|
end
|
@@ -33,7 +32,7 @@ module Datadog
|
|
33
32
|
v.nil? || v.empty?
|
34
33
|
end
|
35
34
|
|
36
|
-
|
35
|
+
result = try_run(persistent_data, ephemeral_data, timeout)
|
37
36
|
stop_ns = Core::Utils::Time.get_time(:nanosecond)
|
38
37
|
|
39
38
|
report_execution(result)
|
@@ -55,19 +54,19 @@ module Datadog
|
|
55
54
|
@mutex.unlock
|
56
55
|
end
|
57
56
|
|
58
|
-
def finalize
|
59
|
-
@
|
57
|
+
def finalize!
|
58
|
+
@waf_context.finalize!
|
60
59
|
end
|
61
60
|
|
62
61
|
private
|
63
62
|
|
64
63
|
def try_run(persistent_data, ephemeral_data, timeout)
|
65
|
-
@
|
66
|
-
rescue WAF::
|
64
|
+
@waf_context.run(persistent_data, ephemeral_data, timeout)
|
65
|
+
rescue WAF::LibDDWAFError => e
|
67
66
|
Datadog.logger.debug { "#{@debug_tag} execution error: #{e} backtrace: #{e.backtrace&.first(3)}" }
|
68
|
-
|
67
|
+
AppSec.telemetry.report(e, description: 'libddwaf-rb internal low-level error')
|
69
68
|
|
70
|
-
|
69
|
+
WAF::Result.new(:err_internal, [], 0, false, [], [])
|
71
70
|
end
|
72
71
|
|
73
72
|
def report_execution(result)
|
@@ -79,7 +78,7 @@ module Datadog
|
|
79
78
|
message = "#{@debug_tag} execution error: #{result.status.inspect}"
|
80
79
|
|
81
80
|
Datadog.logger.debug { message }
|
82
|
-
|
81
|
+
AppSec.telemetry.error(message)
|
83
82
|
end
|
84
83
|
end
|
85
84
|
end
|
data/lib/datadog/appsec.rb
CHANGED
@@ -26,15 +26,12 @@ module Datadog
|
|
26
26
|
components.appsec&.telemetry
|
27
27
|
end
|
28
28
|
|
29
|
-
def
|
30
|
-
components.appsec&.
|
29
|
+
def security_engine
|
30
|
+
components.appsec&.security_engine
|
31
31
|
end
|
32
32
|
|
33
|
-
def reconfigure
|
34
|
-
|
35
|
-
return unless appsec_component
|
36
|
-
|
37
|
-
appsec_component.reconfigure(ruleset: ruleset, telemetry: telemetry)
|
33
|
+
def reconfigure!
|
34
|
+
components.appsec&.reconfigure!
|
38
35
|
end
|
39
36
|
|
40
37
|
def reconfigure_lock(&block)
|
@@ -40,7 +40,23 @@ module Datadog
|
|
40
40
|
add_all!(underflow) unless underflow.nil?
|
41
41
|
|
42
42
|
# Iteratively replace items, to ensure pseudo-random replacement.
|
43
|
-
overflow
|
43
|
+
overflow&.each { |item| replace!(item) }
|
44
|
+
end
|
45
|
+
|
46
|
+
def unshift(*items)
|
47
|
+
# TODO The existing concat implementation does not always append
|
48
|
+
# to the end of the buffer - if the buffer is full, a random
|
49
|
+
# item is deleted and the new item is added in the position of
|
50
|
+
# removed item.
|
51
|
+
# Therefore, if we want to preserve the item order, concat
|
52
|
+
# would also need to be changed to maintain order.
|
53
|
+
# With the existing implementation, the idea is to not move
|
54
|
+
# existing items around, which is what sets unshift apart from
|
55
|
+
# concat to begin with.
|
56
|
+
#
|
57
|
+
# Since this method currently delegates to +concat+, it does not
|
58
|
+
# have a matching definition in the thread-safe worker.
|
59
|
+
concat(items)
|
44
60
|
end
|
45
61
|
|
46
62
|
# Stored items are returned and the local buffer is reset.
|
@@ -78,7 +94,7 @@ module Datadog
|
|
78
94
|
underflow = nil
|
79
95
|
overflow = nil
|
80
96
|
|
81
|
-
overflow_size = @max_size > 0 ? (@items.length + items.length) - @max_size : 0
|
97
|
+
overflow_size = (@max_size > 0) ? (@items.length + items.length) - @max_size : 0
|
82
98
|
|
83
99
|
if overflow_size > 0
|
84
100
|
# Items will overflow
|
@@ -0,0 +1,52 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require_relative 'ext'
|
4
|
+
|
5
|
+
module Datadog
|
6
|
+
module Core
|
7
|
+
module Configuration
|
8
|
+
# Immutable container for the resulting settings
|
9
|
+
class AgentSettings
|
10
|
+
# IPv6 regular expression from
|
11
|
+
# https://stackoverflow.com/questions/53497/regular-expression-that-matches-valid-ipv6-addresses
|
12
|
+
# Does not match IPv4 addresses.
|
13
|
+
IPV6_REGEXP = /\A(([0-9a-fA-F]{1,4}:){7,7}[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,7}:|([0-9a-fA-F]{1,4}:){1,6}:[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,5}(:[0-9a-fA-F]{1,4}){1,2}|([0-9a-fA-F]{1,4}:){1,4}(:[0-9a-fA-F]{1,4}){1,3}|([0-9a-fA-F]{1,4}:){1,3}(:[0-9a-fA-F]{1,4}){1,4}|([0-9a-fA-F]{1,4}:){1,2}(:[0-9a-fA-F]{1,4}){1,5}|[0-9a-fA-F]{1,4}:((:[0-9a-fA-F]{1,4}){1,6})|:((:[0-9a-fA-F]{1,4}){1,7}|:)|fe80:(:[0-9a-fA-F]{0,4}){0,4}%[0-9a-zA-Z]{1,}|::(ffff(:0{1,4}){0,1}:){0,1}((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])|([0-9a-fA-F]{1,4}:){1,4}:((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\z)/.freeze # rubocop:disable Layout/LineLength
|
14
|
+
|
15
|
+
attr_reader :adapter, :ssl, :hostname, :port, :uds_path, :timeout_seconds
|
16
|
+
|
17
|
+
def initialize(adapter: nil, ssl: nil, hostname: nil, port: nil, uds_path: nil, timeout_seconds: nil)
|
18
|
+
@adapter = adapter
|
19
|
+
@ssl = ssl
|
20
|
+
@hostname = hostname
|
21
|
+
@port = port
|
22
|
+
@uds_path = uds_path
|
23
|
+
@timeout_seconds = timeout_seconds
|
24
|
+
freeze
|
25
|
+
end
|
26
|
+
|
27
|
+
def url
|
28
|
+
case adapter
|
29
|
+
when Datadog::Core::Configuration::Ext::Agent::HTTP::ADAPTER
|
30
|
+
hostname = self.hostname
|
31
|
+
hostname = "[#{hostname}]" if IPV6_REGEXP.match?(hostname)
|
32
|
+
"#{ssl ? "https" : "http"}://#{hostname}:#{port}/"
|
33
|
+
when Datadog::Core::Configuration::Ext::Agent::UnixSocket::ADAPTER
|
34
|
+
"unix://#{uds_path}"
|
35
|
+
else
|
36
|
+
raise ArgumentError, "Unexpected adapter: #{adapter}"
|
37
|
+
end
|
38
|
+
end
|
39
|
+
|
40
|
+
def ==(other)
|
41
|
+
self.class == other.class &&
|
42
|
+
adapter == other.adapter &&
|
43
|
+
ssl == other.ssl &&
|
44
|
+
hostname == other.hostname &&
|
45
|
+
port == other.port &&
|
46
|
+
uds_path == other.uds_path &&
|
47
|
+
timeout_seconds == other.timeout_seconds
|
48
|
+
end
|
49
|
+
end
|
50
|
+
end
|
51
|
+
end
|
52
|
+
end
|
@@ -4,6 +4,7 @@ require 'uri'
|
|
4
4
|
|
5
5
|
require_relative 'settings'
|
6
6
|
require_relative 'ext'
|
7
|
+
require_relative 'agent_settings'
|
7
8
|
require_relative '../transport/ext'
|
8
9
|
|
9
10
|
module Datadog
|
@@ -19,49 +20,6 @@ module Datadog
|
|
19
20
|
# Whenever there is a conflict (different configurations are provided in different orders), it MUST warn the users
|
20
21
|
# about it and pick a value based on the following priority: code > environment variable > defaults.
|
21
22
|
class AgentSettingsResolver
|
22
|
-
# Immutable container for the resulting settings
|
23
|
-
class AgentSettings
|
24
|
-
attr_reader :adapter, :ssl, :hostname, :port, :uds_path, :timeout_seconds
|
25
|
-
|
26
|
-
def initialize(adapter: nil, ssl: nil, hostname: nil, port: nil, uds_path: nil, timeout_seconds: nil)
|
27
|
-
@adapter = adapter
|
28
|
-
@ssl = ssl
|
29
|
-
@hostname = hostname
|
30
|
-
@port = port
|
31
|
-
@uds_path = uds_path
|
32
|
-
@timeout_seconds = timeout_seconds
|
33
|
-
freeze
|
34
|
-
end
|
35
|
-
|
36
|
-
def url
|
37
|
-
case adapter
|
38
|
-
when Datadog::Core::Configuration::Ext::Agent::HTTP::ADAPTER
|
39
|
-
hostname = self.hostname
|
40
|
-
hostname = "[#{hostname}]" if hostname =~ IPV6_REGEXP
|
41
|
-
"#{ssl ? 'https' : 'http'}://#{hostname}:#{port}/"
|
42
|
-
when Datadog::Core::Configuration::Ext::Agent::UnixSocket::ADAPTER
|
43
|
-
"unix://#{uds_path}"
|
44
|
-
else
|
45
|
-
raise ArgumentError, "Unexpected adapter: #{adapter}"
|
46
|
-
end
|
47
|
-
end
|
48
|
-
|
49
|
-
def ==(other)
|
50
|
-
self.class == other.class &&
|
51
|
-
adapter == other.adapter &&
|
52
|
-
ssl == other.ssl &&
|
53
|
-
hostname == other.hostname &&
|
54
|
-
port == other.port &&
|
55
|
-
uds_path == other.uds_path &&
|
56
|
-
timeout_seconds == other.timeout_seconds
|
57
|
-
end
|
58
|
-
end
|
59
|
-
|
60
|
-
# IPv6 regular expression from
|
61
|
-
# https://stackoverflow.com/questions/53497/regular-expression-that-matches-valid-ipv6-addresses
|
62
|
-
# Does not match IPv4 addresses.
|
63
|
-
IPV6_REGEXP = /\A(([0-9a-fA-F]{1,4}:){7,7}[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,7}:|([0-9a-fA-F]{1,4}:){1,6}:[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,5}(:[0-9a-fA-F]{1,4}){1,2}|([0-9a-fA-F]{1,4}:){1,4}(:[0-9a-fA-F]{1,4}){1,3}|([0-9a-fA-F]{1,4}:){1,3}(:[0-9a-fA-F]{1,4}){1,4}|([0-9a-fA-F]{1,4}:){1,2}(:[0-9a-fA-F]{1,4}){1,5}|[0-9a-fA-F]{1,4}:((:[0-9a-fA-F]{1,4}){1,6})|:((:[0-9a-fA-F]{1,4}){1,7}|:)|fe80:(:[0-9a-fA-F]{0,4}){0,4}%[0-9a-zA-Z]{1,}|::(ffff(:0{1,4}){0,1}:){0,1}((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])|([0-9a-fA-F]{1,4}:){1,4}:((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\z)/.freeze # rubocop:disable Layout/LineLength
|
64
|
-
|
65
23
|
def self.call(settings, logger: Datadog.logger)
|
66
24
|
new(settings, logger: logger).send(:call)
|
67
25
|
end
|
@@ -158,7 +116,7 @@ module Datadog
|
|
158
116
|
value: settings.agent.timeout_seconds,
|
159
117
|
),
|
160
118
|
try_parsing_as_integer(
|
161
|
-
friendly_name: "#{Datadog::Core::Configuration::Ext::Agent::ENV_DEFAULT_TIMEOUT_SECONDS} "\
|
119
|
+
friendly_name: "#{Datadog::Core::Configuration::Ext::Agent::ENV_DEFAULT_TIMEOUT_SECONDS} " \
|
162
120
|
'environment variable',
|
163
121
|
value: ENV[Datadog::Core::Configuration::Ext::Agent::ENV_DEFAULT_TIMEOUT_SECONDS],
|
164
122
|
)
|
@@ -338,13 +296,13 @@ module Datadog
|
|
338
296
|
log_warning(
|
339
297
|
'Configuration mismatch: values differ between ' \
|
340
298
|
"#{detected_configurations_in_priority_order
|
341
|
-
.map { |config| "#{config.friendly_name} (#{config.value.inspect})" }.join(
|
299
|
+
.map { |config| "#{config.friendly_name} (#{config.value.inspect})" }.join(" and ")}" \
|
342
300
|
". Using #{detected_configurations_in_priority_order.first.value.inspect} and ignoring other configuration."
|
343
301
|
)
|
344
302
|
end
|
345
303
|
|
346
304
|
def log_warning(message)
|
347
|
-
logger
|
305
|
+
logger&.warn(message)
|
348
306
|
end
|
349
307
|
|
350
308
|
def http_scheme?(uri)
|
@@ -1,6 +1,7 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
3
|
require_relative 'agent_settings_resolver'
|
4
|
+
require_relative 'components_state'
|
4
5
|
require_relative 'ext'
|
5
6
|
require_relative '../diagnostics/environment_logger'
|
6
7
|
require_relative '../diagnostics/health'
|
@@ -8,7 +9,6 @@ require_relative '../logger'
|
|
8
9
|
require_relative '../runtime/metrics'
|
9
10
|
require_relative '../telemetry/component'
|
10
11
|
require_relative '../workers/runtime_metrics'
|
11
|
-
|
12
12
|
require_relative '../remote/component'
|
13
13
|
require_relative '../../tracing/component'
|
14
14
|
require_relative '../../profiling/component'
|
@@ -16,7 +16,6 @@ require_relative '../../appsec/component'
|
|
16
16
|
require_relative '../../di/component'
|
17
17
|
require_relative '../../error_tracking/component'
|
18
18
|
require_relative '../crashtracking/component'
|
19
|
-
|
20
19
|
require_relative '../environment/agent_info'
|
21
20
|
require_relative '../process_discovery'
|
22
21
|
|
@@ -30,7 +29,7 @@ module Datadog
|
|
30
29
|
|
31
30
|
def build_health_metrics(settings, logger, telemetry)
|
32
31
|
settings = settings.health_metrics
|
33
|
-
options = {
|
32
|
+
options = {enabled: settings.enabled}
|
34
33
|
options[:statsd] = settings.statsd unless settings.statsd.nil?
|
35
34
|
|
36
35
|
Core::Diagnostics::Health::Metrics.new(telemetry: telemetry, logger: logger, **options)
|
@@ -44,7 +43,7 @@ module Datadog
|
|
44
43
|
end
|
45
44
|
|
46
45
|
def build_runtime_metrics(settings, logger, telemetry)
|
47
|
-
options = {
|
46
|
+
options = {enabled: settings.runtime_metrics.enabled}
|
48
47
|
options[:statsd] = settings.runtime_metrics.statsd unless settings.runtime_metrics.statsd.nil?
|
49
48
|
options[:services] = [settings.service] unless settings.service.nil?
|
50
49
|
options[:experimental_runtime_id_enabled] = settings.runtime_metrics.experimental_runtime_id_enabled
|
@@ -127,14 +126,15 @@ module Datadog
|
|
127
126
|
@dynamic_instrumentation = Datadog::DI::Component.build(settings, agent_settings, @logger, telemetry: telemetry)
|
128
127
|
@error_tracking = Datadog::ErrorTracking::Component.build(settings, @tracer, @logger)
|
129
128
|
@environment_logger_extra[:dynamic_instrumentation_enabled] = !!@dynamic_instrumentation
|
130
|
-
|
131
|
-
# @process_discovery_fd = Core::ProcessDiscovery.get_and_store_metadata(settings, @logger)
|
129
|
+
@process_discovery_fd = Core::ProcessDiscovery.get_and_store_metadata(settings, @logger)
|
132
130
|
|
133
131
|
self.class.configure_tracing(settings)
|
134
132
|
end
|
135
133
|
|
136
134
|
# Starts up components
|
137
135
|
def startup!(settings, old_state: nil)
|
136
|
+
telemetry.start(old_state&.telemetry_enabled?)
|
137
|
+
|
138
138
|
if settings.profiling.enabled
|
139
139
|
if profiler
|
140
140
|
profiler.start
|
@@ -145,7 +145,7 @@ module Datadog
|
|
145
145
|
end
|
146
146
|
end
|
147
147
|
|
148
|
-
if settings.remote.enabled && old_state&.
|
148
|
+
if settings.remote.enabled && old_state&.remote_started?
|
149
149
|
# The library was reconfigured and previously it already started
|
150
150
|
# the remote component (i.e., it received at least one request
|
151
151
|
# through the installed Rack middleware which started the remote).
|
@@ -163,20 +163,20 @@ module Datadog
|
|
163
163
|
# and avoid tearing down parts still in use.
|
164
164
|
def shutdown!(replacement = nil)
|
165
165
|
# Shutdown remote configuration
|
166
|
-
remote
|
166
|
+
remote&.shutdown!
|
167
167
|
|
168
168
|
# Shutdown DI after remote, since remote config triggers DI operations.
|
169
169
|
dynamic_instrumentation&.shutdown!
|
170
170
|
|
171
171
|
# Decommission AppSec
|
172
|
-
appsec
|
172
|
+
appsec&.shutdown!
|
173
173
|
|
174
174
|
# Shutdown the old tracer, unless it's still being used.
|
175
175
|
# (e.g. a custom tracer instance passed in.)
|
176
|
-
tracer.shutdown! unless replacement && tracer
|
176
|
+
tracer.shutdown! unless replacement && tracer.equal?(replacement.tracer)
|
177
177
|
|
178
178
|
# Shutdown old profiler
|
179
|
-
profiler
|
179
|
+
profiler&.shutdown!
|
180
180
|
|
181
181
|
# Shutdown workers
|
182
182
|
runtime_metrics.stop(true, close_metrics: false)
|
@@ -194,24 +194,31 @@ module Datadog
|
|
194
194
|
health_metrics.statsd
|
195
195
|
].compact.uniq
|
196
196
|
|
197
|
-
new_statsd =
|
198
|
-
|
199
|
-
|
200
|
-
|
201
|
-
|
202
|
-
|
203
|
-
|
204
|
-
|
197
|
+
new_statsd = if replacement
|
198
|
+
[
|
199
|
+
replacement.runtime_metrics.metrics.statsd,
|
200
|
+
replacement.health_metrics.statsd
|
201
|
+
].compact.uniq
|
202
|
+
else
|
203
|
+
[]
|
204
|
+
end
|
205
205
|
|
206
206
|
unused_statsd = (old_statsd - (old_statsd & new_statsd))
|
207
207
|
unused_statsd.each(&:close)
|
208
208
|
|
209
|
-
# enqueue closing event before stopping telemetry so it will be
|
210
|
-
telemetry.emit_closing! unless replacement
|
211
|
-
telemetry.
|
209
|
+
# enqueue closing event before stopping telemetry so it will be sent out on shutdown
|
210
|
+
telemetry.emit_closing! unless replacement&.telemetry&.enabled
|
211
|
+
telemetry.shutdown!
|
212
212
|
|
213
|
-
|
214
|
-
|
213
|
+
@process_discovery_fd&.shutdown!
|
214
|
+
end
|
215
|
+
|
216
|
+
# Returns the current state of various components.
|
217
|
+
def state
|
218
|
+
ComponentsState.new(
|
219
|
+
telemetry_enabled: telemetry.enabled,
|
220
|
+
remote_started: remote&.started?,
|
221
|
+
)
|
215
222
|
end
|
216
223
|
end
|
217
224
|
end
|
@@ -0,0 +1,23 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Datadog
|
4
|
+
module Core
|
5
|
+
module Configuration
|
6
|
+
# Stores the state of component tree when replacing the tree.
|
7
|
+
class ComponentsState
|
8
|
+
def initialize(telemetry_enabled:, remote_started:)
|
9
|
+
@telemetry_enabled = !!telemetry_enabled
|
10
|
+
@remote_started = !!remote_started
|
11
|
+
end
|
12
|
+
|
13
|
+
def telemetry_enabled?
|
14
|
+
@telemetry_enabled
|
15
|
+
end
|
16
|
+
|
17
|
+
def remote_started?
|
18
|
+
@remote_started
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|