datadog 2.35.0 → 2.37.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +85 -1
- data/ext/datadog_profiling_native_extension/clock_id_from_mach.c +5 -12
- data/ext/datadog_profiling_native_extension/collectors_cpu_and_wall_time_worker.c +71 -31
- data/ext/datadog_profiling_native_extension/collectors_discrete_dynamic_sampler.c +1 -1
- data/ext/datadog_profiling_native_extension/collectors_idle_sampling_helper.c +18 -4
- data/ext/datadog_profiling_native_extension/collectors_stack.c +37 -18
- data/ext/datadog_profiling_native_extension/collectors_stack.h +8 -2
- data/ext/datadog_profiling_native_extension/collectors_thread_context.c +596 -341
- data/ext/datadog_profiling_native_extension/collectors_thread_context.h +11 -7
- data/ext/datadog_profiling_native_extension/datadog_ruby_common.c +7 -8
- data/ext/datadog_profiling_native_extension/datadog_ruby_common.h +0 -12
- data/ext/datadog_profiling_native_extension/extconf.rb +13 -2
- data/ext/datadog_profiling_native_extension/gvl_profiling_helper.c +4 -43
- data/ext/datadog_profiling_native_extension/gvl_profiling_helper.h +15 -47
- data/ext/datadog_profiling_native_extension/heap_recorder.c +44 -26
- data/ext/datadog_profiling_native_extension/private_vm_api_access.c +14 -35
- data/ext/datadog_profiling_native_extension/profiling.c +41 -4
- data/ext/datadog_profiling_native_extension/ruby_helpers.c +40 -34
- data/ext/datadog_profiling_native_extension/stack_recorder.c +24 -3
- data/ext/datadog_profiling_native_extension/stack_recorder.h +1 -0
- data/ext/datadog_profiling_native_extension/unsafe_api_calls_check.h +4 -2
- data/ext/libdatadog_api/datadog_ruby_common.c +7 -8
- data/ext/libdatadog_api/datadog_ruby_common.h +0 -12
- data/ext/libdatadog_api/di.c +76 -0
- data/ext/libdatadog_api/extconf.rb +6 -0
- data/ext/libdatadog_extconf_helpers.rb +1 -1
- data/lib/datadog/ai_guard/api_client.rb +6 -4
- data/lib/datadog/appsec/api_security/route_extractor.rb +13 -4
- data/lib/datadog/appsec/component.rb +1 -1
- data/lib/datadog/appsec/configuration.rb +13 -0
- data/lib/datadog/appsec/context.rb +4 -2
- data/lib/datadog/appsec/contrib/aws_lambda/waf_addresses.rb +37 -4
- data/lib/datadog/appsec/contrib/excon/ssrf_detection_middleware.rb +59 -4
- data/lib/datadog/appsec/contrib/faraday/ssrf_detection_middleware.rb +59 -5
- data/lib/datadog/appsec/contrib/graphql/gateway/multiplex.rb +64 -19
- data/lib/datadog/appsec/contrib/graphql/integration.rb +1 -0
- data/lib/datadog/appsec/contrib/rack/buffered_input.rb +83 -0
- data/lib/datadog/appsec/contrib/rack/gateway/request.rb +41 -3
- data/lib/datadog/appsec/contrib/rack/gateway/watcher.rb +24 -7
- data/lib/datadog/appsec/contrib/rack/input_peeker.rb +79 -0
- data/lib/datadog/appsec/contrib/rack/request_middleware.rb +21 -0
- data/lib/datadog/appsec/contrib/rails/gateway/request.rb +35 -1
- data/lib/datadog/appsec/contrib/rails/gateway/watcher.rb +17 -1
- data/lib/datadog/appsec/contrib/rest_client/request_ssrf_detection_patch.rb +60 -7
- data/lib/datadog/appsec/contrib/sinatra/gateway/watcher.rb +24 -3
- data/lib/datadog/appsec/default_header_tags.rb +10 -6
- data/lib/datadog/appsec/ext.rb +1 -0
- data/lib/datadog/appsec/metrics/collector.rb +18 -1
- data/lib/datadog/appsec/metrics/exporter.rb +9 -3
- data/lib/datadog/appsec/remote.rb +9 -3
- data/lib/datadog/appsec/route_normalizer/rails_route_pattern.rb +176 -0
- data/lib/datadog/appsec/route_normalizer/route_pattern.rb +378 -0
- data/lib/datadog/appsec/route_normalizer/route_text.rb +57 -0
- data/lib/datadog/appsec/route_normalizer.rb +80 -0
- data/lib/datadog/appsec/utils/http/body_reader.rb +61 -0
- data/lib/datadog/core/configuration/components.rb +83 -2
- data/lib/datadog/core/configuration/components_state.rb +6 -1
- data/lib/datadog/core/configuration/settings.rb +1 -5
- data/lib/datadog/core/configuration/supported_configurations.rb +4 -0
- data/lib/datadog/core/evp.rb +11 -0
- data/lib/datadog/core/remote/client/capabilities.rb +34 -7
- data/lib/datadog/core/remote/component.rb +1 -1
- data/lib/datadog/core/telemetry/event/app_started.rb +8 -22
- data/lib/datadog/core/utils/at_fork_monkey_patch.rb +1 -1
- data/lib/datadog/core/utils/forking.rb +3 -1
- data/lib/datadog/core/utils/spawn_monkey_patch.rb +3 -1
- data/lib/datadog/core.rb +3 -0
- data/lib/datadog/di/base.rb +7 -2
- data/lib/datadog/di/code_tracker.rb +5 -2
- data/lib/datadog/di/component.rb +110 -36
- data/lib/datadog/di/error.rb +10 -0
- data/lib/datadog/di/fatal_exceptions.rb +26 -0
- data/lib/datadog/di/instrumenter.rb +381 -165
- data/lib/datadog/di/probe.rb +14 -0
- data/lib/datadog/di/probe_file_loader.rb +8 -3
- data/lib/datadog/di/probe_manager.rb +67 -9
- data/lib/datadog/di/probe_notification_builder.rb +4 -1
- data/lib/datadog/di/probe_notifier_worker.rb +52 -32
- data/lib/datadog/di/redactor.rb +16 -1
- data/lib/datadog/di/remote.rb +175 -14
- data/lib/datadog/di/serializer.rb +12 -5
- data/lib/datadog/di/transport/input.rb +8 -4
- data/lib/datadog/di.rb +63 -0
- data/lib/datadog/error_tracking/collector.rb +2 -1
- data/lib/datadog/error_tracking/component.rb +2 -2
- data/lib/datadog/kit/appsec/events/v2.rb +60 -2
- data/lib/datadog/kit/tracing/method_tracer.rb +4 -1
- data/lib/datadog/open_feature/component.rb +29 -6
- data/lib/datadog/open_feature/configuration.rb +8 -0
- data/lib/datadog/open_feature/ext.rb +2 -0
- data/lib/datadog/open_feature/flag_evaluation/aggregator.rb +286 -0
- data/lib/datadog/open_feature/flag_evaluation/writer.rb +433 -0
- data/lib/datadog/open_feature/hooks/flag_eval_evp_hook.rb +101 -0
- data/lib/datadog/open_feature/hooks/{flag_eval_hook.rb → flag_eval_metrics_hook.rb} +1 -1
- data/lib/datadog/open_feature/native_evaluator.rb +22 -0
- data/lib/datadog/open_feature/provider.rb +59 -27
- data/lib/datadog/open_feature/transport.rb +55 -1
- data/lib/datadog/opentelemetry/sdk/logs_exporter.rb +5 -12
- data/lib/datadog/opentelemetry/sdk/metrics_exporter.rb +4 -11
- data/lib/datadog/opentelemetry/sdk/propagator.rb +9 -3
- data/lib/datadog/opentelemetry/sdk/span_processor.rb +4 -1
- data/lib/datadog/opentelemetry/sdk.rb +19 -0
- data/lib/datadog/profiling/collectors/cpu_and_wall_time_worker.rb +1 -1
- data/lib/datadog/profiling/collectors/idle_sampling_helper.rb +5 -3
- data/lib/datadog/profiling/collectors/thread_context.rb +1 -0
- data/lib/datadog/profiling/component.rb +13 -15
- data/lib/datadog/profiling/ext/dir_monkey_patches.rb +3 -3
- data/lib/datadog/ruby_version.rb +25 -0
- data/lib/datadog/symbol_database/component.rb +453 -128
- data/lib/datadog/symbol_database/configuration.rb +12 -4
- data/lib/datadog/symbol_database/extensions.rb +19 -0
- data/lib/datadog/symbol_database/extractor.rb +326 -150
- data/lib/datadog/symbol_database/file_hash.rb +3 -1
- data/lib/datadog/symbol_database/remote.rb +8 -3
- data/lib/datadog/symbol_database/scope_batcher.rb +7 -3
- data/lib/datadog/symbol_database/uploader.rb +9 -2
- data/lib/datadog/symbol_database.rb +22 -0
- data/lib/datadog/tracing/configuration/ext.rb +13 -0
- data/lib/datadog/tracing/configuration/settings.rb +17 -0
- data/lib/datadog/tracing/contrib/configuration/resolver.rb +7 -0
- data/lib/datadog/tracing/contrib/grpc/distributed/propagation.rb +2 -0
- data/lib/datadog/tracing/contrib/grpc.rb +1 -0
- data/lib/datadog/tracing/contrib/http/distributed/propagation.rb +2 -0
- data/lib/datadog/tracing/contrib/http.rb +1 -0
- data/lib/datadog/tracing/contrib/karafka/distributed/propagation.rb +2 -0
- data/lib/datadog/tracing/contrib/karafka.rb +1 -0
- data/lib/datadog/tracing/contrib/rack/middlewares.rb +3 -1
- data/lib/datadog/tracing/contrib/rack/route_inference.rb +3 -1
- data/lib/datadog/tracing/contrib/sidekiq/distributed/propagation.rb +2 -0
- data/lib/datadog/tracing/contrib/sidekiq.rb +1 -0
- data/lib/datadog/tracing/contrib/waterdrop/distributed/propagation.rb +2 -0
- data/lib/datadog/tracing/contrib/waterdrop.rb +1 -0
- data/lib/datadog/tracing/distributed/baggage.rb +2 -1
- data/lib/datadog/tracing/distributed/propagation.rb +33 -1
- data/lib/datadog/tracing/distributed/trace_context.rb +11 -2
- data/lib/datadog/tracing/ext.rb +9 -0
- data/lib/datadog/tracing/remote.rb +34 -3
- data/lib/datadog/tracing/trace_digest.rb +7 -0
- data/lib/datadog/tracing/trace_operation.rb +4 -1
- data/lib/datadog/tracing/tracer.rb +1 -0
- data/lib/datadog/version.rb +1 -1
- data/lib/datadog.rb +5 -1
- metadata +22 -6
|
@@ -46,6 +46,38 @@ module Datadog
|
|
|
46
46
|
Core::Diagnostics::Health::Metrics.new(telemetry: telemetry, logger: logger, **options)
|
|
47
47
|
end
|
|
48
48
|
|
|
49
|
+
# Decides whether to build the SymbolDatabase component, for the
|
|
50
|
+
# tri-state symbol_database.enabled setting: true/false are explicit
|
|
51
|
+
# overrides; nil (the default) mirrors Dynamic Instrumentation by
|
|
52
|
+
# building whenever DI's component was built. dynamic_instrumentation
|
|
53
|
+
# is DI's build result — nil only when DI is explicitly disabled or the
|
|
54
|
+
# runtime can't support it (Rails development mode, missing C
|
|
55
|
+
# extension, remote config off, non-MRI). Under the always-build model
|
|
56
|
+
# it is non-nil even when the DI setting defaults to off, so a built
|
|
57
|
+
# SymbolDatabase component is inert until an upload is actually
|
|
58
|
+
# permitted: the component's own gate (see Component#upload_allowed?)
|
|
59
|
+
# only extracts and uploads when DI is truly active or the customer
|
|
60
|
+
# opted in explicitly. This mirrors DI advertising/building a component
|
|
61
|
+
# by default while installing no probes until it is enabled.
|
|
62
|
+
# force_upload (internal/testing) always builds, since it uploads
|
|
63
|
+
# regardless of DI or the enabled setting.
|
|
64
|
+
# @param settings [Configuration::Settings]
|
|
65
|
+
# @param dynamic_instrumentation [DI::Component, nil]
|
|
66
|
+
# @return [Boolean]
|
|
67
|
+
def enable_symbol_database?(settings, dynamic_instrumentation)
|
|
68
|
+
# The symbol_database settings group is only registered on the full
|
|
69
|
+
# library load path; a partial load (e.g. require 'datadog/di') leaves
|
|
70
|
+
# it absent, in which case the feature cannot be enabled.
|
|
71
|
+
return false unless settings.respond_to?(:symbol_database)
|
|
72
|
+
|
|
73
|
+
# force_upload uploads unconditionally, so the component must be built
|
|
74
|
+
# even when the setting is nil and DI's component was not built.
|
|
75
|
+
return true if settings.symbol_database.internal.force_upload
|
|
76
|
+
|
|
77
|
+
# nil (the default) follows whether DI's component was built.
|
|
78
|
+
Datadog::SymbolDatabase.resolve_enabled(settings.symbol_database.enabled, !dynamic_instrumentation.nil?)
|
|
79
|
+
end
|
|
80
|
+
|
|
49
81
|
def build_logger(settings)
|
|
50
82
|
logger = settings.logger.instance || Core::Logger.new($stderr)
|
|
51
83
|
logger.level = settings.diagnostics.debug ? ::Logger::DEBUG : settings.logger.level
|
|
@@ -174,10 +206,31 @@ module Datadog
|
|
|
174
206
|
@ai_guard = Datadog::AIGuard::Component.build(settings, logger: @logger, telemetry: telemetry)
|
|
175
207
|
@open_feature = OpenFeature::Component.build(settings, agent_settings, logger: @logger, telemetry: telemetry)
|
|
176
208
|
@dynamic_instrumentation = Datadog::DI::Component.build(settings, agent_settings, @logger, telemetry: telemetry)
|
|
177
|
-
|
|
209
|
+
# Only build symbol database when enabled, so a disabled component is
|
|
210
|
+
# never constructed.
|
|
211
|
+
@symbol_database =
|
|
212
|
+
if self.class.enable_symbol_database?(settings, @dynamic_instrumentation)
|
|
213
|
+
# di_active is a live predicate, not a snapshot: DI may start after
|
|
214
|
+
# this component is built (implicit enablement via remote config).
|
|
215
|
+
# Symbol Database holds only this opaque proc, never a DI reference,
|
|
216
|
+
# so the two modules stay independent — the orchestration layer
|
|
217
|
+
# owns the cross-feature knowledge.
|
|
218
|
+
Datadog::SymbolDatabase::Component.build(
|
|
219
|
+
settings, agent_settings, @logger,
|
|
220
|
+
telemetry: telemetry,
|
|
221
|
+
di_active: -> { dynamic_instrumentation&.started? || false },
|
|
222
|
+
)
|
|
223
|
+
end
|
|
178
224
|
@error_tracking = Datadog::ErrorTracking::Component.build(settings, @tracer, @logger)
|
|
179
225
|
@data_streams = self.class.build_data_streams(settings, agent_settings, @logger, @agent_info)
|
|
180
|
-
|
|
226
|
+
# Reflects "the customer configured DI to be on" — true iff the
|
|
227
|
+
# component was built AND the env-var-driven enabled flag is set.
|
|
228
|
+
# This is the post-initialize / pre-startup value visible to tests
|
|
229
|
+
# that don't call startup!. Production overwrites this in startup!
|
|
230
|
+
# below with `dynamic_instrumentation&.started?`, which also accounts
|
|
231
|
+
# for RC-driven enablement carried over via ComponentsState.
|
|
232
|
+
@environment_logger_extra[:dynamic_instrumentation_enabled] =
|
|
233
|
+
!!(@dynamic_instrumentation && settings.dynamic_instrumentation.enabled)
|
|
181
234
|
|
|
182
235
|
# Configure non-privileged components.
|
|
183
236
|
Datadog::Tracing::Contrib::Component.configure(settings)
|
|
@@ -194,6 +247,7 @@ module Datadog
|
|
|
194
247
|
remote&.after_fork
|
|
195
248
|
crashtracker&.update_on_fork
|
|
196
249
|
ProcessDiscovery.after_fork
|
|
250
|
+
symbol_database&.after_fork!
|
|
197
251
|
end
|
|
198
252
|
|
|
199
253
|
# Hot-swaps with a new sampler.
|
|
@@ -227,11 +281,25 @@ module Datadog
|
|
|
227
281
|
remote.start
|
|
228
282
|
end
|
|
229
283
|
|
|
284
|
+
# Start DI component if enabled by env var or if it was implicitly enabled
|
|
285
|
+
# (via RC) in the previous Components instance. dynamic_instrumentation is
|
|
286
|
+
# non-nil only when Component.build's preconditions passed (Ruby 2.6+ MRI,
|
|
287
|
+
# etc.), which is exactly the condition under which di/base.rb is loaded
|
|
288
|
+
# and DI.activate_tracking is defined.
|
|
289
|
+
if dynamic_instrumentation
|
|
290
|
+
if settings.dynamic_instrumentation.enabled || old_state&.di_implicitly_enabled?
|
|
291
|
+
DI.activate_tracking
|
|
292
|
+
dynamic_instrumentation.start!
|
|
293
|
+
end
|
|
294
|
+
end
|
|
295
|
+
|
|
230
296
|
# This should stay here, not in initialize. During reconfiguration, the order of the calls is:
|
|
231
297
|
# initialize new components, shutdown old components, startup new components.
|
|
232
298
|
# Because this is a singleton, if we call it in initialize, it will be shutdown right away.
|
|
233
299
|
Core::ProcessDiscovery.publish(settings)
|
|
234
300
|
|
|
301
|
+
@environment_logger_extra[:dynamic_instrumentation_enabled] = dynamic_instrumentation&.started? || false
|
|
302
|
+
|
|
235
303
|
Core::Diagnostics::EnvironmentLogger.collect_and_log!(@environment_logger_extra)
|
|
236
304
|
end
|
|
237
305
|
|
|
@@ -308,9 +376,22 @@ module Datadog
|
|
|
308
376
|
|
|
309
377
|
# Returns the current state of various components.
|
|
310
378
|
def state
|
|
379
|
+
# di_implicitly_enabled distinguishes RC-driven start from explicit
|
|
380
|
+
# start (env var or programmatic) so that an explicit
|
|
381
|
+
# `enabled = false` on reconfiguration can take effect.
|
|
382
|
+
#
|
|
383
|
+
# using_default? — not `!enabled` — because Datadog.configure has
|
|
384
|
+
# already mutated the singleton settings by the time #state runs,
|
|
385
|
+
# so `!enabled` would treat a customer's explicit
|
|
386
|
+
# `enabled = false` as "implicitly enabled" and restart DI.
|
|
387
|
+
# using_default? asks "did the customer ever touch this setting",
|
|
388
|
+
# which is the right question for RC carry-over.
|
|
389
|
+
di_implicit = dynamic_instrumentation&.started? &&
|
|
390
|
+
@settings.dynamic_instrumentation.using_default?(:enabled)
|
|
311
391
|
ComponentsState.new(
|
|
312
392
|
telemetry_enabled: telemetry.enabled,
|
|
313
393
|
remote_started: remote&.started?,
|
|
394
|
+
di_implicitly_enabled: di_implicit || false,
|
|
314
395
|
)
|
|
315
396
|
end
|
|
316
397
|
end
|
|
@@ -5,9 +5,10 @@ module Datadog
|
|
|
5
5
|
module Configuration
|
|
6
6
|
# Stores the state of component tree when replacing the tree.
|
|
7
7
|
class ComponentsState
|
|
8
|
-
def initialize(telemetry_enabled:, remote_started:)
|
|
8
|
+
def initialize(telemetry_enabled:, remote_started:, di_implicitly_enabled: false)
|
|
9
9
|
@telemetry_enabled = !!telemetry_enabled
|
|
10
10
|
@remote_started = !!remote_started
|
|
11
|
+
@di_implicitly_enabled = !!di_implicitly_enabled
|
|
11
12
|
end
|
|
12
13
|
|
|
13
14
|
def telemetry_enabled?
|
|
@@ -17,6 +18,10 @@ module Datadog
|
|
|
17
18
|
def remote_started?
|
|
18
19
|
@remote_started
|
|
19
20
|
end
|
|
21
|
+
|
|
22
|
+
def di_implicitly_enabled?
|
|
23
|
+
@di_implicitly_enabled
|
|
24
|
+
end
|
|
20
25
|
end
|
|
21
26
|
end
|
|
22
27
|
end
|
|
@@ -14,7 +14,6 @@ require_relative '../../profiling/ext'
|
|
|
14
14
|
|
|
15
15
|
require_relative '../../tracing/configuration/settings'
|
|
16
16
|
require_relative '../../opentelemetry/configuration/settings'
|
|
17
|
-
require_relative '../../symbol_database/configuration'
|
|
18
17
|
|
|
19
18
|
module Datadog
|
|
20
19
|
module Core
|
|
@@ -618,8 +617,7 @@ module Datadog
|
|
|
618
617
|
o.type :bool
|
|
619
618
|
o.env 'DD_PROFILING_SIGHANDLER_SAMPLING_ENABLED'
|
|
620
619
|
o.default do
|
|
621
|
-
|
|
622
|
-
!(RUBY_VERSION.start_with?('3.3.') && Gem::Version.new(RUBY_VERSION) < Gem::Version.new('3.3.4'))
|
|
620
|
+
RubyVersion.is?('>= 3.2.5') && !RubyVersion.is?('>= 3.3', '< 3.3.4')
|
|
623
621
|
end
|
|
624
622
|
end
|
|
625
623
|
|
|
@@ -1131,8 +1129,6 @@ module Datadog
|
|
|
1131
1129
|
extend Datadog::Tracing::Configuration::Settings
|
|
1132
1130
|
|
|
1133
1131
|
extend Datadog::OpenTelemetry::Configuration::Settings
|
|
1134
|
-
|
|
1135
|
-
extend Datadog::SymbolDatabase::Configuration::Settings
|
|
1136
1132
|
end
|
|
1137
1133
|
# standard:enable Metrics/BlockLength
|
|
1138
1134
|
end
|
|
@@ -19,12 +19,14 @@ module Datadog
|
|
|
19
19
|
"DD_API_SECURITY_DOWNSTREAM_BODY_ANALYSIS_SAMPLE_RATE",
|
|
20
20
|
"DD_API_SECURITY_ENABLED",
|
|
21
21
|
"DD_API_SECURITY_ENDPOINT_COLLECTION_ENABLED",
|
|
22
|
+
"DD_API_SECURITY_MAX_DOWNSTREAM_BODY_BYTES",
|
|
22
23
|
"DD_API_SECURITY_MAX_DOWNSTREAM_REQUEST_BODY_ANALYSIS",
|
|
23
24
|
"DD_API_SECURITY_REQUEST_SAMPLE_RATE",
|
|
24
25
|
"DD_API_SECURITY_SAMPLE_DELAY",
|
|
25
26
|
"DD_APM_TRACING_ENABLED",
|
|
26
27
|
"DD_APPSEC_AUTOMATED_USER_EVENTS_TRACKING",
|
|
27
28
|
"DD_APPSEC_AUTO_USER_INSTRUMENTATION_MODE",
|
|
29
|
+
"DD_APPSEC_BODY_PARSING_SIZE_LIMIT",
|
|
28
30
|
"DD_APPSEC_ENABLED",
|
|
29
31
|
"DD_APPSEC_HTTP_BLOCKED_TEMPLATE_HTML",
|
|
30
32
|
"DD_APPSEC_HTTP_BLOCKED_TEMPLATE_JSON",
|
|
@@ -58,6 +60,7 @@ module Datadog
|
|
|
58
60
|
"DD_EXPERIMENTAL_FLAGGING_PROVIDER_ENABLED",
|
|
59
61
|
"DD_EXPERIMENTAL_PROPAGATE_PROCESS_TAGS_ENABLED",
|
|
60
62
|
"DD_EXTERNAL_ENV",
|
|
63
|
+
"DD_FLAGGING_EVALUATION_COUNTS_ENABLED",
|
|
61
64
|
"DD_GIT_COMMIT_SHA",
|
|
62
65
|
"DD_GIT_REPOSITORY_URL",
|
|
63
66
|
"DD_HEALTH_METRICS_ENABLED",
|
|
@@ -270,6 +273,7 @@ module Datadog
|
|
|
270
273
|
"DD_TRACE_PRESTO_ENABLED",
|
|
271
274
|
"DD_TRACE_PRESTO_PEER_SERVICE",
|
|
272
275
|
"DD_TRACE_PRESTO_SERVICE_NAME",
|
|
276
|
+
"DD_TRACE_PROPAGATION_BEHAVIOR_EXTRACT",
|
|
273
277
|
"DD_TRACE_PROPAGATION_EXTRACT_FIRST",
|
|
274
278
|
"DD_TRACE_PROPAGATION_STYLE",
|
|
275
279
|
"DD_TRACE_PROPAGATION_STYLE_EXTRACT",
|
|
@@ -4,6 +4,7 @@ require_relative '../../utils/base64_codec'
|
|
|
4
4
|
require_relative '../../../appsec/remote'
|
|
5
5
|
require_relative '../../../tracing/remote'
|
|
6
6
|
require_relative '../../../di/remote'
|
|
7
|
+
require_relative '../../../symbol_database'
|
|
7
8
|
require_relative '../../../symbol_database/remote'
|
|
8
9
|
require_relative '../../../open_feature/remote'
|
|
9
10
|
|
|
@@ -35,13 +36,43 @@ module Datadog
|
|
|
35
36
|
register_receivers(Datadog::AppSec::Remote.receivers(@telemetry))
|
|
36
37
|
end
|
|
37
38
|
|
|
38
|
-
|
|
39
|
+
# Tracing must register before DI: on a combined RC dispatch,
|
|
40
|
+
# the APM_TRACING handler must run first to call
|
|
41
|
+
# Datadog::DI::Remote.handle_rc_enablement and start the
|
|
42
|
+
# component before the DI receiver processes LIVE_DEBUGGING
|
|
43
|
+
# changes against `component.started?`. Reversing the order
|
|
44
|
+
# silently drops the probe — the remote client only
|
|
45
|
+
# redispatches on content hash changes.
|
|
46
|
+
register_capabilities(Datadog::Tracing::Remote.capabilities)
|
|
47
|
+
register_products(Datadog::Tracing::Remote.products)
|
|
48
|
+
register_receivers(Datadog::Tracing::Remote.receivers(@telemetry))
|
|
49
|
+
|
|
50
|
+
# Skip DI registration entirely when DI is explicitly disabled
|
|
51
|
+
# (DD_DYNAMIC_INSTRUMENTATION_ENABLED=false) or when the runtime
|
|
52
|
+
# cannot run DI (JRuby, Ruby 2.5): in either case no component will
|
|
53
|
+
# run, so advertising bit 38 or the LIVE_DEBUGGING product would
|
|
54
|
+
# invite probe configs and an enable signal the tracer must refuse.
|
|
55
|
+
# When the env var is unset (default) on a supported runtime, DI is
|
|
56
|
+
# registered so RC can enable it.
|
|
57
|
+
if settings.respond_to?(:dynamic_instrumentation) &&
|
|
58
|
+
!Datadog::DI::Remote.explicitly_disabled?(settings) &&
|
|
59
|
+
Datadog::DI.supported_runtime?
|
|
39
60
|
register_capabilities(Datadog::DI::Remote.capabilities)
|
|
40
61
|
register_products(Datadog::DI::Remote.products)
|
|
41
62
|
register_receivers(Datadog::DI::Remote.receivers(@telemetry))
|
|
63
|
+
end
|
|
42
64
|
|
|
43
|
-
|
|
44
|
-
|
|
65
|
+
# Skip symbol database registration on runtimes that cannot run it
|
|
66
|
+
# (JRuby, Ruby < 2.7): DI supports Ruby 2.6 but Symbol Database does
|
|
67
|
+
# not, so advertising LIVE_DEBUGGING_SYMBOL_DB there would invite
|
|
68
|
+
# symbol-upload configs the component can never serve.
|
|
69
|
+
if settings.respond_to?(:symbol_database) && Datadog::SymbolDatabase.supported_runtime?
|
|
70
|
+
# Symbol database follows DI: when unset it advertises whenever DI
|
|
71
|
+
# advertises (mirror the DI branch above, including the unset/default
|
|
72
|
+
# case that RC may enable). An explicit symbol_database.enabled wins.
|
|
73
|
+
di_enabled = settings.respond_to?(:dynamic_instrumentation) &&
|
|
74
|
+
!Datadog::DI::Remote.explicitly_disabled?(settings)
|
|
75
|
+
if Datadog::SymbolDatabase.resolve_enabled(settings.symbol_database.enabled, di_enabled)
|
|
45
76
|
register_capabilities(Datadog::SymbolDatabase::Remote.capabilities)
|
|
46
77
|
register_products(Datadog::SymbolDatabase::Remote.products)
|
|
47
78
|
register_receivers(Datadog::SymbolDatabase::Remote.receivers(@telemetry))
|
|
@@ -53,10 +84,6 @@ module Datadog
|
|
|
53
84
|
register_products(Datadog::OpenFeature::Remote.products)
|
|
54
85
|
register_receivers(Datadog::OpenFeature::Remote.receivers(@telemetry))
|
|
55
86
|
end
|
|
56
|
-
|
|
57
|
-
register_capabilities(Datadog::Tracing::Remote.capabilities)
|
|
58
|
-
register_products(Datadog::Tracing::Remote.products)
|
|
59
|
-
register_receivers(Datadog::Tracing::Remote.receivers(@telemetry))
|
|
60
87
|
end
|
|
61
88
|
|
|
62
89
|
def register_capabilities(capabilities)
|
|
@@ -144,7 +144,7 @@ module Datadog
|
|
|
144
144
|
# timeout and an integer otherwise
|
|
145
145
|
# - before Ruby 3.2, ConditionVariable returns itself
|
|
146
146
|
# so we have to rely on @once having been set
|
|
147
|
-
if
|
|
147
|
+
if RubyVersion.is?('>= 3.2')
|
|
148
148
|
lifted = @condition.wait(@mutex, timeout)
|
|
149
149
|
else
|
|
150
150
|
@condition.wait(@mutex, timeout)
|
|
@@ -57,7 +57,14 @@ module Datadog
|
|
|
57
57
|
enabled: !!components.profiler,
|
|
58
58
|
},
|
|
59
59
|
dynamic_instrumentation: {
|
|
60
|
-
|
|
60
|
+
# With always-build, the DI component is constructed by
|
|
61
|
+
# default (when the runtime supports DI, RC is available, and
|
|
62
|
+
# DI is not explicitly disabled) regardless of whether the
|
|
63
|
+
# customer turned DI on. Its presence therefore no longer
|
|
64
|
+
# indicates the enabled state. started? reflects whether DI was
|
|
65
|
+
# actually enabled by env var or by implicit enablement via
|
|
66
|
+
# remote config.
|
|
67
|
+
enabled: components.dynamic_instrumentation&.started? || false,
|
|
61
68
|
}
|
|
62
69
|
}
|
|
63
70
|
|
|
@@ -146,27 +153,6 @@ module Datadog
|
|
|
146
153
|
)
|
|
147
154
|
end
|
|
148
155
|
|
|
149
|
-
# OpenTelemetry configuration options (using environment variable names)
|
|
150
|
-
otel_exporter_headers_option = resolve_option(settings, 'opentelemetry.exporter.headers')
|
|
151
|
-
otel_exporter_headers_option.values_per_precedence.each do |precedence, value|
|
|
152
|
-
list << conf_value(
|
|
153
|
-
option_telemetry_name(otel_exporter_headers_option),
|
|
154
|
-
# Steep: Value is always a hash for opentelemetry.exporter.headers (ensured by o.type :hash)
|
|
155
|
-
value&.map { |key, header_value| "#{key}=#{header_value}" }&.join(','), # steep:ignore NoMethod
|
|
156
|
-
precedence
|
|
157
|
-
)
|
|
158
|
-
end
|
|
159
|
-
|
|
160
|
-
otel_metrics_headers_option = resolve_option(settings, 'opentelemetry.metrics.headers')
|
|
161
|
-
otel_metrics_headers_option.values_per_precedence.each do |precedence, value|
|
|
162
|
-
list << conf_value(
|
|
163
|
-
option_telemetry_name(otel_metrics_headers_option),
|
|
164
|
-
# Steep: Value is always a hash for opentelemetry.metrics.headers (ensured by o.type :hash)
|
|
165
|
-
value&.map { |key, header_value| "#{key}=#{header_value}" }&.join(','), # steep:ignore NoMethod
|
|
166
|
-
precedence
|
|
167
|
-
)
|
|
168
|
-
end
|
|
169
|
-
|
|
170
156
|
# Add some more custom additional payload values here
|
|
171
157
|
if settings.logger.instance
|
|
172
158
|
logger_instance_option = resolve_option(settings, 'logger.instance')
|
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
+
require_relative '../../ruby_version'
|
|
4
|
+
|
|
3
5
|
module Datadog
|
|
4
6
|
module Core
|
|
5
7
|
module Utils
|
|
@@ -45,7 +47,7 @@ module Datadog
|
|
|
45
47
|
# object will cause forking to not be detected in the fork when it should have.
|
|
46
48
|
#
|
|
47
49
|
# This wrapper prevents this by initializing the fork PID when the object is created.
|
|
48
|
-
if
|
|
50
|
+
if RubyVersion.is?('>= 3')
|
|
49
51
|
def initialize(*args, **kwargs, &block)
|
|
50
52
|
super
|
|
51
53
|
update_fork_pid!
|
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
+
require_relative '../../ruby_version'
|
|
4
|
+
|
|
3
5
|
module Datadog
|
|
4
6
|
module Core
|
|
5
7
|
module Utils
|
|
@@ -19,7 +21,7 @@ module Datadog
|
|
|
19
21
|
# Prepends `Process.spawn` to merge `env_provider` output into the child's environment hash.
|
|
20
22
|
module ProcessSpawnPatch
|
|
21
23
|
# The One and Only Correct Delegation Pattern
|
|
22
|
-
if
|
|
24
|
+
if RubyVersion.is?('>= 3')
|
|
23
25
|
def spawn(*args, **kwargs) # steep:ignore DifferentMethodParameterKind
|
|
24
26
|
super(*SpawnMonkeyPatch.inject_envs(args), **kwargs)
|
|
25
27
|
end
|
data/lib/datadog/core.rb
CHANGED
|
@@ -1,5 +1,8 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
+
# Keep this at the top, this is needed at require-time by some files
|
|
4
|
+
require_relative 'ruby_version'
|
|
5
|
+
|
|
3
6
|
require_relative 'core/deprecations'
|
|
4
7
|
require_relative 'core/configuration/config_helper'
|
|
5
8
|
require_relative 'core/extensions'
|
data/lib/datadog/di/base.rb
CHANGED
|
@@ -8,8 +8,12 @@
|
|
|
8
8
|
# are loaded, and also none of the rest of datadog library which also
|
|
9
9
|
# has contrib code in other products.
|
|
10
10
|
|
|
11
|
+
require_relative 'fatal_exceptions'
|
|
11
12
|
require_relative 'code_tracker'
|
|
12
13
|
|
|
14
|
+
# Needed since this file can be loaded without core
|
|
15
|
+
require_relative '../ruby_version'
|
|
16
|
+
|
|
13
17
|
module Datadog
|
|
14
18
|
# Namespace for Datadog dynamic instrumentation.
|
|
15
19
|
#
|
|
@@ -44,13 +48,14 @@ module Datadog
|
|
|
44
48
|
# DI code.
|
|
45
49
|
def activate_tracking
|
|
46
50
|
# :script_compiled trace point was added in Ruby 2.6.
|
|
47
|
-
return unless
|
|
51
|
+
return unless RubyVersion.is?('>= 2.6')
|
|
48
52
|
|
|
49
53
|
begin
|
|
50
54
|
# Activate code tracking by default because line trace points will not work
|
|
51
55
|
# without it.
|
|
52
56
|
Datadog::DI.activate_tracking!
|
|
53
|
-
rescue => exc
|
|
57
|
+
rescue Exception => exc # standard:disable Lint/RescueException
|
|
58
|
+
Datadog::DI.reraise_if_fatal(exc)
|
|
54
59
|
if defined?(Datadog.logger)
|
|
55
60
|
Datadog.logger.warn { "di: Failed to activate code tracking for DI: #{exc.class}: #{exc.message}" }
|
|
56
61
|
else
|
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
# rubocop:disable Lint/AssignmentInCondition
|
|
4
4
|
|
|
5
5
|
require_relative 'error'
|
|
6
|
+
require_relative 'fatal_exceptions'
|
|
6
7
|
|
|
7
8
|
module Datadog
|
|
8
9
|
module DI
|
|
@@ -120,7 +121,8 @@ module Datadog
|
|
|
120
121
|
end
|
|
121
122
|
end
|
|
122
123
|
nil
|
|
123
|
-
rescue => exc
|
|
124
|
+
rescue Exception => exc # standard:disable Lint/RescueException
|
|
125
|
+
Datadog::DI.reraise_if_fatal(exc)
|
|
124
126
|
# Backfill is best-effort — if it fails, line probes on
|
|
125
127
|
# pre-loaded code won't work but everything else is unaffected.
|
|
126
128
|
if component = DI.current_component
|
|
@@ -191,7 +193,8 @@ module Datadog
|
|
|
191
193
|
# Since this method normally is called from customer applications,
|
|
192
194
|
# rescue any exceptions that might not be handled to not break said
|
|
193
195
|
# customer applications.
|
|
194
|
-
rescue => exc
|
|
196
|
+
rescue Exception => exc # standard:disable Lint/RescueException
|
|
197
|
+
Datadog::DI.reraise_if_fatal(exc)
|
|
195
198
|
# Code tracker may be loaded without the rest of DI,
|
|
196
199
|
# in which case DI.component will not yet be defined,
|
|
197
200
|
# but we will have DI.current_component (set to nil).
|
data/lib/datadog/di/component.rb
CHANGED
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
+
require_relative 'fatal_exceptions'
|
|
4
|
+
|
|
3
5
|
module Datadog
|
|
4
6
|
module DI
|
|
5
7
|
# Component for dynamic instrumentation.
|
|
@@ -15,45 +17,51 @@ module Datadog
|
|
|
15
17
|
class Component
|
|
16
18
|
class << self
|
|
17
19
|
def build(settings, agent_settings, logger, telemetry: nil)
|
|
18
|
-
return unless settings.respond_to?(:dynamic_instrumentation)
|
|
20
|
+
return unless settings.respond_to?(:dynamic_instrumentation)
|
|
19
21
|
|
|
20
|
-
|
|
21
|
-
|
|
22
|
+
# Explicit DD_DYNAMIC_INSTRUMENTATION_ENABLED=false: do not build the
|
|
23
|
+
# component at all. This is customer intent, not a failure — log at
|
|
24
|
+
# debug and emit no telemetry error. Capabilities#register skips the
|
|
25
|
+
# DI RC block under the same condition, so no enable signal arrives.
|
|
26
|
+
if Remote.explicitly_disabled?(settings)
|
|
27
|
+
logger.debug("di: dynamic instrumentation is explicitly disabled (DD_DYNAMIC_INSTRUMENTATION_ENABLED=false); not building component")
|
|
22
28
|
return
|
|
23
29
|
end
|
|
24
30
|
|
|
25
|
-
|
|
31
|
+
reason = DI.unsupported_reason(settings)
|
|
32
|
+
if reason
|
|
33
|
+
# Log level mirrors customer intent: if the customer explicitly
|
|
34
|
+
# opted in via DD_DYNAMIC_INSTRUMENTATION_ENABLED, warn. Otherwise
|
|
35
|
+
# debug — with always-build, this path runs on every tracer boot
|
|
36
|
+
# for every customer, including those who never wanted DI. Spamming
|
|
37
|
+
# warnings to silent users (especially on JRuby or Ruby 2.5) would
|
|
38
|
+
# be noise. Customers who later trigger implicit enablement via the
|
|
39
|
+
# Datadog UI get a symmetric warn from Remote.handle_rc_enablement
|
|
40
|
+
# when the RC enable signal finds no component to start.
|
|
41
|
+
level = explicitly_enabled?(settings) ? :warn : :debug
|
|
42
|
+
logger.public_send(level, "di: dynamic instrumentation is disabled: #{reason}")
|
|
43
|
+
return
|
|
44
|
+
end
|
|
26
45
|
|
|
27
46
|
new(settings, agent_settings, logger, code_tracker: DI.code_tracker, telemetry: telemetry).tap do |component|
|
|
28
47
|
DI.add_current_component(component)
|
|
29
48
|
end
|
|
30
49
|
end
|
|
31
50
|
|
|
32
|
-
#
|
|
33
|
-
#
|
|
34
|
-
#
|
|
35
|
-
#
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
return false
|
|
47
|
-
end
|
|
48
|
-
if RUBY_VERSION < '2.6'
|
|
49
|
-
logger.warn("di: cannot enable dynamic instrumentation: Ruby 2.6+ is required, but running on #{RUBY_VERSION}")
|
|
50
|
-
return false
|
|
51
|
-
end
|
|
52
|
-
unless DI.respond_to?(:exception_message)
|
|
53
|
-
logger.warn("di: cannot enable dynamic instrumentation: C extension is not available")
|
|
54
|
-
return false
|
|
55
|
-
end
|
|
56
|
-
true
|
|
51
|
+
# True when the customer explicitly set
|
|
52
|
+
# DD_DYNAMIC_INSTRUMENTATION_ENABLED=true (or its equivalent in code).
|
|
53
|
+
# Symmetric to {Remote.explicitly_disabled?}.
|
|
54
|
+
#
|
|
55
|
+
# Uses {Datadog::Core::Configuration::Options::InstanceMethods#using_default?}
|
|
56
|
+
# rather than `options[:enabled].default_precedence?` because the option
|
|
57
|
+
# hash is populated lazily on first access; reading the underlying option
|
|
58
|
+
# before {Component.build} touches the value would NoMethodError on nil.
|
|
59
|
+
#
|
|
60
|
+
# @param settings [Datadog::Core::Configuration::Settings]
|
|
61
|
+
# @return [Boolean]
|
|
62
|
+
def explicitly_enabled?(settings)
|
|
63
|
+
!settings.dynamic_instrumentation.using_default?(:enabled) &&
|
|
64
|
+
settings.dynamic_instrumentation.enabled
|
|
57
65
|
end
|
|
58
66
|
end
|
|
59
67
|
|
|
@@ -80,7 +88,11 @@ module Datadog
|
|
|
80
88
|
settings, instrumenter, probe_notification_builder, probe_notifier_worker, logger, probe_repository,
|
|
81
89
|
telemetry: telemetry,
|
|
82
90
|
)
|
|
83
|
-
|
|
91
|
+
# @started transitions are serialized by @lifecycle_mutex so that
|
|
92
|
+
# concurrent RC callbacks (which run on the remote-config thread)
|
|
93
|
+
# cannot race a foreground start! with a background stop!.
|
|
94
|
+
@lifecycle_mutex = Mutex.new
|
|
95
|
+
@started = false
|
|
84
96
|
end
|
|
85
97
|
|
|
86
98
|
attr_reader :settings
|
|
@@ -96,9 +108,61 @@ module Datadog
|
|
|
96
108
|
attr_reader :redactor
|
|
97
109
|
attr_reader :serializer
|
|
98
110
|
|
|
99
|
-
#
|
|
111
|
+
# Starts the DI component: begins accepting probes and
|
|
112
|
+
# processing snapshots.
|
|
113
|
+
#
|
|
114
|
+
# Starts the probe notifier worker thread before enabling the
|
|
115
|
+
# definition trace point, so any future status emission from a
|
|
116
|
+
# trace-point-driven installation has a worker to drain it.
|
|
117
|
+
# Today {ProbeManager#reopen} re-hooks without emitting statuses, so
|
|
118
|
+
# the order is defensive rather than load-bearing. No-op if already
|
|
119
|
+
# started. Serialized by @lifecycle_mutex.
|
|
120
|
+
#
|
|
121
|
+
# @return [void]
|
|
122
|
+
def start!
|
|
123
|
+
@lifecycle_mutex.synchronize do
|
|
124
|
+
return if @started
|
|
125
|
+
|
|
126
|
+
probe_notifier_worker.start
|
|
127
|
+
probe_manager.reopen
|
|
128
|
+
@started = true
|
|
129
|
+
end
|
|
130
|
+
end
|
|
131
|
+
|
|
132
|
+
# Stops the DI component: removes all probes and stops
|
|
133
|
+
# background threads.
|
|
134
|
+
#
|
|
135
|
+
# The component remains alive and can be restarted with {#start!}.
|
|
136
|
+
# Does not clear out the code tracker.
|
|
137
|
+
# No-op if already stopped. Serialized by @lifecycle_mutex.
|
|
138
|
+
#
|
|
139
|
+
# @return [void]
|
|
140
|
+
def stop!
|
|
141
|
+
@lifecycle_mutex.synchronize do
|
|
142
|
+
return unless @started
|
|
143
|
+
|
|
144
|
+
probe_manager.stop
|
|
145
|
+
probe_notifier_worker.stop
|
|
146
|
+
@started = false
|
|
147
|
+
end
|
|
148
|
+
end
|
|
149
|
+
|
|
150
|
+
# Whether the component is currently started.
|
|
151
|
+
#
|
|
152
|
+
# Read by remote config dispatch to decide whether to apply probe
|
|
153
|
+
# changes (changes received while stopped are dropped, since the
|
|
154
|
+
# next start! will reconcile from the latest RC state).
|
|
155
|
+
#
|
|
156
|
+
# @return [Boolean] true if start! has been called and stop! has not
|
|
157
|
+
def started?
|
|
158
|
+
@started
|
|
159
|
+
end
|
|
160
|
+
|
|
161
|
+
# Shuts down dynamic instrumentation permanently.
|
|
100
162
|
#
|
|
101
163
|
# Removes all code hooks and stops background threads.
|
|
164
|
+
# Called by Components#shutdown! during component destruction.
|
|
165
|
+
# Unlike {#stop!}, this is not reversible.
|
|
102
166
|
#
|
|
103
167
|
# Does not clear out the code tracker, because it's only populated
|
|
104
168
|
# by code when code is compiled and therefore, if the code tracker
|
|
@@ -107,21 +171,31 @@ module Datadog
|
|
|
107
171
|
def shutdown!(replacement = nil)
|
|
108
172
|
DI.remove_current_component(self)
|
|
109
173
|
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
174
|
+
# Hold the lifecycle mutex so all transitions of @started are
|
|
175
|
+
# serialized — start! / stop! / shutdown! cannot interleave with
|
|
176
|
+
# one another. Without the mutex an in-flight stop! from an RC
|
|
177
|
+
# callback could complete after shutdown!'s probe_manager.close,
|
|
178
|
+
# producing an inconsistent state.
|
|
179
|
+
@lifecycle_mutex.synchronize do
|
|
180
|
+
@started = false
|
|
181
|
+
probe_manager.clear_hooks
|
|
182
|
+
probe_manager.close
|
|
183
|
+
probe_notifier_worker.stop
|
|
184
|
+
end
|
|
113
185
|
end
|
|
114
186
|
|
|
115
187
|
def parse_probe_spec_and_notify(probe_spec)
|
|
116
188
|
probe = ProbeBuilder.build_from_remote_config(probe_spec)
|
|
117
|
-
rescue => exc
|
|
189
|
+
rescue Exception => exc # standard:disable Lint/RescueException
|
|
190
|
+
Datadog::DI.reraise_if_fatal(exc)
|
|
118
191
|
begin
|
|
119
192
|
probe = Struct.new(:id).new(
|
|
120
193
|
probe_spec['id'],
|
|
121
194
|
)
|
|
122
195
|
payload = probe_notification_builder.build_errored(probe, exc)
|
|
123
196
|
probe_notifier_worker.add_status(payload)
|
|
124
|
-
rescue => nested_exc
|
|
197
|
+
rescue Exception => nested_exc # standard:disable Lint/RescueException
|
|
198
|
+
Datadog::DI.reraise_if_fatal(nested_exc)
|
|
125
199
|
logger.debug { "di: failed to build error notification: #{nested_exc.class}: #{nested_exc.message}" }
|
|
126
200
|
telemetry&.report(nested_exc, description: 'Error building probe error notification')
|
|
127
201
|
raise
|