datadog 2.33.0 → 2.34.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +30 -1
- data/ext/datadog_profiling_native_extension/collectors_cpu_and_wall_time_worker.c +20 -0
- data/ext/datadog_profiling_native_extension/macos_sampler_thread.h +55 -0
- data/lib/datadog/appsec/component.rb +4 -1
- data/lib/datadog/appsec/compressed_json.rb +2 -2
- data/lib/datadog/appsec/contrib/aws_lambda/waf_addresses.rb +3 -3
- data/lib/datadog/appsec/contrib/rack/ext.rb +1 -1
- data/lib/datadog/core/configuration/components.rb +8 -1
- data/lib/datadog/core/configuration/settings.rb +6 -1
- data/lib/datadog/core/configuration/supported_configurations.rb +10 -0
- data/lib/datadog/core/environment/ext.rb +4 -0
- data/lib/datadog/core/environment/identity.rb +15 -1
- data/lib/datadog/core/environment/process.rb +48 -27
- data/lib/datadog/core/remote/client/capabilities.rb +11 -2
- data/lib/datadog/core/remote/transport/http/config.rb +5 -5
- data/lib/datadog/core/telemetry/request.rb +0 -2
- data/lib/datadog/core/transport/response.rb +1 -1
- data/lib/datadog/core/utils/{base64.rb → base64_codec.rb} +3 -2
- data/lib/datadog/core/utils/hash.rb +0 -23
- data/lib/datadog/core/utils/spawn_monkey_patch.rb +46 -16
- data/lib/datadog/data_streams/pathway_context.rb +3 -3
- data/lib/datadog/di/code_tracker.rb +43 -22
- data/lib/datadog/di/contrib/active_record.rb +6 -2
- data/lib/datadog/di/instrumenter.rb +24 -4
- data/lib/datadog/di/probe_notification_builder.rb +1 -1
- data/lib/datadog/di/remote.rb +4 -4
- data/lib/datadog/di/serializer.rb +5 -5
- data/lib/datadog/di/utils.rb +42 -14
- data/lib/datadog/opentelemetry/configuration/settings.rb +65 -0
- data/lib/datadog/opentelemetry/ext.rb +9 -0
- data/lib/datadog/opentelemetry/logs.rb +98 -0
- data/lib/datadog/opentelemetry/metrics.rb +10 -46
- data/lib/datadog/opentelemetry/sdk/configurator.rb +40 -0
- data/lib/datadog/opentelemetry/sdk/logs_exporter.rb +37 -0
- data/lib/datadog/opentelemetry/signal_configuration.rb +53 -0
- data/lib/datadog/opentelemetry.rb +1 -0
- data/lib/datadog/symbol_database/component.rb +409 -0
- data/lib/datadog/symbol_database/configuration.rb +2 -2
- data/lib/datadog/symbol_database/extractor.rb +29 -1
- data/lib/datadog/symbol_database/remote.rb +175 -0
- data/lib/datadog/symbol_database/scope_batcher.rb +8 -0
- data/lib/datadog/symbol_database/service_version.rb +11 -2
- data/lib/datadog/symbol_database/symbol.rb +6 -3
- data/lib/datadog/symbol_database/uploader.rb +62 -8
- data/lib/datadog/tracing/contrib/action_pack/action_dispatch/instrumentation.rb +8 -0
- data/lib/datadog/tracing/contrib/active_record/events/sql.rb +0 -4
- data/lib/datadog/tracing/contrib/active_support/cache/events/cache.rb +0 -4
- data/lib/datadog/tracing/contrib/active_support/cache/instrumentation.rb +0 -4
- data/lib/datadog/tracing/contrib/aws/instrumentation.rb +0 -5
- data/lib/datadog/tracing/contrib/dalli/instrumentation.rb +0 -5
- data/lib/datadog/tracing/contrib/elasticsearch/patcher.rb +0 -5
- data/lib/datadog/tracing/contrib/ethon/easy_patch.rb +0 -5
- data/lib/datadog/tracing/contrib/ethon/multi_patch.rb +0 -8
- data/lib/datadog/tracing/contrib/excon/middleware.rb +0 -5
- data/lib/datadog/tracing/contrib/ext.rb +2 -3
- data/lib/datadog/tracing/contrib/faraday/middleware.rb +0 -5
- data/lib/datadog/tracing/contrib/grpc/datadog_interceptor/client.rb +0 -5
- data/lib/datadog/tracing/contrib/grpc/datadog_interceptor/server.rb +0 -5
- data/lib/datadog/tracing/contrib/http/instrumentation.rb +0 -5
- data/lib/datadog/tracing/contrib/httpclient/instrumentation.rb +0 -5
- data/lib/datadog/tracing/contrib/httprb/instrumentation.rb +0 -5
- data/lib/datadog/tracing/contrib/mongodb/subscribers.rb +0 -5
- data/lib/datadog/tracing/contrib/mysql2/instrumentation.rb +0 -5
- data/lib/datadog/tracing/contrib/opensearch/patcher.rb +0 -5
- data/lib/datadog/tracing/contrib/pg/instrumentation.rb +0 -5
- data/lib/datadog/tracing/contrib/presto/instrumentation.rb +0 -5
- data/lib/datadog/tracing/contrib/racecar/event.rb +0 -5
- data/lib/datadog/tracing/contrib/redis/tags.rb +0 -5
- data/lib/datadog/tracing/contrib/rest_client/request_patch.rb +0 -5
- data/lib/datadog/tracing/contrib/sequel/utils.rb +0 -5
- data/lib/datadog/tracing/contrib/trilogy/instrumentation.rb +0 -5
- data/lib/datadog/tracing/distributed/datadog_tags_codec.rb +0 -13
- data/lib/datadog/tracing/distributed/trace_context.rb +0 -28
- data/lib/datadog/tracing/metadata/ext.rb +3 -0
- data/lib/datadog/tracing/span_operation.rb +13 -0
- data/lib/datadog/tracing/trace_operation.rb +22 -0
- data/lib/datadog/tracing/tracer.rb +6 -0
- data/lib/datadog/version.rb +1 -1
- metadata +12 -5
|
@@ -1,18 +1,19 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
require_relative '../core/configuration/ext'
|
|
4
|
-
require_relative '
|
|
4
|
+
require_relative 'ext'
|
|
5
|
+
require_relative 'signal_configuration'
|
|
5
6
|
|
|
6
7
|
module Datadog
|
|
7
8
|
module OpenTelemetry
|
|
8
9
|
class Metrics
|
|
9
|
-
|
|
10
|
+
include SignalConfiguration
|
|
10
11
|
|
|
11
12
|
def self.initialize!(components)
|
|
12
13
|
new(components).configure_metrics_sdk
|
|
13
14
|
true
|
|
14
15
|
rescue => exc
|
|
15
|
-
components.logger.
|
|
16
|
+
components.logger.warn("Failed to initialize OpenTelemetry metrics: #{exc.class}: #{exc.message}: #{exc.backtrace.join("\n")}")
|
|
16
17
|
false
|
|
17
18
|
end
|
|
18
19
|
|
|
@@ -41,38 +42,9 @@ module Datadog
|
|
|
41
42
|
|
|
42
43
|
private
|
|
43
44
|
|
|
44
|
-
def create_resource
|
|
45
|
-
resource_attributes = {}
|
|
46
|
-
|
|
47
|
-
@settings.tags&.each do |key, value|
|
|
48
|
-
otel_key = case key
|
|
49
|
-
when 'service' then 'service.name'
|
|
50
|
-
when 'env' then 'deployment.environment'
|
|
51
|
-
when 'version' then 'service.version'
|
|
52
|
-
else key
|
|
53
|
-
end
|
|
54
|
-
resource_attributes[otel_key] = value
|
|
55
|
-
end
|
|
56
|
-
|
|
57
|
-
resource_attributes['service.name'] = @settings.service_without_fallback || resource_attributes['service.name'] || Datadog::Core::Environment::Ext::FALLBACK_SERVICE_NAME
|
|
58
|
-
resource_attributes['deployment.environment'] = @settings.env if @settings.env
|
|
59
|
-
resource_attributes['service.version'] = @settings.version if @settings.version
|
|
60
|
-
|
|
61
|
-
hostname = Datadog::Core::Environment::Socket.resolved_hostname(@settings)
|
|
62
|
-
if hostname
|
|
63
|
-
if hostname == @settings.hostname
|
|
64
|
-
resource_attributes['host.name'] = hostname
|
|
65
|
-
elsif !resource_attributes.key?('host.name')
|
|
66
|
-
resource_attributes['host.name'] = hostname
|
|
67
|
-
end
|
|
68
|
-
end
|
|
69
|
-
|
|
70
|
-
::OpenTelemetry::SDK::Resources::Resource.create(resource_attributes)
|
|
71
|
-
end
|
|
72
|
-
|
|
73
45
|
def configure_metric_reader(provider)
|
|
74
46
|
exporter_name = @settings.opentelemetry.metrics.exporter
|
|
75
|
-
return if exporter_name == EXPORTER_NONE
|
|
47
|
+
return if exporter_name == Ext::EXPORTER_NONE
|
|
76
48
|
|
|
77
49
|
configure_otlp_exporter(provider)
|
|
78
50
|
rescue => e
|
|
@@ -88,15 +60,16 @@ module Datadog
|
|
|
88
60
|
require_relative 'sdk/metrics_exporter'
|
|
89
61
|
|
|
90
62
|
metrics_config = @settings.opentelemetry.metrics
|
|
91
|
-
endpoint =
|
|
63
|
+
endpoint = config_or_exporter_fallback(
|
|
64
|
+
signal: :metrics,
|
|
92
65
|
option_name: :endpoint,
|
|
93
66
|
computed_default: default_metrics_endpoint
|
|
94
67
|
)
|
|
95
|
-
timeout =
|
|
96
|
-
headers =
|
|
68
|
+
timeout = config_or_exporter_fallback(signal: :metrics, option_name: :timeout_millis)
|
|
69
|
+
headers = config_or_exporter_fallback(signal: :metrics, option_name: :headers)
|
|
97
70
|
# OpenTelemetry SDK only supports http/protobuf protocol.
|
|
98
71
|
# TODO: Add support for http/json and grpc.
|
|
99
|
-
# protocol =
|
|
72
|
+
# protocol = config_or_exporter_fallback(signal: :metrics, option_name: :protocol)
|
|
100
73
|
exporter = Datadog::OpenTelemetry::SDK::MetricsExporter.new(
|
|
101
74
|
endpoint: endpoint,
|
|
102
75
|
timeout: timeout / 1000.0,
|
|
@@ -112,15 +85,6 @@ module Datadog
|
|
|
112
85
|
rescue LoadError => e
|
|
113
86
|
@logger.warn("Could not load OTLP metrics exporter: #{e.class}: #{e.message}")
|
|
114
87
|
end
|
|
115
|
-
|
|
116
|
-
# Returns metrics config value if explicitly set, otherwise falls back to exporter config or computed default value.
|
|
117
|
-
def get_metrics_config_with_fallback(option_name:, computed_default: nil)
|
|
118
|
-
if @settings.opentelemetry.metrics.using_default?(option_name)
|
|
119
|
-
@settings.opentelemetry.exporter.public_send(option_name) || computed_default
|
|
120
|
-
else
|
|
121
|
-
@settings.opentelemetry.metrics.public_send(option_name)
|
|
122
|
-
end
|
|
123
|
-
end
|
|
124
88
|
end
|
|
125
89
|
end
|
|
126
90
|
end
|
|
@@ -30,6 +30,16 @@ module Datadog
|
|
|
30
30
|
[SpanProcessor.new]
|
|
31
31
|
end
|
|
32
32
|
|
|
33
|
+
# SDK 1.6.0+ calls logs_configuration_hook from configure.
|
|
34
|
+
# https://github.com/open-telemetry/opentelemetry-ruby/blob/opentelemetry-sdk/v1.6.0/sdk/lib/opentelemetry/sdk/configurator.rb#L152
|
|
35
|
+
# Older supported SDK versions do not, so we call it explicitly as a fallback.
|
|
36
|
+
# The flag prevents double-calling on newer SDK versions.
|
|
37
|
+
def configure
|
|
38
|
+
@datadog_logs_hook_called = false
|
|
39
|
+
super
|
|
40
|
+
logs_configuration_hook unless @datadog_logs_hook_called
|
|
41
|
+
end
|
|
42
|
+
|
|
33
43
|
def metrics_configuration_hook
|
|
34
44
|
components = Datadog.send(:components)
|
|
35
45
|
return super unless components.settings.opentelemetry.metrics.enabled
|
|
@@ -45,15 +55,45 @@ module Datadog
|
|
|
45
55
|
super unless success
|
|
46
56
|
end
|
|
47
57
|
|
|
58
|
+
def logs_configuration_hook
|
|
59
|
+
@datadog_logs_hook_called = true
|
|
60
|
+
components = Datadog.send(:components)
|
|
61
|
+
unless components.settings.opentelemetry.logs.enabled
|
|
62
|
+
super if defined?(super)
|
|
63
|
+
return
|
|
64
|
+
end
|
|
65
|
+
|
|
66
|
+
begin
|
|
67
|
+
require 'opentelemetry-logs-sdk'
|
|
68
|
+
rescue LoadError => exc
|
|
69
|
+
components.logger.warn("Failed to load OpenTelemetry logs gems: #{exc.class}: #{exc.message}")
|
|
70
|
+
return
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
success = Datadog::OpenTelemetry::Logs.initialize!(components)
|
|
74
|
+
unless success
|
|
75
|
+
components.logger.warn('Falling back to OpenTelemetry default logs configuration')
|
|
76
|
+
super if defined?(super)
|
|
77
|
+
end
|
|
78
|
+
end
|
|
79
|
+
|
|
48
80
|
# Prepend to ConfiguratorPatch (not Configurator) so our hook runs first.
|
|
49
81
|
begin
|
|
50
82
|
require 'opentelemetry-metrics-sdk' if defined?(OpenTelemetry::SDK) && !defined?(OpenTelemetry::SDK::Metrics::ConfiguratorPatch)
|
|
51
83
|
rescue LoadError
|
|
52
84
|
end
|
|
53
85
|
|
|
86
|
+
begin
|
|
87
|
+
require 'opentelemetry-logs-sdk' if defined?(OpenTelemetry::SDK) && !defined?(OpenTelemetry::SDK::Logs::ConfiguratorPatch)
|
|
88
|
+
rescue LoadError
|
|
89
|
+
end
|
|
90
|
+
|
|
54
91
|
if defined?(::OpenTelemetry::SDK::Metrics::ConfiguratorPatch)
|
|
55
92
|
::OpenTelemetry::SDK::Metrics::ConfiguratorPatch.prepend(self) unless ::OpenTelemetry::SDK::Metrics::ConfiguratorPatch.ancestors.include?(self)
|
|
56
93
|
end
|
|
94
|
+
if defined?(::OpenTelemetry::SDK::Logs::ConfiguratorPatch)
|
|
95
|
+
::OpenTelemetry::SDK::Logs::ConfiguratorPatch.prepend(self) unless ::OpenTelemetry::SDK::Logs::ConfiguratorPatch.ancestors.include?(self)
|
|
96
|
+
end
|
|
57
97
|
::OpenTelemetry::SDK::Configurator.prepend(self) unless ::OpenTelemetry::SDK::Configurator.ancestors.include?(self)
|
|
58
98
|
end
|
|
59
99
|
end
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'opentelemetry/exporter/otlp_logs'
|
|
4
|
+
|
|
5
|
+
module Datadog
|
|
6
|
+
module OpenTelemetry
|
|
7
|
+
module SDK
|
|
8
|
+
class LogsExporter < ::OpenTelemetry::Exporter::OTLP::Logs::LogsExporter
|
|
9
|
+
METRIC_EXPORT_ATTEMPTS = 'otel.logs_export_attempts'
|
|
10
|
+
METRIC_EXPORT_SUCCESSES = 'otel.logs_export_successes'
|
|
11
|
+
METRIC_EXPORT_FAILURES = 'otel.logs_export_failures'
|
|
12
|
+
METRIC_LOG_RECORDS = 'otel.log_records'
|
|
13
|
+
TELEMETRY_NAMESPACE = 'tracers'
|
|
14
|
+
TELEMETRY_TAGS = {'protocol' => 'http', 'encoding' => 'protobuf'}.freeze
|
|
15
|
+
|
|
16
|
+
def export(log_records, timeout: nil)
|
|
17
|
+
telemetry&.inc(TELEMETRY_NAMESPACE, METRIC_EXPORT_ATTEMPTS, 1, tags: TELEMETRY_TAGS)
|
|
18
|
+
telemetry&.inc(TELEMETRY_NAMESPACE, METRIC_LOG_RECORDS, log_records.size, tags: TELEMETRY_TAGS)
|
|
19
|
+
result = super
|
|
20
|
+
metric_name = (result == 0) ? METRIC_EXPORT_SUCCESSES : METRIC_EXPORT_FAILURES
|
|
21
|
+
telemetry&.inc(TELEMETRY_NAMESPACE, metric_name, 1, tags: TELEMETRY_TAGS)
|
|
22
|
+
result
|
|
23
|
+
rescue => e
|
|
24
|
+
Datadog.logger.warn("Failed to export OpenTelemetry Logs: #{e.class}: #{e.message}")
|
|
25
|
+
telemetry&.inc(TELEMETRY_NAMESPACE, METRIC_EXPORT_FAILURES, 1, tags: TELEMETRY_TAGS)
|
|
26
|
+
raise
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
private
|
|
30
|
+
|
|
31
|
+
def telemetry
|
|
32
|
+
Datadog.send(:components).telemetry
|
|
33
|
+
end
|
|
34
|
+
end
|
|
35
|
+
end
|
|
36
|
+
end
|
|
37
|
+
end
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative '../core/configuration/ext'
|
|
4
|
+
require_relative '../core/environment/socket'
|
|
5
|
+
|
|
6
|
+
module Datadog
|
|
7
|
+
module OpenTelemetry
|
|
8
|
+
# Shared resource building and signal-specific config fallback logic for Logs and Metrics.
|
|
9
|
+
module SignalConfiguration
|
|
10
|
+
private
|
|
11
|
+
|
|
12
|
+
def create_resource
|
|
13
|
+
resource_attributes = {}
|
|
14
|
+
|
|
15
|
+
@settings.tags&.each do |key, value| # steep:ignore
|
|
16
|
+
otel_key = case key
|
|
17
|
+
when 'service' then 'service.name'
|
|
18
|
+
when 'env' then 'deployment.environment'
|
|
19
|
+
when 'version' then 'service.version'
|
|
20
|
+
else key
|
|
21
|
+
end
|
|
22
|
+
resource_attributes[otel_key] = value
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
resource_attributes['service.name'] = @settings.service_without_fallback || resource_attributes['service.name'] || Datadog::Core::Environment::Ext::FALLBACK_SERVICE_NAME # steep:ignore
|
|
26
|
+
resource_attributes['deployment.environment'] = @settings.env if @settings.env # steep:ignore
|
|
27
|
+
resource_attributes['service.version'] = @settings.version if @settings.version # steep:ignore
|
|
28
|
+
|
|
29
|
+
hostname = Datadog::Core::Environment::Socket.resolved_hostname(@settings) # steep:ignore
|
|
30
|
+
if hostname
|
|
31
|
+
if hostname == @settings.hostname # steep:ignore
|
|
32
|
+
resource_attributes['host.name'] = hostname
|
|
33
|
+
elsif !resource_attributes.key?('host.name')
|
|
34
|
+
resource_attributes['host.name'] = hostname
|
|
35
|
+
end
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
::OpenTelemetry::SDK::Resources::Resource.create(resource_attributes)
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
# Returns the signal-specific option value when explicitly set,
|
|
42
|
+
# otherwise falls back to the general OTLP exporter config or computed_default.
|
|
43
|
+
def config_or_exporter_fallback(signal:, option_name:, computed_default: nil)
|
|
44
|
+
signal_settings = @settings.opentelemetry.public_send(signal) # steep:ignore
|
|
45
|
+
if signal_settings.using_default?(option_name)
|
|
46
|
+
@settings.opentelemetry.exporter.public_send(option_name) || computed_default # steep:ignore
|
|
47
|
+
else
|
|
48
|
+
signal_settings.public_send(option_name)
|
|
49
|
+
end
|
|
50
|
+
end
|
|
51
|
+
end
|
|
52
|
+
end
|
|
53
|
+
end
|
|
@@ -23,6 +23,7 @@ require_relative 'opentelemetry/sdk/configurator' if defined?(OpenTelemetry::SDK
|
|
|
23
23
|
require_relative 'opentelemetry/sdk/trace/span' if defined?(OpenTelemetry::SDK)
|
|
24
24
|
|
|
25
25
|
require_relative 'opentelemetry/metrics' if defined?(OpenTelemetry::SDK::Metrics)
|
|
26
|
+
require_relative 'opentelemetry/logs' if defined?(OpenTelemetry::SDK::Logs)
|
|
26
27
|
|
|
27
28
|
module Datadog
|
|
28
29
|
# Datadog OpenTelemetry integration.
|
|
@@ -0,0 +1,409 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative 'extractor'
|
|
4
|
+
require_relative 'logger'
|
|
5
|
+
require_relative 'scope_batcher'
|
|
6
|
+
require_relative 'uploader'
|
|
7
|
+
require_relative '../core/utils/time'
|
|
8
|
+
|
|
9
|
+
module Datadog
|
|
10
|
+
module SymbolDatabase
|
|
11
|
+
# Main coordinator for symbol database upload functionality.
|
|
12
|
+
#
|
|
13
|
+
# Responsibilities:
|
|
14
|
+
# - Lifecycle management: Initialization, shutdown, upload triggering
|
|
15
|
+
# - Coordination: Connects Extractor → ScopeBatcher → Uploader
|
|
16
|
+
# - Remote config handling: start_upload called by Remote module on config changes
|
|
17
|
+
# - Debounce: extraction is deferred by EXTRACT_DEBOUNCE_INTERVAL seconds so
|
|
18
|
+
# reconfigurations during boot coalesce into a single extraction on the
|
|
19
|
+
# final Component instance.
|
|
20
|
+
#
|
|
21
|
+
# Upload flow:
|
|
22
|
+
# 1. Remote config sends upload_symbols: true (or force_upload mode)
|
|
23
|
+
# 2. start_upload called — schedules extraction EXTRACT_DEBOUNCE_INTERVAL
|
|
24
|
+
# seconds in the future on a per-instance scheduler thread.
|
|
25
|
+
# 3. When the timer fires (no further start_upload calls reset it),
|
|
26
|
+
# extract_and_upload runs: ObjectSpace iteration → Extractor → ScopeBatcher.
|
|
27
|
+
# 4. ScopeBatcher batches and triggers Uploader.
|
|
28
|
+
# 5. A class-level flag is set so subsequent Component instances created via
|
|
29
|
+
# Datadog reconfiguration do not re-upload.
|
|
30
|
+
#
|
|
31
|
+
# Created by: Components#initialize (in Core::Configuration::Components)
|
|
32
|
+
# Accessed by: Remote config receiver via Datadog.send(:components).symbol_database
|
|
33
|
+
# Requires: Remote config enabled (unless force mode)
|
|
34
|
+
#
|
|
35
|
+
# @api private
|
|
36
|
+
class Component
|
|
37
|
+
# Debounce window for extraction. Multiple start_upload calls within this
|
|
38
|
+
# window coalesce; the timer fires once after the window of inactivity.
|
|
39
|
+
# Long enough to absorb reconfiguration cascades during Rails boot.
|
|
40
|
+
EXTRACT_DEBOUNCE_INTERVAL = 5 # seconds
|
|
41
|
+
|
|
42
|
+
# Class-level state: tracks whether any Component instance in this process
|
|
43
|
+
# has performed an extract+upload. Survives Component replacement during
|
|
44
|
+
# Datadog reconfiguration so duplicate uploads are prevented.
|
|
45
|
+
@uploaded_this_process = false
|
|
46
|
+
@upload_done_mutex = Mutex.new
|
|
47
|
+
@upload_done_cv = ConditionVariable.new
|
|
48
|
+
|
|
49
|
+
class << self
|
|
50
|
+
attr_reader :upload_done_mutex, :upload_done_cv
|
|
51
|
+
|
|
52
|
+
# Whether any Component instance in this process has completed an
|
|
53
|
+
# upload. Cross-instance flag — used to dedupe uploads across
|
|
54
|
+
# Component rebuilds within a single Ruby process.
|
|
55
|
+
# @return [Boolean]
|
|
56
|
+
def uploaded_this_process?
|
|
57
|
+
@upload_done_mutex.synchronize { @uploaded_this_process }
|
|
58
|
+
end
|
|
59
|
+
|
|
60
|
+
# Mark the current process as having completed a symbol upload.
|
|
61
|
+
# Called by the Component instance that successfully completes an
|
|
62
|
+
# upload; subsequent start_upload calls on any instance short-circuit.
|
|
63
|
+
# @return [void]
|
|
64
|
+
def mark_uploaded
|
|
65
|
+
@upload_done_mutex.synchronize do
|
|
66
|
+
@uploaded_this_process = true
|
|
67
|
+
@upload_done_cv.broadcast
|
|
68
|
+
end
|
|
69
|
+
end
|
|
70
|
+
|
|
71
|
+
# Reset class-level upload state. Test-only.
|
|
72
|
+
# @api private
|
|
73
|
+
def reset_uploaded_this_process_for_tests!
|
|
74
|
+
@upload_done_mutex.synchronize { @uploaded_this_process = false }
|
|
75
|
+
end
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
# Build a new Component if feature is enabled and dependencies met.
|
|
79
|
+
# @param settings [Configuration::Settings] Tracer settings
|
|
80
|
+
# @param agent_settings [Configuration::AgentSettings] Agent configuration
|
|
81
|
+
# @param logger [Logger] Logger instance
|
|
82
|
+
# @param telemetry [Core::Telemetry::Component, nil] Telemetry component for error reporting
|
|
83
|
+
# @return [Component, nil] Component instance or nil if not enabled/requirements not met
|
|
84
|
+
def self.build(settings, agent_settings, logger, telemetry: nil)
|
|
85
|
+
symdb_logger = SymbolDatabase::Logger.new(settings, logger)
|
|
86
|
+
|
|
87
|
+
unless settings.respond_to?(:symbol_database) && settings.symbol_database.enabled
|
|
88
|
+
symdb_logger.debug("symdb: symbol database upload not enabled, skipping")
|
|
89
|
+
return
|
|
90
|
+
end
|
|
91
|
+
|
|
92
|
+
# Symbol database requires MRI Ruby 2.6+.
|
|
93
|
+
# Configuration accessors (settings.symbol_database.*) remain available on all
|
|
94
|
+
# platforms — only the component (upload) is disabled on unsupported engines/versions.
|
|
95
|
+
# environment_supported? logs the specific reason (engine or version) internally.
|
|
96
|
+
return nil unless environment_supported?(symdb_logger)
|
|
97
|
+
|
|
98
|
+
# Requires remote config (unless force mode)
|
|
99
|
+
if !settings.remote&.enabled && !settings.symbol_database.internal.force_upload
|
|
100
|
+
symdb_logger.debug("symdb: remote config not available and force_upload not set, skipping")
|
|
101
|
+
return nil
|
|
102
|
+
end
|
|
103
|
+
|
|
104
|
+
new(settings, agent_settings, symdb_logger, telemetry: telemetry).tap do |component|
|
|
105
|
+
# Defer extraction if force upload mode — wait for app boot to complete
|
|
106
|
+
component.schedule_deferred_upload if settings.symbol_database.internal.force_upload
|
|
107
|
+
end
|
|
108
|
+
end
|
|
109
|
+
|
|
110
|
+
attr_reader :settings, :logger, :last_upload_time, :last_upload_scope_count, :upload_in_progress
|
|
111
|
+
|
|
112
|
+
# Initialize component.
|
|
113
|
+
# @param settings [Configuration::Settings] Tracer settings
|
|
114
|
+
# @param agent_settings [Configuration::AgentSettings] Agent configuration
|
|
115
|
+
# @param logger [Logger] Logger instance
|
|
116
|
+
# @param telemetry [Core::Telemetry::Component, nil] Telemetry component for error reporting
|
|
117
|
+
def initialize(settings, agent_settings, logger, telemetry: nil)
|
|
118
|
+
@settings = settings
|
|
119
|
+
@agent_settings = agent_settings
|
|
120
|
+
@logger = logger
|
|
121
|
+
@telemetry = telemetry
|
|
122
|
+
|
|
123
|
+
@extractor = Extractor.new(logger: logger, settings: settings)
|
|
124
|
+
@uploader = Uploader.new(settings: settings, agent_settings: agent_settings, logger: logger, telemetry: telemetry)
|
|
125
|
+
@scope_batcher = ScopeBatcher.new(@uploader, logger: logger)
|
|
126
|
+
|
|
127
|
+
@last_upload_time = nil
|
|
128
|
+
@last_upload_scope_count = nil
|
|
129
|
+
@mutex = Mutex.new
|
|
130
|
+
@upload_in_progress = false
|
|
131
|
+
@upload_in_progress_cv = ConditionVariable.new
|
|
132
|
+
@shutdown = false
|
|
133
|
+
|
|
134
|
+
# Per-instance scheduler state. The scheduler thread is started lazily
|
|
135
|
+
# on the first start_upload call.
|
|
136
|
+
@scheduler_mutex = Mutex.new
|
|
137
|
+
@scheduler_cv = ConditionVariable.new
|
|
138
|
+
@scheduled_at = nil
|
|
139
|
+
@scheduler_signaled = false
|
|
140
|
+
@scheduler_thread = nil
|
|
141
|
+
end
|
|
142
|
+
|
|
143
|
+
# Schedule a deferred upload that waits for app boot to complete.
|
|
144
|
+
#
|
|
145
|
+
# In Rails: registers ActiveSupport.on_load(:after_initialize). When the
|
|
146
|
+
# hook has already fired (e.g., this Component was built by a reconfigure
|
|
147
|
+
# after Rails finished initializing), the callback runs immediately.
|
|
148
|
+
#
|
|
149
|
+
# In non-Rails: triggers start_upload immediately.
|
|
150
|
+
#
|
|
151
|
+
# Each Component registers its own callback. Old Components that have
|
|
152
|
+
# been shut down short-circuit in start_upload via @shutdown.
|
|
153
|
+
# Cross-process deduplication is handled by the class-level
|
|
154
|
+
# uploaded_this_process? flag, not by guarding registration.
|
|
155
|
+
#
|
|
156
|
+
# @return [void]
|
|
157
|
+
def schedule_deferred_upload
|
|
158
|
+
if defined?(::ActiveSupport) && defined?(::Rails::Railtie)
|
|
159
|
+
# Capture self — on_load runs the block via instance_exec on the
|
|
160
|
+
# loaded object (Rails::Application), so a bare `start_upload`
|
|
161
|
+
# would resolve against it.
|
|
162
|
+
component = self
|
|
163
|
+
logger = @logger
|
|
164
|
+
::ActiveSupport.on_load(:after_initialize) do
|
|
165
|
+
# Only auto-trigger when Rails has eager-loaded application
|
|
166
|
+
# classes during initialization. In dev (eager_load=false)
|
|
167
|
+
# there is nothing complete to extract; the auto-deferred
|
|
168
|
+
# upload would race with explicit triggers and produce
|
|
169
|
+
# under-extracted uploads.
|
|
170
|
+
if defined?(::Rails) && ::Rails.application&.config&.eager_load # steep:ignore NoMethod
|
|
171
|
+
component.start_upload
|
|
172
|
+
else
|
|
173
|
+
logger.debug { "symdb: skipping auto-deferred upload (eager_load disabled)" }
|
|
174
|
+
end
|
|
175
|
+
end
|
|
176
|
+
else
|
|
177
|
+
start_upload
|
|
178
|
+
end
|
|
179
|
+
end
|
|
180
|
+
|
|
181
|
+
# Whether this component has been shut down.
|
|
182
|
+
# @return [Boolean]
|
|
183
|
+
def shutdown?
|
|
184
|
+
@scheduler_mutex.synchronize { @shutdown }
|
|
185
|
+
end
|
|
186
|
+
|
|
187
|
+
# Schedule symbol upload (triggered by remote config or force mode).
|
|
188
|
+
# The actual extraction is debounced by EXTRACT_DEBOUNCE_INTERVAL seconds —
|
|
189
|
+
# subsequent calls within the window restart the timer.
|
|
190
|
+
# Thread-safe: can be called concurrently from multiple remote config updates.
|
|
191
|
+
# @return [void]
|
|
192
|
+
def start_upload
|
|
193
|
+
return if Component.uploaded_this_process?
|
|
194
|
+
|
|
195
|
+
@scheduler_mutex.synchronize do
|
|
196
|
+
return if @shutdown
|
|
197
|
+
|
|
198
|
+
@scheduled_at = Datadog::Core::Utils::Time.get_time + EXTRACT_DEBOUNCE_INTERVAL
|
|
199
|
+
@scheduler_signaled = true
|
|
200
|
+
@scheduler_cv.signal
|
|
201
|
+
ensure_scheduler_thread
|
|
202
|
+
end
|
|
203
|
+
rescue => e
|
|
204
|
+
@logger.debug { "symdb: error scheduling upload: #{e.class}: #{e.message}" }
|
|
205
|
+
@telemetry&.report(e, description: 'symdb: error scheduling upload')
|
|
206
|
+
end
|
|
207
|
+
|
|
208
|
+
# Stop symbol upload (cancel the scheduler).
|
|
209
|
+
# Thread-safe: can be called concurrently from multiple remote config updates.
|
|
210
|
+
# @return [void]
|
|
211
|
+
def stop_upload
|
|
212
|
+
@scheduler_mutex.synchronize do
|
|
213
|
+
@scheduled_at = nil
|
|
214
|
+
@scheduler_signaled = true
|
|
215
|
+
@scheduler_cv.signal
|
|
216
|
+
end
|
|
217
|
+
end
|
|
218
|
+
|
|
219
|
+
# Block until any Component in this process has finished an extract+upload,
|
|
220
|
+
# or until the timeout elapses. Used by short-lived scripts that trigger
|
|
221
|
+
# an upload via force_upload and need to wait before exiting.
|
|
222
|
+
# @param timeout [Numeric] Maximum seconds to wait
|
|
223
|
+
# @return [Boolean] true if an upload completed; false on timeout
|
|
224
|
+
def wait_for_idle(timeout: 30)
|
|
225
|
+
deadline = Datadog::Core::Utils::Time.get_time + timeout
|
|
226
|
+
Component.upload_done_mutex.synchronize do
|
|
227
|
+
# Read @uploaded_this_process directly: we already hold
|
|
228
|
+
# Component.upload_done_mutex here, and uploaded_this_process?
|
|
229
|
+
# would try to re-acquire it (non-reentrant), deadlocking.
|
|
230
|
+
until Component.instance_variable_get(:@uploaded_this_process)
|
|
231
|
+
remaining = deadline - Datadog::Core::Utils::Time.get_time
|
|
232
|
+
return false if remaining <= 0
|
|
233
|
+
Component.upload_done_cv.wait(Component.upload_done_mutex, remaining)
|
|
234
|
+
end
|
|
235
|
+
end
|
|
236
|
+
true
|
|
237
|
+
end
|
|
238
|
+
|
|
239
|
+
# Shutdown component and cleanup resources.
|
|
240
|
+
# Cancels the per-instance scheduler so any pending debounced extraction
|
|
241
|
+
# is dropped. Waits for an in-flight extraction to complete before
|
|
242
|
+
# returning. Does not touch class-level state, so a sibling Component
|
|
243
|
+
# built after shutdown can still upload.
|
|
244
|
+
# @return [void]
|
|
245
|
+
def shutdown!
|
|
246
|
+
@scheduler_mutex.synchronize do
|
|
247
|
+
@shutdown = true
|
|
248
|
+
@scheduler_signaled = true
|
|
249
|
+
@scheduler_cv.signal
|
|
250
|
+
end
|
|
251
|
+
@scheduler_thread&.join(5)
|
|
252
|
+
@scheduler_thread = nil
|
|
253
|
+
|
|
254
|
+
@mutex.synchronize do
|
|
255
|
+
if @upload_in_progress
|
|
256
|
+
@upload_in_progress_cv.wait(@mutex, 5)
|
|
257
|
+
end
|
|
258
|
+
end
|
|
259
|
+
|
|
260
|
+
@scope_batcher.shutdown
|
|
261
|
+
end
|
|
262
|
+
|
|
263
|
+
private
|
|
264
|
+
|
|
265
|
+
# Check whether the runtime environment supports symbol database upload.
|
|
266
|
+
# Only MRI Ruby 2.6+ is supported. JRuby and TruffleRuby are not supported
|
|
267
|
+
# because ObjectSpace iteration and Method#source_location behave differently.
|
|
268
|
+
# Configuration accessors remain available on all platforms — this only gates
|
|
269
|
+
# the component (upload) itself.
|
|
270
|
+
# @param logger [Logger]
|
|
271
|
+
# @return [Boolean]
|
|
272
|
+
def self.environment_supported?(logger)
|
|
273
|
+
if RUBY_ENGINE != 'ruby'
|
|
274
|
+
logger.debug { "symdb: not supported on #{RUBY_ENGINE}, skipping" }
|
|
275
|
+
return false
|
|
276
|
+
end
|
|
277
|
+
if RUBY_VERSION < '2.6'
|
|
278
|
+
logger.debug { "symdb: requires Ruby 2.6+, running #{RUBY_VERSION}, skipping" }
|
|
279
|
+
return false
|
|
280
|
+
end
|
|
281
|
+
true
|
|
282
|
+
end
|
|
283
|
+
private_class_method :environment_supported?
|
|
284
|
+
|
|
285
|
+
# Start the scheduler thread if not already running.
|
|
286
|
+
# Must be called from within @scheduler_mutex.synchronize.
|
|
287
|
+
# @return [void]
|
|
288
|
+
def ensure_scheduler_thread
|
|
289
|
+
return if @scheduler_thread&.alive?
|
|
290
|
+
@scheduler_thread = Thread.new { scheduler_loop }
|
|
291
|
+
end
|
|
292
|
+
|
|
293
|
+
# Scheduler thread main loop. Waits for the debounce window to elapse,
|
|
294
|
+
# then runs extract_and_upload exactly once for this Component.
|
|
295
|
+
# @return [void]
|
|
296
|
+
def scheduler_loop
|
|
297
|
+
loop do
|
|
298
|
+
# should_fire = true means the debounce deadline elapsed without further
|
|
299
|
+
# signals; extract_and_upload runs once after the mutex is released.
|
|
300
|
+
should_fire = false
|
|
301
|
+
|
|
302
|
+
@scheduler_mutex.synchronize do
|
|
303
|
+
return if @shutdown
|
|
304
|
+
return if Component.uploaded_this_process?
|
|
305
|
+
|
|
306
|
+
# Copy to local so Steep narrows `Float?` to `Float` in the else branch.
|
|
307
|
+
# Steep does not track narrowing on instance variables across nil checks.
|
|
308
|
+
scheduled_at = @scheduled_at
|
|
309
|
+
if scheduled_at.nil?
|
|
310
|
+
# Nothing scheduled (e.g. stop_upload cleared it). Wait
|
|
311
|
+
# indefinitely for a signal, then re-evaluate on next loop.
|
|
312
|
+
@scheduler_signaled = false
|
|
313
|
+
@scheduler_cv.wait(@scheduler_mutex)
|
|
314
|
+
else
|
|
315
|
+
remaining = scheduled_at - Datadog::Core::Utils::Time.get_time
|
|
316
|
+
if remaining > 0
|
|
317
|
+
# Wait until the debounce deadline. Any signal (start_upload,
|
|
318
|
+
# stop_upload, shutdown!) wakes us early; we always re-loop
|
|
319
|
+
# and recompute rather than firing immediately on wake.
|
|
320
|
+
@scheduler_signaled = false
|
|
321
|
+
@scheduler_cv.wait(@scheduler_mutex, remaining)
|
|
322
|
+
else
|
|
323
|
+
# Deadline elapsed without further signal — fire after releasing the mutex.
|
|
324
|
+
should_fire = true
|
|
325
|
+
end
|
|
326
|
+
end
|
|
327
|
+
end
|
|
328
|
+
|
|
329
|
+
# `next` inside `synchronize` only exits the synchronize block — not the
|
|
330
|
+
# surrounding loop. Use an explicit flag so the loop only fires
|
|
331
|
+
# extract_and_upload when the debounce deadline has actually elapsed.
|
|
332
|
+
next unless should_fire
|
|
333
|
+
|
|
334
|
+
# Outside the mutex.
|
|
335
|
+
return if @shutdown
|
|
336
|
+
if Component.uploaded_this_process?
|
|
337
|
+
return
|
|
338
|
+
end
|
|
339
|
+
|
|
340
|
+
extract_and_upload
|
|
341
|
+
Component.mark_uploaded
|
|
342
|
+
return
|
|
343
|
+
end
|
|
344
|
+
rescue => e
|
|
345
|
+
@logger.debug { "symdb: scheduler error: #{e.class}: #{e.message}" }
|
|
346
|
+
@telemetry&.report(e, description: 'symdb: scheduler error')
|
|
347
|
+
end
|
|
348
|
+
|
|
349
|
+
# Extract symbols from all loaded modules and upload.
|
|
350
|
+
# @return [void]
|
|
351
|
+
def extract_and_upload
|
|
352
|
+
@mutex.synchronize { @upload_in_progress = true }
|
|
353
|
+
|
|
354
|
+
begin
|
|
355
|
+
@logger.trace { "symdb: starting extraction and upload" }
|
|
356
|
+
start_time = Datadog::Core::Utils::Time.get_time
|
|
357
|
+
|
|
358
|
+
# Extract symbols from all loaded modules grouped by source file.
|
|
359
|
+
# extract_all handles ObjectSpace iteration, filtering, and FQN-based nesting.
|
|
360
|
+
file_scopes = @extractor.extract_all
|
|
361
|
+
extracted_count = 0
|
|
362
|
+
file_scopes.each do |scope|
|
|
363
|
+
@scope_batcher.add_scope(scope)
|
|
364
|
+
extracted_count += 1
|
|
365
|
+
log_scope_tree(scope, 0)
|
|
366
|
+
end
|
|
367
|
+
|
|
368
|
+
@logger.debug do
|
|
369
|
+
extraction_duration = Datadog::Core::Utils::Time.get_time - start_time
|
|
370
|
+
targetable_count = count_targetable_methods(file_scopes)
|
|
371
|
+
"symdb: extracted #{extracted_count} scopes (#{targetable_count} methods with targetable lines) in #{'%.2f' % extraction_duration}s"
|
|
372
|
+
end
|
|
373
|
+
|
|
374
|
+
# Flush any remaining scopes (triggers upload)
|
|
375
|
+
@scope_batcher.flush
|
|
376
|
+
|
|
377
|
+
@last_upload_time = Datadog::Core::Utils::Time.now
|
|
378
|
+
@last_upload_scope_count = extracted_count
|
|
379
|
+
rescue => e
|
|
380
|
+
@logger.debug { "symdb: extraction error: #{e.class}: #{e.message}" }
|
|
381
|
+
@telemetry&.report(e, description: 'symdb: extraction error')
|
|
382
|
+
ensure
|
|
383
|
+
@mutex.synchronize do
|
|
384
|
+
@upload_in_progress = false
|
|
385
|
+
@upload_in_progress_cv.signal
|
|
386
|
+
end
|
|
387
|
+
end
|
|
388
|
+
end
|
|
389
|
+
|
|
390
|
+
def log_scope_tree(scope, depth)
|
|
391
|
+
indent = ' ' * depth
|
|
392
|
+
@logger.trace { "symdb: #{indent}#{scope.scope_type} #{scope.name}" }
|
|
393
|
+
scope.scopes&.each { |child| log_scope_tree(child, depth + 1) }
|
|
394
|
+
end
|
|
395
|
+
|
|
396
|
+
def count_targetable_methods(file_scopes)
|
|
397
|
+
count = 0
|
|
398
|
+
file_scopes.each do |file_scope|
|
|
399
|
+
file_scope.scopes&.each do |class_or_module|
|
|
400
|
+
class_or_module.scopes&.each do |method_scope|
|
|
401
|
+
count += 1 if method_scope.scope_type == 'METHOD' && method_scope.targetable_lines?
|
|
402
|
+
end
|
|
403
|
+
end
|
|
404
|
+
end
|
|
405
|
+
count
|
|
406
|
+
end
|
|
407
|
+
end
|
|
408
|
+
end
|
|
409
|
+
end
|
|
@@ -7,7 +7,7 @@ module Datadog
|
|
|
7
7
|
# Configuration settings for symbol database upload feature.
|
|
8
8
|
#
|
|
9
9
|
# Public environment variable:
|
|
10
|
-
# - DD_SYMBOL_DATABASE_UPLOAD_ENABLED (default:
|
|
10
|
+
# - DD_SYMBOL_DATABASE_UPLOAD_ENABLED (default: false) - Feature gate
|
|
11
11
|
#
|
|
12
12
|
# Extended into: Core::Configuration::Settings (via extend)
|
|
13
13
|
# Accessed as: Datadog.configuration.symbol_database.enabled
|
|
@@ -31,7 +31,7 @@ module Datadog
|
|
|
31
31
|
option :enabled do |o|
|
|
32
32
|
o.type :bool
|
|
33
33
|
o.env 'DD_SYMBOL_DATABASE_UPLOAD_ENABLED'
|
|
34
|
-
o.default
|
|
34
|
+
o.default false
|
|
35
35
|
end
|
|
36
36
|
|
|
37
37
|
# Settings in the 'internal' group are for internal Datadog
|