ddtrace 1.20.0 → 1.22.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (113) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +115 -1
  3. data/LICENSE-3rdparty.csv +1 -1
  4. data/bin/ddprofrb +15 -0
  5. data/bin/ddtracerb +3 -1
  6. data/ext/{ddtrace_profiling_loader/ddtrace_profiling_loader.c → datadog_profiling_loader/datadog_profiling_loader.c} +2 -2
  7. data/ext/{ddtrace_profiling_loader → datadog_profiling_loader}/extconf.rb +3 -3
  8. data/ext/{ddtrace_profiling_native_extension → datadog_profiling_native_extension}/collectors_cpu_and_wall_time_worker.c +238 -61
  9. data/ext/{ddtrace_profiling_native_extension → datadog_profiling_native_extension}/collectors_discrete_dynamic_sampler.c +145 -72
  10. data/ext/{ddtrace_profiling_native_extension → datadog_profiling_native_extension}/collectors_discrete_dynamic_sampler.h +17 -5
  11. data/ext/{ddtrace_profiling_native_extension → datadog_profiling_native_extension}/collectors_thread_context.c +97 -4
  12. data/ext/{ddtrace_profiling_native_extension → datadog_profiling_native_extension}/extconf.rb +2 -2
  13. data/ext/{ddtrace_profiling_native_extension → datadog_profiling_native_extension}/heap_recorder.c +45 -3
  14. data/ext/{ddtrace_profiling_native_extension → datadog_profiling_native_extension}/heap_recorder.h +7 -1
  15. data/ext/{ddtrace_profiling_native_extension → datadog_profiling_native_extension}/http_transport.c +15 -19
  16. data/ext/{ddtrace_profiling_native_extension → datadog_profiling_native_extension}/native_extension_helpers.rb +4 -4
  17. data/ext/{ddtrace_profiling_native_extension → datadog_profiling_native_extension}/private_vm_api_access.c +14 -0
  18. data/ext/{ddtrace_profiling_native_extension → datadog_profiling_native_extension}/private_vm_api_access.h +4 -0
  19. data/ext/{ddtrace_profiling_native_extension → datadog_profiling_native_extension}/profiling.c +1 -1
  20. data/ext/{ddtrace_profiling_native_extension → datadog_profiling_native_extension}/ruby_helpers.c +10 -0
  21. data/ext/{ddtrace_profiling_native_extension → datadog_profiling_native_extension}/ruby_helpers.h +2 -0
  22. data/ext/{ddtrace_profiling_native_extension → datadog_profiling_native_extension}/stack_recorder.c +7 -9
  23. data/lib/datadog/appsec/contrib/rack/request_middleware.rb +43 -13
  24. data/lib/datadog/appsec/event.rb +1 -1
  25. data/lib/datadog/auto_instrument.rb +3 -0
  26. data/lib/datadog/core/configuration/components.rb +7 -6
  27. data/lib/datadog/core/configuration/option.rb +8 -6
  28. data/lib/datadog/core/configuration/settings.rb +130 -63
  29. data/lib/datadog/core/configuration.rb +20 -4
  30. data/lib/datadog/core/diagnostics/environment_logger.rb +4 -3
  31. data/lib/datadog/core/environment/git.rb +25 -0
  32. data/lib/datadog/core/environment/identity.rb +18 -48
  33. data/lib/datadog/core/environment/platform.rb +7 -1
  34. data/lib/datadog/core/git/ext.rb +2 -23
  35. data/lib/datadog/core/remote/client/capabilities.rb +1 -1
  36. data/lib/datadog/core/remote/negotiation.rb +2 -2
  37. data/lib/datadog/core/remote/transport/http/config.rb +1 -1
  38. data/lib/datadog/core/remote/worker.rb +7 -4
  39. data/lib/datadog/core/telemetry/client.rb +18 -10
  40. data/lib/datadog/core/telemetry/emitter.rb +9 -13
  41. data/lib/datadog/core/telemetry/event.rb +247 -57
  42. data/lib/datadog/core/telemetry/ext.rb +1 -0
  43. data/lib/datadog/core/telemetry/heartbeat.rb +1 -3
  44. data/lib/datadog/core/telemetry/http/ext.rb +4 -1
  45. data/lib/datadog/core/telemetry/http/transport.rb +9 -4
  46. data/lib/datadog/core/telemetry/request.rb +59 -0
  47. data/lib/datadog/core/transport/ext.rb +2 -0
  48. data/lib/datadog/core/utils/url.rb +25 -0
  49. data/lib/datadog/profiling/collectors/code_provenance.rb +10 -4
  50. data/lib/datadog/profiling/collectors/cpu_and_wall_time_worker.rb +31 -0
  51. data/lib/datadog/profiling/collectors/info.rb +101 -0
  52. data/lib/datadog/profiling/component.rb +34 -28
  53. data/lib/datadog/profiling/exporter.rb +19 -5
  54. data/lib/datadog/profiling/ext.rb +2 -0
  55. data/lib/datadog/profiling/flush.rb +6 -3
  56. data/lib/datadog/profiling/http_transport.rb +5 -1
  57. data/lib/datadog/profiling/load_native_extension.rb +19 -6
  58. data/lib/datadog/profiling/native_extension.rb +1 -1
  59. data/lib/datadog/profiling/tag_builder.rb +5 -0
  60. data/lib/datadog/profiling/tasks/exec.rb +3 -3
  61. data/lib/datadog/profiling/tasks/help.rb +3 -3
  62. data/lib/datadog/profiling.rb +13 -2
  63. data/lib/datadog/tracing/contrib/action_mailer/events/deliver.rb +1 -1
  64. data/lib/datadog/tracing/contrib/active_record/configuration/resolver.rb +11 -4
  65. data/lib/datadog/tracing/contrib/concurrent_ruby/async_patch.rb +20 -0
  66. data/lib/datadog/tracing/contrib/concurrent_ruby/patcher.rb +11 -1
  67. data/lib/datadog/tracing/contrib/configurable.rb +1 -1
  68. data/lib/datadog/tracing/contrib/extensions.rb +6 -2
  69. data/lib/datadog/tracing/contrib/pg/instrumentation.rb +11 -4
  70. data/lib/datadog/tracing/sampling/matcher.rb +23 -3
  71. data/lib/datadog/tracing/sampling/rule.rb +7 -2
  72. data/lib/datadog/tracing/sampling/rule_sampler.rb +2 -0
  73. data/lib/datadog/tracing/trace_operation.rb +1 -2
  74. data/lib/datadog/tracing/transport/http.rb +1 -0
  75. data/lib/datadog/tracing/transport/trace_formatter.rb +31 -0
  76. data/lib/ddtrace/version.rb +1 -1
  77. metadata +55 -62
  78. data/ext/ddtrace_profiling_native_extension/pid_controller.c +0 -57
  79. data/ext/ddtrace_profiling_native_extension/pid_controller.h +0 -45
  80. data/lib/datadog/core/telemetry/collector.rb +0 -250
  81. data/lib/datadog/core/telemetry/v1/app_event.rb +0 -59
  82. data/lib/datadog/core/telemetry/v1/application.rb +0 -92
  83. data/lib/datadog/core/telemetry/v1/configuration.rb +0 -25
  84. data/lib/datadog/core/telemetry/v1/dependency.rb +0 -43
  85. data/lib/datadog/core/telemetry/v1/host.rb +0 -59
  86. data/lib/datadog/core/telemetry/v1/install_signature.rb +0 -38
  87. data/lib/datadog/core/telemetry/v1/integration.rb +0 -64
  88. data/lib/datadog/core/telemetry/v1/product.rb +0 -36
  89. data/lib/datadog/core/telemetry/v1/telemetry_request.rb +0 -106
  90. data/lib/datadog/core/telemetry/v2/app_client_configuration_change.rb +0 -41
  91. data/lib/datadog/core/telemetry/v2/request.rb +0 -29
  92. data/lib/datadog/profiling/diagnostics/environment_logger.rb +0 -39
  93. /data/ext/{ddtrace_profiling_native_extension → datadog_profiling_native_extension}/NativeExtensionDesign.md +0 -0
  94. /data/ext/{ddtrace_profiling_native_extension → datadog_profiling_native_extension}/clock_id.h +0 -0
  95. /data/ext/{ddtrace_profiling_native_extension → datadog_profiling_native_extension}/clock_id_from_pthread.c +0 -0
  96. /data/ext/{ddtrace_profiling_native_extension → datadog_profiling_native_extension}/clock_id_noop.c +0 -0
  97. /data/ext/{ddtrace_profiling_native_extension → datadog_profiling_native_extension}/collectors_dynamic_sampling_rate.c +0 -0
  98. /data/ext/{ddtrace_profiling_native_extension → datadog_profiling_native_extension}/collectors_dynamic_sampling_rate.h +0 -0
  99. /data/ext/{ddtrace_profiling_native_extension → datadog_profiling_native_extension}/collectors_gc_profiling_helper.c +0 -0
  100. /data/ext/{ddtrace_profiling_native_extension → datadog_profiling_native_extension}/collectors_gc_profiling_helper.h +0 -0
  101. /data/ext/{ddtrace_profiling_native_extension → datadog_profiling_native_extension}/collectors_idle_sampling_helper.c +0 -0
  102. /data/ext/{ddtrace_profiling_native_extension → datadog_profiling_native_extension}/collectors_idle_sampling_helper.h +0 -0
  103. /data/ext/{ddtrace_profiling_native_extension → datadog_profiling_native_extension}/collectors_stack.c +0 -0
  104. /data/ext/{ddtrace_profiling_native_extension → datadog_profiling_native_extension}/collectors_stack.h +0 -0
  105. /data/ext/{ddtrace_profiling_native_extension → datadog_profiling_native_extension}/collectors_thread_context.h +0 -0
  106. /data/ext/{ddtrace_profiling_native_extension → datadog_profiling_native_extension}/helpers.h +0 -0
  107. /data/ext/{ddtrace_profiling_native_extension → datadog_profiling_native_extension}/libdatadog_helpers.c +0 -0
  108. /data/ext/{ddtrace_profiling_native_extension → datadog_profiling_native_extension}/libdatadog_helpers.h +0 -0
  109. /data/ext/{ddtrace_profiling_native_extension → datadog_profiling_native_extension}/setup_signal_handler.c +0 -0
  110. /data/ext/{ddtrace_profiling_native_extension → datadog_profiling_native_extension}/setup_signal_handler.h +0 -0
  111. /data/ext/{ddtrace_profiling_native_extension → datadog_profiling_native_extension}/stack_recorder.h +0 -0
  112. /data/ext/{ddtrace_profiling_native_extension → datadog_profiling_native_extension}/time_helpers.c +0 -0
  113. /data/ext/{ddtrace_profiling_native_extension → datadog_profiling_native_extension}/time_helpers.h +0 -0
@@ -22,6 +22,7 @@ module Datadog
22
22
  # **NOTE**: This should only be used for testing; disabling the dynamic sampling rate will increase the
23
23
  # profiler overhead!
24
24
  dynamic_sampling_rate_enabled: true,
25
+ skip_idle_samples_for_testing: false,
25
26
  idle_sampling_helper: IdleSamplingHelper.new
26
27
  )
27
28
  unless dynamic_sampling_rate_enabled
@@ -39,11 +40,14 @@ module Datadog
39
40
  dynamic_sampling_rate_enabled,
40
41
  dynamic_sampling_rate_overhead_target_percentage,
41
42
  allocation_profiling_enabled,
43
+ skip_idle_samples_for_testing,
42
44
  )
43
45
  @worker_thread = nil
44
46
  @failure_exception = nil
45
47
  @start_stop_mutex = Mutex.new
46
48
  @idle_sampling_helper = idle_sampling_helper
49
+ @wait_until_running_mutex = Mutex.new
50
+ @wait_until_running_condition = ConditionVariable.new
47
51
  end
48
52
 
49
53
  def start(on_failure_proc: nil)
@@ -100,6 +104,33 @@ module Datadog
100
104
  def stats
101
105
  self.class._native_stats(self)
102
106
  end
107
+
108
+ def stats_and_reset_not_thread_safe
109
+ stats = self.stats
110
+ self.class._native_stats_reset_not_thread_safe(self)
111
+ stats
112
+ end
113
+
114
+ # Useful for testing, to e.g. make sure the profiler is running before we start running some code we want to observe
115
+ def wait_until_running(timeout_seconds: 5)
116
+ @wait_until_running_mutex.synchronize do
117
+ return true if self.class._native_is_running?(self)
118
+
119
+ @wait_until_running_condition.wait(@wait_until_running_mutex, timeout_seconds)
120
+
121
+ if self.class._native_is_running?(self)
122
+ true
123
+ else
124
+ raise "Timeout waiting for #{self.class.name} to start (waited for #{timeout_seconds} seconds)"
125
+ end
126
+ end
127
+ end
128
+
129
+ private
130
+
131
+ def signal_running
132
+ @wait_until_running_mutex.synchronize { @wait_until_running_condition.broadcast }
133
+ end
103
134
  end
104
135
  end
105
136
  end
@@ -0,0 +1,101 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'set'
4
+ require 'time'
5
+
6
+ module Datadog
7
+ module Profiling
8
+ module Collectors
9
+ # Collects information of relevance for profiler. This will get sent alongside
10
+ # the profile and show up in the UI or potentially influence processing in some way.
11
+ #
12
+ # Information is currently collected and frozen at construction time. A full collector
13
+ # could be seen as overkill for this case but it allows us to centralize information
14
+ # gathering and easily support more flexible/dynamic info collection in the future.
15
+ class Info
16
+ def initialize(settings)
17
+ @profiler_info = nil
18
+ @info = {
19
+ platform: collect_platform_info,
20
+ runtime: collect_runtime_info,
21
+ application: collect_application_info(settings),
22
+ profiler: collect_profiler_info(settings),
23
+ }.freeze
24
+ end
25
+
26
+ attr_reader :info
27
+
28
+ private
29
+
30
+ # Instead of trying to figure out real process start time by checking
31
+ # /proc or some other complex/non-portable way, approximate start time
32
+ # by time of requirement of this file.
33
+ START_TIME = Time.now.utc.freeze
34
+
35
+ def collect_platform_info
36
+ @platform_info ||= {
37
+ container_id: Datadog::Core::Environment::Container.container_id,
38
+ hostname: Datadog::Core::Environment::Platform.hostname,
39
+ kernel_name: Datadog::Core::Environment::Platform.kernel_name,
40
+ kernel_release: Datadog::Core::Environment::Platform.kernel_release,
41
+ kernel_version: Datadog::Core::Environment::Platform.kernel_version
42
+ }.freeze
43
+ end
44
+
45
+ def collect_runtime_info
46
+ @runtime_info ||= {
47
+ engine: Datadog::Core::Environment::Identity.lang_engine,
48
+ version: Datadog::Core::Environment::Identity.lang_version,
49
+ platform: Datadog::Core::Environment::Identity.lang_platform,
50
+ }.freeze
51
+ end
52
+
53
+ def collect_application_info(settings)
54
+ @application_info ||= {
55
+ start_time: START_TIME.iso8601,
56
+ env: settings.env,
57
+ service: settings.service,
58
+ version: settings.version,
59
+ }.freeze
60
+ end
61
+
62
+ def collect_profiler_info(settings)
63
+ unless @profiler_info
64
+ lib_datadog_gem = ::Gem.loaded_specs['libdatadog']
65
+ @profiler_info = {
66
+ version: Datadog::Core::Environment::Identity.tracer_version,
67
+ libdatadog: "#{lib_datadog_gem.version}-#{lib_datadog_gem.platform}",
68
+ settings: collect_settings_recursively(settings.profiling),
69
+ }.freeze
70
+ end
71
+ @profiler_info
72
+ end
73
+
74
+ # The settings/option model isn't directly serializable because
75
+ # of subsettings and options that link to full blown custom object
76
+ # instances without proper serialization.
77
+ # This method navigates a settings object recursively, converting
78
+ # it into more basic types that are trivially convertible to JSON.
79
+ def collect_settings_recursively(v)
80
+ v = v.options_hash if v.respond_to?(:options_hash)
81
+
82
+ if v.nil? || v.is_a?(Symbol) || v.is_a?(Numeric) || v.is_a?(String) || v.equal?(true) || v.equal?(false)
83
+ Core::Utils::SafeDup.frozen_or_dup(v)
84
+ elsif v.is_a?(Hash)
85
+ collected_hash = v.each_with_object({}) do |(key, value), hash|
86
+ collected_value = collect_settings_recursively(value)
87
+ hash[key] = collected_value
88
+ end
89
+ collected_hash.freeze
90
+ elsif v.is_a?(Enumerable)
91
+ collected_list = v
92
+ .map { |value| collect_settings_recursively(value) }
93
+ collected_list.freeze
94
+ else
95
+ v.inspect
96
+ end
97
+ end
98
+ end
99
+ end
100
+ end
101
+ end
@@ -8,11 +8,7 @@ module Datadog
8
8
  # * Code Hotspots panel in the trace viewer, as well as scoping a profile down to a span
9
9
  # * Endpoint aggregation in the profiler UX, including normalization (resource per endpoint call)
10
10
  def self.build_profiler_component(settings:, agent_settings:, optional_tracer:) # rubocop:disable Metrics/MethodLength
11
- require_relative '../profiling/diagnostics/environment_logger'
12
-
13
- Profiling::Diagnostics::EnvironmentLogger.collect_and_log!
14
-
15
- return unless settings.profiling.enabled
11
+ return [nil, { profiling_enabled: false }] unless settings.profiling.enabled
16
12
 
17
13
  # Workaround for weird dependency direction: the Core::Configuration::Components class currently has a
18
14
  # dependency on individual products, in this case the Profiler.
@@ -32,7 +28,8 @@ module Datadog
32
28
  # done, then profiling may not be loaded, and thus to avoid this issue we do a require here (which is a
33
29
  # no-op if profiling is already loaded).
34
30
  require_relative '../profiling'
35
- return unless Profiling.supported?
31
+
32
+ return [nil, { profiling_enabled: false }] unless Profiling.supported?
36
33
 
37
34
  # Activate forking extensions
38
35
  Profiling::Tasks::Setup.new.run
@@ -40,7 +37,7 @@ module Datadog
40
37
  # NOTE: Please update the Initialization section of ProfilingDevelopment.md with any changes to this method
41
38
 
42
39
  no_signals_workaround_enabled = no_signals_workaround_enabled?(settings)
43
- timeline_enabled = settings.profiling.advanced.experimental_timeline_enabled
40
+ timeline_enabled = settings.profiling.advanced.timeline_enabled
44
41
  allocation_profiling_enabled = enable_allocation_profiling?(settings)
45
42
  heap_sample_every = get_heap_sample_every(settings)
46
43
  heap_profiling_enabled = enable_heap_profiling?(settings, allocation_profiling_enabled, heap_sample_every)
@@ -72,11 +69,11 @@ module Datadog
72
69
  heap_sample_every: heap_sample_every,
73
70
  }.freeze
74
71
 
75
- exporter = build_profiler_exporter(settings, recorder, internal_metadata: internal_metadata)
72
+ exporter = build_profiler_exporter(settings, recorder, worker, internal_metadata: internal_metadata)
76
73
  transport = build_profiler_transport(settings, agent_settings)
77
74
  scheduler = Profiling::Scheduler.new(exporter: exporter, transport: transport, interval: upload_period_seconds)
78
75
 
79
- Profiling::Profiler.new(worker: worker, scheduler: scheduler)
76
+ [Profiling::Profiler.new(worker: worker, scheduler: scheduler), { profiling_enabled: true }]
80
77
  end
81
78
 
82
79
  private_class_method def self.build_thread_context_collector(settings, recorder, optional_tracer, timeline_enabled)
@@ -89,12 +86,15 @@ module Datadog
89
86
  )
90
87
  end
91
88
 
92
- private_class_method def self.build_profiler_exporter(settings, recorder, internal_metadata:)
89
+ private_class_method def self.build_profiler_exporter(settings, recorder, worker, internal_metadata:)
90
+ info_collector = Profiling::Collectors::Info.new(settings)
93
91
  code_provenance_collector =
94
92
  (Profiling::Collectors::CodeProvenance.new if settings.profiling.advanced.code_provenance_enabled)
95
93
 
96
94
  Profiling::Exporter.new(
97
95
  pprof_recorder: recorder,
96
+ worker: worker,
97
+ info_collector: info_collector,
98
98
  code_provenance_collector: code_provenance_collector,
99
99
  internal_metadata: internal_metadata,
100
100
  )
@@ -111,19 +111,30 @@ module Datadog
111
111
  end
112
112
 
113
113
  private_class_method def self.enable_gc_profiling?(settings)
114
- # See comments on the setting definition for more context on why it exists.
115
- if settings.profiling.advanced.force_enable_gc_profiling
116
- if Gem::Version.new(RUBY_VERSION) >= Gem::Version.new('3')
117
- Datadog.logger.debug(
118
- 'Profiling time/resources spent in Garbage Collection force enabled. Do not use Ractors in combination ' \
119
- 'with this option as profiles will be incomplete.'
120
- )
121
- end
114
+ return false unless settings.profiling.advanced.gc_enabled
122
115
 
123
- true
124
- else
125
- false
116
+ # SEVERE - Only with Ractors
117
+ # On Ruby versions 3.0 (all), 3.1.0 to 3.1.3, and 3.2.0 to 3.2.2 gc profiling can trigger a VM bug
118
+ # that causes a segmentation fault during garbage collection of Ractors
119
+ # (https://bugs.ruby-lang.org/issues/18464). We don't allow enabling gc profiling on such Rubies.
120
+ # This bug is fixed on Ruby versions 3.1.4, 3.2.3 and 3.3.0.
121
+ if RUBY_VERSION.start_with?('3.0.') ||
122
+ (RUBY_VERSION.start_with?('3.1.') && RUBY_VERSION < '3.1.4') ||
123
+ (RUBY_VERSION.start_with?('3.2.') && RUBY_VERSION < '3.2.3')
124
+ Datadog.logger.warn(
125
+ "Current Ruby version (#{RUBY_VERSION}) has a VM bug where enabling GC profiling would cause "\
126
+ 'crashes (https://bugs.ruby-lang.org/issues/18464). GC profiling has been disabled.'
127
+ )
128
+ return false
129
+ elsif RUBY_VERSION.start_with?('3.')
130
+ Datadog.logger.debug(
131
+ 'In all known versions of Ruby 3.x, using Ractors may result in GC profiling unexpectedly ' \
132
+ 'stopping (https://bugs.ruby-lang.org/issues/19112). Note that this stop has no impact in your ' \
133
+ 'application stability or performance. This does not happen if Ractors are not used.'
134
+ )
126
135
  end
136
+
137
+ true
127
138
  end
128
139
 
129
140
  private_class_method def self.get_heap_sample_every(settings)
@@ -135,10 +146,7 @@ module Datadog
135
146
  end
136
147
 
137
148
  private_class_method def self.enable_allocation_profiling?(settings)
138
- unless settings.profiling.advanced.experimental_allocation_enabled
139
- # Allocation profiling disabled, short-circuit out
140
- return false
141
- end
149
+ return false unless settings.profiling.allocation_enabled
142
150
 
143
151
  # Allocation sampling is safe and supported on Ruby 2.x, but has a few caveats on Ruby 3.x.
144
152
 
@@ -179,9 +187,7 @@ module Datadog
179
187
  )
180
188
  end
181
189
 
182
- Datadog.logger.warn(
183
- 'Enabled experimental allocation profiling. This is experimental, not recommended, and will increase overhead!'
184
- )
190
+ Datadog.logger.debug('Enabled allocation profiling')
185
191
 
186
192
  true
187
193
  end
@@ -23,31 +23,39 @@ module Datadog
23
23
  :time_provider,
24
24
  :last_flush_finish_at,
25
25
  :created_at,
26
- :internal_metadata
26
+ :internal_metadata,
27
+ :info_json
27
28
 
28
29
  public
29
30
 
30
31
  def initialize(
31
32
  pprof_recorder:,
33
+ worker:,
34
+ info_collector:,
32
35
  code_provenance_collector:,
33
36
  internal_metadata:,
34
37
  minimum_duration_seconds: PROFILE_DURATION_THRESHOLD_SECONDS,
35
38
  time_provider: Time
36
39
  )
37
40
  @pprof_recorder = pprof_recorder
41
+ @worker = worker
38
42
  @code_provenance_collector = code_provenance_collector
39
43
  @minimum_duration_seconds = minimum_duration_seconds
40
44
  @time_provider = time_provider
41
45
  @last_flush_finish_at = nil
42
46
  @created_at = time_provider.now.utc
43
47
  @internal_metadata = internal_metadata
48
+ # NOTE: At the time of this comment collected info does not change over time so we'll hardcode
49
+ # it on startup to prevent serializing the same info on every flush.
50
+ @info_json = JSON.fast_generate(info_collector.info).freeze
44
51
  end
45
52
 
46
53
  def flush
47
- start, finish, uncompressed_pprof = pprof_recorder.serialize
54
+ worker_stats = @worker.stats_and_reset_not_thread_safe
55
+ start, finish, compressed_pprof = pprof_recorder.serialize
48
56
  @last_flush_finish_at = finish
49
57
 
50
- return if uncompressed_pprof.nil? # We don't want to report empty profiles
58
+ return if compressed_pprof.nil? # We don't want to report empty profiles
51
59
 
52
60
  if duration_below_threshold?(start, finish)
53
61
  Datadog.logger.debug('Skipped exporting profiling events as profile duration is below minimum')
@@ -60,11 +68,17 @@ module Datadog
60
68
  start: start,
61
69
  finish: finish,
62
70
  pprof_file_name: Datadog::Profiling::Ext::Transport::HTTP::PPROF_DEFAULT_FILENAME,
63
- pprof_data: uncompressed_pprof.to_s,
71
+ pprof_data: compressed_pprof.to_s,
64
72
  code_provenance_file_name: Datadog::Profiling::Ext::Transport::HTTP::CODE_PROVENANCE_FILENAME,
65
73
  code_provenance_data: uncompressed_code_provenance,
66
74
  tags_as_array: Datadog::Profiling::TagBuilder.call(settings: Datadog.configuration).to_a,
67
- internal_metadata: internal_metadata,
75
+ internal_metadata: internal_metadata.merge(
76
+ {
77
+ worker_stats: worker_stats,
78
+ gc: GC.stat,
79
+ }
80
+ ),
81
+ info_json: info_json,
68
82
  )
69
83
  end
70
84
 
@@ -23,6 +23,8 @@ module Datadog
23
23
  FORM_FIELD_TAG_RUNTIME_VERSION = 'runtime_version'
24
24
  FORM_FIELD_TAG_SERVICE = 'service'
25
25
  FORM_FIELD_TAG_VERSION = 'version'
26
+ TAG_GIT_REPOSITORY_URL = 'git.repository_url'
27
+ TAG_GIT_COMMIT_SHA = 'git.commit.sha'
26
28
 
27
29
  PPROF_DEFAULT_FILENAME = 'rubyprofile.pprof'
28
30
  CODE_PROVENANCE_FILENAME = 'code-provenance.json'
@@ -14,7 +14,8 @@ module Datadog
14
14
  :code_provenance_file_name,
15
15
  :code_provenance_data, # gzipped json bytes
16
16
  :tags_as_array,
17
- :internal_metadata_json
17
+ :internal_metadata_json,
18
+ :info_json
18
19
 
19
20
  def initialize(
20
21
  start:,
@@ -24,7 +25,8 @@ module Datadog
24
25
  code_provenance_file_name:,
25
26
  code_provenance_data:,
26
27
  tags_as_array:,
27
- internal_metadata:
28
+ internal_metadata:,
29
+ info_json:
28
30
  )
29
31
  @start = start
30
32
  @finish = finish
@@ -33,7 +35,8 @@ module Datadog
33
35
  @code_provenance_file_name = code_provenance_file_name
34
36
  @code_provenance_data = code_provenance_data
35
37
  @tags_as_array = tags_as_array
36
- @internal_metadata_json = JSON.fast_generate(internal_metadata.map { |k, v| [k, v.to_s] }.to_h)
38
+ @internal_metadata_json = JSON.fast_generate(internal_metadata)
39
+ @info_json = info_json
37
40
  end
38
41
  end
39
42
  end
@@ -43,6 +43,8 @@ module Datadog
43
43
 
44
44
  tags_as_array: flush.tags_as_array,
45
45
  internal_metadata_json: flush.internal_metadata_json,
46
+
47
+ info_json: flush.info_json
46
48
  )
47
49
 
48
50
  if status == :ok
@@ -117,7 +119,8 @@ module Datadog
117
119
  code_provenance_file_name:,
118
120
  code_provenance_data:,
119
121
  tags_as_array:,
120
- internal_metadata_json:
122
+ internal_metadata_json:,
123
+ info_json:
121
124
  )
122
125
  self.class._native_do_export(
123
126
  exporter_configuration,
@@ -132,6 +135,7 @@ module Datadog
132
135
  code_provenance_data,
133
136
  tags_as_array,
134
137
  internal_metadata_json,
138
+ info_json,
135
139
  )
136
140
  end
137
141
 
@@ -1,24 +1,37 @@
1
1
  # This file is used to load the profiling native extension. It works in two steps:
2
2
  #
3
- # 1. Load the ddtrace_profiling_loader extension. This extension will be used to load the actual extension, but in
4
- # a special way that avoids exposing native-level code symbols. See `ddtrace_profiling_loader.c` for more details.
3
+ # 1. Load the datadog_profiling_loader extension. This extension will be used to load the actual extension, but in
4
+ # a special way that avoids exposing native-level code symbols. See `datadog_profiling_loader.c` for more details.
5
5
  #
6
- # 2. Use the Datadog::Profiling::Loader exposed by the ddtrace_profiling_loader extension to load the actual
6
+ # 2. Use the Datadog::Profiling::Loader exposed by the datadog_profiling_loader extension to load the actual
7
7
  # profiling native extension.
8
8
  #
9
9
  # All code on this file is on-purpose at the top-level; this makes it so this file is executed only once,
10
10
  # the first time it gets required, to avoid any issues with the native extension being initialized more than once.
11
11
 
12
12
  begin
13
- require "ddtrace_profiling_loader.#{RUBY_VERSION}_#{RUBY_PLATFORM}"
13
+ require "datadog_profiling_loader.#{RUBY_VERSION}_#{RUBY_PLATFORM}"
14
14
  rescue LoadError => e
15
15
  raise LoadError,
16
16
  'Failed to load the profiling loader extension. To fix this, please remove and then reinstall ddtrace ' \
17
17
  "(Details: #{e.message})"
18
18
  end
19
19
 
20
- extension_name = "ddtrace_profiling_native_extension.#{RUBY_VERSION}_#{RUBY_PLATFORM}"
21
- full_file_path = "#{__dir__}/../../#{extension_name}.#{RbConfig::CONFIG['DLEXT']}"
20
+ extension_name = "datadog_profiling_native_extension.#{RUBY_VERSION}_#{RUBY_PLATFORM}"
21
+ file_name = "#{extension_name}.#{RbConfig::CONFIG['DLEXT']}"
22
+ full_file_path = "#{__dir__}/../../#{file_name}"
23
+
24
+ unless File.exist?(full_file_path)
25
+ extension_dir = Gem.loaded_specs['ddtrace'].extension_dir
26
+ candidate_path = "#{extension_dir}/#{file_name}"
27
+ if File.exist?(candidate_path)
28
+ full_file_path = candidate_path
29
+ else # rubocop:disable Style/EmptyElse
30
+ # We found none of the files. This is unexpected. Let's go ahead anyway, the error is going to be reported further
31
+ # down anyway.
32
+ end
33
+ end
34
+
22
35
  init_function_name = "Init_#{extension_name.split('.').first}"
23
36
 
24
37
  status, result = Datadog::Profiling::Loader._native_load(full_file_path, init_function_name)
@@ -3,7 +3,7 @@
3
3
  module Datadog
4
4
  module Profiling
5
5
  # This module contains classes and methods which are implemented using native code in the
6
- # ext/ddtrace_profiling_native_extension folder, as well as some Ruby-level utilities that don't make sense to
6
+ # ext/datadog_profiling_native_extension folder, as well as some Ruby-level utilities that don't make sense to
7
7
  # write using C
8
8
  module NativeExtension
9
9
  private_class_method def self.working?
@@ -1,6 +1,7 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  require_relative '../core/utils'
4
+ require_relative '../core/environment/git'
4
5
 
5
6
  module Datadog
6
7
  module Profiling
@@ -23,6 +24,8 @@ module Datadog
23
24
  runtime_id: Core::Environment::Identity.id,
24
25
  runtime_platform: Core::Environment::Identity.lang_platform,
25
26
  runtime_version: Core::Environment::Identity.lang_version,
27
+ git_repository_url: Core::Environment::Git.git_repository_url,
28
+ git_commit_sha: Core::Environment::Git.git_commit_sha,
26
29
  # User-provided tags
27
30
  user_tags: settings.tags
28
31
  )
@@ -42,6 +45,8 @@ module Datadog
42
45
  tags[FORM_FIELD_TAG_ENV] = env if env
43
46
  tags[FORM_FIELD_TAG_SERVICE] = service if service
44
47
  tags[FORM_FIELD_TAG_VERSION] = version if version
48
+ tags[TAG_GIT_REPOSITORY_URL] = git_repository_url if git_repository_url
49
+ tags[TAG_GIT_COMMIT_SHA] = git_commit_sha if git_commit_sha
45
50
 
46
51
  # Make sure everything is an utf-8 string, to avoid encoding issues in native code/libddprof/further downstream
47
52
  user_tags.merge(tags).map do |key, value|
@@ -1,7 +1,7 @@
1
1
  module Datadog
2
2
  module Profiling
3
3
  module Tasks
4
- # Wraps command with Datadog tracing
4
+ # Wraps command with Datadog profiling
5
5
  class Exec
6
6
  attr_reader :args
7
7
 
@@ -36,10 +36,10 @@ module Datadog
36
36
  def exec_with_error_handling(args)
37
37
  Kernel.exec(*args)
38
38
  rescue Errno::ENOENT => e
39
- Kernel.warn "ddtracerb exec failed: #{e.class.name} #{e.message} (command was '#{args.join(' ')}')"
39
+ Kernel.warn "ddprofrb exec failed: #{e.class.name} #{e.message} (command was '#{args.join(' ')}')"
40
40
  Kernel.exit 127
41
41
  rescue Errno::EACCES, Errno::ENOEXEC => e
42
- Kernel.warn "ddtracerb exec failed: #{e.class.name} #{e.message} (command was '#{args.join(' ')}')"
42
+ Kernel.warn "ddprofrb exec failed: #{e.class.name} #{e.message} (command was '#{args.join(' ')}')"
43
43
  Kernel.exit 126
44
44
  end
45
45
  end
@@ -1,12 +1,12 @@
1
1
  module Datadog
2
2
  module Profiling
3
3
  module Tasks
4
- # Prints help message for usage of `ddtrace`
4
+ # Prints help message for usage of `ddprofrb`
5
5
  class Help
6
6
  def run
7
7
  puts %(
8
- Usage: ddtracerb [command] [arguments]
9
- exec [command]: Executes command with tracing & profiling preloaded.
8
+ Usage: ddprofrb [command] [arguments]
9
+ exec [command]: Executes command with profiling preloaded.
10
10
  help: Prints this help message.
11
11
  )
12
12
  end
@@ -63,6 +63,17 @@ module Datadog
63
63
  !!(profiler.send(:scheduler).running? if profiler)
64
64
  end
65
65
 
66
+ def self.wait_until_running(timeout_seconds: 5)
67
+ profiler = Datadog.send(:components).profiler
68
+ if profiler
69
+ # Use .send(...) to avoid exposing the attr_reader as an API to the outside
70
+ worker = profiler.send(:worker)
71
+ worker.wait_until_running(timeout_seconds: timeout_seconds)
72
+ else
73
+ raise 'Profiler not enabled or available'
74
+ end
75
+ end
76
+
66
77
  private_class_method def self.replace_noop_allocation_count
67
78
  def self.allocation_count # rubocop:disable Lint/NestedMethodDefinition (On purpose!)
68
79
  Datadog::Profiling::Collectors::CpuAndWallTimeWorker._native_allocation_count
@@ -77,7 +88,7 @@ module Datadog
77
88
 
78
89
  private_class_method def self.try_reading_skipped_reason_file(file_api = File)
79
90
  # This file, if it exists, is recorded by extconf.rb during compilation of the native extension
80
- skipped_reason_file = "#{__dir__}/../../ext/ddtrace_profiling_native_extension/skipped_reason.txt"
91
+ skipped_reason_file = "#{__dir__}/../../ext/datadog_profiling_native_extension/skipped_reason.txt"
81
92
 
82
93
  begin
83
94
  return unless file_api.exist?(skipped_reason_file)
@@ -123,13 +134,13 @@ module Datadog
123
134
  return false unless supported?
124
135
 
125
136
  require_relative 'profiling/ext/forking'
137
+ require_relative 'profiling/collectors/info'
126
138
  require_relative 'profiling/collectors/code_provenance'
127
139
  require_relative 'profiling/collectors/cpu_and_wall_time_worker'
128
140
  require_relative 'profiling/collectors/dynamic_sampling_rate'
129
141
  require_relative 'profiling/collectors/idle_sampling_helper'
130
142
  require_relative 'profiling/collectors/stack'
131
143
  require_relative 'profiling/collectors/thread_context'
132
- require_relative 'profiling/diagnostics/environment_logger'
133
144
  require_relative 'profiling/stack_recorder'
134
145
  require_relative 'profiling/exporter'
135
146
  require_relative 'profiling/flush'
@@ -37,7 +37,7 @@ module Datadog
37
37
 
38
38
  span.set_tag(Tracing::Metadata::Ext::TAG_OPERATION, Ext::TAG_OPERATION_DELIVER)
39
39
 
40
- # Since email date can contain PII we disable by default
40
+ # Since email data can contain PII we disable by default
41
41
  # Some of these fields can be either strings or arrays, so we try to normalize
42
42
  # https://github.com/rails/rails/blob/18707ab17fa492eb25ad2e8f9818a320dc20b823/actionmailer/lib/action_mailer/base.rb#L742-L754
43
43
  if configuration[:email_data] == true
@@ -65,9 +65,14 @@ module Datadog
65
65
 
66
66
  config
67
67
  rescue => e
68
+ # Resolving a valid database configuration should not raise an exception,
69
+ # but if it does, it can be due to adding a broken pattern match prior to this call.
70
+ #
71
+ # `db_config` input may contain sensitive information such as passwords,
72
+ # hence provide a succinct summary for the error logging.
68
73
  Datadog.logger.error(
69
- "Failed to resolve ActiveRecord configuration key #{db_config.inspect}. " \
70
- "Cause: #{e.class.name} #{e.message} Source: #{Array(e.backtrace).first}"
74
+ 'Failed to resolve ActiveRecord database configuration. '\
75
+ "Cause: #{e.class.name} Source: #{Array(e.backtrace).first}"
71
76
  )
72
77
 
73
78
  nil
@@ -85,9 +90,11 @@ module Datadog
85
90
  normalized
86
91
  rescue => e
87
92
  Datadog.logger.error(
88
- "Failed to resolve ActiveRecord configuration key #{matcher.inspect}. " \
89
- "Cause: #{e.class.name} #{e.message} Source: #{Array(e.backtrace).first}"
93
+ "Failed to resolve key #{matcher.inspect}. " \
94
+ "Cause: #{e.class.name} Source: #{Array(e.backtrace).first}"
90
95
  )
96
+
97
+ nil
91
98
  end
92
99
 
93
100
  #
@@ -0,0 +1,20 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative 'context_composite_executor_service'
4
+
5
+ module Datadog
6
+ module Tracing
7
+ module Contrib
8
+ module ConcurrentRuby
9
+ # This patches the Async - to wrap executor service using ContextCompositeExecutorService
10
+ module AsyncPatch
11
+ def initialize(delegate)
12
+ super(delegate)
13
+
14
+ @executor = ContextCompositeExecutorService.new(@executor)
15
+ end
16
+ end
17
+ end
18
+ end
19
+ end
20
+ end
@@ -6,7 +6,7 @@ module Datadog
6
6
  module Tracing
7
7
  module Contrib
8
8
  module ConcurrentRuby
9
- # Patcher enables patching of 'Future' class.
9
+ # Patcher enables patching of 'Future' and 'Async' classes.
10
10
  module Patcher
11
11
  include Contrib::Patcher
12
12
 
@@ -21,6 +21,16 @@ module Datadog
21
21
  patch_future
22
22
  require_relative 'promises_future_patch'
23
23
  patch_promises_future
24
+ require_relative 'async_patch'
25
+ async_patch
26
+ end
27
+
28
+ # Propagate tracing context in Concurrent::Async
29
+ def async_patch
30
+ if defined?(::Concurrent::Async)
31
+ # NOTE: AsyncDelegator is a private constant
32
+ ::Concurrent::Async.const_get(:AsyncDelegator).prepend(AsyncPatch)
33
+ end
24
34
  end
25
35
 
26
36
  # Propagate tracing context in Concurrent::Future