ddtrace 1.20.0 → 1.22.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (113) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +115 -1
  3. data/LICENSE-3rdparty.csv +1 -1
  4. data/bin/ddprofrb +15 -0
  5. data/bin/ddtracerb +3 -1
  6. data/ext/{ddtrace_profiling_loader/ddtrace_profiling_loader.c → datadog_profiling_loader/datadog_profiling_loader.c} +2 -2
  7. data/ext/{ddtrace_profiling_loader → datadog_profiling_loader}/extconf.rb +3 -3
  8. data/ext/{ddtrace_profiling_native_extension → datadog_profiling_native_extension}/collectors_cpu_and_wall_time_worker.c +238 -61
  9. data/ext/{ddtrace_profiling_native_extension → datadog_profiling_native_extension}/collectors_discrete_dynamic_sampler.c +145 -72
  10. data/ext/{ddtrace_profiling_native_extension → datadog_profiling_native_extension}/collectors_discrete_dynamic_sampler.h +17 -5
  11. data/ext/{ddtrace_profiling_native_extension → datadog_profiling_native_extension}/collectors_thread_context.c +97 -4
  12. data/ext/{ddtrace_profiling_native_extension → datadog_profiling_native_extension}/extconf.rb +2 -2
  13. data/ext/{ddtrace_profiling_native_extension → datadog_profiling_native_extension}/heap_recorder.c +45 -3
  14. data/ext/{ddtrace_profiling_native_extension → datadog_profiling_native_extension}/heap_recorder.h +7 -1
  15. data/ext/{ddtrace_profiling_native_extension → datadog_profiling_native_extension}/http_transport.c +15 -19
  16. data/ext/{ddtrace_profiling_native_extension → datadog_profiling_native_extension}/native_extension_helpers.rb +4 -4
  17. data/ext/{ddtrace_profiling_native_extension → datadog_profiling_native_extension}/private_vm_api_access.c +14 -0
  18. data/ext/{ddtrace_profiling_native_extension → datadog_profiling_native_extension}/private_vm_api_access.h +4 -0
  19. data/ext/{ddtrace_profiling_native_extension → datadog_profiling_native_extension}/profiling.c +1 -1
  20. data/ext/{ddtrace_profiling_native_extension → datadog_profiling_native_extension}/ruby_helpers.c +10 -0
  21. data/ext/{ddtrace_profiling_native_extension → datadog_profiling_native_extension}/ruby_helpers.h +2 -0
  22. data/ext/{ddtrace_profiling_native_extension → datadog_profiling_native_extension}/stack_recorder.c +7 -9
  23. data/lib/datadog/appsec/contrib/rack/request_middleware.rb +43 -13
  24. data/lib/datadog/appsec/event.rb +1 -1
  25. data/lib/datadog/auto_instrument.rb +3 -0
  26. data/lib/datadog/core/configuration/components.rb +7 -6
  27. data/lib/datadog/core/configuration/option.rb +8 -6
  28. data/lib/datadog/core/configuration/settings.rb +130 -63
  29. data/lib/datadog/core/configuration.rb +20 -4
  30. data/lib/datadog/core/diagnostics/environment_logger.rb +4 -3
  31. data/lib/datadog/core/environment/git.rb +25 -0
  32. data/lib/datadog/core/environment/identity.rb +18 -48
  33. data/lib/datadog/core/environment/platform.rb +7 -1
  34. data/lib/datadog/core/git/ext.rb +2 -23
  35. data/lib/datadog/core/remote/client/capabilities.rb +1 -1
  36. data/lib/datadog/core/remote/negotiation.rb +2 -2
  37. data/lib/datadog/core/remote/transport/http/config.rb +1 -1
  38. data/lib/datadog/core/remote/worker.rb +7 -4
  39. data/lib/datadog/core/telemetry/client.rb +18 -10
  40. data/lib/datadog/core/telemetry/emitter.rb +9 -13
  41. data/lib/datadog/core/telemetry/event.rb +247 -57
  42. data/lib/datadog/core/telemetry/ext.rb +1 -0
  43. data/lib/datadog/core/telemetry/heartbeat.rb +1 -3
  44. data/lib/datadog/core/telemetry/http/ext.rb +4 -1
  45. data/lib/datadog/core/telemetry/http/transport.rb +9 -4
  46. data/lib/datadog/core/telemetry/request.rb +59 -0
  47. data/lib/datadog/core/transport/ext.rb +2 -0
  48. data/lib/datadog/core/utils/url.rb +25 -0
  49. data/lib/datadog/profiling/collectors/code_provenance.rb +10 -4
  50. data/lib/datadog/profiling/collectors/cpu_and_wall_time_worker.rb +31 -0
  51. data/lib/datadog/profiling/collectors/info.rb +101 -0
  52. data/lib/datadog/profiling/component.rb +34 -28
  53. data/lib/datadog/profiling/exporter.rb +19 -5
  54. data/lib/datadog/profiling/ext.rb +2 -0
  55. data/lib/datadog/profiling/flush.rb +6 -3
  56. data/lib/datadog/profiling/http_transport.rb +5 -1
  57. data/lib/datadog/profiling/load_native_extension.rb +19 -6
  58. data/lib/datadog/profiling/native_extension.rb +1 -1
  59. data/lib/datadog/profiling/tag_builder.rb +5 -0
  60. data/lib/datadog/profiling/tasks/exec.rb +3 -3
  61. data/lib/datadog/profiling/tasks/help.rb +3 -3
  62. data/lib/datadog/profiling.rb +13 -2
  63. data/lib/datadog/tracing/contrib/action_mailer/events/deliver.rb +1 -1
  64. data/lib/datadog/tracing/contrib/active_record/configuration/resolver.rb +11 -4
  65. data/lib/datadog/tracing/contrib/concurrent_ruby/async_patch.rb +20 -0
  66. data/lib/datadog/tracing/contrib/concurrent_ruby/patcher.rb +11 -1
  67. data/lib/datadog/tracing/contrib/configurable.rb +1 -1
  68. data/lib/datadog/tracing/contrib/extensions.rb +6 -2
  69. data/lib/datadog/tracing/contrib/pg/instrumentation.rb +11 -4
  70. data/lib/datadog/tracing/sampling/matcher.rb +23 -3
  71. data/lib/datadog/tracing/sampling/rule.rb +7 -2
  72. data/lib/datadog/tracing/sampling/rule_sampler.rb +2 -0
  73. data/lib/datadog/tracing/trace_operation.rb +1 -2
  74. data/lib/datadog/tracing/transport/http.rb +1 -0
  75. data/lib/datadog/tracing/transport/trace_formatter.rb +31 -0
  76. data/lib/ddtrace/version.rb +1 -1
  77. metadata +55 -62
  78. data/ext/ddtrace_profiling_native_extension/pid_controller.c +0 -57
  79. data/ext/ddtrace_profiling_native_extension/pid_controller.h +0 -45
  80. data/lib/datadog/core/telemetry/collector.rb +0 -250
  81. data/lib/datadog/core/telemetry/v1/app_event.rb +0 -59
  82. data/lib/datadog/core/telemetry/v1/application.rb +0 -92
  83. data/lib/datadog/core/telemetry/v1/configuration.rb +0 -25
  84. data/lib/datadog/core/telemetry/v1/dependency.rb +0 -43
  85. data/lib/datadog/core/telemetry/v1/host.rb +0 -59
  86. data/lib/datadog/core/telemetry/v1/install_signature.rb +0 -38
  87. data/lib/datadog/core/telemetry/v1/integration.rb +0 -64
  88. data/lib/datadog/core/telemetry/v1/product.rb +0 -36
  89. data/lib/datadog/core/telemetry/v1/telemetry_request.rb +0 -106
  90. data/lib/datadog/core/telemetry/v2/app_client_configuration_change.rb +0 -41
  91. data/lib/datadog/core/telemetry/v2/request.rb +0 -29
  92. data/lib/datadog/profiling/diagnostics/environment_logger.rb +0 -39
  93. /data/ext/{ddtrace_profiling_native_extension → datadog_profiling_native_extension}/NativeExtensionDesign.md +0 -0
  94. /data/ext/{ddtrace_profiling_native_extension → datadog_profiling_native_extension}/clock_id.h +0 -0
  95. /data/ext/{ddtrace_profiling_native_extension → datadog_profiling_native_extension}/clock_id_from_pthread.c +0 -0
  96. /data/ext/{ddtrace_profiling_native_extension → datadog_profiling_native_extension}/clock_id_noop.c +0 -0
  97. /data/ext/{ddtrace_profiling_native_extension → datadog_profiling_native_extension}/collectors_dynamic_sampling_rate.c +0 -0
  98. /data/ext/{ddtrace_profiling_native_extension → datadog_profiling_native_extension}/collectors_dynamic_sampling_rate.h +0 -0
  99. /data/ext/{ddtrace_profiling_native_extension → datadog_profiling_native_extension}/collectors_gc_profiling_helper.c +0 -0
  100. /data/ext/{ddtrace_profiling_native_extension → datadog_profiling_native_extension}/collectors_gc_profiling_helper.h +0 -0
  101. /data/ext/{ddtrace_profiling_native_extension → datadog_profiling_native_extension}/collectors_idle_sampling_helper.c +0 -0
  102. /data/ext/{ddtrace_profiling_native_extension → datadog_profiling_native_extension}/collectors_idle_sampling_helper.h +0 -0
  103. /data/ext/{ddtrace_profiling_native_extension → datadog_profiling_native_extension}/collectors_stack.c +0 -0
  104. /data/ext/{ddtrace_profiling_native_extension → datadog_profiling_native_extension}/collectors_stack.h +0 -0
  105. /data/ext/{ddtrace_profiling_native_extension → datadog_profiling_native_extension}/collectors_thread_context.h +0 -0
  106. /data/ext/{ddtrace_profiling_native_extension → datadog_profiling_native_extension}/helpers.h +0 -0
  107. /data/ext/{ddtrace_profiling_native_extension → datadog_profiling_native_extension}/libdatadog_helpers.c +0 -0
  108. /data/ext/{ddtrace_profiling_native_extension → datadog_profiling_native_extension}/libdatadog_helpers.h +0 -0
  109. /data/ext/{ddtrace_profiling_native_extension → datadog_profiling_native_extension}/setup_signal_handler.c +0 -0
  110. /data/ext/{ddtrace_profiling_native_extension → datadog_profiling_native_extension}/setup_signal_handler.h +0 -0
  111. /data/ext/{ddtrace_profiling_native_extension → datadog_profiling_native_extension}/stack_recorder.h +0 -0
  112. /data/ext/{ddtrace_profiling_native_extension → datadog_profiling_native_extension}/time_helpers.c +0 -0
  113. /data/ext/{ddtrace_profiling_native_extension → datadog_profiling_native_extension}/time_helpers.h +0 -0
@@ -22,6 +22,7 @@ module Datadog
22
22
  # **NOTE**: This should only be used for testing; disabling the dynamic sampling rate will increase the
23
23
  # profiler overhead!
24
24
  dynamic_sampling_rate_enabled: true,
25
+ skip_idle_samples_for_testing: false,
25
26
  idle_sampling_helper: IdleSamplingHelper.new
26
27
  )
27
28
  unless dynamic_sampling_rate_enabled
@@ -39,11 +40,14 @@ module Datadog
39
40
  dynamic_sampling_rate_enabled,
40
41
  dynamic_sampling_rate_overhead_target_percentage,
41
42
  allocation_profiling_enabled,
43
+ skip_idle_samples_for_testing,
42
44
  )
43
45
  @worker_thread = nil
44
46
  @failure_exception = nil
45
47
  @start_stop_mutex = Mutex.new
46
48
  @idle_sampling_helper = idle_sampling_helper
49
+ @wait_until_running_mutex = Mutex.new
50
+ @wait_until_running_condition = ConditionVariable.new
47
51
  end
48
52
 
49
53
  def start(on_failure_proc: nil)
@@ -100,6 +104,33 @@ module Datadog
100
104
  def stats
101
105
  self.class._native_stats(self)
102
106
  end
107
+
108
+ def stats_and_reset_not_thread_safe
109
+ stats = self.stats
110
+ self.class._native_stats_reset_not_thread_safe(self)
111
+ stats
112
+ end
113
+
114
+ # Useful for testing, to e.g. make sure the profiler is running before we start running some code we want to observe
115
+ def wait_until_running(timeout_seconds: 5)
116
+ @wait_until_running_mutex.synchronize do
117
+ return true if self.class._native_is_running?(self)
118
+
119
+ @wait_until_running_condition.wait(@wait_until_running_mutex, timeout_seconds)
120
+
121
+ if self.class._native_is_running?(self)
122
+ true
123
+ else
124
+ raise "Timeout waiting for #{self.class.name} to start (waited for #{timeout_seconds} seconds)"
125
+ end
126
+ end
127
+ end
128
+
129
+ private
130
+
131
+ def signal_running
132
+ @wait_until_running_mutex.synchronize { @wait_until_running_condition.broadcast }
133
+ end
103
134
  end
104
135
  end
105
136
  end
@@ -0,0 +1,101 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'set'
4
+ require 'time'
5
+
6
+ module Datadog
7
+ module Profiling
8
+ module Collectors
9
+ # Collects information of relevance for profiler. This will get sent alongside
10
+ # the profile and show up in the UI or potentially influence processing in some way.
11
+ #
12
+ # Information is currently collected and frozen at construction time. A full collector
13
+ # could be seen as overkill for this case but it allows us to centralize information
14
+ # gathering and easily support more flexible/dynamic info collection in the future.
15
+ class Info
16
+ def initialize(settings)
17
+ @profiler_info = nil
18
+ @info = {
19
+ platform: collect_platform_info,
20
+ runtime: collect_runtime_info,
21
+ application: collect_application_info(settings),
22
+ profiler: collect_profiler_info(settings),
23
+ }.freeze
24
+ end
25
+
26
+ attr_reader :info
27
+
28
+ private
29
+
30
+ # Instead of trying to figure out real process start time by checking
31
+ # /proc or some other complex/non-portable way, approximate start time
32
+ # by time of requirement of this file.
33
+ START_TIME = Time.now.utc.freeze
34
+
35
+ def collect_platform_info
36
+ @platform_info ||= {
37
+ container_id: Datadog::Core::Environment::Container.container_id,
38
+ hostname: Datadog::Core::Environment::Platform.hostname,
39
+ kernel_name: Datadog::Core::Environment::Platform.kernel_name,
40
+ kernel_release: Datadog::Core::Environment::Platform.kernel_release,
41
+ kernel_version: Datadog::Core::Environment::Platform.kernel_version
42
+ }.freeze
43
+ end
44
+
45
+ def collect_runtime_info
46
+ @runtime_info ||= {
47
+ engine: Datadog::Core::Environment::Identity.lang_engine,
48
+ version: Datadog::Core::Environment::Identity.lang_version,
49
+ platform: Datadog::Core::Environment::Identity.lang_platform,
50
+ }.freeze
51
+ end
52
+
53
+ def collect_application_info(settings)
54
+ @application_info ||= {
55
+ start_time: START_TIME.iso8601,
56
+ env: settings.env,
57
+ service: settings.service,
58
+ version: settings.version,
59
+ }.freeze
60
+ end
61
+
62
+ def collect_profiler_info(settings)
63
+ unless @profiler_info
64
+ lib_datadog_gem = ::Gem.loaded_specs['libdatadog']
65
+ @profiler_info = {
66
+ version: Datadog::Core::Environment::Identity.tracer_version,
67
+ libdatadog: "#{lib_datadog_gem.version}-#{lib_datadog_gem.platform}",
68
+ settings: collect_settings_recursively(settings.profiling),
69
+ }.freeze
70
+ end
71
+ @profiler_info
72
+ end
73
+
74
+ # The settings/option model isn't directly serializable because
75
+ # of subsettings and options that link to full blown custom object
76
+ # instances without proper serialization.
77
+ # This method navigates a settings object recursively, converting
78
+ # it into more basic types that are trivially convertible to JSON.
79
+ def collect_settings_recursively(v)
80
+ v = v.options_hash if v.respond_to?(:options_hash)
81
+
82
+ if v.nil? || v.is_a?(Symbol) || v.is_a?(Numeric) || v.is_a?(String) || v.equal?(true) || v.equal?(false)
83
+ Core::Utils::SafeDup.frozen_or_dup(v)
84
+ elsif v.is_a?(Hash)
85
+ collected_hash = v.each_with_object({}) do |(key, value), hash|
86
+ collected_value = collect_settings_recursively(value)
87
+ hash[key] = collected_value
88
+ end
89
+ collected_hash.freeze
90
+ elsif v.is_a?(Enumerable)
91
+ collected_list = v
92
+ .map { |value| collect_settings_recursively(value) }
93
+ collected_list.freeze
94
+ else
95
+ v.inspect
96
+ end
97
+ end
98
+ end
99
+ end
100
+ end
101
+ end
@@ -8,11 +8,7 @@ module Datadog
8
8
  # * Code Hotspots panel in the trace viewer, as well as scoping a profile down to a span
9
9
  # * Endpoint aggregation in the profiler UX, including normalization (resource per endpoint call)
10
10
  def self.build_profiler_component(settings:, agent_settings:, optional_tracer:) # rubocop:disable Metrics/MethodLength
11
- require_relative '../profiling/diagnostics/environment_logger'
12
-
13
- Profiling::Diagnostics::EnvironmentLogger.collect_and_log!
14
-
15
- return unless settings.profiling.enabled
11
+ return [nil, { profiling_enabled: false }] unless settings.profiling.enabled
16
12
 
17
13
  # Workaround for weird dependency direction: the Core::Configuration::Components class currently has a
18
14
  # dependency on individual products, in this case the Profiler.
@@ -32,7 +28,8 @@ module Datadog
32
28
  # done, then profiling may not be loaded, and thus to avoid this issue we do a require here (which is a
33
29
  # no-op if profiling is already loaded).
34
30
  require_relative '../profiling'
35
- return unless Profiling.supported?
31
+
32
+ return [nil, { profiling_enabled: false }] unless Profiling.supported?
36
33
 
37
34
  # Activate forking extensions
38
35
  Profiling::Tasks::Setup.new.run
@@ -40,7 +37,7 @@ module Datadog
40
37
  # NOTE: Please update the Initialization section of ProfilingDevelopment.md with any changes to this method
41
38
 
42
39
  no_signals_workaround_enabled = no_signals_workaround_enabled?(settings)
43
- timeline_enabled = settings.profiling.advanced.experimental_timeline_enabled
40
+ timeline_enabled = settings.profiling.advanced.timeline_enabled
44
41
  allocation_profiling_enabled = enable_allocation_profiling?(settings)
45
42
  heap_sample_every = get_heap_sample_every(settings)
46
43
  heap_profiling_enabled = enable_heap_profiling?(settings, allocation_profiling_enabled, heap_sample_every)
@@ -72,11 +69,11 @@ module Datadog
72
69
  heap_sample_every: heap_sample_every,
73
70
  }.freeze
74
71
 
75
- exporter = build_profiler_exporter(settings, recorder, internal_metadata: internal_metadata)
72
+ exporter = build_profiler_exporter(settings, recorder, worker, internal_metadata: internal_metadata)
76
73
  transport = build_profiler_transport(settings, agent_settings)
77
74
  scheduler = Profiling::Scheduler.new(exporter: exporter, transport: transport, interval: upload_period_seconds)
78
75
 
79
- Profiling::Profiler.new(worker: worker, scheduler: scheduler)
76
+ [Profiling::Profiler.new(worker: worker, scheduler: scheduler), { profiling_enabled: true }]
80
77
  end
81
78
 
82
79
  private_class_method def self.build_thread_context_collector(settings, recorder, optional_tracer, timeline_enabled)
@@ -89,12 +86,15 @@ module Datadog
89
86
  )
90
87
  end
91
88
 
92
- private_class_method def self.build_profiler_exporter(settings, recorder, internal_metadata:)
89
+ private_class_method def self.build_profiler_exporter(settings, recorder, worker, internal_metadata:)
90
+ info_collector = Profiling::Collectors::Info.new(settings)
93
91
  code_provenance_collector =
94
92
  (Profiling::Collectors::CodeProvenance.new if settings.profiling.advanced.code_provenance_enabled)
95
93
 
96
94
  Profiling::Exporter.new(
97
95
  pprof_recorder: recorder,
96
+ worker: worker,
97
+ info_collector: info_collector,
98
98
  code_provenance_collector: code_provenance_collector,
99
99
  internal_metadata: internal_metadata,
100
100
  )
@@ -111,19 +111,30 @@ module Datadog
111
111
  end
112
112
 
113
113
  private_class_method def self.enable_gc_profiling?(settings)
114
- # See comments on the setting definition for more context on why it exists.
115
- if settings.profiling.advanced.force_enable_gc_profiling
116
- if Gem::Version.new(RUBY_VERSION) >= Gem::Version.new('3')
117
- Datadog.logger.debug(
118
- 'Profiling time/resources spent in Garbage Collection force enabled. Do not use Ractors in combination ' \
119
- 'with this option as profiles will be incomplete.'
120
- )
121
- end
114
+ return false unless settings.profiling.advanced.gc_enabled
122
115
 
123
- true
124
- else
125
- false
116
+ # SEVERE - Only with Ractors
117
+ # On Ruby versions 3.0 (all), 3.1.0 to 3.1.3, and 3.2.0 to 3.2.2 gc profiling can trigger a VM bug
118
+ # that causes a segmentation fault during garbage collection of Ractors
119
+ # (https://bugs.ruby-lang.org/issues/18464). We don't allow enabling gc profiling on such Rubies.
120
+ # This bug is fixed on Ruby versions 3.1.4, 3.2.3 and 3.3.0.
121
+ if RUBY_VERSION.start_with?('3.0.') ||
122
+ (RUBY_VERSION.start_with?('3.1.') && RUBY_VERSION < '3.1.4') ||
123
+ (RUBY_VERSION.start_with?('3.2.') && RUBY_VERSION < '3.2.3')
124
+ Datadog.logger.warn(
125
+ "Current Ruby version (#{RUBY_VERSION}) has a VM bug where enabling GC profiling would cause "\
126
+ 'crashes (https://bugs.ruby-lang.org/issues/18464). GC profiling has been disabled.'
127
+ )
128
+ return false
129
+ elsif RUBY_VERSION.start_with?('3.')
130
+ Datadog.logger.debug(
131
+ 'In all known versions of Ruby 3.x, using Ractors may result in GC profiling unexpectedly ' \
132
+ 'stopping (https://bugs.ruby-lang.org/issues/19112). Note that this stop has no impact in your ' \
133
+ 'application stability or performance. This does not happen if Ractors are not used.'
134
+ )
126
135
  end
136
+
137
+ true
127
138
  end
128
139
 
129
140
  private_class_method def self.get_heap_sample_every(settings)
@@ -135,10 +146,7 @@ module Datadog
135
146
  end
136
147
 
137
148
  private_class_method def self.enable_allocation_profiling?(settings)
138
- unless settings.profiling.advanced.experimental_allocation_enabled
139
- # Allocation profiling disabled, short-circuit out
140
- return false
141
- end
149
+ return false unless settings.profiling.allocation_enabled
142
150
 
143
151
  # Allocation sampling is safe and supported on Ruby 2.x, but has a few caveats on Ruby 3.x.
144
152
 
@@ -179,9 +187,7 @@ module Datadog
179
187
  )
180
188
  end
181
189
 
182
- Datadog.logger.warn(
183
- 'Enabled experimental allocation profiling. This is experimental, not recommended, and will increase overhead!'
184
- )
190
+ Datadog.logger.debug('Enabled allocation profiling')
185
191
 
186
192
  true
187
193
  end
@@ -23,31 +23,39 @@ module Datadog
23
23
  :time_provider,
24
24
  :last_flush_finish_at,
25
25
  :created_at,
26
- :internal_metadata
26
+ :internal_metadata,
27
+ :info_json
27
28
 
28
29
  public
29
30
 
30
31
  def initialize(
31
32
  pprof_recorder:,
33
+ worker:,
34
+ info_collector:,
32
35
  code_provenance_collector:,
33
36
  internal_metadata:,
34
37
  minimum_duration_seconds: PROFILE_DURATION_THRESHOLD_SECONDS,
35
38
  time_provider: Time
36
39
  )
37
40
  @pprof_recorder = pprof_recorder
41
+ @worker = worker
38
42
  @code_provenance_collector = code_provenance_collector
39
43
  @minimum_duration_seconds = minimum_duration_seconds
40
44
  @time_provider = time_provider
41
45
  @last_flush_finish_at = nil
42
46
  @created_at = time_provider.now.utc
43
47
  @internal_metadata = internal_metadata
48
+ # NOTE: At the time of this comment collected info does not change over time so we'll hardcode
49
+ # it on startup to prevent serializing the same info on every flush.
50
+ @info_json = JSON.fast_generate(info_collector.info).freeze
44
51
  end
45
52
 
46
53
  def flush
47
- start, finish, uncompressed_pprof = pprof_recorder.serialize
54
+ worker_stats = @worker.stats_and_reset_not_thread_safe
55
+ start, finish, compressed_pprof = pprof_recorder.serialize
48
56
  @last_flush_finish_at = finish
49
57
 
50
- return if uncompressed_pprof.nil? # We don't want to report empty profiles
58
+ return if compressed_pprof.nil? # We don't want to report empty profiles
51
59
 
52
60
  if duration_below_threshold?(start, finish)
53
61
  Datadog.logger.debug('Skipped exporting profiling events as profile duration is below minimum')
@@ -60,11 +68,17 @@ module Datadog
60
68
  start: start,
61
69
  finish: finish,
62
70
  pprof_file_name: Datadog::Profiling::Ext::Transport::HTTP::PPROF_DEFAULT_FILENAME,
63
- pprof_data: uncompressed_pprof.to_s,
71
+ pprof_data: compressed_pprof.to_s,
64
72
  code_provenance_file_name: Datadog::Profiling::Ext::Transport::HTTP::CODE_PROVENANCE_FILENAME,
65
73
  code_provenance_data: uncompressed_code_provenance,
66
74
  tags_as_array: Datadog::Profiling::TagBuilder.call(settings: Datadog.configuration).to_a,
67
- internal_metadata: internal_metadata,
75
+ internal_metadata: internal_metadata.merge(
76
+ {
77
+ worker_stats: worker_stats,
78
+ gc: GC.stat,
79
+ }
80
+ ),
81
+ info_json: info_json,
68
82
  )
69
83
  end
70
84
 
@@ -23,6 +23,8 @@ module Datadog
23
23
  FORM_FIELD_TAG_RUNTIME_VERSION = 'runtime_version'
24
24
  FORM_FIELD_TAG_SERVICE = 'service'
25
25
  FORM_FIELD_TAG_VERSION = 'version'
26
+ TAG_GIT_REPOSITORY_URL = 'git.repository_url'
27
+ TAG_GIT_COMMIT_SHA = 'git.commit.sha'
26
28
 
27
29
  PPROF_DEFAULT_FILENAME = 'rubyprofile.pprof'
28
30
  CODE_PROVENANCE_FILENAME = 'code-provenance.json'
@@ -14,7 +14,8 @@ module Datadog
14
14
  :code_provenance_file_name,
15
15
  :code_provenance_data, # gzipped json bytes
16
16
  :tags_as_array,
17
- :internal_metadata_json
17
+ :internal_metadata_json,
18
+ :info_json
18
19
 
19
20
  def initialize(
20
21
  start:,
@@ -24,7 +25,8 @@ module Datadog
24
25
  code_provenance_file_name:,
25
26
  code_provenance_data:,
26
27
  tags_as_array:,
27
- internal_metadata:
28
+ internal_metadata:,
29
+ info_json:
28
30
  )
29
31
  @start = start
30
32
  @finish = finish
@@ -33,7 +35,8 @@ module Datadog
33
35
  @code_provenance_file_name = code_provenance_file_name
34
36
  @code_provenance_data = code_provenance_data
35
37
  @tags_as_array = tags_as_array
36
- @internal_metadata_json = JSON.fast_generate(internal_metadata.map { |k, v| [k, v.to_s] }.to_h)
38
+ @internal_metadata_json = JSON.fast_generate(internal_metadata)
39
+ @info_json = info_json
37
40
  end
38
41
  end
39
42
  end
@@ -43,6 +43,8 @@ module Datadog
43
43
 
44
44
  tags_as_array: flush.tags_as_array,
45
45
  internal_metadata_json: flush.internal_metadata_json,
46
+
47
+ info_json: flush.info_json
46
48
  )
47
49
 
48
50
  if status == :ok
@@ -117,7 +119,8 @@ module Datadog
117
119
  code_provenance_file_name:,
118
120
  code_provenance_data:,
119
121
  tags_as_array:,
120
- internal_metadata_json:
122
+ internal_metadata_json:,
123
+ info_json:
121
124
  )
122
125
  self.class._native_do_export(
123
126
  exporter_configuration,
@@ -132,6 +135,7 @@ module Datadog
132
135
  code_provenance_data,
133
136
  tags_as_array,
134
137
  internal_metadata_json,
138
+ info_json,
135
139
  )
136
140
  end
137
141
 
@@ -1,24 +1,37 @@
1
1
  # This file is used to load the profiling native extension. It works in two steps:
2
2
  #
3
- # 1. Load the ddtrace_profiling_loader extension. This extension will be used to load the actual extension, but in
4
- # a special way that avoids exposing native-level code symbols. See `ddtrace_profiling_loader.c` for more details.
3
+ # 1. Load the datadog_profiling_loader extension. This extension will be used to load the actual extension, but in
4
+ # a special way that avoids exposing native-level code symbols. See `datadog_profiling_loader.c` for more details.
5
5
  #
6
- # 2. Use the Datadog::Profiling::Loader exposed by the ddtrace_profiling_loader extension to load the actual
6
+ # 2. Use the Datadog::Profiling::Loader exposed by the datadog_profiling_loader extension to load the actual
7
7
  # profiling native extension.
8
8
  #
9
9
  # All code on this file is on-purpose at the top-level; this makes it so this file is executed only once,
10
10
  # the first time it gets required, to avoid any issues with the native extension being initialized more than once.
11
11
 
12
12
  begin
13
- require "ddtrace_profiling_loader.#{RUBY_VERSION}_#{RUBY_PLATFORM}"
13
+ require "datadog_profiling_loader.#{RUBY_VERSION}_#{RUBY_PLATFORM}"
14
14
  rescue LoadError => e
15
15
  raise LoadError,
16
16
  'Failed to load the profiling loader extension. To fix this, please remove and then reinstall ddtrace ' \
17
17
  "(Details: #{e.message})"
18
18
  end
19
19
 
20
- extension_name = "ddtrace_profiling_native_extension.#{RUBY_VERSION}_#{RUBY_PLATFORM}"
21
- full_file_path = "#{__dir__}/../../#{extension_name}.#{RbConfig::CONFIG['DLEXT']}"
20
+ extension_name = "datadog_profiling_native_extension.#{RUBY_VERSION}_#{RUBY_PLATFORM}"
21
+ file_name = "#{extension_name}.#{RbConfig::CONFIG['DLEXT']}"
22
+ full_file_path = "#{__dir__}/../../#{file_name}"
23
+
24
+ unless File.exist?(full_file_path)
25
+ extension_dir = Gem.loaded_specs['ddtrace'].extension_dir
26
+ candidate_path = "#{extension_dir}/#{file_name}"
27
+ if File.exist?(candidate_path)
28
+ full_file_path = candidate_path
29
+ else # rubocop:disable Style/EmptyElse
30
+ # We found none of the files. This is unexpected. Let's go ahead anyway, the error is going to be reported further
31
+ # down anyway.
32
+ end
33
+ end
34
+
22
35
  init_function_name = "Init_#{extension_name.split('.').first}"
23
36
 
24
37
  status, result = Datadog::Profiling::Loader._native_load(full_file_path, init_function_name)
@@ -3,7 +3,7 @@
3
3
  module Datadog
4
4
  module Profiling
5
5
  # This module contains classes and methods which are implemented using native code in the
6
- # ext/ddtrace_profiling_native_extension folder, as well as some Ruby-level utilities that don't make sense to
6
+ # ext/datadog_profiling_native_extension folder, as well as some Ruby-level utilities that don't make sense to
7
7
  # write using C
8
8
  module NativeExtension
9
9
  private_class_method def self.working?
@@ -1,6 +1,7 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  require_relative '../core/utils'
4
+ require_relative '../core/environment/git'
4
5
 
5
6
  module Datadog
6
7
  module Profiling
@@ -23,6 +24,8 @@ module Datadog
23
24
  runtime_id: Core::Environment::Identity.id,
24
25
  runtime_platform: Core::Environment::Identity.lang_platform,
25
26
  runtime_version: Core::Environment::Identity.lang_version,
27
+ git_repository_url: Core::Environment::Git.git_repository_url,
28
+ git_commit_sha: Core::Environment::Git.git_commit_sha,
26
29
  # User-provided tags
27
30
  user_tags: settings.tags
28
31
  )
@@ -42,6 +45,8 @@ module Datadog
42
45
  tags[FORM_FIELD_TAG_ENV] = env if env
43
46
  tags[FORM_FIELD_TAG_SERVICE] = service if service
44
47
  tags[FORM_FIELD_TAG_VERSION] = version if version
48
+ tags[TAG_GIT_REPOSITORY_URL] = git_repository_url if git_repository_url
49
+ tags[TAG_GIT_COMMIT_SHA] = git_commit_sha if git_commit_sha
45
50
 
46
51
  # Make sure everything is an utf-8 string, to avoid encoding issues in native code/libddprof/further downstream
47
52
  user_tags.merge(tags).map do |key, value|
@@ -1,7 +1,7 @@
1
1
  module Datadog
2
2
  module Profiling
3
3
  module Tasks
4
- # Wraps command with Datadog tracing
4
+ # Wraps command with Datadog profiling
5
5
  class Exec
6
6
  attr_reader :args
7
7
 
@@ -36,10 +36,10 @@ module Datadog
36
36
  def exec_with_error_handling(args)
37
37
  Kernel.exec(*args)
38
38
  rescue Errno::ENOENT => e
39
- Kernel.warn "ddtracerb exec failed: #{e.class.name} #{e.message} (command was '#{args.join(' ')}')"
39
+ Kernel.warn "ddprofrb exec failed: #{e.class.name} #{e.message} (command was '#{args.join(' ')}')"
40
40
  Kernel.exit 127
41
41
  rescue Errno::EACCES, Errno::ENOEXEC => e
42
- Kernel.warn "ddtracerb exec failed: #{e.class.name} #{e.message} (command was '#{args.join(' ')}')"
42
+ Kernel.warn "ddprofrb exec failed: #{e.class.name} #{e.message} (command was '#{args.join(' ')}')"
43
43
  Kernel.exit 126
44
44
  end
45
45
  end
@@ -1,12 +1,12 @@
1
1
  module Datadog
2
2
  module Profiling
3
3
  module Tasks
4
- # Prints help message for usage of `ddtrace`
4
+ # Prints help message for usage of `ddprofrb`
5
5
  class Help
6
6
  def run
7
7
  puts %(
8
- Usage: ddtracerb [command] [arguments]
9
- exec [command]: Executes command with tracing & profiling preloaded.
8
+ Usage: ddprofrb [command] [arguments]
9
+ exec [command]: Executes command with profiling preloaded.
10
10
  help: Prints this help message.
11
11
  )
12
12
  end
@@ -63,6 +63,17 @@ module Datadog
63
63
  !!(profiler.send(:scheduler).running? if profiler)
64
64
  end
65
65
 
66
+ def self.wait_until_running(timeout_seconds: 5)
67
+ profiler = Datadog.send(:components).profiler
68
+ if profiler
69
+ # Use .send(...) to avoid exposing the attr_reader as an API to the outside
70
+ worker = profiler.send(:worker)
71
+ worker.wait_until_running(timeout_seconds: timeout_seconds)
72
+ else
73
+ raise 'Profiler not enabled or available'
74
+ end
75
+ end
76
+
66
77
  private_class_method def self.replace_noop_allocation_count
67
78
  def self.allocation_count # rubocop:disable Lint/NestedMethodDefinition (On purpose!)
68
79
  Datadog::Profiling::Collectors::CpuAndWallTimeWorker._native_allocation_count
@@ -77,7 +88,7 @@ module Datadog
77
88
 
78
89
  private_class_method def self.try_reading_skipped_reason_file(file_api = File)
79
90
  # This file, if it exists, is recorded by extconf.rb during compilation of the native extension
80
- skipped_reason_file = "#{__dir__}/../../ext/ddtrace_profiling_native_extension/skipped_reason.txt"
91
+ skipped_reason_file = "#{__dir__}/../../ext/datadog_profiling_native_extension/skipped_reason.txt"
81
92
 
82
93
  begin
83
94
  return unless file_api.exist?(skipped_reason_file)
@@ -123,13 +134,13 @@ module Datadog
123
134
  return false unless supported?
124
135
 
125
136
  require_relative 'profiling/ext/forking'
137
+ require_relative 'profiling/collectors/info'
126
138
  require_relative 'profiling/collectors/code_provenance'
127
139
  require_relative 'profiling/collectors/cpu_and_wall_time_worker'
128
140
  require_relative 'profiling/collectors/dynamic_sampling_rate'
129
141
  require_relative 'profiling/collectors/idle_sampling_helper'
130
142
  require_relative 'profiling/collectors/stack'
131
143
  require_relative 'profiling/collectors/thread_context'
132
- require_relative 'profiling/diagnostics/environment_logger'
133
144
  require_relative 'profiling/stack_recorder'
134
145
  require_relative 'profiling/exporter'
135
146
  require_relative 'profiling/flush'
@@ -37,7 +37,7 @@ module Datadog
37
37
 
38
38
  span.set_tag(Tracing::Metadata::Ext::TAG_OPERATION, Ext::TAG_OPERATION_DELIVER)
39
39
 
40
- # Since email date can contain PII we disable by default
40
+ # Since email data can contain PII we disable by default
41
41
  # Some of these fields can be either strings or arrays, so we try to normalize
42
42
  # https://github.com/rails/rails/blob/18707ab17fa492eb25ad2e8f9818a320dc20b823/actionmailer/lib/action_mailer/base.rb#L742-L754
43
43
  if configuration[:email_data] == true
@@ -65,9 +65,14 @@ module Datadog
65
65
 
66
66
  config
67
67
  rescue => e
68
+ # Resolving a valid database configuration should not raise an exception,
69
+ # but if it does, it can be due to adding a broken pattern match prior to this call.
70
+ #
71
+ # `db_config` input may contain sensitive information such as passwords,
72
+ # hence provide a succinct summary for the error logging.
68
73
  Datadog.logger.error(
69
- "Failed to resolve ActiveRecord configuration key #{db_config.inspect}. " \
70
- "Cause: #{e.class.name} #{e.message} Source: #{Array(e.backtrace).first}"
74
+ 'Failed to resolve ActiveRecord database configuration. '\
75
+ "Cause: #{e.class.name} Source: #{Array(e.backtrace).first}"
71
76
  )
72
77
 
73
78
  nil
@@ -85,9 +90,11 @@ module Datadog
85
90
  normalized
86
91
  rescue => e
87
92
  Datadog.logger.error(
88
- "Failed to resolve ActiveRecord configuration key #{matcher.inspect}. " \
89
- "Cause: #{e.class.name} #{e.message} Source: #{Array(e.backtrace).first}"
93
+ "Failed to resolve key #{matcher.inspect}. " \
94
+ "Cause: #{e.class.name} Source: #{Array(e.backtrace).first}"
90
95
  )
96
+
97
+ nil
91
98
  end
92
99
 
93
100
  #
@@ -0,0 +1,20 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative 'context_composite_executor_service'
4
+
5
+ module Datadog
6
+ module Tracing
7
+ module Contrib
8
+ module ConcurrentRuby
9
+ # This patches the Async - to wrap executor service using ContextCompositeExecutorService
10
+ module AsyncPatch
11
+ def initialize(delegate)
12
+ super(delegate)
13
+
14
+ @executor = ContextCompositeExecutorService.new(@executor)
15
+ end
16
+ end
17
+ end
18
+ end
19
+ end
20
+ end
@@ -6,7 +6,7 @@ module Datadog
6
6
  module Tracing
7
7
  module Contrib
8
8
  module ConcurrentRuby
9
- # Patcher enables patching of 'Future' class.
9
+ # Patcher enables patching of 'Future' and 'Async' classes.
10
10
  module Patcher
11
11
  include Contrib::Patcher
12
12
 
@@ -21,6 +21,16 @@ module Datadog
21
21
  patch_future
22
22
  require_relative 'promises_future_patch'
23
23
  patch_promises_future
24
+ require_relative 'async_patch'
25
+ async_patch
26
+ end
27
+
28
+ # Propagate tracing context in Concurrent::Async
29
+ def async_patch
30
+ if defined?(::Concurrent::Async)
31
+ # NOTE: AsyncDelegator is a private constant
32
+ ::Concurrent::Async.const_get(:AsyncDelegator).prepend(AsyncPatch)
33
+ end
24
34
  end
25
35
 
26
36
  # Propagate tracing context in Concurrent::Future