datadog 2.16.0 → 2.18.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (164) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +72 -1
  3. data/ext/datadog_profiling_native_extension/collectors_cpu_and_wall_time_worker.c +12 -46
  4. data/ext/datadog_profiling_native_extension/collectors_stack.c +227 -49
  5. data/ext/datadog_profiling_native_extension/collectors_stack.h +19 -3
  6. data/ext/datadog_profiling_native_extension/collectors_thread_context.c +63 -12
  7. data/ext/datadog_profiling_native_extension/collectors_thread_context.h +1 -0
  8. data/ext/datadog_profiling_native_extension/encoded_profile.c +22 -12
  9. data/ext/datadog_profiling_native_extension/encoded_profile.h +1 -0
  10. data/ext/datadog_profiling_native_extension/extconf.rb +7 -0
  11. data/ext/datadog_profiling_native_extension/heap_recorder.c +239 -363
  12. data/ext/datadog_profiling_native_extension/heap_recorder.h +4 -6
  13. data/ext/datadog_profiling_native_extension/http_transport.c +45 -72
  14. data/ext/datadog_profiling_native_extension/libdatadog_helpers.c +22 -0
  15. data/ext/datadog_profiling_native_extension/libdatadog_helpers.h +8 -5
  16. data/ext/datadog_profiling_native_extension/private_vm_api_access.c +1 -0
  17. data/ext/datadog_profiling_native_extension/private_vm_api_access.h +6 -3
  18. data/ext/datadog_profiling_native_extension/ruby_helpers.c +1 -13
  19. data/ext/datadog_profiling_native_extension/ruby_helpers.h +2 -10
  20. data/ext/datadog_profiling_native_extension/stack_recorder.c +156 -60
  21. data/ext/libdatadog_api/crashtracker.c +10 -3
  22. data/ext/libdatadog_api/extconf.rb +2 -2
  23. data/ext/libdatadog_api/library_config.c +54 -12
  24. data/ext/libdatadog_api/library_config.h +6 -0
  25. data/ext/libdatadog_api/macos_development.md +3 -3
  26. data/ext/libdatadog_api/process_discovery.c +2 -7
  27. data/ext/libdatadog_extconf_helpers.rb +2 -2
  28. data/lib/datadog/appsec/api_security/lru_cache.rb +56 -0
  29. data/lib/datadog/appsec/api_security/route_extractor.rb +65 -0
  30. data/lib/datadog/appsec/api_security/sampler.rb +59 -0
  31. data/lib/datadog/appsec/api_security.rb +23 -0
  32. data/lib/datadog/appsec/assets/waf_rules/recommended.json +257 -85
  33. data/lib/datadog/appsec/assets/waf_rules/strict.json +10 -78
  34. data/lib/datadog/appsec/component.rb +30 -54
  35. data/lib/datadog/appsec/configuration/settings.rb +60 -2
  36. data/lib/datadog/appsec/context.rb +6 -6
  37. data/lib/datadog/appsec/contrib/devise/tracking_middleware.rb +1 -1
  38. data/lib/datadog/appsec/contrib/rack/request_middleware.rb +27 -16
  39. data/lib/datadog/appsec/processor/rule_loader.rb +5 -6
  40. data/lib/datadog/appsec/remote.rb +15 -55
  41. data/lib/datadog/appsec/security_engine/engine.rb +194 -0
  42. data/lib/datadog/appsec/security_engine/runner.rb +10 -11
  43. data/lib/datadog/appsec.rb +4 -7
  44. data/lib/datadog/core/buffer/random.rb +18 -2
  45. data/lib/datadog/core/configuration/agent_settings.rb +52 -0
  46. data/lib/datadog/core/configuration/agent_settings_resolver.rb +4 -46
  47. data/lib/datadog/core/configuration/components.rb +31 -24
  48. data/lib/datadog/core/configuration/components_state.rb +23 -0
  49. data/lib/datadog/core/configuration/option.rb +27 -27
  50. data/lib/datadog/core/configuration/option_definition.rb +4 -4
  51. data/lib/datadog/core/configuration/options.rb +1 -1
  52. data/lib/datadog/core/configuration/settings.rb +32 -20
  53. data/lib/datadog/core/configuration/stable_config.rb +1 -2
  54. data/lib/datadog/core/configuration.rb +16 -16
  55. data/lib/datadog/core/crashtracking/component.rb +2 -1
  56. data/lib/datadog/core/crashtracking/tag_builder.rb +4 -22
  57. data/lib/datadog/core/encoding.rb +1 -1
  58. data/lib/datadog/core/environment/cgroup.rb +10 -12
  59. data/lib/datadog/core/environment/container.rb +38 -40
  60. data/lib/datadog/core/environment/ext.rb +6 -6
  61. data/lib/datadog/core/environment/identity.rb +3 -3
  62. data/lib/datadog/core/environment/platform.rb +3 -3
  63. data/lib/datadog/core/error.rb +11 -9
  64. data/lib/datadog/core/logger.rb +2 -2
  65. data/lib/datadog/core/metrics/client.rb +12 -14
  66. data/lib/datadog/core/metrics/logging.rb +5 -5
  67. data/lib/datadog/core/process_discovery/tracer_memfd.rb +15 -0
  68. data/lib/datadog/core/process_discovery.rb +5 -1
  69. data/lib/datadog/core/rate_limiter.rb +4 -2
  70. data/lib/datadog/core/remote/client.rb +32 -31
  71. data/lib/datadog/core/remote/component.rb +3 -3
  72. data/lib/datadog/core/remote/configuration/digest.rb +7 -7
  73. data/lib/datadog/core/remote/configuration/path.rb +1 -1
  74. data/lib/datadog/core/remote/configuration/repository.rb +12 -0
  75. data/lib/datadog/core/remote/transport/http/client.rb +1 -1
  76. data/lib/datadog/core/remote/transport/http/config.rb +21 -5
  77. data/lib/datadog/core/remote/transport/http/negotiation.rb +1 -1
  78. data/lib/datadog/core/runtime/metrics.rb +3 -3
  79. data/lib/datadog/core/tag_builder.rb +56 -0
  80. data/lib/datadog/core/telemetry/component.rb +39 -24
  81. data/lib/datadog/core/telemetry/emitter.rb +7 -1
  82. data/lib/datadog/core/telemetry/event/app_client_configuration_change.rb +66 -0
  83. data/lib/datadog/core/telemetry/event/app_closing.rb +18 -0
  84. data/lib/datadog/core/telemetry/event/app_dependencies_loaded.rb +33 -0
  85. data/lib/datadog/core/telemetry/event/app_heartbeat.rb +18 -0
  86. data/lib/datadog/core/telemetry/event/app_integrations_change.rb +58 -0
  87. data/lib/datadog/core/telemetry/event/app_started.rb +269 -0
  88. data/lib/datadog/core/telemetry/event/base.rb +40 -0
  89. data/lib/datadog/core/telemetry/event/distributions.rb +18 -0
  90. data/lib/datadog/core/telemetry/event/generate_metrics.rb +43 -0
  91. data/lib/datadog/core/telemetry/event/log.rb +76 -0
  92. data/lib/datadog/core/telemetry/event/message_batch.rb +42 -0
  93. data/lib/datadog/core/telemetry/event/synth_app_client_configuration_change.rb +43 -0
  94. data/lib/datadog/core/telemetry/event.rb +17 -475
  95. data/lib/datadog/core/telemetry/logger.rb +5 -4
  96. data/lib/datadog/core/telemetry/logging.rb +11 -5
  97. data/lib/datadog/core/telemetry/metric.rb +3 -3
  98. data/lib/datadog/core/telemetry/transport/http/telemetry.rb +2 -2
  99. data/lib/datadog/core/telemetry/transport/telemetry.rb +0 -1
  100. data/lib/datadog/core/telemetry/worker.rb +48 -27
  101. data/lib/datadog/core/transport/http/adapters/net.rb +17 -2
  102. data/lib/datadog/core/transport/http/adapters/test.rb +2 -1
  103. data/lib/datadog/core/transport/http/builder.rb +14 -14
  104. data/lib/datadog/core/transport/http/env.rb +8 -0
  105. data/lib/datadog/core/utils/at_fork_monkey_patch.rb +6 -6
  106. data/lib/datadog/core/utils/duration.rb +32 -32
  107. data/lib/datadog/core/utils/forking.rb +2 -2
  108. data/lib/datadog/core/utils/network.rb +6 -6
  109. data/lib/datadog/core/utils/only_once_successful.rb +16 -5
  110. data/lib/datadog/core/utils/time.rb +10 -2
  111. data/lib/datadog/core/utils/truncation.rb +21 -0
  112. data/lib/datadog/core/utils.rb +7 -0
  113. data/lib/datadog/core/vendor/multipart-post/multipart/post/composite_read_io.rb +1 -1
  114. data/lib/datadog/core/vendor/multipart-post/multipart/post/multipartable.rb +8 -8
  115. data/lib/datadog/core/vendor/multipart-post/multipart/post/parts.rb +7 -7
  116. data/lib/datadog/core/worker.rb +1 -1
  117. data/lib/datadog/core/workers/async.rb +9 -10
  118. data/lib/datadog/di/instrumenter.rb +52 -2
  119. data/lib/datadog/di/probe_notification_builder.rb +31 -41
  120. data/lib/datadog/di/probe_notifier_worker.rb +9 -1
  121. data/lib/datadog/di/serializer.rb +6 -2
  122. data/lib/datadog/di/transport/http/input.rb +10 -0
  123. data/lib/datadog/di/transport/input.rb +10 -2
  124. data/lib/datadog/error_tracking/component.rb +2 -2
  125. data/lib/datadog/profiling/collectors/code_provenance.rb +18 -9
  126. data/lib/datadog/profiling/collectors/cpu_and_wall_time_worker.rb +4 -0
  127. data/lib/datadog/profiling/collectors/idle_sampling_helper.rb +1 -0
  128. data/lib/datadog/profiling/collectors/thread_context.rb +16 -1
  129. data/lib/datadog/profiling/component.rb +7 -9
  130. data/lib/datadog/profiling/ext.rb +0 -13
  131. data/lib/datadog/profiling/flush.rb +1 -1
  132. data/lib/datadog/profiling/http_transport.rb +3 -8
  133. data/lib/datadog/profiling/profiler.rb +2 -0
  134. data/lib/datadog/profiling/scheduler.rb +10 -2
  135. data/lib/datadog/profiling/stack_recorder.rb +5 -5
  136. data/lib/datadog/profiling/tag_builder.rb +5 -41
  137. data/lib/datadog/profiling/tasks/setup.rb +2 -0
  138. data/lib/datadog/tracing/contrib/action_pack/action_controller/instrumentation.rb +15 -0
  139. data/lib/datadog/tracing/contrib/action_pack/action_dispatch/instrumentation.rb +19 -12
  140. data/lib/datadog/tracing/contrib/action_pack/ext.rb +2 -0
  141. data/lib/datadog/tracing/contrib/active_support/cache/events/cache.rb +4 -1
  142. data/lib/datadog/tracing/contrib/active_support/cache/instrumentation.rb +33 -0
  143. data/lib/datadog/tracing/contrib/active_support/cache/patcher.rb +4 -0
  144. data/lib/datadog/tracing/contrib/active_support/cache/redis.rb +2 -4
  145. data/lib/datadog/tracing/contrib/aws/instrumentation.rb +10 -0
  146. data/lib/datadog/tracing/contrib/aws/parsed_context.rb +5 -1
  147. data/lib/datadog/tracing/contrib/http/instrumentation.rb +1 -5
  148. data/lib/datadog/tracing/contrib/httpclient/instrumentation.rb +1 -5
  149. data/lib/datadog/tracing/contrib/httprb/instrumentation.rb +1 -5
  150. data/lib/datadog/tracing/contrib/lograge/patcher.rb +4 -2
  151. data/lib/datadog/tracing/contrib/patcher.rb +5 -2
  152. data/lib/datadog/tracing/contrib/sidekiq/ext.rb +1 -0
  153. data/lib/datadog/tracing/contrib/sidekiq/server_tracer.rb +5 -2
  154. data/lib/datadog/tracing/contrib/support.rb +28 -0
  155. data/lib/datadog/tracing/metadata/errors.rb +4 -4
  156. data/lib/datadog/tracing/sync_writer.rb +1 -1
  157. data/lib/datadog/tracing/trace_operation.rb +12 -4
  158. data/lib/datadog/tracing/tracer.rb +6 -2
  159. data/lib/datadog/version.rb +1 -1
  160. metadata +31 -12
  161. data/lib/datadog/appsec/assets/waf_rules/processors.json +0 -321
  162. data/lib/datadog/appsec/assets/waf_rules/scanners.json +0 -1023
  163. data/lib/datadog/appsec/processor/rule_merger.rb +0 -171
  164. data/lib/datadog/appsec/processor.rb +0 -107
@@ -52,6 +52,17 @@ module Datadog
52
52
  # (however, Probe instances can be replaced by OpenStruct instances
53
53
  # providing the same interface with not much effort).
54
54
  #
55
+ # Instrumenter (this class) is responsible for building snapshots.
56
+ # This is because to capture values on method entry, those values need to
57
+ # be duplicated or serialized into immutable values to prevent their
58
+ # modification by the instrumented method. Therefore this class must
59
+ # do at least some serialization/snapshot building and to keep the code
60
+ # well-encapsulated, all serialization/snapshot building should thus be
61
+ # initiated from this class rather than downstream code.
62
+ #
63
+ # As a consequence of Instrumenter building snapshots, it should not
64
+ # expose TracePoint objects to any downstream code.
65
+ #
55
66
  # @api private
56
67
  class Instrumenter
57
68
  def initialize(settings, serializer, logger, code_tracker: nil, telemetry: nil)
@@ -111,7 +122,8 @@ module Datadog
111
122
  # Arguments may be mutated by the method, therefore
112
123
  # they need to be serialized prior to method invocation.
113
124
  entry_args = if probe.capture_snapshot?
114
- serializer.serialize_args(args, kwargs,
125
+ instance_vars = Instrumenter.get_instance_variables(self)
126
+ serializer.serialize_args(args, kwargs, instance_vars,
115
127
  depth: probe.max_capture_depth || settings.dynamic_instrumentation.max_capture_depth,
116
128
  attribute_count: probe.max_capture_attribute_count || settings.dynamic_instrumentation.max_capture_attribute_count)
117
129
  end
@@ -149,6 +161,7 @@ module Datadog
149
161
  # TODO capture arguments at exit
150
162
  # & is to stop steep complaints, block is always present here.
151
163
  block&.call(probe: probe, rv: rv, duration: duration, caller_locations: caller_locs,
164
+ instance_vars: probe.capture_snapshot? ? Instrumenter.get_instance_variables(self) : nil,
152
165
  serialized_entry_args: entry_args)
153
166
  rv
154
167
  else
@@ -298,8 +311,20 @@ module Datadog
298
311
  probe.file == tp.path || probe.file_matches?(tp.path)
299
312
  )
300
313
  if rate_limiter.nil? || rate_limiter.allow?
314
+ locals = if probe.capture_snapshot?
315
+ serializer.serialize_vars(Instrumenter.get_local_variables(tp),
316
+ depth: probe.max_capture_depth || settings.dynamic_instrumentation.max_capture_depth,
317
+ attribute_count: probe.max_capture_attribute_count || settings.dynamic_instrumentation.max_capture_attribute_count,)
318
+ end
319
+ instance_vars = if probe.capture_snapshot?
320
+ serializer.serialize_vars(Instrumenter.get_instance_variables(tp.self),
321
+ depth: probe.max_capture_depth || settings.dynamic_instrumentation.max_capture_depth,
322
+ attribute_count: probe.max_capture_attribute_count || settings.dynamic_instrumentation.max_capture_attribute_count,)
323
+ end
301
324
  # & is to stop steep complaints, block is always present here.
302
- block&.call(probe: probe, trace_point: tp, caller_locations: caller_locations)
325
+ block&.call(probe: probe,
326
+ locals: locals, instance_vars: instance_vars,
327
+ path: tp.path, caller_locations: caller_locations)
303
328
  end
304
329
  end
305
330
  rescue => exc
@@ -371,6 +396,31 @@ module Datadog
371
396
  end
372
397
  end
373
398
 
399
+ class << self
400
+ def get_instance_variables(object)
401
+ {}.tap do |hash|
402
+ object.instance_variables.each do |var|
403
+ hash[var] = object.instance_variable_get(var)
404
+ end
405
+ end
406
+ end
407
+
408
+ def get_local_variables(trace_point)
409
+ # binding appears to be constructed on access, therefore
410
+ # 1) we should attempt to cache it and
411
+ # 2) we should not call +binding+ until we actually need variable values.
412
+ binding = trace_point.binding
413
+
414
+ # steep hack - should never happen
415
+ return {} unless binding
416
+
417
+ binding.local_variables.each_with_object({}) do |name, map|
418
+ value = binding.local_variable_get(name)
419
+ map[name] = value
420
+ end
421
+ end
422
+ end
423
+
374
424
  private
375
425
 
376
426
  attr_reader :lock
@@ -39,39 +39,50 @@ module Datadog
39
39
  end
40
40
 
41
41
  # Duration is in seconds.
42
+ # path is the actual path of the instrumented file.
42
43
  def build_executed(probe,
43
- trace_point: nil, rv: nil, duration: nil, caller_locations: nil,
44
- args: nil, kwargs: nil, serialized_entry_args: nil)
45
- snapshot = if probe.line? && probe.capture_snapshot?
46
- if trace_point.nil?
47
- raise "Cannot create snapshot because there is no trace point"
48
- end
49
- get_local_variables(trace_point)
50
- end
51
- # TODO check how many stack frames we should be keeping/sending,
52
- # this should be all frames for enriched probes and no frames for
53
- # non-enriched probes?
54
- build_snapshot(probe, rv: rv, snapshot: snapshot,
44
+ path: nil, rv: nil, duration: nil, caller_locations: nil,
45
+ locals: nil, args: nil, kwargs: nil, instance_vars: nil,
46
+ serialized_entry_args: nil)
47
+ build_snapshot(probe, rv: rv, locals: locals,
55
48
  # Actual path of the instrumented file.
56
- path: trace_point&.path,
57
- duration: duration, caller_locations: caller_locations, args: args, kwargs: kwargs,
49
+ path: path,
50
+ duration: duration,
51
+ # TODO check how many stack frames we should be keeping/sending,
52
+ # this should be all frames for enriched probes and no frames for
53
+ # non-enriched probes?
54
+ caller_locations: caller_locations,
55
+ args: args, kwargs: kwargs, instance_vars: instance_vars,
58
56
  serialized_entry_args: serialized_entry_args)
59
57
  end
60
58
 
61
- def build_snapshot(probe, rv: nil, snapshot: nil, path: nil,
62
- duration: nil, caller_locations: nil, args: nil, kwargs: nil,
59
+ def build_snapshot(probe, rv: nil, locals: nil, path: nil,
60
+ duration: nil, caller_locations: nil,
61
+ args: nil, kwargs: nil, instance_vars: nil,
63
62
  serialized_entry_args: nil)
64
63
  # TODO also verify that non-capturing probe does not pass
65
64
  # snapshot or vars/args into this method
66
65
  captures = if probe.capture_snapshot?
67
66
  if probe.method?
67
+ return_arguments = {
68
+ "@return": serializer.serialize_value(rv,
69
+ depth: probe.max_capture_depth || settings.dynamic_instrumentation.max_capture_depth,
70
+ attribute_count: probe.max_capture_attribute_count || settings.dynamic_instrumentation.max_capture_attribute_count),
71
+ }
72
+ if instance_vars
73
+ return_arguments.update(
74
+ serializer.serialize_vars(instance_vars,
75
+ depth: probe.max_capture_depth || settings.dynamic_instrumentation.max_capture_depth,
76
+ attribute_count: probe.max_capture_attribute_count || settings.dynamic_instrumentation.max_capture_attribute_count,)
77
+ )
78
+ end
68
79
  {
69
80
  entry: {
70
81
  # standard:disable all
71
82
  arguments: if serialized_entry_args
72
83
  serialized_entry_args
73
84
  else
74
- (args || kwargs) && serializer.serialize_args(args, kwargs,
85
+ (args || kwargs) && serializer.serialize_args(args, kwargs, instance_vars,
75
86
  depth: probe.max_capture_depth || settings.dynamic_instrumentation.max_capture_depth,
76
87
  attribute_count: probe.max_capture_attribute_count || settings.dynamic_instrumentation.max_capture_attribute_count)
77
88
  end,
@@ -79,20 +90,14 @@ module Datadog
79
90
  # standard:enable all
80
91
  },
81
92
  return: {
82
- arguments: {
83
- "@return": serializer.serialize_value(rv,
84
- depth: probe.max_capture_depth || settings.dynamic_instrumentation.max_capture_depth,
85
- attribute_count: probe.max_capture_attribute_count || settings.dynamic_instrumentation.max_capture_attribute_count),
86
- },
93
+ arguments: return_arguments,
87
94
  throwable: nil,
88
95
  },
89
96
  }
90
97
  elsif probe.line?
91
98
  {
92
- lines: snapshot && {
93
- probe.line_no => {locals: serializer.serialize_vars(snapshot,
94
- depth: probe.max_capture_depth || settings.dynamic_instrumentation.max_capture_depth,
95
- attribute_count: probe.max_capture_attribute_count || settings.dynamic_instrumentation.max_capture_attribute_count,)},
99
+ lines: locals && {
100
+ probe.line_no => {locals: locals.merge(instance_vars || {})},
96
101
  },
97
102
  }
98
103
  end
@@ -194,21 +199,6 @@ module Datadog
194
199
  (Core::Utils::Time.now.to_f * 1000).to_i
195
200
  end
196
201
 
197
- def get_local_variables(trace_point)
198
- # binding appears to be constructed on access, therefore
199
- # 1) we should attempt to cache it and
200
- # 2) we should not call +binding+ until we actually need variable values.
201
- binding = trace_point.binding
202
-
203
- # steep hack - should never happen
204
- return {} unless binding
205
-
206
- binding.local_variables.each_with_object({}) do |name, map|
207
- value = binding.local_variable_get(name)
208
- map[name] = value
209
- end
210
- end
211
-
212
202
  def active_trace
213
203
  if defined?(Datadog::Tracing)
214
204
  Datadog::Tracing.active_trace
@@ -183,7 +183,15 @@ module Datadog
183
183
  end
184
184
 
185
185
  def do_send_snapshot(batch)
186
- snapshot_transport.send_input(batch)
186
+ snapshot_transport.send_input(batch, tags)
187
+ end
188
+
189
+ def tags
190
+ # DEV: The tags could be cached but they need to be recreated
191
+ # when process forks (since the child receives new runtime IDs).
192
+ Core::TagBuilder.tags(settings).merge(
193
+ 'debugger_version' => Core::Environment::Identity.gem_datadog_version,
194
+ )
187
195
  end
188
196
 
189
197
  [
@@ -82,7 +82,11 @@ module Datadog
82
82
  # between positional and keyword arguments. We convert positional
83
83
  # arguments to keyword arguments ("arg1", "arg2", ...) and ensure
84
84
  # the positional arguments are listed first.
85
- def serialize_args(args, kwargs,
85
+ #
86
+ # Instance variables are technically a hash just like kwargs,
87
+ # we take them as a separate parameter to avoid a hash merge
88
+ # in upstream code.
89
+ def serialize_args(args, kwargs, instance_vars,
86
90
  depth: settings.dynamic_instrumentation.max_capture_depth,
87
91
  attribute_count: settings.dynamic_instrumentation.max_capture_attribute_count)
88
92
  counter = 0
@@ -91,7 +95,7 @@ module Datadog
91
95
  # Conversion to symbol is needed here to put args ahead of
92
96
  # kwargs when they are merged below.
93
97
  c[:"arg#{counter}"] = value
94
- end.update(kwargs)
98
+ end.update(kwargs).update(instance_vars)
95
99
  serialize_vars(combined, depth: depth, attribute_count: attribute_count)
96
100
  end
97
101
 
@@ -53,6 +53,16 @@ module Datadog
53
53
  # Encode body & type
54
54
  env.headers[HEADER_CONTENT_TYPE] = encoder.content_type
55
55
  env.body = env.request.parcel.data
56
+ env.query = {
57
+ # DEV: In theory we could serialize the tags here
58
+ # rather than requiring them to be pre-serialized.
59
+ # In practice the tags should be relatively static
60
+ # (they would change when process forks, and hostname
61
+ # could change at any time but probably we should ignore
62
+ # those changes), therefore serializing the tags
63
+ # every time would be wasteful.
64
+ ddtags: env.request.serialized_tags,
65
+ }
56
66
 
57
67
  super
58
68
  end
@@ -12,6 +12,13 @@ module Datadog
12
12
  end
13
13
 
14
14
  class Request < Datadog::Core::Transport::Request
15
+ attr_reader :serialized_tags
16
+
17
+ def initialize(parcel, serialized_tags)
18
+ super(parcel)
19
+
20
+ @serialized_tags = serialized_tags
21
+ end
15
22
  end
16
23
 
17
24
  class Transport
@@ -28,10 +35,11 @@ module Datadog
28
35
  @apis[HTTP::API::INPUT]
29
36
  end
30
37
 
31
- def send_input(payload)
38
+ def send_input(payload, tags)
32
39
  json = JSON.dump(payload)
33
40
  parcel = EncodedParcel.new(json)
34
- request = Request.new(parcel)
41
+ serialized_tags = Core::TagBuilder.serialize_tags(tags)
42
+ request = Request.new(parcel, serialized_tags)
35
43
 
36
44
  response = @client.send_input_payload(request)
37
45
  unless response.ok?
@@ -33,9 +33,9 @@ module Datadog
33
33
  if RUBY_ENGINE != 'ruby'
34
34
  logger.warn("error tracking: cannot enable error tracking: MRI is required, but running on #{RUBY_ENGINE}")
35
35
  false
36
- elsif RUBY_VERSION < '2.6'
36
+ elsif RUBY_VERSION < '2.7'
37
37
  logger.warn(
38
- "error tracking: cannot enable error tracking: Ruby 2.6+ is required, but running
38
+ "error tracking: cannot enable error tracking: Ruby 2.7+ is required, but running
39
39
  on #{RUBY_VERSION}"
40
40
  )
41
41
  false
@@ -14,7 +14,10 @@ module Datadog
14
14
  #
15
15
  # This class acts both as a collector (collecting data) as well as a recorder (records/serializes it)
16
16
  class CodeProvenance
17
- def initialize(standard_library_path: RbConfig::CONFIG.fetch("rubylibdir"))
17
+ def initialize(
18
+ standard_library_path: RbConfig::CONFIG.fetch("rubylibdir"),
19
+ ruby_native_filename: Datadog::Profiling::Collectors::Stack._native_ruby_native_filename
20
+ )
18
21
  @libraries_by_name = {}
19
22
  @libraries_by_path = {}
20
23
  @seen_files = Set.new
@@ -26,6 +29,7 @@ module Datadog
26
29
  name: "stdlib",
27
30
  version: RUBY_VERSION,
28
31
  path: standard_library_path,
32
+ extra_path: ruby_native_filename,
29
33
  )
30
34
  )
31
35
  end
@@ -37,12 +41,8 @@ module Datadog
37
41
  self
38
42
  end
39
43
 
40
- def generate
41
- seen_libraries
42
- end
43
-
44
44
  def generate_json
45
- JSON.fast_generate(v1: seen_libraries.to_a)
45
+ JSON.generate(v1: seen_libraries.to_a)
46
46
  end
47
47
 
48
48
  private
@@ -79,7 +79,15 @@ module Datadog
79
79
  loaded_specs.each do |spec|
80
80
  next if libraries_by_name.key?(spec.name)
81
81
 
82
- record_library(Library.new(kind: "library", name: spec.name, version: spec.version, path: spec.gem_dir))
82
+ record_library(
83
+ Library.new(
84
+ kind: "library",
85
+ name: spec.name,
86
+ version: spec.version,
87
+ path: spec.gem_dir,
88
+ extra_path: (spec.extension_dir if spec.extensions.any?),
89
+ )
90
+ )
83
91
  recorded_library = true
84
92
  end
85
93
 
@@ -110,11 +118,12 @@ module Datadog
110
118
  class Library
111
119
  attr_reader :kind, :name, :version
112
120
 
113
- def initialize(kind:, name:, version:, path:)
121
+ def initialize(kind:, name:, version:, path:, extra_path: nil)
122
+ extra_path = nil if extra_path&.empty?
114
123
  @kind = kind.freeze
115
124
  @name = name.dup.freeze
116
125
  @version = version.to_s.dup.freeze
117
- @paths = [path.dup.freeze].freeze
126
+ @paths = [path.dup.freeze, extra_path.dup.freeze].compact.freeze
118
127
  freeze
119
128
  end
120
129
 
@@ -33,6 +33,9 @@ module Datadog
33
33
  Datadog.logger.warn(
34
34
  "Profiling dynamic sampling rate disabled. This should only be used for testing, and will increase overhead!"
35
35
  )
36
+ Datadog::Core::Telemetry::Logger.error(
37
+ "Profiling dynamic sampling rate disabled. This should only be used for testing, and will increase overhead!"
38
+ )
36
39
  end
37
40
 
38
41
  self.class._native_initialize(
@@ -77,6 +80,7 @@ module Datadog
77
80
  "Cause: #{e.class.name} #{e.message} Location: #{Array(e.backtrace).first}"
78
81
  )
79
82
  on_failure_proc&.call
83
+ Datadog::Core::Telemetry::Logger.report(e, description: "CpuAndWallTimeWorker thread error", pii_safe: true)
80
84
  end
81
85
  @worker_thread.name = self.class.name # Repeated from above to make sure thread gets named asap
82
86
  @worker_thread.thread_variable_set(:fork_safe, true)
@@ -41,6 +41,7 @@ module Datadog
41
41
  "IdleSamplingHelper thread error. " \
42
42
  "Cause: #{e.class.name} #{e.message} Location: #{Array(e.backtrace).first}"
43
43
  )
44
+ Datadog::Core::Telemetry::Logger.report(e, description: "IdleSamplingHelper thread error", pii_safe: true)
44
45
  end
45
46
  @worker_thread.name = self.class.name # Repeated from above to make sure thread gets named asap
46
47
  @worker_thread.thread_variable_set(:fork_safe, true)
@@ -21,7 +21,8 @@ module Datadog
21
21
  endpoint_collection_enabled:,
22
22
  timeline_enabled:,
23
23
  waiting_for_gvl_threshold_ns:,
24
- otel_context_enabled:
24
+ otel_context_enabled:,
25
+ native_filenames_enabled:
25
26
  )
26
27
  tracer_context_key = safely_extract_context_key_from(tracer)
27
28
  self.class._native_initialize(
@@ -33,6 +34,7 @@ module Datadog
33
34
  timeline_enabled: timeline_enabled,
34
35
  waiting_for_gvl_threshold_ns: waiting_for_gvl_threshold_ns,
35
36
  otel_context_enabled: otel_context_enabled,
37
+ native_filenames_enabled: validate_native_filenames(native_filenames_enabled),
36
38
  )
37
39
  end
38
40
 
@@ -44,6 +46,7 @@ module Datadog
44
46
  timeline_enabled: false,
45
47
  waiting_for_gvl_threshold_ns: 10_000_000,
46
48
  otel_context_enabled: false,
49
+ native_filenames_enabled: true,
47
50
  **options
48
51
  )
49
52
  new(
@@ -54,6 +57,7 @@ module Datadog
54
57
  timeline_enabled: timeline_enabled,
55
58
  waiting_for_gvl_threshold_ns: waiting_for_gvl_threshold_ns,
56
59
  otel_context_enabled: otel_context_enabled,
60
+ native_filenames_enabled: native_filenames_enabled,
57
61
  **options,
58
62
  )
59
63
  end
@@ -81,6 +85,17 @@ module Datadog
81
85
  context = provider.instance_variable_get(:@context)
82
86
  context&.instance_variable_get(:@key)
83
87
  end
88
+
89
+ def validate_native_filenames(native_filenames_enabled)
90
+ if native_filenames_enabled && !Datadog::Profiling::Collectors::Stack._native_filenames_available?
91
+ Datadog.logger.debug(
92
+ "Native filenames are enabled, but the required dladdr API was not available. Disabling native filenames."
93
+ )
94
+ false
95
+ else
96
+ native_filenames_enabled
97
+ end
98
+ end
84
99
  end
85
100
  end
86
101
  end
@@ -96,6 +96,7 @@ module Datadog
96
96
  timeline_enabled: timeline_enabled,
97
97
  waiting_for_gvl_threshold_ns: settings.profiling.advanced.waiting_for_gvl_threshold_ns,
98
98
  otel_context_enabled: settings.profiling.advanced.preview_otel_context_enabled,
99
+ native_filenames_enabled: settings.profiling.advanced.native_filenames_enabled,
99
100
  )
100
101
  end
101
102
 
@@ -221,13 +222,14 @@ module Datadog
221
222
  end
222
223
 
223
224
  unless allocation_profiling_enabled
224
- raise ArgumentError, "Heap profiling requires allocation profiling to be enabled"
225
+ logger.warn(
226
+ "Heap profiling was requested but allocation profiling is not enabled. " \
227
+ "Heap profiling has been disabled."
228
+ )
229
+ return false
225
230
  end
226
231
 
227
- logger.warn(
228
- "Enabled experimental heap profiling: heap_sample_rate=#{heap_sample_rate}. This is experimental, not " \
229
- "recommended, and will increase overhead!"
230
- )
232
+ logger.debug("Enabled heap profiling: heap_sample_rate=#{heap_sample_rate}")
231
233
 
232
234
  true
233
235
  end
@@ -237,10 +239,6 @@ module Datadog
237
239
 
238
240
  return false unless heap_profiling_enabled && heap_size_profiling_enabled
239
241
 
240
- logger.warn(
241
- "Enabled experimental heap size profiling. This is experimental, not recommended, and will increase overhead!"
242
- )
243
-
244
242
  true
245
243
  end
246
244
 
@@ -11,20 +11,7 @@ module Datadog
11
11
 
12
12
  module Transport
13
13
  module HTTP
14
- FORM_FIELD_TAG_ENV = "env"
15
- FORM_FIELD_TAG_HOST = "host"
16
- FORM_FIELD_TAG_LANGUAGE = "language"
17
- FORM_FIELD_TAG_PID = "process_id"
18
14
  FORM_FIELD_TAG_PROFILER_VERSION = "profiler_version"
19
- FORM_FIELD_TAG_RUNTIME = "runtime"
20
- FORM_FIELD_TAG_RUNTIME_ENGINE = "runtime_engine"
21
- FORM_FIELD_TAG_RUNTIME_ID = "runtime-id"
22
- FORM_FIELD_TAG_RUNTIME_PLATFORM = "runtime_platform"
23
- FORM_FIELD_TAG_RUNTIME_VERSION = "runtime_version"
24
- FORM_FIELD_TAG_SERVICE = "service"
25
- FORM_FIELD_TAG_VERSION = "version"
26
- TAG_GIT_REPOSITORY_URL = "git.repository_url"
27
- TAG_GIT_COMMIT_SHA = "git.commit.sha"
28
15
 
29
16
  CODE_PROVENANCE_FILENAME = "code-provenance.json"
30
17
  end
@@ -32,7 +32,7 @@ module Datadog
32
32
  @code_provenance_file_name = code_provenance_file_name
33
33
  @code_provenance_data = code_provenance_data
34
34
  @tags_as_array = tags_as_array
35
- @internal_metadata_json = JSON.fast_generate(internal_metadata)
35
+ @internal_metadata_json = JSON.generate(internal_metadata)
36
36
  @info_json = info_json
37
37
  end
38
38
  end
@@ -29,12 +29,7 @@ module Datadog
29
29
  status, result = self.class._native_do_export(
30
30
  exporter_configuration,
31
31
  @upload_timeout_milliseconds,
32
- flush,
33
- # TODO: This is going to be removed once we move to libdatadog 17
34
- flush.start.tv_sec,
35
- flush.start.tv_nsec,
36
- flush.finish.tv_sec,
37
- flush.finish.tv_nsec,
32
+ flush
38
33
  )
39
34
 
40
35
  if status == :ok
@@ -42,7 +37,7 @@ module Datadog
42
37
  Datadog.logger.debug("Successfully reported profiling data")
43
38
  true
44
39
  else
45
- Datadog.logger.error(
40
+ Datadog.logger.warn(
46
41
  "Failed to report profiling data (#{config_without_api_key}): " \
47
42
  "server returned unexpected HTTP #{result} status code"
48
43
  )
@@ -52,7 +47,7 @@ module Datadog
52
47
  false
53
48
  end
54
49
  else
55
- Datadog.logger.error("Failed to report profiling data (#{config_without_api_key}): #{result}")
50
+ Datadog.logger.warn("Failed to report profiling data (#{config_without_api_key}): #{result}")
56
51
  Datadog::Core::Telemetry::Logger.error("Failed to report profiling data")
57
52
  false
58
53
  end
@@ -50,6 +50,8 @@ module Datadog
50
50
  "Detected issue with profiler (#{failed_component} component), stopping profiling. " \
51
51
  "See previous log messages for details."
52
52
  )
53
+ Datadog::Core::Telemetry::Logger
54
+ .error("Detected issue with profiler (#{failed_component} component), stopping profiling")
53
55
 
54
56
  # We explicitly not stop the crash tracker in this situation, under the assumption that, if a component failed,
55
57
  # we're operating in a degraded state and crash tracking may still be helpful.
@@ -37,6 +37,7 @@ module Datadog
37
37
  @exporter = exporter
38
38
  @transport = transport
39
39
  @profiler_failed = false
40
+ @stop_requested = false
40
41
 
41
42
  # Workers::Async::Thread settings
42
43
  self.fork_policy = fork_policy
@@ -67,6 +68,7 @@ module Datadog
67
68
  "Cause: #{e.class.name} #{e.message} Location: #{Array(e.backtrace).first}"
68
69
  )
69
70
  on_failure_proc&.call
71
+ Datadog::Core::Telemetry::Logger.report(e, description: "Profiling::Scheduler thread error")
70
72
  raise
71
73
  ensure
72
74
  Datadog.logger.debug("#flush was interrupted or failed before it could complete") if interrupted
@@ -88,7 +90,7 @@ module Datadog
88
90
  end
89
91
 
90
92
  def work_pending?
91
- !profiler_failed && exporter.can_flush?
93
+ !profiler_failed && exporter.can_flush? && (run_loop? || !stop_requested?)
92
94
  end
93
95
 
94
96
  def reset_after_fork
@@ -132,14 +134,20 @@ module Datadog
132
134
  begin
133
135
  transport.export(flush)
134
136
  rescue => e
135
- Datadog.logger.error(
137
+ Datadog.logger.warn(
136
138
  "Unable to report profile. Cause: #{e.class.name} #{e.message} Location: #{Array(e.backtrace).first}"
137
139
  )
138
140
  Datadog::Core::Telemetry::Logger.report(e, description: "Unable to report profile")
139
141
  end
140
142
 
143
+ @stop_requested = !run_loop?
144
+
141
145
  true
142
146
  end
147
+
148
+ def stop_requested?
149
+ @stop_requested
150
+ end
143
151
  end
144
152
  end
145
153
  end
@@ -23,7 +23,7 @@ module Datadog
23
23
  # This isn't something we expect to happen normally, but because it would break the assumptions of the
24
24
  # C-level mutexes (that there is a single serializer thread), we add it here as an extra safeguard against it
25
25
  # accidentally happening.
26
- @no_concurrent_synchronize_mutex = Mutex.new
26
+ @no_concurrent_serialize_mutex = Mutex.new
27
27
 
28
28
  self.class._native_initialize(
29
29
  self_instance: self,
@@ -60,7 +60,7 @@ module Datadog
60
60
  end
61
61
 
62
62
  def serialize
63
- status, result = @no_concurrent_synchronize_mutex.synchronize { self.class._native_serialize(self) }
63
+ status, result = @no_concurrent_serialize_mutex.synchronize { self.class._native_serialize(self) }
64
64
 
65
65
  if status == :ok
66
66
  start, finish, encoded_profile, profile_stats = result
@@ -71,15 +71,15 @@ module Datadog
71
71
  else
72
72
  error_message = result
73
73
 
74
- Datadog.logger.error("Failed to serialize profiling data: #{error_message}")
75
- Datadog::Core::Telemetry::Logger.error("Failed to serialize profiling data")
74
+ Datadog.logger.warn("Failed to serialize profiling data: #{error_message}")
75
+ Datadog::Core::Telemetry::Logger.error("Failed to serialize profiling data (#{error_message})")
76
76
 
77
77
  nil
78
78
  end
79
79
  end
80
80
 
81
81
  def serialize!
82
- status, result = @no_concurrent_synchronize_mutex.synchronize { self.class._native_serialize(self) }
82
+ status, result = @no_concurrent_serialize_mutex.synchronize { self.class._native_serialize(self) }
83
83
 
84
84
  if status == :ok
85
85
  _start, _finish, encoded_profile = result