datadog 2.31.0 → 2.33.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (202) hide show
  1. checksums.yaml +4 -4
  2. data/ext/datadog_profiling_native_extension/clock_id.h +9 -1
  3. data/ext/datadog_profiling_native_extension/clock_id_from_mach.c +73 -0
  4. data/ext/datadog_profiling_native_extension/clock_id_from_pthread.c +1 -1
  5. data/ext/datadog_profiling_native_extension/collectors_cpu_and_wall_time_worker.c +17 -7
  6. data/ext/datadog_profiling_native_extension/collectors_thread_context.c +16 -5
  7. data/ext/datadog_profiling_native_extension/collectors_thread_context.h +6 -0
  8. data/ext/datadog_profiling_native_extension/extconf.rb +8 -4
  9. data/ext/datadog_profiling_native_extension/http_transport.c +10 -5
  10. data/ext/datadog_profiling_native_extension/stack_recorder.c +3 -9
  11. data/ext/datadog_profiling_native_extension/time_helpers.h +1 -0
  12. data/ext/libdatadog_api/crashtracker.c +2 -0
  13. data/ext/libdatadog_api/di.c +48 -0
  14. data/ext/libdatadog_api/extconf.rb +7 -4
  15. data/ext/libdatadog_extconf_helpers.rb +38 -1
  16. data/lib/datadog/ai_guard/autoload.rb +10 -0
  17. data/lib/datadog/ai_guard/component.rb +1 -1
  18. data/lib/datadog/ai_guard/configuration.rb +105 -2
  19. data/lib/datadog/ai_guard/contrib/auto_instrument.rb +24 -0
  20. data/lib/datadog/ai_guard/contrib/rack/integration.rb +42 -0
  21. data/lib/datadog/ai_guard/contrib/rack/patcher.rb +26 -0
  22. data/lib/datadog/ai_guard/contrib/rack/request_middleware.rb +83 -0
  23. data/lib/datadog/ai_guard/contrib/rails/integration.rb +41 -0
  24. data/lib/datadog/ai_guard/contrib/rails/patcher.rb +97 -0
  25. data/lib/datadog/ai_guard/evaluation.rb +2 -0
  26. data/lib/datadog/ai_guard/ext.rb +2 -0
  27. data/lib/datadog/ai_guard.rb +8 -0
  28. data/lib/datadog/appsec/autoload.rb +1 -1
  29. data/lib/datadog/appsec/component.rb +1 -1
  30. data/lib/datadog/appsec/configuration.rb +414 -1
  31. data/lib/datadog/appsec/contrib/aws_lambda/gateway/watcher.rb +75 -0
  32. data/lib/datadog/appsec/contrib/aws_lambda/integration.rb +39 -0
  33. data/lib/datadog/appsec/contrib/aws_lambda/patcher.rb +30 -0
  34. data/lib/datadog/appsec/contrib/aws_lambda/waf_addresses.rb +111 -0
  35. data/lib/datadog/appsec/contrib/devise/patches/signin_tracking_patch.rb +2 -1
  36. data/lib/datadog/appsec/contrib/rack/gateway/request.rb +1 -1
  37. data/lib/datadog/appsec/contrib/rails/patcher.rb +2 -2
  38. data/lib/datadog/appsec/metrics/telemetry.rb +13 -1
  39. data/lib/datadog/appsec/security_engine/runner.rb +1 -1
  40. data/lib/datadog/appsec/trace_keeper.rb +18 -6
  41. data/lib/datadog/appsec/utils/http/url_encoded.rb +2 -2
  42. data/lib/datadog/appsec.rb +1 -0
  43. data/lib/datadog/core/configuration/components.rb +1 -1
  44. data/lib/datadog/core/configuration/settings.rb +13 -0
  45. data/lib/datadog/core/configuration/supported_configurations.rb +4 -0
  46. data/lib/datadog/core/configuration.rb +1 -1
  47. data/lib/datadog/core/contrib/rails/utils.rb +1 -1
  48. data/lib/datadog/core/crashtracking/component.rb +3 -3
  49. data/lib/datadog/core/diagnostics/environment_logger.rb +3 -1
  50. data/lib/datadog/core/environment/container.rb +2 -2
  51. data/lib/datadog/core/environment/ext.rb +1 -0
  52. data/lib/datadog/core/environment/socket.rb +13 -0
  53. data/lib/datadog/core/feature_flags.rb +1 -1
  54. data/lib/datadog/core/metrics/client.rb +5 -5
  55. data/lib/datadog/core/remote/client.rb +1 -1
  56. data/lib/datadog/core/remote/component.rb +2 -2
  57. data/lib/datadog/core/runtime/metrics.rb +1 -1
  58. data/lib/datadog/core/telemetry/emitter.rb +1 -1
  59. data/lib/datadog/core/telemetry/event/app_started.rb +2 -2
  60. data/lib/datadog/core/transport/http.rb +2 -0
  61. data/lib/datadog/core/utils.rb +1 -1
  62. data/lib/datadog/core/workers/async.rb +1 -1
  63. data/lib/datadog/core.rb +1 -1
  64. data/lib/datadog/data_streams/configuration.rb +40 -1
  65. data/lib/datadog/data_streams/pathway_context.rb +1 -1
  66. data/lib/datadog/data_streams/processor.rb +1 -1
  67. data/lib/datadog/data_streams.rb +1 -1
  68. data/lib/datadog/di/base.rb +8 -5
  69. data/lib/datadog/di/code_tracker.rb +179 -1
  70. data/lib/datadog/di/component.rb +1 -1
  71. data/lib/datadog/di/configuration.rb +235 -2
  72. data/lib/datadog/di/instrumenter.rb +46 -26
  73. data/lib/datadog/di/probe_builder.rb +1 -1
  74. data/lib/datadog/di/probe_file_loader.rb +2 -2
  75. data/lib/datadog/di/probe_manager.rb +6 -6
  76. data/lib/datadog/di/probe_notification_builder.rb +1 -1
  77. data/lib/datadog/di/probe_notifier_worker.rb +2 -2
  78. data/lib/datadog/di/remote.rb +6 -6
  79. data/lib/datadog/di/serializer.rb +1 -1
  80. data/lib/datadog/di/transport/input.rb +3 -3
  81. data/lib/datadog/error_tracking/configuration.rb +55 -2
  82. data/lib/datadog/kit/enable_core_dumps.rb +1 -1
  83. data/lib/datadog/open_feature/component.rb +18 -1
  84. data/lib/datadog/open_feature/evaluation_engine.rb +3 -3
  85. data/lib/datadog/open_feature/exposures/reporter.rb +1 -1
  86. data/lib/datadog/open_feature/exposures/worker.rb +1 -1
  87. data/lib/datadog/open_feature/hooks/flag_eval_hook.rb +49 -0
  88. data/lib/datadog/open_feature/metrics/flag_eval_metrics.rb +149 -0
  89. data/lib/datadog/open_feature/provider.rb +19 -1
  90. data/lib/datadog/open_feature/remote.rb +1 -1
  91. data/lib/datadog/open_feature/transport.rb +1 -1
  92. data/lib/datadog/opentelemetry/metrics.rb +13 -4
  93. data/lib/datadog/opentelemetry/sdk/configurator.rb +1 -1
  94. data/lib/datadog/opentelemetry/sdk/id_generator.rb +16 -10
  95. data/lib/datadog/opentelemetry/sdk/metrics_exporter.rb +1 -1
  96. data/lib/datadog/profiling/collectors/code_provenance.rb +35 -9
  97. data/lib/datadog/profiling/collectors/cpu_and_wall_time_worker.rb +31 -2
  98. data/lib/datadog/profiling/collectors/idle_sampling_helper.rb +8 -2
  99. data/lib/datadog/profiling/collectors/info.rb +16 -3
  100. data/lib/datadog/profiling/component.rb +3 -6
  101. data/lib/datadog/profiling/exporter.rb +37 -12
  102. data/lib/datadog/profiling/ext.rb +0 -2
  103. data/lib/datadog/profiling/flush.rb +21 -12
  104. data/lib/datadog/profiling/http_transport.rb +12 -1
  105. data/lib/datadog/profiling/load_native_extension.rb +1 -1
  106. data/lib/datadog/profiling/profiler.rb +13 -1
  107. data/lib/datadog/profiling/scheduler.rb +2 -2
  108. data/lib/datadog/profiling/stack_recorder.rb +0 -4
  109. data/lib/datadog/profiling/tasks/exec.rb +8 -3
  110. data/lib/datadog/profiling/tasks/help.rb +1 -0
  111. data/lib/datadog/profiling/tasks/setup.rb +2 -2
  112. data/lib/datadog/single_step_instrument.rb +1 -1
  113. data/lib/datadog/symbol_database/configuration.rb +65 -0
  114. data/lib/datadog/symbol_database/extractor.rb +906 -0
  115. data/lib/datadog/symbol_database/file_hash.rb +46 -0
  116. data/lib/datadog/symbol_database/logger.rb +43 -0
  117. data/lib/datadog/symbol_database/scope.rb +102 -0
  118. data/lib/datadog/symbol_database/scope_batcher.rb +280 -0
  119. data/lib/datadog/symbol_database/service_version.rb +57 -0
  120. data/lib/datadog/symbol_database/symbol.rb +66 -0
  121. data/lib/datadog/symbol_database/transport/http/endpoint.rb +28 -0
  122. data/lib/datadog/symbol_database/transport/http.rb +45 -0
  123. data/lib/datadog/symbol_database/transport.rb +54 -0
  124. data/lib/datadog/symbol_database/uploader.rb +169 -0
  125. data/lib/datadog/symbol_database.rb +49 -0
  126. data/lib/datadog/tracing/buffer.rb +3 -3
  127. data/lib/datadog/tracing/configuration/settings.rb +1 -1
  128. data/lib/datadog/tracing/contrib/action_pack/action_controller/instrumentation.rb +5 -3
  129. data/lib/datadog/tracing/contrib/action_view/events/render_template.rb +1 -1
  130. data/lib/datadog/tracing/contrib/active_job/events/discard.rb +1 -1
  131. data/lib/datadog/tracing/contrib/active_job/events/enqueue.rb +1 -1
  132. data/lib/datadog/tracing/contrib/active_job/events/enqueue_at.rb +1 -1
  133. data/lib/datadog/tracing/contrib/active_job/events/enqueue_retry.rb +1 -1
  134. data/lib/datadog/tracing/contrib/active_job/events/perform.rb +1 -1
  135. data/lib/datadog/tracing/contrib/active_job/events/retry_stopped.rb +1 -1
  136. data/lib/datadog/tracing/contrib/active_model_serializers/events/render.rb +1 -1
  137. data/lib/datadog/tracing/contrib/active_model_serializers/events/serialize.rb +1 -1
  138. data/lib/datadog/tracing/contrib/active_record/configuration/resolver.rb +2 -2
  139. data/lib/datadog/tracing/contrib/active_record/events/instantiation.rb +1 -1
  140. data/lib/datadog/tracing/contrib/active_record/events/sql.rb +1 -1
  141. data/lib/datadog/tracing/contrib/active_record/utils.rb +1 -1
  142. data/lib/datadog/tracing/contrib/active_support/cache/events/cache.rb +1 -1
  143. data/lib/datadog/tracing/contrib/active_support/notifications/subscription.rb +2 -2
  144. data/lib/datadog/tracing/contrib/aws/instrumentation.rb +1 -1
  145. data/lib/datadog/tracing/contrib/component.rb +1 -1
  146. data/lib/datadog/tracing/contrib/configuration/resolver.rb +7 -4
  147. data/lib/datadog/tracing/contrib/dalli/quantize.rb +1 -1
  148. data/lib/datadog/tracing/contrib/elasticsearch/patcher.rb +1 -1
  149. data/lib/datadog/tracing/contrib/excon/middleware.rb +2 -2
  150. data/lib/datadog/tracing/contrib/extensions.rb +9 -0
  151. data/lib/datadog/tracing/contrib/faraday/middleware.rb +2 -2
  152. data/lib/datadog/tracing/contrib/grape/endpoint.rb +5 -5
  153. data/lib/datadog/tracing/contrib/grpc/datadog_interceptor/client.rb +2 -2
  154. data/lib/datadog/tracing/contrib/grpc/datadog_interceptor/server.rb +2 -2
  155. data/lib/datadog/tracing/contrib/http/instrumentation.rb +2 -2
  156. data/lib/datadog/tracing/contrib/httpclient/instrumentation.rb +6 -2
  157. data/lib/datadog/tracing/contrib/httprb/instrumentation.rb +2 -2
  158. data/lib/datadog/tracing/contrib/kafka/instrumentation/consumer.rb +2 -2
  159. data/lib/datadog/tracing/contrib/kafka/instrumentation/producer.rb +2 -2
  160. data/lib/datadog/tracing/contrib/karafka/patcher.rb +1 -1
  161. data/lib/datadog/tracing/contrib/mongodb/subscribers.rb +3 -3
  162. data/lib/datadog/tracing/contrib/opensearch/patcher.rb +1 -1
  163. data/lib/datadog/tracing/contrib/presto/instrumentation.rb +3 -3
  164. data/lib/datadog/tracing/contrib/rack/configuration/settings.rb +6 -0
  165. data/lib/datadog/tracing/contrib/rack/ext.rb +27 -0
  166. data/lib/datadog/tracing/contrib/rack/patcher.rb +1 -1
  167. data/lib/datadog/tracing/contrib/rack/request_queue.rb +1 -1
  168. data/lib/datadog/tracing/contrib/rack/trace_proxy_middleware.rb +117 -1
  169. data/lib/datadog/tracing/contrib/rails/log_injection.rb +1 -1
  170. data/lib/datadog/tracing/contrib/rails/runner.rb +1 -1
  171. data/lib/datadog/tracing/contrib/rake/instrumentation.rb +2 -2
  172. data/lib/datadog/tracing/contrib/redis/quantize.rb +1 -1
  173. data/lib/datadog/tracing/contrib/redis/tags.rb +1 -1
  174. data/lib/datadog/tracing/contrib/sidekiq/utils.rb +1 -1
  175. data/lib/datadog/tracing/contrib/stripe/request.rb +1 -1
  176. data/lib/datadog/tracing/contrib.rb +8 -0
  177. data/lib/datadog/tracing/diagnostics/environment_logger.rb +3 -1
  178. data/lib/datadog/tracing/distributed/baggage.rb +59 -5
  179. data/lib/datadog/tracing/distributed/datadog.rb +11 -11
  180. data/lib/datadog/tracing/distributed/datadog_tags_codec.rb +1 -1
  181. data/lib/datadog/tracing/distributed/propagation.rb +2 -2
  182. data/lib/datadog/tracing/distributed/trace_context.rb +74 -32
  183. data/lib/datadog/tracing/event.rb +1 -1
  184. data/lib/datadog/tracing/metadata/tagging.rb +2 -2
  185. data/lib/datadog/tracing/pipeline.rb +1 -1
  186. data/lib/datadog/tracing/remote.rb +1 -1
  187. data/lib/datadog/tracing/sampling/rule.rb +1 -1
  188. data/lib/datadog/tracing/sampling/rule_sampler.rb +2 -2
  189. data/lib/datadog/tracing/sampling/span/rule_parser.rb +2 -2
  190. data/lib/datadog/tracing/span_operation.rb +3 -3
  191. data/lib/datadog/tracing/trace_operation.rb +4 -4
  192. data/lib/datadog/tracing/tracer.rb +6 -8
  193. data/lib/datadog/tracing/transport/io/client.rb +1 -1
  194. data/lib/datadog/tracing/workers.rb +2 -1
  195. data/lib/datadog/version.rb +1 -1
  196. metadata +33 -12
  197. data/ext/datadog_profiling_native_extension/clock_id_noop.c +0 -21
  198. data/lib/datadog/ai_guard/configuration/settings.rb +0 -113
  199. data/lib/datadog/appsec/configuration/settings.rb +0 -423
  200. data/lib/datadog/data_streams/configuration/settings.rb +0 -49
  201. data/lib/datadog/di/configuration/settings.rb +0 -243
  202. data/lib/datadog/error_tracking/configuration/settings.rb +0 -63
@@ -22,11 +22,114 @@ module Datadog
22
22
  class CodeTracker
23
23
  def initialize
24
24
  @registry = {}
25
+ @per_method_registry = {}
25
26
  @trace_point_lock = Mutex.new
26
27
  @registry_lock = Mutex.new
27
28
  @compiled_trace_point = nil
28
29
  end
29
30
 
31
+ # Populates the registry with iseqs for files that were loaded
32
+ # before code tracking started.
33
+ #
34
+ # Uses the all_iseqs C extension to walk the Ruby object space and
35
+ # find instruction sequences for already-loaded code. Whole-file
36
+ # iseqs are stored in the main registry; per-method/block/class
37
+ # iseqs are stored in per_method_registry as fallback for files
38
+ # whose whole-file iseq was GC'd.
39
+ #
40
+ # See docs/DynamicInstrumentationDevelopment.md "Iseq Lifecycle and GC"
41
+ # for which iseq types survive GC and implications for backfill.
42
+ #
43
+ # Whole-file detection uses two strategies:
44
+ # - Ruby 3.1+: DI.iseq_type (wraps rb_iseq_type) returns :top for
45
+ # require/load and :main for the entry script. This is precise.
46
+ # - Ruby < 3.1: falls back to first_lineno == 0, which is true for
47
+ # whole-file iseqs from require/load (INT2FIX(0) in Ruby's
48
+ # rb_iseq_new_top and rb_iseq_new_main) and false for
49
+ # method/block/class definitions (first_lineno >= 1).
50
+ # InstructionSequence.compile passes first_lineno = 1 by default,
51
+ # so eval'd code is not matched. Both strategies produce the same
52
+ # result in practice.
53
+ #
54
+ # Does not overwrite iseqs already in the registry (from
55
+ # :script_compiled), since those are guaranteed to be whole-file
56
+ # iseqs and are authoritative.
57
+ #
58
+ # @return [void]
59
+ def backfill_registry
60
+ iseqs = DI.file_iseqs
61
+ have_iseq_type = DI.respond_to?(:iseq_type)
62
+ registry_lock.synchronize do
63
+ iseqs.each do |iseq|
64
+ path = iseq.absolute_path
65
+ next unless path
66
+
67
+ whole_file = if have_iseq_type
68
+ type = DI.iseq_type(iseq)
69
+ # Require first_lineno == 0 to exclude compile_file/compile
70
+ # iseqs. These are :top type but have first_lineno == 1 and
71
+ # produce iseq objects distinct from require-produced iseqs.
72
+ # Targeted TracePoints are bound to the specific iseq object
73
+ # — a probe on a compile_file iseq silently never fires when
74
+ # the require-produced code runs.
75
+ (type == :top || type == :main) && iseq.first_lineno == 0
76
+ else
77
+ iseq.first_lineno == 0
78
+ end
79
+
80
+ if whole_file
81
+ # Ruby 3.2.9+ creates dummy profiler iseqs during require/load
82
+ # (rb_iseq_alloc_with_dummy_path in iseq.c). These have type
83
+ # :top, first_lineno == 0, and the same absolute_path as the
84
+ # real iseq — but iseq_size == 0 (no bytecode). A targeted
85
+ # TracePoint on a dummy iseq can't find child iseqs and raises
86
+ # ArgumentError "can not enable any hooks". Filter them out:
87
+ # a real top-level iseq always has at least one trace event.
88
+ next if iseq.trace_points.empty?
89
+
90
+ # Do not overwrite entries from :script_compiled — those are
91
+ # captured at load time and are authoritative.
92
+ next if registry.key?(path)
93
+
94
+ registry[path] = iseq
95
+ else
96
+ # Skip top-level script iseqs (:top/:main) produced by
97
+ # RubyVM::InstructionSequence.compile_file and .compile
98
+ # (compile source to bytecode without executing it).
99
+ # These represent the file body,
100
+ # not a method or block. They pass the first_lineno check
101
+ # (lineno != 0) but a targeted TracePoint bound to one
102
+ # of these never fires for method-level code — the
103
+ # user's probe silently produces no snapshots.
104
+ #
105
+ # On Ruby < 3.1 (no iseq_type), we cannot distinguish
106
+ # these from method iseqs, so they leak into
107
+ # per_method_registry. If iseq_for_line selects a leaked
108
+ # top-level iseq instead of the real method iseq, the
109
+ # probe installs but silently never fires — same failure
110
+ # as above. This requires the application to call
111
+ # compile_file and hold the result, which is rare outside
112
+ # tooling like bootsnap (which discards it).
113
+ next if have_iseq_type && (type == :top || type == :main)
114
+
115
+ # Store per-method/block/class iseqs as fallback for files
116
+ # whose whole-file iseq was GC'd. These can be used to
117
+ # target line probes on lines within their range.
118
+ (per_method_registry[path] ||= []) << iseq
119
+ end
120
+ end
121
+ end
122
+ nil
123
+ rescue => exc
124
+ # Backfill is best-effort — if it fails, line probes on
125
+ # pre-loaded code won't work but everything else is unaffected.
126
+ if component = DI.current_component
127
+ component.logger.debug { "di: backfill_registry failed: #{exc.class}: #{exc.message}" }
128
+ component.telemetry&.report(exc, description: "backfill_registry failed")
129
+ end
130
+ nil
131
+ end
132
+
30
133
  # Starts tracking loaded code.
31
134
  #
32
135
  # This method should generally be called early in application boot
@@ -94,7 +197,7 @@ module Datadog
94
197
  # but we will have DI.current_component (set to nil).
95
198
  if component = DI.current_component
96
199
  raise if component.settings.dynamic_instrumentation.internal.propagate_all_exceptions
97
- component.logger.debug { "di: unhandled exception in script_compiled trace point: #{exc.class}: #{exc}" }
200
+ component.logger.debug { "di: unhandled exception in script_compiled trace point: #{exc.class}: #{exc.message}" }
98
201
  component.telemetry&.report(exc, description: "Unhandled exception in script_compiled trace point")
99
202
  # TODO test this path
100
203
  else
@@ -104,6 +207,12 @@ module Datadog
104
207
  # TODO test this path
105
208
  end
106
209
  end
210
+
211
+ # Backfill the registry with iseqs for files that were loaded
212
+ # before tracking started. This must happen after the trace
213
+ # point is enabled so that any files loaded concurrently are
214
+ # captured by the trace point (backfill won't overwrite them).
215
+ backfill_registry
107
216
  end
108
217
  end
109
218
 
@@ -160,6 +269,51 @@ module Datadog
160
269
  end
161
270
  end
162
271
 
272
+ # Returns a [path, iseq] pair for a line probe target, or nil.
273
+ #
274
+ # First checks the whole-file iseq registry (via iseqs_for_path_suffix).
275
+ # If no whole-file iseq exists, searches the per-method iseq registry
276
+ # for an iseq whose trace_points include the target line.
277
+ #
278
+ # @param suffix [String] file path or suffix to match
279
+ # @param line [Integer] target line number
280
+ # @return [Array(String, RubyVM::InstructionSequence), nil]
281
+ def iseq_for_line(suffix, line)
282
+ # Try whole-file iseq first — it always covers all lines.
283
+ result = iseqs_for_path_suffix(suffix)
284
+ return result if result
285
+
286
+ # Fall back to per-method iseqs.
287
+ registry_lock.synchronize do
288
+ # Resolve the path using the per-method registry keys.
289
+ path = resolve_path_suffix(suffix, per_method_registry.keys)
290
+ return nil unless path
291
+
292
+ iseqs = per_method_registry[path]
293
+ return nil unless iseqs
294
+
295
+ # Only match event types the instrumenter subscribes to
296
+ # (:line, :return, :b_return — see hook_line). Lines that
297
+ # only carry :call (e.g. a `def` line within the defined
298
+ # method's own iseq, not the enclosing scope) have no
299
+ # subscribed event at that position; TracePoint#enable
300
+ # raises because it cannot bind an enabled event there.
301
+ matches = iseqs.select do |iseq|
302
+ iseq.trace_points.any? do |tp_line, event|
303
+ tp_line == line && (event == :line || event == :return || event == :b_return)
304
+ end
305
+ end
306
+ # When multiple iseqs contain the target line (e.g. a method
307
+ # and an inline block sharing the same line), picking one
308
+ # would silently miss executions in the other context.
309
+ # Raise so the probe is recorded as failed with a clear error.
310
+ if matches.length > 1
311
+ raise Error::MultiplePathsMatch, "Multiple code locations match line #{line}"
312
+ end
313
+ matches.first ? [path, matches.first] : nil
314
+ end
315
+ end
316
+
163
317
  # Stops tracking code that is being loaded.
164
318
  #
165
319
  # This method should ordinarily never be called - if a file is loaded
@@ -186,6 +340,7 @@ module Datadog
186
340
  def clear
187
341
  registry_lock.synchronize do
188
342
  registry.clear
343
+ per_method_registry.clear
189
344
  end
190
345
  end
191
346
 
@@ -195,8 +350,31 @@ module Datadog
195
350
  # objects representing compiled code of those files.
196
351
  attr_reader :registry
197
352
 
353
+ # Mapping from paths to arrays of per-method/block/class iseqs.
354
+ # Used as fallback when the whole-file iseq has been GC'd.
355
+ attr_reader :per_method_registry
356
+
198
357
  attr_reader :trace_point_lock
199
358
  attr_reader :registry_lock
359
+
360
+ # Resolves a path suffix against a set of known paths.
361
+ # Returns the matching path or nil.
362
+ #
363
+ # Must be called within registry_lock.
364
+ def resolve_path_suffix(suffix, paths)
365
+ # Exact match.
366
+ return suffix if paths.include?(suffix)
367
+
368
+ # Suffix match.
369
+ suffix = suffix.dup
370
+ loop do
371
+ matches = paths.select { |p| Utils.path_matches_suffix?(p, suffix) }
372
+ raise Error::MultiplePathsMatch, "Multiple paths matched requested suffix" if matches.length > 1
373
+ return matches.first if matches.any?
374
+ return nil unless suffix.include?('/')
375
+ suffix.sub!(%r{.*/+}, '')
376
+ end
377
+ end
200
378
  end
201
379
  end
202
380
  end
@@ -122,7 +122,7 @@ module Datadog
122
122
  payload = probe_notification_builder.build_errored(probe, exc)
123
123
  probe_notifier_worker.add_status(payload)
124
124
  rescue => nested_exc
125
- logger.debug { "di: failed to build error notification: #{nested_exc.class}: #{nested_exc}" }
125
+ logger.debug { "di: failed to build error notification: #{nested_exc.class}: #{nested_exc.message}" }
126
126
  telemetry&.report(nested_exc, description: 'Error building probe error notification')
127
127
  raise
128
128
  end
@@ -4,8 +4,241 @@ module Datadog
4
4
  module DI
5
5
  # Configuration for DI
6
6
  module Configuration
7
+ # Settings
8
+ module Settings
9
+ def self.extended(base)
10
+ base = base.singleton_class unless base.is_a?(Class)
11
+ add_settings!(base)
12
+ end
13
+
14
+ def self.add_settings!(base)
15
+ base.class_eval do
16
+ settings :dynamic_instrumentation do
17
+ option :enabled do |o|
18
+ o.type :bool
19
+ # The environment variable has an "internal" prefix so that
20
+ # any customers that have the "proper" environment variable
21
+ # turned on (i.e. DD_DYNAMIC_INSTRUMENTATION_ENABLED)
22
+ # do not enable Ruby DI until the latter is ready for
23
+ # customer testing.
24
+ o.env "DD_DYNAMIC_INSTRUMENTATION_ENABLED"
25
+ o.default false
26
+ end
27
+
28
+ # An array of variable and key names to redact in addition to
29
+ # the built-in list of identifiers.
30
+ #
31
+ # The names will be normalized by removing the following
32
+ # symbols: _, -, @, $, and then matched to the complete
33
+ # variable or key name while ignoring the case.
34
+ # For example, specifying pass_word will match password and
35
+ # PASSWORD, and specifying PASSWORD will match pass_word.
36
+ # Note that, while the at sign (@) is used in Ruby to refer
37
+ # to instance variables, it does not have any significance
38
+ # for this setting (and is removed before matching identifiers).
39
+ option :redacted_identifiers do |o|
40
+ o.env "DD_DYNAMIC_INSTRUMENTATION_REDACTED_IDENTIFIERS"
41
+ o.env_parser do |value|
42
+ value&.split(",")&.map(&:strip)
43
+ end
44
+
45
+ o.type :array
46
+ o.default []
47
+ end
48
+
49
+ # An array of variable and key names to exclude from the
50
+ # built-in redaction list.
51
+ #
52
+ # This allows users to capture values of variables that would
53
+ # otherwise be redacted by the default identifier list.
54
+ # For example, if an application has a "session" variable
55
+ # that does not contain sensitive data, "session" can be added
56
+ # to this list to exclude it from redaction.
57
+ #
58
+ # The names will be normalized the same way as redacted_identifiers,
59
+ # by removing the following symbols: _, -, @, $, and then matched
60
+ # against the complete variable or key name while ignoring the case.
61
+ option :redaction_excluded_identifiers do |o|
62
+ o.env "DD_DYNAMIC_INSTRUMENTATION_REDACTION_EXCLUDED_IDENTIFIERS"
63
+ o.env_parser do |value|
64
+ value&.split(",")&.map(&:strip)
65
+ end
66
+
67
+ o.type :array
68
+ o.default []
69
+ end
70
+
71
+ # An array of class names, values of which will be redacted from
72
+ # dynamic instrumentation snapshots. Example: FooClass.
73
+ # If a name is suffixed by '*', it becomes a wildcard and
74
+ # instances of any class whose name begins with the specified
75
+ # prefix will be redacted (example: Foo*).
76
+ #
77
+ # The names must all be fully-qualified, if any prefix of a
78
+ # class name is configured to be redacted, the value will be
79
+ # subject to redaction. For example, if Foo* is in the
80
+ # redacted class name list, instances of Foo, FooBar,
81
+ # Foo::Bar are all subject to redaction, but Bar::Foo will
82
+ # not be subject to redaction.
83
+ #
84
+ # Leading double-colon is permitted but has no effect,
85
+ # because the names are always considered to be fully-qualified.
86
+ # For example, adding ::Foo to the list will redact instances
87
+ # of Foo.
88
+ #
89
+ # Trailing colons should not be used because they will trigger
90
+ # exact match behavior but Ruby class names do not have
91
+ # trailing colons. For example, Foo:: will not cause anything
92
+ # to be redacted. Use Foo::* to redact all classes under
93
+ # the Foo module.
94
+ option :redacted_type_names do |o|
95
+ o.env "DD_DYNAMIC_INSTRUMENTATION_REDACTED_TYPES"
96
+ o.env_parser do |value|
97
+ value&.split(",")&.map(&:strip)
98
+ end
99
+
100
+ o.type :array
101
+ o.default []
102
+ end
103
+
104
+ # Maximum number of object or collection traversals that
105
+ # will be permitted when serializing captured values.
106
+ option :max_capture_depth do |o|
107
+ o.type :int
108
+ o.default 3
109
+ end
110
+
111
+ # Maximum number of collection (Array and Hash) elements
112
+ # that will be captured. Arrays and hashes that have more
113
+ # elements will be truncated to this many elements.
114
+ option :max_capture_collection_size do |o|
115
+ o.type :int
116
+ o.default 100
117
+ end
118
+
119
+ # Strings longer than this length will be truncated to this
120
+ # length in dynamic instrumentation snapshots.
121
+ #
122
+ # Note that while all values are stringified during
123
+ # serialization, only values which are originally instances
124
+ # of the String class are subject to this length limit.
125
+ option :max_capture_string_length do |o|
126
+ o.type :int
127
+ o.default 255
128
+ end
129
+
130
+ # Maximim number of attributes that will be captured for
131
+ # a single non-primitive value.
132
+ option :max_capture_attribute_count do |o|
133
+ o.type :int
134
+ o.default 20
135
+ end
136
+
137
+ # Settings in the 'internal' group are for internal Datadog
138
+ # use only, and are needed to test dynamic instrumentation or
139
+ # experiment with features not released to customers.
140
+ settings :internal do
141
+ # This option instructs dynamic instrumentation to use
142
+ # untargeted trace points when installing line probes and
143
+ # code tracking is not active.
144
+ # WARNING: untargeted trace points carry a massive performance
145
+ # penalty for the entire file in which a line probe is placed.
146
+ #
147
+ # If this option is set to false, which is the default,
148
+ # dynamic instrumentation will add probes that reference
149
+ # unknown files to the list of pending probes, and when
150
+ # the respective files are loaded, the line probes will be
151
+ # installed using targeted trace points. If the file in
152
+ # question is already loaded when the probe is received
153
+ # (for example, it is in a third-party library loaded during
154
+ # application boot), and code tracking was not active when
155
+ # the file was loaded, such files will not be instrumentable
156
+ # via line probes.
157
+ #
158
+ # If this option is set to true, dynamic instrumentation will
159
+ # install untargeted trace points for all line probes,
160
+ # regardless of whether the referenced file is loaded.
161
+ # This permits instrumenting code which was loaded prior to
162
+ # code tracking being activated and instrumenting lines when
163
+ # code tracking is not activated at all. However, untargeted
164
+ # trace points are extremely slow and will greatly degrade
165
+ # performance of *all* code executed while they are installed,
166
+ # not just the instrumentation target.
167
+ option :untargeted_trace_points do |o|
168
+ o.type :bool
169
+ o.default false
170
+ end
171
+
172
+ # If true, all of the catch-all rescue blocks in DI
173
+ # will propagate the exceptions onward.
174
+ # WARNING: for internal Datadog use only - this will break
175
+ # the DI product and potentially the library in general in
176
+ # a multitude of ways, cause resource leakage, permanent
177
+ # performance decreases, etc.
178
+ option :propagate_all_exceptions do |o|
179
+ o.type :bool
180
+ o.default false
181
+ end
182
+
183
+ # Minimum interval, in seconds, between probe status and
184
+ # snapshot submissions to the agent. Probe notifier worker will
185
+ # batch together payloads submitted during each interval.
186
+ # A longer interval reduces the overhead imposed by dynamic
187
+ # instrumentation on the application, but also increases the
188
+ # time when application code cannot run (when the batches are
189
+ # being sent out by the probe notifier worker) and creates a
190
+ # possibility of dropping payloads if the queue gets too long.
191
+ option :min_send_interval do |o|
192
+ o.type :float
193
+ o.default 3
194
+ end
195
+
196
+ # Number of snapshots that can be stored in the probe
197
+ # notifier worker queue. Larger capacity runs the risk of
198
+ # creating snapshots that exceed the agent's request size
199
+ # limit. Smaller capacity increases the risk of dropping
200
+ # snapshots.
201
+ option :snapshot_queue_capacity do |o|
202
+ o.type :int
203
+ o.default 100
204
+ end
205
+
206
+ # Enable dynamic instrumentation in development environments.
207
+ # Currently DI does not fully implement support for code
208
+ # unloading and reloading, and is not supported in
209
+ # non-production environments.
210
+ option :development do |o|
211
+ o.type :bool
212
+ o.default false
213
+ end
214
+
215
+ # Enable logging of dynamic instrumentation activity.
216
+ # This is quite verbose.
217
+ option :trace_logging do |o|
218
+ o.type :bool
219
+ o.default false
220
+
221
+ # Use the same environment variable as the rest of
222
+ # dd-trace-rb logging for now. Could change to a
223
+ # dedicated environment variable in the future but
224
+ # will likely need a way to turn on remote config
225
+ # debugging (since DI uses RC for configuration).
226
+ o.env 'DD_TRACE_DEBUG'
227
+ end
228
+
229
+ # If the CPU time consumed by the thread performing instrumentation
230
+ # exceeds this amount, the offending probe will be automatically disabled.
231
+ # Set to nil to disable the circuit breaker.
232
+ # Set to zero to disable every probe after it executes once.
233
+ option :max_processing_time do |o|
234
+ o.type :float
235
+ o.default 0.5
236
+ end
237
+ end
238
+ end
239
+ end
240
+ end
241
+ end
7
242
  end
8
243
  end
9
244
  end
10
-
11
- require_relative "configuration/settings"
@@ -89,6 +89,11 @@ module Datadog
89
89
  # from the method but from outside of the method).
90
90
  Location = Struct.new(:path, :lineno, :label)
91
91
 
92
+ # Method probes can only target instance methods. The implementation uses
93
+ # Module#prepend with a module that defines an instance method matching the
94
+ # probe's target — class/singleton methods (def self.foo, module_function)
95
+ # are not reachable via prepend on the class itself. Line probes are
96
+ # unaffected since they install via TracePoint, not method dispatch.
92
97
  def hook_method(probe, responder)
93
98
  lock.synchronize do
94
99
  if probe.instrumentation_module
@@ -150,7 +155,7 @@ module Datadog
150
155
  rescue => nested_exc
151
156
  raise if settings.dynamic_instrumentation.internal.propagate_all_exceptions
152
157
 
153
- instrumenter.logger.debug { "di: error in probe condition evaluation failed callback: #{nested_exc.class}: #{nested_exc}" }
158
+ instrumenter.logger.debug { "di: error in probe condition evaluation failed callback: #{nested_exc.class}: #{nested_exc.message}" }
154
159
  instrumenter.telemetry&.report(nested_exc, description: "Error in probe condition evaluation failed callback")
155
160
  end
156
161
  else
@@ -158,7 +163,7 @@ module Datadog
158
163
 
159
164
  raise if settings.dynamic_instrumentation.internal.propagate_all_exceptions
160
165
 
161
- instrumenter.logger.debug { "di: error evaluating condition without context (tracer bug?): #{exc.class}: #{exc}" }
166
+ instrumenter.logger.debug { "di: error evaluating condition without context (tracer bug?): #{exc.class}: #{exc.message}" }
162
167
  instrumenter.telemetry&.report(exc, description: "Error evaluating condition without context")
163
168
  # If execution gets here, there is probably a bug in the tracer.
164
169
  end
@@ -245,7 +250,7 @@ module Datadog
245
250
  rescue => di_exc
246
251
  raise if settings.dynamic_instrumentation.internal.propagate_all_exceptions
247
252
 
248
- instrumenter.logger.debug { "di: unhandled exception in method probe: #{di_exc.class}: #{di_exc}" }
253
+ instrumenter.logger.debug { "di: unhandled exception in method probe: #{di_exc.class}: #{di_exc.message}" }
249
254
  instrumenter.telemetry&.report(di_exc, description: "Unhandled exception in method probe")
250
255
  end
251
256
 
@@ -337,7 +342,11 @@ module Datadog
337
342
  # Steep: Complex type narrowing (before calling hook_line,
338
343
  # we check that probe.line? is true which itself checks that probe.file is not nil)
339
344
  # Annotation do not work here as `file` is a method on probe, not a local variable.
340
- ret = code_tracker.iseqs_for_path_suffix(probe.file) # steep:ignore ArgumentTypeMismatch
345
+ ret = if code_tracker.respond_to?(:iseq_for_line)
346
+ code_tracker.iseq_for_line(probe.file, line_no) # steep:ignore ArgumentTypeMismatch
347
+ else
348
+ code_tracker.iseqs_for_path_suffix(probe.file) # steep:ignore ArgumentTypeMismatch
349
+ end
341
350
  unless ret
342
351
  if permit_untargeted_trace_points
343
352
  # Continue withoout targeting the trace point.
@@ -355,16 +364,16 @@ module Datadog
355
364
  # to instrument and install the hook when the file in
356
365
  # question is loaded (and hopefully, by then code tracking
357
366
  # is active, otherwise the line will never be instrumented.)
358
- raise_if_probe_in_loaded_features(probe)
359
- raise Error::DITargetNotDefined, "File not in code tracker registry: #{probe.file}"
367
+ raise_if_probe_in_loaded_features(probe, line_no, code_tracker)
368
+ raise Error::DITargetNotDefined, "File not in code tracker registry: #{probe.file}:#{line_no}"
360
369
  end
361
370
  end
362
371
  elsif !permit_untargeted_trace_points
363
372
  # Same as previous comment, if untargeted trace points are not
364
373
  # explicitly defined, and we do not have code tracking, do not
365
374
  # instrument the method.
366
- raise_if_probe_in_loaded_features(probe)
367
- raise Error::DITargetNotDefined, "File not in code tracker registry: #{probe.file}"
375
+ raise_if_probe_in_loaded_features(probe, line_no, nil)
376
+ raise Error::DITargetNotDefined, "File not in code tracker registry: #{probe.file}:#{line_no}"
368
377
  end
369
378
 
370
379
  if ret
@@ -536,7 +545,7 @@ module Datadog
536
545
  rescue => nested_exc
537
546
  raise if settings.dynamic_instrumentation.internal.propagate_all_exceptions
538
547
 
539
- logger.debug { "di: error in probe condition evaluation failed callback: #{nested_exc.class}: #{nested_exc}" }
548
+ logger.debug { "di: error in probe condition evaluation failed callback: #{nested_exc.class}: #{nested_exc.message}" }
540
549
  telemetry&.report(nested_exc, description: "Error in probe condition evaluation failed callback")
541
550
  end
542
551
 
@@ -546,7 +555,7 @@ module Datadog
546
555
 
547
556
  raise if settings.dynamic_instrumentation.internal.propagate_all_exceptions
548
557
 
549
- logger.debug { "di: error evaluating condition without context (tracer bug?): #{exc.class}: #{exc}" }
558
+ logger.debug { "di: error evaluating condition without context (tracer bug?): #{exc.class}: #{exc.message}" }
550
559
  telemetry&.report(exc, description: "Error evaluating condition without context")
551
560
  # If execution gets here, there is probably a bug in the tracer.
552
561
  end
@@ -568,7 +577,7 @@ module Datadog
568
577
  check_and_disable_if_exceeded(probe, responder, di_start_time)
569
578
  rescue => exc
570
579
  raise if settings.dynamic_instrumentation.internal.propagate_all_exceptions
571
- logger.debug { "di: unhandled exception in line trace point: #{exc.class}: #{exc}" }
580
+ logger.debug { "di: unhandled exception in line trace point: #{exc.class}: #{exc.message}" }
572
581
  telemetry&.report(exc, description: "Unhandled exception in line trace point")
573
582
  # TODO test this path
574
583
  end
@@ -606,23 +615,34 @@ module Datadog
606
615
  end
607
616
  end
608
617
 
609
- def raise_if_probe_in_loaded_features(probe)
618
+ def raise_if_probe_in_loaded_features(probe, line_no, code_tracker)
610
619
  return unless probe.file
611
620
 
612
- # If the probe file is in the list of loaded files
613
- # (as per $LOADED_FEATURES, using either exact or suffix match),
614
- # raise an error indicating that
615
- # code tracker is missing the loaded file because the file
616
- # won't be loaded again (DI only works in production environments
617
- # that do not normally reload code).
618
- if $LOADED_FEATURES.include?(probe.file)
619
- raise Error::DITargetNotInRegistry, "File loaded but is not in code tracker registry: #{probe.file}"
621
+ # Find the loaded path matching the probe file.
622
+ loaded_path = if $LOADED_FEATURES.include?(probe.file)
623
+ probe.file
624
+ else
625
+ # Expensive suffix check.
626
+ $LOADED_FEATURES.find { |path| Utils.path_matches_suffix?(path, probe.file) }
620
627
  end
621
- # Ths is an expensive check
622
- $LOADED_FEATURES.each do |path|
623
- if Utils.path_matches_suffix?(path, probe.file)
624
- raise Error::DITargetNotInRegistry, "File matching probe path (#{probe.file}) was loaded and is not in code tracker registry: #{path}"
625
- end
628
+
629
+ return unless loaded_path
630
+
631
+ # Distinguish between "no iseqs at all" and "has per-method iseqs
632
+ # but none cover this line".
633
+ has_per_method = code_tracker&.send(:instance_variable_defined?, :@per_method_registry) &&
634
+ code_tracker.send(:per_method_registry).key?(loaded_path)
635
+
636
+ if has_per_method
637
+ raise Error::DITargetNotInRegistry,
638
+ "File #{loaded_path} is loaded and has per-method iseqs, " \
639
+ "but none cover line #{line_no}. " \
640
+ "The line may be in file-level setup code outside any method."
641
+ else
642
+ raise Error::DITargetNotInRegistry,
643
+ "File #{loaded_path} is loaded but has no surviving iseqs " \
644
+ "(whole-file iseq was garbage collected and no per-method iseqs remain). " \
645
+ "Line probes cannot target this file."
626
646
  end
627
647
  end
628
648
 
@@ -630,7 +650,7 @@ module Datadog
630
650
  def symbolize_class_name(cls_name)
631
651
  Object.const_get(cls_name)
632
652
  rescue NameError => exc
633
- raise Error::DITargetNotDefined, "Class not defined: #{cls_name}: #{exc.class}: #{exc}"
653
+ raise Error::DITargetNotDefined, "Class not defined: #{cls_name}: #{exc.class}: #{exc.message}"
634
654
  end
635
655
  end
636
656
  end
@@ -58,7 +58,7 @@ module Datadog
58
58
  condition: cond,
59
59
  )
60
60
  rescue KeyError => exc
61
- raise ArgumentError, "Malformed remote configuration entry for probe: #{exc.class}: #{exc}: #{config}"
61
+ raise ArgumentError, "Malformed remote configuration entry for probe: #{exc.class}: #{exc.message}: #{config}"
62
62
  end
63
63
 
64
64
  def build_template_segments(segments)
@@ -50,7 +50,7 @@ module Datadog
50
50
  rescue => exc
51
51
  raise if component.settings.dynamic_instrumentation.internal.propagate_all_exceptions
52
52
 
53
- component.logger.debug { "di: unhandled exception adding #{probe.type} probe at #{probe.location} (#{probe.id}) in DI probe file loader: #{exc.class}: #{exc}" }
53
+ component.logger.debug { "di: unhandled exception adding #{probe.type} probe at #{probe.location} (#{probe.id}) in DI probe file loader: #{exc.class}: #{exc.message}" }
54
54
  component.telemetry&.report(exc, description: "Unhandled exception adding probe in DI probe file loader")
55
55
 
56
56
  # TODO test this path
@@ -64,7 +64,7 @@ module Datadog
64
64
  raise
65
65
  end
66
66
 
67
- component.logger.debug { "di: unhandled exception handling a probe in DI probe file loader: #{exc.class}: #{exc}" }
67
+ component.logger.debug { "di: unhandled exception handling a probe in DI probe file loader: #{exc.class}: #{exc.message}" }
68
68
  component.telemetry&.report(exc, description: "Unhandled exception handling probe in DI probe file loader")
69
69
  end
70
70
  rescue