datadog 2.39.0 → 2.41.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (158) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +97 -44
  3. data/ext/datadog_profiling_native_extension/collectors_stack.c +219 -148
  4. data/ext/datadog_profiling_native_extension/collectors_stack.h +16 -2
  5. data/ext/datadog_profiling_native_extension/collectors_thread_context.c +12 -7
  6. data/ext/datadog_profiling_native_extension/crashtracking_runtime_stacks.c +5 -4
  7. data/ext/datadog_profiling_native_extension/datadog_ruby_common.c +19 -11
  8. data/ext/datadog_profiling_native_extension/datadog_ruby_common.h +7 -0
  9. data/ext/datadog_profiling_native_extension/extconf.rb +12 -7
  10. data/ext/datadog_profiling_native_extension/native_extension_helpers.rb +3 -3
  11. data/ext/datadog_profiling_native_extension/private_vm_api_access.c +284 -76
  12. data/ext/datadog_profiling_native_extension/private_vm_api_access.h +16 -10
  13. data/ext/libdatadog_api/datadog_ruby_common.c +19 -11
  14. data/ext/libdatadog_api/datadog_ruby_common.h +7 -0
  15. data/ext/libdatadog_api/init.c +4 -0
  16. data/ext/libdatadog_api/trace_exporter.c +1312 -0
  17. data/ext/libdatadog_api/trace_exporter.h +5 -0
  18. data/ext/libdatadog_extconf_helpers.rb +1 -1
  19. data/lib/datadog/ai_guard/api_client.rb +1 -1
  20. data/lib/datadog/ai_guard/evaluation/request.rb +8 -8
  21. data/lib/datadog/ai_guard/evaluation.rb +11 -3
  22. data/lib/datadog/ai_guard/ext.rb +3 -1
  23. data/lib/datadog/appsec/actions_handler/serializable_backtrace.rb +21 -42
  24. data/lib/datadog/appsec/api_security/endpoint_collection/grape_route_serializer.rb +1 -1
  25. data/lib/datadog/appsec/api_security/endpoint_collection/rails_route_serializer.rb +1 -1
  26. data/lib/datadog/appsec/api_security/endpoint_collection/sinatra_route_serializer.rb +1 -1
  27. data/lib/datadog/appsec/configuration.rb +11 -2
  28. data/lib/datadog/appsec/context.rb +1 -1
  29. data/lib/datadog/appsec/contrib/active_record/instrumentation.rb +1 -1
  30. data/lib/datadog/appsec/contrib/aws_lambda/waf_addresses.rb +2 -2
  31. data/lib/datadog/appsec/contrib/devise/configuration.rb +1 -1
  32. data/lib/datadog/appsec/contrib/devise/patcher.rb +1 -1
  33. data/lib/datadog/appsec/contrib/excon/ssrf_detection_middleware.rb +2 -2
  34. data/lib/datadog/appsec/contrib/faraday/ssrf_detection_middleware.rb +2 -2
  35. data/lib/datadog/appsec/contrib/graphql/gateway/watcher.rb +1 -1
  36. data/lib/datadog/appsec/contrib/rack/ext.rb +2 -2
  37. data/lib/datadog/appsec/contrib/rack/gateway/watcher.rb +2 -2
  38. data/lib/datadog/appsec/contrib/rack/patcher.rb +1 -1
  39. data/lib/datadog/appsec/contrib/rack/request_body_middleware.rb +6 -3
  40. data/lib/datadog/appsec/contrib/rails/gateway/watcher.rb +2 -2
  41. data/lib/datadog/appsec/contrib/rails/request_middleware.rb +2 -1
  42. data/lib/datadog/appsec/contrib/rest_client/request_ssrf_detection_patch.rb +2 -2
  43. data/lib/datadog/appsec/contrib/sinatra/gateway/watcher.rb +2 -2
  44. data/lib/datadog/appsec/contrib/sinatra/request_middleware.rb +2 -1
  45. data/lib/datadog/appsec/processor/rule_loader.rb +12 -12
  46. data/lib/datadog/appsec/response.rb +1 -1
  47. data/lib/datadog/appsec/route_normalizer/route_pattern.rb +3 -3
  48. data/lib/datadog/appsec/security_engine/engine.rb +3 -3
  49. data/lib/datadog/core/configuration/components.rb +2 -2
  50. data/lib/datadog/core/configuration/option_definition.rb +1 -1
  51. data/lib/datadog/core/configuration/options.rb +1 -1
  52. data/lib/datadog/core/configuration/settings.rb +33 -41
  53. data/lib/datadog/core/configuration/supported_configurations.rb +3 -0
  54. data/lib/datadog/core/diagnostics/environment_logger.rb +36 -1
  55. data/lib/datadog/core/environment/gc.rb +1 -1
  56. data/lib/datadog/core/environment/vm_cache.rb +1 -1
  57. data/lib/datadog/core/environment/yjit.rb +2 -2
  58. data/lib/datadog/core/metrics/helpers.rb +1 -1
  59. data/lib/datadog/core/metrics/options.rb +2 -2
  60. data/lib/datadog/core/process_discovery.rb +1 -1
  61. data/lib/datadog/core/remote/client.rb +1 -1
  62. data/lib/datadog/core/remote/configuration/repository.rb +2 -2
  63. data/lib/datadog/core/telemetry/configuration_value.rb +38 -0
  64. data/lib/datadog/core/telemetry/event/app_client_configuration_change.rb +3 -1
  65. data/lib/datadog/core/telemetry/event/app_endpoints_loaded.rb +1 -1
  66. data/lib/datadog/core/telemetry/event/app_started.rb +5 -24
  67. data/lib/datadog/core/telemetry/event/generate_metrics.rb +1 -1
  68. data/lib/datadog/core/telemetry/event/log.rb +2 -2
  69. data/lib/datadog/core/telemetry/event/synth_app_client_configuration_change.rb +1 -1
  70. data/lib/datadog/core/telemetry/metric.rb +2 -2
  71. data/lib/datadog/core/telemetry/request.rb +2 -2
  72. data/lib/datadog/core/transport/http.rb +1 -1
  73. data/lib/datadog/core/transport/transport.rb +1 -1
  74. data/lib/datadog/core/utils/at_fork_monkey_patch.rb +170 -18
  75. data/lib/datadog/core/utils/sequence.rb +3 -6
  76. data/lib/datadog/core/utils/time.rb +16 -54
  77. data/lib/datadog/data_streams/processor.rb +14 -14
  78. data/lib/datadog/di/el/compiler.rb +88 -12
  79. data/lib/datadog/di/el/evaluator.rb +82 -2
  80. data/lib/datadog/di/el/expression.rb +13 -2
  81. data/lib/datadog/di/instrumenter.rb +5 -4
  82. data/lib/datadog/di/probe_builder.rb +7 -6
  83. data/lib/datadog/di/probe_notification_builder.rb +2 -2
  84. data/lib/datadog/di/serializer.rb +29 -4
  85. data/lib/datadog/error_tracking/component.rb +1 -1
  86. data/lib/datadog/kit/appsec/events/v2.rb +2 -2
  87. data/lib/datadog/open_feature/exposures/batch_builder.rb +1 -1
  88. data/lib/datadog/open_feature/exposures/event.rb +5 -5
  89. data/lib/datadog/profiling/collectors/info.rb +1 -1
  90. data/lib/datadog/profiling/collectors/thread_context.rb +6 -13
  91. data/lib/datadog/profiling/component.rb +1 -0
  92. data/lib/datadog/profiling/tasks/exec.rb +1 -1
  93. data/lib/datadog/profiling.rb +4 -0
  94. data/lib/datadog/symbol_database/extractor.rb +6 -6
  95. data/lib/datadog/tracing/component.rb +21 -0
  96. data/lib/datadog/tracing/configuration/dynamic/option.rb +4 -0
  97. data/lib/datadog/tracing/configuration/dynamic.rb +5 -3
  98. data/lib/datadog/tracing/configuration/ext.rb +1 -0
  99. data/lib/datadog/tracing/configuration/settings.rb +16 -0
  100. data/lib/datadog/tracing/contrib/action_cable/events.rb +1 -1
  101. data/lib/datadog/tracing/contrib/action_mailer/events.rb +1 -1
  102. data/lib/datadog/tracing/contrib/action_pack/action_controller/instrumentation.rb +2 -2
  103. data/lib/datadog/tracing/contrib/action_view/events.rb +1 -1
  104. data/lib/datadog/tracing/contrib/active_model_serializers/events.rb +1 -1
  105. data/lib/datadog/tracing/contrib/active_record/configuration/resolver.rb +1 -1
  106. data/lib/datadog/tracing/contrib/active_record/events.rb +1 -1
  107. data/lib/datadog/tracing/contrib/active_support/cache/events/cache.rb +2 -2
  108. data/lib/datadog/tracing/contrib/aws/ext.rb +1 -1
  109. data/lib/datadog/tracing/contrib/aws/services.rb +1 -1
  110. data/lib/datadog/tracing/contrib/elasticsearch/quantize.rb +1 -1
  111. data/lib/datadog/tracing/contrib/ext.rb +3 -3
  112. data/lib/datadog/tracing/contrib/grpc/datadog_interceptor/client.rb +1 -1
  113. data/lib/datadog/tracing/contrib/grpc/datadog_interceptor/server.rb +1 -1
  114. data/lib/datadog/tracing/contrib/grpc/distributed/propagation.rb +1 -1
  115. data/lib/datadog/tracing/contrib/hanami/router_tracing.rb +2 -1
  116. data/lib/datadog/tracing/contrib/kafka/events/connection/request.rb +4 -0
  117. data/lib/datadog/tracing/contrib/kafka/events/produce_operation/send_messages.rb +7 -2
  118. data/lib/datadog/tracing/contrib/kafka/events/producer/deliver_messages.rb +7 -2
  119. data/lib/datadog/tracing/contrib/kafka/events.rb +1 -1
  120. data/lib/datadog/tracing/contrib/karafka/distributed/propagation.rb +1 -1
  121. data/lib/datadog/tracing/contrib/mongodb/parsers.rb +1 -1
  122. data/lib/datadog/tracing/contrib/opensearch/ext.rb +1 -1
  123. data/lib/datadog/tracing/contrib/opensearch/quantize.rb +1 -1
  124. data/lib/datadog/tracing/contrib/patchable.rb +1 -1
  125. data/lib/datadog/tracing/contrib/patcher.rb +1 -1
  126. data/lib/datadog/tracing/contrib/que/tracer.rb +1 -1
  127. data/lib/datadog/tracing/contrib/racecar/events.rb +1 -1
  128. data/lib/datadog/tracing/contrib/rack/configuration/settings.rb +1 -1
  129. data/lib/datadog/tracing/contrib/rack/header_tagging.rb +1 -1
  130. data/lib/datadog/tracing/contrib/rack/patcher.rb +1 -1
  131. data/lib/datadog/tracing/contrib/rack/trace_proxy_middleware.rb +1 -1
  132. data/lib/datadog/tracing/contrib/redis/configuration/resolver.rb +1 -1
  133. data/lib/datadog/tracing/contrib/redis/quantize.rb +36 -2
  134. data/lib/datadog/tracing/contrib/redis/trace_middleware.rb +9 -3
  135. data/lib/datadog/tracing/contrib/sequel/database.rb +1 -5
  136. data/lib/datadog/tracing/contrib/sequel/utils.rb +1 -1
  137. data/lib/datadog/tracing/contrib/sidekiq/distributed/propagation.rb +1 -1
  138. data/lib/datadog/tracing/contrib/sinatra/configuration/settings.rb +1 -1
  139. data/lib/datadog/tracing/contrib/sneakers/tracer.rb +1 -1
  140. data/lib/datadog/tracing/contrib/utils/quantization/hash_formatter.rb +1 -1
  141. data/lib/datadog/tracing/contrib/waterdrop/distributed/propagation.rb +1 -1
  142. data/lib/datadog/tracing/correlation.rb +2 -2
  143. data/lib/datadog/tracing/diagnostics/environment_logger.rb +2 -2
  144. data/lib/datadog/tracing/distributed/baggage.rb +1 -2
  145. data/lib/datadog/tracing/distributed/propagation_policy.rb +5 -0
  146. data/lib/datadog/tracing/metadata/tagging.rb +1 -1
  147. data/lib/datadog/tracing/remote.rb +2 -2
  148. data/lib/datadog/tracing/span.rb +1 -1
  149. data/lib/datadog/tracing/span_operation.rb +1 -1
  150. data/lib/datadog/tracing/sync_writer.rb +8 -3
  151. data/lib/datadog/tracing/tracer.rb +8 -0
  152. data/lib/datadog/tracing/transport/io/traces.rb +1 -1
  153. data/lib/datadog/tracing/transport/native/response.rb +71 -0
  154. data/lib/datadog/tracing/transport/native.rb +387 -0
  155. data/lib/datadog/tracing/transport/trace_formatter.rb +1 -1
  156. data/lib/datadog/tracing/writer.rb +17 -4
  157. data/lib/datadog/version.rb +1 -1
  158. metadata +12 -7
@@ -6,8 +6,30 @@ module Datadog
6
6
  # Monkey patches `Kernel#fork` and similar functions, adding an `at_fork` callback mechanism which
7
7
  # is used to restart observability after the VM forks (e.g. in multiprocess Ruby apps).
8
8
  module AtForkMonkeyPatch
9
- AT_FORK_CHILD_BLOCKS = [] # rubocop:disable Style/MutableConstant -- Used to store blocks to run, mutable by design.
10
- private_constant :AT_FORK_CHILD_BLOCKS
9
+ class Callback
10
+ attr_reader :block, :group
11
+
12
+ def initialize(block, group = nil)
13
+ @block = block
14
+ @group = group
15
+ freeze
16
+ end
17
+ end
18
+ private_constant :Callback
19
+
20
+ EMPTY_AT_FORK_BLOCKS = {
21
+ before: [].freeze,
22
+ parent: [].freeze,
23
+ child: [].freeze,
24
+ }.freeze
25
+ private_constant :EMPTY_AT_FORK_BLOCKS
26
+
27
+ @at_fork_blocks = EMPTY_AT_FORK_BLOCKS
28
+
29
+ # Serializes copy-on-write registry updates. Fork dispatch only reads the
30
+ # latest immutable registry reference and never acquires this mutex.
31
+ AT_FORK_REGISTRY_MUTEX = Mutex.new
32
+ private_constant :AT_FORK_REGISTRY_MUTEX
11
33
 
12
34
  def self.supported?
13
35
  Process.respond_to?(:fork)
@@ -31,43 +53,147 @@ module Datadog
31
53
  true
32
54
  end
33
55
 
34
- def self.run_at_fork_blocks(stage)
35
- raise(ArgumentError, "Unsupported stage #{stage}") unless stage == :child
56
+ # Runs the callbacks copied for one fork lifecycle. Before callbacks
57
+ # stop at the first failure; parent and child callbacks all run before
58
+ # the first failure is re-raised.
59
+ def self.run_at_fork_blocks(stage, snapshot:, started: nil)
60
+ callbacks = blocks_for(snapshot, stage)
61
+ if stage == :before
62
+ return callbacks.each do |callback|
63
+ started[callback.group] = true if started && callback.group
64
+ callback.block.call
65
+ end
66
+ end
67
+
68
+ error = nil
69
+ callbacks.each do |callback|
70
+ next if started && callback.group && !started.key?(callback.group)
36
71
 
37
- AT_FORK_CHILD_BLOCKS.each(&:call)
72
+ callback.block.call
73
+ rescue Exception => e # rubocop:disable Lint/RescueException -- finish cleanup, then re-raise the first failure
74
+ error ||= e
75
+ end
76
+ raise error if error
38
77
  end
39
78
 
79
+ def self.run_parent_cleanup(snapshot, started, fork_error)
80
+ run_at_fork_blocks(:parent, snapshot: snapshot, started: started)
81
+ rescue Exception => cleanup_error # rubocop:disable Lint/RescueException -- preserve the original fork failure
82
+ Datadog.logger.warn do
83
+ "Parent at-fork cleanup failed while handling #{fork_error.class}: " \
84
+ "#{cleanup_error.class}: #{cleanup_error.message}"
85
+ end
86
+ end
87
+ private_class_method :run_parent_cleanup
88
+
89
+ # Registers a block to run at the given fork +stage+ (+:before+,
90
+ # +:parent+, or +:child+).
91
+ #
92
+ # Returns the registered block so callers can keep a handle to it and
93
+ # later deregister it via {.remove_at_fork}.
40
94
  def self.at_fork(stage, &block)
41
- raise(ArgumentError, "Unsupported stage #{stage}") unless stage == :child
42
95
  raise(ArgumentError, "Missing block argument") unless block
43
96
 
44
- AT_FORK_CHILD_BLOCKS << block
97
+ AT_FORK_REGISTRY_MUTEX.synchronize do
98
+ current = @at_fork_blocks
99
+ @at_fork_blocks = current.merge(
100
+ stage => (blocks_for(current, stage) + [Callback.new(block)]).freeze
101
+ ).freeze
102
+ end
45
103
 
46
- true
104
+ block
105
+ end
106
+
107
+ # Registers one before/parent/child callback triplet atomically relative
108
+ # to the snapshots taken by fork dispatch.
109
+ def self.at_fork_blocks(before:, parent:, child:)
110
+ blocks = {before: before, parent: parent, child: child}
111
+ group = Object.new.freeze
112
+ AT_FORK_REGISTRY_MUTEX.synchronize do
113
+ current = @at_fork_blocks
114
+ @at_fork_blocks = {
115
+ before: (current.fetch(:before) + [Callback.new(before, group)]).freeze,
116
+ parent: (current.fetch(:parent) + [Callback.new(parent, group)]).freeze,
117
+ child: (current.fetch(:child) + [Callback.new(child, group)]).freeze,
118
+ }.freeze
119
+ end
120
+ blocks
47
121
  end
48
122
 
123
+ # Deregisters a block previously registered with {.at_fork} for the given
124
+ # +stage+. It is a no-op (does not raise) when +block+ was never
125
+ # registered (or was already removed). Raises +ArgumentError+ for an
126
+ # unknown stage, matching the {.at_fork} contract.
127
+ def self.remove_at_fork(stage, block)
128
+ AT_FORK_REGISTRY_MUTEX.synchronize do
129
+ current = @at_fork_blocks
130
+ callbacks = blocks_for(current, stage)
131
+ @at_fork_blocks = current.merge(
132
+ stage => callbacks.reject { |callback| callback.block.equal?(block) }.freeze
133
+ ).freeze
134
+ end
135
+
136
+ nil
137
+ end
138
+
139
+ # Returns the immutable callback registry for one fork lifecycle.
140
+ # Copy-on-write updates publish a new reference, so registrations made
141
+ # afterwards apply only to the next lifecycle without locking this read.
142
+ def self.snapshot_at_fork_blocks
143
+ @at_fork_blocks
144
+ end
145
+
146
+ def self.replace_at_fork_blocks(snapshot)
147
+ AT_FORK_REGISTRY_MUTEX.synchronize { @at_fork_blocks = snapshot }
148
+ end
149
+ private_class_method :replace_at_fork_blocks
150
+
151
+ def self.blocks_for(snapshot, stage)
152
+ case stage
153
+ when :before, :parent, :child then snapshot.fetch(stage)
154
+ else raise(ArgumentError, "Unsupported stage #{stage}")
155
+ end
156
+ end
157
+ private_class_method :blocks_for
158
+
49
159
  # Adds `at_fork` behavior; see parent module for details.
50
160
  module KernelMonkeyPatch
51
161
  def fork
162
+ snapshot = AtForkMonkeyPatch.snapshot_at_fork_blocks
163
+ started = {}
164
+
52
165
  # If a block is provided, it must be wrapped to trigger callbacks.
53
166
  child_block = if block_given?
54
167
  proc do
55
- AtForkMonkeyPatch.run_at_fork_blocks(:child)
168
+ AtForkMonkeyPatch.run_at_fork_blocks(:child, snapshot: snapshot)
56
169
 
57
170
  # Invoke original block
58
171
  yield
59
172
  end
60
173
  end
61
174
 
62
- # Start fork
63
- # If a block is provided, use the wrapped version.
64
- result = child_block.nil? ? super : super(&child_block)
175
+ begin
176
+ # Run pre-fork callbacks in the parent, just before forking.
177
+ AtForkMonkeyPatch.run_at_fork_blocks(:before, snapshot: snapshot, started: started)
178
+
179
+ # Start fork. If a block is provided, use the wrapped version.
180
+ result = child_block.nil? ? super : super(&child_block)
181
+ rescue Exception => e # rubocop:disable Lint/RescueException -- re-raised unchanged; we only need to run parent cleanup first
182
+ # The fork or a before-fork callback failed and we are still in
183
+ # the parent. Restore any state set up by earlier callbacks.
184
+ AtForkMonkeyPatch.send(:run_parent_cleanup, snapshot, started, e)
185
+ raise
186
+ end
65
187
 
66
188
  # When fork gets called without a block, it returns twice:
67
189
  # If we're in the fork, result = nil: trigger child callbacks.
68
- # If we're in the parent, result = pid: we do nothing.
190
+ # If we're in the parent, result = pid: trigger parent callbacks.
69
191
  # (If it gets called with a block, it only returns on the parent)
70
- AtForkMonkeyPatch.run_at_fork_blocks(:child) if result.nil?
192
+ if result.nil?
193
+ AtForkMonkeyPatch.run_at_fork_blocks(:child, snapshot: snapshot)
194
+ else
195
+ AtForkMonkeyPatch.run_at_fork_blocks(:parent, snapshot: snapshot)
196
+ end
71
197
 
72
198
  result
73
199
  end
@@ -78,9 +204,23 @@ module Datadog
78
204
  # Hook provided by Ruby 3.1+ for observability libraries that want to know about fork, see
79
205
  # https://github.com/ruby/ruby/pull/5017 and https://bugs.ruby-lang.org/issues/17795
80
206
  def _fork
81
- pid = super
207
+ snapshot = AtForkMonkeyPatch.snapshot_at_fork_blocks
208
+ started = {}
209
+ begin
210
+ AtForkMonkeyPatch.run_at_fork_blocks(:before, snapshot: snapshot, started: started)
211
+ pid = super
212
+ rescue Exception => e # rubocop:disable Lint/RescueException -- re-raised unchanged; we only need to run parent cleanup first
213
+ # The fork or a before-fork callback failed, so no child was
214
+ # created. Restore state set up by any earlier callbacks.
215
+ AtForkMonkeyPatch.send(:run_parent_cleanup, snapshot, started, e)
216
+ raise
217
+ end
82
218
 
83
- AtForkMonkeyPatch.run_at_fork_blocks(:child) if pid == 0
219
+ if pid == 0
220
+ AtForkMonkeyPatch.run_at_fork_blocks(:child, snapshot: snapshot)
221
+ else
222
+ AtForkMonkeyPatch.run_at_fork_blocks(:parent, snapshot: snapshot)
223
+ end
84
224
 
85
225
  pid
86
226
  end
@@ -89,9 +229,21 @@ module Datadog
89
229
  # keeps executing code in the child process, killing off the parent, thus effectively replacing it.
90
230
  # This is not covered by `_fork` and thus we have some extra code for it.
91
231
  def daemon(*args)
92
- result = super
232
+ snapshot = AtForkMonkeyPatch.snapshot_at_fork_blocks
233
+ started = {}
234
+ begin
235
+ AtForkMonkeyPatch.run_at_fork_blocks(:before, snapshot: snapshot, started: started)
236
+ result = super
237
+ rescue Exception => e # rubocop:disable Lint/RescueException -- re-raised unchanged; we only need to run parent cleanup first
238
+ # `daemon` or a before-fork callback failed, so the original
239
+ # process survives. Restore state set up by earlier callbacks.
240
+ AtForkMonkeyPatch.send(:run_parent_cleanup, snapshot, started, e)
241
+ raise
242
+ end
93
243
 
94
- AtForkMonkeyPatch.run_at_fork_blocks(:child)
244
+ # `daemon` kills the parent, so there is no surviving parent to run
245
+ # `:parent` callbacks in; only the child continues executing.
246
+ AtForkMonkeyPatch.run_at_fork_blocks(:child, snapshot: snapshot)
95
247
 
96
248
  result
97
249
  end
@@ -5,18 +5,15 @@ module Datadog
5
5
  module Utils
6
6
  # Generates values from a consistent sequence
7
7
  class Sequence
8
- def initialize(seed = 0, &block)
8
+ def initialize(seed = 0)
9
9
  @seed = seed
10
10
  @current = seed
11
- @next_item = block
12
11
  end
13
12
 
14
13
  def next
15
- # Steep: https://github.com/soutaro/steep/issues/477
16
- # @type ivar @next_item: ^(::Integer) -> Integer
17
- next_item = @next_item ? @next_item.call(@current) : @current
14
+ current = @current
18
15
  @current += 1
19
- next_item
16
+ current
20
17
  end
21
18
 
22
19
  def reset!
@@ -5,7 +5,16 @@ module Datadog
5
5
  module Utils
6
6
  # Common database-related utility functions.
7
7
  module Time
8
- module_function
8
+ # timecop monkey-patches Time.now, Process.clock_gettime, etc:
9
+ # https://github.com/travisjeffery/timecop/blob/v0.9.11/lib/timecop/time_extensions.rb
10
+ # It keeps the original methods as aliases.
11
+ # We want the real not-mocked time here, so we use those aliases to the original method if present.
12
+ # If not present we assume these methods have not been monkey-patched and are the original.
13
+
14
+ original_clock_gettime_name = Process.respond_to?(:clock_gettime_without_mock) ? :clock_gettime_without_mock : :clock_gettime
15
+ define_singleton_method(:original_clock_gettime, &Process.method(original_clock_gettime_name))
16
+
17
+ MONOTONIC_CLOCK_ID = RUBY_PLATFORM.include?("darwin") ? Process::CLOCK_MONOTONIC_RAW : Process::CLOCK_MONOTONIC
9
18
 
10
19
  # Current monotonic time
11
20
  # On macOS, CLOCK_MONOTONIC only has microsecond precision,
@@ -13,71 +22,24 @@ module Datadog
13
22
  #
14
23
  # @param unit [Symbol] unit for the resulting value, same as ::Process#clock_gettime, defaults to :float_second
15
24
  # @return [Float|Integer] timestamp in the requested unit, since some unspecified starting point
16
- MONOTONIC_CLOCK_ID = RUBY_PLATFORM.include?("darwin") ? Process::CLOCK_MONOTONIC_RAW : Process::CLOCK_MONOTONIC
17
-
18
- def get_time(unit = :float_second)
19
- Process.clock_gettime(MONOTONIC_CLOCK_ID, unit)
25
+ def self.get_time(unit = :float_second)
26
+ original_clock_gettime(MONOTONIC_CLOCK_ID, unit)
20
27
  end
21
28
 
22
29
  # Current wall time.
23
30
  #
24
31
  # @return [Time] current time object
25
- def now
26
- ::Time.now
27
- end
28
-
29
- # Overrides the implementation of `#now
30
- # with the provided callable.
31
- #
32
- # Overriding the method `#now` instead of
33
- # indirectly calling `block` removes
34
- # one level of method call overhead.
35
- #
36
- # @param block [Proc] block that returns a `Time` object representing the current wall time
37
- def now_provider=(block)
38
- class << self
39
- # Avoid method redefinition warning.
40
- # `rescue nil` is added in case customers remove the method
41
- # themselves to squelch the warning.
42
- begin
43
- remove_method(:now)
44
- rescue
45
- nil
46
- end
47
- end
48
- define_singleton_method(:now, &block)
49
- end
50
-
51
- # Overrides the implementation of `#get_time
52
- # with the provided callable.
53
- #
54
- # Overriding the method `#get_time` instead of
55
- # indirectly calling `block` removes
56
- # one level of method call overhead.
57
- #
58
- # @param block [Proc] block that accepts unit and returns timestamp in the requested unit
59
- def get_time_provider=(block)
60
- class << self
61
- # Avoid method redefinition warning
62
- # `rescue nil` is added in case customers remove the method
63
- # themselves to squelch the warning.
64
- begin
65
- remove_method(:get_time)
66
- rescue
67
- nil
68
- end
69
- end
70
- define_singleton_method(:get_time, &block)
71
- end
32
+ original_time_now_name = ::Time.respond_to?(:now_without_mock_time) ? :now_without_mock_time : :now
33
+ define_singleton_method(:now, &::Time.method(original_time_now_name))
72
34
 
73
- def measure(unit = :float_second)
35
+ def self.measure(unit = :float_second)
74
36
  before = get_time(unit)
75
37
  yield
76
38
  after = get_time(unit)
77
39
  after - before
78
40
  end
79
41
 
80
- def as_utc_epoch_ns(time)
42
+ def self.as_utc_epoch_ns(time)
81
43
  # we use #to_r instead of #to_f because Float doesn't have enough precision to represent exact nanoseconds, see
82
44
  # https://rubyapi.org/3.0/o/time#method-i-to_f
83
45
  (time.to_r * 1_000_000_000).to_i
@@ -79,7 +79,7 @@ module Datadog
79
79
  topic: topic,
80
80
  partition: partition,
81
81
  offset: offset,
82
- timestamp_ns: (now.to_f * 1e9).to_i
82
+ timestamp_ns: (now.to_f * 1e9).to_i,
83
83
  }
84
84
  )
85
85
  true
@@ -98,7 +98,7 @@ module Datadog
98
98
  topic: topic,
99
99
  partition: partition,
100
100
  offset: offset,
101
- timestamp: now
101
+ timestamp: now,
102
102
  }
103
103
  )
104
104
  true
@@ -186,7 +186,7 @@ module Datadog
186
186
 
187
187
  bucket[:latest_produce_offsets][partition_key] = [
188
188
  event[:offset],
189
- bucket[:latest_produce_offsets][partition_key] || 0
189
+ bucket[:latest_produce_offsets][partition_key] || 0,
190
190
  ].max
191
191
  end
192
192
 
@@ -196,7 +196,7 @@ module Datadog
196
196
  partition: event[:partition],
197
197
  offset: event[:offset],
198
198
  timestamp: event[:timestamp],
199
- timestamp_sec: event[:timestamp].to_f
199
+ timestamp_sec: event[:timestamp].to_f,
200
200
  }
201
201
 
202
202
  timestamp_ns = (event[:timestamp].to_f * 1e9).to_i
@@ -216,7 +216,7 @@ module Datadog
216
216
  expected_offset: previous_offset + 1,
217
217
  actual_offset: event[:offset],
218
218
  gap_size: event[:offset] - previous_offset - 1,
219
- timestamp_sec: event[:timestamp].to_f
219
+ timestamp_sec: event[:timestamp].to_f,
220
220
  }
221
221
  end
222
222
 
@@ -321,7 +321,7 @@ module Datadog
321
321
  "TracerVersion" => Datadog::VERSION::STRING,
322
322
  "Lang" => "ruby",
323
323
  "Stats" => stats_buckets,
324
- "Hostname" => hostname
324
+ "Hostname" => hostname,
325
325
  } # : ::Hash[::String, (::String | ::Array[::String])]
326
326
 
327
327
  payload["ProcessTags"] = Core::Environment::Process.tags if @settings.experimental_propagate_process_tags_enabled
@@ -400,7 +400,7 @@ module Datadog
400
400
  full_pathway_latency_sec: full_pathway_latency_sec,
401
401
  payload_size: payload_size,
402
402
  tags: tags,
403
- timestamp_sec: timestamp_sec
403
+ timestamp_sec: timestamp_sec,
404
404
  }
405
405
  )
406
406
  true
@@ -449,14 +449,14 @@ module Datadog
449
449
  topic, partition = key.split(":", 2)
450
450
  backlogs << {
451
451
  "Tags" => ["type:kafka_produce", "topic:#{topic}", "partition:#{partition}"],
452
- "Value" => offset
452
+ "Value" => offset,
453
453
  }
454
454
  end
455
455
  bucket[:latest_commit_offsets].each do |key, offset|
456
456
  group, topic, partition = key.split(":", 3)
457
457
  backlogs << {
458
458
  "Tags" => ["type:kafka_commit", "consumer_group:#{group}", "topic:#{topic}", "partition:#{partition}"],
459
- "Value" => offset
459
+ "Value" => offset,
460
460
  }
461
461
  end
462
462
 
@@ -464,7 +464,7 @@ module Datadog
464
464
  "Start" => bucket_time_ns,
465
465
  "Duration" => @bucket_size_ns,
466
466
  "Stats" => bucket_stats,
467
- "Backlogs" => backlogs + serialize_consumer_backlogs
467
+ "Backlogs" => backlogs + serialize_consumer_backlogs,
468
468
  }
469
469
  end
470
470
 
@@ -479,9 +479,9 @@ module Datadog
479
479
  "Tags" => [
480
480
  "type:kafka_commit",
481
481
  "topic:#{stat[:topic]}",
482
- "partition:#{stat[:partition]}"
482
+ "partition:#{stat[:partition]}",
483
483
  ],
484
- "Value" => stat[:offset]
484
+ "Value" => stat[:offset],
485
485
  }
486
486
  end
487
487
  end
@@ -494,7 +494,7 @@ module Datadog
494
494
  {
495
495
  pathway_stats: {},
496
496
  latest_produce_offsets: {},
497
- latest_commit_offsets: {}
497
+ latest_commit_offsets: {},
498
498
  }
499
499
  end
500
500
 
@@ -503,7 +503,7 @@ module Datadog
503
503
  edge_latency: Datadog::Core::DDSketch.new,
504
504
  full_pathway_latency: Datadog::Core::DDSketch.new,
505
505
  payload_size_sum: 0,
506
- payload_size_count: 0
506
+ payload_size_count: 0,
507
507
  }
508
508
  end
509
509
  end
@@ -16,8 +16,42 @@ module Datadog
16
16
  #
17
17
  # @api private
18
18
  class Compiler
19
+ # Compiles +ast+ into eval'able Ruby source.
20
+ #
21
+ # Returns the compiled source and companion compiled Regexp
22
+ # objects.
23
+ #
24
+ # @param ast [untyped] expression AST from the probe definition.
25
+ # @return [Array(String, Array<Regexp>)] the compiled Ruby source and
26
+ # the precompiled Regexps, indexed in the order the compiled code
27
+ # references them.
19
28
  def compile(ast)
20
- compile_partial(ast)
29
+ regexps = []
30
+ code = compile_partial(ast, regexps)
31
+ [code, regexps]
32
+ end
33
+
34
+ # Returns the identifier that +ast+ directly references at its top
35
+ # level, so the message path can redact direct references the same
36
+ # way snapshots redact by name.
37
+ #
38
+ # @param ast [untyped] top-level expression AST from a probe definition.
39
+ # @return [String, nil] the variable name for a `ref`/`iref`, the field
40
+ # name for a `getmember`, or the string key for an `index`; nil when
41
+ # the top-level expression is not a direct reference.
42
+ def redaction_identifier(ast)
43
+ return nil unless Hash === ast && ast.length == 1
44
+
45
+ entry = ast.first
46
+ return nil if entry.nil?
47
+
48
+ op, target = entry
49
+ case op
50
+ when "ref"
51
+ target if String === target
52
+ when "getmember", "index"
53
+ target[1] if Array === target && target.length == 2 && String === target[1]
54
+ end
21
55
  end
22
56
 
23
57
  private
@@ -37,9 +71,13 @@ module Datadog
37
71
  len isEmpty isUndefined
38
72
  ].freeze # steep:ignore IncompatibleAssignment
39
73
 
74
+ # `matches` is also a two-argument method but is special-cased in
75
+ # #compile_partial so that regular expressions can be precompiled
76
+ # once, so it is not listed here.
77
+ #
40
78
  # Steep: https://github.com/soutaro/steep/issues/363
41
79
  TWO_ARG_METHODS = %w[
42
- startsWith endsWith contains matches
80
+ startsWith endsWith contains
43
81
  getmember index instanceof
44
82
  ].freeze # steep:ignore IncompatibleAssignment
45
83
 
@@ -49,13 +87,18 @@ module Datadog
49
87
  "or" => "||",
50
88
  }.freeze
51
89
 
52
- def compile_partial(ast)
90
+ # @param ast [untyped] AST node to compile.
91
+ # @param regexps [Array<Regexp>] output array that collects the
92
+ # companion precompiled Regexp objects.
93
+ # @return [String] compiled Ruby source for +ast+.
94
+ def compile_partial(ast, regexps)
53
95
  case ast
54
96
  when Hash
55
- if ast.length != 1
97
+ entry = ast.first
98
+ if ast.length != 1 || entry.nil?
56
99
  raise DI::Error::InvalidExpression, "Expected hash of length 1: #{ast}"
57
100
  end
58
- op, target = ast.first
101
+ op, target = entry
59
102
  case op
60
103
  when "ref"
61
104
  unless String === target
@@ -94,20 +137,33 @@ module Datadog
94
137
  end
95
138
  when *SINGLE_ARG_METHODS
96
139
  method_name = op.gsub(/[A-Z]/) { |m| "_#{m.downcase}" }
97
- "#{method_name}(#{compile_partial(target)}, '#{var_name_maybe(target)}')"
140
+ "#{method_name}(#{compile_partial(target, regexps)}, '#{var_name_maybe(target)}')"
141
+ when "matches"
142
+ unless Array === target && target.length == 2
143
+ raise DI::Error::InvalidExpression, "Improper matches syntax"
144
+ end
145
+ first, second = target
146
+ if String === second
147
+ # Match against a literal regular expression (string).
148
+ index = precompile_regexp(second, regexps)
149
+ "matches_compiled(#{compile_partial(first, regexps)}, #{index})"
150
+ else
151
+ # Match against a (complex) expression.
152
+ "matches(#{compile_partial(first, regexps)}, (#{compile_partial(second, regexps)}))"
153
+ end
98
154
  when *TWO_ARG_METHODS
99
155
  method_name = op.gsub(/[A-Z]/) { |m| "_#{m.downcase}" }
100
156
  unless Array === target && target.length == 2
101
157
  raise DI::Error::InvalidExpression, "Improper #{op} syntax"
102
158
  end
103
159
  first, second = target
104
- "#{method_name}(#{compile_partial(first)}, (#{compile_partial(second)}))"
160
+ "#{method_name}(#{compile_partial(first, regexps)}, (#{compile_partial(second, regexps)}))"
105
161
  when *MULTI_ARG_METHODS.keys
106
162
  unless Array === target && target.length >= 1
107
163
  raise DI::Error::InvalidExpression, "Improper #{op} syntax"
108
164
  end
109
165
  compiled_targets = target.map do |item|
110
- "(#{compile_partial(item)})"
166
+ "(#{compile_partial(item, regexps)})"
111
167
  end
112
168
  compiled_op = MULTI_ARG_METHODS[op]
113
169
  "(#{compiled_targets.join(" #{compiled_op} ")})"
@@ -115,18 +171,21 @@ module Datadog
115
171
  unless Array === target && target.length == 3
116
172
  raise DI::Error::InvalidExpression, "Improper #{op} syntax"
117
173
  end
118
- "#{op}(#{target.map { |arg| "(#{compile_partial(arg)})" }.join(", ")})"
174
+ "#{op}(#{target.map { |arg| "(#{compile_partial(arg, regexps)})" }.join(", ")})"
119
175
  when "not"
120
- "!(#{compile_partial(target)})"
176
+ "!(#{compile_partial(target, regexps)})"
121
177
  when *OPERATORS.keys
122
178
  unless Array === target && target.length == 2
123
179
  raise DI::Error::InvalidExpression, "Improper #{op} syntax"
124
180
  end
125
181
  first, second = target
126
182
  operator = OPERATORS.fetch(op)
127
- "(#{compile_partial(first)}) #{operator} (#{compile_partial(second)})"
183
+ "(#{compile_partial(first, regexps)}) #{operator} (#{compile_partial(second, regexps)})"
128
184
  when "any", "all", "filter"
129
- "#{op}(#{compile_partial(target.first)}) { |current_item, current_key, current_value| #{compile_partial(target.last)} }"
185
+ unless Array === target && target.length == 2
186
+ raise DI::Error::InvalidExpression, "Improper #{op} syntax"
187
+ end
188
+ "#{op}(#{compile_partial(target.first, regexps)}) { |current_item, current_key, current_value| #{compile_partial(target.last, regexps)} }"
130
189
  else
131
190
  raise DI::Error::InvalidExpression, "Unknown operation: #{op}"
132
191
  end
@@ -162,6 +221,23 @@ module Datadog
162
221
  def escape(needle)
163
222
  needle.gsub("\\") { "\\\\" }.gsub('"') { "\\\"" }.gsub("#") { "\\#" }
164
223
  end
224
+
225
+ # Compile a literal regular expression +regexp_str+ at
226
+ # instrumentation time. Append it to +regexps+, returning its index for
227
+ # Evaluator#matches_compiled to look up.
228
+ #
229
+ # @param regexp_str [String] regular expression source.
230
+ # @param regexps [Array<Regexp>] output array to append the compiled
231
+ # regexp to.
232
+ # @return [Integer] index into +regexps+.
233
+ # @raise [DI::Error::InvalidExpression] if +needle+ is not a valid regexp.
234
+ def precompile_regexp(regexp_str, regexps)
235
+ index = regexps.length
236
+ regexps << Evaluator.compile_regexp(regexp_str)
237
+ index
238
+ rescue RegexpError => exc
239
+ raise DI::Error::InvalidExpression, "Invalid regular expression in matches: #{exc.class}: #{exc.message}"
240
+ end
165
241
  end
166
242
  end
167
243
  end