datadog 2.40.0 → 2.42.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (170) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +120 -59
  3. data/ext/datadog_profiling_native_extension/NativeExtensionDesign.md +12 -24
  4. data/ext/datadog_profiling_native_extension/collectors_cpu_and_wall_time_worker.c +110 -61
  5. data/ext/datadog_profiling_native_extension/collectors_stack.c +13 -5
  6. data/ext/datadog_profiling_native_extension/collectors_thread_context.c +141 -103
  7. data/ext/datadog_profiling_native_extension/collectors_thread_context.h +7 -3
  8. data/ext/datadog_profiling_native_extension/datadog_ruby_common.h +0 -10
  9. data/ext/datadog_profiling_native_extension/extconf.rb +52 -93
  10. data/ext/datadog_profiling_native_extension/heap_recorder.c +338 -255
  11. data/ext/datadog_profiling_native_extension/heap_recorder.h +46 -31
  12. data/ext/datadog_profiling_native_extension/native_extension_helpers.rb +3 -27
  13. data/ext/datadog_profiling_native_extension/private_vm_api_access.c +526 -443
  14. data/ext/datadog_profiling_native_extension/private_vm_api_access.h +15 -3
  15. data/ext/datadog_profiling_native_extension/profiling.c +2 -0
  16. data/ext/datadog_profiling_native_extension/ruby_helpers.c +1 -79
  17. data/ext/datadog_profiling_native_extension/ruby_helpers.h +0 -7
  18. data/ext/datadog_profiling_native_extension/stack_recorder.c +93 -61
  19. data/ext/datadog_profiling_native_extension/stack_recorder.h +12 -4
  20. data/ext/libdatadog_api/datadog_ruby_common.h +0 -10
  21. data/ext/libdatadog_api/di.c +10 -0
  22. data/ext/libdatadog_api/extconf.rb +3 -0
  23. data/ext/libdatadog_api/init.c +2 -0
  24. data/ext/libdatadog_api/otel_thread_context.c +232 -0
  25. data/ext/libdatadog_api/otel_thread_context.h +5 -0
  26. data/ext/libdatadog_api/trace_exporter.c +409 -71
  27. data/ext/libdatadog_extconf_helpers.rb +1 -1
  28. data/lib/datadog/ai_guard/api_client.rb +1 -1
  29. data/lib/datadog/ai_guard/evaluation/request.rb +8 -8
  30. data/lib/datadog/ai_guard/evaluation.rb +10 -3
  31. data/lib/datadog/ai_guard/ext.rb +1 -1
  32. data/lib/datadog/appsec/actions_handler/serializable_backtrace.rb +21 -42
  33. data/lib/datadog/appsec/api_security/endpoint_collection/grape_route_serializer.rb +1 -1
  34. data/lib/datadog/appsec/api_security/endpoint_collection/rails_route_serializer.rb +1 -1
  35. data/lib/datadog/appsec/api_security/endpoint_collection/sinatra_route_serializer.rb +1 -1
  36. data/lib/datadog/appsec/assets/blocked.html +1 -108
  37. data/lib/datadog/appsec/configuration.rb +2 -2
  38. data/lib/datadog/appsec/context.rb +1 -1
  39. data/lib/datadog/appsec/contrib/active_record/instrumentation.rb +1 -1
  40. data/lib/datadog/appsec/contrib/aws_lambda/waf_addresses.rb +2 -2
  41. data/lib/datadog/appsec/contrib/devise/configuration.rb +1 -1
  42. data/lib/datadog/appsec/contrib/excon/ssrf_detection_middleware.rb +2 -2
  43. data/lib/datadog/appsec/contrib/faraday/ssrf_detection_middleware.rb +2 -2
  44. data/lib/datadog/appsec/contrib/graphql/gateway/watcher.rb +1 -1
  45. data/lib/datadog/appsec/contrib/rack/ext.rb +2 -2
  46. data/lib/datadog/appsec/contrib/rack/gateway/watcher.rb +2 -2
  47. data/lib/datadog/appsec/contrib/rails/gateway/watcher.rb +2 -2
  48. data/lib/datadog/appsec/contrib/rest_client/request_ssrf_detection_patch.rb +2 -2
  49. data/lib/datadog/appsec/contrib/sinatra/gateway/watcher.rb +2 -2
  50. data/lib/datadog/appsec/processor/rule_loader.rb +12 -12
  51. data/lib/datadog/appsec/response.rb +1 -1
  52. data/lib/datadog/appsec/route_normalizer/route_pattern.rb +3 -3
  53. data/lib/datadog/appsec/security_engine/engine.rb +3 -3
  54. data/lib/datadog/core/configuration/components.rb +3 -2
  55. data/lib/datadog/core/configuration/option_definition.rb +1 -1
  56. data/lib/datadog/core/configuration/options.rb +1 -1
  57. data/lib/datadog/core/crashtracking/component.rb +5 -1
  58. data/lib/datadog/core/diagnostics/environment_logger.rb +36 -1
  59. data/lib/datadog/core/metrics/helpers.rb +1 -1
  60. data/lib/datadog/core/metrics/options.rb +2 -2
  61. data/lib/datadog/core/process_discovery.rb +1 -1
  62. data/lib/datadog/core/remote/client.rb +1 -1
  63. data/lib/datadog/core/remote/configuration/repository.rb +2 -2
  64. data/lib/datadog/core/telemetry/configuration_value.rb +38 -0
  65. data/lib/datadog/core/telemetry/event/app_client_configuration_change.rb +3 -1
  66. data/lib/datadog/core/telemetry/event/app_endpoints_loaded.rb +1 -1
  67. data/lib/datadog/core/telemetry/event/app_started.rb +5 -24
  68. data/lib/datadog/core/telemetry/event/generate_metrics.rb +1 -1
  69. data/lib/datadog/core/telemetry/event/log.rb +2 -2
  70. data/lib/datadog/core/telemetry/metric.rb +2 -2
  71. data/lib/datadog/core/telemetry/request.rb +2 -2
  72. data/lib/datadog/core/transport/http.rb +1 -1
  73. data/lib/datadog/core/transport/transport.rb +1 -1
  74. data/lib/datadog/core/utils/at_fork_monkey_patch.rb +127 -43
  75. data/lib/datadog/data_streams/pathway_context.rb +20 -22
  76. data/lib/datadog/data_streams/processor.rb +45 -14
  77. data/lib/datadog/di/el/compiler.rb +23 -0
  78. data/lib/datadog/di/el/expression.rb +8 -5
  79. data/lib/datadog/di/instrumenter.rb +46 -5
  80. data/lib/datadog/di/logger.rb +2 -2
  81. data/lib/datadog/di/probe.rb +9 -1
  82. data/lib/datadog/di/probe_builder.rb +5 -4
  83. data/lib/datadog/di/probe_notification_builder.rb +3 -2
  84. data/lib/datadog/di/remote.rb +3 -3
  85. data/lib/datadog/di/serializer.rb +29 -4
  86. data/lib/datadog/error_tracking/component.rb +1 -1
  87. data/lib/datadog/kit/appsec/events/v2.rb +2 -2
  88. data/lib/datadog/open_feature/evaluation_engine.rb +29 -3
  89. data/lib/datadog/open_feature/exposures/batch_builder.rb +1 -1
  90. data/lib/datadog/open_feature/exposures/event.rb +15 -8
  91. data/lib/datadog/open_feature/ext.rb +19 -0
  92. data/lib/datadog/open_feature/flag_evaluation/aggregator.rb +236 -80
  93. data/lib/datadog/open_feature/flag_evaluation/writer.rb +179 -68
  94. data/lib/datadog/open_feature/hooks/flag_eval_evp_hook.rb +24 -21
  95. data/lib/datadog/open_feature/native_evaluator.rb +33 -6
  96. data/lib/datadog/open_feature/noop_evaluator.rb +5 -0
  97. data/lib/datadog/open_feature/provider.rb +11 -2
  98. data/lib/datadog/opentelemetry/sdk/propagator.rb +1 -1
  99. data/lib/datadog/opentelemetry/trace.rb +3 -0
  100. data/lib/datadog/profiling/collectors/cpu_and_wall_time_worker.rb +3 -0
  101. data/lib/datadog/profiling/collectors/info.rb +1 -1
  102. data/lib/datadog/profiling/collectors/thread_context.rb +0 -4
  103. data/lib/datadog/profiling/component.rb +8 -16
  104. data/lib/datadog/profiling/tasks/exec.rb +1 -1
  105. data/lib/datadog/symbol_database/extractor.rb +6 -6
  106. data/lib/datadog/tracing/configuration/dynamic/option.rb +4 -0
  107. data/lib/datadog/tracing/configuration/dynamic.rb +5 -3
  108. data/lib/datadog/tracing/contrib/action_cable/events.rb +1 -1
  109. data/lib/datadog/tracing/contrib/action_mailer/events.rb +1 -1
  110. data/lib/datadog/tracing/contrib/action_pack/action_controller/instrumentation.rb +2 -2
  111. data/lib/datadog/tracing/contrib/action_view/events.rb +1 -1
  112. data/lib/datadog/tracing/contrib/active_model_serializers/events.rb +1 -1
  113. data/lib/datadog/tracing/contrib/active_record/configuration/resolver.rb +1 -1
  114. data/lib/datadog/tracing/contrib/active_record/events/sql.rb +1 -0
  115. data/lib/datadog/tracing/contrib/active_record/events.rb +1 -1
  116. data/lib/datadog/tracing/contrib/active_support/cache/events/cache.rb +2 -2
  117. data/lib/datadog/tracing/contrib/aws/ext.rb +1 -1
  118. data/lib/datadog/tracing/contrib/aws/services.rb +1 -1
  119. data/lib/datadog/tracing/contrib/elasticsearch/quantize.rb +1 -1
  120. data/lib/datadog/tracing/contrib/ext.rb +3 -3
  121. data/lib/datadog/tracing/contrib/grpc/datadog_interceptor/client.rb +1 -1
  122. data/lib/datadog/tracing/contrib/grpc/datadog_interceptor/server.rb +1 -1
  123. data/lib/datadog/tracing/contrib/grpc/distributed/propagation.rb +1 -1
  124. data/lib/datadog/tracing/contrib/kafka/events.rb +1 -1
  125. data/lib/datadog/tracing/contrib/karafka/distributed/propagation.rb +1 -1
  126. data/lib/datadog/tracing/contrib/mongodb/parsers.rb +1 -1
  127. data/lib/datadog/tracing/contrib/opensearch/ext.rb +1 -1
  128. data/lib/datadog/tracing/contrib/opensearch/quantize.rb +1 -1
  129. data/lib/datadog/tracing/contrib/patchable.rb +1 -1
  130. data/lib/datadog/tracing/contrib/patcher.rb +1 -1
  131. data/lib/datadog/tracing/contrib/que/tracer.rb +1 -1
  132. data/lib/datadog/tracing/contrib/racecar/events.rb +1 -1
  133. data/lib/datadog/tracing/contrib/rack/configuration/settings.rb +1 -1
  134. data/lib/datadog/tracing/contrib/rack/header_tagging.rb +1 -1
  135. data/lib/datadog/tracing/contrib/rack/patcher.rb +1 -1
  136. data/lib/datadog/tracing/contrib/rack/trace_proxy_middleware.rb +1 -1
  137. data/lib/datadog/tracing/contrib/redis/configuration/resolver.rb +1 -1
  138. data/lib/datadog/tracing/contrib/redis/quantize.rb +36 -2
  139. data/lib/datadog/tracing/contrib/redis/trace_middleware.rb +9 -3
  140. data/lib/datadog/tracing/contrib/sequel/utils.rb +1 -1
  141. data/lib/datadog/tracing/contrib/sidekiq/distributed/propagation.rb +1 -1
  142. data/lib/datadog/tracing/contrib/sinatra/configuration/settings.rb +1 -1
  143. data/lib/datadog/tracing/contrib/sneakers/tracer.rb +1 -1
  144. data/lib/datadog/tracing/contrib/utils/quantization/hash_formatter.rb +1 -1
  145. data/lib/datadog/tracing/contrib/waterdrop/distributed/propagation.rb +1 -1
  146. data/lib/datadog/tracing/correlation.rb +2 -2
  147. data/lib/datadog/tracing/diagnostics/environment_logger.rb +2 -2
  148. data/lib/datadog/tracing/distributed/baggage.rb +1 -3
  149. data/lib/datadog/tracing/distributed/datadog.rb +3 -3
  150. data/lib/datadog/tracing/distributed/propagation.rb +3 -0
  151. data/lib/datadog/tracing/distributed/trace_context.rb +14 -271
  152. data/lib/datadog/tracing/distributed/trace_state/datadog.rb +233 -0
  153. data/lib/datadog/tracing/distributed/trace_state/ext.rb +44 -0
  154. data/lib/datadog/tracing/distributed/trace_state/open_telemetry.rb +156 -0
  155. data/lib/datadog/tracing/distributed/trace_state.rb +121 -0
  156. data/lib/datadog/tracing/metadata/tagging.rb +1 -1
  157. data/lib/datadog/tracing/otel_thread_context.rb +30 -0
  158. data/lib/datadog/tracing/remote.rb +197 -29
  159. data/lib/datadog/tracing/sampling/rule_sampler.rb +2 -0
  160. data/lib/datadog/tracing/span.rb +1 -1
  161. data/lib/datadog/tracing/span_operation.rb +1 -1
  162. data/lib/datadog/tracing/trace_digest.rb +22 -4
  163. data/lib/datadog/tracing/trace_operation.rb +22 -10
  164. data/lib/datadog/tracing/tracer.rb +5 -5
  165. data/lib/datadog/tracing/transport/io/traces.rb +1 -1
  166. data/lib/datadog/tracing/transport/native.rb +78 -54
  167. data/lib/datadog/tracing/writer.rb +1 -1
  168. data/lib/datadog/version.rb +1 -1
  169. metadata +15 -8
  170. data/lib/datadog/tracing/distributed/datadog_tags_codec.rb +0 -69
@@ -98,40 +98,38 @@ module Datadog
98
98
  end
99
99
  private_class_method :decode
100
100
 
101
- # Encode an unsigned 64-bit integer using LEB128 variable-length encoding.
101
+ # Encode a signed 64-bit integer using ZigZag + LEB128 variable-length encoding.
102
102
  #
103
- # This implements unsigned LEB128 (Little Endian Base 128) encoding as specified
104
- # in DWARF5 standard section 7.6:
105
- # https://dwarfstd.org/doc/DWARF5.pdf#page=301
103
+ # dd-trace-go's DSM pathway encoder (internal/datastreams/propagator.go, via
104
+ # sketches-go's ddsketch/encoding package) ZigZag-maps the signed value before
105
+ # applying plain unsigned LEB128 (DWARF5 section 7.6:
106
+ # https://dwarfstd.org/doc/DWARF5.pdf#page=301). This implementation previously
107
+ # skipped the ZigZag step, which silently decoded to the wrong timestamp on any
108
+ # dd-trace-go consumer -- see https://github.com/DataDog/dd-trace-go/issues/5163.
106
109
  #
107
- # Each byte uses 7 bits for data and 1 bit to indicate continuation.
108
- # The high bit is set if more bytes follow, clear for the final byte.
110
+ # ZigZag maps signed values to unsigned ones so small-magnitude negative numbers
111
+ # still encode to few bytes: 0, -1, 1, -2, 2, ... -> 0, 1, 2, 3, 4, ...
109
112
  #
110
- # @param value [Integer] Unsigned integer value to encode
113
+ # @param value [Integer] Signed integer value to encode
111
114
  # @return [String] Binary string of encoded bytes
112
115
  def encode_var_int_64(value)
116
+ zigzag = (value << 1) ^ (value >> 63)
113
117
  bytes = []
114
- while value >= 0x80
115
- bytes << ((value & 0x7F) | 0x80)
116
- value >>= 7
118
+ while zigzag >= 0x80
119
+ bytes << ((zigzag & 0x7F) | 0x80)
120
+ zigzag >>= 7
117
121
  end
118
- bytes << value
122
+ bytes << zigzag
119
123
  bytes.pack("C*")
120
124
  end
121
125
 
122
- # Decode an unsigned LEB128 variable-length integer from IO stream.
126
+ # Decode a signed 64-bit integer encoded with ZigZag + LEB128 from an IO stream.
123
127
  #
124
- # This implements unsigned LEB128 (Little Endian Base 128) decoding as specified
125
- # in DWARF5 standard section 7.6:
126
- # https://dwarfstd.org/doc/DWARF5.pdf#page=301
127
- #
128
- # VarInt format: Each byte uses 7 bits for data, 1 bit for continuation
129
- # - High bit set = more bytes follow
130
- # - High bit clear = final byte
131
- # - Data bits accumulated in little-endian order
128
+ # See #encode_var_int_64 for why the ZigZag un-map is required to match
129
+ # dd-trace-go's wire format.
132
130
  #
133
131
  # @param io [StringIO] IO stream to read from
134
- # @return [Integer, nil] Decoded unsigned integer, or nil if malformed
132
+ # @return [Integer, nil] Decoded signed integer, or nil if malformed
135
133
  def self.decode_varint(io)
136
134
  value = 0
137
135
  shift = 0
@@ -143,7 +141,7 @@ module Datadog
143
141
  value |= (byte & 0x7F) << shift
144
142
 
145
143
  # If high bit is clear, we're done
146
- return value unless (byte & 0x80).nonzero?
144
+ return (value >> 1) ^ -(value & 1) unless (byte & 0x80).nonzero?
147
145
 
148
146
  shift += 7
149
147
 
@@ -62,10 +62,27 @@ module Datadog
62
62
 
63
63
  super()
64
64
  self.loop_base_interval = interval
65
+ # Without this, a preload-then-fork deployment model (e.g. Puma cluster mode with
66
+ # preload_app!, or Sidekiq Enterprise's sidekiqswarm with SIDEKIQ_PRELOAD_APP) leaves
67
+ # every forked child with a dead flush thread: Ruby threads don't survive fork, and the
68
+ # default FORK_POLICY_STOP means #perform would otherwise just mark the worker stopped
69
+ # instead of restarting it. See #restart_flush_thread.
70
+ self.fork_policy = Core::Workers::Async::Thread::FORK_POLICY_RESTART
65
71
 
66
72
  perform
67
73
  end
68
74
 
75
+ # Restarts the flush thread if a fork has been detected since this processor was created.
76
+ # Called by Components#after_fork. Safe to call even when no fork occurred: #perform (via
77
+ # Workers::Async::Thread) only restarts the worker when +forked?+ is true. Deliberately not
78
+ # named +after_fork+: Workers::Async::Thread already defines a protected +after_fork+
79
+ # template method that #perform's internal restart path (+restart_after_fork+) calls on
80
+ # every restart -- overriding it here would recurse.
81
+ def restart_flush_thread
82
+ discard_inherited_state_after_fork! if forked?
83
+ perform
84
+ end
85
+
69
86
  # Track Kafka produce offset for lag monitoring
70
87
  # @param topic [String] The Kafka topic name
71
88
  # @param partition [Integer] The partition number
@@ -79,7 +96,7 @@ module Datadog
79
96
  topic: topic,
80
97
  partition: partition,
81
98
  offset: offset,
82
- timestamp_ns: (now.to_f * 1e9).to_i
99
+ timestamp_ns: (now.to_f * 1e9).to_i,
83
100
  }
84
101
  )
85
102
  true
@@ -98,7 +115,7 @@ module Datadog
98
115
  topic: topic,
99
116
  partition: partition,
100
117
  offset: offset,
101
- timestamp: now
118
+ timestamp: now,
102
119
  }
103
120
  )
104
121
  true
@@ -186,7 +203,7 @@ module Datadog
186
203
 
187
204
  bucket[:latest_produce_offsets][partition_key] = [
188
205
  event[:offset],
189
- bucket[:latest_produce_offsets][partition_key] || 0
206
+ bucket[:latest_produce_offsets][partition_key] || 0,
190
207
  ].max
191
208
  end
192
209
 
@@ -196,7 +213,7 @@ module Datadog
196
213
  partition: event[:partition],
197
214
  offset: event[:offset],
198
215
  timestamp: event[:timestamp],
199
- timestamp_sec: event[:timestamp].to_f
216
+ timestamp_sec: event[:timestamp].to_f,
200
217
  }
201
218
 
202
219
  timestamp_ns = (event[:timestamp].to_f * 1e9).to_i
@@ -216,7 +233,7 @@ module Datadog
216
233
  expected_offset: previous_offset + 1,
217
234
  actual_offset: event[:offset],
218
235
  gap_size: event[:offset] - previous_offset - 1,
219
- timestamp_sec: event[:timestamp].to_f
236
+ timestamp_sec: event[:timestamp].to_f,
220
237
  }
221
238
  end
222
239
 
@@ -302,6 +319,20 @@ module Datadog
302
319
  PathwayContext.decode_b64(encoded_ctx)
303
320
  end
304
321
 
322
+ # A fork copies whatever was buffered in the parent (unflushed events, aggregated
323
+ # buckets, consumer stats) into the child. If the parent process is still alive and
324
+ # flushing (e.g. a supervisor process that stays up after forking workers), letting
325
+ # the child flush that same inherited data as well would double-report it to the
326
+ # agent. Discard it instead: losing a few seconds of pre-fork stats is preferable to
327
+ # corrupting pipeline throughput/lag numbers with duplicates.
328
+ def discard_inherited_state_after_fork!
329
+ @event_buffer.pop
330
+ @stats_mutex.synchronize do
331
+ @buckets.clear
332
+ @consumer_stats.clear
333
+ end
334
+ end
335
+
305
336
  def flush_stats
306
337
  stats_buckets = @stats_mutex.synchronize do
307
338
  return if @buckets.empty? && @consumer_stats.empty?
@@ -321,7 +352,7 @@ module Datadog
321
352
  "TracerVersion" => Datadog::VERSION::STRING,
322
353
  "Lang" => "ruby",
323
354
  "Stats" => stats_buckets,
324
- "Hostname" => hostname
355
+ "Hostname" => hostname,
325
356
  } # : ::Hash[::String, (::String | ::Array[::String])]
326
357
 
327
358
  payload["ProcessTags"] = Core::Environment::Process.tags if @settings.experimental_propagate_process_tags_enabled
@@ -400,7 +431,7 @@ module Datadog
400
431
  full_pathway_latency_sec: full_pathway_latency_sec,
401
432
  payload_size: payload_size,
402
433
  tags: tags,
403
- timestamp_sec: timestamp_sec
434
+ timestamp_sec: timestamp_sec,
404
435
  }
405
436
  )
406
437
  true
@@ -449,14 +480,14 @@ module Datadog
449
480
  topic, partition = key.split(":", 2)
450
481
  backlogs << {
451
482
  "Tags" => ["type:kafka_produce", "topic:#{topic}", "partition:#{partition}"],
452
- "Value" => offset
483
+ "Value" => offset,
453
484
  }
454
485
  end
455
486
  bucket[:latest_commit_offsets].each do |key, offset|
456
487
  group, topic, partition = key.split(":", 3)
457
488
  backlogs << {
458
489
  "Tags" => ["type:kafka_commit", "consumer_group:#{group}", "topic:#{topic}", "partition:#{partition}"],
459
- "Value" => offset
490
+ "Value" => offset,
460
491
  }
461
492
  end
462
493
 
@@ -464,7 +495,7 @@ module Datadog
464
495
  "Start" => bucket_time_ns,
465
496
  "Duration" => @bucket_size_ns,
466
497
  "Stats" => bucket_stats,
467
- "Backlogs" => backlogs + serialize_consumer_backlogs
498
+ "Backlogs" => backlogs + serialize_consumer_backlogs,
468
499
  }
469
500
  end
470
501
 
@@ -479,9 +510,9 @@ module Datadog
479
510
  "Tags" => [
480
511
  "type:kafka_commit",
481
512
  "topic:#{stat[:topic]}",
482
- "partition:#{stat[:partition]}"
513
+ "partition:#{stat[:partition]}",
483
514
  ],
484
- "Value" => stat[:offset]
515
+ "Value" => stat[:offset],
485
516
  }
486
517
  end
487
518
  end
@@ -494,7 +525,7 @@ module Datadog
494
525
  {
495
526
  pathway_stats: {},
496
527
  latest_produce_offsets: {},
497
- latest_commit_offsets: {}
528
+ latest_commit_offsets: {},
498
529
  }
499
530
  end
500
531
 
@@ -503,7 +534,7 @@ module Datadog
503
534
  edge_latency: Datadog::Core::DDSketch.new,
504
535
  full_pathway_latency: Datadog::Core::DDSketch.new,
505
536
  payload_size_sum: 0,
506
- payload_size_count: 0
537
+ payload_size_count: 0,
507
538
  }
508
539
  end
509
540
  end
@@ -31,6 +31,29 @@ module Datadog
31
31
  [code, regexps]
32
32
  end
33
33
 
34
+ # Returns the identifier that +ast+ directly references at its top
35
+ # level, so the message path can redact direct references the same
36
+ # way snapshots redact by name.
37
+ #
38
+ # @param ast [untyped] top-level expression AST from a probe definition.
39
+ # @return [String, nil] the variable name for a `ref`/`iref`, the field
40
+ # name for a `getmember`, or the string key for an `index`; nil when
41
+ # the top-level expression is not a direct reference.
42
+ def redaction_identifier(ast)
43
+ return nil unless Hash === ast && ast.length == 1
44
+
45
+ entry = ast.first
46
+ return nil if entry.nil?
47
+
48
+ op, target = entry
49
+ case op
50
+ when "ref"
51
+ target if String === target
52
+ when "getmember", "index"
53
+ target[1] if Array === target && target.length == 2 && String === target[1]
54
+ end
55
+ end
56
+
34
57
  private
35
58
 
36
59
  # Steep: https://github.com/soutaro/steep/issues/363
@@ -12,12 +12,17 @@ module Datadog
12
12
  # @param regexps [Array<Regexp>] precompiled `matches` regexps (see
13
13
  # Compiler#precompile_regexp), the second element returned by
14
14
  # Compiler#compile.
15
- def initialize(dsl_expr, compiled_expr, regexps = [])
15
+ # @param redaction_identifier [String, nil] the identifier this
16
+ # expression directly references at its top level (see
17
+ # Compiler#redaction_identifier); nil when it is not a direct
18
+ # reference.
19
+ def initialize(dsl_expr, compiled_expr, regexps: [], redaction_identifier: nil)
16
20
  unless String === compiled_expr
17
21
  raise ArgumentError, "compiled_expr must be a string"
18
22
  end
19
23
 
20
24
  @dsl_expr = dsl_expr
25
+ @redaction_identifier = redaction_identifier
21
26
 
22
27
  cls = Class.new(Evaluator)
23
28
  cls.class_exec do
@@ -28,14 +33,12 @@ module Datadog
28
33
  end
29
34
  RUBY
30
35
  end
31
- # cls inherits Evaluator#initialize(regexps), but Steep types
32
- # Class#new as () -> untyped and cannot see that initializer
33
- # through this dynamically created subclass.
34
- @evaluator = cls.new(regexps) # steep:ignore UnexpectedPositionalArgument
36
+ @evaluator = cls.new(regexps)
35
37
  end
36
38
 
37
39
  attr_reader :dsl_expr
38
40
  attr_reader :evaluator
41
+ attr_reader :redaction_identifier
39
42
 
40
43
  def evaluate(context)
41
44
  @evaluator.evaluate(context)
@@ -1,5 +1,6 @@
1
1
  # frozen_string_literal: true
2
2
 
3
+ require_relative "../core/rate_limiter"
3
4
  require_relative "../core/utils/time"
4
5
  require_relative "../ruby_version"
5
6
  require_relative "fatal_exceptions"
@@ -69,12 +70,22 @@ module Datadog
69
70
  #
70
71
  # @api private
71
72
  class Instrumenter
73
+ # Maximum number of capturing-probe snapshots admitted per second across the
74
+ # whole process.
75
+ GLOBAL_SNAPSHOT_RATE_LIMIT = 20
76
+
77
+ # Maximum number of non-capturing probe emissions admitted per second across
78
+ # the whole process.
79
+ GLOBAL_LOG_RATE_LIMIT = 5000
80
+
72
81
  def initialize(settings, serializer, logger, code_tracker: nil, telemetry: nil)
73
82
  @settings = settings
74
83
  @serializer = serializer
75
84
  @logger = logger
76
85
  @telemetry = telemetry
77
86
  @code_tracker = code_tracker
87
+ @global_snapshot_rate_limiter = Datadog::Core::TokenBucket.new(GLOBAL_SNAPSHOT_RATE_LIMIT)
88
+ @global_log_rate_limiter = Datadog::Core::TokenBucket.new(GLOBAL_LOG_RATE_LIMIT)
78
89
 
79
90
  @lock = Mutex.new
80
91
  end
@@ -91,12 +102,31 @@ module Datadog
91
102
  # Component#start! assigns the now-current tracker here.
92
103
  attr_writer :code_tracker
93
104
 
105
+ # Process-wide snapshot rate limiter.
106
+ # @return [Datadog::Core::TokenBucket]
107
+ attr_reader :global_snapshot_rate_limiter
108
+ # Process-wide log rate limiter.
109
+ # @return [Datadog::Core::TokenBucket]
110
+ attr_reader :global_log_rate_limiter
111
+
94
112
  def capture_expression_evaluator
95
113
  @capture_expression_evaluator ||= CaptureExpressionEvaluator.new(
96
114
  settings: settings, serializer: serializer, logger: logger, telemetry: telemetry,
97
115
  )
98
116
  end
99
117
 
118
+ # Returns the process-wide rate limiter for the probe.
119
+ #
120
+ # @param probe [Probe] the probe whose invocation is being rate limited
121
+ # @return [Datadog::Core::TokenBucket] the shared limiter for the probe's category
122
+ def probe_global_rate_limiter(probe)
123
+ if probe.capturing?
124
+ global_snapshot_rate_limiter
125
+ else
126
+ global_log_rate_limiter
127
+ end
128
+ end
129
+
100
130
  # This is a substitute for Thread::Backtrace::Location
101
131
  # which does not have a public constructor.
102
132
  # Used for the fabricated stack frame for the method itself
@@ -160,6 +190,7 @@ module Datadog
160
190
  end
161
191
 
162
192
  loc = target_method&.source_location
193
+ # @type var instrumenter: Instrumenter
163
194
  instrumenter = self
164
195
 
165
196
  mod = Module.new do
@@ -193,7 +224,7 @@ module Datadog
193
224
  # user probe on Proc#call cannot intercept the trampoline, and no
194
225
  # Proc is allocated for the block.
195
226
  if RubyVersion.is?(">= 3")
196
- define_method(method_name) do |*args, **kwargs, &target_block| # steep:ignore NoMethod
227
+ define_method(method_name) do |*args, **kwargs, &target_block|
197
228
  # steep:ignore FallbackAny below: Steep cannot narrow the
198
229
  # **kwargs parameter inside this define_method block, so it
199
230
  # falls back to untyped at the super and run_method_probe sites.
@@ -211,7 +242,7 @@ module Datadog
211
242
  end
212
243
  end
213
244
  else
214
- define_method(method_name) do |*args, &target_block| # steep:ignore NoMethod
245
+ define_method(method_name) do |*args, &target_block|
215
246
  if DI.in_probe?
216
247
  return super(*args, &target_block)
217
248
  end
@@ -230,7 +261,7 @@ module Datadog
230
261
  super(*args, &target_block)
231
262
  end
232
263
  end
233
- ruby2_keywords(method_name) if respond_to?(:ruby2_keywords, true) # steep:ignore NoMethod
264
+ ruby2_keywords(method_name) if respond_to?(:ruby2_keywords, true)
234
265
  end
235
266
  end
236
267
 
@@ -461,7 +492,7 @@ module Datadog
461
492
  # @param args [Array] positional arguments passed to the probed method
462
493
  # @param kwargs [Hash{Symbol => Object}] keyword arguments passed to the probed method
463
494
  # @param target_block [Proc, nil] block argument passed to the probed method
464
- # @param target_self [Object] the receiver of the probed method invocation
495
+ # @param target_self [any] the receiver of the probed method invocation
465
496
  # @param probe [Datadog::DI::Probe] the probe whose callback this invocation runs
466
497
  # @param responder [#probe_executed_callback, #probe_condition_evaluation_failed_callback] callback target invoked with the built Context
467
498
  # @param loc [Array(String, Integer), nil] source location of the probed method, or nil for virtual/lazily-defined methods
@@ -528,7 +559,12 @@ module Datadog
528
559
  end
529
560
 
530
561
  rate_limiter = probe.rate_limiter
531
- if continue and rate_limiter.nil? || rate_limiter.allow?
562
+ admitted = continue && (rate_limiter.nil? || rate_limiter.allow?)
563
+ if admitted && !probe_global_rate_limiter(probe).allow?
564
+ admitted = false
565
+ logger.trace { "di: #{probe.type} probe #{probe.id}: skipping due to global rate limit" }
566
+ end
567
+ if admitted
532
568
  # Arguments may be mutated by the method, therefore
533
569
  # they need to be serialized prior to method invocation.
534
570
  serialized_entry_args = if probe.capture_snapshot?
@@ -793,6 +829,11 @@ module Datadog
793
829
  # and check that it is in fact set.
794
830
  return if probe.rate_limiter && !probe.rate_limiter.allow?
795
831
 
832
+ unless probe_global_rate_limiter(probe).allow?
833
+ logger.trace { "di: #{probe.type} probe #{probe.id}: skipping due to global rate limit" }
834
+ return
835
+ end
836
+
796
837
  # The context creation is relatively expensive and we don't
797
838
  # want to run it if the callback won't be executed due to the
798
839
  # rate limit.
@@ -20,9 +20,9 @@ module Datadog
20
20
 
21
21
  def_delegators :target, :debug
22
22
 
23
- def trace(&block)
23
+ def trace
24
24
  if settings.dynamic_instrumentation.internal.trace_logging
25
- debug(&block)
25
+ debug { yield }
26
26
  end
27
27
  end
28
28
  end
@@ -97,7 +97,7 @@ module Datadog
97
97
  @evaluate_at = evaluate_at
98
98
  @condition = condition
99
99
 
100
- @rate_limit = rate_limit || ((@capture_snapshot || !@capture_expressions.empty?) ? 1 : 5000)
100
+ @rate_limit = rate_limit || (capturing? ? 1 : 5000)
101
101
  @rate_limiter = Datadog::Core::TokenBucket.new(@rate_limit)
102
102
 
103
103
  # At most one report per second.
@@ -166,6 +166,14 @@ module Datadog
166
166
  !@capture_expressions.empty?
167
167
  end
168
168
 
169
+ # Whether this probe captures user data, either as a full snapshot
170
+ # or via capture expressions.
171
+ #
172
+ # @return [Boolean]
173
+ def capturing?
174
+ capture_snapshot? || capture_expressions?
175
+ end
176
+
169
177
  def evaluate_at_entry?
170
178
  evaluate_at == :entry
171
179
  end
@@ -46,7 +46,7 @@ module Datadog
46
46
  raise ArgumentError, "Malformed condition specification for probe: #{config}"
47
47
  end
48
48
  compiled, regexps = EL::Compiler.new.compile(cond_spec["json"])
49
- EL::Expression.new(cond_spec["dsl"], compiled, regexps)
49
+ EL::Expression.new(cond_spec["dsl"], compiled, regexps: regexps)
50
50
  end
51
51
  capture_expressions = build_capture_expressions(config["captureExpressions"])
52
52
  capture_expressions = dedup_capture_expressions(capture_expressions, config["id"], logger)
@@ -101,7 +101,7 @@ module Datadog
101
101
  raise ArgumentError, "captureExpressions entry #{name}: missing or malformed expr"
102
102
  end
103
103
  compiled, regexps = EL::Compiler.new.compile(expr_spec["json"])
104
- expr = EL::Expression.new(expr_spec["dsl"], compiled, regexps)
104
+ expr = EL::Expression.new(expr_spec["dsl"], compiled, regexps: regexps)
105
105
  limits = build_capture_limits(entry["capture"])
106
106
  CaptureExpression.new(name: name, expr: expr, limits: limits)
107
107
  end
@@ -152,8 +152,9 @@ module Datadog
152
152
  unless dsl = segment["dsl"]
153
153
  raise ArgumentError, "Missing dsl for json in segment: #{segment}"
154
154
  end
155
- compiled, regexps = EL::Compiler.new.compile(ast)
156
- EL::Expression.new(dsl, compiled, regexps)
155
+ compiler = EL::Compiler.new
156
+ compiled, regexps = compiler.compile(ast)
157
+ EL::Expression.new(dsl, compiled, regexps: regexps, redaction_identifier: compiler.redaction_identifier(ast))
157
158
  else
158
159
  # TODO report to telemetry?
159
160
  end
@@ -179,7 +179,7 @@ module Datadog
179
179
  if status == "ERROR"
180
180
  diagnostics[:exception] = { # steep:ignore
181
181
  type: exception ? exception.class.name : "Error",
182
- message: exception ? exception.message : message
182
+ message: exception ? exception.message : message,
183
183
  }
184
184
  end
185
185
 
@@ -379,6 +379,7 @@ module Datadog
379
379
  thread_id: nil,
380
380
  version: 2,
381
381
  },
382
+ runtime_id: Core::Environment::Identity.id,
382
383
  # TODO add tests that the trace/span id is correctly propagated
383
384
  "dd.trace_id": active_trace&.id&.to_s,
384
385
  "dd.span_id": active_span&.id&.to_s,
@@ -405,7 +406,7 @@ module Datadog
405
406
  when String
406
407
  segment
407
408
  when EL::Expression
408
- serializer.serialize_value_for_message(segment.evaluate(context))
409
+ serializer.serialize_value_for_message(segment.evaluate(context), name: segment.redaction_identifier)
409
410
  else
410
411
  raise ArgumentError, "Invalid template segment type: #{segment}"
411
412
  end
@@ -22,7 +22,7 @@ module Datadog
22
22
  # when DI is explicitly disabled, or the runtime cannot run DI
23
23
  # (JRuby, Ruby 2.5), that block — including this bit — is skipped.
24
24
  # The enable signal itself is delivered in APM_TRACING payloads and
25
- # routed here by Tracing::Remote.process_config.
25
+ # routed here by Tracing::Remote.merge_and_apply_configs.
26
26
  CAPABILITIES = [
27
27
  1 << 38, # APM_TRACING_ENABLE_DYNAMIC_INSTRUMENTATION: Implicit DI enablement
28
28
  ].freeze
@@ -37,7 +37,7 @@ module Datadog
37
37
 
38
38
  # Entry point for the RC-driven DI enable/disable path.
39
39
  #
40
- # Invoked from {Datadog::Tracing::Remote.process_config} when an
40
+ # Invoked from {Datadog::Tracing::Remote.merge_and_apply_configs} when an
41
41
  # APM_TRACING payload carries `dynamic_instrumentation_enabled`. Runs
42
42
  # on the remote-config thread; never raises.
43
43
  #
@@ -45,7 +45,7 @@ module Datadog
45
45
  # false to stop it. The `DD_DYNAMIC_INSTRUMENTATION_ENABLED=false`
46
46
  # env var blocks an enable here (see {.explicitly_disabled?}).
47
47
  # @param repository [Datadog::Core::Remote::Configuration::Repository, nil]
48
- # the RC repository, passed by {Datadog::Tracing::Remote.process_config}
48
+ # the RC repository, passed by {Datadog::Tracing::Remote.apply_lib_config}
49
49
  # so that a stopped->started transition can reconcile against probes that
50
50
  # were delivered in an earlier poll. nil when called outside the RC
51
51
  # dispatch path (e.g. unit tests), in which case no reconcile happens.
@@ -50,6 +50,13 @@ module Datadog
50
50
  # return a safe stub rather than tear the process down.
51
51
  SERIALIZABLE_FATAL_EXCEPTION_CLASSES = [SystemExit, SignalException].freeze
52
52
 
53
+ # Placeholder emitted by #serialize_value_for_message in place of a value
54
+ # whose identifier or type matches the redaction configuration. This
55
+ # mirrors, for the human-readable log-probe message path, the
56
+ # notCapturedReason: "redactedIdent"/"redactedType" gating that
57
+ # #serialize_value applies on the snapshot path.
58
+ REDACTED_VALUE_FOR_MESSAGE = "[redacted]"
59
+
53
60
  # Third-party library integration / custom serializers.
54
61
  #
55
62
  # Dynamic instrumentation has limited payload sizes, and for efficiency
@@ -375,9 +382,17 @@ module Datadog
375
382
  #
376
383
  # We also use the Ruby-like syntax for symbols, which don't exist
377
384
  # in other languages.
378
- def serialize_value_for_message(value, depth = 1)
385
+ #
386
+ # +name+, when given, is the identifier the template expression
387
+ # references at its top level; a redacted identifier yields the
388
+ # redaction placeholder, mirroring #serialize_value on the snapshot path.
389
+ def serialize_value_for_message(value, depth: 1, name: nil)
379
390
  # This method is more verbose than "normal" Ruby code to avoid
380
391
  # array allocations.
392
+
393
+ return REDACTED_VALUE_FOR_MESSAGE if redactor.redact_type?(value)
394
+ return REDACTED_VALUE_FOR_MESSAGE if name && redactor.redact_identifier?(name)
395
+
381
396
  case value
382
397
  when NilClass
383
398
  "nil"
@@ -398,7 +413,7 @@ module Datadog
398
413
  value = value_
399
414
  end
400
415
  "[" + value.map do |item|
401
- serialize_value_for_message(item, depth - 1)
416
+ serialize_value_for_message(item, depth: depth - 1)
402
417
  end.join(", ") + "]"
403
418
  when Hash
404
419
  return "..." if depth <= 0
@@ -413,7 +428,12 @@ module Datadog
413
428
  truncated = true
414
429
  end
415
430
  serialized = keys.map do |key|
416
- "#{serialize_value_for_message(key, depth - 1)} => #{serialize_value_for_message(value[key], depth - 1)}"
431
+ serialized_value = if (String === key || Symbol === key) && redactor.redact_identifier?(key)
432
+ REDACTED_VALUE_FOR_MESSAGE
433
+ else
434
+ serialize_value_for_message(value[key], depth: depth - 1)
435
+ end
436
+ "#{serialize_value_for_message(key, depth: depth - 1)} => #{serialized_value}"
417
437
  end
418
438
  if truncated
419
439
  serialized[serialized.length] = serialized[serialized.length - 1]
@@ -435,7 +455,12 @@ module Datadog
435
455
  serialized = vars.map do |var|
436
456
  # +var+ here is always the instance variable name which is a
437
457
  # symbol, we do not need to run it through our serializer.
438
- "#{var}=#{serialize_value_for_message(value.send(:instance_variable_get, var), depth - 1)}"
458
+ serialized_value = if redactor.redact_identifier?(var)
459
+ REDACTED_VALUE_FOR_MESSAGE
460
+ else
461
+ serialize_value_for_message(value.send(:instance_variable_get, var), depth: depth - 1)
462
+ end
463
+ "#{var}=#{serialized_value}"
439
464
  end
440
465
  if truncated
441
466
  serialized << serialized.last
@@ -154,7 +154,7 @@ module Datadog
154
154
  attributes = {
155
155
  "exception.type" => formatted_exception.type,
156
156
  "exception.message" => formatted_exception.message,
157
- "exception.stacktrace" => formatted_exception.backtrace
157
+ "exception.stacktrace" => formatted_exception.backtrace,
158
158
  }
159
159
  Datadog::Tracing::SpanEvent.new("exception", attributes: attributes)
160
160
  end
@@ -19,7 +19,7 @@ module Datadog
19
19
  TELEMETRY_METRICS_EVENTS_INTO_TYPES = {
20
20
  LOGIN_SUCCESS_EVENT => "login_success",
21
21
  LOGIN_FAILURE_EVENT => "login_failure",
22
- SIGNUP_EVENT => "signup"
22
+ SIGNUP_EVENT => "signup",
23
23
  }.freeze
24
24
 
25
25
  class << self
@@ -241,7 +241,7 @@ module Datadog
241
241
 
242
242
  tags = {
243
243
  event_type: TELEMETRY_METRICS_EVENTS_INTO_TYPES[event],
244
- sdk_version: TELEMETRY_METRICS_SDK_VERSION
244
+ sdk_version: TELEMETRY_METRICS_SDK_VERSION,
245
245
  }
246
246
  telemetry.inc(TELEMETRY_METRICS_NAMESPACE, TELEMETRY_METRICS_SDK_EVENT, 1, tags: tags)
247
247
  end