datadog 2.45.0 → 2.47.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (78) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +49 -1
  3. data/ext/datadog_profiling_native_extension/collectors_stack.c +31 -14
  4. data/ext/datadog_profiling_native_extension/private_vm_api_access.c +5 -1
  5. data/ext/datadog_profiling_native_extension/private_vm_api_access.h +5 -0
  6. data/ext/datadog_profiling_native_extension/profiling.c +6 -0
  7. data/ext/libdatadog_api/di.c +30 -5
  8. data/ext/libdatadog_api/extconf.rb +2 -0
  9. data/ext/libdatadog_api/otel_thread_context.c +20 -14
  10. data/ext/libdatadog_api/process_discovery.c +15 -1
  11. data/ext/libdatadog_api/trace_exporter.c +105 -25
  12. data/ext/libdatadog_extconf_helpers.rb +1 -1
  13. data/lib/datadog/appsec/contrib/rails/gateway/watcher.rb +29 -2
  14. data/lib/datadog/appsec/contrib/rails/patches/render_to_body_patch.rb +8 -7
  15. data/lib/datadog/appsec/contrib/sinatra/gateway/watcher.rb +14 -2
  16. data/lib/datadog/appsec/contrib/sinatra/patches/json_patch.rb +8 -7
  17. data/lib/datadog/appsec/security_engine/engine.rb +2 -1
  18. data/lib/datadog/core/configuration/components.rb +40 -18
  19. data/lib/datadog/core/configuration/components_state.rb +8 -1
  20. data/lib/datadog/core/configuration/supported_configurations.rb +7 -2
  21. data/lib/datadog/core/process_discovery.rb +5 -1
  22. data/lib/datadog/core/remote/client/capabilities.rb +34 -29
  23. data/lib/datadog/core/remote/client.rb +44 -3
  24. data/lib/datadog/core/remote/component.rb +28 -12
  25. data/lib/datadog/core/remote/dispatcher.rb +47 -6
  26. data/lib/datadog/core/workers/runtime_metrics.rb +1 -1
  27. data/lib/datadog/di/boot.rb +1 -0
  28. data/lib/datadog/di/capture_expression_evaluator/result.rb +30 -0
  29. data/lib/datadog/di/capture_expression_evaluator.rb +39 -9
  30. data/lib/datadog/di/component.rb +108 -3
  31. data/lib/datadog/di/configuration.rb +53 -2
  32. data/lib/datadog/di/context.rb +7 -2
  33. data/lib/datadog/di/contrib/active_record.rb +8 -2
  34. data/lib/datadog/di/evaluation_failure.rb +28 -0
  35. data/lib/datadog/di/guardrails_telemetry/reason.rb +87 -0
  36. data/lib/datadog/di/guardrails_telemetry.rb +203 -0
  37. data/lib/datadog/di/instrumenter.rb +65 -15
  38. data/lib/datadog/di/logger.rb +30 -8
  39. data/lib/datadog/di/probe_manager.rb +48 -5
  40. data/lib/datadog/di/probe_notification_builder.rb +28 -26
  41. data/lib/datadog/di/probe_notifier_worker.rb +56 -6
  42. data/lib/datadog/di/remote.rb +46 -2
  43. data/lib/datadog/di/serializer.rb +147 -55
  44. data/lib/datadog/di/snapshot_event.rb +38 -0
  45. data/lib/datadog/di/transport/http.rb +13 -3
  46. data/lib/datadog/di/transport/input.rb +32 -7
  47. data/lib/datadog/open_feature/activation.rb +268 -0
  48. data/lib/datadog/open_feature/component.rb +31 -15
  49. data/lib/datadog/open_feature/configuration.rb +6 -0
  50. data/lib/datadog/open_feature/hooks/evaluation_telemetry_opt_out_hook.rb +25 -0
  51. data/lib/datadog/open_feature/provider.rb +304 -8
  52. data/lib/datadog/open_feature.rb +31 -1
  53. data/lib/datadog/profiling/collectors/info.rb +37 -6
  54. data/lib/datadog/profiling/component.rb +22 -20
  55. data/lib/datadog/profiling/exporter.rb +3 -5
  56. data/lib/datadog/tracing/component.rb +21 -2
  57. data/lib/datadog/tracing/configuration/ext.rb +2 -0
  58. data/lib/datadog/tracing/configuration/settings.rb +14 -1
  59. data/lib/datadog/tracing/context_provider.rb +8 -0
  60. data/lib/datadog/tracing/contrib/ethon/easy_patch.rb +10 -5
  61. data/lib/datadog/tracing/contrib/excon/middleware.rb +10 -3
  62. data/lib/datadog/tracing/contrib/faraday/middleware.rb +10 -4
  63. data/lib/datadog/tracing/contrib/http/instrumentation.rb +16 -1
  64. data/lib/datadog/tracing/contrib/httpclient/instrumentation.rb +8 -2
  65. data/lib/datadog/tracing/contrib/httprb/instrumentation.rb +10 -4
  66. data/lib/datadog/tracing/contrib/rest_client/request_patch.rb +10 -3
  67. data/lib/datadog/tracing/contrib/utils/quantization/http.rb +102 -0
  68. data/lib/datadog/tracing/otel_thread_context.rb +3 -5
  69. data/lib/datadog/tracing/remote.rb +50 -19
  70. data/lib/datadog/tracing/transport/http/telemetry.rb +94 -0
  71. data/lib/datadog/tracing/transport/http/traces.rb +2 -0
  72. data/lib/datadog/tracing/transport/native/telemetry.rb +17 -0
  73. data/lib/datadog/tracing/transport/native.rb +16 -1
  74. data/lib/datadog/tracing/transport/telemetry.rb +62 -0
  75. data/lib/datadog/tracing/transport/traces.rb +82 -15
  76. data/lib/datadog/version.rb +1 -1
  77. metadata +19 -10
  78. data/lib/datadog/appsec/utils/hash_coercion.rb +0 -23
@@ -1,15 +1,16 @@
1
1
  # frozen_string_literal: true
2
2
 
3
+ require "date"
3
4
  require_relative "redactor"
5
+ require_relative "guardrails_telemetry"
6
+ require_relative "../core/utils/time"
4
7
 
5
8
  module Datadog
6
9
  module DI
7
- # Serializes captured snapshot to primitive types, which are subsequently
10
+ # Serializes captured snapshots to primitive types, which are subsequently
8
11
  # serialized to JSON and sent to the backend.
9
12
  #
10
- # This class performs actual removal of sensitive values from the
11
- # snapshots. It uses Redactor to determine which values are sensitive
12
- # and need to be removed.
13
+ # Sensitive values are removed from the snapshots.
13
14
  #
14
15
  # Serializer normally ought not to invoke user (application) code,
15
16
  # to guarantee predictable performance. However, objects like ActiveRecord
@@ -19,7 +20,7 @@ module Datadog
19
20
  # thus they won't be captured at all). To accommodate complex objects,
20
21
  # there is an extension mechanism implemented permitting registration
21
22
  # of serializer callbacks for arbitrary types. Applications and libraries
22
- # definining such serializer callbacks should be very careful to
23
+ # defining such serializer callbacks should be very careful to
23
24
  # have predictable performance and avoid exceptions and infinite loops
24
25
  # and other such issues.
25
26
  #
@@ -36,25 +37,21 @@ module Datadog
36
37
  # efficient but there would be additional overhead from passing this
37
38
  # parameter all the time and the API would get more complex.
38
39
  #
39
- # Note: "self" cannot be used as a parameter name in Ruby, therefore
40
- # there should never be a conflict between instance variable
41
- # serialization and method parameters.
42
- #
43
40
  # @api private
44
41
  class Serializer
45
- # Exception classes that should never be caught during serialization.
46
- # These represent fatal conditions (signals, interrupts, system exit)
47
- # that must propagate to the caller. NoMemoryError is deliberately
48
- # excluded (unlike DI::FATAL_EXCEPTION_CLASSES): serialization of large
49
- # or deeply recursive objects can exhaust memory, and the serializer must
50
- # return a safe stub rather than tear the process down.
42
+ # Exception classes that must propagate to the caller during
43
+ # serialization: fatal conditions (signals, interrupts, system exit).
44
+ # NoMemoryError is deliberately excluded (unlike
45
+ # DI::FATAL_EXCEPTION_CLASSES): serialization of large or deeply
46
+ # recursive objects can exhaust memory, and the serializer must return
47
+ # a safe stub rather than terminate the process.
51
48
  SERIALIZABLE_FATAL_EXCEPTION_CLASSES = [SystemExit, SignalException].freeze
52
49
 
53
50
  # Placeholder emitted by #serialize_value_for_message in place of a value
54
- # whose identifier or type matches the redaction configuration. This
55
- # mirrors, for the human-readable log-probe message path, the
56
- # notCapturedReason: "redactedIdent"/"redactedType" gating that
57
- # #serialize_value applies on the snapshot path.
51
+ # whose identifier or type matches the redaction configuration. The
52
+ # message-path counterpart of the +notCapturedReason+
53
+ # "redactedIdent"/"redactedType" markers that #serialize_value produces
54
+ # on the snapshot path.
58
55
  REDACTED_VALUE_FOR_MESSAGE = "[redacted]"
59
56
 
60
57
  # Third-party library integration / custom serializers.
@@ -99,20 +96,63 @@ module Datadog
99
96
  # hex string, or UTF-8 with replacement characters) before returning
100
97
  # it from the custom serializer.
101
98
  @@flat_registry = []
102
- def self.register(condition: nil, &block)
99
+ # Registers a custom serializer. The block serializes each value
100
+ # for which +condition+ returns a truthy value.
101
+ #
102
+ # @param condition [Proc] receives the value being serialized and
103
+ # returns whether this serializer applies to it
104
+ # @param block [Proc] serializer for the values +condition+ accepts
105
+ # @raise [ArgumentError] if +condition+ is not a Proc or +block+ is absent
106
+ # @return [void]
107
+ def self.register(condition:, &block)
108
+ unless condition.is_a?(Proc)
109
+ raise ArgumentError, "Serializer.register condition must be a Proc, got #{condition.class}"
110
+ end
111
+
112
+ unless block
113
+ raise ArgumentError, "Serializer.register requires a serializer block"
114
+ end
115
+
103
116
  @@flat_registry << {condition: condition, proc: block}
117
+ nil
104
118
  end
105
119
 
106
- def initialize(settings, redactor, telemetry: nil)
120
+ # Initializes the serializer.
121
+ #
122
+ # @param settings [Datadog::Core::Configuration::Settings] tracer settings
123
+ # @param redactor [Redactor] redactor gating captured values
124
+ # @param logger [DI::Logger] logger for contained serialization failures
125
+ # @param guardrails_telemetry [GuardrailsTelemetry] emitter for the canonical capture-incomplete metric
126
+ # @param telemetry [Datadog::Core::Telemetry::Component, nil] component serializer errors are reported through
127
+ # @return [void]
128
+ def initialize(settings, redactor, logger:, guardrails_telemetry:, telemetry: nil)
107
129
  @settings = settings
108
130
  @redactor = redactor
131
+ @logger = logger
132
+ @guardrails_telemetry = guardrails_telemetry
109
133
  @telemetry = telemetry
110
134
  end
111
135
 
112
136
  attr_reader :settings
113
137
  attr_reader :redactor
138
+ # The logger.
139
+ # @return [DI::Logger]
140
+ attr_reader :logger
141
+ # The guardrails metric emitter.
142
+ # @return [GuardrailsTelemetry]
143
+ attr_reader :guardrails_telemetry
114
144
  attr_reader :telemetry
115
145
 
146
+ # Emits the canonical capture-incomplete metric for a capture limit
147
+ # enforced while serializing a snapshot, with the snapshot event
148
+ # type.
149
+ #
150
+ # @param reason [String] a GuardrailsTelemetry::Reason constant naming the enforced limit
151
+ # @return [void]
152
+ def record_capture_incomplete(reason)
153
+ guardrails_telemetry.capture_incomplete(reason: reason, event_type: GuardrailsTelemetry::EVENT_TYPE_SNAPSHOT)
154
+ end
155
+
116
156
  def combine_args(args, kwargs, target_self)
117
157
  counter = 0
118
158
  combined = args.each_with_object({}) do |value, c|
@@ -133,31 +173,39 @@ module Datadog
133
173
  # arguments to keyword arguments ("arg1", "arg2", ...) and ensure
134
174
  # the positional arguments are listed first.
135
175
  #
136
- # Instance variables are technically a hash just like kwargs,
137
- # we take them as a separate parameter to avoid a hash merge
138
- # in upstream code.
176
+ # @param args [Array] positional arguments passed to the probed method
177
+ # @param kwargs [Hash{Symbol => Object}] keyword arguments passed to the probed method
178
+ # @param target_self [any] receiver of the probed method invocation
179
+ # @param deadline [Float] absolute capture deadline in thread CPU clock seconds
180
+ # @return [Hash{Symbol => Hash}] serialized arguments keyed by argument name
139
181
  def serialize_args(args, kwargs, target_self,
182
+ deadline:,
140
183
  depth: settings.dynamic_instrumentation.max_capture_depth,
141
184
  attribute_count: settings.dynamic_instrumentation.max_capture_attribute_count,
142
185
  length: nil,
143
186
  collection_size: nil)
144
187
  combined = combine_args(args, kwargs, target_self)
145
188
  serialize_vars(combined, depth: depth, attribute_count: attribute_count,
146
- length: length, collection_size: collection_size)
189
+ length: length, collection_size: collection_size, deadline: deadline,)
147
190
  end
148
191
 
149
192
  # Serializes variables captured by a line probe.
150
193
  #
151
194
  # These are normally local variables that exist on a particular line
152
195
  # of executed code.
196
+ #
197
+ # @param vars [Hash{Symbol => Object}] variables to serialize, keyed by variable name
198
+ # @param deadline [Float] absolute capture deadline in thread CPU clock seconds
199
+ # @return [Hash{Symbol => Hash}] serialized variables keyed by variable name
153
200
  def serialize_vars(vars,
201
+ deadline:,
154
202
  depth: settings.dynamic_instrumentation.max_capture_depth,
155
203
  attribute_count: settings.dynamic_instrumentation.max_capture_attribute_count,
156
204
  length: nil,
157
205
  collection_size: nil)
158
206
  vars.each_with_object({}) do |(k, v), agg|
159
207
  agg[k] = serialize_value(v, name: k, depth: depth, attribute_count: attribute_count,
160
- length: length, collection_size: collection_size)
208
+ length: length, collection_size: collection_size, deadline: deadline,)
161
209
  end
162
210
  end
163
211
 
@@ -173,7 +221,13 @@ module Datadog
173
221
  # (integers, strings, arrays, hashes).
174
222
  #
175
223
  # Respects string length, collection size and traversal depth limits.
176
- def serialize_value(value, name: nil,
224
+ #
225
+ # @param value [any] value to serialize
226
+ # @param name [Symbol, String, nil] name the value is bound to, used for redaction
227
+ # @param deadline [Float] absolute capture deadline in thread CPU clock seconds
228
+ # @return [Hash{Symbol => Object}] serialized value; the {type:, notCapturedReason: "timeout"} stub when the deadline is exceeded
229
+ def serialize_value(value, deadline:,
230
+ name: nil,
177
231
  depth: settings.dynamic_instrumentation.max_capture_depth,
178
232
  attribute_count: nil,
179
233
  length: nil,
@@ -182,6 +236,12 @@ module Datadog
182
236
  attribute_count ||= settings.dynamic_instrumentation.max_capture_attribute_count
183
237
  cls = type || value.class
184
238
  begin
239
+ if deadline_exceeded?(deadline)
240
+ telemetry&.inc(DI::TELEMETRY_NAMESPACE, "serialized_values_skipped_by_timeout", 1)
241
+ record_capture_incomplete(GuardrailsTelemetry::Reason::CAPTURE_TIMEOUT)
242
+ return {type: class_name(cls), notCapturedReason: "timeout"}
243
+ end
244
+
185
245
  if redactor.redact_type?(value)
186
246
  return {type: class_name(cls), notCapturedReason: "redactedType"}
187
247
  end
@@ -213,7 +273,7 @@ module Datadog
213
273
 
214
274
  if condition_result
215
275
  serializer_proc = entry.fetch(:proc)
216
- return serializer_proc.call(self, value, name: nil, depth: depth)
276
+ return serializer_proc.call(self, value, name: nil, depth: depth, deadline: deadline)
217
277
  end
218
278
  end
219
279
  end
@@ -227,8 +287,6 @@ module Datadog
227
287
  when Integer, Float, TrueClass, FalseClass
228
288
  serialized.update(value: value.to_s)
229
289
  when Time
230
- # This path also handles DateTime values although they do not need
231
- # to be explicitly added to the case statement.
232
290
  serialized.update(value: value.iso8601)
233
291
  when Date
234
292
  serialized.update(value: value.to_s)
@@ -246,27 +304,24 @@ module Datadog
246
304
  end
247
305
 
248
306
  # Handle binary strings and invalid UTF-8 by escaping to JSON-safe format.
249
- # See escape_binary_string for details on the escaping format.
250
- #
251
- # Truncate binary data BEFORE escaping to avoid cutting mid-escape-sequence.
252
307
  # For regular strings, the limit is applied to string length in characters.
253
308
  length ||= settings.dynamic_instrumentation.max_capture_string_length
254
309
  max = length
255
310
 
256
311
  if value.encoding == Encoding::BINARY || !value.valid_encoding?
257
- # Truncate binary data BEFORE escaping to avoid cutting mid-escape-sequence
312
+ # Truncate binary data before escaping to avoid cutting mid-escape-sequence
258
313
  # For invalid encodings, use bytesize instead of length to avoid encoding errors
259
314
  original_size = value.bytesize
260
315
  if original_size > max
261
316
  serialized.update(truncated: true, size: original_size)
317
+ record_capture_incomplete(GuardrailsTelemetry::Reason::STRING_LENGTH)
262
318
  value = value.byteslice(0...max)
263
319
  end
264
320
  value = escape_binary_string(value) # steep:ignore ArgumentTypeMismatch
265
- false # Already converted to a new string
266
321
  else
267
- # Truncate non-binary strings
268
322
  if value.length > max
269
323
  serialized.update(truncated: true, size: value.length)
324
+ record_capture_incomplete(GuardrailsTelemetry::Reason::STRING_LENGTH)
270
325
  value = value[0...max]
271
326
  need_dup = false
272
327
  end
@@ -278,23 +333,32 @@ module Datadog
278
333
  when Array
279
334
  if depth <= 0
280
335
  serialized.update(notCapturedReason: "depth")
336
+ record_capture_incomplete(GuardrailsTelemetry::Reason::DEPTH)
281
337
  else
282
338
  collection_size ||= settings.dynamic_instrumentation.max_capture_collection_size
283
339
  max = collection_size
284
340
  if max != 0 && value.length > max
285
341
  serialized.update(notCapturedReason: "collectionSize", size: value.length)
342
+ record_capture_incomplete(GuardrailsTelemetry::Reason::COLLECTION_SIZE)
286
343
  # same steep failure with array slices.
287
344
  # https://github.com/soutaro/steep/issues/1219
288
345
  value = value[0...max] || []
289
346
  end
290
- entries = value.map do |elt|
291
- serialize_value(elt, depth: depth - 1, length: length, collection_size: collection_size, attribute_count: attribute_count)
347
+ entries = []
348
+ value.each do |elt|
349
+ entry = serialize_value(elt, depth: depth - 1, length: length, collection_size: collection_size, attribute_count: attribute_count, deadline: deadline)
350
+ entries << entry
351
+ if entry[:notCapturedReason] == "timeout"
352
+ serialized.update(notCapturedReason: "timeout")
353
+ break
354
+ end
292
355
  end
293
356
  serialized.update(elements: entries)
294
357
  end
295
358
  when Hash
296
359
  if depth <= 0
297
360
  serialized.update(notCapturedReason: "depth")
361
+ record_capture_incomplete(GuardrailsTelemetry::Reason::DEPTH)
298
362
  else
299
363
  collection_size ||= settings.dynamic_instrumentation.max_capture_collection_size
300
364
  max = collection_size
@@ -303,17 +367,28 @@ module Datadog
303
367
  value.each do |k, v|
304
368
  if max != 0 && cur >= max
305
369
  serialized.update(notCapturedReason: "collectionSize", size: value.length)
370
+ record_capture_incomplete(GuardrailsTelemetry::Reason::COLLECTION_SIZE)
306
371
  break
307
372
  end
308
373
  cur += 1
309
- entries << [serialize_value(k, depth: depth - 1, length: length, collection_size: collection_size, attribute_count: attribute_count),
310
- serialize_value(v, name: k, depth: depth - 1, length: length, collection_size: collection_size, attribute_count: attribute_count)]
374
+ key_entry = serialize_value(k, depth: depth - 1, length: length, collection_size: collection_size, attribute_count: attribute_count, deadline: deadline)
375
+ if key_entry[:notCapturedReason] == "timeout"
376
+ serialized.update(notCapturedReason: "timeout")
377
+ break
378
+ end
379
+ value_entry = serialize_value(v, name: k, depth: depth - 1, length: length, collection_size: collection_size, attribute_count: attribute_count, deadline: deadline)
380
+ entries << [key_entry, value_entry]
381
+ if value_entry[:notCapturedReason] == "timeout"
382
+ serialized.update(notCapturedReason: "timeout")
383
+ break
384
+ end
311
385
  end
312
386
  serialized.update(entries: entries)
313
387
  end
314
388
  else
315
389
  if depth <= 0
316
390
  serialized.update(notCapturedReason: "depth")
391
+ record_capture_incomplete(GuardrailsTelemetry::Reason::DEPTH)
317
392
  else
318
393
  fields = {}
319
394
  cur = 0
@@ -334,40 +409,42 @@ module Datadog
334
409
  #
335
410
  # For consistency, we should have some kind of stable order of
336
411
  # instance variables on all supported Ruby runtimes, so that the UI
337
- # stays consistent. Given that initial implementation of Ruby DI
338
- # does not support JRuby, we don't handle JRuby's lack of ordering
339
- # of #instance_variables here, but if JRuby is supported in the
340
- # future this may need to be addressed.
412
+ # stays consistent. Ruby DI does not support JRuby, so JRuby's lack
413
+ # of ordering of #instance_variables is not handled here.
341
414
  ivars = value.instance_variables
342
415
 
343
416
  ivars.each do |ivar|
344
417
  if cur >= attribute_count
345
418
  serialized.update(notCapturedReason: "fieldCount", fields: fields)
419
+ record_capture_incomplete(GuardrailsTelemetry::Reason::FIELD_COUNT)
346
420
  break
347
421
  end
348
422
  cur += 1
349
- fields[ivar] = serialize_value(value.instance_variable_get(ivar), name: ivar, depth: depth - 1, length: length, collection_size: collection_size, attribute_count: attribute_count)
423
+ entry = serialize_value(value.instance_variable_get(ivar), name: ivar, depth: depth - 1, length: length, collection_size: collection_size, attribute_count: attribute_count, deadline: deadline)
424
+ fields[ivar] = entry
425
+ if entry[:notCapturedReason] == "timeout"
426
+ serialized.update(notCapturedReason: "timeout")
427
+ break
428
+ end
350
429
  end
351
430
  serialized.update(fields: fields)
352
431
  end
353
432
  end
354
433
  serialized
355
434
  rescue Exception => exc # standard:disable Lint/RescueException
356
- # Re-raise fatal exceptions that should not be caught
357
- # (signals, interrupts, system exit)
358
435
  raise if SERIALIZABLE_FATAL_EXCEPTION_CLASSES.any? { |klass| exc.is_a?(klass) }
359
436
 
360
- # Catch all other exceptions including SystemStackError and NoMemoryError.
361
- # These inherit from Exception (not StandardError) but can occur during
362
- # serialization (e.g., infinite recursion in custom serializers, memory
363
- # exhaustion from large objects) and should return a safe structure
364
- # rather than propagating to the transport layer.
437
+ # Catch all remaining exceptions: SystemStackError and NoMemoryError can
438
+ # occur during serialization (e.g. a custom serializer recursing
439
+ # infinitely) and produce the safe stub.
440
+ record_capture_incomplete(GuardrailsTelemetry::Reason::RUNTIME_ERROR)
441
+ logger.debug { "di: error serializing value of #{class_name(cls)}: #{exc.class}: #{exc.message}" }
365
442
  telemetry&.report(exc, description: "Error serializing")
366
443
  {type: class_name(cls), notSerializedReason: exc.to_s}
367
444
  end
368
445
  end
369
446
 
370
- # This method is used for serializing arbitrary values into log messages.
447
+ # Serializes a value for interpolation into a log-probe message.
371
448
  # Because the output is meant to be human-readable, we cannot use
372
449
  # the "normal" serialization format which is meant to be machine-readable.
373
450
  # Serialize objects with depth of 1 and include the class name.
@@ -386,6 +463,8 @@ module Datadog
386
463
  # +name+, when given, is the identifier the template expression
387
464
  # references at its top level; a redacted identifier yields the
388
465
  # redaction placeholder, mirroring #serialize_value on the snapshot path.
466
+ # Message rendering is capped by depth (1) and
467
+ # MAX_MESSAGE_COLLECTION_SIZE / MAX_MESSAGE_ATTRIBUTE_COUNT.
389
468
  def serialize_value_for_message(value, depth: 1, name: nil)
390
469
  # This method is more verbose than "normal" Ruby code to avoid
391
470
  # array allocations.
@@ -475,13 +554,26 @@ module Datadog
475
554
  raise if SERIALIZABLE_FATAL_EXCEPTION_CLASSES.any? { |klass| exc.is_a?(klass) }
476
555
 
477
556
  telemetry&.report(exc, description: "Error serializing for message")
478
- # TODO class_name(foo) can also fail, which we don't handle here.
479
- # Telemetry reporting could potentially also fail?
557
+ # TODO handle class_name raising here as well.
480
558
  "#<#{class_name(value.class)}: serialization error>"
481
559
  end
482
560
 
561
+ # Computes the absolute thread CPU time deadline (in seconds) for a
562
+ # capture point from max_time_to_serialize_ms.
563
+ #
564
+ # @return [Float] absolute thread CPU clock reading, in seconds, at
565
+ # which the capture budget is exhausted
566
+ def serialization_deadline
567
+ Process.clock_gettime(Process::CLOCK_THREAD_CPUTIME_ID, :float_second) +
568
+ settings.dynamic_instrumentation.max_time_to_serialize_ms / 1000.0
569
+ end
570
+
483
571
  private
484
572
 
573
+ def deadline_exceeded?(deadline)
574
+ Process.clock_gettime(Process::CLOCK_THREAD_CPUTIME_ID, :float_second) >= deadline
575
+ end
576
+
485
577
  MAX_MESSAGE_COLLECTION_SIZE = 3
486
578
  MAX_MESSAGE_ATTRIBUTE_COUNT = 5
487
579
 
@@ -0,0 +1,38 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Datadog
4
+ module DI
5
+ # A snapshot payload queued for sending to the agent, paired with
6
+ # the guardrails event type of the probe that produced it, so drop
7
+ # metrics can be tagged with the probe's classification.
8
+ # @api private
9
+ class SnapshotEvent
10
+ # Initializes the event.
11
+ #
12
+ # @param payload [Hash] the snapshot payload to send to the agent
13
+ # @param event_type [String] one of GuardrailsTelemetry::EVENT_TYPE_SNAPSHOT and EVENT_TYPE_LOG
14
+ # @return [void]
15
+ def initialize(payload:, event_type:)
16
+ @payload = payload
17
+ @event_type = event_type
18
+ end
19
+
20
+ # The snapshot payload to send to the agent.
21
+ # @return [Hash]
22
+ attr_reader :payload
23
+
24
+ # The guardrails event type of the probe that produced the payload.
25
+ # @return [String] one of GuardrailsTelemetry::EVENT_TYPE_SNAPSHOT and EVENT_TYPE_LOG
26
+ attr_reader :event_type
27
+
28
+ # Compares two events by payload and event type.
29
+ #
30
+ # @param other [Object] the object to compare with
31
+ # @return [Boolean] whether the other object is an event with the
32
+ # same payload and event type
33
+ def ==(other)
34
+ other.is_a?(SnapshotEvent) && other.payload == payload && other.event_type == event_type
35
+ end
36
+ end
37
+ end
38
+ end
@@ -47,11 +47,20 @@ module Datadog
47
47
  end.to_transport(DI::Transport::Diagnostics::Transport)
48
48
  end
49
49
 
50
- # Builds a new Transport::HTTP::Client with default settings
51
- # Pass a block to override any settings.
50
+ # Builds the snapshot input transport. Pass a block to override
51
+ # any transport settings.
52
+ #
53
+ # @param agent_settings [Datadog::Core::Configuration::AgentSettings] agent connection settings
54
+ # @param logger [DI::Logger] logger for transport diagnostics
55
+ # @param headers [Hash{String => String}, nil] headers to send with every request
56
+ # @param telemetry [Datadog::Core::Telemetry::Component, nil] component transport errors are reported through
57
+ # @param guardrails_telemetry [GuardrailsTelemetry] emitter for the canonical guardrails drop metric
58
+ # @return [DI::Transport::Input::Transport]
59
+ # @yield [builder] block overriding the default transport builder settings
52
60
  def self.input(
53
61
  agent_settings:,
54
62
  logger:,
63
+ guardrails_telemetry:,
55
64
  headers: nil,
56
65
  telemetry: nil
57
66
  )
@@ -72,7 +81,8 @@ module Datadog
72
81
  builder.to_api_instances,
73
82
  builder.default_api,
74
83
  logger: logger,
75
- telemetry: telemetry
84
+ telemetry: telemetry,
85
+ guardrails_telemetry: guardrails_telemetry,
76
86
  )
77
87
  end
78
88
  end
@@ -8,6 +8,7 @@ require_relative "../../core/transport/request"
8
8
  require_relative "../../core/transport/transport"
9
9
  require_relative "../error"
10
10
  require_relative "../fatal_exceptions"
11
+ require_relative "../guardrails_telemetry"
11
12
  require_relative "http/input"
12
13
 
13
14
  module Datadog
@@ -26,10 +27,22 @@ module Datadog
26
27
 
27
28
  class Transport < Core::Transport::Transport
28
29
  attr_reader :telemetry
30
+ # The guardrails metric emitter.
31
+ # @return [GuardrailsTelemetry]
32
+ attr_reader :guardrails_telemetry
29
33
 
30
- def initialize(apis, default_api, logger:, telemetry: nil)
34
+ # Initializes the transport.
35
+ #
36
+ # @param apis [Datadog::Core::Transport::HTTP::API::Map] API instances by name
37
+ # @param default_api [String] name of the API requests are sent through when unspecified
38
+ # @param logger [Datadog::Core::Logger] logger for transport diagnostics
39
+ # @param guardrails_telemetry [GuardrailsTelemetry] emitter for the canonical payload-drop metric
40
+ # @param telemetry [Datadog::Core::Telemetry::Component, nil] component transport errors are reported through
41
+ # @return [void]
42
+ def initialize(apis, default_api, logger:, guardrails_telemetry:, telemetry: nil)
31
43
  super(apis, default_api, logger: logger)
32
44
  @telemetry = telemetry
45
+ @guardrails_telemetry = guardrails_telemetry
33
46
  end
34
47
 
35
48
  # The limit on an individual snapshot payload, aka "log line",
@@ -62,19 +75,24 @@ module Datadog
62
75
  # Large snapshots (> 1MB) are dropped. Batches are split into chunks
63
76
  # of ~2MB each to avoid large network requests.
64
77
  #
65
- # @param payload [Array<Hash>] Array of snapshot payloads
78
+ # @param payload [Array<SnapshotEvent>] the snapshot events to send, each paired with its probe's guardrails event type
66
79
  # @param tags [Hash] Tags to send with the snapshots
67
80
  # @param on_serialization_error [Proc] Called with (probe_id, exception)
68
81
  # when a snapshot fails to serialize.
82
+ # @return [Array<SnapshotEvent>] the events that were sent
69
83
  def send_input(payload, tags, on_serialization_error:)
70
84
  serialized_tags = Core::TagBuilder.serialize_tags(tags)
71
85
 
72
86
  # Serialize each snapshot individually to isolate failures
73
87
  encoded_snapshots = []
74
- payload.each do |snapshot|
75
- encoded = encoder.encode(snapshot)
76
- if encoded.length > MAX_SERIALIZED_SNAPSHOT_SIZE
77
- logger.debug { "di: dropping too big snapshot" }
88
+ dropped_events = []
89
+ payload.each do |event|
90
+ encoded = encoder.encode(event.payload)
91
+ if encoded.bytesize > MAX_SERIALIZED_SNAPSHOT_SIZE
92
+ logger.debug do
93
+ "di: dropping too big snapshot (#{GuardrailsTelemetry::Reason::PAYLOAD_TOO_LARGE})"
94
+ end
95
+ dropped_events << [event, encoded.bytesize]
78
96
  next
79
97
  end
80
98
  encoded_snapshots << encoded
@@ -82,7 +100,7 @@ module Datadog
82
100
  Datadog::DI.reraise_if_fatal(exc)
83
101
  # Serialization failed for this snapshot - report via callback
84
102
  # This catches JSON::GeneratorError, Encoding errors, TypeError, etc.
85
- probe_id = snapshot.dig(:debugger, :snapshot, :probe, :id)
103
+ probe_id = event.payload.dig(:debugger, :snapshot, :probe, :id)
86
104
  logger.debug { "di: JSON encoding failed for snapshot (probe #{probe_id}): #{exc.class}: #{exc.message}" }
87
105
  telemetry&.report(exc, description: "JSON encoding failed for snapshot")
88
106
 
@@ -97,6 +115,13 @@ module Datadog
97
115
  end
98
116
  end
99
117
 
118
+ dropped_events.each do |event, bytes|
119
+ guardrails_telemetry.dropped(
120
+ reason: GuardrailsTelemetry::Reason::PAYLOAD_TOO_LARGE,
121
+ event_type: event.event_type, bytes: bytes,
122
+ )
123
+ end
124
+
100
125
  return payload if encoded_snapshots.empty?
101
126
 
102
127
  Datadog::Core::Chunker.chunk_by_size(