datadog 2.32.0 → 2.34.0

data/lib/datadog/symbol_database/remote.rb

@@ -0,0 +1,175 @@
+# frozen_string_literal: true
+
+module Datadog
+  module SymbolDatabase
+    # Provides remote configuration integration for symbol database.
+    #
+    # Responsibilities:
+    # - Registers with Core::Remote as a receiver for LIVE_DEBUGGING_SYMBOL_DB product
+    # - Processes remote config changes (insert/update/delete)
+    # - Calls Component.start_upload when upload_symbols: true
+    # - Calls Component.stop_upload when config deleted or upload_symbols: false
+    #
+    # Flow:
+    # 1. Remote config system calls receiver with repository and changes
+    # 2. For each change, process_change called
+    # 3. parse_config extracts upload_symbols flag
+    # 4. enable_upload or disable_upload called on component
+    #
+    # Created by: Symbol database initialization
+    # Accessed by: Core::Remote system when configurations change
+    # Requires: Component must exist (accessed via Datadog.send(:components).symbol_database)
+    #
+    # @api private
+    module Remote
+      PRODUCT = 'LIVE_DEBUGGING_SYMBOL_DB'
+
+      class << self
+        # Declare products this receiver handles.
+        # @return [Array<String>] Product names
+        def products
+          [PRODUCT]
+        end
+
+        # Declare capabilities for this receiver.
+        # @return [Array] Capabilities (none for symbol database)
+        def capabilities
+          []
+        end
+
+        # Create receivers for remote configuration.
+        # @return [Array<Receiver>] Array of receivers
+        def receivers(_telemetry)
+          receiver do |repository, changes|
+            telemetry = lookup_telemetry
+            component = begin
+              Datadog.send(:components, allow_initialization: false)&.symbol_database
+            rescue => e
+              Datadog.logger.debug { "symdb: failed to look up component in RC receiver: #{e.class}: #{e.message}" }
+              telemetry&.report(e, description: 'symdb: failed to look up component in RC receiver')
+              nil
+            end
+
+            if component
+              changes.each do |change|
+                process_change(component, change, telemetry)
+              end
+            end
+          end
+        end
+
+        # Create a single receiver for the product.
+        # @param products [Array<String>] Product names to match
+        # @return [Array<Receiver>] Receiver array
+        def receiver(products = [PRODUCT], &block)
+          matcher = Core::Remote::Dispatcher::Matcher::Product.new(products)
+          [Core::Remote::Dispatcher::Receiver.new(matcher, &block)]
+        end
+
+        private
+
+        # Look up the telemetry component for error reporting. Returns nil if the
+        # component tree isn't built yet (very early boot) or the lookup raises.
+        # `allow_initialization: false` avoids triggering component-tree construction
+        # from inside an RC receiver callback.
+        # @return [Core::Telemetry::Component, nil]
+        # @api private
+        def lookup_telemetry
+          Datadog.send(:components, allow_initialization: false)&.telemetry
+        rescue
+          nil
+        end
+
+        # Process a single configuration change.
+        # @param component [Component] Symbol database component
+        # @param change [Change] Configuration change (:insert, :update, :delete)
+        # @param telemetry [Core::Telemetry::Component, nil] Telemetry for error reporting
+        # @return [void]
+        # @api private
+        def process_change(component, change, telemetry)
+          case change.type
+          when :insert
+            # @type var change: ::Datadog::Core::Remote::Configuration::Repository::Change::Inserted
+            enable_upload(component, change.content)
+            change.content.applied
+          when :update
+            # @type var change: ::Datadog::Core::Remote::Configuration::Repository::Change::Updated
+            disable_upload(component)
+            enable_upload(component, change.content)
+            change.content.applied
+          when :delete
+            # @type var change: ::Datadog::Core::Remote::Configuration::Repository::Change::Deleted
+            disable_upload(component)
+            change.previous&.applied
+          else
+            component.logger.debug { "symdb: unrecognized change type: #{change.type}" }
+            # Steep cannot narrow `change.content` from a respond_to? check — it sees
+            # the Repository::Change union type where `Deleted` lacks `content`.
+            change.content.errored("Unrecognized change type: #{change.type}") if change.respond_to?(:content) # steep:ignore NoMethod
+          end
+        rescue => e
+          component.logger.debug { "symdb: error processing remote config change: #{e.class}: #{e.message}" }
+          telemetry&.report(e, description: 'symdb: error processing remote config change')
+          # Rescue runs regardless of which branch raised — Steep cannot narrow the
+          # union type from a respond_to? check.
+          content_obj = change.respond_to?(:content) ? change.content : change.previous # steep:ignore NoMethod
+          content_obj&.errored(e.to_s)
+        end
+
+        # Enable upload if config has upload_symbols: true.
+        # @param component [Component] Symbol database component
+        # @param content [Content] Remote config content
+        # @return [void]
+        # @api private
+        def enable_upload(component, content)
+          config = parse_config(content, component.logger)
+
+          unless config
+            return
+          end
+
+          if config['upload_symbols']
+            component.logger.debug { "symdb: upload enabled via remote config" }
+            component.start_upload
+          else
+            component.logger.debug { "symdb: upload disabled in config" }
+          end
+        end
+
+        # Disable upload.
+        # @param component [Component] Symbol database component
+        # @return [void]
+        # @api private
+        def disable_upload(component)
+          component.logger.debug { "symdb: upload disabled via remote config" }
+          component.stop_upload
+        end
+
+        # Parse and validate remote config content.
+        # @param content [Content] Remote config content
+        # @param logger [SymbolDatabase::Logger] Logger for invalid-config diagnostics
+        # @return [Hash, nil] Parsed config or nil if invalid
+        # @api private
+        #
+        # JSON::ParserError is intentionally NOT rescued here — it propagates to
+        # process_change's rescue, which logs and reports to telemetry. Catching
+        # it locally would swallow the error from telemetry observability.
+        def parse_config(content, logger)
+          config = JSON.parse(content.data)
+
+          unless config.is_a?(Hash)
+            logger.debug { "symdb: invalid config format: expected Hash, got #{config.class}" }
+            return nil
+          end
+
+          unless config.key?('upload_symbols')
+            logger.debug { "symdb: missing 'upload_symbols' key in config" }
+            return nil
+          end
+
+          config
+        end
+      end
+    end
+  end
+end

data/lib/datadog/symbol_database/scope.rb

@@ -22,9 +22,11 @@ module Datadog
         # - nil: not computed (source unreadable, native/C-extension method)
         # - []: computed but no executable lines found (comments/whitespace only)
         # - non-empty: computed, contains executable line ranges
-        # nil and [] both serialize as injectible_lines?: false on METHOD
-        # scopes. Key is absent on non-METHOD scopes.
-        :injectible_lines,
+        # nil and [] both serialize as has_injectible_lines: false on METHOD
+        # scopes. Key is absent on non-METHOD scopes. The wire format key
+        # name keeps the historical spelling +injectible+ for backend
+        # compatibility; the Ruby identifier is +targetable_lines+.
+        :targetable_lines,
         :language_specifics, :symbols, :scopes
 
       # Initialize a new Scope
@@ -33,7 +35,7 @@ module Datadog
       # @param source_file [String, nil] Path to source file
       # @param start_line [Integer, nil] Starting line number (UNKNOWN_MIN_LINE for unknown)
       # @param end_line [Integer, nil] Ending line number (UNKNOWN_MAX_LINE for entire file)
-      # @param injectible_lines [Array<Hash>, nil] Ranges of executable lines [{start:, end:}]
+      # @param targetable_lines [Array<Hash>, nil] Ranges of executable lines [{start:, end:}]
       # @param language_specifics [Hash, nil] Ruby-specific metadata
       # @param symbols [Array<Symbol>, nil] Symbols defined in this scope
       # @param scopes [Array<Scope>, nil] Nested child scopes
@@ -43,7 +45,7 @@ module Datadog
         source_file: nil,
         start_line: nil,
         end_line: nil,
-        injectible_lines: nil,
+        targetable_lines: nil,
         language_specifics: nil,
         symbols: nil,
         scopes: nil
@@ -53,15 +55,15 @@ module Datadog
         @source_file = source_file
         @start_line = start_line
         @end_line = end_line
-        @injectible_lines = injectible_lines
+        @targetable_lines = targetable_lines
         @language_specifics = language_specifics || {}
         @symbols = symbols || []
         @scopes = scopes || []
       end
 
-      # @return [Boolean] true when injectible_lines is non-nil and non-empty
-      def injectible_lines?
-        !injectible_lines.nil? && !injectible_lines.empty?
+      # @return [Boolean] true when targetable_lines is non-nil and non-empty
+      def targetable_lines?
+        !targetable_lines.nil? && !targetable_lines.empty?
       end
 
       # Convert scope to Hash for JSON serialization.
@@ -79,11 +81,13 @@ module Datadog
           scopes: scopes.empty? ? nil : scopes.map(&:to_h),
         }
         h.compact!
-        # Injectable lines only on METHOD scopes (per spec — not on CLASS/MODULE/FILE).
+        # Targetable lines only on METHOD scopes (per spec — not on CLASS/MODULE/FILE).
         # Always emit has_injectible_lines (even when false) on METHOD scopes.
+        # Wire format keeps the historical spelling +injectible+; Ruby identifier
+        # is +targetable_lines+.
         if scope_type == 'METHOD'
-          h[:has_injectible_lines] = injectible_lines? # steep:ignore ArgumentTypeMismatch
-          h[:injectible_lines] = injectible_lines if injectible_lines && !injectible_lines.empty?
+          h[:has_injectible_lines] = targetable_lines? # steep:ignore ArgumentTypeMismatch
+          h[:injectible_lines] = targetable_lines if targetable_lines && !targetable_lines.empty?
         end
         h
       end

data/lib/datadog/symbol_database/scope_batcher.rb

@@ -0,0 +1,288 @@
+# frozen_string_literal: true
+
+require 'set'
+
+module Datadog
+  module SymbolDatabase
+    # Batches extracted scopes and triggers uploads at appropriate times.
+    #
+    # Implements two upload triggers:
+    # 1. Size-based: Immediate upload when 400 scopes collected (MAX_SCOPES)
+    # 2. Time-based: Upload after 1 second of inactivity (debounce timer, not periodic)
+    #
+    # Also provides:
+    # - Deduplication: Tracks uploaded module names to prevent re-uploads
+    # - File limiting: Stops after 10,000 files to prevent runaway extraction
+    # - Thread safety: Mutex-protected state for concurrent access
+    #
+    # Timer implementation: A single long-lived thread waits on a ConditionVariable
+    # with a timeout. Each add_scope signals the CV to reset the deadline. When the
+    # timeout expires without a signal, the timer fires and flushes the batch.
+    # This avoids creating/destroying a thread per add_scope call.
+    #
+    # Flow: Extractor → add_scope → (batch or timer) → Uploader
+    # Created by: Component (during initialization)
+    # Calls: Uploader.upload_scopes when batch full or timer fires
+    #
+    # @api private
+    class ScopeBatcher
+      # Maximum scopes per batch before triggering immediate upload.
+      # This matches the batch size used in Java and Python tracers to ensure
+      # consistent upload behavior across languages.
+      MAX_SCOPES = 400
+      INACTIVITY_TIMEOUT = 1.0  # seconds
+      # Maximum unique files to track before stopping extraction.
+      # This prevents runaway memory usage in applications with very large
+      # numbers of loaded classes (e.g., heavily modularized Rails apps).
+      MAX_FILES = 10_000
+      # Seconds to wait for the timer thread to exit when joining during
+      # shutdown or reset. Bounded so a misbehaving thread cannot hang the
+      # caller indefinitely.
+      TIMER_JOIN_TIMEOUT = 5
+
+      # Initialize batching context.
+      # @param uploader [Uploader] Uploader instance for triggering uploads
+      # @param logger [Logger] Logger for diagnostics
+      # @param on_upload [Proc, nil] Optional callback called after upload (for testing)
+      # @param timer_enabled [Boolean] Enable async timer (default true, false for tests)
+      def initialize(uploader, logger:, on_upload: nil, timer_enabled: true)
+        @uploader = uploader
+        @logger = logger
+        @on_upload = on_upload
+        @timer_enabled = timer_enabled
+        @scopes = []
+        @mutex = Mutex.new
+        @file_count = 0
+        @uploaded_modules = Set.new
+
+        # Timer state: single long-lived thread + ConditionVariable for debounce.
+        # @timer_signaled is set to true on each add_scope and cleared by the timer
+        # thread after waking. This flag is needed because ConditionVariable#wait
+        # does not distinguish signal vs timeout on Ruby < 3.2 (returns self in both
+        # cases). The flag gives a portable way to detect whether the wakeup was a
+        # signal (reset deadline) or a timeout (fire the timer).
+        @timer_cv = ConditionVariable.new
+        @timer_thread = nil
+        @timer_stopped = false
+        @timer_signaled = false
+      end
+
+      # Add a scope to the batch.
+      # Triggers immediate upload if batch reaches 400 scopes.
+      # Resets inactivity timer if batch not full.
+      # @param scope [Scope] The scope to add
+      # @return [void]
+      def add_scope(scope)
+        # @type var scopes_to_upload: ::Array[Scope]?
+        scopes_to_upload = nil
+
+        @mutex.synchronize do
+          # Check file limit (counts only unique accepted files; duplicates are
+          # filtered by the dedup check below and do not consume the budget).
+          if @file_count >= MAX_FILES
+            @logger.debug { "symdb: file limit (#{MAX_FILES}) reached, ignoring scope: #{scope.name}" }
+            return
+          end
+
+          # Check if already uploaded — duplicates do not count toward MAX_FILES
+          # so a re-extraction scenario does not exhaust the budget for unique scopes.
+          if @uploaded_modules.include?(scope.name)
+            @logger.trace { "symdb: skipping #{scope.name}: already uploaded" }
+            return
+          end
+
+          # Marked uploaded before perform_upload runs. This is intentional:
+          # symdb extraction is a one-shot operation per process — extract_all
+          # walks ObjectSpace once and never revisits a module within the same
+          # process. The Set deduplicates within a single extraction run, not
+          # across upload attempts. Failed uploads are not retried; symbols
+          # from a failed batch are lost until the next process restart, by
+          # design (matches Python and Go; Java retries via OkHttp, .NET via
+          # exponential backoff — Ruby does neither).
+          @uploaded_modules.add(scope.name)
+          @file_count += 1
+
+          # Add the scope
+          @scopes << scope
+
+          # Check if batch size reached (AFTER adding)
+          if @scopes.size >= MAX_SCOPES
+            # Prepare for upload (clear within mutex)
+            scopes_to_upload = @scopes.dup
+            @scopes.clear
+          end
+
+          # Signal the timer thread to reset its inactivity deadline.
+          # If batch was full, this is harmless — the timer will just
+          # re-check and find an empty batch if it fires.
+          ensure_timer_running
+          @timer_signaled = true
+          @timer_cv.signal
+        end
+
+        # Upload outside mutex (if batch was full)
+        perform_upload(scopes_to_upload) if scopes_to_upload
+      rescue => e
+        @logger.debug { "symdb: failed to add scope: #{e.class}: #{e.message}" }
+        # Don't propagate, continue operation
+      end
+
+      # Force upload of current batch immediately.
+      # @return [void]
+      def flush
+        # @type var scopes_to_upload: ::Array[Scope]?
+        scopes_to_upload = nil
+
+        @mutex.synchronize do
+          return if @scopes.empty?
+
+          scopes_to_upload = @scopes.dup
+          @scopes.clear
+        end
+
+        perform_upload(scopes_to_upload)
+      end
+
+      # Shutdown and upload remaining scopes.
+      # @return [void]
+      def shutdown
+        # @type var scopes_to_upload: ::Array[Scope]?
+        scopes_to_upload = nil
+        # @type var thread_to_join: ::Thread?
+        thread_to_join = nil
+
+        @mutex.synchronize do
+          @timer_stopped = true
+          @timer_cv.signal  # Wake the timer thread so it exits
+
+          # Capture the timer thread under the mutex so a concurrent add_scope
+          # cannot create a new thread that we'd accidentally orphan when we
+          # nil the field below.
+          thread_to_join = @timer_thread
+          @timer_thread = nil
+
+          scopes_to_upload = @scopes.dup
+          @scopes.clear
+        end
+
+        # Join the timer thread outside the mutex.
+        # The thread checks @timer_stopped and exits when signaled.
+        thread_to_join&.join(TIMER_JOIN_TIMEOUT)
+
+        # Upload outside mutex
+        perform_upload(scopes_to_upload) unless scopes_to_upload.nil? || scopes_to_upload.empty?
+      end
+
+      # Check if scopes are pending upload.
+      # @return [Boolean] true if scopes waiting in batch
+      def scopes_pending?
+        @mutex.synchronize { @scopes.any? }
+      end
+
+      # Get current batch size.
+      # @return [Integer] Number of scopes in current batch
+      def size
+        @mutex.synchronize { @scopes.size }
+      end
+
+      private
+
+      # Reset state. Private so production code cannot accidentally invoke it;
+      # tests call via +send(:reset)+.
+      # @return [void]
+      def reset
+        # @type var thread_to_join: ::Thread?
+        thread_to_join = nil
+
+        @mutex.synchronize do
+          @scopes.clear
+          @timer_stopped = true
+          @timer_cv.signal
+          @file_count = 0
+          @uploaded_modules.clear
+
+          # Capture under the mutex (see shutdown for rationale).
+          thread_to_join = @timer_thread
+          @timer_thread = nil
+        end
+
+        thread_to_join&.join(TIMER_JOIN_TIMEOUT)
+
+        # Allow timer to be restarted after reset
+        @mutex.synchronize do
+          @timer_stopped = false
+          @timer_signaled = false
+        end
+      end
+
+      # Start the timer thread if not already running.
+      # Must be called from within @mutex.synchronize.
+      # @return [void]
+      def ensure_timer_running
+        return unless @timer_enabled
+        return if @timer_thread&.alive?
+
+        @timer_stopped = false
+        @timer_signaled = false
+
+        @timer_thread = Thread.new do
+          timer_loop
+        end
+      end
+
+      # Timer thread main loop. Waits on the ConditionVariable with a timeout.
+      # Each signal resets the deadline (debounce). When the wait times out
+      # (no signal within INACTIVITY_TIMEOUT), the batch is flushed.
+      #
+      # Uses @timer_signaled flag instead of ConditionVariable#wait return value
+      # because Ruby < 3.2 returns self for both signal and timeout (no way to
+      # distinguish). The flag is set by add_scope before signaling, and cleared
+      # by the timer thread after waking.
+      # @return [void]
+      def timer_loop
+        loop do
+          should_flush = false
+
+          @mutex.synchronize do
+            return if @timer_stopped
+
+            @timer_signaled = false
+            @timer_cv.wait(@mutex, INACTIVITY_TIMEOUT)
+
+            return if @timer_stopped
+
+            if @timer_signaled
+              # Woke up because add_scope signaled — loop back to re-wait with
+              # a fresh timeout. This implements the debounce: the timeout resets
+              # on every scope addition.
+              next # steep:ignore BreakTypeMismatch
+            end
+
+            # Timed out (no signal within INACTIVITY_TIMEOUT). If there are
+            # scopes pending, flush them. Otherwise, loop back and wait again.
+            should_flush = !@scopes.empty?
+          end
+
+          if should_flush
+            flush
+          end
+        end
+      rescue => e
+        @logger.debug { "symdb: timer thread error: #{e.class}: #{e.message}" }
+      end
+
+      # Perform upload via uploader.
+      # @param scopes [Array<Scope>] Scopes to upload
+      # @return [void]
+      def perform_upload(scopes)
+        return if scopes.nil? || scopes.empty?
+
+        @uploader.upload_scopes(scopes)
+        @on_upload&.call(scopes)  # Notify tests after upload
+      rescue => e
+        @logger.debug { "symdb: upload failed: #{e.class}: #{e.message}" }
+        # Don't propagate, uploader handles retries
+      end
+    end
+  end
+end

data/lib/datadog/symbol_database/service_version.rb

@@ -16,23 +16,29 @@ module Datadog
     #
     # @api private
     class ServiceVersion
-      attr_reader :service, :env, :version, :language, :scopes
+      attr_reader :service, :env, :version, :language, :scopes, :upload_id, :batch_num, :final
 
       # Initialize a new ServiceVersion
       # @param service [String] Service name (required, from DD_SERVICE)
-      # @param env [String] Environment (from DD_ENV, defaults to "none")
-      # @param version [String] Version (from DD_VERSION, defaults to "none")
+      # @param env [String, nil] Environment (from DD_ENV, passed through unchanged)
+      # @param version [String, nil] Version (from DD_VERSION, passed through unchanged)
       # @param scopes [Array<Scope>] Top-level scopes (required)
+      # @param upload_id [String, nil] UUID identifying the logical upload (shared by all batches)
+      # @param batch_num [Integer, nil] 1-indexed batch number within the upload
+      # @param final [Boolean, nil] true if this is the last batch of the upload
       # @raise [ArgumentError] if service empty or scopes not an array
-      def initialize(service:, env:, version:, scopes:)
+      def initialize(service:, env:, version:, scopes:, upload_id: nil, batch_num: nil, final: nil)
         raise ArgumentError, 'service is required' if service.nil? || service.empty?
         raise ArgumentError, 'scopes must be an array' unless scopes.is_a?(Array)
 
         @service = service
-        @env = env.to_s.empty? ? 'none' : env.to_s
-        @version = version.to_s.empty? ? 'none' : version.to_s
+        @env = env
+        @version = version
         @language = 'ruby'
         @scopes = scopes
+        @upload_id = upload_id
+        @batch_num = batch_num
+        @final = final
       end
 
       # Convert service version to Hash for JSON serialization.
@@ -44,6 +50,9 @@ module Datadog
           version: version,
           language: language,
           scopes: scopes.map(&:to_h),
+          upload_id: upload_id,
+          batch_num: batch_num,
+          final: final,
         }
       end
 

data/lib/datadog/symbol_database/symbol.rb

@@ -42,17 +42,20 @@ module Datadog
       end
 
       # Convert symbol to Hash for JSON serialization.
-      # Removes nil values to reduce payload size.
+      # The `type` key is always present per the symdb JSON schema — emit nil
+      # when the type cannot be determined (the common case in Ruby since
+      # parameters and instance variables carry no declared type).
+      # `language_specifics` is omitted when nil to reduce payload size.
       # @return [Hash] Symbol as hash with symbol keys
       def to_h
+        # @type var h: Hash[::Symbol, untyped]
         h = {
           symbol_type: symbol_type,
           name: name,
           line: line,
           type: type,
-          language_specifics: language_specifics,
         }
-        h.compact!
+        h[:language_specifics] = language_specifics if language_specifics
         h
       end