datadog 2.31.0 → 2.33.0

data/lib/datadog/symbol_database/file_hash.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+require 'digest/sha1'
+
+module Datadog
+  module SymbolDatabase
+    # Computes Git-style SHA-1 hashes of Ruby source files for backend commit inference.
+    #
+    # Uses Git's blob hash algorithm: SHA1("blob <size>\0<content>")
+    # Hashes enable the backend to correlate runtime code with Git repository history,
+    # identifying which commit is actually deployed.
+    #
+    # Called by: Extractor (when building MODULE scopes)
+    # Stores result in: Scope's language_specifics[:file_hash]
+    # Returns: 40-character hex string or nil if file unreadable
+    #
+    # @api private
+    module FileHash
+      module_function
+
+      # Compute Git-style SHA-1 hash of a file.
+      # Uses Git's blob hash algorithm: SHA1("blob <size>\0<content>")
+      # Returns nil on any error (file not found, permission denied, etc.)
+      #
+      # @param file_path [String] Path to the file
+      # @param logger [#debug] Logger for error reporting
+      # @return [String, nil] 40-character hex-encoded SHA-1 hash, or nil if error
+      def compute(file_path, logger:)
+        return nil unless file_path
+        return nil unless File.exist?(file_path)
+
+        content = File.read(file_path, mode: 'rb')
+        size = content.bytesize
+        git_blob = "blob #{size}\0#{content}"
+
+        # SHA-1 is required here to match Git's blob hash format for commit inference.
+        # This is not a security vulnerability - we're computing file content hashes
+        # to match against Git objects, not using SHA-1 for authentication/integrity.
+        Digest::SHA1.hexdigest(git_blob)  # nosemgrep: ruby.lang.security.weak-hashes-sha1.weak-hashes-sha1
+      rescue => e
+        logger.debug { "symdb: file hash failed for #{file_path}: #{e.class}: #{e.message}" }
+        nil
+      end
+    end
+  end
+end

data/lib/datadog/symbol_database/logger.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+require 'forwardable'
+
+module Datadog
+  module SymbolDatabase
+    # Logger facade that adds a config-gated +trace+ method.
+    #
+    # Wraps any logger (customer-provided or default) and delegates
+    # standard methods. The +trace+ method is a sub-debug level that
+    # is a no-op unless DD_TRACE_DEBUG is set, avoiding overhead for
+    # high-frequency log sites (per-module filtering, dedup checks).
+    #
+    # @api private
+    class Logger
+      extend Forwardable
+
+      # @param settings [Configuration::Settings] Tracer settings (reads trace_logging flag)
+      # @param target [::Logger] Underlying logger to delegate to
+      def initialize(settings, target)
+        @settings = settings
+        @target = target
+      end
+
+      attr_reader :settings
+
+      # Only debug and warn are delegated by design — symbol database
+      # extraction logs only at debug (high-volume diagnostics) and warn
+      # (user-actionable problems). Adding info/error would invite
+      # log-level drift; explicit additions can be made if needed.
+      def_delegators :@target, :debug, :warn
+
+      # Log at trace level (sub-debug). No-op unless DD_TRACE_DEBUG is set.
+      # @yield Block that returns the log message string
+      # @return [void]
+      def trace(&block)
+        if settings.symbol_database.internal.trace_logging
+          debug(&block)
+        end
+      end
+    end
+  end
+end

data/lib/datadog/symbol_database/scope.rb

@@ -0,0 +1,102 @@
+# frozen_string_literal: true
+
+require 'json'
+
+module Datadog
+  module SymbolDatabase
+    # Represents a scope in the hierarchical symbol structure (FILE → MODULE/CLASS → METHOD).
+    #
+    # Scopes form a tree structure representing Ruby code organization. Each scope contains:
+    # - Metadata: name, source file, line range, scope type (MODULE/CLASS/METHOD/etc.)
+    # - Symbols: Variables, constants, parameters defined in this scope
+    # - Nested scopes: Child scopes (e.g., methods within a class)
+    #
+    # Created by: Extractor (during symbol extraction)
+    # Used by: ScopeBatcher (batching), ServiceVersion (wrapping for upload)
+    # Serialized to: JSON via to_h/to_json for upload to agent
+    #
+    # @api private
+    class Scope
+      attr_reader :scope_type, :name, :source_file, :start_line, :end_line,
+        # Ranges of executable lines [{start:, end:}]. Three states:
+        # - nil: not computed (source unreadable, native/C-extension method)
+        # - []: computed but no executable lines found (comments/whitespace only)
+        # - non-empty: computed, contains executable line ranges
+        # nil and [] both serialize as has_injectible_lines: false on METHOD
+        # scopes. Key is absent on non-METHOD scopes. The wire format key
+        # name keeps the historical spelling +injectible+ for backend
+        # compatibility; the Ruby identifier is +targetable_lines+.
+        :targetable_lines,
+        :language_specifics, :symbols, :scopes
+
+      # Initialize a new Scope
+      # @param scope_type [String] Type of scope (FILE, MODULE, CLASS, METHOD)
+      # @param name [String, nil] Name of the scope (class name, method name, etc.)
+      # @param source_file [String, nil] Path to source file
+      # @param start_line [Integer, nil] Starting line number (UNKNOWN_MIN_LINE for unknown)
+      # @param end_line [Integer, nil] Ending line number (UNKNOWN_MAX_LINE for entire file)
+      # @param targetable_lines [Array<Hash>, nil] Ranges of executable lines [{start:, end:}]
+      # @param language_specifics [Hash, nil] Ruby-specific metadata
+      # @param symbols [Array<Symbol>, nil] Symbols defined in this scope
+      # @param scopes [Array<Scope>, nil] Nested child scopes
+      def initialize(
+        scope_type:,
+        name: nil,
+        source_file: nil,
+        start_line: nil,
+        end_line: nil,
+        targetable_lines: nil,
+        language_specifics: nil,
+        symbols: nil,
+        scopes: nil
+      )
+        @scope_type = scope_type
+        @name = name
+        @source_file = source_file
+        @start_line = start_line
+        @end_line = end_line
+        @targetable_lines = targetable_lines
+        @language_specifics = language_specifics || {}
+        @symbols = symbols || []
+        @scopes = scopes || []
+      end
+
+      # @return [Boolean] true when targetable_lines is non-nil and non-empty
+      def targetable_lines?
+        !targetable_lines.nil? && !targetable_lines.empty?
+      end
+
+      # Convert scope to Hash for JSON serialization.
+      # Removes nil values to reduce payload size.
+      # @return [Hash] Scope as hash with symbol keys
+      def to_h
+        h = {
+          scope_type: scope_type,
+          name: name,
+          source_file: source_file,
+          start_line: start_line,
+          end_line: end_line,
+          language_specifics: language_specifics.empty? ? nil : language_specifics,
+          symbols: symbols.empty? ? nil : symbols.map(&:to_h),
+          scopes: scopes.empty? ? nil : scopes.map(&:to_h),
+        }
+        h.compact!
+        # Targetable lines only on METHOD scopes (per spec — not on CLASS/MODULE/FILE).
+        # Always emit has_injectible_lines (even when false) on METHOD scopes.
+        # Wire format keeps the historical spelling +injectible+; Ruby identifier
+        # is +targetable_lines+.
+        if scope_type == 'METHOD'
+          h[:has_injectible_lines] = targetable_lines? # steep:ignore ArgumentTypeMismatch
+          h[:injectible_lines] = targetable_lines if targetable_lines && !targetable_lines.empty?
+        end
+        h
+      end
+
+      # Serialize scope to JSON.
+      # @return [String] JSON string representation
+      def to_json(_state = nil)
+        JSON.generate(to_h)
+      end
+    end
+  end
+end

data/lib/datadog/symbol_database/scope_batcher.rb

@@ -0,0 +1,280 @@
+# frozen_string_literal: true
+
+require 'set'
+
+module Datadog
+  module SymbolDatabase
+    # Batches extracted scopes and triggers uploads at appropriate times.
+    #
+    # Implements two upload triggers:
+    # 1. Size-based: Immediate upload when 400 scopes collected (MAX_SCOPES)
+    # 2. Time-based: Upload after 1 second of inactivity (debounce timer, not periodic)
+    #
+    # Also provides:
+    # - Deduplication: Tracks uploaded module names to prevent re-uploads
+    # - File limiting: Stops after 10,000 files to prevent runaway extraction
+    # - Thread safety: Mutex-protected state for concurrent access
+    #
+    # Timer implementation: A single long-lived thread waits on a ConditionVariable
+    # with a timeout. Each add_scope signals the CV to reset the deadline. When the
+    # timeout expires without a signal, the timer fires and flushes the batch.
+    # This avoids creating/destroying a thread per add_scope call.
+    #
+    # Flow: Extractor → add_scope → (batch or timer) → Uploader
+    # Created by: Component (during initialization)
+    # Calls: Uploader.upload_scopes when batch full or timer fires
+    #
+    # @api private
+    class ScopeBatcher
+      # Maximum scopes per batch before triggering immediate upload.
+      # This matches the batch size used in Java and Python tracers to ensure
+      # consistent upload behavior across languages.
+      MAX_SCOPES = 400
+      INACTIVITY_TIMEOUT = 1.0  # seconds
+      # Maximum unique files to track before stopping extraction.
+      # This prevents runaway memory usage in applications with very large
+      # numbers of loaded classes (e.g., heavily modularized Rails apps).
+      MAX_FILES = 10_000
+      # Seconds to wait for the timer thread to exit when joining during
+      # shutdown or reset. Bounded so a misbehaving thread cannot hang the
+      # caller indefinitely.
+      TIMER_JOIN_TIMEOUT = 5
+
+      # Initialize batching context.
+      # @param uploader [Uploader] Uploader instance for triggering uploads
+      # @param logger [Logger] Logger for diagnostics
+      # @param on_upload [Proc, nil] Optional callback called after upload (for testing)
+      # @param timer_enabled [Boolean] Enable async timer (default true, false for tests)
+      def initialize(uploader, logger:, on_upload: nil, timer_enabled: true)
+        @uploader = uploader
+        @logger = logger
+        @on_upload = on_upload
+        @timer_enabled = timer_enabled
+        @scopes = []
+        @mutex = Mutex.new
+        @file_count = 0
+        @uploaded_modules = Set.new
+
+        # Timer state: single long-lived thread + ConditionVariable for debounce.
+        # @timer_signaled is set to true on each add_scope and cleared by the timer
+        # thread after waking. This flag is needed because ConditionVariable#wait
+        # does not distinguish signal vs timeout on Ruby < 3.2 (returns self in both
+        # cases). The flag gives a portable way to detect whether the wakeup was a
+        # signal (reset deadline) or a timeout (fire the timer).
+        @timer_cv = ConditionVariable.new
+        @timer_thread = nil
+        @timer_stopped = false
+        @timer_signaled = false
+      end
+
+      # Add a scope to the batch.
+      # Triggers immediate upload if batch reaches 400 scopes.
+      # Resets inactivity timer if batch not full.
+      # @param scope [Scope] The scope to add
+      # @return [void]
+      def add_scope(scope)
+        # @type var scopes_to_upload: ::Array[Scope]?
+        scopes_to_upload = nil
+
+        @mutex.synchronize do
+          # Check file limit (counts only unique accepted files; duplicates are
+          # filtered by the dedup check below and do not consume the budget).
+          if @file_count >= MAX_FILES
+            @logger.debug { "symdb: file limit (#{MAX_FILES}) reached, ignoring scope: #{scope.name}" }
+            return
+          end
+
+          # Check if already uploaded — duplicates do not count toward MAX_FILES
+          # so a re-extraction scenario does not exhaust the budget for unique scopes.
+          if @uploaded_modules.include?(scope.name)
+            @logger.trace { "symdb: skipping #{scope.name}: already uploaded" }
+            return
+          end
+
+          @uploaded_modules.add(scope.name)
+          @file_count += 1
+
+          # Add the scope
+          @scopes << scope
+
+          # Check if batch size reached (AFTER adding)
+          if @scopes.size >= MAX_SCOPES
+            # Prepare for upload (clear within mutex)
+            scopes_to_upload = @scopes.dup
+            @scopes.clear
+          end
+
+          # Signal the timer thread to reset its inactivity deadline.
+          # If batch was full, this is harmless — the timer will just
+          # re-check and find an empty batch if it fires.
+          ensure_timer_running
+          @timer_signaled = true
+          @timer_cv.signal
+        end
+
+        # Upload outside mutex (if batch was full)
+        perform_upload(scopes_to_upload) if scopes_to_upload
+      rescue => e
+        @logger.debug { "symdb: failed to add scope: #{e.class}: #{e.message}" }
+        # Don't propagate, continue operation
+      end
+
+      # Force upload of current batch immediately.
+      # @return [void]
+      def flush
+        # @type var scopes_to_upload: ::Array[Scope]?
+        scopes_to_upload = nil
+
+        @mutex.synchronize do
+          return if @scopes.empty?
+
+          scopes_to_upload = @scopes.dup
+          @scopes.clear
+        end
+
+        perform_upload(scopes_to_upload)
+      end
+
+      # Shutdown and upload remaining scopes.
+      # @return [void]
+      def shutdown
+        # @type var scopes_to_upload: ::Array[Scope]?
+        scopes_to_upload = nil
+        # @type var thread_to_join: ::Thread?
+        thread_to_join = nil
+
+        @mutex.synchronize do
+          @timer_stopped = true
+          @timer_cv.signal  # Wake the timer thread so it exits
+
+          # Capture the timer thread under the mutex so a concurrent add_scope
+          # cannot create a new thread that we'd accidentally orphan when we
+          # nil the field below.
+          thread_to_join = @timer_thread
+          @timer_thread = nil
+
+          scopes_to_upload = @scopes.dup
+          @scopes.clear
+        end
+
+        # Join the timer thread outside the mutex.
+        # The thread checks @timer_stopped and exits when signaled.
+        thread_to_join&.join(TIMER_JOIN_TIMEOUT)
+
+        # Upload outside mutex
+        perform_upload(scopes_to_upload) unless scopes_to_upload.nil? || scopes_to_upload.empty?
+      end
+
+      # Check if scopes are pending upload.
+      # @return [Boolean] true if scopes waiting in batch
+      def scopes_pending?
+        @mutex.synchronize { @scopes.any? }
+      end
+
+      # Get current batch size.
+      # @return [Integer] Number of scopes in current batch
+      def size
+        @mutex.synchronize { @scopes.size }
+      end
+
+      private
+
+      # Reset state. Private so production code cannot accidentally invoke it;
+      # tests call via +send(:reset)+.
+      # @return [void]
+      def reset
+        # @type var thread_to_join: ::Thread?
+        thread_to_join = nil
+
+        @mutex.synchronize do
+          @scopes.clear
+          @timer_stopped = true
+          @timer_cv.signal
+          @file_count = 0
+          @uploaded_modules.clear
+
+          # Capture under the mutex (see shutdown for rationale).
+          thread_to_join = @timer_thread
+          @timer_thread = nil
+        end
+
+        thread_to_join&.join(TIMER_JOIN_TIMEOUT)
+
+        # Allow timer to be restarted after reset
+        @mutex.synchronize do
+          @timer_stopped = false
+          @timer_signaled = false
+        end
+      end
+
+      # Start the timer thread if not already running.
+      # Must be called from within @mutex.synchronize.
+      # @return [void]
+      def ensure_timer_running
+        return unless @timer_enabled
+        return if @timer_thread&.alive?
+
+        @timer_stopped = false
+        @timer_signaled = false
+
+        @timer_thread = Thread.new do
+          timer_loop
+        end
+      end
+
+      # Timer thread main loop. Waits on the ConditionVariable with a timeout.
+      # Each signal resets the deadline (debounce). When the wait times out
+      # (no signal within INACTIVITY_TIMEOUT), the batch is flushed.
+      #
+      # Uses @timer_signaled flag instead of ConditionVariable#wait return value
+      # because Ruby < 3.2 returns self for both signal and timeout (no way to
+      # distinguish). The flag is set by add_scope before signaling, and cleared
+      # by the timer thread after waking.
+      # @return [void]
+      def timer_loop
+        loop do
+          should_flush = false
+
+          @mutex.synchronize do
+            return if @timer_stopped
+
+            @timer_signaled = false
+            @timer_cv.wait(@mutex, INACTIVITY_TIMEOUT)
+
+            return if @timer_stopped
+
+            if @timer_signaled
+              # Woke up because add_scope signaled — loop back to re-wait with
+              # a fresh timeout. This implements the debounce: the timeout resets
+              # on every scope addition.
+              next # steep:ignore BreakTypeMismatch
+            end
+
+            # Timed out (no signal within INACTIVITY_TIMEOUT). If there are
+            # scopes pending, flush them. Otherwise, loop back and wait again.
+            should_flush = !@scopes.empty?
+          end
+
+          if should_flush
+            flush
+          end
+        end
+      rescue => e
+        @logger.debug { "symdb: timer thread error: #{e.class}: #{e.message}" }
+      end
+
+      # Perform upload via uploader.
+      # @param scopes [Array<Scope>] Scopes to upload
+      # @return [void]
+      def perform_upload(scopes)
+        return if scopes.nil? || scopes.empty?
+
+        @uploader.upload_scopes(scopes)
+        @on_upload&.call(scopes)  # Notify tests after upload
+      rescue => e
+        @logger.debug { "symdb: upload failed: #{e.class}: #{e.message}" }
+        # Don't propagate, uploader handles retries
+      end
+    end
+  end
+end

data/lib/datadog/symbol_database/service_version.rb

@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+require 'json'
+
+module Datadog
+  module SymbolDatabase
+    # Top-level container wrapping scopes for upload to the agent.
+    #
+    # ServiceVersion is the root object serialized to JSON for symbol database uploads.
+    # Contains service metadata (name, env, version) and all extracted scopes.
+    # The language field identifies the tracer.
+    #
+    # Created by: Uploader (wraps scopes array before serialization)
+    # Contains: Array of top-level Scope objects (FILE scopes)
+    # Serialized to: JSON via to_json, then GZIP compressed for upload
+    #
+    # @api private
+    class ServiceVersion
+      attr_reader :service, :env, :version, :language, :scopes
+
+      # Initialize a new ServiceVersion
+      # @param service [String] Service name (required, from DD_SERVICE)
+      # @param env [String, nil] Environment (from DD_ENV, passed through unchanged)
+      # @param version [String, nil] Version (from DD_VERSION, passed through unchanged)
+      # @param scopes [Array<Scope>] Top-level scopes (required)
+      # @raise [ArgumentError] if service empty or scopes not an array
+      def initialize(service:, env:, version:, scopes:)
+        raise ArgumentError, 'service is required' if service.nil? || service.empty?
+        raise ArgumentError, 'scopes must be an array' unless scopes.is_a?(Array)
+
+        @service = service
+        @env = env
+        @version = version
+        @language = 'ruby'
+        @scopes = scopes
+      end
+
+      # Convert service version to Hash for JSON serialization.
+      # @return [Hash] ServiceVersion as hash with symbol keys
+      def to_h
+        {
+          service: service,
+          env: env,
+          version: version,
+          language: language,
+          scopes: scopes.map(&:to_h),
+        }
+      end
+
+      # Serialize service version to JSON.
+      # @return [String] JSON string representation
+      def to_json(_state = nil)
+        JSON.generate(to_h)
+      end
+    end
+  end
+end

data/lib/datadog/symbol_database/symbol.rb

@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+require 'json'
+
+module Datadog
+  module SymbolDatabase
+    # Represents a symbol (variable, parameter, field, constant) within a scope.
+    #
+    # Symbols are the actual identifiers extracted from Ruby code:
+    # - Instance variables (@var) - FIELD type
+    # - Class variables (@@var) - STATIC_FIELD type
+    # - Constants (CONST) - STATIC_FIELD type
+    # - Method parameters (arg) - ARG type
+    # - Local variables (var) - LOCAL type (not yet implemented)
+    #
+    # Created by: Extractor (during class/method introspection)
+    # Contained in: Scope objects (symbols array)
+    # Serialized to: JSON via to_h/to_json
+    #
+    # @api private
+    class Symbol
+      attr_reader :symbol_type, :name, :line, :type, :language_specifics
+
+      # Initialize a new Symbol
+      # @param symbol_type [String] Type: FIELD, STATIC_FIELD, ARG, LOCAL
+      # @param name [String] Symbol name (variable name, parameter name)
+      # @param line [Integer] Line number (UNKNOWN_MIN_LINE for entire scope, UNKNOWN_MAX_LINE for method-level only)
+      # @param type [String, nil] Type annotation (optional, Ruby is dynamic)
+      # @param language_specifics [Hash, nil] Symbol-specific metadata
+      def initialize(
+        symbol_type:,
+        name:,
+        line:,
+        type: nil,
+        language_specifics: nil
+      )
+        @symbol_type = symbol_type
+        @name = name
+        @line = line
+        @type = type
+        @language_specifics = language_specifics
+      end
+
+      # Convert symbol to Hash for JSON serialization.
+      # Removes nil values to reduce payload size.
+      # @return [Hash] Symbol as hash with symbol keys
+      def to_h
+        h = {
+          symbol_type: symbol_type,
+          name: name,
+          line: line,
+          type: type,
+          language_specifics: language_specifics,
+        }
+        h.compact!
+        h
+      end
+
+      # Serialize symbol to JSON.
+      # @return [String] JSON string representation
+      def to_json(_state = nil)
+        JSON.generate(to_h)
+      end
+    end
+  end
+end

data/lib/datadog/symbol_database/transport/http/endpoint.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+require_relative '../../../core/transport/http/api/endpoint'
+
+module Datadog
+  module SymbolDatabase
+    module Transport
+      module HTTP
+        module API
+          # POST endpoint for symbol database uploads. Multipart form-data
+          # is triggered by setting `env.form` on the request (handled in
+          # `Core::Transport::HTTP::Adapters::Net`); the multipart library
+          # sets Content-Type itself.
+          class Endpoint < Datadog::Core::Transport::HTTP::API::Endpoint
+            attr_reader :encoder
+
+            # @param path [String] URL path for the endpoint
+            # @param encoder [#encode] Encoder for request data
+            def initialize(path, encoder)
+              super(:post, path)
+              @encoder = encoder
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/symbol_database/transport/http.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+require_relative '../../core/encoding'
+require_relative '../../core/transport/http'
+require_relative 'http/endpoint'
+require_relative '../transport'
+
+module Datadog
+  module SymbolDatabase
+    module Transport
+      # Namespace for HTTP transport components
+      module HTTP
+        # POST endpoint for the agent's symbol database intake.
+        # Multipart form-data is dispatched via `env.form` from the
+        # `Symbols::Client` subclass.
+        SYMBOLS_ENDPOINT = API::Endpoint.new(
+          '/symdb/v1/input',
+          Datadog::Core::Encoding::JSONEncoder,
+        )
+
+        # Builds a transport for the symbols upload endpoint.
+        # @param agent_settings [Core::Configuration::AgentSettingsResolver::AgentSettings]
+        #   Agent connection settings (host, port, timeout, etc.)
+        # @param logger [Logger] Logger instance
+        # @param headers [Hash, nil] Optional additional headers
+        # @return [Symbols::Transport] Transport for the symbols endpoint
+        def self.symbols(
+          agent_settings:,
+          logger:,
+          headers: nil
+        )
+          Core::Transport::HTTP.build(
+            logger: logger,
+            agent_settings: agent_settings,
+            headers: headers,
+          ) do |transport|
+            transport.api 'symbols', SYMBOLS_ENDPOINT, default: true
+
+            yield(transport) if block_given?
+          end.to_transport(SymbolDatabase::Transport::Symbols::Transport)
+        end
+      end
+    end
+  end
+end