datadog 2.17.0 → 2.37.0

data/lib/datadog/symbol_database/service_version.rb

@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+require 'json'
+
+module Datadog
+  module SymbolDatabase
+    # Top-level container wrapping scopes for upload to the agent.
+    #
+    # ServiceVersion is the root object serialized to JSON for symbol database uploads.
+    # Contains service metadata (name, env, version) and all extracted scopes.
+    # The language field identifies the tracer.
+    #
+    # Created by: Uploader (wraps scopes array before serialization)
+    # Contains: Array of top-level Scope objects (FILE scopes)
+    # Serialized to: JSON via to_json, then GZIP compressed for upload
+    #
+    # @api private
+    class ServiceVersion
+      attr_reader :service, :env, :version, :language, :scopes, :upload_id, :batch_num, :final
+
+      # Initialize a new ServiceVersion
+      # @param service [String] Service name (required, from DD_SERVICE)
+      # @param env [String, nil] Environment (from DD_ENV, passed through unchanged)
+      # @param version [String, nil] Version (from DD_VERSION, passed through unchanged)
+      # @param scopes [Array<Scope>] Top-level scopes (required)
+      # @param upload_id [String, nil] UUID identifying the logical upload (shared by all batches)
+      # @param batch_num [Integer, nil] 1-indexed batch number within the upload
+      # @param final [Boolean, nil] true if this is the last batch of the upload
+      # @raise [ArgumentError] if service empty or scopes not an array
+      def initialize(service:, env:, version:, scopes:, upload_id: nil, batch_num: nil, final: nil)
+        raise ArgumentError, 'service is required' if service.nil? || service.empty?
+        raise ArgumentError, 'scopes must be an array' unless scopes.is_a?(Array)
+
+        @service = service
+        @env = env
+        @version = version
+        @language = 'ruby'
+        @scopes = scopes
+        @upload_id = upload_id
+        @batch_num = batch_num
+        @final = final
+      end
+
+      # Convert service version to Hash for JSON serialization.
+      # @return [Hash] ServiceVersion as hash with symbol keys
+      def to_h
+        {
+          service: service,
+          env: env,
+          version: version,
+          language: language,
+          scopes: scopes.map(&:to_h),
+          upload_id: upload_id,
+          batch_num: batch_num,
+          final: final,
+        }
+      end
+
+      # Serialize service version to JSON.
+      # @return [String] JSON string representation
+      def to_json(_state = nil)
+        JSON.generate(to_h)
+      end
+    end
+  end
+end

data/lib/datadog/symbol_database/symbol.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+require 'json'
+
+module Datadog
+  module SymbolDatabase
+    # Represents a symbol (variable, parameter, field, constant) within a scope.
+    #
+    # Symbols are the actual identifiers extracted from Ruby code:
+    # - Instance variables (@var) - FIELD type
+    # - Class variables (@@var) - STATIC_FIELD type
+    # - Constants (CONST) - STATIC_FIELD type
+    # - Method parameters (arg) - ARG type
+    # - Local variables (var) - LOCAL type (not yet implemented)
+    #
+    # Created by: Extractor (during class/method introspection)
+    # Contained in: Scope objects (symbols array)
+    # Serialized to: JSON via to_h/to_json
+    #
+    # @api private
+    class Symbol
+      attr_reader :symbol_type, :name, :line, :type, :language_specifics
+
+      # Initialize a new Symbol
+      # @param symbol_type [String] Type: FIELD, STATIC_FIELD, ARG, LOCAL
+      # @param name [String] Symbol name (variable name, parameter name)
+      # @param line [Integer] Line number (UNKNOWN_MIN_LINE for entire scope, UNKNOWN_MAX_LINE for method-level only)
+      # @param type [String, nil] Type annotation (optional, Ruby is dynamic)
+      # @param language_specifics [Hash, nil] Symbol-specific metadata
+      def initialize(
+        symbol_type:,
+        name:,
+        line:,
+        type: nil,
+        language_specifics: nil
+      )
+        @symbol_type = symbol_type
+        @name = name
+        @line = line
+        @type = type
+        @language_specifics = language_specifics
+      end
+
+      # Convert symbol to Hash for JSON serialization.
+      # The `type` key is always present per the symdb JSON schema — emit nil
+      # when the type cannot be determined (the common case in Ruby since
+      # parameters and instance variables carry no declared type).
+      # `language_specifics` is omitted when nil to reduce payload size.
+      # @return [Hash] Symbol as hash with symbol keys
+      def to_h
+        # @type var h: Hash[::Symbol, untyped]
+        h = {
+          symbol_type: symbol_type,
+          name: name,
+          line: line,
+          type: type,
+        }
+        h[:language_specifics] = language_specifics if language_specifics
+        h
+      end
+
+      # Serialize symbol to JSON.
+      # @return [String] JSON string representation
+      def to_json(_state = nil)
+        JSON.generate(to_h)
+      end
+    end
+  end
+end

data/lib/datadog/symbol_database/transport/http/endpoint.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+require_relative '../../../core/transport/http/api/endpoint'
+
+module Datadog
+  module SymbolDatabase
+    module Transport
+      module HTTP
+        module API
+          # POST endpoint for symbol database uploads. Multipart form-data
+          # is triggered by setting `env.form` on the request (handled in
+          # `Core::Transport::HTTP::Adapters::Net`); the multipart library
+          # sets Content-Type itself.
+          class Endpoint < Datadog::Core::Transport::HTTP::API::Endpoint
+            attr_reader :encoder
+
+            # @param path [String] URL path for the endpoint
+            # @param encoder [#encode] Encoder for request data
+            def initialize(path, encoder)
+              super(:post, path)
+              @encoder = encoder
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/symbol_database/transport/http.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+require_relative '../../core/encoding'
+require_relative '../../core/transport/http'
+require_relative 'http/endpoint'
+require_relative '../transport'
+
+module Datadog
+  module SymbolDatabase
+    module Transport
+      # Namespace for HTTP transport components
+      module HTTP
+        # POST endpoint for the agent's symbol database intake.
+        # Multipart form-data is dispatched via `env.form` from the
+        # `Symbols::Client` subclass.
+        SYMBOLS_ENDPOINT = API::Endpoint.new(
+          '/symdb/v1/input',
+          Datadog::Core::Encoding::JSONEncoder,
+        )
+
+        # Builds a transport for the symbols upload endpoint.
+        # @param agent_settings [Core::Configuration::AgentSettingsResolver::AgentSettings]
+        #   Agent connection settings (host, port, timeout, etc.)
+        # @param logger [Logger] Logger instance
+        # @param headers [Hash, nil] Optional additional headers
+        # @return [Symbols::Transport] Transport for the symbols endpoint
+        def self.symbols(
+          agent_settings:,
+          logger:,
+          headers: nil
+        )
+          Core::Transport::HTTP.build(
+            logger: logger,
+            agent_settings: agent_settings,
+            headers: headers,
+          ) do |transport|
+            transport.api 'symbols', SYMBOLS_ENDPOINT, default: true
+
+            yield(transport) if block_given?
+          end.to_transport(SymbolDatabase::Transport::Symbols::Transport)
+        end
+      end
+    end
+  end
+end

data/lib/datadog/symbol_database/transport.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+require_relative '../core/transport/request'
+require_relative '../core/transport/transport'
+require_relative '../core/transport/http/client'
+require_relative '../core/transport/http/env'
+
+module Datadog
+  module SymbolDatabase
+    module Transport
+      # Transport classes for the symbols upload endpoint.
+      # Mirrors the per-purpose split used by DI (`DI::Transport::Input::*`).
+      module Symbols
+        # Request wrapper carrying the multipart form for a symbols upload.
+        class Request < Core::Transport::Request
+          attr_reader :form
+
+          # @param form [Hash] Multipart form data with UploadIO objects
+          def initialize(form)
+            @form = form
+            # Multipart upload — no parcel; the Net adapter reads form data
+            # off the env directly.
+            super(nil)
+          end
+        end
+
+        # HTTP client for symbol database uploads.
+        # Extends Core::Transport::HTTP::Client to set `env.form`, which the
+        # Net adapter (lib/datadog/core/transport/http/adapters/net.rb) detects
+        # and dispatches as multipart/form-data.
+        class Client < Core::Transport::HTTP::Client
+          # @param request [Request] Symbols upload request
+          # @return [Core::Transport::HTTP::Env] Env with form data set
+          def build_env(request)
+            Core::Transport::HTTP::Env.new(request, form: request.form)
+          end
+        end
+
+        # Transport for the symbols upload endpoint.
+        class Transport < Core::Transport::Transport
+          self.http_client_class = Client
+
+          # Send a symbols upload to the agent.
+          # @param form [Hash] Multipart form data with UploadIO objects
+          # @return [Core::Transport::Response] Response from agent
+          def send_symbols(form)
+            request = Request.new(form)
+            client.send_request(:symbols, request)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/symbol_database/uploader.rb

@@ -0,0 +1,230 @@
+# frozen_string_literal: true
+
+require 'json'
+require 'securerandom'
+require 'zlib'
+require 'stringio'
+require_relative '../core/environment/identity'
+require_relative '../core/vendor/multipart-post/multipart/post/composite_read_io'
+require_relative '../tracing/ext'
+require_relative 'service_version'
+require_relative 'transport/http'
+require_relative '../di/fatal_exceptions'
+
+module Datadog
+  module SymbolDatabase
+    # Uploads symbol database payloads to the Datadog agent via HTTP multipart.
+    #
+    # Handles the complete upload process:
+    # 1. Wraps scopes in ServiceVersion (adds service/env/version metadata)
+    # 2. Serializes to JSON
+    # 3. Compresses with GZIP (always, ~40:1 ratio expected)
+    # 4. Builds multipart form: event.json (metadata) + symbols_{pid}.json.gz (data)
+    # 5. POSTs to agent at /symdb/v1/input via Core::Transport::HTTP
+    # No retries — single attempt. Any failure is logged at debug and discarded.
+    #
+    # Uses Core::Transport::HTTP infrastructure (consistent with DI, Profiling, DataStreams).
+    # Headers: DD-API-KEY, Datadog-Container-ID, Datadog-Entity-ID (automatic from transport)
+    #
+    # Called by: ScopeBatcher.perform_upload (when batch ready)
+    # Calls: Transport::HTTP for network, Zlib for compression
+    #
+    # @api private
+    class Uploader
+      MAX_PAYLOAD_SIZE = 50 * 1024 * 1024  # 50MB
+
+      # Initialize uploader.
+      # @param settings [Configuration::Settings] Tracer settings (for service, env, version metadata)
+      # @param agent_settings [Configuration::AgentSettings] Agent connection settings
+      # @param logger [Logger] Logger instance
+      # @param telemetry [Telemetry, nil] Optional telemetry component for error reporting
+      def initialize(settings:, agent_settings:, logger:, telemetry: nil)
+        @settings = settings
+        @agent_settings = agent_settings
+        @logger = logger
+        @telemetry = telemetry
+
+        @transport = Transport::HTTP.symbols(
+          agent_settings: agent_settings,
+          logger: @logger,
+        )
+
+        # Protects the lazy initialization of @upload_pid/@upload_id and the
+        # @batch_num increment in next_upload_metadata. ScopeBatcher calls
+        # upload_scopes outside its own mutex, so two threads (e.g. a size-
+        # triggered flush and the timer/shutdown path) can race here.
+        @metadata_mutex = Mutex.new
+      end
+
+      # Upload a batch of scopes to the agent.
+      # Wraps in ServiceVersion, serializes to JSON, compresses with GZIP,
+      # builds multipart form, and POSTs to /symdb/v1/input via transport.
+      # No retries — single attempt, matching Python behavior.
+      # @param scopes [Array<Scope>] Scopes to upload
+      # @return [void]
+      def upload_scopes(scopes)
+        return if scopes.empty?
+
+        upload_id, batch_num = next_upload_metadata
+        json_data = build_symbol_payload(scopes, upload_id: upload_id, batch_num: batch_num)
+        compressed_data = Zlib.gzip(json_data)
+
+        # Emitted unconditionally so the rare oversized case is observable.
+        @telemetry&.distribution(
+          Tracing::Ext::TELEMETRY_METRICS_NAMESPACE,
+          'symbol_database.payload_size',
+          compressed_data.bytesize
+        )
+
+        # Symbols for very large applications (>50MB after gzip) are dropped:
+        # the upload is skipped and the customer sees no autocomplete /
+        # symbol probe results for those classes. Java handles the same case
+        # by splitting the payload across multiple requests; we have not
+        # implemented splitting here. Deferred for a post-MVP follow-up.
+        if compressed_data.bytesize > MAX_PAYLOAD_SIZE
+          @logger.debug { "symdb: payload too large: #{compressed_data.bytesize}/#{MAX_PAYLOAD_SIZE} bytes, skipping" }
+          return
+        end
+
+        perform_http_upload(compressed_data, scopes.size, upload_id: upload_id, batch_num: batch_num)
+      rescue Exception => e # standard:disable Lint/RescueException
+        Datadog::DI.reraise_if_fatal(e)
+        @logger.debug { "symdb: upload failed: #{e.class}: #{e.message}" }
+        @telemetry&.report(e, description: 'symdb: upload failed')
+      end
+
+      private
+
+      # Build JSON payload from scopes.
+      # @param scopes [Array<Scope>] Scopes to serialize
+      # @param upload_id [String] UUID identifying the logical upload
+      # @param batch_num [Integer] 1-indexed batch number within the upload
+      # @return [String] JSON string
+      def build_symbol_payload(scopes, upload_id:, batch_num:)
+        ServiceVersion.new(
+          service: @settings.service,
+          env: @settings.env,
+          version: @settings.version,
+          scopes: scopes,
+          upload_id: upload_id,
+          batch_num: batch_num,
+          # Always false: the Ruby tracer continuously uploads new code
+          # as files are loaded; there is no defined end-of-upload point.
+          final: false,
+        ).to_json
+      end
+
+      # Perform HTTP POST with multipart form-data via transport layer.
+      # @param compressed_data [String] GZIP compressed JSON payload
+      # @param scope_count [Integer] Number of scopes (for logging)
+      # @param upload_id [String] UUID identifying the logical upload
+      # @param batch_num [Integer] 1-indexed batch number within the upload
+      # @return [void]
+      def perform_http_upload(compressed_data, scope_count, upload_id:, batch_num:)
+        form = build_multipart_form(compressed_data, upload_id: upload_id, batch_num: batch_num)
+        response = @transport.send_symbols(form)
+        handle_response(response, scope_count)
+      end
+
+      # Build multipart form-data with event metadata and compressed symbols.
+      # @param compressed_data [String] GZIP compressed JSON payload
+      # @param upload_id [String] UUID identifying the logical upload
+      # @param batch_num [Integer] 1-indexed batch number within the upload
+      # @return [Hash] Form data hash with UploadIO objects
+      def build_multipart_form(compressed_data, upload_id:, batch_num:)
+        event_io = StringIO.new(
+          build_event_metadata(compressed_data.bytesize, upload_id: upload_id, batch_num: batch_num),
+        )
+        file_io = StringIO.new(compressed_data)
+
+        event_upload = Datadog::Core::Vendor::Multipart::Post::UploadIO.new(
+          event_io,
+          'application/json',
+          'event.json',
+        )
+
+        file_upload = Datadog::Core::Vendor::Multipart::Post::UploadIO.new(
+          file_io,
+          'application/gzip',
+          "symbols_#{Process.pid}.json.gz",
+        )
+
+        {
+          'event' => event_upload,
+          'file' => file_upload,
+        }
+      end
+
+      # Build event.json metadata part.
+      # @param attachment_size [Integer] Size of the compressed attachment in bytes
+      # @param upload_id [String] UUID identifying the logical upload
+      # @param batch_num [Integer] 1-indexed batch number within the upload
+      # @return [String] JSON string for event metadata
+      def build_event_metadata(attachment_size, upload_id:, batch_num:)
+        JSON.generate(
+          ddsource: 'ruby',
+          service: @settings.service,
+          version: @settings.version,
+          language: 'ruby',
+          runtimeId: Datadog::Core::Environment::Identity.id,
+          parentId: nil,  # Fork tracking deferred for MVP
+          type: 'symdb',
+          uploadId: upload_id,
+          batchNum: batch_num,
+          # Always false: the Ruby tracer continuously uploads new code
+          # as files are loaded; there is no defined end-of-upload point.
+          final: false,
+          attachmentSize: attachment_size,
+        )
+      end
+
+      # Return [upload_id, batch_num] for the current process. upload_id is
+      # generated lazily on first use and shared by all batches uploaded from
+      # the process. A forked child observes a different Process.pid and
+      # therefore gets a fresh upload_id and batch counter.
+      # Synchronized: ScopeBatcher releases its mutex before calling
+      # upload_scopes, so this can be entered concurrently.
+      # @return [Array<(String, Integer)>]
+      def next_upload_metadata
+        @metadata_mutex.synchronize do
+          if @upload_pid != Process.pid
+            @upload_pid = Process.pid
+            @upload_id = SecureRandom.uuid.freeze
+            @batch_num = 0
+          end
+          @batch_num += 1 # steep:ignore NoMethod
+          # @upload_id and @batch_num are non-nil after the branch above; refine for Steep.
+          upload_id = @upload_id or raise('unreachable: @upload_id should be set')
+          batch_num = @batch_num or raise('unreachable: @batch_num should be set')
+          [upload_id, batch_num]
+        end
+      end
+
+      # Handle HTTP response and track metrics.
+      # @param response [Core::Transport::Response] HTTP response from agent
+      # @param scope_count [Integer] Number of scopes uploaded
+      # @return [Boolean] true if successful, false otherwise
+      def handle_response(response, scope_count)
+        if response.internal_error?
+          @logger.debug { "symdb: upload failed: #{response.error.class}: #{response.error}" } # steep:ignore NoMethod
+          return false
+        end
+
+        case response.code
+        when 200..299
+          @logger.debug { "symdb: uploaded #{scope_count} scopes successfully" }
+          true
+        when 429
+          @logger.debug { "symdb: upload rejected: rate limited (429)" }
+          false
+        when 500..599
+          @logger.debug { "symdb: upload rejected: server error (#{response.code})" }
+          false
+        else
+          @logger.debug { "symdb: upload rejected: #{response.code}" }
+          false
+        end
+      end
+    end
+  end
+end

data/lib/datadog/symbol_database.rb

@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+require_relative 'ruby_version'
+
+module Datadog
+  # Namespace for Datadog symbol database upload.
+  #
+  # @api private
+  module SymbolDatabase
+    # Sentinel value for unknown or unavailable minimum line number.
+    #
+    # Used for:
+    # 1. start_line when exact line cannot be determined (e.g., modules without methods)
+    # 2. Symbol line numbers for FIELD, STATIC_FIELD, ARG symbols to indicate
+    #    the symbol is available throughout the entire enclosing scope
+    #
+    # Backend behavior: line=0 means symbol completes in every line of the scope
+    #
+    # Reference: Symbol Database Backend RFC, section "Edge Cases"
+    # - "We use 0 for FIELD, STATIC_FIELD and ARG. It means that the symbol
+    #   will be completed in every line of the enclosing scope (CLASS or METHOD)."
+    #
+    # @see https://www.postgresql.org/docs/current/datatype-numeric.html
+    UNKNOWN_MIN_LINE = 0
+
+    # Sentinel value for unknown or unavailable maximum line number.
+    #
+    # Used for:
+    # 1. end_line when exact boundaries cannot be determined (e.g., modules, classes
+    #    without methods, fallback when introspection fails)
+    # 2. LOCAL symbol line numbers when exact line is unknown (future feature)
+    #
+    # Value: 2147483647 (PostgreSQL signed INT_MAX, 2^31 - 1)
+    #
+    # Backend behavior:
+    # - For scopes: indicates "entire file" or "unknown end"
+    # - For LOCAL symbols (future): included in method probe completions but excluded
+    #   from line probe completions
+    #
+    # Protocol specification:
+    # - "If the symbols of the scope should be available to all lines in the
+    #   source_file of the scope, use start_line = 0 and end_line = 2147483647
+    #   (maximum signed integer, postgres int max)."
+    # - "For LOCAL symbols, we use 2147483647 (signed int max) to avoid completing
+    #   the symbol for line probes, but keep it in the method for method probe completions."
+    #
+    # Reference: Symbol Database Backend RFC, section "Scope" and "Edge Cases"
+    # @see https://www.postgresql.org/docs/current/datatype-numeric.html
+    UNKNOWN_MAX_LINE = 2147483647
+
+    # Collapses the symbol_database.enabled tri-state setting to a boolean.
+    # An explicit true/false wins; nil (unconfigured) yields the caller-supplied
+    # fallback.
+    # @param setting_value [Boolean, nil] the symbol_database.enabled setting
+    # @param di_fallback [Boolean] value used when the setting is unconfigured
+    # @return [Boolean]
+    def self.resolve_enabled(setting_value, di_fallback)
+      setting_value.nil? ? di_fallback : setting_value
+    end
+
+    # Whether the current Ruby runtime can run symbol database extraction:
+    # MRI (CRuby) on Ruby 2.7 or later. Used by the remote-config layer to avoid
+    # advertising the product on runtimes where Component.build would return nil
+    # (e.g. Ruby 2.6, which Dynamic Instrumentation supports but Symbol Database
+    # does not).
+    # @return [Boolean]
+    def self.supported_runtime?
+      RUBY_ENGINE == 'ruby' && RubyVersion.is?('>= 2.7')
+    end
+  end
+end

data/lib/datadog/tracing/analytics.rb

@@ -16,7 +16,7 @@ module Datadog
         def set_measured(span_op, value = true)
           return if span_op.nil?
 
-          value = value == true || value == 1 ? 1 : 0
+          value = (value == true || value == 1) ? 1 : 0
           span_op.set_metric(Metadata::Ext::Analytics::TAG_MEASURED, value)
         end
       end

data/lib/datadog/tracing/buffer.rb

@@ -54,9 +54,9 @@ module Datadog
         @buffer_accepted_lengths += trace.length
 
         @buffer_spans += trace.length
-      rescue StandardError => e
+      rescue => e
         Datadog.logger.debug(
-          "Failed to measure queue accept. Cause: #{e.class.name} #{e.message} Source: #{Array(e.backtrace).first}"
+          "Failed to measure queue accept. Cause: #{e.class}: #{e.message} Source: #{Array(e.backtrace).first}"
         )
       end
 
@@ -64,9 +64,9 @@ module Datadog
         @buffer_dropped += 1
 
         @buffer_spans -= trace.length
-      rescue StandardError => e
+      rescue => e
         Datadog.logger.debug(
-          "Failed to measure queue drop. Cause: #{e.class.name} #{e.message} Source: #{Array(e.backtrace).first}"
+          "Failed to measure queue drop. Cause: #{e.class}: #{e.message} Source: #{Array(e.backtrace).first}"
         )
       end
 
@@ -89,9 +89,9 @@ module Datadog
         @buffer_accepted_lengths = 0
         @buffer_dropped = 0
         @buffer_spans = 0
-      rescue StandardError => e
+      rescue => e
         Datadog.logger.debug(
-          "Failed to measure queue. Cause: #{e.class.name} #{e.message} Source: #{Array(e.backtrace).first}"
+          "Failed to measure queue. Cause: #{e.class}: #{e.message} Source: #{Array(e.backtrace).first}"
         )
       end
     end
@@ -121,9 +121,9 @@ module Datadog
     # TODO We should restructure this module, so that classes are not declared at top-level ::Datadog.
     # TODO Making such a change is potentially breaking for users manually configuring the tracer.
     TraceBuffer = if Core::Environment::Ext::RUBY_ENGINE == 'ruby'
-                    CRubyTraceBuffer
-                  else
-                    ThreadSafeTraceBuffer
-                  end
+      CRubyTraceBuffer
+    else
+      ThreadSafeTraceBuffer
+    end
   end
 end