datadog 2.2.0 → 2.3.0

data/lib/datadog/core/crashtracking/agent_base_url.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+require_relative '../configuration/ext'
+
+module Datadog
+  module Core
+    module Crashtracking
+      # This module provides a method to resolve the base URL of the agent
+      module AgentBaseUrl
+        def self.resolve(agent_settings)
+          case agent_settings.adapter
+          when Datadog::Core::Configuration::Ext::Agent::HTTP::ADAPTER
+            "#{agent_settings.ssl ? 'https' : 'http'}://#{agent_settings.hostname}:#{agent_settings.port}/"
+          when Datadog::Core::Configuration::Ext::Agent::UnixSocket::ADAPTER
+            "unix://#{agent_settings.uds_path}"
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/core/crashtracking/component.rb

@@ -0,0 +1,111 @@
+# frozen_string_literal: true
+
+require 'libdatadog'
+
+require_relative 'tag_builder'
+require_relative 'agent_base_url'
+require_relative '../utils/only_once'
+require_relative '../utils/at_fork_monkey_patch'
+
+module Datadog
+  module Core
+    module Crashtracking
+      # Used to report Ruby VM crashes.
+      #
+      # NOTE: The crashtracker native state is a singleton;
+      # so even if you create multiple instances of `Crashtracking::Component` and start them,
+      # it only works as "last writer wins". Same for stop -- there's only one state, so calling stop
+      # on it will stop the crash tracker, regardless of which instance started it.
+      #
+      # Methods prefixed with _native_ are implemented in `crashtracker.c`
+      class Component
+        LIBDATADOG_API_FAILURE =
+          begin
+            require "libdatadog_api.#{RUBY_VERSION[/\d+.\d+/]}_#{RUBY_PLATFORM}"
+            nil
+          rescue LoadError => e
+            e.message
+          end
+
+        ONLY_ONCE = Core::Utils::OnlyOnce.new
+
+        def self.build(settings, agent_settings, logger:)
+          tags = TagBuilder.call(settings)
+          agent_base_url = AgentBaseUrl.resolve(agent_settings)
+          logger.warn('Missing agent base URL; cannot enable crash tracking') unless agent_base_url
+
+          ld_library_path = ::Libdatadog.ld_library_path
+          logger.warn('Missing ld_library_path; cannot enable crash tracking') unless ld_library_path
+
+          path_to_crashtracking_receiver_binary = ::Libdatadog.path_to_crashtracking_receiver_binary
+          unless path_to_crashtracking_receiver_binary
+            logger.warn('Missing path_to_crashtracking_receiver_binary; cannot enable crash tracking')
+          end
+
+          return unless agent_base_url
+          return unless ld_library_path
+          return unless path_to_crashtracking_receiver_binary
+
+          new(
+            tags: tags,
+            agent_base_url: agent_base_url,
+            ld_library_path: ld_library_path,
+            path_to_crashtracking_receiver_binary: path_to_crashtracking_receiver_binary,
+            logger: logger
+          ).tap(&:start)
+        end
+
+        def initialize(tags:, agent_base_url:, ld_library_path:, path_to_crashtracking_receiver_binary:, logger:)
+          @tags = tags
+          @agent_base_url = agent_base_url
+          @ld_library_path = ld_library_path
+          @path_to_crashtracking_receiver_binary = path_to_crashtracking_receiver_binary
+          @logger = logger
+        end
+
+        def start
+          Utils::AtForkMonkeyPatch.apply!
+
+          start_or_update_on_fork(action: :start)
+          ONLY_ONCE.run do
+            Utils::AtForkMonkeyPatch.at_fork(:child) do
+              # Must NOT reference `self` here, as only the first instance will
+              # be captured by the ONLY_ONCE and we want to pick the latest active one
+              # (which may have different tags or agent config)
+              Datadog.send(:components).crashtracker&.update_on_fork
+            end
+          end
+        end
+
+        def update_on_fork
+          start_or_update_on_fork(action: :update_on_fork)
+        end
+
+        def stop
+          self.class._native_stop
+          logger.debug('Crash tracking stopped successfully')
+        rescue => e
+          logger.error("Failed to stop crash tracking: #{e.message}")
+        end
+
+        private
+
+        attr_reader :tags, :agent_base_url, :ld_library_path, :path_to_crashtracking_receiver_binary, :logger
+
+        def start_or_update_on_fork(action:)
+          self.class._native_start_or_update_on_fork(
+            action: action,
+            exporter_configuration: [:agent, agent_base_url],
+            path_to_crashtracking_receiver_binary: path_to_crashtracking_receiver_binary,
+            ld_library_path: ld_library_path,
+            tags_as_array: tags.to_a,
+            upload_timeout_seconds: 1
+          )
+          logger.debug("Crash tracking #{action} successfully")
+        rescue => e
+          logger.error("Failed to #{action} crash tracking: #{e.message}")
+        end
+      end
+    end
+  end
+end

data/lib/datadog/core/crashtracking/tag_builder.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+require_relative '../utils'
+require_relative '../environment/socket'
+require_relative '../environment/identity'
+require_relative '../environment/git'
+
+module Datadog
+  module Core
+    module Crashtracking
+      # This module builds a hash of tags
+      module TagBuilder
+        def self.call(settings)
+          hash = {
+            'host' => Environment::Socket.hostname,
+            'process_id' => Process.pid.to_s,
+            'runtime_engine' => Environment::Identity.lang_engine,
+            'runtime-id' => Environment::Identity.id,
+            'runtime_platform' => Environment::Identity.lang_platform,
+            'runtime_version' => Environment::Identity.lang_version,
+            'env' => settings.env,
+            'service' => settings.service,
+            'version' => settings.version,
+            'git.repository_url' => Environment::Git.git_repository_url,
+            'git.commit.sha' => Environment::Git.git_commit_sha,
+            'is_crash' => 'true',
+            'language' => 'ruby',
+            'library_version' => Core::Environment::Identity.gem_datadog_version,
+          }.compact
+
+          # Make sure everything is an utf-8 string, to avoid encoding issues in downstream
+          settings.tags.merge(hash).each_with_object({}) do |(key, value), h|
+            h[Utils.utf8_encode(key)] = Utils.utf8_encode(value)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/core/diagnostics/environment_logger.rb

@@ -8,11 +8,17 @@ module Datadog
   module Core
     module Diagnostics
       # Base class for EnvironmentLoggers - should allow for easy reporting by users to Datadog support.
+      #
+      # The EnvironmentLogger should not pollute the logs in a development environment.
       module EnvironmentLogging
         def log_configuration!(prefix, data)
           logger.info("DATADOG CONFIGURATION - #{prefix} - #{data}")
         end
 
+        def log_debug!(prefix, data)
+          logger.debug("DATADOG CONFIGURATION - #{prefix} - #{data}")
+        end
+
         def log_error!(prefix, type, error)
           logger.warn("DATADOG ERROR - #{prefix} - #{type}: #{error}")
         end
@@ -27,21 +33,12 @@ module Datadog
         def log?
           startup_logs_enabled = Datadog.configuration.diagnostics.startup_logs.enabled
           if startup_logs_enabled.nil?
-            !repl? && !rspec? # Suppress logs if we are running in a REPL or rspec
+            # Do not pollute the logs in a development environment.
+            !Datadog::Core::Environment::Execution.development?
           else
             startup_logs_enabled
           end
         end
-
-        REPL_PROGRAM_NAMES = %w[irb pry].freeze
-
-        def repl?
-          REPL_PROGRAM_NAMES.include?($PROGRAM_NAME)
-        end
-
-        def rspec?
-          $PROGRAM_NAME.end_with?('rspec')
-        end
       end
 
       # Collects and logs Core diagnostic information

data/lib/datadog/core/telemetry/component.rb

@@ -2,8 +2,11 @@
 
 require_relative 'emitter'
 require_relative 'event'
+require_relative 'http/transport'
 require_relative 'metrics_manager'
 require_relative 'worker'
+
+require_relative '../configuration/ext'
 require_relative '../utils/forking'
 
 module Datadog
@@ -15,6 +18,41 @@ module Datadog
 
         include Core::Utils::Forking
 
+        def self.build(settings, agent_settings, logger)
+          enabled = settings.telemetry.enabled
+          agentless_enabled = settings.telemetry.agentless_enabled
+
+          if !agentless_enabled && agent_settings.adapter != Datadog::Core::Configuration::Ext::Agent::HTTP::ADAPTER
+            enabled = false
+            logger.debug { "Telemetry disabled. Agent network adapter not supported: #{agent_settings.adapter}" }
+          end
+
+          if agentless_enabled && settings.api_key.nil?
+            enabled = false
+            logger.debug { 'Telemetry disabled. Agentless telemetry requires an DD_API_KEY variable to be set.' }
+          end
+
+          transport = if agentless_enabled
+                        Datadog::Core::Telemetry::Http::Transport.build_agentless_transport(
+                          api_key: settings.api_key,
+                          dd_site: settings.site,
+                          url_override: settings.telemetry.agentless_url_override
+                        )
+                      else
+                        Datadog::Core::Telemetry::Http::Transport.build_agent_transport(agent_settings)
+                      end
+
+          Telemetry::Component.new(
+            http_transport: transport,
+            enabled: enabled,
+            metrics_enabled: enabled && settings.telemetry.metrics_enabled,
+            heartbeat_interval_seconds: settings.telemetry.heartbeat_interval_seconds,
+            metrics_aggregation_interval_seconds: settings.telemetry.metrics_aggregation_interval_seconds,
+            dependency_collection: settings.telemetry.dependency_collection,
+            shutdown_timeout_seconds: settings.telemetry.shutdown_timeout_seconds,
+          )
+        end
+
         # @param enabled [Boolean] Determines whether telemetry events should be sent to the API
         # @param metrics_enabled [Boolean] Determines whether telemetry metrics should be sent to the API
         # @param heartbeat_interval_seconds [Float] How frequently heartbeats will be reported, in seconds.
@@ -24,6 +62,8 @@ module Datadog
           heartbeat_interval_seconds:,
           metrics_aggregation_interval_seconds:,
           dependency_collection:,
+          http_transport:,
+          shutdown_timeout_seconds:,
           enabled: true,
           metrics_enabled: true
         )
@@ -39,9 +79,10 @@ module Datadog
             enabled: @enabled,
             heartbeat_interval_seconds: heartbeat_interval_seconds,
             metrics_aggregation_interval_seconds: metrics_aggregation_interval_seconds,
-            emitter: Emitter.new,
+            emitter: Emitter.new(http_transport: http_transport),
             metrics_manager: @metrics_manager,
-            dependency_collection: dependency_collection
+            dependency_collection: dependency_collection,
+            shutdown_timeout: shutdown_timeout_seconds
           )
           @worker.start
         end
@@ -70,6 +111,12 @@ module Datadog
           @worker.enqueue(Event::AppIntegrationsChange.new)
         end
 
+        def log!(event)
+          return unless @enabled || forked?
+
+          @worker.enqueue(event)
+        end
+
         # Report configuration changes caused by Remote Configuration.
         def client_configuration_change!(changes)
           return if !@enabled || forked?

data/lib/datadog/core/telemetry/emitter.rb

@@ -16,22 +16,20 @@ module Datadog
 
         # @param http_transport [Datadog::Core::Telemetry::Http::Transport] Transport object that can be used to send
         #   telemetry requests via the agent
-        def initialize(http_transport: Datadog::Core::Telemetry::Http::Transport.new)
+        def initialize(http_transport:)
           @http_transport = http_transport
         end
 
         # Retrieves and emits a TelemetryRequest object based on the request type specified
         def request(event)
-          begin
-            seq_id = self.class.sequence.next
-            payload = Request.build_payload(event, seq_id)
-            res = @http_transport.request(request_type: event.type, payload: payload.to_json)
-            Datadog.logger.debug { "Telemetry sent for event `#{event.type}` (code: #{res.code.inspect})" }
-            res
-          rescue => e
-            Datadog.logger.debug("Unable to send telemetry request for event `#{event.type rescue 'unknown'}`: #{e}")
-            Telemetry::Http::InternalErrorResponse.new(e)
-          end
+          seq_id = self.class.sequence.next
+          payload = Request.build_payload(event, seq_id)
+          res = @http_transport.request(request_type: event.type, payload: payload.to_json)
+          Datadog.logger.debug { "Telemetry sent for event `#{event.type}` (code: #{res.code.inspect})" }
+          res
+        rescue => e
+          Datadog.logger.debug("Unable to send telemetry request for event `#{event.type rescue 'unknown'}`: #{e}")
+          Telemetry::Http::InternalErrorResponse.new(e)
         end
 
         # Initializes a Sequence object to track seq_id if not already initialized; else returns stored

data/lib/datadog/core/telemetry/event.rb

@@ -18,7 +18,9 @@ module Datadog
           # The type of the event.
           # It must be one of the stings defined in the Telemetry V2
           # specification for event names.
-          def type; end
+          def type
+            raise NotImplementedError, 'Must be implemented by subclass'
+          end
 
           # The JSON payload for the event.
           def payload
@@ -332,6 +334,35 @@ module Datadog
           end
         end
 
+        # Telemetry class for the 'logs' event
+        class Log < Base
+          LEVELS = {
+            error: 'ERROR',
+            debug: 'DEBUG',
+            warn: 'WARN',
+          }.freeze
+
+          def type
+            'logs'
+          end
+
+          def initialize(message:, level:)
+            super()
+            @message = message
+            @level = LEVELS.fetch(level) { |k| raise ArgumentError, "Invalid log level :#{k}" }
+          end
+
+          def payload
+            {
+              logs: [{
+                message: @message,
+                level: @level,
+                # More optional fields to be added here...
+              }]
+            }
+          end
+        end
+
         # Telemetry class for the 'distributions' event
         class Distributions < GenerateMetrics
           def type

data/lib/datadog/core/telemetry/ext.rb

@@ -12,6 +12,7 @@ module Datadog
         ENV_INSTALL_ID = 'DD_INSTRUMENTATION_INSTALL_ID'
         ENV_INSTALL_TYPE = 'DD_INSTRUMENTATION_INSTALL_TYPE'
         ENV_INSTALL_TIME = 'DD_INSTRUMENTATION_INSTALL_TIME'
+        ENV_AGENTLESS_URL_OVERRIDE = 'DD_TELEMETRY_AGENTLESS_URL'
       end
     end
   end

data/lib/datadog/core/telemetry/http/adapters/net.rb

@@ -34,19 +34,17 @@ module Datadog
             end
 
             def post(env)
-              begin
-                post = ::Net::HTTP::Post.new(env.path, env.headers)
-                post.body = env.body
-
-                http_response = open do |http|
-                  http.request(post)
-                end
-
-                Response.new(http_response)
-              rescue StandardError => e
-                Datadog.logger.debug("Unable to send telemetry event to agent: #{e}")
-                Telemetry::Http::InternalErrorResponse.new(e)
+              post = ::Net::HTTP::Post.new(env.path, env.headers)
+              post.body = env.body
+
+              http_response = open do |http|
+                http.request(post)
               end
+
+              Response.new(http_response)
+            rescue StandardError => e
+              Datadog.logger.debug("Unable to send telemetry event to agent: #{e}")
+              Telemetry::Http::InternalErrorResponse.new(e)
             end
 
             # Data structure for an HTTP Response

data/lib/datadog/core/telemetry/http/ext.rb

@@ -17,7 +17,10 @@ module Datadog
           CONTENT_TYPE_APPLICATION_JSON = 'application/json'
           API_VERSION = 'v2'
 
+          AGENTLESS_HOST_PREFIX = 'instrumentation-telemetry-intake'
+
           AGENT_ENDPOINT = '/telemetry/proxy/api/v2/apmtelemetry'
+          AGENTLESS_ENDPOINT = '/api/v2/apmtelemetry'
         end
       end
     end

data/lib/datadog/core/telemetry/http/transport.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require_relative '../../configuration/settings'
+require_relative '../../environment/ext'
 require_relative '../../transport/ext'
 require_relative 'env'
 require_relative 'ext'
@@ -13,18 +14,42 @@ module Datadog
         # Class to send telemetry data to Telemetry API
         # Currently only supports the HTTP protocol.
         class Transport
+          def self.build_agent_transport(agent_settings)
+            Transport.new(
+              host: agent_settings.hostname,
+              port: agent_settings.port,
+              path: Http::Ext::AGENT_ENDPOINT
+            )
+          end
+
+          def self.build_agentless_transport(api_key:, dd_site:, url_override: nil)
+            url = url_override || "https://#{Http::Ext::AGENTLESS_HOST_PREFIX}.#{dd_site}:443"
+
+            uri = URI.parse(url)
+            raise "Invalid agentless mode URL: #{url}" if uri.host.nil?
+
+            Transport.new(
+              host: uri.host,
+              port: uri.port || 80,
+              path: Http::Ext::AGENTLESS_ENDPOINT,
+              ssl: uri.scheme == 'https' || uri.port == 443,
+              api_key: api_key
+            )
+          end
+
           attr_reader \
             :host,
             :port,
             :ssl,
-            :path
-
-          def initialize
-            agent_settings = Configuration::AgentSettingsResolver.call(Datadog.configuration)
-            @host = agent_settings.hostname
-            @port = agent_settings.port
-            @ssl = false
-            @path = Http::Ext::AGENT_ENDPOINT
+            :path,
+            :api_key
+
+          def initialize(host:, port:, path:, ssl: false, api_key: nil)
+            @host = host
+            @port = port
+            @ssl = ssl
+            @path = path
+            @api_key = api_key
           end
 
           def request(request_type:, payload:)
@@ -38,7 +63,7 @@ module Datadog
           private
 
           def headers(request_type:, api_version: Http::Ext::API_VERSION)
-            {
+            result = {
               Core::Transport::Ext::HTTP::HEADER_DD_INTERNAL_UNTRACED_REQUEST => '1',
               Ext::HEADER_CONTENT_TYPE => Http::Ext::CONTENT_TYPE_APPLICATION_JSON,
               Ext::HEADER_DD_TELEMETRY_API_VERSION => api_version,
@@ -49,6 +74,10 @@ module Datadog
               # Enable debug mode for telemetry
               # HEADER_TELEMETRY_DEBUG_ENABLED => 'true',
             }
+
+            result[Ext::HEADER_DD_API_KEY] = api_key unless api_key.nil?
+
+            result
           end
 
           def adapter

data/lib/datadog/core/telemetry/logging.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+require_relative 'event'
+
+module Datadog
+  module Core
+    module Telemetry
+      # Logging interface for sending telemetry logs.
+      #
+      # Reporting internal error so that we can fix them.
+      # IMPORTANT: Make sure to not log any sensitive information.
+      module Logging
+        module_function
+
+        def report(exception, level:)
+          # Annoymous exceptions to be logged as <Class:0x00007f8b1c0b3b40>
+          message = exception.class.name || exception.class.inspect
+
+          event = Event::Log.new(
+            message: message,
+            level: level
+          )
+
+          if (telemetry = Datadog.send(:components).telemetry)
+            telemetry.log!(event)
+          else
+            Datadog.logger.debug do
+              "Attempting to send telemetry log when telemetry component is not ready: #{message}"
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/core/utils/at_fork_monkey_patch.rb

@@ -0,0 +1,102 @@
+# frozen_string_literal: true
+
+module Datadog
+  module Core
+    module Utils
+      # Monkey patches `Kernel#fork` and similar functions, adding an `at_fork` callback mechanism which
+      # is used to restart observability after the VM forks (e.g. in multiprocess Ruby apps).
+      module AtForkMonkeyPatch
+        AT_FORK_CHILD_BLOCKS = [] # rubocop:disable Style/MutableConstant Used to store blocks to run, mutable by design.
+        private_constant :AT_FORK_CHILD_BLOCKS
+
+        def self.supported?
+          Process.respond_to?(:fork)
+        end
+
+        def self.apply!
+          return false unless supported?
+
+          if RUBY_VERSION < '3.1'
+            [
+              ::Process.singleton_class, # Process.fork
+              ::Kernel.singleton_class,  # Kernel.fork
+              ::Object,                  # fork without explicit receiver (it's defined as a method in ::Kernel)
+              # Note: Modifying Object as we do here is irreversible. During tests, this
+              # change will stick around even if we otherwise stub `Process` and `Kernel`
+            ].each { |target| target.prepend(KernelMonkeyPatch) }
+          end
+
+          ::Process.singleton_class.prepend(ProcessMonkeyPatch)
+
+          true
+        end
+
+        def self.run_at_fork_blocks(stage)
+          raise(ArgumentError, "Unsupported stage #{stage}") unless stage == :child
+
+          AT_FORK_CHILD_BLOCKS.each(&:call)
+        end
+
+        def self.at_fork(stage, &block)
+          raise(ArgumentError, "Unsupported stage #{stage}") unless stage == :child
+          raise(ArgumentError, 'Missing block argument') unless block
+
+          AT_FORK_CHILD_BLOCKS << block
+
+          true
+        end
+
+        # Adds `at_fork` behavior; see parent module for details.
+        module KernelMonkeyPatch
+          def fork
+            # If a block is provided, it must be wrapped to trigger callbacks.
+            child_block = if block_given?
+                            proc do
+                              AtForkMonkeyPatch.run_at_fork_blocks(:child)
+
+                              # Invoke original block
+                              yield
+                            end
+                          end
+
+            # Start fork
+            # If a block is provided, use the wrapped version.
+            result = child_block.nil? ? super : super(&child_block)
+
+            # When fork gets called without a block, it returns twice:
+            # If we're in the fork, result = nil: trigger child callbacks.
+            # If we're in the parent, result = pid: we do nothing.
+            # (If it gets called with a block, it only returns on the parent)
+            AtForkMonkeyPatch.run_at_fork_blocks(:child) if result.nil?
+
+            result
+          end
+        end
+
+        # Adds `at_fork` behavior; see parent module for details.
+        module ProcessMonkeyPatch
+          # Hook provided by Ruby 3.1+ for observability libraries that want to know about fork, see
+          # https://github.com/ruby/ruby/pull/5017 and https://bugs.ruby-lang.org/issues/17795
+          def _fork
+            pid = super
+
+            AtForkMonkeyPatch.run_at_fork_blocks(:child) if pid == 0
+
+            pid
+          end
+
+          # A call to Process.daemon ( https://rubyapi.org/3.1/o/process#method-c-daemon ) forks the current process and
+          # keeps executing code in the child process, killing off the parent, thus effectively replacing it.
+          # This is not covered by `_fork` and thus we have some extra code for it.
+          def daemon(*args)
+            result = super
+
+            AtForkMonkeyPatch.run_at_fork_blocks(:child)
+
+            result
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/kit/appsec/events.rb

@@ -53,13 +53,11 @@ module Datadog
           # @param user_exists [bool] Whether the user id that did a login attempt exists.
           # @param others [Hash<String || Symbol, String>] Additional free-form
           #   event information to attach to the trace.
-          def track_login_failure(trace = nil, span = nil, user_id:, user_exists:, **others)
+          def track_login_failure(trace = nil, span = nil, user_exists:, user_id: nil, **others)
             set_trace_and_span_context('track_login_failure', trace, span) do |active_trace, active_span|
-              raise ArgumentError, 'user_id cannot be nil' if user_id.nil?
-
               track(LOGIN_FAILURE_EVENT, active_trace, active_span, **others)
 
-              active_span.set_tag('appsec.events.users.login.failure.usr.id', user_id)
+              active_span.set_tag('appsec.events.users.login.failure.usr.id', user_id) if user_id
               active_span.set_tag('appsec.events.users.login.failure.usr.exists', user_exists)
             end
           end