datadog 2.4.0 → 2.44.0

data/ext/libdatadog_api/trace_exporter.h

@@ -0,0 +1,5 @@
+#pragma once
+
+#include "datadog_ruby_common.h"
+
+void trace_exporter_init(VALUE tracing_module);

data/ext/libdatadog_extconf_helpers.rb

@@ -1,14 +1,17 @@
 # frozen_string_literal: true
 
-require 'rubygems'
-require 'pathname'
+require "rubygems"
+require "pathname"
+require "shellwords"
 
 module Datadog
   # Contains a bunch of shared helpers that get used during building of extensions that link to libdatadog
+  #
+  # Note: Specs for this file currently live in `spec/datadog/profiling/native_extension_helpers_spec.rb`.
   module LibdatadogExtconfHelpers
     # Used to make sure the correct gem version gets loaded, as extconf.rb does not get run with "bundle exec" and thus
     # may see multiple libdatadog versions. See https://github.com/DataDog/dd-trace-rb/pull/2531 for the horror story.
-    LIBDATADOG_VERSION = '~> 12.0.0.1.0'
+    LIBDATADOG_VERSION = "~> 44.0.1.1.0"
 
     # Used as an workaround for a limitation with how dynamic linking works in environments where the datadog gem and
     # libdatadog are moved after the extension gets compiled.
@@ -21,9 +24,9 @@ module Datadog
     # Linux: e.g. `readelf -d datadog_profiling_native_extension.2.7.3_x86_64-linux.so`.
     #
     # In older versions of the datadog gem, we only set as runpath an absolute path to libdatadog.
-    # (This gets set automatically by the call
-    # to `pkg_config('datadog_profiling_with_rpath')` in `extconf.rb`). This worked fine as long as libdatadog was **NOT**
-    # moved from the folder it was present at datadog gem installation/linking time.
+    # (This gets set automatically by the call to `configure_libdatadog` in `extconf.rb`).
+    # This worked fine as long as libdatadog was **NOT** moved from the folder it was present at
+    # datadog gem installation/linking time.
     #
     # Unfortunately, environments such as Heroku and AWS Elastic Beanstalk move gems around in the filesystem after
     # installation. Thus, the profiling native extension could not be loaded in these environments
@@ -45,12 +48,12 @@ module Datadog
     # we could setup when doing a `require`.
     #
     def self.libdatadog_folder_relative_to_native_lib_folder(
-      current_folder:,
+      extconf_folder:,
       libdatadog_pkgconfig_folder: Libdatadog.pkgconfig_folder
     )
       return unless libdatadog_pkgconfig_folder
 
-      native_lib_folder = "#{current_folder}/../../lib/"
+      native_lib_folder = "#{extconf_folder}/../../lib/"
       libdatadog_lib_folder = "#{libdatadog_pkgconfig_folder}/../"
 
       Pathname.new(libdatadog_lib_folder).relative_path_from(Pathname.new(native_lib_folder)).to_s
@@ -102,17 +105,59 @@ module Datadog
       end
     end
 
-    # mkmf sets $PKGCONFIG after the `pkg_config` gets used in extconf.rb. When `pkg_config` is unsuccessful, we use
-    # this helper to decide if we can show more specific error message vs a generic "something went wrong".
-    def self.pkg_config_missing?(command: $PKGCONFIG) # rubocop:disable Style/GlobalVars
-      pkg_config_available = command && xsystem("#{command} --version")
+    # Directly configures mkmf to link against libdatadog by setting $INCFLAGS, $LDFLAGS, and $libs.
+    #
+    # This replaces the previous use of mkmf's `pkg_config("datadog_profiling_with_rpath")`, removing the need for
+    # the pkg-config system tool to be installed.
+    #
+    # The extconf_folder argument should be the __dir__ of the calling extconf.rb, used to compute relative rpaths.
+    # The logger argument is the mkmf Logging module, dependency-injected to make testing easier.
+    # rubocop:disable Style/GlobalVars
+    def self.configure_libdatadog(
+      extconf_folder:,
+      libdatadog_pkgconfig_folder: Libdatadog.pkgconfig_folder,
+      gem_dir: Gem.dir,
+      logger: Logging,
+      target_incflags: $INCFLAGS,
+      target_ldflags: $LDFLAGS,
+      target_libs: $libs
+    )
+      return unless libdatadog_pkgconfig_folder
+
+      # The lib and include folders are at a fixed relative path from pkgconfig_folder
+      libdir = "#{libdatadog_pkgconfig_folder}/../../lib"
+      includedir = "#{libdatadog_pkgconfig_folder}/../../include"
 
-      pkg_config_available != true
+      target_incflags << " -I#{Shellwords.escape(includedir)}"
+      target_ldflags << " -L#{Shellwords.escape(libdir)} -Wl,-rpath,#{Shellwords.escape(libdir)}"
+      target_libs << " -ldatadog_profiling"
+
+      # Add extra relative rpaths using $ORIGIN to handle environments where gems are moved after installation.
+      # The excessive escaping is needed to get these special characters through Make and the shell untouched.
+      extra_relative_rpaths = [
+        libdatadog_folder_relative_to_native_lib_folder(
+          extconf_folder: extconf_folder,
+          libdatadog_pkgconfig_folder: libdatadog_pkgconfig_folder,
+        ),
+        *libdatadog_folder_relative_to_ruby_extensions_folders(
+          gem_dir: gem_dir,
+          libdatadog_pkgconfig_folder: libdatadog_pkgconfig_folder,
+        ),
+      ]
+      extra_relative_rpaths.each { |folder| target_ldflags << " -Wl,-rpath,$$$\\\\{ORIGIN\\}/#{Shellwords.escape(folder)}" }
+
+      # includedir/libdir/ldflags might have a `%` in there and `logger.message` bottoms out on `printf` so we need to
+      # use the `message("%s", ...)` format otherwise the logger will complain
+      logger.message("%s", "linking with libdatadog (include=#{includedir}, lib=#{libdir})\n")
+      logger.message("%s", "[datadog] $LDFLAGS were set to: #{target_ldflags.inspect}\n")
+
+      true
     end
+    # rubocop:enable Style/GlobalVars
 
     def self.try_loading_libdatadog
-      gem 'libdatadog', LIBDATADOG_VERSION
-      require 'libdatadog'
+      gem "libdatadog", LIBDATADOG_VERSION
+      require "libdatadog"
       nil
     rescue Exception => e # rubocop:disable Lint/RescueException
       if block_given?
@@ -122,9 +167,62 @@ module Datadog
       end
     end
 
-    def self.libdatadog_issue?
-      try_loading_libdatadog { |_exception| return true }
-      Libdatadog.pkgconfig_folder.nil?
+    # Adds a C preprocessor define with the libdatadog version used at compile time.
+    # This allows runtime verification that the loaded libdatadog matches what was compiled against.
+    # rubocop:disable Style/GlobalVars
+    def self.add_libdatadog_version_define
+      $defs << %(-DEXPECTED_LIBDATADOG_VERSION=\\"#{Libdatadog::VERSION}\\")
+    end
+    # rubocop:enable Style/GlobalVars
+
+    # When setting up the extension build fails (e.g. a missing header), the actual failure ends up in mkmf.log
+    # and is not printed, which is confusing and complicates debugging. To fix that, this monkey patch actually
+    # prints the failure.
+    def self.dump_mkmf_log_on_failure!
+      # Doesn't work on 2.5/2.6 and not worth extra complexity to support
+      return if RUBY_VERSION < "2.7"
+
+      MakeMakefile.prepend(DumpMkmfLogOnFailure)
+    end
+
+    # rubocop:disable Style/GlobalVars,Style/StderrPuts
+    # See `dump_mkmf_log_on_failure!` for details.
+    module DumpMkmfLogOnFailure
+      def mkmf_failed(path)
+        unless $makefile_created || File.exist?("Makefile")
+          log_path = File.expand_path("mkmf.log")
+          if File.exist?(log_path)
+            # The full log is very verbose so let's grab only the last check which should be the one that failed
+            entries = File.read(log_path).split(/^-{20}$\n?/)
+            last_entry = entries.reverse.find { |e| !e.strip.empty? } || ""
+
+            $stderr.puts(
+              "+------------------------------------------------------------------------------+\n" \
+              "| There was an issue setting up extension build:                               |\n" \
+              "+------------------------------------------------------------------------------+" \
+              "#{last_entry.chomp}" \
+              "+------------------------------------------------------------------------------+\n" \
+              "| Full failure log is at #{log_path}\n" \
+              "+------------------------------------------------------------------------------+\n" \
+            )
+          end
+        end
+        super
+      end
+    end
+    # rubocop:enable Style/GlobalVars,Style/StderrPuts
+
+    # Note: This helper is currently only used in the `libdatadog_api/extconf.rb` BUT still lives here to enable testing.
+    def self.load_libdatadog_or_get_issue
+      try_loading_libdatadog do |exception|
+        return "There was an error loading `libdatadog`: #{exception.class} #{exception.message}"
+      end
+
+      unless Libdatadog.pkgconfig_folder
+        "The `libdatadog` gem installed on your system is missing binaries for your platform variant. " \
+          "Your platform: " \
+          "`#{Libdatadog.current_platform}`; available binaries: `#{Libdatadog.available_binaries.join("`, `")}`"
+      end
     end
   end
 end

data/lib/datadog/ai_guard/autoload.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+if %w[1 true].include?((Datadog::DATADOG_ENV["DD_AI_GUARD_ENABLED"] || "").downcase)
+  begin
+    require_relative "contrib/auto_instrument"
+    Datadog::AIGuard::Contrib::AutoInstrument.patch_all
+  rescue => e
+    Kernel.warn("[datadog] AI Guard failed to auto-instrument. error: #{e.class}: #{e.message}")
+  end
+end

data/lib/datadog/ai_guard/component.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+require "json"
+
+require_relative "ext"
+require_relative "http_client"
+require_relative "evaluation"
+require_relative "evaluation/request"
+require_relative "evaluation/response"
+require_relative "evaluation/client"
+require_relative "evaluation/result"
+require_relative "evaluation/outcome"
+require_relative "evaluation/no_op_result"
+require_relative "evaluation/message"
+require_relative "evaluation/tool_call"
+require_relative "evaluation/content_part"
+require_relative "evaluation/message_builder"
+require_relative "redaction"
+require_relative "redaction/result"
+require_relative "redaction/replacements"
+require_relative "metrics/telemetry"
+
+module Datadog
+  module AIGuard
+    # Component for API Guard product
+    class Component
+      attr_reader :http_client, :logger, :telemetry
+
+      def self.build(settings, logger:, telemetry:)
+        return unless settings.respond_to?(:ai_guard) && settings.ai_guard.enabled
+
+        http_client = HTTPClient.new(
+          endpoint: settings.ai_guard.endpoint,
+          api_key: settings.api_key,
+          application_key: settings.ai_guard.app_key,
+          timeout: settings.ai_guard.timeout_ms / 1_000
+        )
+
+        new(http_client, logger: logger, telemetry: telemetry)
+      end
+
+      def initialize(http_client, logger:, telemetry:)
+        @http_client = http_client
+        @logger = logger
+        @telemetry = telemetry
+      end
+
+      def shutdown!
+        # no-op
+      end
+    end
+  end
+end

data/lib/datadog/ai_guard/configuration/ext.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module Datadog
+  module AIGuard
+    module Configuration
+      # This module contains constants for AI Guard component
+      module Ext
+        ENV_AI_GUARD_ENABLED = "DD_AI_GUARD_ENABLED"
+        ENV_AI_GUARD_REDACTION_ENABLED = "DD_AI_GUARD_REDACTION_ENABLED"
+
+        ENV_AI_GUARD_ENDPOINT = "DD_AI_GUARD_ENDPOINT"
+        ENV_AI_GUARD_TIMEOUT = "DD_AI_GUARD_TIMEOUT"
+        ENV_AI_GUARD_MAX_CONTENT_SIZE = "DD_AI_GUARD_MAX_CONTENT_SIZE"
+        ENV_AI_GUARD_MAX_MESSAGES_LENGTH = "DD_AI_GUARD_MAX_MESSAGES_LENGTH"
+        ENV_APP_KEY = "DD_APP_KEY"
+      end
+    end
+  end
+end

data/lib/datadog/ai_guard/configuration.rb

@@ -0,0 +1,128 @@
+# frozen_string_literal: true
+
+require "uri"
+require_relative "configuration/ext"
+
+module Datadog
+  module AIGuard
+    # Configuration module for AI Guard
+    module Configuration
+      # AI Guard specific settings
+      module Settings
+        def self.extended(base)
+          base = base.singleton_class unless base.is_a?(Class)
+          add_settings!(base)
+        end
+
+        def self.add_settings!(base)
+          base.class_eval do
+            # AI Guard specific configurations.
+            # @public_api
+            #
+            # Steep does not update `self` for this `class_eval` block.
+            # @type self: Datadog::Core::Configuration::Base::_DslContext
+            settings :ai_guard do
+              # Enable AI Guard.
+              #
+              # You can use this option to skip calls to AI Guard API without having to remove library as a whole.
+              #
+              # @default `DD_AI_GUARD_ENABLED`, otherwise `false`
+              # @return [Boolean]
+              option :enabled do |o|
+                o.type :bool
+                o.env Ext::ENV_AI_GUARD_ENABLED
+                o.default false
+              end
+
+              # Redact sensitive data from evaluated messages.
+              #
+              # When enabled, values flagged by AI Guard are replaced before the messages are
+              # stored in the span or returned to the caller. Disable to skip redaction entirely.
+              #
+              # @default `DD_AI_GUARD_REDACTION_ENABLED`, otherwise `true`
+              # @return [Boolean]
+              option :redaction_enabled do |o|
+                o.type :bool
+                o.env Ext::ENV_AI_GUARD_REDACTION_ENABLED
+                o.default true
+              end
+
+              define_method(:instrument) do |integration_name|
+                return unless enabled # steep:ignore
+
+                if (registered_integration = Datadog::AIGuard::Contrib::Integration.registry[integration_name])
+                  klass = registered_integration.klass
+                  if klass.loaded? && klass.compatible?
+                    instance = klass.new
+                    instance.patcher.patch unless instance.patcher.patched?
+                  end
+                end
+              end
+
+              # AI Guard API endpoint path.
+              #
+              # @default `DD_AI_GUARD_ENDPOINT`, otherwise `nil`
+              # @return [String, nil]
+              option :endpoint do |o|
+                o.type :string, nilable: true
+                o.env Ext::ENV_AI_GUARD_ENDPOINT
+
+                o.setter do |value|
+                  next unless value
+
+                  uri = URI(value.to_s)
+                  raise ArgumentError, "Please provide an absolute URI that includes a protocol" unless uri.absolute?
+
+                  uri.to_s.delete_suffix("/")
+                end
+              end
+
+              # Datadog Application key.
+              #
+              # @default `DD_APP_KEY` environment variable, otherwise `nil`
+              # @return [String, nil]
+              option :app_key do |o|
+                o.type :string, nilable: true
+                o.env Ext::ENV_APP_KEY
+                o.skip_telemetry true
+              end
+
+              # Request timeout in milliseconds.
+              #
+              # @default `DD_AI_GUARD_TIMEOUT`, otherwise 10 000 ms
+              # @return [Integer]
+              option :timeout_ms do |o|
+                o.type :int
+                o.env Ext::ENV_AI_GUARD_TIMEOUT
+                o.default 10_000
+              end
+
+              # Maximum content size in bytes.
+              # Content that exceeds the maximum allowed size is truncated before
+              # being stored in the current span context.
+              #
+              # @default `DD_AI_GUARD_MAX_CONTENT_SIZE`, otherwise 524 228 bytes
+              # @return [Integer]
+              option :max_content_size_bytes do |o|
+                o.type :int
+                o.env Ext::ENV_AI_GUARD_MAX_CONTENT_SIZE
+                o.default 512 * 1024
+              end
+
+              # Maximum number of messages.
+              # Older messages are omitted once the message limit is reached.
+              #
+              # @default `DD_AI_GUARD_MAX_MESSAGES_LENGTH`, otherwise 16 messages
+              # @return [Integer]
+              option :max_messages_length do |o|
+                o.type :int
+                o.env Ext::ENV_AI_GUARD_MAX_MESSAGES_LENGTH
+                o.default 16
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/ai_guard/contrib/auto_instrument.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module Datadog
+  module AIGuard
+    module Contrib
+      # Auto-instrumentation for AI Guard integrations
+      module AutoInstrument
+        def self.patch_all
+          integrations = []
+
+          Datadog::AIGuard::Contrib::Integration.registry.each_value do |integration|
+            next unless integration.klass.auto_instrument?
+
+            integrations << integration.name
+          end
+
+          integrations.each do |integration_name|
+            Datadog.configuration.ai_guard.instrument(integration_name)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/ai_guard/contrib/integration.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+module Datadog
+  module AIGuard
+    module Contrib
+      # Base provides features that are shared across all integrations
+      module Integration
+        RegisteredIntegration = Struct.new(:name, :klass, :options)
+
+        @registry = {}
+
+        # Class-level methods for Integration
+        module ClassMethods
+          def register_as(name, options = {})
+            Integration.register(self, name, options)
+          end
+
+          def compatible?
+            true
+          end
+        end
+
+        def self.included(base)
+          base.extend(ClassMethods)
+        end
+
+        def self.register(integration, name, options)
+          @registry[name] = RegisteredIntegration.new(name, integration, options)
+        end
+
+        def self.registry
+          @registry
+        end
+      end
+    end
+  end
+end

data/lib/datadog/ai_guard/contrib/rack/integration.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+require_relative "../integration"
+require_relative "patcher"
+require_relative "request_middleware"
+
+module Datadog
+  module AIGuard
+    module Contrib
+      module Rack
+        # Rack integration for AI Guard
+        class Integration
+          include Datadog::AIGuard::Contrib::Integration
+
+          MINIMUM_VERSION = Gem::Version.new("1.1.0")
+
+          register_as :rack, auto_patch: false
+
+          def self.version
+            Gem.loaded_specs["rack"]&.version
+          end
+
+          def self.loaded?
+            !defined?(::Rack).nil?
+          end
+
+          def self.compatible?
+            super && !!(version&.>= MINIMUM_VERSION)
+          end
+
+          def self.auto_instrument?
+            false
+          end
+
+          def patcher
+            Patcher
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/ai_guard/contrib/rack/patcher.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+module Datadog
+  module AIGuard
+    module Contrib
+      module Rack
+        # Patcher for Rack integration
+        module Patcher
+          module_function
+
+          def patched?
+            !!Patcher.instance_variable_get(:@patched)
+          end
+
+          def target_version
+            Integration.version
+          end
+
+          def patch
+            Patcher.instance_variable_set(:@patched, true)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/ai_guard/contrib/rack/request_middleware.rb

@@ -0,0 +1,97 @@
+# frozen_string_literal: true
+
+require_relative "../../../tracing/client_ip"
+require_relative "../../../tracing/contrib/rack/header_collection"
+require_relative "../../../tracing/metadata/ext"
+require_relative "../../ext"
+
+module Datadog
+  module AIGuard
+    module Contrib
+      module Rack
+        # AI Guard Rack middleware.
+        #
+        # At request entry the middleware stores some request attributes for
+        # on the active trace under the `_dd.ai_guard.` prefix.
+        #
+        # Later when AI Guard evaluation is performed, those attributes are
+        # mirrored on AI Guard span.
+        class RequestMiddleware
+          def initialize(app, opt = {})
+            @app = app
+          end
+
+          def call(env)
+            trace = Datadog::Tracing.active_trace
+            return @app.call(env) unless trace
+
+            store_anomaly_detection_tags!(trace, env)
+
+            @app.call(env)
+          ensure
+            # @type var trace: Datadog::Tracing::TraceSegment?
+            # Steep: https://github.com/soutaro/steep/issues/919
+            if trace
+              tag_client_ip_on_request_span!(trace) if ai_guard_executed?(trace)
+
+              clean_up_ai_guard_temp_tags!(trace)
+            end
+          end
+
+          private
+
+          # steep:ignore:start
+          def store_anomaly_detection_tags!(trace, env)
+            remote_ip = env["REMOTE_ADDR"]
+            trace.set_tag(Datadog::AIGuard::Ext::TRACE_NETWORK_CLIENT_IP_TAG, remote_ip) if remote_ip
+
+            headers = Datadog::Tracing::Contrib::Rack::Header::RequestHeaderCollection.new(env)
+            resolved_client_ip = Datadog::Tracing::ClientIp.extract_client_ip(headers, remote_ip)
+            trace.set_tag(Datadog::AIGuard::Ext::TRACE_HTTP_CLIENT_IP_TAG, resolved_client_ip) if resolved_client_ip
+
+            user_agent = env["HTTP_USER_AGENT"]
+            trace.set_tag(Datadog::AIGuard::Ext::TRACE_HTTP_USERAGENT_TAG, user_agent) if user_agent
+          rescue => e
+            Datadog::AIGuard.telemetry&.report(e, description: "AI Guard: failed to get request attributes")
+          end
+          # steep:ignore:end
+
+          # steep:ignore:start
+          def clean_up_ai_guard_temp_tags!(trace)
+            Ext::TRACE_ANOMALY_DETECTION_TAGS.each do |tag|
+              trace.clear_tag(tag)
+            end
+
+            trace.clear_tag(Datadog::AIGuard::Ext::TRACE_EXECUTED_TAG)
+          end
+          # steep:ignore:end
+
+          # AI Guard's evaluation flow sets `_dd.ai_guard.executed` on the
+          # trace whenever an AI Guard span is created during the request.
+          # steep:ignore:start
+          def ai_guard_executed?(trace)
+            trace.get_tag(Datadog::AIGuard::Ext::TRACE_EXECUTED_TAG) == "1"
+          end
+          # steep:ignore:end
+
+          # steep:ignore:start
+          def tag_client_ip_on_request_span!(trace)
+            span = Datadog::Tracing.active_span
+            return unless span
+
+            client_ip = trace.get_tag(Datadog::AIGuard::Ext::TRACE_HTTP_CLIENT_IP_TAG)
+            if client_ip && span.get_tag(Datadog::Tracing::Metadata::Ext::HTTP::TAG_CLIENT_IP).nil?
+              span.set_tag(Datadog::Tracing::Metadata::Ext::HTTP::TAG_CLIENT_IP, client_ip)
+            end
+
+            network_client_ip = trace.get_tag(Datadog::AIGuard::Ext::TRACE_NETWORK_CLIENT_IP_TAG)
+            span["network.client.ip"] = network_client_ip if network_client_ip
+          rescue => e
+            Datadog::AIGuard.telemetry&.report(e, description: "AI Guard: failed to tag client IP on root span")
+          end
+          # steep:ignore:end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/ai_guard/contrib/rails/integration.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+require_relative "../integration"
+require_relative "patcher"
+
+module Datadog
+  module AIGuard
+    module Contrib
+      module Rails
+        # Rails integration for AI Guard
+        class Integration
+          include Datadog::AIGuard::Contrib::Integration
+
+          MINIMUM_VERSION = Gem::Version.new("4")
+
+          register_as :rails, auto_patch: false
+
+          def self.version
+            Gem.loaded_specs["railties"]&.version
+          end
+
+          def self.loaded?
+            !defined?(::Rails).nil?
+          end
+
+          def self.compatible?
+            super && !!(version&.>= MINIMUM_VERSION)
+          end
+
+          def self.auto_instrument?
+            true
+          end
+
+          def patcher
+            Patcher
+          end
+        end
+      end
+    end
+  end
+end