appsignal 5.0.0.rc.1-java → 5.0.0.rc.2-java

data/lib/appsignal/cli/diagnose.rb

@@ -189,13 +189,19 @@ module Appsignal
 
         def configure_appsignal(options)
           env_option = options.fetch(:environment, nil)
+          # Mark app as Rails app
+          data[:app][:rails] = true if rails_present?
           # Try and load the Rails app, if any.
           # This will configure AppSignal through the config file or an
           # initializer.
-          require_rails_app_if_present(env_option)
+          require_rails_app_if_present(env_option) do |error|
+            data[:app][:load_error] =
+              "#{error.class}: #{error.message}\n#{error.backtrace.join("\n")}"
+          end
 
           # No config loaded yet, try loading as normal
           Appsignal._load_config!(env_option) unless Appsignal.config
+          Appsignal.config.apply_overrides
           Appsignal._start_logger
           Appsignal.config.write_to_environment
           Appsignal.internal_logger.info("Starting AppSignal diagnose")
@@ -467,16 +473,7 @@ module Appsignal
           config = Appsignal.config
           data[:config] = {
             :options => config.config_hash.merge(:env => config.env),
-            :sources => {
-              :default => Appsignal::Config::DEFAULT_CONFIG,
-              :system => config.system_config,
-              :loaders => config.loaders_config,
-              :initial => config.initial_config,
-              :file => config.file_config,
-              :env => config.env_config,
-              :override => config.override_config,
-              :dsl => config.dsl_config
-            }
+            :sources => config.config_sources
           }
           print_config_options(config)
         end
@@ -627,43 +624,6 @@ module Appsignal
           puts "    Read error: #{path[:read_error]}"
           print_empty_line
         end
-
-        def print_empty_line
-          puts "\n"
-        end
-
-        def require_rails_app_if_present(env_option)
-          return unless rails_present?
-
-          # Set the environment given as an option to the diagnose CLI so the
-          # Rails app uses it when loaded.
-          ENV["_APPSIGNAL_CONFIG_FILE_ENV"] = env_option
-          # Mark app as Rails app
-          data[:app][:rails] = true
-          # Manually require the railtie, because it wasn't loaded when the CLI
-          # started and AppSignal loaded, because the `Rails` constant wasn't
-          # present.
-          require "appsignal/integrations/railtie"
-          # Start the Rails app, including railties and initializers.
-          require Appsignal::Utils::RailsHelper.environment_config_path
-        rescue LoadError, StandardError => error
-          print_empty_line
-          puts "ERROR: Error encountered while loading the Rails app"
-          puts "#{error.class}: #{error.message}"
-          puts error.backtrace
-          data[:app][:load_error] =
-            "#{error.class}: #{error.message}\n#{error.backtrace.join("\n")}"
-        ensure
-          ENV.delete("_APPSIGNAL_CONFIG_FILE_ENV")
-        end
-
-        def rails_present?
-          # Try and load the Rails gem
-          require "rails"
-          true
-        rescue LoadError
-          false
-        end
       end
     end
   end

data/lib/appsignal/cli/helpers.rb

@@ -36,6 +36,51 @@ module Appsignal
         "\e[#{color_code}m#{text}\e[#{reset_color_code}m"
       end
 
+      def print_empty_line
+        puts "\n"
+      end
+
+      def rails_present?
+        require "rails"
+        true
+      rescue LoadError
+        false
+      end
+
+      # The Rails gem being loadable says nothing about the directory the
+      # command runs in, so check for the app itself as well.
+      def rails_app_present?
+        rails_present? &&
+          File.exist?(Appsignal::Utils::RailsHelper.environment_config_path)
+      end
+
+      def load_rails_app(environment)
+        # Pass the environment given as a command line option to the app, so
+        # that the AppSignal config file uses it when the app loads it.
+        ENV["_APPSIGNAL_CONFIG_FILE_ENV"] = environment
+        # Require the railtie manually. It was not loaded when AppSignal
+        # loaded, because the `Rails` constant was not present at that point.
+        require "appsignal/integrations/railtie"
+        # Start the Rails app, including its railties and initializers.
+        require Appsignal::Utils::RailsHelper.environment_config_path
+      ensure
+        ENV.delete("_APPSIGNAL_CONFIG_FILE_ENV")
+      end
+
+      # Yields the error when the app fails to load, so that a command can
+      # report it in its own way.
+      def require_rails_app_if_present(environment)
+        return unless rails_app_present?
+
+        load_rails_app(environment)
+      rescue LoadError, StandardError => error
+        print_empty_line
+        puts "ERROR: Error encountered while loading the Rails app"
+        puts "#{error.class}: #{error.message}"
+        puts error.backtrace
+        yield error if block_given?
+      end
+
       def periods
         3.times do
           print "."

data/lib/appsignal/config.rb

@@ -134,6 +134,11 @@ module Appsignal
       :instrument_sequel => true,
       :instrument_shoryuken => true,
       :instrument_sidekiq => true,
+      :keep_request_environment => [],
+      :keep_request_headers => %w[
+        accept accept-charset accept-encoding accept-language cache-control
+        connection content-length range
+      ],
       :log => "file",
       :logging_endpoint => "https://appsignal-endpoint.net",
       :ownership_set_namespace => false,
@@ -146,10 +151,10 @@ module Appsignal
       ],
       :response_headers => [],
       :send_environment_metadata => true,
-      :send_function_parameters => nil,
+      :send_function_parameters => true,
       :send_params => true,
-      :send_request_payload => nil,
-      :send_request_query_parameters => nil,
+      :send_request_payload => true,
+      :send_request_query_parameters => true,
       :send_session_data => true,
       :service_name => nil,
       :sidekiq_report_errors => "all",
@@ -171,6 +176,18 @@ module Appsignal
       "trace" => ::Logger::DEBUG
     }.freeze
 
+    # Environment variables set by deployment platforms that name the
+    # revision that is being deployed, in the order the agent reads them.
+    # The agent detects the revision this way as well, but only for the data
+    # it sends itself, so the gem has to do it for collector mode.
+    # @!visibility private
+    PLATFORM_REVISION_ENV_VARS = [
+      "HEROKU_SLUG_COMMIT",
+      "RENDER_GIT_COMMIT",
+      "KAMAL_VERSION",
+      "CONTAINER_VERSION" # Scalingo
+    ].freeze
+
     # @!visibility private
     STRING_OPTIONS = {
       :activejob_report_errors => "APPSIGNAL_ACTIVEJOB_REPORT_ERRORS",
@@ -254,6 +271,8 @@ module Appsignal
       :ignore_errors => "APPSIGNAL_IGNORE_ERRORS",
       :ignore_logs => "APPSIGNAL_IGNORE_LOGS",
       :ignore_namespaces => "APPSIGNAL_IGNORE_NAMESPACES",
+      :keep_request_environment => "APPSIGNAL_KEEP_REQUEST_ENVIRONMENT",
+      :keep_request_headers => "APPSIGNAL_KEEP_REQUEST_HEADERS",
       :request_headers => "APPSIGNAL_REQUEST_HEADERS",
       :response_headers => "APPSIGNAL_RESPONSE_HEADERS"
     }.freeze
@@ -275,15 +294,14 @@ module Appsignal
     # @!visibility private
     MIN_RUBY_VERSION_FOR_COLLECTOR_MODE = "3.1"
 
-    # Configuration options that only have an effect when the integration is
-    # in collector mode. When the agent is in use, setting any of these emits
-    # a warning at startup.
     # @!visibility private
     COLLECTOR_ONLY_OPTIONS = [
       :filter_attributes,
       :filter_function_parameters,
       :filter_request_payload,
       :filter_request_query_parameters,
+      :keep_request_environment,
+      :keep_request_headers,
       :response_headers,
       :send_function_parameters,
       :send_request_payload,
@@ -291,16 +309,45 @@ module Appsignal
       :service_name
     ].freeze
 
-    # Existing AppSignal options that only affect the agent's handling of
-    # trace data. In collector mode these don't filter anything; users need
-    # their collector-mode equivalents (see COLLECTOR_ONLY_OPTIONS).
     # @!visibility private
-    AGENT_ONLY_TRACE_OPTIONS = [
-      :filter_metadata,
-      :filter_parameters,
-      :send_params
+    COLLECTOR_ONLY_REPLACED_OPTIONS = {
+      :filter_function_parameters => :filter_parameters,
+      :filter_request_payload => :filter_parameters,
+      :filter_request_query_parameters => :filter_parameters,
+      :keep_request_environment => :request_headers,
+      :keep_request_headers => :request_headers,
+      :send_function_parameters => :send_params,
+      :send_request_payload => :send_params,
+      :send_request_query_parameters => :send_params
+    }.freeze
+
+    # @!visibility private
+    DEPRECATED_COLLECTOR_OPTIONS = {
+      :filter_parameters => {
+        :filter_request_payload => :derived_as_is,
+        :filter_function_parameters => :derived_as_is,
+        :filter_request_query_parameters => :derived_as_is
+      },
+      :request_headers => {
+        :keep_request_headers => :derived_header_names,
+        :keep_request_environment => :derived_environment_keys
+      },
+      :send_params => {
+        :send_request_payload => :derived_as_is,
+        :send_request_query_parameters => :derived_as_is,
+        :send_function_parameters => :derived_as_is
+      }
+    }.freeze
+
+    # @!visibility private
+    SOURCE_ORDER = [
+      :default, :derived, :system, :loaders, :initial, :file, :env, :override,
+      :dsl
     ].freeze
 
+    # @!visibility private
+    APPLICATION_SOURCES = [:initial, :file, :env, :dsl].freeze
+
     # @!visibility private
     attr_reader :root_path, :env, :config_hash
 
@@ -312,8 +359,23 @@ module Appsignal
     #
     # Used by the diagnose report to list which value was read from which source.
     # @!visibility private
-    attr_reader :system_config, :loaders_config, :initial_config, :file_config,
-      :env_config, :override_config, :dsl_config
+    attr_reader :derived_config, :system_config, :loaders_config,
+      :initial_config, :file_config, :env_config, :override_config, :dsl_config
+
+    # @!visibility private
+    def config_sources
+      {
+        :default => DEFAULT_CONFIG,
+        :derived => derived_config,
+        :system => system_config,
+        :loaders => loaders_config,
+        :initial => initial_config,
+        :file => file_config,
+        :env => env_config,
+        :override => override_config,
+        :dsl => dsl_config
+      }
+    end
 
     # Initialize a new AppSignal configuration object.
     #
@@ -349,6 +411,7 @@ module Appsignal
       @initial_config = {}
       @file_config = {}
       @env_config = {}
+      @derived_config = {}
       @override_config = {}
       @dsl_config = {} # Can be set using `Appsignal.configure`
 
@@ -569,6 +632,10 @@ module Appsignal
       ENV["_APPSIGNAL_ENABLE_HOST_METRICS"]          = config_hash[:enable_host_metrics].to_s
       ENV["_APPSIGNAL_ENABLE_STATSD"]                = config_hash[:enable_statsd].to_s
       ENV["_APPSIGNAL_ENABLE_NGINX_METRICS"]         = config_hash[:enable_nginx_metrics].to_s
+      # The gem never sends OpenTelemetry data to the agent, so it has no
+      # reason to listen for it. Written out rather than left to the agent's
+      # own default, so a value set in the environment cannot turn it on.
+      ENV["_APPSIGNAL_ENABLE_OPENTELEMETRY_HTTP"]    = "false"
       ENV["_APPSIGNAL_APP_ENV"]                      = env
       ENV["_APPSIGNAL_FILES_WORLD_ACCESSIBLE"]       = config_hash[:files_world_accessible].to_s
       ENV["_APPSIGNAL_FILTER_PARAMETERS"]            = config_hash[:filter_parameters].join(",")
@@ -585,6 +652,7 @@ module Appsignal
       ENV["_APPSIGNAL_LOG_LEVEL"]                    = config_hash[:log_level]
       ENV["_APPSIGNAL_LOG_FILE_PATH"]                = log_file_path.to_s if log_file_path
       ENV["_APPSIGNAL_LOGGING_ENDPOINT"]             = config_hash[:logging_endpoint]
+      ENV["_APPSIGNAL_PLATFORM"]                     = config_hash[:platform].to_s
       ENV["_APPSIGNAL_PROCESS_NAME"]                 = $PROGRAM_NAME
       ENV["_APPSIGNAL_PUSH_API_ENDPOINT"]            = config_hash[:endpoint]
       ENV["_APPSIGNAL_PUSH_API_KEY"]                 = config_hash[:push_api_key]
@@ -604,9 +672,11 @@ module Appsignal
       merge(options)
     end
 
-    # Apply any overrides for invalid settings.
     # @!visibility private
     def apply_overrides
+      @derived_config = determine_derived
+      merge(derived_config)
+
       @override_config = determine_overrides
       merge(override_config)
     end
@@ -645,16 +715,11 @@ module Appsignal
     # @!visibility private
     def warn_for_mode_mismatch
       if collector_mode_configured?
-        warn_user_modified(AGENT_ONLY_TRACE_OPTIONS) do |option|
-          "The collector is in use. The '#{option}' configuration option is " \
-            "only used by the agent for trace data and will be ignored."
+        warn_user_modified(DEPRECATED_COLLECTOR_OPTIONS.keys) do |option|
+          deprecated_collector_option_message(option)
         end
       else
-        warn_user_modified(COLLECTOR_ONLY_OPTIONS) do |option|
-          "The agent is in use. The '#{option}' configuration option is " \
-            "only used by the collector and will be ignored. Set " \
-            "'collector_endpoint' to use the collector."
-        end
+        warn_collector_only_options
       end
     end
 
@@ -679,12 +744,49 @@ module Appsignal
 
     private
 
-    # Yield a warning for each option in `options` whose effective value
-    # differs from the default. Setting an option to its default value is
-    # a no-op, so we don't warn about it.
+    def deprecated_collector_option_message(option)
+      replacements = DEPRECATED_COLLECTOR_OPTIONS.fetch(option).keys
+      message = "The collector is in use. The '#{option}' configuration " \
+        "option is deprecated in collector mode. It is replaced by " \
+        "#{quoted_option_list(replacements)}."
+
+      derived = replacements.select { |name| derived_config.key?(name) }
+      return message if derived.empty?
+
+      values = derived.map { |name| "\n  #{name}: #{derived_config[name].inspect}" }
+      "#{message} Set these options to keep reporting what this application " \
+        "reports now:#{values.join}"
+    end
+
+    def quoted_option_list(names)
+      names = names.map { |name| "'#{name}'" }
+      return names.first if names.length == 1
+
+      "#{names[0..-2].join(", ")} and #{names.last}"
+    end
+
+    def warn_collector_only_options
+      options = COLLECTOR_ONLY_OPTIONS.select { |option| configured?(option) }
+      return if options.empty?
+
+      options.each do |option|
+        logger.warn(
+          "The agent is in use. The '#{option}' configuration option is " \
+            "only used by the collector and will be ignored."
+        )
+
+        replacement = COLLECTOR_ONLY_REPLACED_OPTIONS[option]
+        logger.warn("Use the '#{replacement}' option instead.") if replacement
+      end
+
+      logger.info(
+        "To use the collector, set the 'collector_endpoint' configuration option."
+      )
+    end
+
     def warn_user_modified(options)
       options.each do |option|
-        next if config_hash[option] == DEFAULT_CONFIG[option]
+        next unless configured?(option)
 
         logger.warn(yield(option))
       end
@@ -710,13 +812,57 @@ module Appsignal
 
         hash[:enable_at_exit_hook] = "always" if Appsignal::Extension.running_in_container?
 
-        # Set revision from REVISION file if present in project root
-        # This helps with Capistrano and Hatchbox.io deployments
-        revision_from_file = detect_revision_from_file
-        hash[:revision] = revision_from_file if revision_from_file
+        # Set the revision from a REVISION file in the project root, which
+        # helps with Capistrano and Hatchbox.io deployments, or from the
+        # environment variable the deployment platform sets.
+        revision = detect_revision_from_file || detect_revision_from_platform
+        hash[:revision] = revision if revision
+
+        hostname = detect_hostname
+        hash[:hostname] = hostname if hostname
+
+        platform = detect_platform
+        hash[:platform] = platform if platform
       end
     end
 
+    # Detect the platform the application is deployed on, the way the agent
+    # does. The agent only detects it for the data it reports itself, so the
+    # gem has to do it for collector mode. There is no config option for it,
+    # because these are the only two platforms that can be recognized.
+    def detect_platform
+      return "dokku" unless ENV.fetch("DOKKU_ROOT", nil).to_s.empty?
+      return "heroku" unless ENV.fetch("DYNO", nil).to_s.empty?
+
+      nil
+    end
+
+    # Detect the hostname the way the agent does: the Heroku dyno name first,
+    # then the name the host reports for itself. The agent only detects it for
+    # the data it reports itself, so the gem has to do it for collector mode.
+    def detect_hostname
+      dyno = ENV.fetch("DYNO", nil)
+      return dyno unless dyno.to_s.empty?
+
+      hostname = Socket.gethostname
+      hostname unless hostname.to_s.empty?
+    rescue SystemCallError => e
+      logger.debug "Unable to detect the hostname: #{e.class}: #{e.message}"
+      nil
+    end
+
+    def detect_revision_from_platform
+      PLATFORM_REVISION_ENV_VARS.each do |env_var|
+        revision = ENV.fetch(env_var, nil)
+        next if revision.to_s.empty?
+
+        logger.debug "Detected revision from the #{env_var} environment variable"
+        return revision
+      end
+
+      nil
+    end
+
     def detect_revision_from_file
       return unless root_path
 
@@ -854,9 +1000,68 @@ module Appsignal
         config[:sidekiq_report_errors] = "all"
       end
 
+      # A config file holding a YAML null gives an array option a `nil`, which
+      # nothing downstream expects. Correct it to the empty list the option means.
+      ARRAY_OPTIONS.each_key do |option|
+        config[option] = [] if config_hash[option].nil?
+      end
+
       config
     end
 
+    def determine_derived
+      derived = {}
+
+      DEPRECATED_COLLECTOR_OPTIONS.each do |option, replacements|
+        next unless configured?(option)
+
+        replacements.each do |replacement, derivation|
+          next if set_above_derived?(replacement)
+
+          derived[replacement] = send(derivation, config_hash[option])
+        end
+      end
+
+      derived
+    end
+
+    def derived_as_is(value)
+      value
+    end
+
+    def derived_header_names(value)
+      Array(value).filter_map do |key|
+        Appsignal::Utils::RequestHeaders.header_name(key)
+      end
+    end
+
+    def derived_environment_keys(value)
+      Array(value).reject do |key|
+        Appsignal::Utils::RequestHeaders.header_name(key) ||
+          Appsignal::Utils::RequestHeaders::TRANSLATED_ENV_KEYS.include?(key)
+      end
+    end
+
+    def configured?(option)
+      config_hash[option] != DEFAULT_CONFIG[option] && user_set?(option)
+    end
+
+    def user_set?(option)
+      set_by_any?(APPLICATION_SOURCES, option)
+    end
+
+    def set_above_derived?(option)
+      set_by_any?(sources_above_derived, option)
+    end
+
+    def sources_above_derived
+      SOURCE_ORDER[(SOURCE_ORDER.index(:derived) + 1)..] - [:override]
+    end
+
+    def set_by_any?(sources, option)
+      sources.any? { |source| config_sources.fetch(source).key?(option) }
+    end
+
     def merge(new_config)
       new_config.each do |key, value|
         logger.debug("Config key '#{key}' is being overwritten") unless config_hash[key].nil?
@@ -881,14 +1086,19 @@ module Appsignal
     # @see AppSignal Ruby gem configuration
     #   https://docs.appsignal.com/ruby/configuration.html
     class ConfigDSL
-      # @!visibility private
-      # @return [Hash] Hash containing the DSL option values
-      attr_reader :dsl_options
-
       # @!visibility private
       def initialize(config)
         @config = config
         @dsl_options = {}
+        @assigned_options = Set.new
+      end
+
+      # @!visibility private
+      # @return [Hash] Hash containing the DSL option values
+      def dsl_options
+        @dsl_options.reject do |key, value|
+          !@assigned_options.include?(key) && unchanged_option?(key, value)
+        end
       end
 
       # Returns the application's root path.
@@ -1074,6 +1284,12 @@ module Appsignal
       #   @return [Array<String>] Ignore log messages by substrings
       # @!attribute [rw] ignore_namespaces
       #   @return [Array<String>] Ignore traces by namespaces
+      # @!attribute [rw] keep_request_environment
+      #   @return [Array<String>] Rack environment keys to report in collector
+      #     mode, named the way Rack names them
+      # @!attribute [rw] keep_request_headers
+      #   @return [Array<String>] HTTP request headers to report in collector
+      #     mode, named the way OpenTelemetry names them
       # @!attribute [rw] request_headers
       #   @return [Array<String>] HTTP request headers to include in error reports
 
@@ -1128,14 +1344,23 @@ module Appsignal
         if @dsl_options.key?(key)
           @dsl_options[key]
         else
-          @dsl_options[key] = @config[key].dup
+          @dsl_options[key] = initial_option_value(key)
         end
       end
 
       def update_option(key, value)
+        @assigned_options << key
         @dsl_options[key] = value
       end
 
+      def initial_option_value(key)
+        @config[key].dup
+      end
+
+      def unchanged_option?(key, value)
+        value == initial_option_value(key)
+      end
+
       # Parse tags from various input formats and validate values
       # @param value [Hash, nil] input tags
       # @return [Hash] validated tags with string keys

data/lib/appsignal/demo.rb

@@ -77,21 +77,23 @@ module Appsignal
       end
 
       def add_headers_to(transaction)
-        transaction.add_headers(
+        transaction.add_request_headers(
+          "accept" => "text/html,application/xhtml+xml",
+          "accept-encoding" => "gzip, deflate, sdch",
+          "accept-language" => "en-US,en;q=0.8,nl;q=0.6",
+          "cache-control" => "max-age=0",
+          "connection" => "keep-alive",
+          "user-agent" => "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0)",
+          "referer" => "http://appsignal.com/accounts"
+        )
+        transaction.add_request_environment(
           "REMOTE_ADDR" => "127.0.0.1",
           "REQUEST_METHOD" => "GET",
           "SERVER_NAME" => "localhost",
           "SERVER_PORT" => "80",
           "SERVER_PROTOCOL" => "HTTP/1.1",
           "REQUEST_PATH" => "/hello",
-          "PATH_INFO" => "/hello",
-          "HTTP_ACCEPT" => "text/html,application/xhtml+xml",
-          "HTTP_ACCEPT_ENCODING" => "gzip, deflate, sdch",
-          "HTTP_ACCEPT_LANGUAGE" => "en-US,en;q=0.8,nl;q=0.6",
-          "HTTP_CACHE_CONTROL" => "max-age=0",
-          "HTTP_CONNECTION" => "keep-alive",
-          "HTTP_USER_AGENT" => "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0)",
-          "HTTP_REFERER" => "http://appsignal.com/accounts"
+          "PATH_INFO" => "/hello"
         )
       end
     end