logstash-core 5.0.0.alpha6.snapshot5-java → 5.0.0-java

data/lib/logstash/environment.rb

@@ -17,9 +17,9 @@ module LogStash
 
   [
             Setting::String.new("node.name", Socket.gethostname),
-            Setting::String.new("path.config", nil, false),
+    Setting::NullableString.new("path.config", nil, false),
  Setting::WritableDirectory.new("path.data", ::File.join(LogStash::Environment::LOGSTASH_HOME, "data")),
-            Setting::String.new("config.string", nil, false),
+    Setting::NullableString.new("config.string", nil, false),
            Setting::Boolean.new("config.test_and_exit", false),
            Setting::Boolean.new("config.reload.automatic", false),
            Setting::Numeric.new("config.reload.interval", 3), # in seconds
@@ -31,12 +31,11 @@ module LogStash
            Setting::Numeric.new("pipeline.batch.delay", 5), # in milliseconds
            Setting::Boolean.new("pipeline.unsafe_shutdown", false),
                     Setting.new("path.plugins", Array, []),
-            Setting::String.new("interactive", nil, false),
+    Setting::NullableString.new("interactive", nil, false),
            Setting::Boolean.new("config.debug", false),
-            Setting::String.new("log.level", "warn", true, ["fatal", "error", "warn", "debug", "info", "trace"]),
+            Setting::String.new("log.level", "info", true, ["fatal", "error", "warn", "debug", "info", "trace"]),
            Setting::Boolean.new("version", false),
            Setting::Boolean.new("help", false),
-            Setting::String.new("path.log", nil, false),
             Setting::String.new("log.format", "plain", true, ["json", "plain"]),
             Setting::String.new("http.host", "127.0.0.1"),
             Setting::PortRange.new("http.port", 9600..9700),

data/lib/logstash/instrument/collector.rb

@@ -104,5 +104,9 @@ module LogStash module Instrument
       logger.debug("Collector: Sending snapshot to observers", :created_at => created_at) if logger.debug?
       notify_observers(snapshot_metric)
     end
+
+    def clear(keypath)
+      @metric_store.prune(keypath)
+    end
   end
 end; end

data/lib/logstash/instrument/metric_store.rb

@@ -80,8 +80,7 @@ module LogStash module Instrument
     # @param [Array] The path where values should be located
     # @return [Hash]
     def get_with_path(path)
-      key_paths = path.gsub(/^#{KEY_PATH_SEPARATOR}+/, "").split(KEY_PATH_SEPARATOR)
-      get(*key_paths)
+      get(*key_paths(path))
     end
 
     # Similar to `get_with_path` but use symbols instead of string
@@ -180,11 +179,28 @@ module LogStash module Instrument
     end
     alias_method :all, :each
 
+    def prune(path)
+      key_paths = key_paths(path).map {|k| k.to_sym }
+      @structured_lookup_mutex.synchronize do
+        keys_to_delete = @fast_lookup.keys.select {|namespace, _| (key_paths - namespace).empty? }
+        keys_to_delete.each {|k| @fast_lookup.delete(k) }
+        delete_from_map(@store, key_paths)
+      end
+    end
+
+    def size
+      @fast_lookup.size
+    end
+
     private
     def get_all
       @fast_lookup.values
     end
 
+    def key_paths(path)
+      path.gsub(/^#{KEY_PATH_SEPARATOR}+/, "").split(KEY_PATH_SEPARATOR)
+    end
+
     # This method take an array of keys and recursively search the metric store structure
     # and return a filtered hash of the structure. This method also take into consideration
     # getting two different branchs.
@@ -294,5 +310,14 @@ module LogStash module Instrument
       new_map = map.fetch_or_store(current) { Concurrent::Map.new }
       return fetch_or_store_namespace_recursively(new_map, namespaces_path, idx + 1)
     end
+
+    def delete_from_map(map, keys)
+      key = keys.first
+      if keys.size == 1
+        map.delete(key)
+      else
+        delete_from_map(map[key], keys[1..-1]) unless map[key].nil?
+      end
+    end
   end
 end; end

data/lib/logstash/logging/logger.rb

@@ -6,6 +6,8 @@ module LogStash
       java_import org.apache.logging.log4j.Level
       java_import org.apache.logging.log4j.LogManager
       java_import org.apache.logging.log4j.core.config.Configurator
+      @@config_mutex = Mutex.new
+      @@logging_context = nil
 
       def initialize(name)
         @logger = LogManager.getLogger(name)
@@ -59,13 +61,22 @@ module LogStash
         @logger.trace(message, data)
       end
 
-      # Point logging at a specific path.
       def self.configure_logging(level, path = LogManager::ROOT_LOGGER_NAME)
-        Configurator.setLevel(path, Level.toLevel(level))
-      end # def configure_logging
+        @@config_mutex.synchronize { Configurator.setLevel(path, Level.valueOf(level)) }
+      rescue Exception => e
+        raise ArgumentError, "invalid level[#{level}] for logger[#{path}]"
+      end
 
       def self.initialize(config_location)
-        Configurator.initialize(nil, config_location)
+        @@config_mutex.synchronize do
+          if @@logging_context.nil?
+            @@logging_context = Configurator.initialize(nil, config_location)
+          end
+        end
+      end
+
+      def self.get_logging_context
+        return @@logging_context
       end
     end
   end

data/lib/logstash/patches/puma.rb

@@ -0,0 +1,44 @@
+# encoding: utf-8
+#
+# Patch to replace the usage of STDERR and STDOUT
+# see: https://github.com/elastic/logstash/issues/5912
+module LogStash
+  class NullLogger
+    def self.debug(message)
+    end
+  end
+
+  # Puma uses by default the STDERR an the STDOUT for all his error
+  # handling, the server class accept custom a events object that can accept custom io object,
+  # so I just wrap the logger into an IO like object.
+  class IOWrappedLogger
+    def initialize(new_logger)
+      @logger_lock = Mutex.new
+      @logger = new_logger
+    end
+
+    def sync=(v)
+      # noop
+    end
+
+    def logger=(logger)
+      @logger_lock.synchronize { @logger = logger }
+    end
+
+    def puts(str)
+      # The logger only accept a str as the first argument
+      @logger_lock.synchronize { @logger.debug(str.to_s) }
+    end
+    alias_method :write, :puts
+    alias_method :<<, :puts
+  end
+
+end
+
+# Reopen the puma class to create a scoped STDERR and STDOUT
+# This operation is thread safe since its done at the class level
+# and force JRUBY to flush his classes cache.
+module Puma
+  STDERR = LogStash::IOWrappedLogger.new(LogStash::NullLogger)
+  STDOUT = LogStash::IOWrappedLogger.new(LogStash::NullLogger)
+end

data/lib/logstash/pipeline.rb

@@ -162,12 +162,12 @@ module LogStash; class Pipeline
     wait_inputs
     transition_to_stopped
 
-    @logger.info("Input plugins stopped! Will shutdown filter/output workers.")
+    @logger.debug("Input plugins stopped! Will shutdown filter/output workers.")
 
     shutdown_flusher
     shutdown_workers
 
-    @logger.info("Pipeline #{@pipeline_id} has been shutdown")
+    @logger.debug("Pipeline #{@pipeline_id} has been shutdown")
 
     # exit code
     return 0
@@ -257,14 +257,9 @@ module LogStash; class Pipeline
   def filter_batch(batch)
     batch.each do |event|
       if event.is_a?(Event)
-        filtered = filter_func(event)
-        filtered.each do |e|
-          #these are both original and generated events
-          if e.cancelled?
-            batch.cancel(e)
-          else
-            batch.merge(e)
-          end
+        filter_func(event).each do |e|
+          # these are both original and generated events
+          batch.merge(e) unless e.cancelled?
         end
       end
     end
@@ -375,9 +370,9 @@ module LogStash; class Pipeline
 
     before_stop.call if block_given?
 
-    @logger.info "Closing inputs"
+    @logger.debug "Closing inputs"
     @inputs.each(&:do_stop)
-    @logger.info "Closed inputs"
+    @logger.debug "Closed inputs"
   end # def shutdown
 
   # After `shutdown` is called from an external thread this is called from the main thread to
@@ -492,9 +487,7 @@ module LogStash; class Pipeline
   def flush_filters_to_batch(batch, options = {})
     options[:final] = batch.shutdown_signal_received?
     flush_filters(options) do |event|
-      if event.cancelled?
-        batch.cancel(event)
-      else
+      unless event.cancelled?
         @logger.debug? and @logger.debug("Pushing flushed events", :event => event)
         batch.merge(event)
       end

data/lib/logstash/runner.rb

@@ -19,11 +19,12 @@ require "logstash/version"
 
 class LogStash::Runner < Clamp::StrictCommand
   include LogStash::Util::Loggable
-  # The `path.settings` need to be defined in the runner instead of the `logstash-core/lib/logstash/environment.rb`
+  # The `path.settings` and `path.logs` need to be defined in the runner instead of the `logstash-core/lib/logstash/environment.rb`
   # because the `Environment::LOGSTASH_HOME` doesn't exist in the context of the `logstash-core` gem.
   #
   # See issue https://github.com/elastic/logstash/issues/5361
   LogStash::SETTINGS.register(LogStash::Setting::String.new("path.settings", ::File.join(LogStash::Environment::LOGSTASH_HOME, "config")))
+  LogStash::SETTINGS.register(LogStash::Setting::String.new("path.logs", ::File.join(LogStash::Environment::LOGSTASH_HOME, "logs")))
 
   # Node Settings
   option ["-n", "--node.name"], "NAME",
@@ -77,9 +78,10 @@ class LogStash::Runner < Clamp::StrictCommand
     :default => LogStash::SETTINGS.get_default("path.plugins")
 
   # Logging Settings
-  option ["-l", "--path.log"], "FILE",
+  option ["-l", "--path.logs"], "PATH",
     I18n.t("logstash.runner.flag.log"),
-    :attribute_name => "path.log"
+    :attribute_name => "path.logs",
+    :default => LogStash::SETTINGS.get_default("path.logs")
 
   option "--log.level", "LEVEL", I18n.t("logstash.runner.flag.log_level"),
     :default => LogStash::SETTINGS.get_default("log.level"),
@@ -163,20 +165,12 @@ class LogStash::Runner < Clamp::StrictCommand
     rescue => e
       # abort unless we're just looking for the help
       if (["--help", "-h"] & args).empty?
-        $stderr.puts "INFO: Logstash has a new settings file which defines start up time settings. This file is typically located in $LS_HOME/config or /etc/logstash. If you installed Logstash through a package and are starting it manually please specify the location to this settings file by passing in \"--path.settings=/path/..\" in the command line options"
+        $stderr.puts "INFO: Logstash requires a setting file which is typically located in $LS_HOME/config or /etc/logstash. If you installed Logstash through a package and are starting it manually please specify the location to this settings file by passing in \"--path.settings=/path/..\""
         $stderr.puts "ERROR: Failed to load settings file from \"path.settings\". Aborting... path.setting=#{LogStash::SETTINGS.get("path.settings")}, exception=#{e.class}, message=>#{e.message}"
         return 1
       end
     end
 
-    # Configure Logstash logging facility, this need to be done before everything else to
-    # make sure the logger has the correct settings and the log level is correctly defined.
-    # TODO(talevy): cleanly support `path.logs` setting in log4j
-    unless java.lang.System.getProperty("log4j.configurationFile")
-      log4j_config_location = setting("path.settings") + "/log4j2.properties"
-      LogStash::Logging::Logger::initialize(log4j_config_location)
-    end
-
     super(*[args])
   end
 
@@ -185,7 +179,23 @@ class LogStash::Runner < Clamp::StrictCommand
     require "logstash/util/java_version"
     require "stud/task"
 
+    # Configure Logstash logging facility, this need to be done before everything else to
+    # make sure the logger has the correct settings and the log level is correctly defined.
+    java.lang.System.setProperty("ls.logs", setting("path.logs"))
+    java.lang.System.setProperty("ls.log.format", setting("log.format"))
+    java.lang.System.setProperty("ls.log.level", setting("log.level"))
+    unless java.lang.System.getProperty("log4j.configurationFile")
+      log4j_config_location = ::File.join(setting("path.settings"), "log4j2.properties")
+      LogStash::Logging::Logger::initialize("file://" + log4j_config_location)
+    end
+    # override log level that may have been introduced from a custom log4j config file
     LogStash::Logging::Logger::configure_logging(setting("log.level"))
+    
+    # Adding this here because a ton of LS users install LS via packages and try to manually 
+    # start Logstash using bin/logstash. See #5986. I think not logging to console is good for 
+    # services, but until LS users re-learn that logs end up in path.logs, we should keep this 
+    # message. Otherwise we'll be answering the same question again and again.
+    puts "Sending Logstash logs to #{setting("path.logs")} which is now configured via log4j2.properties."
 
     if setting("config.debug") && logger.debug?
       logger.warn("--config.debug was specified, but log.level was not set to \'debug\'! No config info will be logged.")
@@ -194,14 +204,14 @@ class LogStash::Runner < Clamp::StrictCommand
     LogStash::Util::set_thread_name(self.class.name)
 
     if RUBY_VERSION < "1.9.2"
-      $stderr.puts "Ruby 1.9.2 or later is required. (You are running: " + RUBY_VERSION + ")"
+      logger.fatal "Ruby 1.9.2 or later is required. (You are running: " + RUBY_VERSION + ")"
       return 1
     end
 
     # Exit on bad java versions
     java_version = LogStash::Util::JavaVersion.version
     if LogStash::Util::JavaVersion.bad_java_version?(java_version)
-      $stderr.puts "Java version 1.8.0 or later is required. (You are running: #{java_version})"
+      logger.fatal "Java version 1.8.0 or later is required. (You are running: #{java_version})"
       return 1
     end
 
@@ -216,7 +226,9 @@ class LogStash::Runner < Clamp::StrictCommand
 
     return start_shell(setting("interactive"), binding) if setting("interactive")
 
-    @settings.format_settings.each {|line| logger.info(line) }
+    @settings.validate_all
+
+    @settings.format_settings.each {|line| logger.debug(line) }
 
     if setting("config.string").nil? && setting("path.config").nil?
       fail(I18n.t("logstash.runner.missing-configuration"))
@@ -233,6 +245,7 @@ class LogStash::Runner < Clamp::StrictCommand
       begin
         LogStash::Pipeline.new(config_str)
         puts "Configuration OK"
+        logger.info "Using config.test_and_exit mode. Config Validation Result: OK. Exiting Logstash"
         return 0
       rescue => e
         logger.fatal I18n.t("logstash.runner.invalid-configuration", :error => e.message)
@@ -252,7 +265,8 @@ class LogStash::Runner < Clamp::StrictCommand
     @agent_task = Stud::Task.new { @agent.execute }
 
     # no point in enabling config reloading before the agent starts
-    sighup_id = trap_sighup()
+    # also windows doesn't have SIGHUP. we can skip it
+    sighup_id = LogStash::Environment.windows? ? nil : trap_sighup()
 
     agent_return = @agent_task.wait
 
@@ -280,7 +294,7 @@ class LogStash::Runner < Clamp::StrictCommand
   def show_version
     show_version_logstash
 
-    if logger.is_info_enabled
+    if logger.info?
       show_version_ruby
       show_version_java if LogStash::Environment.jruby?
       show_gems if logger.debug?

data/lib/logstash/settings.rb

@@ -95,6 +95,12 @@ module LogStash
       self.merge(flatten_hash(settings))
     end
 
+    def validate_all
+      @settings.each do |name, setting|
+        setting.validate_value
+      end
+    end
+
     private
     def read_yaml(path)
       YAML.safe_load(IO.read(path)) || {}
@@ -123,8 +129,9 @@ module LogStash
       @validator_proc = validator_proc
       @value = nil
       @value_is_set = false
+      @strict = strict
 
-      validate(default) if strict
+      validate(default) if @strict
       @default = default
     end
 
@@ -136,8 +143,12 @@ module LogStash
       @value_is_set
     end
 
+    def strict?
+      @strict
+    end
+
     def set(value)
-      validate(value)
+      validate(value) if @strict
       @value = value
       @value_is_set = true
       @value
@@ -167,12 +178,18 @@ module LogStash
       self.to_hash == other.to_hash
     end
 
-    private
-    def validate(value)
-      if !value.is_a?(@klass)
-        raise ArgumentError.new("Setting \"#{@name}\" must be a #{@klass}. Received: #{value} (#{value.class})")
-      elsif @validator_proc && !@validator_proc.call(value)
-        raise ArgumentError.new("Failed to validate setting \"#{@name}\" with value: #{value}")
+    def validate_value
+      validate(value)
+    end
+
+    protected
+    def validate(input)
+      if !input.is_a?(@klass)
+        raise ArgumentError.new("Setting \"#{@name}\" must be a #{@klass}. Received: #{input} (#{input.class})")
+      end
+
+      if @validator_proc && !@validator_proc.call(input)
+        raise ArgumentError.new("Failed to validate setting \"#{@name}\" with value: #{input}")
       end
     end
 
@@ -351,6 +368,13 @@ module LogStash
       end
     end
 
+    class NullableString < String
+      def validate(value)
+        return if value.nil?
+        super(value)
+      end
+    end
+
     class ExistingFilePath < Setting
       def initialize(name, default=nil, strict=true)
         super(name, ::String, default, strict) do |file_path|
@@ -364,7 +388,7 @@ module LogStash
     end
 
     class WritableDirectory < Setting
-      def initialize(name, default=nil, strict=true)
+      def initialize(name, default=nil, strict=false)
         super(name, ::String, default, strict) do |path|
           if ::File.directory?(path) && ::File.writable?(path)
             true
@@ -378,3 +402,4 @@ module LogStash
 
   SETTINGS = Settings.new
 end
+

data/lib/logstash/util/wrapped_synchronous_queue.rb

@@ -90,9 +90,14 @@ module LogStash; module Util
       def take_batch
         @mutex.synchronize do
           batch = ReadBatch.new(@queue, @batch_size, @wait_for)
-          add_starting_metrics(batch)
           set_current_thread_inflight_batch(batch)
-          start_clock
+
+          # We dont actually have any events to work on so lets
+          # not bother with recording metrics for them
+          if batch.size > 0
+            add_starting_metrics(batch)
+            start_clock
+          end
           batch
         end
       end
@@ -116,8 +121,10 @@ module LogStash; module Util
       end
 
       def stop_clock
-        @inflight_clocks[Thread.current].each(&:stop)
-        @inflight_clocks.delete(Thread.current)
+        unless @inflight_clocks[Thread.current].nil?
+          @inflight_clocks[Thread.current].each(&:stop)
+          @inflight_clocks.delete(Thread.current)
+        end
       end
 
       def add_starting_metrics(batch)
@@ -141,7 +148,10 @@ module LogStash; module Util
         @shutdown_signal_received = false
         @flush_signal_received = false
         @originals = Hash.new
-        @cancelled = Hash.new
+
+        # TODO: disabled for https://github.com/elastic/logstash/issues/6055 - will have to properly refactor
+        # @cancelled = Hash.new
+
         @generated = Hash.new
         @iterating_temp = Hash.new
         @iterating = false # Atomic Boolean maybe? Although batches are not shared across threads
@@ -161,17 +171,22 @@ module LogStash; module Util
       end
 
       def cancel(event)
-        @cancelled[event] = true
+        # TODO: disabled for https://github.com/elastic/logstash/issues/6055 - will have to properly refactor
+        raise("cancel is unsupported")
+        # @cancelled[event] = true
       end
 
       def each(&blk)
         # take care not to cause @originals or @generated to change during iteration
         @iterating = true
+
+        # below the checks for @cancelled.include?(e) have been replaced by e.cancelled?
+        # TODO: for https://github.com/elastic/logstash/issues/6055 = will have to properly refactor
         @originals.each do |e, _|
-          blk.call(e) unless @cancelled.include?(e)
+          blk.call(e) unless e.cancelled?
         end
         @generated.each do |e, _|
-          blk.call(e) unless @cancelled.include?(e)
+          blk.call(e) unless e.cancelled?
         end
         @iterating = false
         update_generated
@@ -190,7 +205,9 @@ module LogStash; module Util
       end
 
       def cancelled_size
-        @cancelled.size
+      # TODO: disabled for https://github.com/elastic/logstash/issues/6055 = will have to properly refactor
+      raise("cancelled_size is unsupported ")
+        # @cancelled.size
       end
 
       def shutdown_signal_received?