logstash-core 5.0.2-java → 5.1.1.1-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 5f86c1c793b45c1e8757388c7520d098f6f1bb7d
-  data.tar.gz: b05ed9d9784cd5e56b67df54a2c5786cbcd86fed
+  metadata.gz: c1c528ad65660126cc95b61d32483ba720122505
+  data.tar.gz: ed0b7dbcd0d18786caa99f1eacc6b44cf6def997
 SHA512:
-  metadata.gz: 1edcabdf1f7b5176c1de416af340ce427e04280af8c1655bff40ba6e5f32d07c9866ff9dbe98a212fc37d4a39657d9be22cad2a7dcde81150909ae6bba0158b5
-  data.tar.gz: 088175a7b550c772cb56b982bc0d37de92e9ec01db6be34e784d525b35fd0a84b530f1d01f2f52b5b32e09cb72b628b9e1e55f8402a8d52245c490919f9678c8
+  metadata.gz: acd07b85def1d3754b60f147c5403a7cb624e0bee8a812dfffe7e6dc09ecb2b651d8a689f243a2efaec965b1a3b4a42eafe1f9d442dfa5626e9cc9ae3713ef90
+  data.tar.gz: 410e63863b1679c7c6be251b3a08f278cb990f77fc26e0fc2236e79a87eeb7a154553ded30876cf3983aa4d4b2c654592c9a889b4cffa8fb8978822f193a9716

data/gemspec_jars.rb

@@ -0,0 +1,9 @@
+# This file is generated by Gradle as part of the build process. It extracts the build.gradle
+# runtime dependencies to generate this gemspec dependencies file to be eval'ed by the gemspec
+# for the jar-dependencies requirements.
+
+gem.requirements << "jar org.apache.logging.log4j:log4j-1.2-api, 2.6.2"
+gem.requirements << "jar org.apache.logging.log4j:log4j-api, 2.6.2"
+gem.requirements << "jar org.apache.logging.log4j:log4j-core, 2.6.2"
+gem.requirements << "jar com.fasterxml.jackson.core:jackson-core, 2.7.4"
+gem.requirements << "jar com.fasterxml.jackson.core:jackson-databind, 2.7.4"

data/lib/logstash-core/logstash-core.rb

@@ -1,3 +1,25 @@
 # encoding: utf-8
+
+require "java"
+
 module LogStash
 end
+
+require "logstash-core_jars"
+
+# local dev setup
+classes_dir = File.expand_path("../../../build/classes/main", __FILE__)
+resources_dir = File.expand_path("../../../build/resources/main", __FILE__)
+
+if File.directory?(classes_dir) && File.directory?(resources_dir)
+  # if in local dev setup, add target to classpath
+  $CLASSPATH << classes_dir unless $CLASSPATH.include?(classes_dir)
+  $CLASSPATH << resources_dir unless $CLASSPATH.include?(resources_dir)
+else
+  # otherwise use included jar
+  begin
+    require "logstash-core/logstash-core.jar"
+  rescue Exception => e
+    raise("Error loading logstash-core/logstash-core.jar file, cause: #{e.message}")
+  end
+end

data/lib/logstash-core/version.rb

@@ -5,4 +5,4 @@
 # Note to authors: this should not include dashes because 'gem' barfs if
 # you include a dash in the version string.
 
-LOGSTASH_CORE_VERSION = "5.0.2"
+LOGSTASH_CORE_VERSION = "5.1.1.1"

data/lib/logstash-core_jars.rb

@@ -0,0 +1,20 @@
+# this is a generated file, to avoid over-writing it just delete this comment
+begin
+  require 'jar_dependencies'
+rescue LoadError
+  require 'org/apache/logging/log4j/log4j-core/2.6.2/log4j-core-2.6.2.jar'
+  require 'org/apache/logging/log4j/log4j-api/2.6.2/log4j-api-2.6.2.jar'
+  require 'com/fasterxml/jackson/core/jackson-core/2.7.4/jackson-core-2.7.4.jar'
+  require 'org/apache/logging/log4j/log4j-1.2-api/2.6.2/log4j-1.2-api-2.6.2.jar'
+  require 'com/fasterxml/jackson/core/jackson-annotations/2.7.0/jackson-annotations-2.7.0.jar'
+  require 'com/fasterxml/jackson/core/jackson-databind/2.7.4/jackson-databind-2.7.4.jar'
+end
+
+if defined? Jars
+  require_jar( 'org.apache.logging.log4j', 'log4j-core', '2.6.2' )
+  require_jar( 'org.apache.logging.log4j', 'log4j-api', '2.6.2' )
+  require_jar( 'com.fasterxml.jackson.core', 'jackson-core', '2.7.4' )
+  require_jar( 'org.apache.logging.log4j', 'log4j-1.2-api', '2.6.2' )
+  require_jar( 'com.fasterxml.jackson.core', 'jackson-annotations', '2.7.0' )
+  require_jar( 'com.fasterxml.jackson.core', 'jackson-databind', '2.7.4' )
+end

data/lib/logstash/agent.rb

@@ -9,6 +9,7 @@ require "logstash/instrument/collector"
 require "logstash/instrument/metric"
 require "logstash/pipeline"
 require "logstash/webserver"
+require "logstash/event_dispatcher"
 require "stud/trap"
 require "logstash/config/loader"
 require "uri"
@@ -21,12 +22,12 @@ class LogStash::Agent
   include LogStash::Util::Loggable
   STARTED_AT = Time.now.freeze
 
-  attr_reader :metric, :node_name, :pipelines, :settings, :webserver
+  attr_reader :metric, :name, :pipelines, :settings, :webserver, :dispatcher
   attr_accessor :logger
 
   # initialize method for LogStash::Agent
   # @param params [Hash] potential parameters are:
-  #   :node_name [String] - identifier for the agent
+  #   :name [String] - identifier for the agent
   #   :auto_reload [Boolean] - enable reloading of pipelines
   #   :reload_interval [Integer] - reload pipelines every X seconds
   def initialize(settings = LogStash::SETTINGS)
@@ -35,10 +36,12 @@ class LogStash::Agent
     @auto_reload = setting("config.reload.automatic")
 
     @pipelines = {}
-    @node_name = setting("node.name")
+    @name = setting("node.name")
     @http_host = setting("http.host")
     @http_port = setting("http.port")
     @http_environment = setting("http.environment")
+    # Generate / load the persistent uuid
+    id
 
     @config_loader = LogStash::Config::Loader.new(@logger)
     @reload_interval = setting("config.reload.interval")
@@ -50,6 +53,10 @@ class LogStash::Agent
     configure_metrics_collectors
 
     @reload_metric = metric.namespace([:stats, :pipelines])
+
+    @dispatcher = LogStash::EventDispatcher.new(self)
+    LogStash::PLUGIN_REGISTRY.hooks.register_emitter(self.class, dispatcher)
+    dispatcher.fire(:after_initialize)
   end
 
   def execute
@@ -97,7 +104,8 @@ class LogStash::Agent
 
   def reload_state!
     @upgrade_mutex.synchronize do
-      @pipelines.each do |pipeline_id, _|
+      @pipelines.each do |pipeline_id, pipeline|
+        next if pipeline.settings.get("config.reload.automatic") == false
         begin
           reload_pipeline!(pipeline_id)
         rescue => e
@@ -130,8 +138,42 @@ class LogStash::Agent
     shutdown_pipelines
   end
 
-  def node_uuid
-    @node_uuid ||= SecureRandom.uuid
+  def id
+    return @id if @id
+
+    uuid = nil
+    if ::File.exists?(id_path)
+      begin
+        uuid = ::File.open(id_path) {|f| f.each_line.first.chomp }
+      rescue => e
+        logger.warn("Could not open persistent UUID file!",
+                    :path => id_path,
+                    :error => e.message,
+                    :class => e.class.name)
+      end
+    end
+
+    if !uuid
+      uuid = SecureRandom.uuid
+      logger.info("No persistent UUID file found. Generating new UUID",
+                  :uuid => uuid,
+                  :path => id_path)
+      begin
+        ::File.open(id_path, 'w') {|f| f.write(uuid) }
+      rescue => e
+        logger.warn("Could not write persistent UUID file! Will use ephemeral UUID",
+                    :uuid => uuid,
+                    :path => id_path,
+                    :error => e.message,
+                    :class => e.class.name)
+      end
+    end
+
+    @id = uuid
+  end
+
+  def id_path
+    @id_path ||= ::File.join(settings.get("path.data"), "uuid")
   end
 
   def running_pipelines?
@@ -161,7 +203,7 @@ class LogStash::Agent
                 @logger.debug("Agent: Configuring metric collection")
                 LogStash::Instrument::Metric.new(@collector)
               else
-                LogStash::Instrument::NullMetric.new
+                LogStash::Instrument::NullMetric.new(@collector)
               end
 
 
@@ -244,12 +286,12 @@ class LogStash::Agent
     return unless pipeline.is_a?(LogStash::Pipeline)
     return if pipeline.ready?
     @logger.debug("starting pipeline", :id => id)
-    Thread.new do
+    t = Thread.new do
       LogStash::Util.set_thread_name("pipeline.#{id}")
       begin
         pipeline.run
       rescue => e
-        @reload_metric.namespace([id.to_sym, :reloads]) do |n|
+        @reload_metric.namespace([id.to_sym, :reloads]).tap do |n|
           n.increment(:failures)
           n.gauge(:last_error, { :message => e.message, :backtrace => e.backtrace})
           n.gauge(:last_failure_timestamp, LogStash::Timestamp.now)
@@ -257,7 +299,15 @@ class LogStash::Agent
         @logger.error("Pipeline aborted due to error", :exception => e, :backtrace => e.backtrace)
       end
     end
-    sleep 0.01 until pipeline.ready?
+    while true do
+      if !t.alive?
+        return false
+      elsif pipeline.ready?
+        return true
+      else
+        sleep 0.01
+      end
+    end
   end
 
   def stop_pipeline(id)
@@ -289,10 +339,11 @@ class LogStash::Agent
     stop_pipeline(pipeline_id)
     reset_pipeline_metrics(pipeline_id)
     @pipelines[pipeline_id] = new_pipeline
-    start_pipeline(pipeline_id)
-    @reload_metric.namespace([pipeline_id.to_sym, :reloads]).tap do |n|
-      n.increment(:successes)
-      n.gauge(:last_success_timestamp, LogStash::Timestamp.now)
+    if start_pipeline(pipeline_id) # pipeline started successfuly
+      @reload_metric.namespace([pipeline_id.to_sym, :reloads]).tap do |n|
+        n.increment(:successes)
+        n.gauge(:last_success_timestamp, LogStash::Timestamp.now)
+      end
     end
   end
 

data/lib/logstash/api/commands/default_metadata.rb

@@ -7,7 +7,8 @@ module LogStash
     module Commands
       class DefaultMetadata < Commands::Base
         def all
-          {:host => host, :version => version, :http_address => http_address}
+          {:host => host, :version => version, :http_address => http_address,
+           :id => service.agent.id, :name => service.agent.name}
         end
 
         def host

data/lib/logstash/api/commands/stats.rb

@@ -15,7 +15,8 @@ module LogStash
               :peak_count
             ),
             :mem => memory,
-            :gc => gc
+            :gc => gc,
+            :uptime_in_millis => service.get_shallow(:jvm, :uptime_in_millis)
           }
         end
 
@@ -26,7 +27,7 @@ module LogStash
             :peak_open_file_descriptors,
             :max_file_descriptors,
             [:mem, [:total_virtual_in_bytes]],
-            [:cpu, [:total_in_millis, :percent]]
+            [:cpu, [:total_in_millis, :percent, :load_average]]
           )
         end
 

data/lib/logstash/config/file.rb

@@ -2,7 +2,6 @@
 require "logstash/namespace"
 require "logstash/config/grammar"
 require "logstash/config/config_ast"
-require "logstash/config/registry"
 require "logstash/errors"
 require "logger"
 

data/lib/logstash/config/loader.rb

@@ -66,6 +66,7 @@ module LogStash; module Config; class Loader
     config = ""
     if Dir.glob(path).length == 0
       @logger.info("No config files found in path", :path => path)
+      return config
     end
 
     encoding_issue_files = []

data/lib/logstash/config/mixin.rb

@@ -1,6 +1,5 @@
 # encoding: utf-8
 require "logstash/namespace"
-require "logstash/config/registry"
 require "logstash/plugins/registry"
 require "logstash/logging"
 require "logstash/util/password"
@@ -73,6 +72,7 @@ module LogStash::Config::Mixin
                      "in the future. #{extra} If you have any questions " +
                      "about this, please visit the #logstash channel " +
                      "on freenode irc.", :name => name, :plugin => self)
+
       end
       if opts && opts[:obsolete]
         extra = opts[:obsolete].is_a?(String) ? opts[:obsolete] : ""
@@ -181,11 +181,7 @@ module LogStash::Config::Mixin
     # If no name given (nil), return the current name.
     def config_name(name = nil)
       @config_name = name if !name.nil?
-      LogStash::Config::Registry.registry[@config_name] = self
-      if self.respond_to?("plugin_type")
-        declare_plugin(self.plugin_type, @config_name)
-      end
-      return @config_name
+      @config_name
     end
     alias_method :config_plugin, :config_name
 

data/lib/logstash/environment.rb

@@ -23,7 +23,7 @@ module LogStash
            Setting::Boolean.new("config.test_and_exit", false),
            Setting::Boolean.new("config.reload.automatic", false),
            Setting::Numeric.new("config.reload.interval", 3), # in seconds
-           Setting::Boolean.new("metric.collect", true) {|v| v == true }, # metric collection cannot be disabled
+           Setting::Boolean.new("metric.collect", true),
             Setting::String.new("pipeline.id", "main"),
    Setting::PositiveInteger.new("pipeline.workers", LogStash::Config::CpuCoreStrategy.maximum),
    Setting::PositiveInteger.new("pipeline.output.workers", 1),
@@ -40,14 +40,33 @@ module LogStash
             Setting::String.new("http.host", "127.0.0.1"),
             Setting::PortRange.new("http.port", 9600..9700),
             Setting::String.new("http.environment", "production"),
+            Setting::String.new("queue.type", "memory", true, ["persisted", "memory", "memory_acked"]),
+            Setting::Bytes.new("queue.page_capacity", "250mb"),
+            Setting::Bytes.new("queue.max_bytes", "1024mb"),
+            Setting::Numeric.new("queue.max_events", 0), # 0 is unlimited
+            Setting::Numeric.new("queue.checkpoint.acks", 1024), # 0 is unlimited
+            Setting::Numeric.new("queue.checkpoint.writes", 1024), # 0 is unlimited
+            Setting::Numeric.new("queue.checkpoint.interval", 1000), # 0 is no time-based checkpointing
+            Setting::TimeValue.new("slowlog.threshold.warn", "-1"),
+            Setting::TimeValue.new("slowlog.threshold.info", "-1"),
+            Setting::TimeValue.new("slowlog.threshold.debug", "-1"),
+            Setting::TimeValue.new("slowlog.threshold.trace", "-1")
   ].each {|setting| SETTINGS.register(setting) }
 
+  # Compute the default queue path based on `path.data`
+  default_queue_file_path = ::File.join(SETTINGS.get("path.data"), "queue")
+  SETTINGS.register Setting::WritableDirectory.new("path.queue", default_queue_file_path)
+
   module Environment
     extend self
 
     LOGSTASH_CORE = ::File.expand_path(::File.join(::File.dirname(__FILE__), "..", ".."))
     LOGSTASH_ENV = (ENV["LS_ENV"] || 'production').to_s.freeze
 
+    LINUX_OS_RE = /linux/
+    WINDOW_OS_RE = /mswin|msys|mingw|cygwin|bccwin|wince|emc/
+    MACOS_OS_RE = /darwin/
+
     def env
       LOGSTASH_ENV
     end
@@ -110,7 +129,11 @@ module LogStash
     end
 
     def windows?
-      ::Gem.win_platform?
+      RbConfig::CONFIG['host_os'] =~ WINDOW_OS_RE
+    end
+
+    def linux?
+      RbConfig::CONFIG['host_os'] =~ LINUX_OS_RE
     end
 
     def locales_path(path)

data/lib/logstash/event_dispatcher.rb

@@ -0,0 +1,40 @@
+# encoding: utf-8
+module LogStash
+  class EventDispatcher
+    java_import "java.util.concurrent.CopyOnWriteArrayList"
+
+    attr_reader :emitter
+
+    def initialize(emitter)
+      @emitter = emitter
+      @listeners = CopyOnWriteArrayList.new
+    end
+
+    # This operation is slow because we use a CopyOnWriteArrayList
+    # But the majority of the addition will be done at bootstrap time
+    # So add_listener shouldn't be called often at runtime.
+    #
+    # On the other hand the notification could be called really often.
+    def add_listener(listener)
+      @listeners.add(listener)
+    end
+
+    # This operation is slow because we use a `CopyOnWriteArrayList` as the backend, instead of a
+    # ConcurrentHashMap, but since we are mostly adding stuff and iterating the `CopyOnWriteArrayList`
+    # should provide a better performance.
+    #
+    # See note on add_listener, this method shouldn't be called really often.
+    def remove_listener(listener)
+      @listeners.remove(listener)
+    end
+
+    def fire(method_name, *arguments)
+      @listeners.each do |listener|
+        if listener.respond_to?(method_name)
+          listener.send(method_name, emitter, *arguments)
+        end
+      end
+    end
+    alias_method :execute, :fire
+  end
+end

data/lib/logstash/filter_delegator.rb

@@ -20,7 +20,7 @@ module LogStash
       @filter = klass.new(plugin_args)
 
       # Scope the metrics to the plugin
-      namespaced_metric = metric.namespace("#{@klass.config_name}_#{@id}".to_sym)
+      namespaced_metric = metric.namespace(@id.to_sym)
       @filter.metric = namespaced_metric
 
       @metric_events = namespaced_metric.namespace(:events)

data/lib/logstash/filters/base.rb

@@ -139,6 +139,14 @@ class LogStash::Filters::Base < LogStash::Plugin
     raise "#{self.class}#filter must be overidden"
   end # def filter
 
+  public
+  def do_filter(event, &block)
+    time = java.lang.System.nanoTime
+    filter(event, &block)
+    @slow_logger.on_event("event processing time", @original_params, event, java.lang.System.nanoTime - time)
+  end
+
+
   # in 1.5.0 multi_filter is meant to be used in the generated filter function in LogStash::Config::AST::Plugin only
   # and is temporary until we refactor the filter method interface to accept events list and return events list,
   # just list in multi_filter see https://github.com/elastic/logstash/issues/2872.
@@ -153,7 +161,7 @@ class LogStash::Filters::Base < LogStash::Plugin
     events.each do |event|
       unless event.cancelled?
         result << event
-        filter(event){|new_event| result << new_event}
+        do_filter(event){|new_event| result << new_event}
       end
     end
     result
@@ -161,7 +169,7 @@ class LogStash::Filters::Base < LogStash::Plugin
 
   public
   def execute(event, &block)
-    filter(event, &block)
+    do_filter(event, &block)
   end # def execute
 
   public