logstash-core 5.0.0.alpha5.snapshot1-java → 5.0.0.alpha6.snapshot1-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 31b35e84fcbac8fa8ccb7ac01aac834645c04084
-  data.tar.gz: 3c5d884562c45235f897279458b3e0ee8a840d4b
+  metadata.gz: e89e8c25486877ae001285428c6e6d9fbe3dfc6b
+  data.tar.gz: 5e6318f6ae9f263864588fa2361f5e9ebdc94fb4
 SHA512:
-  metadata.gz: 2355f8393a9f37222e45f7698fdb2b3cf10be0a52cfb132a3bbaf546dadf11e05f4c2b8f60a216409b5bed7b49eba79803270dbac561217a0e36ac107e2348b2
-  data.tar.gz: 5642f18d54cfb80d5b159575f1ba59b652ec2d57606a5609bda4644740367d7f82187718728c9ccc6854d944291855ec48f54d66b957e3e6e736c52d9c2b9339
+  metadata.gz: af57be94f22a74d3ffbbde98eca0eb92fa503c15254b666c5fdfe518c401af10260203b520b9ce26f2b9043c93fdd2289f2af5ecb1387deaf7f65bdfa6ad1a19
+  data.tar.gz: 9d6a1a9a69f77fb49cb0ec3f5d2be1f8a0f2f3a460590a4722ae071cd711fb5a792baa663c233a0f06b94dbea6e7a09a3fd9c0f60ee0ac7d488097b5a96f2a52

data/lib/logstash-core/version.rb

@@ -5,4 +5,4 @@
 # Note to authors: this should not include dashes because 'gem' barfs if
 # you include a dash in the version string.
 
-LOGSTASH_CORE_VERSION = "5.0.0.alpha5.snapshot1"
+LOGSTASH_CORE_VERSION = "5.0.0.alpha6.snapshot1"

data/lib/logstash/agent.rb

@@ -135,7 +135,7 @@ class LogStash::Agent
 
   private
   def start_webserver
-    options = {:http_host => @http_host, :http_port => @http_port, :http_environment => @http_environment }
+    options = {:http_host => @http_host, :http_ports => @http_port, :http_environment => @http_environment }
     @webserver = LogStash::WebServer.new(@logger, self, options)
     Thread.new(@webserver) do |webserver|
       LogStash::Util.set_thread_name("Api Webserver")

data/lib/logstash/api/commands/default_metadata.rb

@@ -9,7 +9,7 @@ module LogStash
         def all
           {:host => host, :version => version, :http_address => http_address}
         end
-        
+
         def host
           Socket.gethostname
         end

data/lib/logstash/api/commands/hot_threads_reporter.rb

@@ -1,6 +1,7 @@
 # encoding: utf-8
 
 class HotThreadsReport
+  STRING_SEPARATOR_LENGTH = 80.freeze
   HOT_THREADS_STACK_TRACES_SIZE_DEFAULT = 10.freeze
 
   def initialize(cmd, options)
@@ -13,19 +14,16 @@ class HotThreadsReport
   def to_s
     hash = to_hash[:hot_threads]
     report =  "#{I18n.t("logstash.web_api.hot_threads.title", :hostname => hash[:hostname], :time => hash[:time], :top_count => @thread_dump.top_count )} \n"
-    report << '=' * 80
+    report << '=' * STRING_SEPARATOR_LENGTH
     report << "\n"
     hash[:threads].each do |thread|
-      thread_report = ""
-      thread_report = "#{I18n.t("logstash.web_api.
-                                hot_threads.thread_title", :percent_of_cpu_time => thread[:percent_of_cpu_time], :thread_state => thread[:state], :thread_name => thread[:name])} \n"
-      thread_report = "#{thread[:percent_of_cpu_time]} % of of cpu usage by #{thread[:state]} thread named '#{thread[:name]}'\n"
+      thread_report = "#{I18n.t("logstash.web_api.hot_threads.thread_title", :percent_of_cpu_time => thread[:percent_of_cpu_time], :thread_state => thread[:state], :thread_name => thread[:name])} \n"
       thread_report << "#{thread[:path]}\n" if thread[:path]
       thread[:traces].each do |trace|
         thread_report << "\t#{trace}\n"
       end
       report << thread_report
-      report << '-' * 80
+      report << '-' * STRING_SEPARATOR_LENGTH
       report << "\n"
     end
     report
@@ -57,5 +55,4 @@ class HotThreadsReport
   def cpu_time(hash)
     hash["cpu.time"] / 1000000.0
   end
-
 end

data/lib/logstash/api/commands/node.rb

@@ -20,7 +20,7 @@ module LogStash
         def pipeline
           extract_metrics(
             [:stats, :pipelines, :main, :config],
-            :workers, :batch_size, :batch_delay
+            :workers, :batch_size, :batch_delay, :config_reload_automatic, :config_reload_interval
           )
         end
 
@@ -35,27 +35,28 @@ module LogStash
 
         def jvm
           memory_bean = ManagementFactory.getMemoryMXBean()
+
           {
             :pid =>  ManagementFactory.getRuntimeMXBean().getName().split("@").first.to_i,
             :version => java.lang.System.getProperty("java.version"),
             :vm_name => java.lang.System.getProperty("java.vm.name"),
             :vm_version => java.lang.System.getProperty("java.version"),
             :vm_vendor => java.lang.System.getProperty("java.vendor"),
-            :vm_name => java.lang.System.getProperty("java.vm.name"),            
+            :vm_name => java.lang.System.getProperty("java.vm.name"),
             :start_time_in_millis => started_at,
             :mem => {
               :heap_init_in_bytes => (memory_bean.getHeapMemoryUsage().getInit() < 0 ? 0 : memory_bean.getHeapMemoryUsage().getInit()),
               :heap_max_in_bytes => (memory_bean.getHeapMemoryUsage().getMax() < 0 ? 0 : memory_bean.getHeapMemoryUsage().getMax()),
               :non_heap_init_in_bytes => (memory_bean.getNonHeapMemoryUsage().getInit() < 0 ? 0 : memory_bean.getNonHeapMemoryUsage().getInit()),
               :non_heap_max_in_bytes => (memory_bean.getNonHeapMemoryUsage().getMax() < 0 ? 0 : memory_bean.getNonHeapMemoryUsage().getMax())
-            }
+            },
+            :gc_collectors => ManagementFactory.getGarbageCollectorMXBeans().collect(&:getName)
           }
         end
 
         def hot_threads(options={})
           HotThreadsReport.new(self, options)
         end
-
       end
     end
   end

data/lib/logstash/api/commands/stats.rb

@@ -14,7 +14,8 @@ module LogStash
               :count,
               :peak_count
             ),
-            :mem => memory
+            :mem => memory,
+            :gc => gc
           }
         end
 
@@ -32,7 +33,7 @@ module LogStash
         def events
           extract_metrics(
             [:stats, :events],
-            :in, :filtered, :out
+            :in, :filtered, :out, :duration_in_millis
           )
         end
 
@@ -59,6 +60,10 @@ module LogStash
           }
         end
 
+        def gc
+          service.get_shallow(:jvm, :gc)
+        end
+
         def hot_threads(options={})
           HotThreadsReport.new(self, options)
         end
@@ -70,7 +75,7 @@ module LogStash
             # Turn the `plugins` stats hash into an array of [ {}, {}, ... ]
             # This is to produce an array of data points, one point for each
             # plugin instance.
-            return [] unless stats[:plugins].include?(plugin_type)
+            return [] unless stats[:plugins] && stats[:plugins].include?(plugin_type)
             stats[:plugins][plugin_type].collect do |id, data|
               { :id => id }.merge(data)
             end

data/lib/logstash/api/modules/base.rb

@@ -34,6 +34,11 @@ module LogStash
           text = as == :string ? "" : {}
           respond_with(text, :as => as)
         end
+
+        protected
+        def human?
+          params.has_key?("human") && (params["human"].nil? || as_boolean(params["human"]) == true)
+        end
       end
     end
   end

data/lib/logstash/api/modules/node.rb

@@ -14,7 +14,7 @@ module LogStash
 
           options = {
             :ignore_idle_threads => as_boolean(ignore_idle_threads),
-            :human => params.has_key?("human")
+            :human => human?
           }
           options[:threads] = params["threads"].to_i if params.has_key?("threads")
 
@@ -26,7 +26,6 @@ module LogStash
           selected_fields = extract_fields(params["filter"].to_s.strip)
           respond_with node.all(selected_fields)
         end
-
       end
     end
   end

data/lib/logstash/api/modules/node_stats.rb

@@ -12,8 +12,7 @@ module LogStash
           payload = {
             :jvm => jvm_payload,
             :process => process_payload,
-            :mem => mem_payload,
-            :pipeline => pipeline_payload
+            :pipeline => pipeline_payload,
           }
           respond_with(payload, {:filter => params["filter"]})
         end

data/lib/logstash/codecs/base.rb

@@ -18,6 +18,7 @@ module LogStash::Codecs; class Base < LogStash::Plugin
     super
     config_init(@params)
     register if respond_to?(:register)
+    setup_multi_encode!
   end
 
   public
@@ -28,10 +29,37 @@ module LogStash::Codecs; class Base < LogStash::Plugin
   alias_method :<<, :decode
 
   public
+  # DEPRECATED: Prefer defining encode_sync or multi_encode
   def encode(event)
-    raise "#{self.class}#encode must be overidden"
+    encoded = multi_encode([event])
+    encoded.each {|event,data| @on_event.call(event,data) }
   end # def encode
 
+  public
+  # Relies on the codec being synchronous (which they all are!)
+  # We need a better long term design here, but this is an improvement
+  # over the current API for shared plugins
+  # It is best if the codec implements this directly
+  def multi_encode(events)
+    if @has_encode_sync              
+      events.map {|event| [event, self.encode_sync(event)]}
+    else
+      batch = Thread.current[:logstash_output_codec_batch] ||= []
+      batch.clear
+      
+      events.each {|event| self.encode(event) }
+      batch
+    end
+  end
+
+  def setup_multi_encode!
+    @has_encode_sync = self.methods.include?(:encode_sync)
+
+    on_event do |event, data|
+      Thread.current[:logstash_output_codec_batch] << [event, data]
+    end
+  end
+
   public
   def close; end;
 

data/lib/logstash/config/mixin.rb

@@ -209,7 +209,7 @@ module LogStash::Config::Mixin
 
       name = name.to_s if name.is_a?(Symbol)
       @config[name] = opts  # ok if this is empty
-
+      
       if name.is_a?(String)
         define_method(name) { instance_variable_get("@#{name}") }
         define_method("#{name}=") { |v| instance_variable_set("@#{name}", v) }

data/lib/logstash/environment.rb

@@ -21,12 +21,12 @@ module LogStash
             Setting::String.new("config.string", nil, false),
            Setting::Boolean.new("config.test_and_exit", false),
            Setting::Boolean.new("config.reload.automatic", false),
-           Setting::Numeric.new("config.reload.interval", 3),
+           Setting::Numeric.new("config.reload.interval", 3), # in seconds
            Setting::Boolean.new("metric.collect", true) {|v| v == true }, # metric collection cannot be disabled
             Setting::String.new("pipeline.id", "main"),
-           Setting::Numeric.new("pipeline.workers", LogStash::Config::CpuCoreStrategy.maximum),
-           Setting::Numeric.new("pipeline.output.workers", 1),
-           Setting::Numeric.new("pipeline.batch.size", 125),
+   Setting::PositiveInteger.new("pipeline.workers", LogStash::Config::CpuCoreStrategy.maximum),
+   Setting::PositiveInteger.new("pipeline.output.workers", 1),
+   Setting::PositiveInteger.new("pipeline.batch.size", 125),
            Setting::Numeric.new("pipeline.batch.delay", 5), # in milliseconds
            Setting::Boolean.new("pipeline.unsafe_shutdown", false),
                     Setting.new("path.plugins", Array, []),
@@ -38,7 +38,7 @@ module LogStash
             Setting::String.new("path.log", nil, false),
             Setting::String.new("log.format", "plain", true, ["json", "plain"]),
             Setting::String.new("http.host", "127.0.0.1"),
-              Setting::Port.new("http.port", 9600),
+            Setting::PortRange.new("http.port", 9600..9700),
             Setting::String.new("http.environment", "production"),
   ].each {|setting| SETTINGS.register(setting) }
 

data/lib/logstash/filter_delegator.rb

@@ -13,15 +13,14 @@ module LogStash
     ]
     def_delegators :@filter, *DELEGATED_METHODS
 
-    def initialize(logger, klass, metric, *args)
-      options = args.reduce({}, :merge)
-
+    def initialize(logger, klass, metric, plugin_args)
       @logger = logger
       @klass = klass
-      @filter = klass.new(options)
+      @id = plugin_args["id"]
+      @filter = klass.new(plugin_args)
 
       # Scope the metrics to the plugin
-      namespaced_metric = metric.namespace(@filter.plugin_unique_name.to_sym)
+      namespaced_metric = metric.namespace("#{@klass.config_name}_#{@id}".to_sym)
       @filter.metric = namespaced_metric
 
       @metric_events = namespaced_metric.namespace(:events)

data/lib/logstash/instrument/periodic_poller/jvm.rb

@@ -1,18 +1,36 @@
 
 # encoding: utf-8
 require "logstash/instrument/periodic_poller/base"
-require 'jrmonitor'
+require "jrmonitor"
+require "set"
 
 java_import 'java.lang.management.ManagementFactory'
 java_import 'java.lang.management.OperatingSystemMXBean'
+java_import 'java.lang.management.GarbageCollectorMXBean'
 java_import 'com.sun.management.UnixOperatingSystemMXBean'
 java_import 'javax.management.MBeanServer'
 java_import 'javax.management.ObjectName'
 java_import 'javax.management.AttributeList'
 java_import 'javax.naming.directory.Attribute'
 
+
 module LogStash module Instrument module PeriodicPoller
   class JVM < Base
+    class GarbageCollectorName
+      YOUNG_GC_NAMES = Set.new(["Copy", "PS Scavenge", "ParNew", "G1 Young Generation"])
+      OLD_GC_NAMES = Set.new(["MarkSweepCompact", "PS MarkSweep", "ConcurrentMarkSweep", "G1 Old Generation"])
+
+      YOUNG = :young
+      OLD = :old
+
+      def self.get(gc_name)
+        if YOUNG_GC_NAMES.include?(gc_name)
+          YOUNG
+        elsif(OLD_GC_NAMES.include?(gc_name))
+          OLD
+        end
+      end
+    end
 
     attr_reader :metric
 
@@ -22,31 +40,46 @@ module LogStash module Instrument module PeriodicPoller
     end
 
     def collect
-      raw = JRMonitor.memory.generate      
+      raw = JRMonitor.memory.generate
       collect_heap_metrics(raw)
       collect_non_heap_metrics(raw)
       collect_pools_metrics(raw)
       collect_threads_metrics
       collect_process_metrics
+      collect_gc_stats
     end
 
     private
 
-    def collect_threads_metrics      
+    def collect_gc_stats
+      garbage_collectors = ManagementFactory.getGarbageCollectorMXBeans()
+
+      garbage_collectors.each do |collector|
+        name = GarbageCollectorName.get(collector.getName())
+        if name.nil?
+          logger.error("Unknown garbage collector name", :name => name)
+        else
+          metric.gauge([:jvm, :gc, :collectors, name], :collection_count, collector.getCollectionCount())
+          metric.gauge([:jvm, :gc, :collectors, name], :collection_time_in_millis, collector.getCollectionTime())
+        end
+      end
+    end
+
+    def collect_threads_metrics
       threads = JRMonitor.threads.generate
-      
+
       current = threads.count
       if @peak_threads.nil? || @peak_threads < current
         @peak_threads = current
-      end      
-      
-      metric.gauge([:jvm, :threads], :count, threads.count)     
+      end
+
+      metric.gauge([:jvm, :threads], :count, threads.count)
       metric.gauge([:jvm, :threads], :peak_count, @peak_threads)
     end
 
     def collect_process_metrics
       process_metrics = JRMonitor.process.generate
-      
+
       path = [:jvm, :process]
 
 
@@ -91,6 +124,7 @@ module LogStash module Instrument module PeriodicPoller
       end
     end
 
+
     def build_pools_metrics(data)
       heap = data["heap"]
       old  = {}
@@ -129,9 +163,8 @@ module LogStash module Instrument module PeriodicPoller
         :committed_in_bytes => 0,
         :max_in_bytes => 0,
         :peak_used_in_bytes => 0,
-        :peak_max_in_bytes  => 0
+        :peak_max_in_bytes => 0
       }
     end
-
   end
 end; end; end

data/lib/logstash/output_delegator.rb

@@ -1,192 +1,57 @@
-# encoding: utf-8
-require "concurrent/atomic/atomic_fixnum"
-java_import "java.util.concurrent.CopyOnWriteArrayList"
+require "logstash/output_delegator_strategy_registry"
+
+require "logstash/output_delegator_strategies/shared"
+require "logstash/output_delegator_strategies/single"
+require "logstash/output_delegator_strategies/legacy"
 
-# This class goes hand in hand with the pipeline to provide a pool of
-# free workers to be used by pipeline worker threads. The pool is
-# internally represented with a SizedQueue set the the size of the number
-# of 'workers' the output plugin is configured with.
-#
-# This plugin also records some basic statistics
 module LogStash class OutputDelegator
-  attr_reader :workers, :config, :threadsafe
+  attr_reader :metric, :metric_events, :strategy, :namespaced_metric, :metric_events , :plugin_args, :strategy_registry
 
-  # The *args this takes are the same format that a Outputs::Base takes. A list of hashes with parameters in them
-  # Internally these just get merged together into a single hash
-  def initialize(logger, klass, default_worker_count, metric, *plugin_args)
+  def initialize(logger, output_class, metric, strategy_registry, plugin_args)
     @logger = logger
-    @threadsafe = klass.threadsafe?
-    @config = plugin_args.reduce({}, :merge)
-    @klass = klass
-    @workers = java.util.concurrent.CopyOnWriteArrayList.new
-    @default_worker_count = default_worker_count
-    @registered = false
-
-    # Create an instance of the input so we can fetch the identifier
-    output = @klass.new(@config)
-
-    # Scope the metrics to the plugin
-    namespaced_metric = metric.namespace(output.plugin_unique_name.to_sym)
-    output.metric = namespaced_metric
-
-    @metric_events = namespaced_metric.namespace(:events)
-    namespaced_metric.gauge(:name, config_name)
+    @output_class = output_class
+    @metric = metric
+    @plugin_args = plugin_args
+    @strategy_registry = strategy_registry
+    raise ArgumentError, "No strategy registry specified" unless strategy_registry
+    raise ArgumentError, "No ID specified! Got args #{plugin_args}" unless id
+    
+    build_strategy!
 
-    @events_received = Concurrent::AtomicFixnum.new(0)
+    @namespaced_metric = metric.namespace(id.to_sym)
+    @metric_events = @namespaced_metric.namespace(:events)
+    @namespaced_metric.gauge(:name, id)
   end
 
-  def threadsafe?
-    !!@threadsafe
+  def config_name
+    @output_class.config_name
   end
 
-  def warn_on_worker_override!
-    # The user has configured extra workers, but this plugin doesn't support it :(
-    if worker_limits_overriden?
-      message = @klass.workers_not_supported_message
-      warning_meta = {:plugin => @klass.config_name, :worker_count => @config["workers"]}
-      if message
-        warning_meta[:message] = message
-        @logger.warn(I18n.t("logstash.pipeline.output-worker-unsupported-with-message", warning_meta))
-      else
-        @logger.warn(I18n.t("logstash.pipeline.output-worker-unsupported", warning_meta))
-      end
-    end
+  def concurrency
+    @output_class.concurrency
   end
 
-  def worker_limits_overriden?
-    @config["workers"] && @config["workers"] > 1 && @klass.workers_not_supported?
+  def build_strategy!
+    @strategy = strategy_registry.
+                  class_for(self.concurrency).
+                  new(@logger, @output_class, @metric, @plugin_args)
   end
 
-  def target_worker_count
-    # Remove in 5.0 after all plugins upgraded to use class level declarations
-    raise ArgumentError, "Attempted to detect target worker count before instantiating a worker to test for legacy workers_not_supported!" if @workers.size == 0
-
-    if @threadsafe || @klass.workers_not_supported?
-      1
-    else
-      @config["workers"] || @default_worker_count
-    end
-  end
-
-  def config_name
-    @klass.config_name
+  def id
+    @plugin_args["id"]
   end
 
   def register
-    raise ArgumentError, "Attempted to register #{self} twice!" if @registered
-    @registered = true
-    # We define this as an array regardless of threadsafety
-    # to make reporting simpler, even though a threadsafe plugin will just have
-    # a single instance
-    #
-    # Older plugins invoke the instance method Outputs::Base#workers_not_supported
-    # To detect these we need an instance to be created first :()
-    # TODO: In the next major version after 2.x remove support for this
-    @workers << @klass.new(@config)
-    @workers.first.register # Needed in case register calls `workers_not_supported`
-
-    @logger.debug("Will start workers for output", :worker_count => target_worker_count, :class => @klass.name)
-
-    # Threadsafe versions don't need additional workers
-    setup_additional_workers!(target_worker_count) unless @threadsafe
-    # We skip the first worker because that's pre-registered to deal with legacy workers_not_supported
-    @workers.subList(1,@workers.size).each(&:register)
-    setup_multi_receive!
-  end
-
-  def setup_additional_workers!(target_worker_count)
-    warn_on_worker_override!
-
-    (target_worker_count - 1).times do
-      inst = @klass.new(@config)
-      inst.metric = @metric
-      @workers << inst
-    end
-
-    # This queue is used to manage sharing across threads
-    @worker_queue = SizedQueue.new(target_worker_count)
-    @workers.each {|w| @worker_queue << w }
+    @strategy.register
   end
 
-  def setup_multi_receive!
-    # One might wonder why we don't use something like
-    # define_singleton_method(:multi_receive, method(:threadsafe_multi_receive)
-    # and the answer is this is buggy on Jruby 1.7.x . It works 98% of the time!
-    # The other 2% you get weird errors about rebinding to the same object
-    # Until we switch to Jruby 9.x keep the define_singleton_method parts
-    # the way they are, with a block
-    # See https://github.com/jruby/jruby/issues/3582
-    if threadsafe?
-      @threadsafe_worker = @workers.first
-      define_singleton_method(:multi_receive) do |events|
-        threadsafe_multi_receive(events)
-      end
-    else
-      define_singleton_method(:multi_receive) do |events|
-        worker_multi_receive(events)
-      end
-    end
-  end
-
-  def threadsafe_multi_receive(events)
-    @events_received.increment(events.length)
+  def multi_receive(events)
     @metric_events.increment(:in, events.length)
-
-    clock = @metric_events.time(:duration_in_millis)
-    @threadsafe_worker.multi_receive(events)
-    clock.stop
+    @strategy.multi_receive(events)
     @metric_events.increment(:out, events.length)
   end
 
-  def worker_multi_receive(events)
-    @events_received.increment(events.length)
-    @metric_events.increment(:in, events.length)
-
-    worker = @worker_queue.pop
-    begin
-      clock = @metric_events.time(:duration_in_millis)
-      worker.multi_receive(events)
-      clock.stop
-      @metric_events.increment(:out, events.length)
-    ensure
-      @worker_queue.push(worker)
-    end
-  end
-
   def do_close
-    @logger.debug("closing output delegator", :klass => @klass.name)
-
-    if @threadsafe
-      @workers.each(&:do_close)
-    else
-      worker_count.times do
-        worker = @worker_queue.pop
-        worker.do_close
-      end
-    end
+    @strategy.do_close
   end
-
-  def events_received
-    @events_received.value
-  end
-
-  # There's no concept of 'busy' workers for a threadsafe plugin!
-  def busy_workers
-    if @threadsafe
-      0
-    else
-      # The pipeline reporter can run before the outputs are registered trying to pull a value here
-      # In that case @worker_queue is empty, we just return 0
-      return 0 unless @worker_queue
-      @workers.size - @worker_queue.size
-    end
-  end
-
-  def worker_count
-    @workers.size
-  end
-
-  private
-  # Needed for testing, so private
-  attr_reader :threadsafe_worker, :worker_queue
-end end
+end; end