logstash-core 5.0.0.alpha2.snapshot2-java → 5.0.0.alpha3-java

data/lib/logstash/environment.rb

@@ -1,7 +1,36 @@
 # encoding: utf-8
 require "logstash/errors"
+require "logstash/config/cpu_core_strategy"
+require "logstash/settings"
 
 module LogStash
+
+  [
+            Setting::String.new("node.name", Socket.gethostname),
+            Setting::String.new("path.config", nil, false),
+            Setting::String.new("config.string", nil, false),
+           Setting::Boolean.new("config.test_and_exit", false),
+           Setting::Boolean.new("config.reload.automatic", false),
+           Setting::Numeric.new("config.reload.interval", 3),
+           Setting::Boolean.new("metric.collect", true) {|v| v == true }, # metric collection cannot be disabled
+            Setting::String.new("pipeline.id", "main"),
+           Setting::Numeric.new("pipeline.workers", LogStash::Config::CpuCoreStrategy.maximum),
+           Setting::Numeric.new("pipeline.output.workers", 1),
+           Setting::Numeric.new("pipeline.batch.size", 125),
+           Setting::Numeric.new("pipeline.batch.delay", 5), # in milliseconds
+           Setting::Boolean.new("pipeline.unsafe_shutdown", false),
+                    Setting.new("path.plugins", Array, []),
+            Setting::String.new("interactive", nil, false),
+           Setting::Boolean.new("config.debug", false),
+            Setting::String.new("log.level", "warn", true, ["quiet", "verbose", "warn", "debug"]),
+           Setting::Boolean.new("version", false),
+           Setting::Boolean.new("help", false),
+            Setting::String.new("path.log", nil, false),
+            Setting::String.new("log.format", "plain", true, ["json", "plain"]),
+            Setting::String.new("http.host", "127.0.0.1"),
+              Setting::Port.new("http.port", 9600),
+  ].each {|setting| SETTINGS.register(setting) }
+
   module Environment
     extend self
 

data/lib/logstash/filters/base.rb

@@ -187,12 +187,12 @@ class LogStash::Filters::Base < LogStash::Plugin
     # this is important because a construct like event["tags"].delete(tag) will not work
     # in the current Java event implementation. see https://github.com/elastic/logstash/issues/4140
     @remove_tag.each do |tag|
-      tags = event["tags"]
+      tags = event.get("tags")
       break if tags.nil? || tags.empty?
       tag = event.sprintf(tag)
       @logger.debug? and @logger.debug("filters/#{self.class.name}: removing tag", :tag => tag)
       tags.delete(tag)
-      event["tags"] = tags
+      event.set("tags", tags)
     end
   end # def filter_matched
 

data/lib/logstash/inputs/base.rb

@@ -83,7 +83,7 @@ class LogStash::Inputs::Base < LogStash::Plugin
 
   public
   def do_stop
-    @logger.debug("stopping", :plugin => self)
+    @logger.debug("stopping", :plugin => self.class.name)
     @stop_called.make_true
     stop
   end
@@ -97,7 +97,7 @@ class LogStash::Inputs::Base < LogStash::Plugin
   protected
   def decorate(event)
     # Only set 'type' if not already set. This is backwards-compatible behavior
-    event["type"] = @type if @type && !event.include?("type")
+    event.set("type", @type) if @type && !event.include?("type")
 
     LogStash::Util::Decorators.add_fields(@add_field,event,"inputs/#{self.class.name}")
     LogStash::Util::Decorators.add_tags(@tags,event,"inputs/#{self.class.name}")

data/lib/logstash/instrument/collector.rb

@@ -75,7 +75,7 @@ module LogStash module Instrument
       logger.error("Collector: Something went wrong went sending data to the observers",
                    :execution_time => time_of_execution,
                    :result => result,
-                   :exception => exception)
+                   :exception => exception.class.name)
     end
 
     # Snapshot the current Metric Store and return it immediately,

data/lib/logstash/instrument/metric_store.rb

@@ -86,7 +86,7 @@ module LogStash module Instrument
 
     # Similar to `get_with_path` but use symbols instead of string
     #
-    # @param [Array<Symbol>
+    # @param [Array<Symbol>]
     # @return [Hash]
     def get(*key_paths)
       # Normalize the symbols access
@@ -100,6 +100,16 @@ module LogStash module Instrument
       new_hash
     end
 
+    # Retrieve values like `get`, but don't return them fully nested.
+    # This means that if you call `get_shallow(:foo, :bar)` the result will not
+    # be nested inside of `{:foo {:bar => values}`.
+    #
+    # @param [Array<Symbol>]
+    # @return [Hash]
+    def get_shallow(*key_paths)
+      key_paths.reduce(get(*key_paths)) {|acc, p| acc[p]}
+    end
+
     # Return all the individuals Metric,
     # This call mimic a Enum's each if a block is provided
     #

data/lib/logstash/instrument/periodic_poller/base.rb

@@ -37,6 +37,8 @@ module LogStash module Instrument module PeriodicPoller
       logger.debug("PeriodicPoller: Starting",
                    :polling_interval => @options[:polling_interval],
                    :polling_timeout => @options[:polling_timeout]) if logger.debug?
+      
+      collect # Collect data right away if possible
       @task.execute
     end
 

data/lib/logstash/instrument/periodic_poller/jvm.rb

@@ -1,6 +1,15 @@
+
 # encoding: utf-8
 require "logstash/instrument/periodic_poller/base"
-require 'monitoring'
+require 'jrmonitor'
+
+java_import 'java.lang.management.ManagementFactory'
+java_import 'java.lang.management.OperatingSystemMXBean'
+java_import 'com.sun.management.UnixOperatingSystemMXBean'
+java_import 'javax.management.MBeanServer'
+java_import 'javax.management.ObjectName'
+java_import 'javax.management.AttributeList'
+java_import 'javax.naming.directory.Attribute'
 
 module LogStash module Instrument module PeriodicPoller
   class JVM < Base
@@ -13,14 +22,50 @@ module LogStash module Instrument module PeriodicPoller
     end
 
     def collect
-      raw = JRMonitor.memory.generate
+      raw = JRMonitor.memory.generate      
       collect_heap_metrics(raw)
       collect_non_heap_metrics(raw)
       collect_pools_metrics(raw)
+      collect_threads_metrics
+      collect_process_metrics
     end
 
     private
 
+    def collect_threads_metrics      
+      threads = JRMonitor.threads.generate
+      
+      current = threads.count
+      if @peak_threads.nil? || @peak_threads < current
+        @peak_threads = current
+      end      
+      
+      metric.gauge([:jvm, :threads], :count, threads.count)     
+      metric.gauge([:jvm, :threads], :peak_count, @peak_threads)
+    end
+
+    def collect_process_metrics
+      process_metrics = JRMonitor.process.generate
+      
+      path = [:jvm, :process]
+
+
+      open_fds = process_metrics["open_file_descriptors"]
+      if @peak_open_fds.nil? || open_fds > @peak_open_fds
+        @peak_open_fds = open_fds
+      end
+      metric.gauge(path, :open_file_descriptors, open_fds)
+      metric.gauge(path, :peak_open_file_descriptors, @peak_open_fds)
+      metric.gauge(path, :max_file_descriptors, process_metrics["max_file_descriptors"])
+
+      cpu_path = path + [:cpu]
+      cpu_metrics = process_metrics["cpu"]
+      metric.gauge(cpu_path, :percent, cpu_metrics["process_percent"])
+      metric.gauge(cpu_path, :total_in_millis, cpu_metrics["total_in_millis"])
+
+      metric.gauge(path + [:mem], :total_virtual_in_bytes, process_metrics["mem"]["total_virtual_in_bytes"])
+    end
+
     def collect_heap_metrics(data)
       heap = aggregate_information_for(data["heap"].values)
       heap[:used_percent] = (heap[:used_in_bytes] / heap[:max_in_bytes].to_f)*100.0

data/lib/logstash/logging/json.rb

@@ -0,0 +1,21 @@
+# encoding: utf-8
+require "logstash/namespace"
+require "logstash/logging"
+require "logstash/json"
+
+module LogStash; class Logging; class JSON
+  def initialize(io)
+    raise ArgumentError, "Expected IO, got #{io.class.name}" unless io.is_a?(IO)
+
+    @io = io
+    @lock = Mutex.new
+  end
+
+  def <<(obj)
+    serialized = LogStash::Json.dump(obj)
+    @lock.synchronize do
+      @io.puts(serialized)
+      @io.flush
+    end
+  end
+end; end; end

data/lib/logstash/output_delegator.rb

@@ -82,7 +82,7 @@ module LogStash class OutputDelegator
     @workers << @klass.new(@config)
     @workers.first.register # Needed in case register calls `workers_not_supported`
 
-    @logger.debug("Will start workers for output", :worker_count => target_worker_count, :class => @klass)
+    @logger.debug("Will start workers for output", :worker_count => target_worker_count, :class => @klass.name)
 
     # Threadsafe versions don't need additional workers
     setup_additional_workers!(target_worker_count) unless @threadsafe
@@ -147,7 +147,7 @@ module LogStash class OutputDelegator
   end
 
   def do_close
-    @logger.debug("closing output delegator", :klass => @klass)
+    @logger.debug("closing output delegator", :klass => @klass.name)
 
     if @threadsafe
       @workers.each(&:do_close)

data/lib/logstash/patches/clamp.rb

@@ -0,0 +1,69 @@
+require 'clamp'
+require 'logstash/environment'
+
+module Clamp
+  module Attribute
+    class Instance
+      def default_from_environment
+        # we don't want uncontrolled var injection from the environment
+        # since we're establishing that settings can be pulled from only three places:
+        # 1. default settings
+        # 2. yaml file
+        # 3. cli arguments
+      end
+    end
+  end
+
+  module Option
+
+    module StrictDeclaration
+
+      include Clamp::Attribute::Declaration
+
+      # Instead of letting Clamp set up accessors for the options
+      # weŕe going to tightly controlling them through
+      # LogStash::SETTINGS
+      def define_simple_writer_for(option, &block)
+        LogStash::SETTINGS.get(option.attribute_name)
+        define_method(option.write_method) do |value|
+          value = instance_exec(value, &block) if block
+          LogStash::SETTINGS.set_value(option.attribute_name, value)
+        end
+      end
+
+      def define_reader_for(option)
+        define_method(option.read_method) do
+          LogStash::SETTINGS.get_value(option.attribute_name)
+        end
+      end
+
+    end
+
+    class Definition
+      # Allow boolean flags to optionally receive a true/false argument
+      # to explicitly set them, i.e.
+      # --long.flag.name       => sets flag to true
+      # --long.flag.name true  => sets flag to true
+      # --long.flag.name false => sets flag to false
+      # --long.flag.name=true  => sets flag to true
+      # --long.flag.name=false => sets flag to false
+      def extract_value(switch, arguments)
+        if flag? && (arguments.first.nil? || arguments.first.match("^-"))
+          flag_value(switch)
+        else
+          arguments.shift
+        end
+      end
+    end
+  end
+
+  # Create a subclass of Clamp::Command that enforces the use of
+  # LogStash::SETTINGS for setting validation
+  class StrictCommand < Command
+    class << self
+      include ::Clamp::Option::StrictDeclaration
+    end
+  end
+end
+
+

data/lib/logstash/pipeline.rb

@@ -9,8 +9,6 @@ require "logstash/config/file"
 require "logstash/filters/base"
 require "logstash/inputs/base"
 require "logstash/outputs/base"
-require "logstash/config/cpu_core_strategy"
-require "logstash/util/defaults_printer"
 require "logstash/shutdown_watcher"
 require "logstash/util/wrapped_synchronous_queue"
 require "logstash/pipeline_reporter"
@@ -22,7 +20,7 @@ require "logstash/output_delegator"
 require "logstash/filter_delegator"
 
 module LogStash; class Pipeline
- attr_reader :inputs,
+  attr_reader :inputs,
     :filters,
     :outputs,
     :worker_threads,
@@ -30,39 +28,25 @@ module LogStash; class Pipeline
     :events_filtered,
     :reporter,
     :pipeline_id,
-    :metric,
     :logger,
     :started_at,
     :thread,
     :config_str,
-    :original_settings
-
-  DEFAULT_OUTPUT_WORKERS = 1
-
-  DEFAULT_SETTINGS = {
-    :default_pipeline_workers => LogStash::Config::CpuCoreStrategy.maximum,
-    :pipeline_batch_size => 125,
-    :pipeline_batch_delay => 5, # in milliseconds
-    :flush_interval => 5, # in seconds
-    :flush_timeout_interval => 60, # in seconds
-    :debug_config => false,
-    :allow_env => false
-  }
+    :settings
+  attr_accessor :metric
+
   MAX_INFLIGHT_WARN_THRESHOLD = 10_000
 
   RELOAD_INCOMPATIBLE_PLUGINS = [
     "LogStash::Inputs::Stdin"
   ]
 
-  def initialize(config_str, settings = {})
+  def initialize(config_str, settings = LogStash::SETTINGS)
     @config_str = config_str
-    @original_settings = settings
     @logger = Cabin::Channel.get(LogStash)
-    @pipeline_id = settings[:pipeline_id] || self.object_id
-    @settings = DEFAULT_SETTINGS.clone
-    settings.each {|setting, value| configure(setting, value) }
+    @settings = settings
+    @pipeline_id = @settings.get_value("pipeline.id") || self.object_id
     @reporter = LogStash::PipelineReporter.new(@logger, self)
-    @allow_env = settings[:allow_env]
 
     @inputs = nil
     @filters = nil
@@ -70,15 +54,9 @@ module LogStash; class Pipeline
 
     @worker_threads = []
 
-    # Metric object should be passed upstream, multiple pipeline share the same metric
-    # and collector only the namespace will changes.
-    # If no metric is given, we use a `NullMetric` for all internal calls.
-    # We also do this to make the changes backward compatible with previous testing of the
-    # pipeline.
-    #
-    # This need to be configured before we evaluate the code to make
-    # sure the metric instance is correctly send to the plugin.
-    @metric = settings.fetch(:metric, Instrument::NullMetric.new)
+    # This needs to be configured before we evaluate the code to make
+    # sure the metric instance is correctly send to the plugins to make the namespace scoping work
+    @metric = settings.get_value("metric.collect") ? Instrument::Metric.new : Instrument::NullMetric.new 
 
     grammar = LogStashConfigParser.new
     @config = grammar.parse(config_str)
@@ -94,7 +72,7 @@ module LogStash; class Pipeline
     # The config code is hard to represent as a log message...
     # So just print it.
 
-    if @settings[:debug_config] && logger.debug?
+    if @settings.get_value("config.debug") && logger.debug?
       logger.debug("Compiled pipeline code", :code => code)
     end
 
@@ -123,38 +101,29 @@ module LogStash; class Pipeline
     @ready.value
   end
 
-  def configure(setting, value)
-    @settings[setting] = value
-  end
-
   def safe_pipeline_worker_count
-    default = DEFAULT_SETTINGS[:default_pipeline_workers]
-    thread_count = @settings[:pipeline_workers] #override from args "-w 8" or config
+    default = @settings.get_default("pipeline.workers")
+    pipeline_workers = @settings.get("pipeline.workers") #override from args "-w 8" or config
     safe_filters, unsafe_filters = @filters.partition(&:threadsafe?)
+    plugins = unsafe_filters.collect { |f| f.config_name }
 
-    if unsafe_filters.any?
-      plugins = unsafe_filters.collect { |f| f.config_name }
-      case thread_count
-      when nil
-        # user did not specify a worker thread count
-        # warn if the default is multiple
-
-        if default > 1
-          @logger.warn("Defaulting pipeline worker threads to 1 because there are some filters that might not work with multiple worker threads",
-                       :count_was => default, :filters => plugins)
-        end
+    return pipeline_workers if unsafe_filters.empty?
 
-        1 # can't allow the default value to propagate if there are unsafe filters
-      when 0, 1
-        1
-      else
+    if @settings.set?("pipeline.workers")
+      if pipeline_workers > 1
         @logger.warn("Warning: Manual override - there are filters that might not work with multiple worker threads",
-                     :worker_threads => thread_count, :filters => plugins)
-        thread_count # allow user to force this even if there are unsafe filters
+                     :worker_threads => pipeline_workers, :filters => plugins)
       end
     else
-      thread_count || default
+      # user did not specify a worker thread count
+      # warn if the default is multiple
+      if default > 1
+        @logger.warn("Defaulting pipeline worker threads to 1 because there are some filters that might not work with multiple worker threads",
+                     :count_was => default, :filters => plugins)
+        return 1 # can't allow the default value to propagate if there are unsafe filters
+      end
     end
+    pipeline_workers
   end
 
   def filters?
@@ -164,9 +133,8 @@ module LogStash; class Pipeline
   def run
     @started_at = Time.now
 
-    LogStash::Util.set_thread_name("[#{pipeline_id}]-pipeline-manager")
-    @logger.terminal(LogStash::Util::DefaultsPrinter.print(@settings))
     @thread = Thread.current
+    LogStash::Util.set_thread_name("[#{pipeline_id}]-pipeline-manager")
 
     start_workers
 
@@ -217,15 +185,15 @@ module LogStash; class Pipeline
       @filters.each {|f| f.register }
 
       pipeline_workers = safe_pipeline_worker_count
-      batch_size = @settings[:pipeline_batch_size]
-      batch_delay = @settings[:pipeline_batch_delay]
+      batch_size = @settings.get("pipeline.batch.size")
+      batch_delay = @settings.get("pipeline.batch.delay")
       max_inflight = batch_size * pipeline_workers
       @logger.info("Starting pipeline",
-                   :id => self.pipeline_id,
-                   :pipeline_workers => pipeline_workers,
-                   :batch_size => batch_size,
-                   :batch_delay => batch_delay,
-                   :max_inflight => max_inflight)
+                   "id" => self.pipeline_id,
+                   "pipeline.workers" => pipeline_workers,
+                   "pipeline.batch.size" => batch_size,
+                   "pipeline.batch.delay" => batch_delay,
+                   "pipeline.max_inflight" => max_inflight)
       if max_inflight > MAX_INFLIGHT_WARN_THRESHOLD
         @logger.warn "CAUTION: Recommended inflight events max exceeded! Logstash will run with up to #{max_inflight} events in memory in your current configuration. If your message sizes are large this may cause instability with the default heap size. Please consider setting a non-standard heap size, changing the batch size (currently #{batch_size}), or changing the number of pipeline workers (currently #{pipeline_workers})"
       end
@@ -438,7 +406,7 @@ module LogStash; class Pipeline
   def shutdown_workers
     # Each worker thread will receive this exactly once!
     @worker_threads.each do |t|
-      @logger.debug("Pushing shutdown", :thread => t)
+      @logger.debug("Pushing shutdown", :thread => t.inspect)
       @input_queue.push(LogStash::SHUTDOWN)
     end
 
@@ -453,14 +421,13 @@ module LogStash; class Pipeline
 
   def plugin(plugin_type, name, *args)
     args << {} if args.empty?
-    args.first.merge!(LogStash::Config::Mixin::ALLOW_ENV_FLAG => @allow_env)
 
     pipeline_scoped_metric = metric.namespace([:stats, :pipelines, pipeline_id.to_s.to_sym, :plugins])
 
     klass = LogStash::Plugin.lookup(plugin_type, name)
 
     if plugin_type == "output"
-      LogStash::OutputDelegator.new(@logger, klass, DEFAULT_OUTPUT_WORKERS, pipeline_scoped_metric.namespace(:outputs), *args)
+      LogStash::OutputDelegator.new(@logger, klass, @settings.get("pipeline.output.workers"), pipeline_scoped_metric.namespace(:outputs), *args)
     elsif plugin_type == "filter"
       LogStash::FilterDelegator.new(@logger, klass, pipeline_scoped_metric.namespace(:filters), *args)
     else