logstash-core 5.0.0.alpha5.snapshot1-java → 5.0.0.alpha6.snapshot1-java

data/lib/logstash/output_delegator_strategies/legacy.rb

@@ -0,0 +1,29 @@
+# Remove this in Logstash 6.0
+module LogStash module OutputDelegatorStrategies class Legacy
+  attr_reader :worker_count, :workers
+  
+  def initialize(logger, klass, metric, plugin_args)
+    @worker_count = (plugin_args["workers"] || 1).to_i
+    @workers = @worker_count.times.map {|t| klass.new(plugin_args)}
+    @worker_queue = SizedQueue.new(@worker_count)
+    @workers.each {|w| @worker_queue << w}
+  end
+  
+  def register
+    @workers.each(&:register)
+  end
+  
+  def multi_receive(events)
+    worker = @worker_queue.pop
+    worker.multi_receive(events)
+  ensure
+    @worker_queue << worker if worker
+  end
+
+  def do_close
+    # No mutex needed since this is only called when the pipeline is clear
+    @workers.each(&:do_close)
+  end
+
+  ::LogStash::OutputDelegatorStrategyRegistry.instance.register(:legacy, self)
+end; end; end

data/lib/logstash/output_delegator_strategies/shared.rb

@@ -0,0 +1,20 @@
+module LogStash module OutputDelegatorStrategies class Shared
+  def initialize(logger, klass, metric, plugin_args)
+    @output = klass.new(plugin_args)
+  end
+  
+  def register
+    @output.register
+  end
+
+  def multi_receive(events)
+    @output.multi_receive(events)
+  end
+
+  def do_close    
+    @output.do_close
+  end
+
+  ::LogStash::OutputDelegatorStrategyRegistry.instance.register(:shared, self)  
+end; end; end
+

data/lib/logstash/output_delegator_strategies/single.rb

@@ -0,0 +1,23 @@
+module LogStash module OutputDelegatorStrategies class Single
+  def initialize(logger, klass, metric, plugin_args)
+    @output = klass.new(plugin_args)
+    @mutex = Mutex.new
+  end
+
+  def register
+    @output.register
+  end
+  
+  def multi_receive(events)
+    @mutex.synchronize do
+      @output.multi_receive(events)
+    end
+  end
+
+  def do_close
+    # No mutex needed since this is only called when the pipeline is clear
+    @output.do_close
+  end
+
+  ::LogStash::OutputDelegatorStrategyRegistry.instance.register(:single, self)
+end; end; end

data/lib/logstash/output_delegator_strategy_registry.rb

@@ -0,0 +1,36 @@
+module LogStash; class OutputDelegatorStrategyRegistry
+  class InvalidStrategyError < StandardError; end
+                   
+  # This is generally used as a singleton
+  # Except perhaps during testing
+  def self.instance
+    @instance ||= self.new
+  end
+
+  def initialize()
+    @map = {}
+  end
+
+  def classes
+    @map.values
+  end
+
+  def types
+    @map.keys
+  end
+  
+  def class_for(type)
+    klass = @map[type]
+
+    if !klass
+      raise InvalidStrategyError, "Could not find output delegator strategy of type '#{type}'. Valid strategies: #{@strategy_registry.types}"
+    end
+
+    klass
+  end
+
+  def register(type, klass)
+    @map[type] = klass
+  end
+
+end; end

data/lib/logstash/outputs/base.rb

@@ -20,42 +20,37 @@ class LogStash::Outputs::Base < LogStash::Plugin
 
   # The codec used for output data. Output codecs are a convenient method for encoding your data before it leaves the output, without needing a separate filter in your Logstash pipeline.
   config :codec, :validate => :codec, :default => "plain"
+  # TODO remove this in Logstash 6.0
+  # when we no longer support the :legacy type
+  # This is hacky, but it can only be herne
+  config :workers, :type => :number, :default => 1
+  
+  # Set or return concurrency type
+  def self.concurrency(type=nil)
+    if type
+      @concurrency = type
+    else
+      @concurrency || :legacy # default is :legacyo
+    end
+  end
 
-  # The number of workers to use for this output.
-  # Note that this setting may not be useful for all outputs.
-  config :workers, :validate => :number, :default => 1
-
-  attr_reader :worker_plugins, :available_workers, :workers, :worker_plugins, :workers_not_supported
-
+  # Deprecated: Favor `concurrency :shared`
   def self.declare_threadsafe!
-    declare_workers_not_supported!
-    @threadsafe = true
+    concurrency :shared
   end
 
+  # Deprecated: Favor `#concurrency`
   def self.threadsafe?
-    @threadsafe == true
+    concurrency == :shared
   end
 
+  # Deprecated: Favor `concurrency :single`
+  # Remove in Logstash 6.0.0
   def self.declare_workers_not_supported!(message=nil)
-    @workers_not_supported_message = message
-    @workers_not_supported = true
-  end
-
-  def self.workers_not_supported_message
-    @workers_not_supported_message
-  end
-
-  def self.workers_not_supported?
-    !!@workers_not_supported
+    concurrency :single
   end
 
   public
-  # TODO: Remove this in the next major version after Logstash 2.x
-  # Post 2.x it should raise an error and tell people to use the class level
-  # declaration
-  def workers_not_supported(message=nil)
-    self.class.declare_workers_not_supported!(message)
-  end
 
   def self.plugin_type
     "output"
@@ -66,9 +61,15 @@ class LogStash::Outputs::Base < LogStash::Plugin
     super
     config_init(@params)
 
+    if self.workers != 1
+      raise LogStash::ConfigurationError, "You are using a plugin that doesn't support workers but have set the workers value explicitly! This plugin uses the #{concurrency} and doesn't need this option"
+    end
+
     # If we're running with a single thread we must enforce single-threaded concurrency by default
     # Maybe in a future version we'll assume output plugins are threadsafe
     @single_worker_mutex = Mutex.new
+    
+    @receives_encoded = self.methods.include?(:multi_receive_encoded)
   end
 
   public
@@ -84,7 +85,19 @@ class LogStash::Outputs::Base < LogStash::Plugin
   public
   # To be overriden in implementations
   def multi_receive(events)
-    events.each {|event| receive(event) }
+    if @receives_encoded
+      self.multi_receive_encoded(codec.multi_encode(events))
+    else
+      events.each {|event| receive(event) }
+    end
+  end
+
+  def codec
+    params["codec"]
+  end
+
+  def concurrency
+    self.class.concurrency
   end
 
   private

data/lib/logstash/patches/clamp.rb

@@ -63,6 +63,12 @@ module Clamp
     class << self
       include ::Clamp::Option::StrictDeclaration
     end
+
+    def handle_remaining_arguments
+      unless remaining_arguments.empty?
+        signal_usage_error "Unknown command '#{remaining_arguments.first}'"
+      end
+    end
   end
 end
 

data/lib/logstash/pipeline.rb

@@ -32,6 +32,7 @@ module LogStash; class Pipeline
     :started_at,
     :thread,
     :config_str,
+    :config_hash,
     :settings,
     :metric,
     :filter_queue_client,
@@ -45,11 +46,18 @@ module LogStash; class Pipeline
 
   def initialize(config_str, settings = LogStash::SETTINGS, namespaced_metric = nil)
     @config_str = config_str
+    @config_hash = Digest::SHA1.hexdigest(@config_str)
+    # Every time #plugin is invoked this is incremented to give each plugin
+    # a unique id when auto-generating plugin ids
+    @plugin_counter ||= 0 
+    
     @logger = Cabin::Channel.get(LogStash)
     @settings = settings
     @pipeline_id = @settings.get_value("pipeline.id") || self.object_id
     @reporter = LogStash::PipelineReporter.new(@logger, self)
 
+    # A list of plugins indexed by id
+    @plugins_by_id = {}
     @inputs = nil
     @filters = nil
     @outputs = nil
@@ -87,7 +95,7 @@ module LogStash; class Pipeline
     queue = LogStash::Util::WrappedSynchronousQueue.new
     @input_queue_client = queue.write_client
     @filter_queue_client = queue.read_client
-    # Note that @infilght_batches as a central mechanism for tracking inflight
+    # Note that @inflight_batches as a central mechanism for tracking inflight
     # batches will fail if we have multiple read clients here.
     @filter_queue_client.set_events_metric(metric.namespace([:stats, :events]))
     @filter_queue_client.set_pipeline_metric(
@@ -198,6 +206,8 @@ module LogStash; class Pipeline
       config_metric.gauge(:workers, pipeline_workers)
       config_metric.gauge(:batch_size, batch_size)
       config_metric.gauge(:batch_delay, batch_delay)
+      config_metric.gauge(:config_reload_automatic, @settings.get("config.reload.automatic"))
+      config_metric.gauge(:config_reload_interval, @settings.get("config.reload.interval"))
 
       @logger.info("Starting pipeline",
                    "id" => self.pipeline_id,
@@ -288,7 +298,10 @@ module LogStash; class Pipeline
     end
     # Now that we have our output to event mapping we can just invoke each output
     # once with its list of events
-    output_events_map.each { |output, events| output.multi_receive(events) }
+    output_events_map.each do |output, events|
+      output.multi_receive(events)
+    end
+    
     @filter_queue_client.add_output_metrics(batch)
   end
 
@@ -387,23 +400,38 @@ module LogStash; class Pipeline
   end
 
   def plugin(plugin_type, name, *args)
-    args << {} if args.empty?
+    @plugin_counter += 1
+
+    # Collapse the array of arguments into a single merged hash
+    args = args.reduce({}, &:merge)
+
+    id = if args["id"].nil? || args["id"].empty?
+           args["id"] = "#{@config_hash}-#{@plugin_counter}"
+         else
+           args["id"]
+         end
 
+    raise LogStash::ConfigurationError, "Two plugins have the id '#{id}', please fix this conflict" if @plugins_by_id[id]
+    
     pipeline_scoped_metric = metric.namespace([:stats, :pipelines, pipeline_id.to_s.to_sym, :plugins])
 
     klass = LogStash::Plugin.lookup(plugin_type, name)
 
-    if plugin_type == "output"
-      LogStash::OutputDelegator.new(@logger, klass, @settings.get("pipeline.output.workers"), pipeline_scoped_metric.namespace(:outputs), *args)
-    elsif plugin_type == "filter"
-      LogStash::FilterDelegator.new(@logger, klass, pipeline_scoped_metric.namespace(:filters), *args)
-    else
-      new_plugin = klass.new(*args)
-      inputs_metric = pipeline_scoped_metric.namespace(:inputs)
-      namespaced_metric = inputs_metric.namespace(new_plugin.plugin_unique_name.to_sym)
-      new_plugin.metric = namespaced_metric
-      new_plugin
-    end
+    # Scope plugins of type 'input' to 'inputs'
+    type_scoped_metric = pipeline_scoped_metric.namespace("#{plugin_type}s".to_sym)
+    plugin = if plugin_type == "output"
+               OutputDelegator.new(@logger, klass, type_scoped_metric,
+                                   ::LogStash::OutputDelegatorStrategyRegistry.instance,
+                                   args)
+             elsif plugin_type == "filter"
+               LogStash::FilterDelegator.new(@logger, klass, type_scoped_metric, args)
+             else # input
+               input_plugin = klass.new(args)
+               input_plugin.metric = type_scoped_metric.namespace(id)
+               input_plugin
+             end
+    
+    @plugins_by_id[id] = plugin
   end
 
   # for backward compatibility in devutils for the rspec helpers, this method is not used

data/lib/logstash/pipeline_reporter.rb

@@ -60,7 +60,6 @@ module LogStash; class PipelineReporter
       {
         :events_filtered => events_filtered,
         :events_consumed => events_consumed,
-        :worker_count => pipeline.worker_threads.size,
         :inflight_count => inflight_count,
         :worker_states => worker_states_snap,
         :output_info => output_info,
@@ -100,15 +99,10 @@ module LogStash; class PipelineReporter
 
   def output_info
     pipeline.outputs.map do |output_delegator|
-      is_multi_worker = output_delegator.worker_count > 1
-
       {
         :type => output_delegator.config_name,
-        :config => output_delegator.config,
-        :is_multi_worker => is_multi_worker,
-        :events_received => output_delegator.events_received,
-        :workers => output_delegator.workers,
-        :busy_workers => output_delegator.busy_workers
+        :plugin_args => output_delegator.plugin_args,
+        :concurrency => output_delegator.concurrency,        
       }
     end
   end

data/lib/logstash/plugin.rb

@@ -45,8 +45,11 @@ class LogStash::Plugin
     self.class.name == other.class.name && @params == other.params
   end
 
-  def initialize(params=nil)
+  def initialize(params=nil)    
     @params = LogStash::Util.deep_clone(params)
+    # The id should always be defined normally, but in tests that might not be the case
+    # In the future we may make this more strict in the Plugin API
+    @params["id"] ||= "#{self.class.config_name}_#{SecureRandom.uuid}"
     @logger = Cabin::Channel.get(LogStash)
   end
 
@@ -57,15 +60,7 @@ class LogStash::Plugin
   #
   # @return [String] A plugin ID
   def id
-    (@params["id"].nil? || @params["id"].empty?) ? SecureRandom.uuid : @params["id"]
-  end
-
-  # Return a unique_name, This is composed by the name of
-  # the plugin and the generated ID (of the configured one)
-  #
-  # @return [String] a unique name
-  def plugin_unique_name
-    "#{config_name}_#{id}"
+    @params["id"]
   end
 
   # close is called during shutdown, after the plugin worker
@@ -127,6 +122,7 @@ class LogStash::Plugin
     LogStash::Registry.instance.lookup(type ,name) do |plugin_klass, plugin_name|
       is_a_plugin?(plugin_klass, plugin_name)
     end
+    
   rescue LoadError, NameError => e
     logger.debug("Problems loading the plugin with", :type => type, :name => name, :path => path)
     raise(LogStash::PluginLoadingError, I18n.t("logstash.pipeline.plugin-loading-error", :type => type, :name => name, :path => path, :error => e.to_s))

data/lib/logstash/runner.rb

@@ -47,17 +47,17 @@ class LogStash::Runner < Clamp::StrictCommand
   option ["-w", "--pipeline.workers"], "COUNT",
     I18n.t("logstash.runner.flag.pipeline-workers"),
     :attribute_name => "pipeline.workers",
-    :default => LogStash::SETTINGS.get_default("pipeline.workers"), &:to_i
+    :default => LogStash::SETTINGS.get_default("pipeline.workers")
 
   option ["-b", "--pipeline.batch.size"], "SIZE",
     I18n.t("logstash.runner.flag.pipeline-batch-size"),
     :attribute_name => "pipeline.batch.size",
-    :default => LogStash::SETTINGS.get_default("pipeline.batch.size"), &:to_i
+    :default => LogStash::SETTINGS.get_default("pipeline.batch.size")
 
   option ["-u", "--pipeline.batch.delay"], "DELAY_IN_MS",
     I18n.t("logstash.runner.flag.pipeline-batch-delay"),
     :attribute_name => "pipeline.batch.delay",
-    :default => LogStash::SETTINGS.get_default("pipeline.batch.delay"), &:to_i
+    :default => LogStash::SETTINGS.get_default("pipeline.batch.delay")
 
   option ["--pipeline.unsafe_shutdown"], :flag,
     I18n.t("logstash.runner.flag.unsafe_shutdown"),
@@ -110,7 +110,7 @@ class LogStash::Runner < Clamp::StrictCommand
   option ["--config.reload.interval"], "RELOAD_INTERVAL",
     I18n.t("logstash.runner.flag.reload_interval"),
     :attribute_name => "config.reload.interval",
-    :default => LogStash::SETTINGS.get_default("config.reload.interval"), &:to_i
+    :default => LogStash::SETTINGS.get_default("config.reload.interval")
 
   option ["--http.host"], "HTTP_HOST",
     I18n.t("logstash.runner.flag.http_host"),
@@ -120,7 +120,7 @@ class LogStash::Runner < Clamp::StrictCommand
   option ["--http.port"], "HTTP_PORT",
     I18n.t("logstash.runner.flag.http_port"),
     :attribute_name => "http.port",
-    :default => LogStash::SETTINGS.get_default("http.port"), &:to_i
+    :default => LogStash::SETTINGS.get_default("http.port")
 
   option ["--log.format"], "FORMAT",
     I18n.t("logstash.runner.flag.log_format"),
@@ -148,10 +148,13 @@ class LogStash::Runner < Clamp::StrictCommand
     begin
       LogStash::SETTINGS.from_yaml(LogStash::SETTINGS.get("path.settings"))
     rescue => e
-      @logger.subscribe(STDOUT)
-      @logger.warn("Logstash has a new settings file which defines start up time settings. This file is typically located in $LS_HOME/config or /etc/logstash. If you installed Logstash through a package and are starting it manually please specify the location to this settings file by passing in \"--path.settings=/path/..\" in the command line options")
-      @logger.fatal("Failed to load settings file from \"path.settings\". Aborting...", "path.settings" => LogStash::SETTINGS.get("path.settings"), "exception" => e.class, "message" => e.message)
-      exit(-1)
+      # abort unless we're just looking for the help
+      if (["--help", "-h"] & args).empty?
+        @logger.subscribe(STDOUT)
+        @logger.warn("Logstash has a new settings file which defines start up time settings. This file is typically located in $LS_HOME/config or /etc/logstash. If you installed Logstash through a package and are starting it manually please specify the location to this settings file by passing in \"--path.settings=/path/..\" in the command line options")
+        @logger.fatal("Failed to load settings file from \"path.settings\". Aborting...", "path.settings" => LogStash::SETTINGS.get("path.settings"), "exception" => e.class, "message" => e.message)
+        return 1
+      end
     end
 
     super(*[args])