logstash-core 5.2.2-java → 5.3.0-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 72571fd4dcaa96000b854ac8362522c0db5d3e82
-  data.tar.gz: 6ff9cf5d47543adfb8e45d6b28163bebe9ce40ac
+  metadata.gz: 32438210f44fc19cf00eefbd7dc9b439936be0dd
+  data.tar.gz: 67e8f76e6c310aa08de14edce2170e8658c08d6f
 SHA512:
-  metadata.gz: a7aaa38c146c061f56e6654b70bcfda14c89516ebca3a98fdad2f7b347b423bc78eae89cef47547ee6a1aaa725fda4ab2c9849942ec1619eecba862996d34458
-  data.tar.gz: 3111f88041c5367e1724c741c625875c3bbe0e8035107b86ac60fb7651922c8b4efa5f71d02495674549eec75d5a34fbcb6ddf80c49b515e6bd73eedf64c5ffb
+  metadata.gz: b9a50f2852a3cee3dcf045d56f5650e4d521e0e792fb557e326e1fd70b42dd57f843ebc43c1c5d74d967d706e0d51a73ffc0a87a402ef32426f6128e404a0a42
+  data.tar.gz: 2f03c599450bf46ae2db2a367360710a5b693dfcf3f1d81b0f342ebcbf69de2c0e87a2a79619ae8c73c4914659b58dabcd8be5cb798db80e3e18230f0ee248b0

data/lib/logstash-core/version.rb

@@ -5,4 +5,4 @@
 # Note to authors: this should not include dashes because 'gem' barfs if
 # you include a dash in the version string.
 
-LOGSTASH_CORE_VERSION = "5.2.2"
+LOGSTASH_CORE_VERSION = "5.3.0"

data/lib/logstash/agent.rb

@@ -88,13 +88,13 @@ class LogStash::Agent
   # @param pipeline_id [String] pipeline string identifier
   # @param settings [Hash] settings that will be passed when creating the pipeline.
   #   keys should be symbols such as :pipeline_workers and :pipeline_batch_delay
-  def register_pipeline(pipeline_id, settings = @settings)
+  def register_pipeline(settings)
     pipeline_settings = settings.clone
-    pipeline_settings.set("pipeline.id", pipeline_id)
+    pipeline_id = pipeline_settings.get("pipeline.id")
 
     pipeline = create_pipeline(pipeline_settings)
     return unless pipeline.is_a?(LogStash::Pipeline)
-    if @auto_reload && pipeline.non_reloadable_plugins.any?
+    if @auto_reload && !pipeline.reloadable?
       @logger.error(I18n.t("logstash.agent.non_reloadable_config_register"),
                     :pipeline_id => pipeline_id,
                     :plugins => pipeline.non_reloadable_plugins.map(&:class))
@@ -222,16 +222,13 @@ class LogStash::Agent
     @collector = LogStash::Instrument::Collector.new
 
     @metric = if collect_metrics?
-                @logger.debug("Agent: Configuring metric collection")
-                LogStash::Instrument::Metric.new(@collector)
-              else
-                LogStash::Instrument::NullMetric.new(@collector)
-              end
-
+      @logger.debug("Agent: Configuring metric collection")
+      LogStash::Instrument::Metric.new(@collector)
+    else
+      LogStash::Instrument::NullMetric.new(@collector)
+    end
 
-    @periodic_pollers = LogStash::Instrument::PeriodicPollers.new(@metric,
-                                                                  settings.get("queue.type"),
-                                                                  self)
+    @periodic_pollers = LogStash::Instrument::PeriodicPollers.new(@metric, settings.get("queue.type"), self)
     @periodic_pollers.start
   end
 
@@ -261,13 +258,18 @@ class LogStash::Agent
     end
   end
 
+  # create a new pipeline with the given settings and config, if the pipeline initialization failed
+  # increment the failures metrics
+  # @param settings [Settings] the setting for the new pipelines
+  # @param config [String] the configuration string or nil to fetch the configuration per settings
+  # @return [Pipeline] the new pipeline or nil if it failed
   def create_pipeline(settings, config = nil)
     if config.nil?
       begin
         config = fetch_config(settings)
       rescue => e
         @logger.error("failed to fetch pipeline configuration", :message => e.message)
-        return
+        return nil
       end
     end
 
@@ -275,7 +277,7 @@ class LogStash::Agent
       LogStash::Pipeline.new(config, settings, metric)
     rescue => e
       increment_reload_failures_metrics(settings.get("pipeline.id"), e.message, e.backtrace)
-      return
+      return nil
     end
   end
 
@@ -297,7 +299,7 @@ class LogStash::Agent
 
     # check if this pipeline is not reloadable. it should not happen as per the check below
     # but keep it here as a safety net if a reloadable pipeline was releoaded with a non reloadable pipeline
-    if old_pipeline.non_reloadable_plugins.any?
+    if !old_pipeline.reloadable?
       @logger.error("pipeline is not reloadable", :pipeline => id)
       return
     end
@@ -312,7 +314,7 @@ class LogStash::Agent
     end
 
     # check if the new pipeline will be reloadable in which case we want to log that as an error and abort
-    if pipeline_validator.non_reloadable_plugins.any?
+    if !pipeline_validator.reloadable?
       @logger.error(I18n.t("logstash.agent.non_reloadable_config_reload"), :pipeline_id => id, :plugins => pipeline_validator.non_reloadable_plugins.map(&:class))
       increment_reload_failures_metrics(id, "non reloadable pipeline")
       return
@@ -351,7 +353,7 @@ class LogStash::Agent
 
     # check if the new pipeline will be reloadable in which case we want to log that as an error and abort. this should normally not
     # happen since the check should be done in reload_pipeline! prior to get here.
-    if new_pipeline.non_reloadable_plugins.any?
+    if !new_pipeline.reloadable?
       @logger.error(I18n.t("logstash.agent.non_reloadable_config_reload"), :pipeline_id => pipeline_id, :plugins => new_pipeline.non_reloadable_plugins.map(&:class))
       increment_reload_failures_metrics(pipeline_id, "non reloadable pipeline")
       new_pipeline.close
@@ -393,6 +395,8 @@ class LogStash::Agent
           n.gauge(:last_failure_timestamp, LogStash::Timestamp.now)
         end
         @logger.error("Pipeline aborted due to error", :exception => e, :backtrace => e.backtrace)
+
+        # TODO: this is weird, why dont we return directly here? any reason we need to enter the while true loop below?!
       end
     end
     while true do

data/lib/logstash/api/commands/node.rb

@@ -17,11 +17,12 @@ module LogStash
           payload
         end
 
-        def pipeline
-          extract_metrics(
-            [:stats, :pipelines, :main, :config],
+        def pipeline(pipeline_id = LogStash::SETTINGS.get("pipeline.id").to_sym)
+          stats = extract_metrics(
+            [:stats, :pipelines, pipeline_id, :config],
             :workers, :batch_size, :batch_delay, :config_reload_automatic, :config_reload_interval
           )
+          stats.merge(:id => pipeline_id)
         end
 
         def os

data/lib/logstash/api/commands/stats.rb

@@ -45,9 +45,10 @@ module LogStash
           )
         end
 
-        def pipeline
-          stats = service.get_shallow(:stats, :pipelines)
-          PluginsStats.report(stats)
+        def pipeline(pipeline_id = LogStash::SETTINGS.get("pipeline.id").to_sym)
+          stats = service.get_shallow(:stats, :pipelines, pipeline_id)
+          stats = PluginsStats.report(stats)
+          stats.merge(:id => pipeline_id)
         end
 
         def memory
@@ -98,9 +99,6 @@ module LogStash
           end
 
           def report(stats)
-            # Only one pipeline right now.
-            stats = stats[:main]
-
             {
               :events => stats[:events],
               :plugins => {

data/lib/logstash/filter_delegator.rb

@@ -9,7 +9,8 @@ module LogStash
       :threadsafe?,
       :do_close,
       :do_stop,
-      :periodic_flush
+      :periodic_flush,
+      :reloadable?
     ]
     def_delegators :@filter, *DELEGATED_METHODS
 

data/lib/logstash/output_delegator.rb

@@ -29,6 +29,10 @@ module LogStash class OutputDelegator
     @output_class.config_name
   end
 
+  def reloadable?
+    @output_class.reloadable?
+  end
+
   def concurrency
     @output_class.concurrency
   end

data/lib/logstash/pipeline.rb

@@ -5,8 +5,6 @@ require "concurrent"
 require "logstash/namespace"
 require "logstash/errors"
 require "logstash-core/logstash-core"
-require "logstash/util/wrapped_acked_queue"
-require "logstash/util/wrapped_synchronous_queue"
 require "logstash/event"
 require "logstash/config/file"
 require "logstash/filters/base"
@@ -21,16 +19,13 @@ require "logstash/instrument/namespaced_null_metric"
 require "logstash/instrument/collector"
 require "logstash/output_delegator"
 require "logstash/filter_delegator"
+require "logstash/queue_factory"
 
 module LogStash; class BasePipeline
   include LogStash::Util::Loggable
 
   attr_reader :config_str, :config_hash, :inputs, :filters, :outputs, :pipeline_id
 
-  RELOAD_INCOMPATIBLE_PLUGINS = [
-      "LogStash::Inputs::Stdin"
-  ]
-
   def initialize(config_str, settings = SETTINGS)
     @logger = self.logger
     @config_str = config_str
@@ -75,27 +70,25 @@ module LogStash; class BasePipeline
     args = args.reduce({}, &:merge)
 
     id = if args["id"].nil? || args["id"].empty?
-           args["id"] = "#{@config_hash}-#{@plugin_counter}"
-         else
-           args["id"]
-         end
+      args["id"] = "#{@config_hash}-#{@plugin_counter}"
+    else
+      args["id"]
+    end
 
     raise ConfigurationError, "Two plugins have the id '#{id}', please fix this conflict" if @plugins_by_id[id]
-
     @plugins_by_id[id] = true
 
     # use NullMetric if called in the BasePipeline context otherwise use the @metric value
     metric = @metric || Instrument::NullMetric.new
 
     pipeline_scoped_metric = metric.namespace([:stats, :pipelines, pipeline_id.to_s.to_sym, :plugins])
-
     # Scope plugins of type 'input' to 'inputs'
     type_scoped_metric = pipeline_scoped_metric.namespace("#{plugin_type}s".to_sym)
 
     klass = Plugin.lookup(plugin_type, name)
 
     if plugin_type == "output"
-      OutputDelegator.new(@logger, klass, type_scoped_metric, OutputDelegatorStrategyRegistry.instance,  args)
+      OutputDelegator.new(@logger, klass, type_scoped_metric,  OutputDelegatorStrategyRegistry.instance, args)
     elsif plugin_type == "filter"
       FilterDelegator.new(@logger, klass, type_scoped_metric, args)
     else # input
@@ -105,10 +98,12 @@ module LogStash; class BasePipeline
     end
   end
 
+  def reloadable?
+    non_reloadable_plugins.empty?
+  end
+
   def non_reloadable_plugins
-    (inputs + filters + outputs).select do |plugin|
-      RELOAD_INCOMPATIBLE_PLUGINS.include?(plugin.class.name)
-    end
+    (inputs + filters + outputs).select { |plugin| !plugin.reloadable? }
   end
 end; end
 
@@ -129,13 +124,13 @@ module LogStash; class Pipeline < BasePipeline
   MAX_INFLIGHT_WARN_THRESHOLD = 10_000
 
   def initialize(config_str, settings = SETTINGS, namespaced_metric = nil)
-    # This needs to be configured before we evaluate the code to make
+    # This needs to be configured before we call super which will evaluate the code to make
     # sure the metric instance is correctly send to the plugins to make the namespace scoping work
     @metric = if namespaced_metric
-                settings.get("metric.collect") ? namespaced_metric : Instrument::NullMetric.new(namespaced_metric.collector)
-              else
-                Instrument::NullMetric.new
-              end
+      settings.get("metric.collect") ? namespaced_metric : Instrument::NullMetric.new(namespaced_metric.collector)
+    else
+      Instrument::NullMetric.new
+    end
 
     @settings = settings
     @reporter = PipelineReporter.new(@logger, self)
@@ -143,7 +138,13 @@ module LogStash; class Pipeline < BasePipeline
 
     super(config_str, settings)
 
-    @queue = build_queue_from_settings
+    begin
+      @queue = LogStash::QueueFactory.create(settings)
+    rescue => e
+      @logger.error("Logstash failed to create queue", "exception" => e.message, "backtrace" => e.backtrace)
+      raise e
+    end
+
     @input_queue_client = @queue.write_client
     @filter_queue_client = @queue.read_client
     @signal_queue = Queue.new
@@ -164,32 +165,6 @@ module LogStash; class Pipeline < BasePipeline
     @flushing = Concurrent::AtomicReference.new(false)
   end # def initialize
 
-  def build_queue_from_settings
-    queue_type = settings.get("queue.type")
-    queue_page_capacity = settings.get("queue.page_capacity")
-    queue_max_bytes = settings.get("queue.max_bytes")
-    queue_max_events = settings.get("queue.max_events")
-    checkpoint_max_acks = settings.get("queue.checkpoint.acks")
-    checkpoint_max_writes = settings.get("queue.checkpoint.writes")
-    checkpoint_max_interval = settings.get("queue.checkpoint.interval")
-
-    if queue_type == "memory_acked"
-      # memory_acked is used in tests/specs
-      LogStash::Util::WrappedAckedQueue.create_memory_based("", queue_page_capacity, queue_max_events, queue_max_bytes)
-    elsif queue_type == "memory"
-      # memory is the legacy and default setting
-      LogStash::Util::WrappedSynchronousQueue.new()
-    elsif queue_type == "persisted"
-      # persisted is the disk based acked queue
-      queue_path = settings.get("path.queue")
-      LogStash::Util::WrappedAckedQueue.create_file_based(queue_path, queue_page_capacity, queue_max_events, checkpoint_max_writes, checkpoint_max_acks, checkpoint_max_interval, queue_max_bytes)
-    else
-      raise(ConfigurationError, "invalid queue.type setting")
-    end
-  end
-
-  private :build_queue_from_settings
-
   def ready?
     @ready.value
   end
@@ -390,8 +365,9 @@ module LogStash; class Pipeline < BasePipeline
     # Users need to check their configuration or see if there is a bug in the
     # plugin.
     @logger.error("Exception in pipelineworker, the pipeline stopped processing new events, please check your filter configuration and restart Logstash.",
-                  "exception" => e, "backtrace" => e.backtrace)
-    raise
+                  "exception" => e.message, "backtrace" => e.backtrace)
+
+    raise e
   end
 
   # Take an array of events and send them to the correct output
@@ -450,7 +426,7 @@ module LogStash; class Pipeline < BasePipeline
     rescue => e
       if plugin.stop?
         @logger.debug("Input plugin raised exception during shutdown, ignoring it.",
-                      :plugin => plugin.class.config_name, :exception => e,
+                      :plugin => plugin.class.config_name, :exception => e.message,
                       :backtrace => e.backtrace)
         return
       end
@@ -458,12 +434,12 @@ module LogStash; class Pipeline < BasePipeline
       # otherwise, report error and restart
       if @logger.debug?
         @logger.error(I18n.t("logstash.pipeline.worker-error-debug",
-                             :plugin => plugin.inspect, :error => e.to_s,
+                             :plugin => plugin.inspect, :error => e.message,
                              :exception => e.class,
                              :stacktrace => e.backtrace.join("\n")))
       else
         @logger.error(I18n.t("logstash.pipeline.worker-error",
-                             :plugin => plugin.inspect, :error => e))
+                             :plugin => plugin.inspect, :error => e.message))
       end
 
       # Assuming the failure that caused this exception is transient,
@@ -628,4 +604,4 @@ module LogStash; class Pipeline < BasePipeline
       :flushing => @flushing
     }
   end
-end; end
+end end

data/lib/logstash/plugin.rb

@@ -96,6 +96,14 @@ class LogStash::Plugin
     end
   end
 
+  def reloadable?
+    self.class.reloadable?
+  end
+
+  def self.reloadable?
+    true
+  end
+
   def debug_info
     [self.class.to_s, original_params]
   end

data/lib/logstash/queue_factory.rb

@@ -0,0 +1,36 @@
+# encoding: utf-8
+require "fileutils"
+require "logstash/event"
+require "logstash/namespace"
+require "logstash/util/wrapped_acked_queue"
+require "logstash/util/wrapped_synchronous_queue"
+
+module LogStash
+  class QueueFactory
+    def self.create(settings)
+      queue_type = settings.get("queue.type")
+      queue_page_capacity = settings.get("queue.page_capacity")
+      queue_max_bytes = settings.get("queue.max_bytes")
+      queue_max_events = settings.get("queue.max_events")
+      checkpoint_max_acks = settings.get("queue.checkpoint.acks")
+      checkpoint_max_writes = settings.get("queue.checkpoint.writes")
+      checkpoint_max_interval = settings.get("queue.checkpoint.interval")
+
+      case queue_type
+      when "memory_acked"
+        # memory_acked is used in tests/specs
+        LogStash::Util::WrappedAckedQueue.create_memory_based("", queue_page_capacity, queue_max_events, queue_max_bytes)
+      when "memory"
+        # memory is the legacy and default setting
+        LogStash::Util::WrappedSynchronousQueue.new
+      when "persisted"
+        # persisted is the disk based acked queue
+        queue_path = ::File.join(settings.get("path.queue"), settings.get("pipeline.id"))
+        FileUtils.mkdir_p(queue_path)
+        LogStash::Util::WrappedAckedQueue.create_file_based(queue_path, queue_page_capacity, queue_max_events, checkpoint_max_writes, checkpoint_max_acks, checkpoint_max_interval, queue_max_bytes)
+      else
+        raise ConfigurationError, "Invalid setting `#{queue_type}` for `queue.type`, supported types are: 'memory_acked', 'memory', 'persisted'"
+      end
+    end
+  end
+end

data/lib/logstash/runner.rb

@@ -200,7 +200,7 @@ class LogStash::Runner < Clamp::StrictCommand
     # override log level that may have been introduced from a custom log4j config file
     LogStash::Logging::Logger::configure_logging(setting("log.level"))
 
-    if setting("config.debug") && logger.debug?
+    if setting("config.debug") && !logger.debug?
       logger.warn("--config.debug was specified, but log.level was not set to \'debug\'! No config info will be logged.")
     end
 
@@ -261,7 +261,7 @@ class LogStash::Runner < Clamp::StrictCommand
 
     @agent = create_agent(@settings)
 
-    @agent.register_pipeline("main", @settings)
+    @agent.register_pipeline(@settings)
 
     # enable sigint/sigterm before starting the agent
     # to properly handle a stalled agent

data/lib/logstash/version.rb

@@ -11,4 +11,4 @@
 #       eventually this file should be in the root logstash lib fir and dependencies in logstash-core should be
 #       fixed.
 
-LOGSTASH_VERSION = "5.2.2"
+LOGSTASH_VERSION = "5.3.0"

data/spec/api/spec_helper.rb

@@ -59,7 +59,7 @@ class LogStashRunner
   def start
     # We start a pipeline that will generate a finite number of events
     # before starting the expectations
-    agent.register_pipeline("main", @settings)
+    agent.register_pipeline(@settings)
     @agent_task = Stud::Task.new { agent.execute }
     @agent_task.wait
   end

data/spec/conditionals_spec.rb

@@ -287,7 +287,7 @@ describe "conditionals in filter" do
     conditional "!([message] <= 'sample')" do
       sample("apple") { expect(subject.get("tags")).not_to include("success") }
       sample("zebra") { expect(subject.get("tags")).not_to include("failure") }
-      sample("sample") { expect(subject.get("tags")).not_to include("success") }
+      sample("sample") { expect(subject.get("tags")).not_to include("success")}
     end
 
     conditional "!([message] >= 'sample')" do

data/spec/logstash/agent_spec.rb

@@ -9,13 +9,14 @@ require_relative "../support/helpers"
 describe LogStash::Agent do
 
   let(:agent_settings) { LogStash::SETTINGS }
+  let(:default_pipeline_id) { LogStash::SETTINGS.get("pipeline.id") }
   let(:agent_args) { {} }
   let(:pipeline_settings) { agent_settings.clone }
   let(:pipeline_args) { {} }
   let(:config_file) { Stud::Temporary.pathname }
   let(:config_file_txt) { "input { generator { count => 100000 } } output { }" }
 
-    subject { LogStash::Agent.new(agent_settings) }
+  subject { LogStash::Agent.new(agent_settings) }
 
   before :each do
     # This MUST run first, before `subject` is invoked to ensure clean state
@@ -41,7 +42,6 @@ describe LogStash::Agent do
   end
 
   describe "register_pipeline" do
-    let(:pipeline_id) { "main" }
     let(:config_string) { "input { } filter { } output { }" }
     let(:agent_args) do
       {
@@ -61,7 +61,7 @@ describe LogStash::Agent do
         expect(arg1).to eq(config_string)
         expect(arg2.to_hash).to include(agent_args)
       end
-      subject.register_pipeline(pipeline_id, agent_settings)
+      subject.register_pipeline(agent_settings)
     end
   end
 
@@ -79,7 +79,7 @@ describe LogStash::Agent do
     end
   end
 
-    describe "#execute" do
+  describe "#execute" do
     let(:config_file_txt) { "input { generator { count => 100000 } } output { }" }
 
     before :each do
@@ -94,10 +94,9 @@ describe LogStash::Agent do
           "path.config" => config_file
         }
       end
-      let(:pipeline_id) { "main" }
 
       before(:each) do
-        subject.register_pipeline(pipeline_id, pipeline_settings)
+        subject.register_pipeline(pipeline_settings)
       end
 
       context "if state is clean" do
@@ -110,7 +109,10 @@ describe LogStash::Agent do
         it "should not reload_state!" do
           expect(subject).to_not receive(:reload_state!)
           t = Thread.new { subject.execute }
-          sleep 0.1
+
+          # TODO: refactor this. forcing an arbitrary fixed delay for thread concurrency issues is an indication of
+          # a bad test design or missing class functionality.
+          sleep(0.1)
           Stud.stop!(t)
           t.join
           subject.shutdown
@@ -123,11 +125,14 @@ describe LogStash::Agent do
 
           it "does not upgrade the new config" do
             t = Thread.new { subject.execute }
-            sleep 0.01 until subject.running_pipelines? && subject.pipelines.values.first.ready?
+            sleep(0.1) until subject.running_pipelines? && subject.pipelines.values.first.ready?
             expect(subject).to_not receive(:upgrade_pipeline)
             File.open(config_file, "w") { |f| f.puts second_pipeline_config }
             subject.send(:"reload_pipeline!", "main")
-            sleep 0.1
+
+            # TODO: refactor this. forcing an arbitrary fixed delay for thread concurrency issues is an indication of
+            # a bad test design or missing class functionality.
+            sleep(0.1)
             Stud.stop!(t)
             t.join
             subject.shutdown
@@ -139,14 +144,16 @@ describe LogStash::Agent do
 
           it "does upgrade the new config" do
             t = Thread.new { subject.execute }
-            sleep 0.01 until subject.running_pipelines? && subject.pipelines.values.first.ready?
+            sleep(0.1) until subject.running_pipelines? && subject.pipelines.values.first.ready?
             expect(subject).to receive(:upgrade_pipeline).once.and_call_original
             File.open(config_file, "w") { |f| f.puts second_pipeline_config }
             subject.send(:"reload_pipeline!", "main")
-            sleep 0.1
+
+            # TODO: refactor this. forcing an arbitrary fixed delay for thread concurrency issues is an indication of
+            # a bad test design or missing class functionality.
+            sleep(0.1)
             Stud.stop!(t)
             t.join
-
             subject.shutdown
           end
         end
@@ -159,14 +166,16 @@ describe LogStash::Agent do
 
           it "does not try to reload the pipeline" do
             t = Thread.new { subject.execute }
-            sleep 0.01 until subject.running_pipelines? && subject.pipelines.values.first.ready?
+            sleep(0.01) until subject.running_pipelines? && subject.pipelines.values.first.running?
             expect(subject).to_not receive(:reload_pipeline!)
             File.open(config_file, "w") { |f| f.puts second_pipeline_config }
             subject.reload_state!
-            sleep 0.1
+
+            # TODO: refactor this. forcing an arbitrary fixed delay for thread concurrency issues is an indication of
+            # a bad test design or missing class functionality.
+            sleep(0.1)
             Stud.stop!(t)
             t.join
-
             subject.shutdown
           end
         end
@@ -177,14 +186,16 @@ describe LogStash::Agent do
 
           it "tries to reload the pipeline" do
             t = Thread.new { subject.execute }
-            sleep 0.01 until subject.running_pipelines? && subject.pipelines.values.first.ready?
+            sleep(0.01) until subject.running_pipelines? && subject.pipelines.values.first.running?
             expect(subject).to receive(:reload_pipeline!).once.and_call_original
             File.open(config_file, "w") { |f| f.puts second_pipeline_config }
             subject.reload_state!
-            sleep 0.1
+
+            # TODO: refactor this. forcing an arbitrary fixed delay for thread concurrency issues is an indication of
+            # a bad test design or missing class functionality.
+            sleep(0.1)
             Stud.stop!(t)
             t.join
-
             subject.shutdown
           end
         end
@@ -199,19 +210,21 @@ describe LogStash::Agent do
           "path.config" => config_file,
         }
       end
-      let(:pipeline_id) { "main" }
 
       before(:each) do
-        subject.register_pipeline(pipeline_id, pipeline_settings)
+        subject.register_pipeline(pipeline_settings)
       end
 
       context "if state is clean" do
         it "should periodically reload_state" do
           allow(subject).to receive(:clean_state?).and_return(false)
           t = Thread.new { subject.execute }
-          sleep 0.01 until subject.running_pipelines? && subject.pipelines.values.first.ready?
+          sleep(0.01) until subject.running_pipelines? && subject.pipelines.values.first.running?
           expect(subject).to receive(:reload_state!).at_least(2).times
-          sleep 0.1
+
+          # TODO: refactor this. forcing an arbitrary fixed delay for thread concurrency issues is an indication of
+          # a bad test design or missing class functionality.
+          sleep(0.1)
           Stud.stop!(t)
           t.join
           subject.shutdown
@@ -224,10 +237,13 @@ describe LogStash::Agent do
 
           it "does not upgrade the new config" do
             t = Thread.new { subject.execute }
-            sleep 0.01 until subject.running_pipelines? && subject.pipelines.values.first.ready?
+            sleep(0.01) until subject.running_pipelines? && subject.pipelines.values.first.running?
             expect(subject).to_not receive(:upgrade_pipeline)
             File.open(config_file, "w") { |f| f.puts second_pipeline_config }
-            sleep 0.1
+
+            # TODO: refactor this. forcing an arbitrary fixed delay for thread concurrency issues is an indication of
+            # a bad test design or missing class functionality.
+            sleep(0.1)
             Stud.stop!(t)
             t.join
             subject.shutdown
@@ -239,10 +255,13 @@ describe LogStash::Agent do
 
           it "does upgrade the new config" do
             t = Thread.new { subject.execute }
-            sleep 0.01 until subject.running_pipelines? && subject.pipelines.values.first.ready?
+            sleep(0.01) until subject.running_pipelines? && subject.pipelines.values.first.running?
             expect(subject).to receive(:upgrade_pipeline).once.and_call_original
             File.open(config_file, "w") { |f| f.puts second_pipeline_config }
-            sleep 0.1
+
+            # TODO: refactor this. forcing an arbitrary fixed delay for thread concurrency issues is an indication of
+            # a bad test design or missing class functionality.
+            sleep(0.1)
             Stud.stop!(t)
             t.join
             subject.shutdown
@@ -253,7 +272,6 @@ describe LogStash::Agent do
   end
 
   describe "#reload_state!" do
-    let(:pipeline_id) { "main" }
     let(:first_pipeline_config) { "input { } filter { } output { }" }
     let(:second_pipeline_config) { "input { generator {} } filter { } output { }" }
     let(:pipeline_args) { {
@@ -263,7 +281,7 @@ describe LogStash::Agent do
     } }
 
     before(:each) do
-      subject.register_pipeline(pipeline_id, pipeline_settings)
+      subject.register_pipeline(pipeline_settings)
     end
 
     after(:each) do
@@ -273,7 +291,7 @@ describe LogStash::Agent do
     context "when fetching a new state" do
       it "upgrades the state" do
         expect(subject).to receive(:fetch_config).and_return(second_pipeline_config)
-        expect(subject).to receive(:upgrade_pipeline).with(pipeline_id, kind_of(LogStash::Settings), second_pipeline_config)
+        expect(subject).to receive(:upgrade_pipeline).with(default_pipeline_id, kind_of(LogStash::Settings), second_pipeline_config)
         subject.reload_state!
       end
     end
@@ -293,7 +311,6 @@ describe LogStash::Agent do
       "config.reload.interval" => 0.01,
       "config.string" => pipeline_config
     } }
-    let(:pipeline_id) { "main" }
 
     context "environment variable templating" do
       before :each do
@@ -308,14 +325,13 @@ describe LogStash::Agent do
 
       it "doesn't upgrade the state" do
         allow(subject).to receive(:fetch_config).and_return(pipeline_config)
-        subject.register_pipeline(pipeline_id, pipeline_settings)
-        expect(subject.pipelines[pipeline_id].inputs.first.message).to eq("foo-bar")
+        subject.register_pipeline(pipeline_settings)
+        expect(subject.pipelines[default_pipeline_id].inputs.first.message).to eq("foo-bar")
       end
     end
   end
 
   describe "#upgrade_pipeline" do
-    let(:pipeline_id) { "main" }
     let(:pipeline_config) { "input { } filter { } output { }" }
     let(:pipeline_args) { {
       "config.string" => pipeline_config,
@@ -324,10 +340,12 @@ describe LogStash::Agent do
     let(:new_pipeline_config) { "input { generator {} } output { }" }
 
     before(:each) do
-      subject.register_pipeline(pipeline_id, pipeline_settings)
+      subject.register_pipeline(pipeline_settings)
     end
 
     after(:each) do
+      # new pipelines will be created part of the upgrade process so we need
+      # to close any initialized pipelines
       subject.close_pipelines
     end
 
@@ -336,42 +354,51 @@ describe LogStash::Agent do
         allow(subject).to receive(:fetch_config).and_return(new_pipeline_config)
         allow(subject).to receive(:create_pipeline).and_return(nil)
         allow(subject).to receive(:stop_pipeline) do |id|
+          # we register_pipeline but we never execute them so we have to mock #stop_pipeline to
+          # not call Pipeline#shutdown but Pipeline#close
           subject.close_pipeline(id)
         end
       end
 
       it "leaves the state untouched" do
-        subject.send(:"reload_pipeline!", pipeline_id)
-        expect(subject.pipelines[pipeline_id].config_str).to eq(pipeline_config)
+        subject.send(:"reload_pipeline!", default_pipeline_id)
+        expect(subject.pipelines[default_pipeline_id].config_str).to eq(pipeline_config)
       end
 
       context "and current state is empty" do
         it "should not start a pipeline" do
           expect(subject).to_not receive(:start_pipeline)
-          subject.send(:"reload_pipeline!", pipeline_id)
+          subject.send(:"reload_pipeline!", default_pipeline_id)
         end
       end
     end
 
     context "when the upgrade succeeds" do
       let(:new_config) { "input { generator { count => 1 } } output { }" }
+
       before :each do
         allow(subject).to receive(:fetch_config).and_return(new_config)
-        allow(subject).to receive(:start_pipeline)
+        allow(subject).to receive(:start_pipeline).and_return(true)
         allow(subject).to receive(:stop_pipeline) do |id|
-           subject.close_pipeline(id)
+          # we register_pipeline but we never execute them so we have to mock #stop_pipeline to
+          # not call Pipeline#shutdown but Pipeline#close
+          subject.close_pipeline(id)
         end
       end
+
       it "updates the state" do
-        subject.send(:"reload_pipeline!", pipeline_id)
-        expect(subject.pipelines[pipeline_id].config_str).to eq(new_config)
+        subject.send(:"reload_pipeline!", default_pipeline_id)
+        expect(subject.pipelines[default_pipeline_id].config_str).to eq(new_config)
       end
+
       it "starts the pipeline" do
-        expect(subject).to receive(:start_pipeline)
+        expect(subject).to receive(:start_pipeline).and_return(true)
         expect(subject).to receive(:stop_pipeline) do |id|
+          # we register_pipeline but we never execute them so we have to mock #stop_pipeline to
+          # not call Pipeline#shutdown but Pipeline#close
           subject.close_pipeline(id)
         end
-        subject.send(:"reload_pipeline!", pipeline_id)
+        subject.send(:"reload_pipeline!", default_pipeline_id)
       end
     end
   end
@@ -398,9 +425,9 @@ describe LogStash::Agent do
     end
   end
 
-
   context "metrics after config reloading" do
     let!(:config) { "input { generator { } } output { dummyoutput { } }" }
+
     let!(:config_path) do
       f = Stud::Temporary.file
       f.write(config)
@@ -408,6 +435,7 @@ describe LogStash::Agent do
       f.close
       f.path
     end
+
     let(:pipeline_args) do
       {
         "pipeline.workers" => 2,
@@ -433,11 +461,12 @@ describe LogStash::Agent do
     let(:initial_generator_threshold) { 1000 }
     let(:pipeline_thread) do
       Thread.new do
-        subject.register_pipeline("main",  pipeline_settings)
+        subject.register_pipeline(pipeline_settings)
         subject.execute
       end
     end
 
+
     before :each do
       allow(LogStash::Outputs::DroppingDummyOutput).to receive(:new).at_least(:once).with(anything).and_return(dummy_output)
       allow(DummyOutput2).to receive(:new).at_least(:once).with(anything).and_return(dummy_output2)
@@ -451,13 +480,9 @@ describe LogStash::Agent do
       Thread.abort_on_exception = true
 
       pipeline_thread
-      # @t = Thread.new do
-      #   subject.register_pipeline("main",  pipeline_settings)
-      #   subject.execute
-      # end
 
       # wait for some events to reach the dummy_output
-      sleep(0.01) until dummy_output.events_received > initial_generator_threshold
+      sleep(0.1) until dummy_output.events_received > initial_generator_threshold
     end
 
     after :each do
@@ -523,12 +548,10 @@ describe LogStash::Agent do
         value = snapshot.metric_store.get_with_path("/stats/pipelines")[:stats][:pipelines][:main][:reloads][:last_error].value
         expect(value).to be(nil)
       end
-
     end
 
     context "when reloading a bad config" do
       let(:new_config) { "input { generator { count => " }
-      let(:new_config_generator_counter) { 500 }
       before :each do
 
         File.open(config_path, "w") do |f|
@@ -573,7 +596,6 @@ describe LogStash::Agent do
 
     context "when reloading a config that raises exception on pipeline.run" do
       let(:new_config) { "input { generator { count => 10000 } }" }
-      let(:new_config_generator_counter) { 500 }
 
       class BrokenGenerator < LogStash::Inputs::Generator
         def register
@@ -582,14 +604,12 @@ describe LogStash::Agent do
       end
 
       before :each do
-
         allow(LogStash::Plugin).to receive(:lookup).with("input", "generator").and_return(BrokenGenerator)
 
         File.open(config_path, "w") do |f|
           f.write(new_config)
           f.fsync
         end
-
       end
 
       it "does not increase the successful reload count" do
@@ -600,7 +620,7 @@ describe LogStash::Agent do
         }
       end
 
-      it "increases the failured reload count" do
+      it "increases the failures reload count" do
         expect { subject.send(:"reload_pipeline!", "main") }.to change {
           snapshot = subject.metric.collector.snapshot_metric
           reload_metrics = snapshot.metric_store.get_with_path("/stats/pipelines")[:stats][:pipelines][:main][:reloads]

data/spec/logstash/pipeline_spec.rb

@@ -139,7 +139,7 @@ describe LogStash::Pipeline do
 
       pipeline = LogStash::Pipeline.new(config, pipeline_settings_obj)
       t = Thread.new { pipeline.run }
-      sleep 0.1 while !pipeline.ready?
+      sleep(0.1) until pipeline.ready?
       wait(3).for do
         # give us a bit of time to flush the events
         # puts("*****" + output.events.map{|e| e.message}.to_s)
@@ -366,7 +366,6 @@ describe LogStash::Pipeline do
 
       sample(["foo", "bar"]) do
         expect(subject.size).to eq(2)
-
         expect(subject[0].get("message")).to eq("foo\nbar")
         expect(subject[0].get("type")).to be_nil
         expect(subject[1].get("message")).to eq("foo\nbar")
@@ -388,7 +387,9 @@ describe LogStash::Pipeline do
       allow(LogStash::Plugin).to receive(:lookup).with("codec", "plain").and_return(DummyCodec)
       allow(LogStash::Plugin).to receive(:lookup).with("output", "dummyoutput").and_return(::LogStash::Outputs::DummyOutput)
       allow(logger).to receive(:warn)
-      # pipeline must be first called outside the thread context because it lazyly initialize
+
+      # pipeline must be first called outside the thread context because it lazyly initialize and will create a
+      # race condition if called in the thread
       p = pipeline
       t = Thread.new { p.run }
       sleep(0.1) until pipeline.ready?
@@ -596,7 +597,7 @@ describe LogStash::Pipeline do
       Thread.abort_on_exception = true
       pipeline = LogStash::Pipeline.new(config, pipeline_settings_obj)
       t = Thread.new { pipeline.run }
-      sleep 0.1 while !pipeline.ready?
+      sleep(0.1) until pipeline.ready?
       wait(3).for do
         # give us a bit of time to flush the events
         output.events.empty?
@@ -699,6 +700,7 @@ describe LogStash::Pipeline do
         s = subject
         t = Thread.new { s.run }
         sleep(0.1) until subject.ready?
+
         sleep(0.1)
         expect(subject.uptime).to be > 0
         subject.shutdown

data/spec/logstash/plugin_spec.rb

@@ -7,6 +7,41 @@ require "logstash/inputs/base"
 require "logstash/filters/base"
 
 describe LogStash::Plugin do
+  context "reloadable" do
+    context "by default" do
+      subject do
+        Class.new(LogStash::Plugin) do
+        end
+      end
+
+      it "makes .reloadable? return true" do
+        expect(subject.reloadable?).to be_truthy
+      end
+
+      it "makes #reloadable? return true" do
+        expect(subject.new({}).reloadable?).to be_truthy
+      end
+    end
+
+    context "user can overrides" do
+      subject do
+        Class.new(LogStash::Plugin) do
+          def self.reloadable?
+            false
+          end
+        end
+      end
+
+      it "makes .reloadable? return true" do
+        expect(subject.reloadable?).to be_falsey
+      end
+
+      it "makes #reloadable? return true" do
+        expect(subject.new({}).reloadable?).to be_falsey
+      end
+    end
+  end
+
   it "should fail lookup on inexisting type" do
     #expect_any_instance_of(Cabin::Channel).to receive(:debug).once
     expect { LogStash::Plugin.lookup("badbadtype", "badname") }.to raise_error(LogStash::PluginLoadingError)

data/spec/logstash/queue_factory_spec.rb

@@ -0,0 +1,83 @@
+# encoding: utf-8
+require "logstash/queue_factory"
+require "logstash/settings"
+require "stud/temporary"
+
+describe LogStash::QueueFactory do
+  let(:pipeline_id) { "my_pipeline" }
+  let(:settings_array) do
+    [
+      LogStash::Setting::WritableDirectory.new("path.queue", Stud::Temporary.pathname),
+      LogStash::Setting::String.new("queue.type", "memory", true, ["persisted", "memory", "memory_acked"]),
+      LogStash::Setting::Bytes.new("queue.page_capacity", "250mb"),
+      LogStash::Setting::Bytes.new("queue.max_bytes", "1024mb"),
+      LogStash::Setting::Numeric.new("queue.max_events", 0),
+      LogStash::Setting::Numeric.new("queue.checkpoint.acks", 1024),
+      LogStash::Setting::Numeric.new("queue.checkpoint.writes", 1024),
+      LogStash::Setting::Numeric.new("queue.checkpoint.interval", 1000),
+      LogStash::Setting::String.new("pipeline.id", pipeline_id)
+    ]
+  end
+
+  let(:settings) do
+    s = LogStash::Settings.new
+
+    settings_array.each do |setting|
+      s.register(setting)
+    end
+    s
+  end
+
+  subject { described_class }
+
+  context "when `queue.type` is `persisted`" do
+    before do
+      settings.set("queue.type", "persisted")
+    end
+
+    it "returns a `WrappedAckedQueue`" do
+      queue =  subject.create(settings)
+      expect(queue).to be_kind_of(LogStash::Util::WrappedAckedQueue)
+      queue.close
+    end
+
+    describe "per pipeline id subdirectory creation" do
+      let(:queue_path) { ::File.join(settings.get("path.queue"), pipeline_id) }
+
+      after :each do
+        FileUtils.rmdir(queue_path)
+      end
+
+      it "creates a queue directory based on the pipeline id" do
+        expect(Dir.exist?(queue_path)).to be_falsey
+        queue = subject.create(settings)
+        expect(Dir.exist?(queue_path)).to be_truthy
+        queue.close
+      end
+    end
+  end
+
+  context "when `queue.type` is `memory_acked`" do
+    before do
+      settings.set("queue.type", "memory_acked")
+    end
+
+    it "returns a `WrappedAckedQueue`" do
+      queue =  subject.create(settings)
+      expect(queue).to be_kind_of(LogStash::Util::WrappedAckedQueue)
+      queue.close
+    end
+  end
+
+  context "when `queue.type` is `memory`" do
+    before do
+      settings.set("queue.type", "memory")
+    end
+
+    it "returns a `WrappedAckedQueue`" do
+      queue =  subject.create(settings)
+      expect(queue).to be_kind_of(LogStash::Util::WrappedSynchronousQueue)
+      queue.close
+    end
+  end
+end

metadata

@@ -1,21 +1,21 @@
 --- !ruby/object:Gem::Specification
 name: logstash-core
 version: !ruby/object:Gem::Version
-  version: 5.2.2
+  version: 5.3.0
 platform: java
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-02-24 00:00:00.000000000 Z
+date: 2017-03-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 5.2.2
+        version: 5.3.0
   name: logstash-core-event-java
   prerelease: false
   type: :runtime
@@ -23,13 +23,13 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 5.2.2
+        version: 5.3.0
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 5.2.2
+        version: 5.3.0
   name: logstash-core-queue-jruby
   prerelease: false
   type: :runtime
@@ -37,7 +37,7 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 5.2.2
+        version: 5.3.0
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
@@ -405,6 +405,7 @@ files:
 - lib/logstash/plugins/hooks_registry.rb
 - lib/logstash/plugins/registry.rb
 - lib/logstash/program.rb
+- lib/logstash/queue_factory.rb
 - lib/logstash/runner.rb
 - lib/logstash/settings.rb
 - lib/logstash/shutdown_watcher.rb
@@ -482,6 +483,7 @@ files:
 - spec/logstash/plugin_spec.rb
 - spec/logstash/plugins/hooks_registry_spec.rb
 - spec/logstash/plugins/registry_spec.rb
+- spec/logstash/queue_factory_spec.rb
 - spec/logstash/runner_spec.rb
 - spec/logstash/setting_spec.rb
 - spec/logstash/settings/array_coercible_spec.rb
@@ -588,6 +590,7 @@ test_files:
 - spec/logstash/plugin_spec.rb
 - spec/logstash/plugins/hooks_registry_spec.rb
 - spec/logstash/plugins/registry_spec.rb
+- spec/logstash/queue_factory_spec.rb
 - spec/logstash/runner_spec.rb
 - spec/logstash/setting_spec.rb
 - spec/logstash/settings/array_coercible_spec.rb