logstash-core 5.0.0.pre.rc1-java → 5.0.1-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: baef1eb320ed6e41540de4f69bc3cedbbd57cf5f
-  data.tar.gz: 0386098cd8179ccdda67435e30dac956c65385e2
+  metadata.gz: 1d2dff17e7881a7df042ac7b79891f9519331757
+  data.tar.gz: f02d9b9301c6e6b8973b211b2e5f3c8800ed0fd3
 SHA512:
-  metadata.gz: f330b13ca8673330ce1cccc4a1dbbf36e02f4b61ff8f300fe55b26b101738e1ee8f7b9ae3a580acc2fa1e17db2fda2bb99172859ccd52e42558b5285ecb6bced
-  data.tar.gz: a884dc890d1fc02c4af8893971498cb6eb9ed5481f9766791a4e8361356953c617cf8322565ee9a3203ddc99892ca4131cc24b0acb12f2e826bd1e2afc68c995
+  metadata.gz: f012536aacad34585ae14a4810fc6924e6755f239bc86fdb83a72b258c4379d8f4ca386aba22adbc8805ed31c396c28dbc5a625737aa648fd6d4a8f2f3eb3780
+  data.tar.gz: 2a04623696dec22955b2b44fac4a75bf56525ed0d6247ce95cb751d026d916d55c1c0728825e4baedd9b6a280f882fea784eee34d999af67c820aa5771f036ad

data/lib/jars.rb

@@ -4,4 +4,4 @@ require_jar('org.apache.logging.log4j', 'log4j-api', '2.6.2')
 require_jar('org.apache.logging.log4j', 'log4j-core', '2.6.2')
 require_jar('com.fasterxml.jackson.core', 'jackson-core', '2.7.4')
 require_jar('com.fasterxml.jackson.core', 'jackson-databind', '2.7.4')
-require_jar('org.logstash', 'logstash-core', '5.0.0-rc1')
+require_jar('org.logstash', 'logstash-core', '5.0.1')

data/lib/logstash-core/version.rb

@@ -5,4 +5,4 @@
 # Note to authors: this should not include dashes because 'gem' barfs if
 # you include a dash in the version string.
 
-LOGSTASH_CORE_VERSION = "5.0.0-rc1"
+LOGSTASH_CORE_VERSION = "5.0.1"

data/lib/logstash/config/loader.rb

@@ -12,7 +12,18 @@ module LogStash; module Config; class Loader
       # Append the config string.
       # This allows users to provide both -f and -e flags. The combination
       # is rare, but useful for debugging.
-      config_string = config_string + load_config(config_path)
+      loaded_config = load_config(config_path)
+      if loaded_config.empty? && config_string.empty?
+        # If loaded config from `-f` is empty *and* if config string is empty we raise an error
+        fail(I18n.t("logstash.runner.configuration.file-not-found", :path => config_path))
+      end
+
+      # tell the user we are merging, otherwise it is very confusing
+      if !loaded_config.empty? && !config_string.empty?
+        @logger.info("Created final config by merging config string and config path", :path => config_path)
+      end
+
+      config_string = config_string + loaded_config
     else
       # include a default stdin input if no inputs given
       if config_string !~ /input *{/
@@ -52,11 +63,11 @@ module LogStash; module Config; class Loader
     path = ::File.expand_path(path)
     path = ::File.join(path, "*") if ::File.directory?(path)
 
+    config = ""
     if Dir.glob(path).length == 0
-      fail(I18n.t("logstash.runner.configuration.file-not-found", :path => path))
+      @logger.info("No config files found in path", :path => path)
     end
 
-    config = ""
     encoding_issue_files = []
     Dir.glob(path).sort.each do |file|
       next unless ::File.file?(file)

data/lib/logstash/environment.rb

@@ -17,9 +17,9 @@ module LogStash
 
   [
             Setting::String.new("node.name", Socket.gethostname),
-            Setting::String.new("path.config", nil, false),
+    Setting::NullableString.new("path.config", nil, false),
  Setting::WritableDirectory.new("path.data", ::File.join(LogStash::Environment::LOGSTASH_HOME, "data")),
-            Setting::String.new("config.string", nil, false),
+    Setting::NullableString.new("config.string", nil, false),
            Setting::Boolean.new("config.test_and_exit", false),
            Setting::Boolean.new("config.reload.automatic", false),
            Setting::Numeric.new("config.reload.interval", 3), # in seconds
@@ -31,7 +31,7 @@ module LogStash
            Setting::Numeric.new("pipeline.batch.delay", 5), # in milliseconds
            Setting::Boolean.new("pipeline.unsafe_shutdown", false),
                     Setting.new("path.plugins", Array, []),
-            Setting::String.new("interactive", nil, false),
+    Setting::NullableString.new("interactive", nil, false),
            Setting::Boolean.new("config.debug", false),
             Setting::String.new("log.level", "info", true, ["fatal", "error", "warn", "debug", "info", "trace"]),
            Setting::Boolean.new("version", false),

data/lib/logstash/filters/base.rb

@@ -187,14 +187,20 @@ class LogStash::Filters::Base < LogStash::Plugin
     # note below that the tags array field needs to be updated then reassigned to the event.
     # this is important because a construct like event["tags"].delete(tag) will not work
     # in the current Java event implementation. see https://github.com/elastic/logstash/issues/4140
+
+    tags = event.get("tags")
+    return unless tags
+
+    tags = Array(tags)
     @remove_tag.each do |tag|
-      tags = event.get("tags")
-      break if tags.nil? || tags.empty?
+      break if tags.empty?
+
       tag = event.sprintf(tag)
       @logger.debug? and @logger.debug("filters/#{self.class.name}: removing tag", :tag => tag)
       tags.delete(tag)
-      event.set("tags", tags)
     end
+
+    event.set("tags", tags)
   end # def filter_matched
 
   protected

data/lib/logstash/instrument/periodic_poller/base.rb

@@ -8,8 +8,8 @@ module LogStash module Instrument module PeriodicPoller
     include LogStash::Util::Loggable
 
     DEFAULT_OPTIONS = {
-      :polling_interval => 1,
-      :polling_timeout => 60
+      :polling_interval => 5,
+      :polling_timeout => 120
     }
 
     public
@@ -22,11 +22,25 @@ module LogStash module Instrument module PeriodicPoller
     def update(time, result, exception)
       return unless exception
 
-      logger.error("PeriodicPoller: exception",
-                   :poller => self,
-                   :result => result,
-                   :exception => exception,
-                   :executed_at => time)
+      if exception.is_a?(Concurrent::TimeoutError)
+        # On a busy system this can happen, we just log it as a debug
+        # event instead of an error, Some of the JVM calls can take a long time or block.
+        logger.debug("PeriodicPoller: Timeout exception",
+                :poller => self,
+                :result => result,
+                :polling_timeout => @options[:polling_timeout],
+                :polling_interval => @options[:polling_interval],
+                :exception => exception.class,
+                :executed_at => time)
+      else
+        logger.error("PeriodicPoller: exception",
+                :poller => self,
+                :result => result,
+                :exception => exception.class,
+                :polling_timeout => @options[:polling_timeout],
+                :polling_interval => @options[:polling_interval],
+                :executed_at => time)
+      end
     end
 
     def collect
@@ -37,7 +51,7 @@ module LogStash module Instrument module PeriodicPoller
       logger.debug("PeriodicPoller: Starting",
                    :polling_interval => @options[:polling_interval],
                    :polling_timeout => @options[:polling_timeout]) if logger.debug?
-      
+
       collect # Collect data right away if possible
       @task.execute
     end

data/lib/logstash/logging/logger.rb

@@ -1,4 +1,5 @@
 require "logstash/java_integration"
+require "uri"
 
 module LogStash
   module Logging
@@ -6,6 +7,7 @@ module LogStash
       java_import org.apache.logging.log4j.Level
       java_import org.apache.logging.log4j.LogManager
       java_import org.apache.logging.log4j.core.config.Configurator
+      java_import org.apache.logging.log4j.core.config.DefaultConfiguration
       @@config_mutex = Mutex.new
       @@logging_context = nil
 
@@ -70,7 +72,16 @@ module LogStash
       def self.initialize(config_location)
         @@config_mutex.synchronize do
           if @@logging_context.nil?
-            @@logging_context = Configurator.initialize(nil, config_location)
+            file_path = URI(config_location).path
+            if ::File.exists?(file_path)
+              logs_location = java.lang.System.getProperty("ls.logs")
+              puts "Sending Logstash's logs to #{logs_location} which is now configured via log4j2.properties"
+              @@logging_context = Configurator.initialize(nil, config_location)
+            else
+              # fall back to default config
+              puts "Could not find log4j2 configuration at path #{file_path}. Using default config which logs to console"
+              @@logging_context = Configurator.initialize(DefaultConfiguration.new)
+            end
           end
         end
       end

data/lib/logstash/pipeline.rb

@@ -257,14 +257,9 @@ module LogStash; class Pipeline
   def filter_batch(batch)
     batch.each do |event|
       if event.is_a?(Event)
-        filtered = filter_func(event)
-        filtered.each do |e|
-          #these are both original and generated events
-          if e.cancelled?
-            batch.cancel(e)
-          else
-            batch.merge(e)
-          end
+        filter_func(event).each do |e|
+          # these are both original and generated events
+          batch.merge(e) unless e.cancelled?
         end
       end
     end
@@ -492,9 +487,7 @@ module LogStash; class Pipeline
   def flush_filters_to_batch(batch, options = {})
     options[:final] = batch.shutdown_signal_received?
     flush_filters(options) do |event|
-      if event.cancelled?
-        batch.cancel(event)
-      else
+      unless event.cancelled?
         @logger.debug? and @logger.debug("Pushing flushed events", :event => event)
         batch.merge(event)
       end

data/lib/logstash/runner.rb

@@ -162,10 +162,11 @@ class LogStash::Runner < Clamp::StrictCommand
 
     begin
       LogStash::SETTINGS.from_yaml(LogStash::SETTINGS.get("path.settings"))
+    rescue Errno::ENOENT
+      $stderr.puts "WARNING: Could not find logstash.yml which is typically located in $LS_HOME/config or /etc/logstash. You can specify the path using --path.settings. Continuing using the defaults"
     rescue => e
       # abort unless we're just looking for the help
-      if (["--help", "-h"] & args).empty?
-        $stderr.puts "INFO: Logstash requires a setting file which is typically located in $LS_HOME/config or /etc/logstash. If you installed Logstash through a package and are starting it manually please specify the location to this settings file by passing in \"--path.settings=/path/..\""
+      unless cli_help?(args)
         $stderr.puts "ERROR: Failed to load settings file from \"path.settings\". Aborting... path.setting=#{LogStash::SETTINGS.get("path.settings")}, exception=#{e.class}, message=>#{e.message}"
         return 1
       end
@@ -186,16 +187,10 @@ class LogStash::Runner < Clamp::StrictCommand
     java.lang.System.setProperty("ls.log.level", setting("log.level"))
     unless java.lang.System.getProperty("log4j.configurationFile")
       log4j_config_location = ::File.join(setting("path.settings"), "log4j2.properties")
-      LogStash::Logging::Logger::initialize(log4j_config_location)
+      LogStash::Logging::Logger::initialize("file://" + log4j_config_location)
     end
     # override log level that may have been introduced from a custom log4j config file
     LogStash::Logging::Logger::configure_logging(setting("log.level"))
-    
-    # Adding this here because a ton of LS users install LS via packages and try to manually 
-    # start Logstash using bin/logstash. See #5986. I think not logging to console is good for 
-    # services, but until LS users re-learn that logs end up in path.logs, we should keep this 
-    # message. Otherwise we'll be answering the same question again and again.
-    puts "Sending Logstash logs to #{setting("path.logs")} which is now configured via log4j2.properties."
 
     if setting("config.debug") && logger.debug?
       logger.warn("--config.debug was specified, but log.level was not set to \'debug\'! No config info will be logged.")
@@ -226,6 +221,8 @@ class LogStash::Runner < Clamp::StrictCommand
 
     return start_shell(setting("interactive"), binding) if setting("interactive")
 
+    @settings.validate_all
+
     @settings.format_settings.each {|line| logger.debug(line) }
 
     if setting("config.string").nil? && setting("path.config").nil?
@@ -280,7 +277,12 @@ class LogStash::Runner < Clamp::StrictCommand
     show_short_help
     return 1
   rescue => e
-    logger.fatal(I18n.t("oops"), :error => e, :backtrace => e.backtrace)
+    # if logger itself is not initialized
+    if LogStash::Logging::Logger.get_logging_context.nil?
+      $stderr.puts "#{I18n.t("oops")} :error => #{e}, :backtrace => #{e.backtrace}"
+    else
+      logger.fatal(I18n.t("oops"), :error => e, :backtrace => e.backtrace)
+    end
     return 1
   ensure
     Stud::untrap("INT", sigint_id) unless sigint_id.nil?
@@ -408,5 +410,11 @@ class LogStash::Runner < Clamp::StrictCommand
       nil
     end
   end
+  
+  # is the user asking for CLI help subcommand?
+  def cli_help?(args)
+    # I know, double negative
+    !(["--help", "-h"] & args).empty?
+  end
 
 end

data/lib/logstash/settings.rb

@@ -95,6 +95,12 @@ module LogStash
       self.merge(flatten_hash(settings))
     end
 
+    def validate_all
+      @settings.each do |name, setting|
+        setting.validate_value
+      end
+    end
+
     private
     def read_yaml(path)
       YAML.safe_load(IO.read(path)) || {}
@@ -123,8 +129,9 @@ module LogStash
       @validator_proc = validator_proc
       @value = nil
       @value_is_set = false
+      @strict = strict
 
-      validate(default) if strict
+      validate(default) if @strict
       @default = default
     end
 
@@ -136,8 +143,12 @@ module LogStash
       @value_is_set
     end
 
+    def strict?
+      @strict
+    end
+
     def set(value)
-      validate(value)
+      validate(value) if @strict
       @value = value
       @value_is_set = true
       @value
@@ -167,12 +178,18 @@ module LogStash
       self.to_hash == other.to_hash
     end
 
-    private
-    def validate(value)
-      if !value.is_a?(@klass)
-        raise ArgumentError.new("Setting \"#{@name}\" must be a #{@klass}. Received: #{value} (#{value.class})")
-      elsif @validator_proc && !@validator_proc.call(value)
-        raise ArgumentError.new("Failed to validate setting \"#{@name}\" with value: #{value}")
+    def validate_value
+      validate(value)
+    end
+
+    protected
+    def validate(input)
+      if !input.is_a?(@klass)
+        raise ArgumentError.new("Setting \"#{@name}\" must be a #{@klass}. Received: #{input} (#{input.class})")
+      end
+
+      if @validator_proc && !@validator_proc.call(input)
+        raise ArgumentError.new("Failed to validate setting \"#{@name}\" with value: #{input}")
       end
     end
 
@@ -351,6 +368,13 @@ module LogStash
       end
     end
 
+    class NullableString < String
+      def validate(value)
+        return if value.nil?
+        super(value)
+      end
+    end
+
     class ExistingFilePath < Setting
       def initialize(name, default=nil, strict=true)
         super(name, ::String, default, strict) do |file_path|
@@ -364,7 +388,7 @@ module LogStash
     end
 
     class WritableDirectory < Setting
-      def initialize(name, default=nil, strict=true)
+      def initialize(name, default=nil, strict=false)
         super(name, ::String, default, strict) do |path|
           if ::File.directory?(path) && ::File.writable?(path)
             true
@@ -378,3 +402,4 @@ module LogStash
 
   SETTINGS = Settings.new
 end
+

data/lib/logstash/util/decorators.rb

@@ -33,17 +33,20 @@ module LogStash::Util
     end
 
     # tags is an array of string. sprintf syntax can be used.
-    def add_tags(tags, event, pluginname)
-      tags.each do |tag|
-        tag = event.sprintf(tag)
-        self.logger.debug? and self.logger.debug("#{pluginname}: adding tag", "tag" => tag)
+    def add_tags(new_tags, event, pluginname)
+      tags = event.get("tags")
+      tags = tags.nil? ? [] : Array(tags)
+
+      new_tags.each do |new_tag|
+        new_tag = event.sprintf(new_tag)
+        self.logger.debug? and self.logger.debug("#{pluginname}: adding tag", "tag" => new_tag)
         # note below that the tags array field needs to be updated then reassigned to the event.
         # this is important because a construct like event["tags"] << tag will not work
         # in the current Java event implementation. see https://github.com/elastic/logstash/issues/4140
-        tags = event.get("tags") || []
-        tags << tag
-        event.set("tags", tags)
+        tags << new_tag  #unless tags.include?(new_tag)
       end
+
+      event.set("tags", tags)
     end
 
   end # module LogStash::Util::Decorators

data/lib/logstash/util/wrapped_synchronous_queue.rb

@@ -148,7 +148,10 @@ module LogStash; module Util
         @shutdown_signal_received = false
         @flush_signal_received = false
         @originals = Hash.new
-        @cancelled = Hash.new
+
+        # TODO: disabled for https://github.com/elastic/logstash/issues/6055 - will have to properly refactor
+        # @cancelled = Hash.new
+
         @generated = Hash.new
         @iterating_temp = Hash.new
         @iterating = false # Atomic Boolean maybe? Although batches are not shared across threads
@@ -168,17 +171,22 @@ module LogStash; module Util
       end
 
       def cancel(event)
-        @cancelled[event] = true
+        # TODO: disabled for https://github.com/elastic/logstash/issues/6055 - will have to properly refactor
+        raise("cancel is unsupported")
+        # @cancelled[event] = true
       end
 
       def each(&blk)
         # take care not to cause @originals or @generated to change during iteration
         @iterating = true
+
+        # below the checks for @cancelled.include?(e) have been replaced by e.cancelled?
+        # TODO: for https://github.com/elastic/logstash/issues/6055 = will have to properly refactor
         @originals.each do |e, _|
-          blk.call(e) unless @cancelled.include?(e)
+          blk.call(e) unless e.cancelled?
         end
         @generated.each do |e, _|
-          blk.call(e) unless @cancelled.include?(e)
+          blk.call(e) unless e.cancelled?
         end
         @iterating = false
         update_generated
@@ -197,7 +205,9 @@ module LogStash; module Util
       end
 
       def cancelled_size
-        @cancelled.size
+      # TODO: disabled for https://github.com/elastic/logstash/issues/6055 = will have to properly refactor
+      raise("cancelled_size is unsupported ")
+        # @cancelled.size
       end
 
       def shutdown_signal_received?

data/lib/logstash/version.rb

@@ -11,4 +11,4 @@
 #       eventually this file should be in the root logstash lib fir and dependencies in logstash-core should be
 #       fixed.
 
-LOGSTASH_VERSION = "5.0.0-rc1"
+LOGSTASH_VERSION = "5.0.1"

data/locales/en.yml

@@ -112,8 +112,7 @@ en:
           longer available. %{extra} If you have any questions about this, you
           are invited to visit https://discuss.elastic.co/c/logstash and ask.
         file-not-found: |-
-          No config files found: %{path}
-          Can you make sure this path is a logstash config file?
+          No config files found: %{path}. Can you make sure this path is a logstash config file?
         scheme-not-supported: |-
           URI scheme not supported: %{path}
           Either pass a local file path or "file|http://" URI
@@ -215,7 +214,7 @@ en:
           Specifying once will show 'informational'
           logs. Specifying twice will show 'debug'
           logs. This flag is deprecated. You should use
-          --verbose or --debug instead.
+          --log-level=info or --log-level=debug instead.
         version: |+
           Emit the version of logstash and its friends,
           then exit.
@@ -253,7 +252,7 @@ en:
         agent: |+
           Specify an alternate agent plugin name.
         config_debug: |+
-          Print the compiled config ruby code out as a debug log (you must also have --debug enabled).
+          Print the compiled config ruby code out as a debug log (you must also have --log.level=debug enabled).
           WARNING: This will include any 'password' options passed to plugin configs as plaintext, and may result
           in plaintext passwords appearing in your logs!
         log_format: |+
@@ -264,7 +263,7 @@ en:
           DEPRECATED: use --log.level=debug instead.
         verbose: |+
           Set the log level to info.
-          DEPRECATED: use --log.level=verbose instead.
+          DEPRECATED: use --log.level=info instead.
         quiet: |+
           Set the log level to info.
           DEPRECATED: use --log.level=quiet instead.

data/logstash-core.gemspec

@@ -17,7 +17,7 @@ Gem::Specification.new do |gem|
   gem.require_paths = ["lib", "vendor/jars"]
   gem.version       = LOGSTASH_CORE_VERSION
 
-  gem.add_runtime_dependency "logstash-core-event-java", "5.0.0-rc1"
+  gem.add_runtime_dependency "logstash-core-event-java", "5.0.1"
 
   gem.add_runtime_dependency "pry", "~> 0.10.1"  #(Ruby license)
   gem.add_runtime_dependency "stud", "~> 0.0.19" #(Apache 2.0 license)

data/spec/logstash/filters/base_spec.rb

@@ -98,6 +98,39 @@ describe LogStash::Filters::NOOP do
     end
   end
 
+  describe "tags parsing with one tag as string value" do
+    config <<-CONFIG
+    filter {
+      noop {
+        add_tag => ["bar"]
+      }
+    }
+    CONFIG
+
+    sample("type" => "noop") do
+      insist { subject.get("tags") } == ["bar"]
+    end
+
+    sample("type" => "noop", "tags" => "foo") do
+      insist { subject.get("tags") } == ["foo", "bar"]
+    end
+  end
+
+  describe "tags parsing with duplicate tags" do
+    config <<-CONFIG
+    filter {
+      noop {
+        add_tag => ["foo"]
+      }
+    }
+    CONFIG
+
+    sample("type" => "noop", "tags" => "foo") do
+      # this is completely weird but seems to be already expected in other specs
+      insist { subject.get("tags") } == ["foo", "foo"]
+    end
+  end
+
   describe "tags parsing with multiple tags" do
     config <<-CONFIG
     filter {
@@ -133,6 +166,18 @@ describe LogStash::Filters::NOOP do
     }
     CONFIG
 
+    sample("type" => "noop", "tags" => "foo") do
+      insist { subject.get("tags") } == ["foo"]
+    end
+
+    sample("type" => "noop", "tags" => "t2") do
+      insist { subject.get("tags") } == []
+    end
+
+    sample("type" => "noop", "tags" => ["t2"]) do
+      insist { subject.get("tags") } == []
+    end
+
     sample("type" => "noop", "tags" => ["t4"]) do
       insist { subject.get("tags") } == ["t4"]
     end

data/spec/logstash/instrument/periodic_poller/base_spec.rb

@@ -0,0 +1,32 @@
+# encoding: utf-8
+require "logstash/instrument/periodic_poller/base"
+require "logstash/instrument/metric"
+require "logstash/instrument/collector"
+
+describe LogStash::Instrument::PeriodicPoller::Base do
+  let(:metric) { LogStash::Instrument::Metric.new(LogStash::Instrument::Collector.new) }
+  let(:options) { {} }
+
+  subject { described_class.new(metric, options) }
+
+  describe "#update" do
+    it "logs an timeout exception to debug level" do
+      exception = Concurrent::TimeoutError.new
+      expect(subject.logger).to receive(:debug).with(anything, hash_including(:exception => exception.class))
+      subject.update(Time.now, "hola", exception)
+    end
+
+    it "logs any other exception to error level" do
+      exception = Class.new
+      expect(subject.logger).to receive(:error).with(anything, hash_including(:exception => exception.class))
+      subject.update(Time.now, "hola", exception)
+    end
+
+    it "doesnt log anything when no exception is received" do
+      exception = Concurrent::TimeoutError.new
+      expect(subject.logger).not_to receive(:debug).with(anything)
+      expect(subject.logger).not_to receive(:error).with(anything)
+      subject.update(Time.now, "hola", exception)
+    end
+  end
+end

data/spec/logstash/pipeline_spec.rb

@@ -100,6 +100,60 @@ describe LogStash::Pipeline do
     pipeline_settings_obj.reset
   end
 
+
+  describe "event cancellation" do
+    # test harness for https://github.com/elastic/logstash/issues/6055
+
+    let(:output) { DummyOutputWithEventsArray.new }
+
+    before do
+      allow(LogStash::Plugin).to receive(:lookup).with("input", "generator").and_return(LogStash::Inputs::Generator)
+      allow(LogStash::Plugin).to receive(:lookup).with("output", "dummyoutputwitheventsarray").and_return(DummyOutputWithEventsArray)
+      allow(LogStash::Plugin).to receive(:lookup).with("filter", "drop").and_call_original
+      allow(LogStash::Plugin).to receive(:lookup).with("filter", "mutate").and_call_original
+      allow(LogStash::Plugin).to receive(:lookup).with("codec", "plain").and_call_original
+      allow(DummyOutputWithEventsArray).to receive(:new).with(any_args).and_return(output)
+    end
+
+    let(:config) do
+      <<-CONFIG
+        input {
+          generator {
+            lines => ["1", "2", "END"]
+            count => 1
+          }
+        }
+        filter {
+          if [message] == "1" {
+            drop {}
+          }
+          mutate { add_tag => ["notdropped"] }
+        }
+        output { dummyoutputwitheventsarray {} }
+      CONFIG
+    end
+
+    it "should not propage cancelled events from filter to output" do
+      abort_on_exception_state = Thread.abort_on_exception
+      Thread.abort_on_exception = true
+
+      pipeline = LogStash::Pipeline.new(config, pipeline_settings_obj)
+      Thread.new { pipeline.run }
+      sleep 0.1 while !pipeline.ready?
+      wait(3).for do
+        # give us a bit of time to flush the events
+        # puts("*****" + output.events.map{|e| e.message}.to_s)
+        output.events.map{|e| e.get("message")}.include?("END")
+      end.to be_truthy
+      expect(output.events.size).to eq(2)
+      expect(output.events[0].get("tags")).to eq(["notdropped"])
+      expect(output.events[1].get("tags")).to eq(["notdropped"])
+      pipeline.shutdown
+
+      Thread.abort_on_exception = abort_on_exception_state
+    end
+  end
+
   describe "defaulting the pipeline workers based on thread safety" do
     before(:each) do
       allow(LogStash::Plugin).to receive(:lookup).with("input", "dummyinput").and_return(DummyInput)

data/spec/logstash/runner_spec.rb

@@ -353,8 +353,8 @@ describe LogStash::Runner do
     context "if does not exist" do
       let(:args) { ["--path.settings", "/tmp/a/a/a/a", "-e", "input {} output {}"] }
 
-      it "should terminate logstash" do
-        expect(subject.run(args)).to eq(1)
+      it "should not terminate logstash" do
+        expect(subject.run(args)).to eq(nil)
       end
 
       context "but if --help is passed" do

data/spec/logstash/setting_spec.rb

@@ -63,6 +63,7 @@ describe LogStash::Setting do
       end
     end
   end
+
   describe "#set" do
     subject { described_class.new("number", Numeric, 1) }
     it "should change the value of a setting" do
@@ -77,12 +78,33 @@ describe LogStash::Setting do
         expect(subject.set?).to eq(true)
       end
     end
-
     context "when the argument's class does not match @klass" do
       it "should throw an exception" do
         expect { subject.set("not a number") }.to raise_error
       end
     end
+    context "when strict=false" do
+      let(:strict) { false }
+      subject { described_class.new("number", Numeric, 1, strict) }
+      before do
+        expect(subject).not_to receive(:validate)
+      end
+
+      it "should not call #validate" do
+        subject.set(123)
+      end
+    end
+    context "when strict=true" do
+      let(:strict) { true }
+      subject { described_class.new("number", Numeric, 1, strict) }
+      before do
+        expect(subject).to receive(:validate)
+      end
+
+      it "should call #validate" do
+        subject.set(123)
+      end
+    end
   end
 
   describe "#reset" do

data/spec/logstash/settings/string_spec.rb

@@ -4,7 +4,7 @@ require "logstash/settings"
 
 describe LogStash::Setting::String do
   let(:possible_values) { ["a", "b", "c"] }
-  subject { described_class.new("mytext", nil, false, possible_values) }
+  subject { described_class.new("mytext", possible_values.first, true, possible_values) }
   describe "#set" do
     context "when a value is given outside of possible_values" do
       it "should raise an ArgumentError" do

data/spec/logstash/settings_spec.rb

@@ -59,4 +59,31 @@ describe LogStash::Settings do
       expect(subset.get("num.2")).to eq(1000)
     end
   end
+
+  describe "#validate_all" do
+    subject { described_class.new }
+    let(:numeric_setting_name) { "example" }
+    let(:numeric_setting) { LogStash::Setting.new(numeric_setting_name, Numeric, 1, false) }
+
+    before do
+      subject.register(numeric_setting)
+      subject.set_value(numeric_setting_name, value)
+    end
+
+    context "when any setting is invalid" do
+      let(:value) { "some string" }
+
+      it "should fail" do
+        expect { subject.validate_all }.to raise_error
+      end
+    end
+
+    context "when all settings are valid" do
+      let(:value) { 123 }
+
+      it "should succeed" do
+        expect { subject.validate_all }.not_to raise_error
+      end
+    end
+  end
 end

data/spec/logstash/util/wrapped_synchronous_queue_spec.rb

@@ -94,22 +94,23 @@ describe LogStash::Util::WrappedSynchronousQueue do
 
         it "appends batches to the queue" do
           batch = write_client.get_new_batch
-          5.times {|i| batch.push("value-#{i}")}
+          5.times {|i| batch.push(LogStash::Event.new({"message" => "value-#{i}"}))}
           write_client.push_batch(batch)
           read_batch = read_client.take_batch
           expect(read_batch.size).to eq(5)
           i = 0
           read_batch.each do |data|
-            expect(data).to eq("value-#{i}")
-            read_batch.cancel("value-#{i}") if i > 2
-            read_batch.merge("generated-#{i}") if i > 2
+            expect(data.get("message")).to eq("value-#{i}")
+            # read_batch.cancel("value-#{i}") if i > 2     # TODO: disabled for https://github.com/elastic/logstash/issues/6055 - will have to properly refactor
+            data.cancel if i > 2
+            read_batch.merge(LogStash::Event.new({"message" => "generated-#{i}"})) if i > 2
             i += 1
           end
-          expect(read_batch.cancelled_size).to eq(2)
+          # expect(read_batch.cancelled_size).to eq(2) # disabled for https://github.com/elastic/logstash/issues/6055
           i = 0
           read_batch.each do |data|
-            expect(data).to eq("value-#{i}") if i < 3
-            expect(data).to eq("generated-#{i}") if i > 2
+            expect(data.get("message")).to eq("value-#{i}") if i < 3
+            expect(data.get("message")).to eq("generated-#{i}") if i > 2
             i += 1
           end
         end

data/spec/support/mocks_classes.rb

@@ -25,3 +25,25 @@ class DummyOutput < LogStash::Outputs::Base
     @num_closes = 1
   end
 end
+
+class DummyOutputWithEventsArray < LogStash::Outputs::Base
+  config_name "dummyoutput2"
+  milestone 2
+
+  attr_reader :events
+
+  def initialize(params={})
+    super
+    @events = []
+  end
+
+  def register
+  end
+
+  def receive(event)
+    @events << event
+  end
+
+  def close
+  end
+end

metadata

@@ -1,21 +1,21 @@
 --- !ruby/object:Gem::Specification
 name: logstash-core
 version: !ruby/object:Gem::Version
-  version: 5.0.0.pre.rc1
+  version: 5.0.1
 platform: java
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-10-07 00:00:00.000000000 Z
+date: 2016-11-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 5.0.0.pre.rc1
+        version: 5.0.1
   name: logstash-core-event-java
   prerelease: false
   type: :runtime
@@ -23,7 +23,7 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 5.0.0.pre.rc1
+        version: 5.0.1
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
@@ -343,7 +343,6 @@ files:
 - lib/logstash/instrument/periodic_poller/base.rb
 - lib/logstash/instrument/periodic_poller/jvm.rb
 - lib/logstash/instrument/periodic_poller/os.rb
-- lib/logstash/instrument/periodic_poller/periodic_poller_observer.rb
 - lib/logstash/instrument/periodic_pollers.rb
 - lib/logstash/instrument/snapshot.rb
 - lib/logstash/java_integration.rb
@@ -427,6 +426,7 @@ files:
 - spec/logstash/instrument/metric_type/gauge_spec.rb
 - spec/logstash/instrument/namespaced_metric_spec.rb
 - spec/logstash/instrument/null_metric_spec.rb
+- spec/logstash/instrument/periodic_poller/base_spec.rb
 - spec/logstash/instrument/periodic_poller/jvm_spec.rb
 - spec/logstash/java_integration_spec.rb
 - spec/logstash/json_spec.rb
@@ -464,7 +464,7 @@ files:
 - vendor/jars/org/apache/logging/log4j/log4j-1.2-api/2.6.2/log4j-1.2-api-2.6.2.jar
 - vendor/jars/org/apache/logging/log4j/log4j-api/2.6.2/log4j-api-2.6.2.jar
 - vendor/jars/org/apache/logging/log4j/log4j-core/2.6.2/log4j-core-2.6.2.jar
-- vendor/jars/org/logstash/logstash-core/5.0.0-rc1/logstash-core-5.0.0-rc1.jar
+- vendor/jars/org/logstash/logstash-core/5.0.1/logstash-core-5.0.1.jar
 homepage: http://www.elastic.co/guide/en/logstash/current/index.html
 licenses:
 - Apache License (2.0)
@@ -481,9 +481,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">"
+  - - ">="
     - !ruby/object:Gem::Version
-      version: 1.3.1
+      version: '0'
 requirements: []
 rubyforge_project:
 rubygems_version: 2.4.8
@@ -522,6 +522,7 @@ test_files:
 - spec/logstash/instrument/metric_type/gauge_spec.rb
 - spec/logstash/instrument/namespaced_metric_spec.rb
 - spec/logstash/instrument/null_metric_spec.rb
+- spec/logstash/instrument/periodic_poller/base_spec.rb
 - spec/logstash/instrument/periodic_poller/jvm_spec.rb
 - spec/logstash/java_integration_spec.rb
 - spec/logstash/json_spec.rb

data/lib/logstash/instrument/periodic_poller/periodic_poller_observer.rb

@@ -1,19 +0,0 @@
-# encoding: utf-8
-module LogStash module Instrument module PeriodicPoller
-  class PeriodicPollerObserver
-    include LogStash::Util::Loggable
-    
-    def initialize(poller)
-      @poller = poller
-    end
-
-    def update(time, result, exception)
-      if exception
-        logger.error("PeriodicPoller exception", :poller => @poller,
-                     :result => result,
-                     :exception => exception,
-                     :executed_at => time)
-      end
-    end
-  end
-end; end; end