logstash-core 7.0.0.alpha2-java → 7.0.0.beta1-java

data/lib/logstash/pipeline.rb

@@ -107,8 +107,23 @@ module LogStash; class Pipeline < BasePipeline
     @flushing = Concurrent::AtomicReference.new(false)
     @outputs_registered = Concurrent::AtomicBoolean.new(false)
     @worker_shutdown = java.util.concurrent.atomic.AtomicBoolean.new(false)
+
+    # @finished_execution signals that the pipeline thread has finished its execution
+    # regardless of any exceptions; it will always be true when the thread completes
+    @finished_execution = Concurrent::AtomicBoolean.new(false)
+
+    # @finished_run signals that the run methods called in the pipeline thread was completed
+    # without errors and it will NOT be set if the run method exits from an exception; this
+    # is by design and necessary for the wait_until_started semantic
+    @finished_run = Concurrent::AtomicBoolean.new(false)
+
+    @thread = nil
   end # def initialize
 
+  def finished_execution?
+    @finished_execution.true?
+  end
+
   def ready?
     @ready.value
   end
@@ -152,16 +167,19 @@ module LogStash; class Pipeline < BasePipeline
       "pipeline.batch.size" => settings.get("pipeline.batch.size"),
       "pipeline.batch.delay" => settings.get("pipeline.batch.delay")))
 
-    @finished_execution = Concurrent::AtomicBoolean.new(false)
+    @finished_execution.make_false
+    @finished_run.make_false
 
     @thread = Thread.new do
       begin
         LogStash::Util.set_thread_name("pipeline.#{pipeline_id}")
         run
-        @finished_execution.make_true
+        @finished_run.make_true
       rescue => e
         close
         @logger.error("Pipeline aborted due to error", default_logging_keys(:exception => e, :backtrace => e.backtrace))
+      ensure
+        @finished_execution.make_true
       end
     end
 
@@ -176,15 +194,14 @@ module LogStash; class Pipeline < BasePipeline
 
   def wait_until_started
     while true do
-      # This should be changed with an appropriate FSM
-      # It's an edge case, if we have a pipeline with
-      # a generator { count => 1 } its possible that `Thread#alive?` doesn't return true
-      # because the execution of the thread was successful and complete
-      if @finished_execution.true?
+      if @finished_run.true?
+        # it completed run without exception
         return true
-      elsif !thread.alive?
+      elsif thread.nil? || !thread.alive?
+        # some exception occured and the thread is dead
         return false
       elsif running?
+        # fully initialized and running
         return true
       else
         sleep 0.01
@@ -283,9 +300,9 @@ module LogStash; class Pipeline < BasePipeline
 
       pipeline_workers.times do |t|
         thread = Thread.new(batch_size, batch_delay, self) do |_b_size, _b_delay, _pipeline|
+          Util.set_thread_name("[#{pipeline_id}]>worker#{t}")
           _pipeline.worker_loop(_b_size, _b_delay)
         end
-        Util.set_thread_name("[#{pipeline_id}]>worker#{t}")
         @worker_threads << thread
       end
 
@@ -525,7 +542,7 @@ module LogStash; class Pipeline < BasePipeline
 
   # Calculate the uptime in milliseconds
   #
-  # @return [Fixnum] Uptime in milliseconds, 0 if the pipeline is not started
+  # @return [Integer] Uptime in milliseconds, 0 if the pipeline is not started
   def uptime
     return 0 if started_at.nil?
     ((Time.now.to_f - started_at.to_f) * 1000.0).to_i

data/lib/logstash/pipeline_action/base.rb

@@ -12,7 +12,7 @@ module LogStash module PipelineAction
     end
     alias_method :to_s, :inspect
 
-    def execute(agent, pipelines)
+    def execute(agent, pipelines_registry)
       raise "`#execute` Not implemented!"
     end
 

data/lib/logstash/pipeline_action/create.rb

@@ -3,6 +3,7 @@ require "logstash/pipeline_action/base"
 require "logstash/pipeline"
 require "logstash/java_pipeline"
 
+
 module LogStash module PipelineAction
   class Create < Base
     include LogStash::Util::Loggable
@@ -30,33 +31,26 @@ module LogStash module PipelineAction
 
     # The execute assume that the thread safety access of the pipeline
     # is managed by the caller.
-    def execute(agent, pipelines)
-      pipeline =
+    def execute(agent, pipelines_registry)
+      new_pipeline =
         if @pipeline_config.settings.get_value("pipeline.java_execution")
           LogStash::JavaPipeline.new(@pipeline_config, @metric, agent)
         else
           agent.exclusive do
             # The Ruby pipeline initialization is not thread safe because of the module level
             # shared state in LogsStash::Config::AST. When using multiple pipelines this gets
-            # executed simultaneously in different threads and we need to synchonize this initialization.
+            # executed simultaneously in different threads and we need to synchronize this initialization.
             LogStash::Pipeline.new(@pipeline_config, @metric, agent)
           end
         end
 
-      status = nil
-      pipelines.compute(pipeline_id) do |id,value|
-        if value
-          LogStash::ConvergeResult::ActionResult.create(self, true)
-        end
-        status = pipeline.start # block until the pipeline is correctly started or crashed
-        pipeline # The pipeline is successfully started we can add it to the map
+      success = pipelines_registry.create_pipeline(pipeline_id, new_pipeline) do
+        new_pipeline.start # block until the pipeline is correctly started or crashed
       end
 
-
-      LogStash::ConvergeResult::ActionResult.create(self, status)
+      LogStash::ConvergeResult::ActionResult.create(self, success)
     end
 
-
     def to_s
       "PipelineAction::Create<#{pipeline_id}>"
     end

data/lib/logstash/pipeline_action/reload.rb

@@ -20,8 +20,12 @@ module LogStash module PipelineAction
       "PipelineAction::Reload<#{pipeline_id}>"
     end
 
-    def execute(agent, pipelines)
-      old_pipeline = pipelines[pipeline_id]
+    def execute(agent, pipelines_registry)
+      old_pipeline = pipelines_registry.get_pipeline(pipeline_id)
+
+      if old_pipeline.nil?
+        return LogStash::ConvergeResult::FailedAction.new("Cannot reload pipeline, because the pipeline does not exist")
+      end
 
       if !old_pipeline.reloadable?
         return LogStash::ConvergeResult::FailedAction.new("Cannot reload pipeline, because the existing pipeline is not reloadable")
@@ -34,8 +38,8 @@ module LogStash module PipelineAction
           else
             agent.exclusive do
               # The Ruby pipeline initialization is not thread safe because of the module level
-              # shared state in LogsStash::Config::AST. When using multiple pipelines this can gets
-              # executed simultaneously in different threads and we need to synchonize this initialization.
+              # shared state in LogsStash::Config::AST. When using multiple pipelines this gets
+              # executed simultaneously in different threads and we need to synchronize this initialization.
               LogStash::BasePipeline.new(@pipeline_config)
             end
           end
@@ -49,16 +53,35 @@ module LogStash module PipelineAction
 
       logger.info("Reloading pipeline", "pipeline.id" => pipeline_id)
 
-      pipelines.compute(pipeline_id) do |_,pipeline|
-        status = Stop.new(pipeline_id).execute(agent, pipelines)
+      success = pipelines_registry.reload_pipeline(pipeline_id) do
+        # important NOT to explicitly return from block here
+        # the block must emit a success boolean value
 
-        if status
-          return Create.new(@pipeline_config, @metric).execute(agent, pipelines)
-        else
-          return status
-        end
-        pipeline
+        # First shutdown old pipeline
+        old_pipeline.shutdown { LogStash::ShutdownWatcher.start(old_pipeline) }
+        old_pipeline.thread.join
+
+        # Then create a new pipeline
+        new_pipeline =
+          if @pipeline_config.settings.get_value("pipeline.java_execution")
+            LogStash::JavaPipeline.new(@pipeline_config, @metric, agent)
+          else
+            agent.exclusive do
+              # The Ruby pipeline initialization is not thread safe because of the module level
+              # shared state in LogsStash::Config::AST. When using multiple pipelines this gets
+              # executed simultaneously in different threads and we need to synchronize this initialization.
+              LogStash::Pipeline.new(@pipeline_config, @metric, agent)
+            end
+          end
+
+        success = new_pipeline.start # block until the pipeline is correctly started or crashed
+
+        # return success and new_pipeline to registry reload_pipeline
+        [success, new_pipeline]
       end
+
+      LogStash::ConvergeResult::ActionResult.create(self, success)
     end
+
   end
 end end

data/lib/logstash/pipeline_action/stop.rb

@@ -9,15 +9,13 @@ module LogStash module PipelineAction
       @pipeline_id = pipeline_id
     end
 
-    def execute(agent, pipelines)
-      pipelines.compute(pipeline_id) do |_,pipeline|
+    def execute(agent, pipelines_registry)
+      pipelines_registry.terminate_pipeline(pipeline_id) do |pipeline|
         pipeline.shutdown { LogStash::ShutdownWatcher.start(pipeline) }
         pipeline.thread.join
-        nil # delete the pipeline
       end
-      # If we reach this part of the code we have succeeded because
-      # the shutdown call will block.
-      return LogStash::ConvergeResult::SuccessfulAction.new
+
+      LogStash::ConvergeResult::SuccessfulAction.new
     end
 
     def to_s

data/lib/logstash/pipeline_settings.rb

@@ -21,13 +21,13 @@ module LogStash
       "pipeline.batch.delay",
       "pipeline.batch.size",
       "pipeline.id",
-      "pipeline.output.workers",
       "pipeline.reloadable",
       "pipeline.system",
       "pipeline.workers",
       "queue.checkpoint.acks",
       "queue.checkpoint.interval",
       "queue.checkpoint.writes",
+      "queue.checkpoint.retry",
       "queue.drain",
       "queue.max_bytes",
       "queue.max_events",

data/lib/logstash/pipelines_registry.rb

@@ -0,0 +1,166 @@
+# encoding: utf-8
+
+module LogStash
+  class PipelineState
+    attr_reader :pipeline_id, :pipeline
+
+    def initialize(pipeline_id, pipeline)
+      @pipeline_id = pipeline_id
+      @pipeline = pipeline
+      @reloading = Concurrent::AtomicBoolean.new(false)
+    end
+
+    def terminated?
+      # a reloading pipeline is never considered terminated
+      @reloading.false? && @pipeline.finished_execution?
+    end
+
+    def set_reloading(is_reloading)
+      @reloading.value = is_reloading
+    end
+
+    def set_pipeline(pipeline)
+      raise(ArgumentError, "invalid nil pipeline") if pipeline.nil?
+      @pipeline = pipeline
+    end
+  end
+
+  class PipelinesRegistry
+    attr_reader :states
+    include LogStash::Util::Loggable
+
+    def initialize
+      # we leverage the semantic of the Java ConcurrentHashMap for the
+      # compute() method which is atomic; calling compute() concurrently
+      # will block until the other compute finishes so no mutex is necessary
+      # for synchronizing compute calls
+      @states = java.util.concurrent.ConcurrentHashMap.new
+    end
+
+    # Execute the passed creation logic block and create a new state upon success
+    # @param pipeline_id [String, Symbol] the pipeline id
+    # @param pipeline [Pipeline] the new pipeline to create
+    # @param create_block [Block] the creation execution logic
+    #
+    # @yieldreturn [Boolean] the new pipeline creation success
+    #
+    # @return [Boolean] new pipeline creation success
+    def create_pipeline(pipeline_id, pipeline, &create_block)
+      success = false
+
+      @states.compute(pipeline_id) do |_, state|
+        if state
+          if state.terminated?
+            success = yield
+            state.set_pipeline(pipeline)
+          else
+            logger.error("Attempted to create a pipeline that already exists", :pipeline_id => pipeline_id)
+          end
+          state
+        else
+          success = yield
+          success ? PipelineState.new(pipeline_id, pipeline) : nil
+        end
+      end
+
+      success
+    end
+
+    # Execute the passed termination logic block
+    # @param pipeline_id [String, Symbol] the pipeline id
+    # @param stop_block [Block] the termination execution logic
+    #
+    # @yieldparam [Pipeline] the pipeline to terminate
+    def terminate_pipeline(pipeline_id, &stop_block)
+      @states.compute(pipeline_id) do |_, state|
+        if state.nil?
+          logger.error("Attempted to terminate a pipeline that does not exists", :pipeline_id => pipeline_id)
+          nil
+        else
+          yield(state.pipeline)
+          state
+        end
+      end
+    end
+
+    # Execute the passed reloading logic block in the context of the reloading state and set new pipeline in state
+    # @param pipeline_id [String, Symbol] the pipeline id
+    # @param reload_block [Block] the reloading execution logic
+    #
+    # @yieldreturn [Array<Boolean, Pipeline>] the new pipeline creation success and new pipeline object
+    #
+    # @return [Boolean] new pipeline creation success
+    def reload_pipeline(pipeline_id, &reload_block)
+      success = false
+
+      @states.compute(pipeline_id) do |_, state|
+        if state.nil?
+          logger.error("Attempted to reload a pipeline that does not exists", :pipeline_id => pipeline_id)
+          nil
+        else
+          state.set_reloading(true)
+          begin
+            success, new_pipeline = yield
+            state.set_pipeline(new_pipeline)
+          ensure
+            state.set_reloading(false)
+          end
+          state
+        end
+      end
+
+      success
+    end
+
+    # @param pipeline_id [String, Symbol] the pipeline id
+    # @return [Pipeline] the pipeline object or nil if none for pipeline_id
+    def get_pipeline(pipeline_id)
+      state = @states.get(pipeline_id)
+      state.nil? ? nil : state.pipeline
+    end
+
+    # @return [Fixnum] number of items in the states collection
+    def size
+      @states.size
+    end
+
+    # @return [Boolean] true if the states collection is empty.
+    def empty?
+      @states.isEmpty
+    end
+
+    # @return [Hash{String=>Pipeline}]
+    def running_pipelines
+      select_pipelines { |state| !state.terminated? }
+    end
+
+    # @return [Hash{String=>Pipeline}]
+    def non_running_pipelines
+      select_pipelines { |state| state.terminated? }
+    end
+
+    # @return [Hash{String=>Pipeline}]
+    def running_user_defined_pipelines
+      select_pipelines { |state | !state.terminated? && !state.pipeline.system? }
+    end
+
+    private
+
+    # Returns a mapping of pipelines by their ids.
+    # Pipelines can optionally be filtered by their `PipelineState` by passing
+    # a block that returns truthy when a pipeline should be included in the
+    # result.
+    #
+    # @yieldparam [PipelineState]
+    # @yieldreturn [Boolean]
+    #
+    # @return [Hash{String=>Pipeline}]
+    def select_pipelines(&optional_state_filter)
+      @states.each_with_object({}) do |(id, state), memo|
+        if state && (!block_given? || yield(state))
+          memo[id] = state.pipeline
+        end
+      end
+    end
+  end
+end

data/lib/logstash/settings.rb

@@ -368,7 +368,7 @@ module LogStash
         case value
         when ::Range
           value
-        when ::Fixnum
+        when ::Integer
           value..value
         when ::String
           first, last = value.split(PORT_SEPARATOR)
@@ -481,11 +481,11 @@ module LogStash
 
     class Bytes < Coercible
       def initialize(name, default=nil, strict=true)
-        super(name, ::Fixnum, default, strict=true) { |value| valid?(value) }
+        super(name, ::Integer, default, strict=true) { |value| valid?(value) }
       end
 
       def valid?(value)
-        value.is_a?(Fixnum) && value >= 0
+        value.is_a?(::Integer) && value >= 0
       end
 
       def coerce(value)
@@ -508,11 +508,11 @@ module LogStash
 
     class TimeValue < Coercible
       def initialize(name, default, strict=true, &validator_proc)
-        super(name, ::Fixnum, default, strict, &validator_proc)
+        super(name, ::Integer, default, strict, &validator_proc)
       end
 
       def coerce(value)
-        return value if value.is_a?(::Fixnum)
+        return value if value.is_a?(::Integer)
         Util::TimeValue.from_value(value).to_nanos
       end
     end

data/lib/logstash/state_resolver.rb

@@ -10,11 +10,11 @@ module LogStash
       @metric = metric
     end
 
-    def resolve(pipelines, pipeline_configs)
+    def resolve(pipelines_registry, pipeline_configs)
       actions = []
 
       pipeline_configs.each do |pipeline_config|
-        pipeline = pipelines[pipeline_config.pipeline_id]
+        pipeline = pipelines_registry.get_pipeline(pipeline_config.pipeline_id)
 
         if pipeline.nil?
           actions << LogStash::PipelineAction::Create.new(pipeline_config, @metric)
@@ -25,12 +25,12 @@ module LogStash
         end
       end
 
-      running_pipelines = pipeline_configs.collect(&:pipeline_id)
+      configured_pipelines = pipeline_configs.collect(&:pipeline_id)
 
       # If one of the running pipeline is not in the pipeline_configs, we assume that we need to
       # stop it.
-      pipelines.keys
-        .select { |pipeline_id| !running_pipelines.include?(pipeline_id) }
+      pipelines_registry.running_pipelines.keys
+        .select { |pipeline_id| !configured_pipelines.include?(pipeline_id) }
         .each { |pipeline_id| actions << LogStash::PipelineAction::Stop.new(pipeline_id) }
 
       actions.sort # See logstash/pipeline_action.rb

data/lib/logstash/util/duration_formatter.rb

@@ -7,7 +7,7 @@ module LogStash::Util::DurationFormatter
   # a format that a human can understand. This is currently used by
   # the API.
   #
-  # @param [Fixnum] Duration in milliseconds
+  # @param [Integer] Duration in milliseconds
   # @return [String] Duration in human format
   def self.human_format(duration)
     ChronicDuration.output(duration / 1000, CHRONIC_OPTIONS)

data/lib/logstash/util/safe_uri.rb

@@ -26,6 +26,7 @@ class LogStash::Util::SafeURI
            else
              raise ArgumentError, "Expected a string, java.net.URI, or URI, got a #{arg.class} creating a URL"
            end
+    raise ArgumentError, "URI is not valid - host is not specified" if @uri.host.nil?
   end
 
   def to_s

data/lib/logstash/util.rb

@@ -9,6 +9,8 @@ module LogStash::Util
 
   PR_SET_NAME = 15
   def self.set_thread_name(name)
+    previous_name = Java::java.lang.Thread.currentThread.getName() if block_given?
+
     if RUBY_ENGINE == "jruby"
       # Keep java and ruby thread names in sync.
       Java::java.lang.Thread.currentThread.setName(name)
@@ -21,6 +23,14 @@ module LogStash::Util
       # since MRI 1.9, JRuby, and Rubinius use system threads for this.
       LibC.prctl(PR_SET_NAME, name[0..16], 0, 0, 0)
     end
+
+    if block_given?
+      begin
+        yield
+      ensure
+        set_thread_name(previous_name)
+      end
+    end
   end # def set_thread_name
 
   def self.set_thread_plugin(plugin)
@@ -191,7 +201,7 @@ module LogStash::Util
       o.inject({}) {|h, (k,v)| h[k] = deep_clone(v); h }
     when Array
       o.map {|v| deep_clone(v) }
-    when Fixnum, Symbol, IO, TrueClass, FalseClass, NilClass
+    when Integer, Symbol, IO, TrueClass, FalseClass, NilClass
       o
     when LogStash::Codecs::Base
       o.clone

data/locales/en.yml

@@ -289,7 +289,7 @@ en:
         pipeline-workers: |+
           Sets the number of pipeline workers to run.
         java-execution: |+
-          (Beta) Use new Java execution engine.
+          Use Java execution engine.
         pipeline-batch-size: |+
           Size of batches the pipeline is to work in.
         pipeline-batch-delay: |+

data/logstash-core.gemspec

@@ -47,32 +47,29 @@ Gem::Specification.new do |gem|
 
   gem.platform = "java"
 
-  gem.add_runtime_dependency "pry", "~> 0.10.1"  #(Ruby license)
+  gem.add_runtime_dependency "pry", "~> 0.12"  #(Ruby license)
   gem.add_runtime_dependency "stud", "~> 0.0.19" #(Apache 2.0 license)
-  gem.add_runtime_dependency "clamp", "~> 0.6.5" #(MIT license) for command line args/flags
-  gem.add_runtime_dependency "filesize", "0.0.4" #(MIT license) for :bytes config validator
-  gem.add_runtime_dependency "gems", "~> 0.8.3"  #(MIT license)
-  gem.add_runtime_dependency "concurrent-ruby", "~> 1.0", ">= 1.0.5"
-  gem.add_runtime_dependency "rack", '~> 1.6', '>= 1.6.11'
-  gem.add_runtime_dependency "sinatra", '~> 1.4', '>= 1.4.6'
-  gem.add_runtime_dependency 'puma', '~> 2.16'
-  gem.add_runtime_dependency "jruby-openssl", ">= 0.9.20" # >= 0.9.13 Required to support TLSv1.2
-  gem.add_runtime_dependency "chronic_duration", "0.10.6"
+  gem.add_runtime_dependency "clamp", "~> 0.6" #(MIT license) for command line args/flags
+  gem.add_runtime_dependency "filesize", "~> 0.2" #(MIT license) for :bytes config validator
+  gem.add_runtime_dependency "gems", "~> 1"  #(MIT license)
+  gem.add_runtime_dependency "concurrent-ruby", "~> 1"
+  gem.add_runtime_dependency "rack", '~> 1', '>= 1.6.11'
+  gem.add_runtime_dependency "sinatra", '~> 1', '>= 1.4.6'
+  gem.add_runtime_dependency 'puma', '~> 2'
+  gem.add_runtime_dependency "jruby-openssl", "~> 0.10" # >= 0.9.13 Required to support TLSv1.2
+  gem.add_runtime_dependency "chronic_duration", "~> 0.10"
 
-  # TODO(sissel): Treetop 1.5.x doesn't seem to work well, but I haven't
-  # investigated what the cause might be. -Jordan
-  gem.add_runtime_dependency "treetop", "< 1.5.0" #(MIT license)
+  gem.add_runtime_dependency "treetop", "~> 1" #(MIT license)
 
-  # upgrade i18n only post 0.6.11, see https://github.com/svenfuchs/i18n/issues/270
-  gem.add_runtime_dependency "i18n", "= 0.6.9" #(MIT license)
+  gem.add_runtime_dependency "i18n", "~> 1" #(MIT license)
 
   # filetools and rakelib
-  gem.add_runtime_dependency "minitar", "~> 0.6.1"
-  gem.add_runtime_dependency "rubyzip", "~> 1.2.1"
-  gem.add_runtime_dependency "thread_safe", "~> 0.3.5" #(Apache 2.0 license)
+  gem.add_runtime_dependency "minitar", "~> 0.8"
+  gem.add_runtime_dependency "rubyzip", "~> 1"
+  gem.add_runtime_dependency "thread_safe", "~> 0.3.6" #(Apache 2.0 license)
 
   gem.add_runtime_dependency "jrjackson", "= #{ALL_VERSIONS.fetch('jrjackson')}" #(Apache 2.0 license)
 
-  gem.add_runtime_dependency "elasticsearch", "~> 5.0", ">= 5.0.4" # Ruby client for ES (Apache 2.0 license)
-  gem.add_runtime_dependency "manticore", '>= 0.5.4', '< 1.0.0'
+  gem.add_runtime_dependency "elasticsearch", "~> 5"
+  gem.add_runtime_dependency "manticore", '~> 0.6'
 end

data/spec/logstash/acked_queue_concurrent_stress_spec.rb

@@ -14,7 +14,7 @@ describe LogStash::WrappedAckedQueue, :stress_test => true do
     let(:reject_memo_keys) { [:reject_memo_keys, :path, :queue, :writer_threads, :collector, :metric, :reader_threads, :output_strings] }
 
     let(:queue) do
-      described_class.new(path, page_capacity, 0, queue_checkpoint_acks, queue_checkpoint_writes, queue_checkpoint_interval, queue_capacity)
+      described_class.new(path, page_capacity, 0, queue_checkpoint_acks, queue_checkpoint_writes, queue_checkpoint_interval, false, queue_capacity)
     end
 
     let(:writer_threads) do