logstash-core 6.0.0.alpha1-java → 6.0.0.alpha2-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 81ae0f7af1f442de24af8f661aaaeda947a52536
-  data.tar.gz: 87404b518150d21bd9f20605b44ee0008a54efd0
+  metadata.gz: 58e1606b49e2fe865d648c113d2316ba19a4604d
+  data.tar.gz: 7c50f49a82f7bd2dec47546ddb51459e35d44c0c
 SHA512:
-  metadata.gz: f7091e24dce522fe665b44ec723751298635cb4dface96907ac0dd0c840284ae0fcd96cd837eb0aa4193cf5a0a3630b419f81165aa45dd5b5d83abdf58921cba
-  data.tar.gz: 05d429649b6302583ac6017d49e614ad75c98b62191e145ba1e71deb97d8168513fb787b52ea4f6bfdcbbaab6fc0ed528704e94661c2c454244bd9d022ff1636
+  metadata.gz: c8d297ddd76d6f6c2167f934169a041c7b1df0b76a39a661ddc308405159407fa0800a3a41cf0bd8aeca2a1684ebd45482672f83985f442f500b6f4ce6e9edad
+  data.tar.gz: 8e5e40ab3a364110604e825a43f05a83ea1b8da136f50f8ee6ae475edb69d25c6f70e076c20175eabacf6dddb9032e4d111a1d251ed22d9e5f393ee3c06ef90f

data/lib/logstash-core/version.rb

@@ -5,4 +5,4 @@
 # Note to authors: this should not include dashes because 'gem' barfs if
 # you include a dash in the version string.
 
-LOGSTASH_CORE_VERSION = "6.0.0-alpha1"
+LOGSTASH_CORE_VERSION = "6.0.0-alpha2"

data/lib/logstash/agent.rb

@@ -25,7 +25,7 @@ class LogStash::Agent
   include LogStash::Util::Loggable
   STARTED_AT = Time.now.freeze
 
-  attr_reader :metric, :name, :pipelines, :settings, :webserver, :dispatcher
+  attr_reader :metric, :name, :settings, :webserver, :dispatcher
   attr_accessor :logger
 
   # initialize method for LogStash::Agent
@@ -38,7 +38,10 @@ class LogStash::Agent
     @settings = settings
     @auto_reload = setting("config.reload.automatic")
 
+    # Do not use @pipelines directly. Use #with_pipelines which does locking
     @pipelines = {}
+    @pipelines_lock = java.util.concurrent.locks.ReentrantLock.new
+
     @name = setting("node.name")
     @http_host = setting("http.host")
     @http_port = setting("http.port")
@@ -48,14 +51,13 @@ class LogStash::Agent
 
     # This is for backward compatibility in the tests
     if source_loader.nil?
-      @source_loader = LogStash::Config::SOURCE_LOADER
+      @source_loader = LogStash::Config::SourceLoader.new
       @source_loader.add_source(LogStash::Config::Source::Local.new(@settings))
     else
       @source_loader = source_loader
     end
 
     @reload_interval = setting("config.reload.interval")
-    @pipelines_mutex = Mutex.new
 
     @collect_metric = setting("metric.collect")
 
@@ -129,6 +131,17 @@ class LogStash::Agent
     !@running.value
   end
 
+  # Safely perform an operation on the pipelines hash
+  # Using the correct synchronization
+  def with_pipelines
+    begin
+      @pipelines_lock.lock
+      yield @pipelines
+    ensure
+      @pipelines_lock.unlock
+    end
+  end
+
   def converge_state_and_update
     results = @source_loader.fetch
 
@@ -145,13 +158,14 @@ class LogStash::Agent
     # content of it.
     converge_result = nil
 
-    @pipelines_mutex.synchronize do
+    # we don't use the variable here, but we want the locking
+    with_pipelines do |pipelines|
       pipeline_actions = resolve_actions(results.response)
       converge_result = converge_state(pipeline_actions)
+      update_metrics(converge_result)
     end
 
     report_currently_running_pipelines(converge_result)
-    update_metrics(converge_result)
     dispatch_events(converge_result)
 
     converge_result
@@ -220,52 +234,64 @@ class LogStash::Agent
   end
 
   def get_pipeline(pipeline_id)
-    @pipelines_mutex.synchronize do
-      @pipelines[pipeline_id]
+    with_pipelines do |pipelines|
+      pipelines[pipeline_id]
     end
   end
 
   def pipelines_count
-    @pipelines_mutex.synchronize do
+    with_pipelines do |pipelines|
       pipelines.size
     end
   end
 
-  def running_pipelines
-    @pipelines_mutex.synchronize do
-      @pipelines.select {|pipeline_id, _| running_pipeline?(pipeline_id) }
+  def with_running_pipelines
+    with_pipelines do |pipelines|
+      yield pipelines.select {|pipeline_id, _| running_pipeline?(pipeline_id) }
     end
   end
 
   def running_pipelines?
-    @pipelines_mutex.synchronize do
-      @pipelines.select {|pipeline_id, _| running_pipeline?(pipeline_id) }.any?
+    running_pipelines_count > 0
+  end
+
+  def running_pipelines_count
+    with_running_pipelines do |pipelines|
+      pipelines.size
     end
   end
 
   def running_user_defined_pipelines?
-    running_user_defined_pipelines.any?
+    with_running_user_defined_pipelines do |pipelines|
+      pipelines.size > 0
+    end
   end
 
-  def running_user_defined_pipelines
-    @pipelines_mutex.synchronize do
-      @pipelines.select do |_, pipeline|
+  def with_running_user_defined_pipelines
+    with_pipelines do |pipelines|
+      found = pipelines.select do |_, pipeline|
         pipeline.running? && !pipeline.system?
       end
+
+      yield found
     end
   end
 
   def close_pipeline(id)
-    pipeline = @pipelines[id]
-    if pipeline
-      @logger.warn("closing pipeline", :id => id)
-      pipeline.close
+    with_pipelines do |pipelines|
+      pipeline = pipelines[id]
+      if pipeline
+        @logger.warn("closing pipeline", :id => id)
+        pipeline.close
+      end
     end
   end
 
   def close_pipelines
-    @pipelines.each  do |id, _|
-      close_pipeline(id)
+    with_pipelines do |pipelines|
+      pipelines.each  do |id, _|
+        close_pipeline(id)
+      end
     end
   end
 
@@ -308,20 +334,22 @@ class LogStash::Agent
       #
       # This give us a bit more extensibility with the current startup/validation model
       # that we currently have.
-      begin
-        logger.debug("Executing action", :action => action)
-        action_result = action.execute(self, @pipelines)
-        converge_result.add(action, action_result)
-
-        unless action_result.successful?
-          logger.error("Failed to execute action", :id => action.pipeline_id,
-                       :action_type => action_result.class, :message => action_result.message)
+      with_pipelines do |pipelines|
+        begin
+          logger.debug("Executing action", :action => action)
+            action_result = action.execute(self, pipelines)
+          converge_result.add(action, action_result)
+
+          unless action_result.successful?
+            logger.error("Failed to execute action", :id => action.pipeline_id,
+                        :action_type => action_result.class, :message => action_result.message)
+          end
+        rescue SystemExit => e
+          converge_result.add(action, e)
+        rescue Exception => e
+          logger.error("Failed to execute action", :action => action, :exception => e.class.name, :message => e.message)
+          converge_result.add(action, e)
         end
-      rescue SystemExit => e
-        converge_result.add(action, e)
-      rescue Exception => e
-        logger.error("Failed to execute action", :action => action, :exception => e.class.name, :message => e.message)
-        converge_result.add(action, e)
       end
     end
 
@@ -335,13 +363,17 @@ class LogStash::Agent
   end
 
   def resolve_actions(pipeline_configs)
-    @state_resolver.resolve(@pipelines, pipeline_configs)
+    with_pipelines do |pipelines|
+      @state_resolver.resolve(pipelines, pipeline_configs)
+    end
   end
 
   def report_currently_running_pipelines(converge_result)
     if converge_result.success? && converge_result.total > 0
-      number_of_running_pipeline = running_pipelines.size
-      logger.info("Pipelines running", :count => number_of_running_pipeline, :pipelines => running_pipelines.values.collect(&:pipeline_id) )
+      with_running_pipelines do |pipelines|
+        number_of_running_pipeline = pipelines.size
+        logger.info("Pipelines running", :count => number_of_running_pipeline, :pipelines => pipelines.values.collect(&:pipeline_id) )
+      end
     end
   end
 
@@ -394,9 +426,11 @@ class LogStash::Agent
   end
 
   def force_shutdown_pipelines!
-    @pipelines.each do |_, pipeline|
-      # TODO(ph): should it be his own action?
-      pipeline.force_shutdown!
+    with_pipelines do |pipelines|
+      pipelines.each do |_, pipeline|
+        # TODO(ph): should it be his own action?
+        pipeline.force_shutdown!
+      end
     end
   end
 
@@ -406,19 +440,21 @@ class LogStash::Agent
     # In this context I could just call shutdown, but I've decided to
     # use the stop action implementation for that so we have the same code.
     # This also give us some context into why a shutdown is failing
-    @pipelines_mutex.synchronize do
+    with_pipelines do |pipelines|
       pipeline_actions = resolve_actions([]) # We stop all the pipeline, so we converge to a empty state
       converge_state(pipeline_actions)
     end
   end
 
   def running_pipeline?(pipeline_id)
-    thread = @pipelines[pipeline_id].thread
+    thread = get_pipeline(pipeline_id).thread
     thread.is_a?(Thread) && thread.alive?
   end
 
   def clean_state?
-    @pipelines.empty?
+    with_pipelines do |pipelines|
+      pipelines.empty?
+    end
   end
 
   def setting(key)

data/lib/logstash/api/commands/hot_threads_reporter.rb

@@ -1,4 +1,5 @@
 # encoding: utf-8
+java_import 'org.logstash.instrument.reports.ThreadsReport'
 
 class HotThreadsReport
   STRING_SEPARATOR_LENGTH = 80.freeze
@@ -7,8 +8,7 @@ class HotThreadsReport
   def initialize(cmd, options)
     @cmd = cmd
     filter = { :stacktrace_size => options.fetch(:stacktrace_size, HOT_THREADS_STACK_TRACES_SIZE_DEFAULT) }
-    jr_dump = JRMonitor.threads.generate(filter)
-    @thread_dump = ::LogStash::Util::ThreadDump.new(options.merge(:dump => jr_dump))
+    @thread_dump = ::LogStash::Util::ThreadDump.new(options.merge(:dump => ThreadsReport.generate(filter)))
   end
 
   def to_s
@@ -42,7 +42,7 @@ class HotThreadsReport
       _hash["thread.stacktrace"].each do |trace|
         traces << trace
       end
-      thread[:traces] = traces unless traces.empty?
+      thread[:traces] = traces
       hash[:threads] << thread
     end
     { :hot_threads => hash }

data/lib/logstash/api/commands/node.rb

@@ -9,7 +9,7 @@ module LogStash
 
         def all(selected_fields=[])
           payload = {
-            :pipeline => pipeline,
+            :pipelines => pipelines,
             :os => os,
             :jvm => jvm
           }
@@ -17,12 +17,20 @@ module LogStash
           payload
         end
 
-        def pipeline(pipeline_id = LogStash::SETTINGS.get("pipeline.id").to_sym)
-          stats = extract_metrics(
-            [:stats, :pipelines, pipeline_id, :config],
+        def pipelines
+          pipeline_ids = service.get_shallow(:stats, :pipelines).keys
+          pipeline_ids.each_with_object({}) do |pipeline_id, result|
+            result[pipeline_id] = pipeline(pipeline_id)
+          end
+        end
+
+        def pipeline(pipeline_id)
+          extract_metrics(
+            [:stats, :pipelines, pipeline_id.to_sym, :config],
             :workers, :batch_size, :batch_delay, :config_reload_automatic, :config_reload_interval
           )
-          stats.merge(:id => pipeline_id)
+        rescue
+          {}
         end
 
         def os
@@ -40,7 +48,6 @@ module LogStash
           {
             :pid =>  ManagementFactory.getRuntimeMXBean().getName().split("@").first.to_i,
             :version => java.lang.System.getProperty("java.version"),
-            :vm_name => java.lang.System.getProperty("java.vm.name"),
             :vm_version => java.lang.System.getProperty("java.version"),
             :vm_vendor => java.lang.System.getProperty("java.vendor"),
             :vm_name => java.lang.System.getProperty("java.vm.name"),

data/lib/logstash/api/commands/stats.rb

@@ -41,20 +41,26 @@ module LogStash
         def events
           extract_metrics(
             [:stats, :events],
-            :in, :filtered, :out, :duration_in_millis
+            :in, :filtered, :out, :duration_in_millis, :queue_push_duration_in_millis
           )
         end
 
-        def pipeline(pipeline_id = LogStash::SETTINGS.get("pipeline.id").to_sym)
-          stats = service.get_shallow(:stats, :pipelines, pipeline_id)
-          stats = PluginsStats.report(stats)
-          stats.merge(:id => pipeline_id)
+        def pipeline(pipeline_id = nil)
+          if pipeline_id.nil?
+            pipeline_ids = service.get_shallow(:stats, :pipelines).keys
+            pipeline_ids.each_with_object({}) do |pipeline_id, result|
+              result[pipeline_id] = plugins_stats_report(pipeline_id)
+            end
+          else
+            { pipeline_id => plugins_stats_report(pipeline_id) }
+          end
+        rescue # failed to find pipeline
+          {}
         end
 
         def memory
           memory = service.get_shallow(:jvm, :memory)
           {
-            :heap_used_in_bytes => memory[:heap][:used_in_bytes],
             :heap_used_percent => memory[:heap][:used_percent],
             :heap_committed_in_bytes => memory[:heap][:committed_in_bytes],
             :heap_max_in_bytes => memory[:heap][:max_in_bytes],
@@ -85,6 +91,12 @@ module LogStash
           HotThreadsReport.new(self, options)
         end
 
+        private
+        def plugins_stats_report(pipeline_id)
+          stats = service.get_shallow(:stats, :pipelines, pipeline_id.to_sym)
+          PluginsStats.report(stats)
+        end
+
         module PluginsStats
           module_function
 

data/lib/logstash/api/modules/node.rb

@@ -20,6 +20,13 @@ module LogStash
           respond_with(node.hot_threads(options), {:as => as})
         end
 
+        get "/pipelines/:id" do
+          pipeline_id = params["id"]
+          payload = node.pipeline(pipeline_id)
+          halt(404) if payload.empty?
+          respond_with(:pipelines => { pipeline_id => payload } )
+        end
+
          get "/?:filter?" do
            selected_fields = extract_fields(params["filter"].to_s.strip)
            values = node.all(selected_fields)

data/lib/logstash/api/modules/node_stats.rb

@@ -8,12 +8,19 @@ module LogStash
           @stats = factory.build(:stats)
         end
 
+        get "/pipelines/:id?" do
+          payload = pipeline_payload(params["id"])
+          halt(404) if payload.empty?
+          respond_with(:pipelines => payload)
+        end
+
         get "/?:filter?" do
           payload = {
             :jvm => jvm_payload,
             :process => process_payload,
-            :pipeline => pipeline_payload,
-            :reloads => reloads,
+            :events => events_payload,
+            :pipelines => pipeline_payload,
+            :reloads => reloads_payload,
             :os => os_payload
           }
           respond_with(payload, {:filter => params["filter"]})
@@ -32,7 +39,7 @@ module LogStash
           @stats.jvm
         end
 
-        def reloads
+        def reloads_payload
           @stats.reloads
         end
 
@@ -44,8 +51,8 @@ module LogStash
           @stats.memory
         end
 
-        def pipeline_payload
-          @stats.pipeline
+        def pipeline_payload(val = nil)
+          @stats.pipeline(val)
         end
       end
     end

data/lib/logstash/bootstrap_check/default_config.rb

@@ -4,13 +4,9 @@ require "logstash/errors"
 module LogStash module BootstrapCheck
   class DefaultConfig
     def self.check(settings)
-      if settings.get("config.string").nil? && settings.get("path.config").nil?
-        raise LogStash::BootstrapCheckError, I18n.t("logstash.runner.missing-configuration")
-      end
-
-      if settings.get("config.reload.automatic") && settings.get("path.config").nil?
-        # there's nothing to reload
-        raise LogStash::BootstrapCheckError, I18n.t("logstash.runner.reload-without-config-path")
+      # currently none of the checks applies if there are multiple pipelines
+      if settings.get("config.reload.automatic") && settings.get_setting("config.string").set?
+        raise LogStash::BootstrapCheckError, I18n.t("logstash.runner.reload-with-config-string")
       end
     end
   end