logstash-core 5.0.0.alpha4.snapshot2-java → 5.0.0.alpha4.snapshot3-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: bd0ebd412460fd45c6abdef1ae4a21b75f24b6c5
-  data.tar.gz: 4addbb53a17c98501b27f8f8fa33f8475a44bca4
+  metadata.gz: c90549a9a03e8d32ee0eb8146780ad53609abc84
+  data.tar.gz: 14e872af82cfa7cb8dbfc1b5bbce6e8fcd4e8031
 SHA512:
-  metadata.gz: b602623cc49f59c90e0af86c1d43e90e2b611aef12e44d5590a8536c3efd0cf06deab5644f49477fdcc2a4d7615340bef5e4fcac613ddfff9e6e792594511958
-  data.tar.gz: 7dcc49def008f07dae78a07e78c6ba8fa7e2942e30341637aa873068bf71cbc3ccb82557299b915e79934f523eb469d98a4dfaaeb39abcffe715efe66057bfe3
+  metadata.gz: c666d73e6f87dd8de677888232066d807a2878105a40c023ad0a9d9f221eb9fea16ddf8694a3cd692c816ae92537c23a034e54162c71505990b53ba7a710b7d8
+  data.tar.gz: 45d360a8b9896e2ff744d0e84efce8126c33d5c4c6ce1d71a8fdbf89a61ee6940a2a80843cb7613d3eaa8baab677bf7c717009ac93a7c55b559678c840cc37f1

data/lib/logstash-core/version.rb

@@ -5,4 +5,4 @@
 # Note to authors: this should not include dashes because 'gem' barfs if
 # you include a dash in the version string.
 
-LOGSTASH_CORE_VERSION = "5.0.0-alpha4.snapshot2"
+LOGSTASH_CORE_VERSION = "5.0.0-alpha4.snapshot3"

data/lib/logstash/api/commands/stats.rb

@@ -33,6 +33,11 @@ module LogStash
           )
         end
 
+        def pipeline
+          stats = service.get_shallow(:stats, :pipelines)
+          PluginsStats.report(stats)
+        end
+
         def memory
           memory = service.get_shallow(:jvm, :memory)
           {
@@ -49,7 +54,98 @@ module LogStash
               acc
             end
           }
-        end       
+        end
+
+        def hot_threads(options={})
+          HotThreadsReport.new(self, options)
+        end
+
+        class HotThreadsReport
+          HOT_THREADS_STACK_TRACES_SIZE_DEFAULT = 10.freeze
+          
+          def initialize(cmd, options)
+            @cmd = cmd
+            filter = { :stacktrace_size => options.fetch(:stacktrace_size, HOT_THREADS_STACK_TRACES_SIZE_DEFAULT) }
+            jr_dump = JRMonitor.threads.generate(filter)
+            @thread_dump = ::LogStash::Util::ThreadDump.new(options.merge(:dump => jr_dump))
+          end
+          
+          def to_s
+            hash = to_hash
+            report =  "#{I18n.t("logstash.web_api.hot_threads.title", :hostname => hash[:hostname], :time => hash[:time], :top_count => @thread_dump.top_count )} \n"
+            report << '=' * 80
+            report << "\n"
+            hash[:threads].each do |thread|
+              thread_report = ""
+              thread_report = "#{I18n.t("logstash.web_api.
+                                hot_threads.thread_title", :percent_of_cpu_time => thread[:percent_of_cpu_time], :thread_state => thread[:state], :thread_name => thread[:name])} \n"
+              thread_report = "#{thread[:percent_of_cpu_time]} % of of cpu usage by #{thread[:state]} thread named '#{thread[:name]}'\n"
+              thread_report << "#{thread[:path]}\n" if thread[:path]
+              thread[:traces].each do |trace|
+                thread_report << "\t#{trace}\n"
+              end
+              report << thread_report
+              report << '-' * 80
+              report << "\n"
+            end
+            report
+          end
+
+          def to_hash
+            hash = { :hostname => @cmd.hostname, :time => Time.now.iso8601, :busiest_threads => @thread_dump.top_count, :threads => [] }
+            @thread_dump.each do |thread_name, _hash|
+              thread_name, thread_path = _hash["thread.name"].split(": ")
+              thread = { :name => thread_name,
+                         :percent_of_cpu_time => cpu_time_as_percent(_hash),
+                         :state => _hash["thread.state"]
+                       }
+              thread[:path] = thread_path if thread_path
+              traces = []
+              _hash["thread.stacktrace"].each do |trace|
+                traces << trace
+              end
+              thread[:traces] = traces unless traces.empty?
+              hash[:threads] << thread
+            end
+            hash
+          end
+
+          def cpu_time_as_percent(hash)
+            (((cpu_time(hash) / @cmd.uptime * 1.0)*10000).to_i)/100.0
+          end
+
+          def cpu_time(hash)
+            hash["cpu.time"] / 1000000.0
+          end
+        end # class HotThreadsReport
+
+        module PluginsStats
+          module_function
+
+          def plugin_stats(stats, plugin_type)
+            # Turn the `plugins` stats hash into an array of [ {}, {}, ... ]
+            # This is to produce an array of data points, one point for each
+            # plugin instance.
+            return [] unless stats[:plugins].include?(plugin_type)
+            stats[:plugins][plugin_type].collect do |id, data|
+              { :id => id }.merge(data)
+            end
+          end
+
+          def report(stats)
+            # Only one pipeline right now.
+            stats = stats[:main]
+
+            {
+              :events => stats[:events],
+              :pipeline => {
+                :inputs => plugin_stats(stats, :inputs),
+                :filters => plugin_stats(stats, :filters),
+                :outputs => plugin_stats(stats, :outputs)
+              }
+            }
+          end
+        end # module PluginsStats
       end
     end
   end

data/lib/logstash/api/modules/node_stats.rb

@@ -50,6 +50,10 @@ module LogStash
           respond_with :mem => mem_payload
         end
 
+        get "/pipeline" do
+          respond_with :pipeline => pipeline_payload
+        end
+
         private
 
         def events_payload
@@ -67,6 +71,10 @@ module LogStash
         def mem_payload
           @stats.memory
         end
+
+        def pipeline_payload
+          @stats.pipeline
+        end
       end
     end
   end

data/lib/logstash/config/mixin.rb

@@ -430,7 +430,7 @@ module LogStash::Config::Mixin
       result = nil
 
       if validator.nil?
-        return true
+        return true, value
       elsif validator.is_a?(Array)
         value = [*value]
         if value.size > 1

data/lib/logstash/environment.rb

@@ -2,6 +2,7 @@
 require "logstash/errors"
 require "logstash/config/cpu_core_strategy"
 require "logstash/settings"
+require "socket"
 
 module LogStash
 

data/lib/logstash/filter_delegator.rb

@@ -25,6 +25,7 @@ module LogStash
       @filter.metric = metric
 
       @metric_events = namespaced_metric.namespace(:events)
+      namespaced_metric.gauge(:name, config_name)
 
       # Not all the filters will do bufferings
       define_flush_method if @filter.respond_to?(:flush)
@@ -37,7 +38,9 @@ module LogStash
     def multi_filter(events)
       @metric_events.increment(:in, events.size)
 
+      clock = @metric_events.time(:duration_in_millis)
       new_events = @filter.multi_filter(events)
+      clock.stop
 
       # There is no garantee in the context of filter
       # that EVENTS_INT == EVENTS_OUT, see the aggregates and

data/lib/logstash/instrument/metric.rb

@@ -46,7 +46,7 @@ module LogStash module Instrument
     end
 
     def report_time(namespace, key, duration)
-      collector.push(namespace, key, :mean, :increment, duration)
+      collector.push(namespace, key, :counter, :increment, duration)
     end
 
     # This method return a metric instance tied to a specific namespace
@@ -81,7 +81,7 @@ module LogStash module Instrument
     #
     # @see LogStash::Instrument::Metric#time
     class TimedExecution
-      MILLISECONDS = 1_000_000.0.freeze
+      MILLISECONDS = 1_000.0.freeze
 
       def initialize(metric, namespace, key)
         @metric = metric

data/lib/logstash/instrument/namespaced_metric.rb

@@ -45,7 +45,7 @@ module LogStash module Instrument
     end
 
     def namespace(name)
-      NamespacedMetric.new(metric, namespace_name.concat(Array(name)))
+      NamespacedMetric.new(metric, namespace_name + Array(name))
     end
 
     private

data/lib/logstash/output_delegator.rb

@@ -28,6 +28,7 @@ module LogStash class OutputDelegator
     # Scope the metrics to the plugin
     namespaced_metric = metric.namespace(output.plugin_unique_name.to_sym)
     @metric_events = namespaced_metric.namespace(:events)
+    namespaced_metric.gauge(:name, config_name)
 
     @events_received = Concurrent::AtomicFixnum.new(0)
   end
@@ -129,7 +130,9 @@ module LogStash class OutputDelegator
     @events_received.increment(events.length)
     @metric_events.increment(:in, events.length)
 
+    clock = @metric_events.time(:duration_in_millis)
     @threadsafe_worker.multi_receive(events)
+    clock.stop
     @metric_events.increment(:out, events.length)
   end
 
@@ -139,7 +142,9 @@ module LogStash class OutputDelegator
 
     worker = @worker_queue.pop
     begin
+      clock = @metric_events.time(:duration_in_millis)
       worker.multi_receive(events)
+      clock.stop
       @metric_events.increment(:out, events.length)
     ensure
       @worker_queue.push(worker)

data/lib/logstash/version.rb

@@ -11,4 +11,4 @@
 #       eventually this file should be in the root logstash lib fir and dependencies in logstash-core should be
 #       fixed.
 
-LOGSTASH_VERSION = "5.0.0-alpha4.snapshot2"
+LOGSTASH_VERSION = "5.0.0-alpha4.snapshot3"

data/lib/logstash/webserver.rb

@@ -43,6 +43,9 @@ module LogStash
       @server.add_tcp_listener(http_host, http_port)
 
       @server.run.join
+    rescue Errno::EADDRINUSE
+      message = "Logstash tried to bind to port #{@http_port}, but the port is already in use. You can specify a new port by launching logtash with the --http-port option."
+      raise Errno::EADDRINUSE.new(message)
     end
 
     def log(str)

data/logstash-core.gemspec

@@ -17,7 +17,7 @@ Gem::Specification.new do |gem|
   gem.require_paths = ["lib"]
   gem.version       = LOGSTASH_CORE_VERSION.gsub(/-/, '.')
 
-  gem.add_runtime_dependency "logstash-core-event-java", "5.0.0.alpha4.snapshot2"
+  gem.add_runtime_dependency "logstash-core-event-java", "5.0.0.alpha4.snapshot3"
 
   gem.add_runtime_dependency "cabin", "~> 0.8.0" #(Apache 2.0 license)
   gem.add_runtime_dependency "pry", "~> 0.10.1"  #(Ruby license)

data/spec/logstash/filter_delegator_spec.rb

@@ -67,15 +67,22 @@ describe LogStash::FilterDelegator do
     end
 
     context "when the filter buffer events" do
-      it "doesn't increment out" do
+      before do
+        allow(metric).to receive(:increment).with(anything, anything)
+      end
+
+      it "has incremented :in" do
         expect(metric).to receive(:increment).with(:in, events.size)
-        expect(metric).not_to receive(:increment)
+        subject.multi_filter(events)
+      end
 
+      it "has not incremented :out" do
+        expect(metric).not_to receive(:increment).with(:out, anything)
         subject.multi_filter(events)
       end
     end
 
-    context "when the fitler create more events" do
+    context "when the filter create more events" do
       let(:plugin_klass) do
         Class.new(LogStash::Filters::Base) do
           config_name "super_plugin"
@@ -91,6 +98,10 @@ describe LogStash::FilterDelegator do
         end
       end
 
+      before do
+        allow(metric).to receive(:increment).with(anything, anything)
+      end
+
       it "increments the in/out of the metric" do
         expect(metric).to receive(:increment).with(:in, events.size)
         expect(metric).to receive(:increment).with(:out, events.size * 2)
@@ -112,6 +123,10 @@ describe LogStash::FilterDelegator do
       end
     end
 
+    before do
+      allow(metric).to receive(:increment).with(anything, anything)
+    end
+
     it "doesnt define a flush method" do
       expect(subject.respond_to?(:flush)).to be_falsey
     end

data/spec/logstash/instrument/metric_spec.rb

@@ -67,15 +67,15 @@ describe LogStash::Instrument::Metric do
 
   context "#time" do
     let(:sleep_time) { 2 }
-    let(:sleep_time_ms) { sleep_time * 1_000_000 }
+    let(:sleep_time_ms) { sleep_time * 1_000 }
 
     it "records the duration" do
       subject.time(:root, :duration_ms) { sleep(sleep_time) }
 
-      expect(collector.last).to be_within(sleep_time_ms).of(sleep_time_ms + 5000)
+      expect(collector.last).to be_within(sleep_time_ms).of(sleep_time_ms + 5)
       expect(collector[0]).to match(:root)
       expect(collector[1]).to be(:duration_ms)
-      expect(collector[2]).to be(:mean)
+      expect(collector[2]).to be(:counter)
     end
 
     it "returns the value of the executed block" do
@@ -90,7 +90,7 @@ describe LogStash::Instrument::Metric do
       expect(collector.last).to be_within(sleep_time_ms).of(sleep_time_ms + 0.1)
       expect(collector[0]).to match(:root)
       expect(collector[1]).to be(:duration_ms)
-      expect(collector[2]).to be(:mean)
+      expect(collector[2]).to be(:counter)
     end
   end
 

data/spec/logstash/instrument/namespaced_metric_spec.rb

@@ -22,4 +22,11 @@ describe LogStash::Instrument::NamespacedMetric do
   it "returns the value of the block" do
     expect(subject.time(:duration_ms) { "hello" }).to eq("hello")
   end
+
+  it "its doesnt change the original `namespace` when creating a subnamespace" do
+    new_namespace = subject.namespace(:wally)
+
+    expect(subject.namespace_name).to eq([namespace])
+    expect(new_namespace.namespace_name).to eq([:stats, :wally])
+  end
 end

data/spec/logstash/output_delegator_spec.rb

@@ -19,6 +19,7 @@ describe LogStash::OutputDelegator do
       allow(out_klass).to receive(:threadsafe?).and_return(false)
       allow(out_klass).to receive(:workers_not_supported?).and_return(false)
       allow(out_klass).to receive(:name).and_return("example")
+      allow(out_klass).to receive(:config_name)
       allow(out_inst).to receive(:register)
       allow(out_inst).to receive(:multi_receive)
       allow(out_inst).to receive(:metric=).with(any_args)

metadata

@@ -1,21 +1,21 @@
 --- !ruby/object:Gem::Specification
 name: logstash-core
 version: !ruby/object:Gem::Version
-  version: 5.0.0.alpha4.snapshot2
+  version: 5.0.0.alpha4.snapshot3
 platform: java
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-06-15 00:00:00.000000000 Z
+date: 2016-06-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 5.0.0.alpha4.snapshot2
+        version: 5.0.0.alpha4.snapshot3
   name: logstash-core-event-java
   prerelease: false
   type: :runtime
@@ -23,7 +23,7 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 5.0.0.alpha4.snapshot2
+        version: 5.0.0.alpha4.snapshot3
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements: