logstash-core 5.0.0.alpha5.snapshot1-java → 5.0.0.alpha6.snapshot1-java

data/lib/logstash/settings.rb

@@ -176,18 +176,43 @@ module LogStash
       end
     end
 
-    ### Specific settings #####
-
-    class Boolean < Setting
-      def initialize(name, default, strict=true, &validator_proc)
+    class Coercible < Setting
+      def initialize(name, klass, default=nil, strict=true, &validator_proc)
         @name = name
-        @klass = Object
+        unless klass.is_a?(Class)
+          raise ArgumentError.new("Setting \"#{@name}\" must be initialized with a class (received #{klass})")
+        end
+        @klass = klass
+        @validator_proc = validator_proc
         @value = nil
         @value_is_set = false
-        @validator_proc = validator_proc
-        coerced_default = coerce(default)
-        validate(coerced_default)
-        @default = coerced_default
+
+        if strict
+          coerced_default = coerce(default)
+          validate(coerced_default)
+          @default = coerced_default
+        else
+          @default = default
+        end
+      end
+
+      def set(value)
+        coerced_value = coerce(value)
+        validate(coerced_value)
+        @value = coerce(coerced_value)
+        @value_is_set = true
+        @value
+      end
+
+      def coerce(value)
+        raise NotImplementedError.new("Please implement #coerce for #{self.class}")
+      end
+    end
+    ### Specific settings #####
+
+    class Boolean < Coercible
+      def initialize(name, default, strict=true, &validator_proc)
+        super(name, Object, default, strict, &validator_proc)
       end
 
       def coerce(value)
@@ -200,25 +225,104 @@ module LogStash
           raise ArgumentError.new("could not coerce #{value} into a boolean")
         end
       end
+    end
 
-      def set(value)
-        coerced_value = coerce(value)
-        validate(coerced_value)
-        @value = coerce(coerced_value)
-        @value_is_set = true
-        @value
+    class Numeric < Coercible
+      def initialize(name, default=nil, strict=true)
+        super(name, ::Numeric, default, strict)
+      end
+
+      def coerce(v)
+        return v if v.is_a?(::Numeric)
+
+        # I hate these "exceptions as control flow" idioms
+        # but Ruby's `"a".to_i => 0` makes it hard to do anything else.
+        coerced_value = (Integer(v) rescue nil) || (Float(v) rescue nil)
+
+        if coerced_value.nil?
+          raise ArgumentError.new("Failed to coerce value to Numeric. Received #{v} (#{v.class})")
+        else
+          coerced_value
+        end
       end
     end
 
-    class Numeric < Setting
+    class Integer < Coercible
       def initialize(name, default=nil, strict=true)
-        super(name, ::Numeric, default, strict)
+        super(name, ::Integer, default, strict)
+      end
+
+      def coerce(value)
+        return value unless value.is_a?(::String)
+
+        coerced_value = Integer(value) rescue nil
+
+        if coerced_value.nil?
+          raise ArgumentError.new("Failed to coerce value to Integer. Received #{value} (#{value.class})")
+        else
+          coerced_value
+        end
+      end
+    end
+
+    class PositiveInteger < Integer
+      def initialize(name, default=nil, strict=true)
+        super(name, default, strict) do |v|
+          if v > 0
+            true
+          else
+            raise ArgumentError.new("Number must be bigger than 0. Received: #{v}")
+          end
+        end
       end
     end
 
-    class Port < Setting
+    class Port < Integer
+      VALID_PORT_RANGE = 1..65535
+
       def initialize(name, default=nil, strict=true)
-        super(name, ::Numeric, default, strict) {|value| value >= 1 && value <= 65535 }
+        super(name, default, strict) { |value| valid?(value) }
+      end
+
+      def valid?(port)
+        VALID_PORT_RANGE.cover?(port)
+      end
+    end
+
+    class PortRange < Coercible
+      PORT_SEPARATOR = "-"
+
+      def initialize(name, default=nil, strict=true)
+        super(name, ::Range, default, strict=true) { |value| valid?(value) }
+      end
+
+      def valid?(range)
+        Port::VALID_PORT_RANGE.first <= range.first && Port::VALID_PORT_RANGE.last >= range.last
+      end
+
+      def coerce(value)
+        case value
+        when ::Range
+          value
+        when ::Fixnum
+          value..value
+        when ::String
+          first, last = value.split(PORT_SEPARATOR)
+          last = first if last.nil?
+          begin
+            (Integer(first))..(Integer(last))
+          rescue ArgumentError # Trap and reraise a more human error
+            raise ArgumentError.new("Could not coerce #{value} into a port range")
+          end
+        else
+          raise ArgumentError.new("Could not coerce #{value} into a port range")
+        end
+      end
+
+      def validate(value)
+        unless valid?(value)
+          raise ArgumentError.new("Invalid value \"#{value}, valid options are within the range of #{Port::VALID_PORT_RANGE.first}-#{Port::VALID_PORT_RANGE.last}")
+        end
       end
     end
 
@@ -242,7 +346,7 @@ module LogStash
       def validate(value)
         super(value)
         unless @possible_strings.empty? || @possible_strings.include?(value)
-          raise ArgumentError.new("invalid value \"#{value}\". Options are: #{@possible_strings.inspect}")
+          raise ArgumentError.new("Invalid value \"#{value}\". Options are: #{@possible_strings.inspect}")
         end
       end
     end
@@ -270,7 +374,6 @@ module LogStash
         end
       end
     end
-
   end
 
   SETTINGS = Settings.new

data/lib/logstash/util/wrapped_synchronous_queue.rb

@@ -57,6 +57,9 @@ module LogStash; module Util
         # Note that @infilght_batches as a central mechanism for tracking inflight
         # batches will fail if we have multiple read clients in the pipeline.
         @inflight_batches = {}
+
+        # allow the worker thread to report the execution time of the filter + output
+        @inflight_clocks = {}
         @batch_size = batch_size
         @wait_for = wait_for
       end
@@ -89,6 +92,7 @@ module LogStash; module Util
           batch = ReadBatch.new(@queue, @batch_size, @wait_for)
           add_starting_metrics(batch)
           set_current_thread_inflight_batch(batch)
+          start_clock
           batch
         end
       end
@@ -100,11 +104,23 @@ module LogStash; module Util
       def close_batch(batch)
         @mutex.synchronize do
           @inflight_batches.delete(Thread.current)
+          stop_clock
         end
       end
 
+      def start_clock
+        @inflight_clocks[Thread.current] = [
+          @event_metric.time(:duration_in_millis),
+          @pipeline_metric.time(:duration_in_millis)
+        ]
+      end
+
+      def stop_clock
+        @inflight_clocks[Thread.current].each(&:stop)
+        @inflight_clocks.delete(Thread.current)
+      end
+
       def add_starting_metrics(batch)
-        return if @event_metric.nil? || @pipeline_metric.nil?
         @event_metric.increment(:in, batch.starting_size)
         @pipeline_metric.increment(:in, batch.starting_size)
       end

data/lib/logstash/version.rb

@@ -11,4 +11,4 @@
 #       eventually this file should be in the root logstash lib fir and dependencies in logstash-core should be
 #       fixed.
 
-LOGSTASH_VERSION = "5.0.0.alpha5.snapshot1"
+LOGSTASH_VERSION = "5.0.0.alpha6.snapshot1"

data/lib/logstash/webserver.rb

@@ -1,73 +1,84 @@
 # encoding: utf-8
+require "logstash/api/rack_app"
 require "puma"
 require "puma/server"
-require "logstash/api/rack_app"
+require "concurrent"
 
-module LogStash 
+module LogStash
   class WebServer
     extend Forwardable
 
-    attr_reader :logger, :status, :config, :options, :cli_options, :runner, :binder, :events, :http_host, :http_port, :http_environment, :agent
+    attr_reader :logger, :status, :config, :options, :runner, :binder, :events, :http_host, :http_ports, :http_environment, :agent
 
     def_delegator :@runner, :stats
 
     DEFAULT_HOST = "127.0.0.1".freeze
-    DEFAULT_PORT = 9600.freeze
+    DEFAULT_PORTS = (9600..9700).freeze
     DEFAULT_ENVIRONMENT = 'production'.freeze
 
     def initialize(logger, agent, options={})
       @logger = logger
       @agent = agent
       @http_host = options[:http_host] || DEFAULT_HOST
-      @http_port = options[:http_port] || DEFAULT_PORT
+      @http_ports = options[:http_ports] || DEFAULT_PORTS
       @http_environment = options[:http_environment] || DEFAULT_ENVIRONMENT
       @options = {}
-      @cli_options = options.merge({ :rackup => ::File.join(::File.dirname(__FILE__), "api", "init.ru"),
-                                     :binds => ["tcp://#{http_host}:#{http_port}"],
-                                     :debug => logger.debug?,
-                                     # Prevent puma from queueing request when not able to properly handling them,
-                                     # fixed https://github.com/elastic/logstash/issues/4674. See
-                                     # https://github.com/puma/puma/pull/640 for mode internal details in PUMA.
-                                     :queue_requests => false
-      })
-      @status      = nil
+      @status = nil
+      @running = Concurrent::AtomicBoolean.new(false)
     end
 
     def run
-      log "=== puma start: #{Time.now} ==="
+      logger.debug("Starting puma")
 
       stop # Just in case
 
-      app = LogStash::Api::RackApp.app(logger, agent, http_environment)
-      @server = ::Puma::Server.new(app)
-      @server.add_tcp_listener(http_host, http_port)
+      running!
 
-      @server.run.join
-    rescue Errno::EADDRINUSE
-      message = "Logstash tried to bind to port #{@http_port}, but the port is already in use. You can specify a new port by launching logtash with the --http-port option."
-      raise Errno::EADDRINUSE.new(message)
+      http_ports.each_with_index do |port, idx|
+        begin
+          if running?
+            @port = port
+            logger.debug("Trying to start WebServer", :port => @port)
+            start_webserver(@port)
+          else
+            break # we are closing down the server so just get out of the loop
+          end
+        rescue Errno::EADDRINUSE
+          if http_ports.count == 1
+            raise Errno::EADDRINUSE.new(I18n.t("logstash.web_api.cant_bind_to_port", :port => http_ports.first))
+          elsif idx == http_ports.count-1
+            raise Errno::EADDRINUSE.new(I18n.t("logstash.web_api.cant_bind_to_port_in_range", :http_ports => http_ports))
+          end
+        end
+      end
     end
 
-    def log(str)
-      logger.debug(str)
+    def running!
+      @running.make_true
     end
 
-    def error(str)
-      logger.error(str)
+    def running?
+      @running.value
     end
 
     def address
-      "#{http_host}:#{http_port}"
+      "#{http_host}:#{@port}"
     end
-    
-    # Empty method, this method is required because of the puma usage we make through
-    # the Single interface, https://github.com/puma/puma/blob/master/lib/puma/single.rb#L82
-    # for more details. This can always be implemented when we want to keep track of this
-    # bit of data.
-    def write_state; end
 
     def stop(options={})
+      @running.make_false
       @server.stop(true) if @server
     end
+
+    def start_webserver(port)
+      app = LogStash::Api::RackApp.app(logger, agent, http_environment)
+
+      @server = ::Puma::Server.new(app)
+      @server.add_tcp_listener(http_host, port)
+
+      logger.info("Succesfully started Logstash API", :port => @port)
+
+      @server.run.join
+    end
   end
 end

data/locales/en.yml

@@ -73,12 +73,16 @@ en:
       non_reloadable_config_register: |-
         Logstash is not able to start since configuration auto reloading was enabled but the configuration contains plugins that don't support it. Quitting...
     web_api:
+      cant_bind_to_port: |-
+        Logstash tried to bind to port %{port}, but the port is already in use. You can specify a new port by launching logtash with the --http-port option."
+      cant_bind_to_port_in_range: |-
+        Logstash tried to bind to port range %{http_ports}, but all the ports are already in use. You can specify a new port by launching logtash with the --http-port option."
       hot_threads:
         title: |-
           ::: {%{hostname}}
           Hot threads at %{time}, busiestThreads=%{top_count}:
         thread_title: |-
-            %{percent_of_cpu_time} % of cpu usage by %{thread_state} thread named '%{thread_name}'
+          %{percent_of_cpu_time} % of cpu usage, state: %{thread_state}, thread name: '%{thread_name}'
     runner:
       short-help: |-
         usage:

data/logstash-core.gemspec

@@ -17,7 +17,7 @@ Gem::Specification.new do |gem|
   gem.require_paths = ["lib"]
   gem.version       = LOGSTASH_CORE_VERSION
 
-  gem.add_runtime_dependency "logstash-core-event-java", "5.0.0.alpha5.snapshot1"
+  gem.add_runtime_dependency "logstash-core-event-java", "5.0.0.alpha6.snapshot1"
 
   gem.add_runtime_dependency "cabin", "~> 0.8.0" #(Apache 2.0 license)
   gem.add_runtime_dependency "pry", "~> 0.10.1"  #(Ruby license)
@@ -46,7 +46,7 @@ Gem::Specification.new do |gem|
 
   if RUBY_PLATFORM == 'java'
     gem.platform = RUBY_PLATFORM
-    gem.add_runtime_dependency "jrjackson", "~> 0.3.7" #(Apache 2.0 license)
+    gem.add_runtime_dependency "jrjackson", "~> 0.4.0" #(Apache 2.0 license)
   else
     gem.add_runtime_dependency "oj" #(MIT-style license)
   end

data/spec/api/lib/api/node_spec.rb

@@ -39,30 +39,81 @@ describe LogStash::Api::Modules::Node do
     end
 
     context "when asking for human output" do
+      [
+        "/hot_threads?human",
+        "/hot_threads?human=true",
+        "/hot_threads?human=1",
+        "/hot_threads?human=t",
+      ].each do |path|
+
+        before(:all) do
+          do_request { get path }
+        end
+
+        let(:payload) { last_response.body }
+
+        it "should return a text/plain content type" do
+          expect(last_response.content_type).to eq("text/plain;charset=utf-8")
+        end
+
+        it "should return a plain text payload" do
+          expect{ JSON.parse(payload) }.to raise_error
+        end
+      end
+    end
 
+    context "When asking for human output and threads count" do
       before(:all) do
-        do_request { get "/hot_threads?human" }
+        # Make sure we have enough threads for this to work.
+        @threads = []
+        5.times { @threads << Thread.new { loop {} } }
+
+        do_request { get "/hot_threads?human=t&threads=2"}
+      end
+
+      after(:all) do
+        @threads.each { |t| t.kill } rescue nil
       end
 
       let(:payload) { last_response.body }
 
-      it "should return a text/plain content type" do
-        expect(last_response.content_type).to eq("text/plain;charset=utf-8")
+      it "should return information for <= # requested threads" do
+        expect(payload.scan(/thread name/).size).to eq(2)
       end
+    end
 
-      it "should return a plain text payload" do
-        expect{ JSON.parse(payload) }.to raise_error
+    context "when not asking for human output" do
+      [
+        "/hot_threads?human=false",
+        "/hot_threads?human=0",
+        "/hot_threads?human=f",
+      ].each do |path|
+        before(:all) do
+          do_request { get path }
+        end
+
+        it "should return a json payload content type" do
+          expect(last_response.content_type).to eq("application/json")
+        end
+
+        let(:payload) { last_response.body }
+
+        it "should return a json payload" do
+          expect{ JSON.parse(payload) }.not_to raise_error
+        end
       end
     end
 
     describe "Generic JSON testing" do
       extend ResourceDSLMethods
-      
+
       root_structure = {
         "pipeline" => {
           "workers" => Numeric,
           "batch_size" => Numeric,
-          "batch_delay" => Numeric
+          "batch_delay" => Numeric,
+          "config_reload_automatic" => Boolean,
+          "config_reload_interval" => Numeric
         },
         "os" => {
           "name" => String,
@@ -82,7 +133,8 @@ describe LogStash::Api::Modules::Node do
             "heap_max_in_bytes" => Numeric,
             "non_heap_init_in_bytes" => Numeric,
             "non_heap_max_in_bytes" => Numeric
-          }
+        },
+        "gc_collectors" => Array
         },
         "hot_threads"=> {
           "time" => String,
@@ -90,8 +142,8 @@ describe LogStash::Api::Modules::Node do
           "threads" => Array
         }
       }
-      
+
       test_api_and_resources(root_structure, :exclude_from_root => ["hot_threads"])
-    end   
+    end
   end
 end