logstash-core 5.0.0.alpha2.snapshot2-java → 5.0.0.alpha3-java

data/lib/logstash/settings.rb

@@ -0,0 +1,267 @@
+# encoding: utf-8
+
+module LogStash
+  class Settings
+
+    def initialize
+      @settings = {}
+    end
+
+    def register(setting)
+      if @settings.key?(setting.name)
+        raise ArgumentError.new("Setting \"#{setting.name}\" has already been registered as #{setting.inspect}")
+      else
+        @settings[setting.name] = setting
+      end
+    end
+
+    def get_setting(setting_name)
+      setting = @settings[setting_name]
+      raise ArgumentError.new("Setting \"#{setting_name}\" hasn't been registered") if setting.nil?
+      setting
+    end
+
+    def get_subset(setting_regexp)
+      regexp = setting_regexp.is_a?(Regexp) ? setting_regexp : Regexp.new(setting_regexp)
+      settings = self.class.new
+      @settings.each do |setting_name, setting|
+        next unless setting_name.match(regexp)
+        settings.register(setting.clone)
+      end
+      settings
+    end
+
+    def set?(setting_name)
+      get_setting(setting_name).set?
+    end
+
+    def clone
+      get_subset(".*")
+    end
+
+    def get_default(setting_name)
+      get_setting(setting_name).default
+    end
+
+    def get_value(setting_name)
+      get_setting(setting_name).value
+    end
+    alias_method :get, :get_value
+
+    def set_value(setting_name, value)
+      get_setting(setting_name).set(value)
+    end
+    alias_method :set, :set_value
+
+    def to_hash
+      hash = {}
+      @settings.each do |name, setting|
+        hash[name] = setting.value
+      end
+      hash
+    end
+
+    def merge(hash)
+      hash.each {|key, value| set_value(key, value) }
+      self
+    end
+
+    def format_settings
+      output = []
+      output << "-------- Logstash Settings (* means modified) ---------"
+      @settings.each do |setting_name, setting|
+        value = setting.value
+        default_value = setting.default
+        if default_value == value # print setting and its default value
+          output << "#{setting_name}: #{value.inspect}" unless value.nil?
+        elsif default_value.nil? # print setting and warn it has been set
+          output << "*#{setting_name}: #{value.inspect}"
+        elsif value.nil? # default setting not set by user
+          output << "#{setting_name}: #{default_value.inspect}"
+        else # print setting, warn it has been set, and show default value
+          output << "*#{setting_name}: #{value.inspect} (default: #{default_value.inspect})"
+        end
+      end
+      output << "--------------- Logstash Settings -------------------"
+      output
+    end
+
+    def reset
+      @settings.values.each(&:reset)
+    end
+
+    def from_yaml(yaml_path)
+      settings = read_yaml(::File.join(yaml_path, "logstash.yml"))
+      self.merge(flatten_hash(settings))
+    end
+
+    private
+    def read_yaml(path)
+      YAML.safe_load(IO.read(path)) || {}
+    end
+
+    def flatten_hash(h,f="",g={})
+      return g.update({ f => h }) unless h.is_a? Hash
+      if f.empty?
+        h.each { |k,r| flatten_hash(r,k,g) }
+      else
+        h.each { |k,r| flatten_hash(r,"#{f}.#{k}",g) }
+      end
+      g
+    end
+  end
+
+  class Setting
+    attr_reader :name, :default
+
+    def initialize(name, klass, default=nil, strict=true, &validator_proc)
+      @name = name
+      unless klass.is_a?(Class)
+        raise ArgumentError.new("Setting \"#{@name}\" must be initialized with a class (received #{klass})")
+      end
+      @klass = klass
+      @validator_proc = validator_proc
+      @value = nil
+      @value_is_set = false
+
+      validate(default) if strict
+      @default = default
+    end
+
+    def value
+      @value_is_set ? @value : default
+    end
+
+    def set?
+      @value_is_set
+    end
+
+    def set(value)
+      validate(value)
+      @value = value
+      @value_is_set = true
+      @value
+    end
+
+    def reset
+      @value = nil
+      @value_is_set = false
+    end
+
+    def to_hash
+      {
+        "name" => @name,
+        "klass" => @klass,
+        "value" => @value,
+        "value_is_set" => @value_is_set,
+        "default" => @default,
+        # Proc#== will only return true if it's the same obj
+        # so no there's no point in comparing it
+        # also thereś no use case atm to return the proc
+        # so let's not expose it
+        #"validator_proc" => @validator_proc
+      }
+    end
+
+    def ==(other)
+      self.to_hash == other.to_hash
+    end
+
+    private
+    def validate(value)
+      if !value.is_a?(@klass)
+        raise ArgumentError.new("Setting \"#{@name}\" must be a #{@klass}. Received: #{value} (#{value.class})")
+      elsif @validator_proc && !@validator_proc.call(value)
+        raise ArgumentError.new("Failed to validate setting \"#{@name}\" with value: #{value}")
+      end
+    end
+
+    ### Specific settings #####
+
+    class Boolean < Setting
+      def initialize(name, default, strict=true, &validator_proc)
+        @name = name
+        @klass = Object
+        @value = nil
+        @value_is_set = false
+        @validator_proc = validator_proc
+        coerced_default = coerce(default)
+        validate(coerced_default)
+        @default = coerced_default
+      end
+
+      def coerce(value)
+        case value
+        when TrueClass, "true"
+          true
+        when FalseClass, "false"
+          false
+        else
+          raise ArgumentError.new("could not coerce #{value} into a boolean")
+        end
+      end
+
+      def set(value)
+        coerced_value = coerce(value)
+        validate(coerced_value)
+        @value = coerce(coerced_value)
+        @value_is_set = true
+        @value
+      end
+    end
+
+    class String < Setting
+      def initialize(name, default=nil, strict=true)
+        super(name, ::String, default, strict)
+      end
+    end
+
+    class Numeric < Setting
+      def initialize(name, default=nil, strict=true)
+        super(name, ::Numeric, default, strict)
+      end
+    end
+
+    class Port < Setting
+      def initialize(name, default=nil, strict=true)
+        super(name, ::Numeric, default, strict) {|value| value >= 1 && value <= 65535 }
+      end
+    end
+
+    class Validator < Setting
+      def initialize(name, default=nil, strict=true, validator_class=nil)
+        @validator_class = validator_class
+        super(name, ::Object, default, strict)
+      end
+
+      def validate(value)
+        @validator_class.validate(value)
+      end
+    end
+
+    class String < Setting
+      def initialize(name, default=nil, strict=true, possible_strings=[])
+        super(name, ::String, default, strict)
+      end
+
+      def validate(value)
+        super(value) && possible_strings.include?(value)
+      end
+    end
+
+    class ExistingFilePath < Setting
+      def initialize(name, default=nil, strict=true)
+        super(name, ::String, default, strict) do |file_path|
+          if !::File.exists?(file_path)
+            raise ::ArgumentError.new("File \"#{file_path}\" must exist but was not found.")
+          else
+            true
+          end
+        end
+      end
+    end
+
+  end
+
+  SETTINGS = Settings.new
+end

data/lib/logstash/util/decorators.rb

@@ -7,7 +7,7 @@ module LogStash::Util
   # Decorators provides common manipulation on the event data.
   module Decorators
     extend self
-    
+
     @logger = Cabin::Channel.get(LogStash)
 
     # fields is a hash of field => value
@@ -22,11 +22,11 @@ module LogStash::Util
             # note below that the array field needs to be updated then reassigned to the event.
             # this is important because a construct like event[field] << v will not work
             # in the current Java event implementation. see https://github.com/elastic/logstash/issues/4140
-            a = Array(event[field])
+            a = Array(event.get(field))
             a << v
-            event[field] = a
+            event.set(field, a)
           else
-            event[field] = v
+            event.set(field, v)
           end
           @logger.debug? and @logger.debug("#{pluginname}: adding value to field", :field => field, :value => value)
         end
@@ -41,9 +41,9 @@ module LogStash::Util
         # note below that the tags array field needs to be updated then reassigned to the event.
         # this is important because a construct like event["tags"] << tag will not work
         # in the current Java event implementation. see https://github.com/elastic/logstash/issues/4140
-        tags = event["tags"] || []
+        tags = event.get("tags") || []
         tags << tag
-        event["tags"] = tags
+        event.set("tags", tags)
       end
     end
 

data/lib/logstash/util/java_version.rb

@@ -5,16 +5,7 @@ module LogStash::Util::JavaVersion
   def self.logger
     @logger ||= Cabin::Channel.get(LogStash)
   end
-
-  # Print a warning if we're on a bad version of java
-  def self.warn_on_bad_java_version
-    if self.bad_java_version?(self.version)
-      msg = "!!! Please upgrade your java version, the current version '#{self.version}' is not supported. We recommend a minimum version of Java 8"
-      STDERR.puts(msg)
-      logger.warn(msg)
-    end
-  end
-
+  
   # Return the current java version string. Returns nil if this is a non-java platform (e.g. MRI).
   def self.version
     return nil unless LogStash::Environment.jruby?

data/lib/logstash/util/thread_dump.rb

@@ -0,0 +1,55 @@
+# encoding: utf-8
+module LogStash
+  module Util
+    class ThreadDump
+      SKIPPED_THREADS             = [ "Finalizer", "Reference Handler", "Signal Dispatcher" ].freeze
+      THREADS_COUNT_DEFAULT       = 3.freeze
+      IGNORE_IDLE_THREADS_DEFAULT = true.freeze
+
+      attr_reader :top_count, :ignore, :dump
+
+      def initialize(options={})
+        @options   = options
+        @dump = options.fetch(:dump, JRMonitor.threads.generate({}))
+        @top_count = options.fetch(:threads, THREADS_COUNT_DEFAULT)
+        @ignore    = options.fetch(:ignore_idle_threads, IGNORE_IDLE_THREADS_DEFAULT)
+      end
+
+      def each(&block)
+        i=0
+        dump.each_pair do |thread_name, _hash|
+          break if i >= top_count
+          if ignore
+            next if idle_thread?(thread_name, _hash)
+          end
+          block.call(thread_name, _hash)
+          i += 1
+        end
+      end
+
+      def idle_thread?(thread_name, data)
+        idle = false
+        if SKIPPED_THREADS.include?(thread_name)
+          # these are likely JVM dependent
+          idle = true
+        elsif thread_name.match(/Ruby-\d+-JIT-\d+/)
+          # This are internal JRuby JIT threads, 
+          # see java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor for details.
+          idle = true
+        elsif thread_name.match(/pool-\d+-thread-\d+/)
+          # This are threads used by the internal JRuby implementation to dispatch
+          # calls and tasks, see prg.jruby.internal.runtime.methods.DynamicMethod.call
+          idle = true
+        else
+          data["thread.stacktrace"].each do |trace|
+            if trace.start_with?("java.util.concurrent.ThreadPoolExecutor.getTask")
+              idle = true
+              break
+            end
+          end
+        end
+        idle
+      end
+    end
+  end
+end

data/lib/logstash/util/worker_threads_default_printer.rb

@@ -6,8 +6,8 @@ require "logstash/util"
 module LogStash module Util class WorkerThreadsDefaultPrinter
 
   def initialize(settings)
-    @setting = settings.fetch(:pipeline_workers, 0)
-    @default = settings.fetch(:default_pipeline_workers, 0)
+    @setting = settings.fetch('pipeline.workers', 0)
+    @default = settings.fetch('default-pipeline-workers', 0)
   end
 
   def visit(collector)

data/lib/logstash/version.rb

@@ -11,4 +11,4 @@
 #       eventually this file should be in the root logstash lib fir and dependencies in logstash-core should be
 #       fixed.
 
-LOGSTASH_VERSION = "5.0.0-alpha2.snapshot2"
+LOGSTASH_VERSION = "5.0.0-alpha3"

data/lib/logstash/webserver.rb

@@ -1,16 +1,15 @@
 # encoding: utf-8
 require "puma"
-require "puma/single"
-require "puma/binder"
-require "puma/configuration"
-require "puma/commonlogger"
+require "puma/server"
+require "sinatra"
+require "rack"
+require "logstash/api/rack_app"
 
 module LogStash 
   class WebServer
-
     extend Forwardable
 
-    attr_reader :logger, :status, :config, :options, :cli_options, :runner, :binder, :events
+    attr_reader :logger, :status, :config, :options, :cli_options, :runner, :binder, :events, :http_host, :http_port
 
     def_delegator :@runner, :stats
 
@@ -19,8 +18,8 @@ module LogStash
 
     def initialize(logger, options={})
       @logger      = logger
-      http_host    = options[:http_host] || DEFAULT_HOST
-      http_port    = options[:http_port] || DEFAULT_PORT
+      @http_host    = options[:http_host] || DEFAULT_HOST
+      @http_port    = options[:http_port] || DEFAULT_PORT
       @options     = {}
       @cli_options = options.merge({ :rackup => ::File.join(::File.dirname(__FILE__), "api", "init.ru"),
                                      :binds => ["tcp://#{http_host}:#{http_port}"],
@@ -31,24 +30,17 @@ module LogStash
                                      :queue_requests => false
       })
       @status      = nil
-
-      parse_options
-
-      @runner  = nil
-      @events  = ::Puma::Events.strings
-      @binder  = ::Puma::Binder.new(@events)
-      @binder.import_from_env
-
-      set_environment
     end
 
     def run
       log "=== puma start: #{Time.now} ==="
 
-      @runner = Puma::Single.new(self)
-      @status = :run
-      @runner.run
-      stop(:graceful => true)
+      stop # Just in case
+
+      @server = ::Puma::Server.new(LogStash::Api::RackApp.app)
+      @server.add_tcp_listener(http_host, http_port)
+
+      @server.run.join
     end
 
     def log(str)
@@ -58,7 +50,7 @@ module LogStash
     def error(str)
       logger.error(str)
     end
-
+    
     # Empty method, this method is required because of the puma usage we make through
     # the Single interface, https://github.com/puma/puma/blob/master/lib/puma/single.rb#L82
     # for more details. This can always be implemented when we want to keep track of this
@@ -66,33 +58,7 @@ module LogStash
     def write_state; end
 
     def stop(options={})
-      graceful = options.fetch(:graceful, true)
-
-      if graceful
-        @runner.stop_blocked
-      else
-        @runner.stop
-      end rescue nil
-
-      @status = :stop
-      log "=== puma shutdown: #{Time.now} ==="
-    end
-
-    private
-
-    def env
-      @options[:debug] ? "development" : "production"
-    end
-
-    def set_environment
-      @options[:environment] = env
-      ENV['RACK_ENV']        = env
-    end
-
-    def parse_options
-      @config  = ::Puma::Configuration.new(cli_options)
-      @config.load
-      @options = @config.options
+      @server.stop(true) if @server
     end
   end
 end