logstash-core 5.0.0.alpha3.snapshot2-java → 5.0.0.alpha3.snapshot4-java

data/lib/logstash/settings.rb

@@ -0,0 +1,267 @@
+# encoding: utf-8
+
+module LogStash
+  class Settings
+
+    def initialize
+      @settings = {}
+    end
+
+    def register(setting)
+      if @settings.key?(setting.name)
+        raise ArgumentError.new("Setting \"#{setting.name}\" has already been registered as #{setting.inspect}")
+      else
+        @settings[setting.name] = setting
+      end
+    end
+
+    def get_setting(setting_name)
+      setting = @settings[setting_name]
+      raise ArgumentError.new("Setting \"#{setting_name}\" hasn't been registered") if setting.nil?
+      setting
+    end
+
+    def get_subset(setting_regexp)
+      regexp = setting_regexp.is_a?(Regexp) ? setting_regexp : Regexp.new(setting_regexp)
+      settings = self.class.new
+      @settings.each do |setting_name, setting|
+        next unless setting_name.match(regexp)
+        settings.register(setting.clone)
+      end
+      settings
+    end
+
+    def set?(setting_name)
+      get_setting(setting_name).set?
+    end
+
+    def clone
+      get_subset(".*")
+    end
+
+    def get_default(setting_name)
+      get_setting(setting_name).default
+    end
+
+    def get_value(setting_name)
+      get_setting(setting_name).value
+    end
+    alias_method :get, :get_value
+
+    def set_value(setting_name, value)
+      get_setting(setting_name).set(value)
+    end
+    alias_method :set, :set_value
+
+    def to_hash
+      hash = {}
+      @settings.each do |name, setting|
+        hash[name] = setting.value
+      end
+      hash
+    end
+
+    def merge(hash)
+      hash.each {|key, value| set_value(key, value) }
+      self
+    end
+
+    def format_settings
+      output = []
+      output << "-------- Logstash Settings (* means modified) ---------"
+      @settings.each do |setting_name, setting|
+        value = setting.value
+        default_value = setting.default
+        if default_value == value # print setting and its default value
+          output << "#{setting_name}: #{value.inspect}" unless value.nil?
+        elsif default_value.nil? # print setting and warn it has been set
+          output << "*#{setting_name}: #{value.inspect}"
+        elsif value.nil? # default setting not set by user
+          output << "#{setting_name}: #{default_value.inspect}"
+        else # print setting, warn it has been set, and show default value
+          output << "*#{setting_name}: #{value.inspect} (default: #{default_value.inspect})"
+        end
+      end
+      output << "--------------- Logstash Settings -------------------"
+      output
+    end
+
+    def reset
+      @settings.values.each(&:reset)
+    end
+
+    def from_yaml(yaml_path)
+      settings = read_yaml(::File.join(yaml_path, "logstash.yml"))
+      self.merge(flatten_hash(settings))
+    end
+
+    private
+    def read_yaml(path)
+      YAML.safe_load(IO.read(path)) || {}
+    end
+
+    def flatten_hash(h,f="",g={})
+      return g.update({ f => h }) unless h.is_a? Hash
+      if f.empty?
+        h.each { |k,r| flatten_hash(r,k,g) }
+      else
+        h.each { |k,r| flatten_hash(r,"#{f}.#{k}",g) }
+      end
+      g
+    end
+  end
+
+  class Setting
+    attr_reader :name, :default
+
+    def initialize(name, klass, default=nil, strict=true, &validator_proc)
+      @name = name
+      unless klass.is_a?(Class)
+        raise ArgumentError.new("Setting \"#{@name}\" must be initialized with a class (received #{klass})")
+      end
+      @klass = klass
+      @validator_proc = validator_proc
+      @value = nil
+      @value_is_set = false
+
+      validate(default) if strict
+      @default = default
+    end
+
+    def value
+      @value_is_set ? @value : default
+    end
+
+    def set?
+      @value_is_set
+    end
+
+    def set(value)
+      validate(value)
+      @value = value
+      @value_is_set = true
+      @value
+    end
+
+    def reset
+      @value = nil
+      @value_is_set = false
+    end
+
+    def to_hash
+      {
+        "name" => @name,
+        "klass" => @klass,
+        "value" => @value,
+        "value_is_set" => @value_is_set,
+        "default" => @default,
+        # Proc#== will only return true if it's the same obj
+        # so no there's no point in comparing it
+        # also thereś no use case atm to return the proc
+        # so let's not expose it
+        #"validator_proc" => @validator_proc
+      }
+    end
+
+    def ==(other)
+      self.to_hash == other.to_hash
+    end
+
+    private
+    def validate(value)
+      if !value.is_a?(@klass)
+        raise ArgumentError.new("Setting \"#{@name}\" must be a #{@klass}. Received: #{value} (#{value.class})")
+      elsif @validator_proc && !@validator_proc.call(value)
+        raise ArgumentError.new("Failed to validate setting \"#{@name}\" with value: #{value}")
+      end
+    end
+
+    ### Specific settings #####
+
+    class Boolean < Setting
+      def initialize(name, default, strict=true, &validator_proc)
+        @name = name
+        @klass = Object
+        @value = nil
+        @value_is_set = false
+        @validator_proc = validator_proc
+        coerced_default = coerce(default)
+        validate(coerced_default)
+        @default = coerced_default
+      end
+
+      def coerce(value)
+        case value
+        when TrueClass, "true"
+          true
+        when FalseClass, "false"
+          false
+        else
+          raise ArgumentError.new("could not coerce #{value} into a boolean")
+        end
+      end
+
+      def set(value)
+        coerced_value = coerce(value)
+        validate(coerced_value)
+        @value = coerce(coerced_value)
+        @value_is_set = true
+        @value
+      end
+    end
+
+    class String < Setting
+      def initialize(name, default=nil, strict=true)
+        super(name, ::String, default, strict)
+      end
+    end
+
+    class Numeric < Setting
+      def initialize(name, default=nil, strict=true)
+        super(name, ::Numeric, default, strict)
+      end
+    end
+
+    class Port < Setting
+      def initialize(name, default=nil, strict=true)
+        super(name, ::Numeric, default, strict) {|value| value >= 1 && value <= 65535 }
+      end
+    end
+
+    class Validator < Setting
+      def initialize(name, default=nil, strict=true, validator_class=nil)
+        @validator_class = validator_class
+        super(name, ::Object, default, strict)
+      end
+
+      def validate(value)
+        @validator_class.validate(value)
+      end
+    end
+
+    class String < Setting
+      def initialize(name, default=nil, strict=true, possible_strings=[])
+        super(name, ::String, default, strict)
+      end
+
+      def validate(value)
+        super(value) && possible_strings.include?(value)
+      end
+    end
+
+    class ExistingFilePath < Setting
+      def initialize(name, default=nil, strict=true)
+        super(name, ::String, default, strict) do |file_path|
+          if !::File.exists?(file_path)
+            raise ::ArgumentError.new("File \"#{file_path}\" must exist but was not found.")
+          else
+            true
+          end
+        end
+      end
+    end
+
+  end
+
+  SETTINGS = Settings.new
+end

data/lib/logstash/util/decorators.rb

@@ -7,7 +7,7 @@ module LogStash::Util
   # Decorators provides common manipulation on the event data.
   module Decorators
     extend self
-    
+
     @logger = Cabin::Channel.get(LogStash)
 
     # fields is a hash of field => value
@@ -22,11 +22,11 @@ module LogStash::Util
             # note below that the array field needs to be updated then reassigned to the event.
             # this is important because a construct like event[field] << v will not work
             # in the current Java event implementation. see https://github.com/elastic/logstash/issues/4140
-            a = Array(event[field])
+            a = Array(event.get(field))
             a << v
-            event[field] = a
+            event.set(field, a)
           else
-            event[field] = v
+            event.set(field, v)
           end
           @logger.debug? and @logger.debug("#{pluginname}: adding value to field", :field => field, :value => value)
         end
@@ -41,9 +41,9 @@ module LogStash::Util
         # note below that the tags array field needs to be updated then reassigned to the event.
         # this is important because a construct like event["tags"] << tag will not work
         # in the current Java event implementation. see https://github.com/elastic/logstash/issues/4140
-        tags = event["tags"] || []
+        tags = event.get("tags") || []
         tags << tag
-        event["tags"] = tags
+        event.set("tags", tags)
       end
     end
 

data/lib/logstash/util/java_version.rb

@@ -5,16 +5,7 @@ module LogStash::Util::JavaVersion
   def self.logger
     @logger ||= Cabin::Channel.get(LogStash)
   end
-
-  # Print a warning if we're on a bad version of java
-  def self.warn_on_bad_java_version
-    if self.bad_java_version?(self.version)
-      msg = "!!! Please upgrade your java version, the current version '#{self.version}' is not supported. We recommend a minimum version of Java 8"
-      STDERR.puts(msg)
-      logger.warn(msg)
-    end
-  end
-
+  
   # Return the current java version string. Returns nil if this is a non-java platform (e.g. MRI).
   def self.version
     return nil unless LogStash::Environment.jruby?

data/lib/logstash/util/worker_threads_default_printer.rb

@@ -6,8 +6,8 @@ require "logstash/util"
 module LogStash module Util class WorkerThreadsDefaultPrinter
 
   def initialize(settings)
-    @setting = settings.fetch(:pipeline_workers, 0)
-    @default = settings.fetch(:default_pipeline_workers, 0)
+    @setting = settings.fetch('pipeline.workers', 0)
+    @default = settings.fetch('default-pipeline-workers', 0)
   end
 
   def visit(collector)

data/lib/logstash/version.rb

@@ -11,4 +11,4 @@
 #       eventually this file should be in the root logstash lib fir and dependencies in logstash-core should be
 #       fixed.
 
-LOGSTASH_VERSION = "5.0.0-alpha3.snapshot2"
+LOGSTASH_VERSION = "5.0.0-alpha3.snapshot4"

data/locales/en.yml

@@ -71,12 +71,8 @@ en:
       non_reloadable_config_reload: >-
         Unable to reload configuration because it does not support dynamic reloading
       non_reloadable_config_register: |-
-        Logstash was not able to load configuration since it does not support
-        dynamic reloading and -r or --auto-reload flag was enabled
+        Logstash is not able to start since configuration auto reloading was enabled but the configuration contains plugins that don't support it. Quitting...
     web_api:
-      flag:
-        http_host: Web API binding host
-        http_port: Web API http port
       hot_threads:
         title: |-
           ::: {%{hostname}}
@@ -182,10 +178,8 @@ en:
           the empty string for the '-e' flag.
         configtest: |+
           Check configuration for valid syntax and then exit.
-        allow-env: |+
-          EXPERIMENTAL. Enables templating of environment variable
-          values. Instances of "${VAR}" in strings will be replaced
-          with the respective environment variable value named "VAR".
+        http_host: Web API binding host
+        http_port: Web API http port
         pipeline-workers: |+
           Sets the number of pipeline workers to run.
         pipeline-batch-size: |+
@@ -193,6 +187,9 @@ en:
         pipeline-batch-delay: |+
           When creating pipeline batches, how long to wait while polling
           for the next event.
+        path_settings: |+
+          Directory containing logstash.yml file. This can also be
+          set through the LS_SETTINGS_DIR environment variable.
         auto_reload: |+
           Monitor configuration changes and reload
           whenever it is changed.
@@ -221,15 +218,12 @@ en:
           'PATH/logstash/TYPE/NAME.rb' where TYPE is
           'inputs' 'filters', 'outputs' or 'codecs'
           and NAME is the name of the plugin.
-        quiet: |+
-          Quieter logstash logging. This causes only 
-          errors to be emitted.
-        verbose: |+
-          More verbose logging. This causes 'info' 
-          level logs to be emitted.
-        debug: |+
-          Most verbose logging. This causes 'debug'
-          level logs to be emitted.
+        log_level: |+
+          Set the log level for logstash. Possible values are:
+            - quiet => :error
+            - verbose => :info
+            - debug => :debug
+            - warn => :warn
         unsafe_shutdown: |+
           Force logstash to exit during shutdown even
           if there are still inflight events in memory.
@@ -243,7 +237,10 @@ en:
           it will default to the current hostname.
         agent: |+
           Specify an alternate agent plugin name.
-        debug_config: |+
+        config_debug: |+
           Print the compiled config ruby code out as a debug log (you must also have --debug enabled).
           WARNING: This will include any 'password' options passed to plugin configs as plaintext, and may result
-          in plaintext passwords appearing in your logs!
+          in plaintext passwords appearing in your logs!
+        log_format: |+
+          Specify if Logstash should write its own logs in JSON form (one
+          event per line) or in plain text (using Ruby's Object#inspect)

data/logstash-core.gemspec

@@ -17,7 +17,7 @@ Gem::Specification.new do |gem|
   gem.require_paths = ["lib"]
   gem.version       = LOGSTASH_CORE_VERSION.gsub(/-/, '.')
 
-  gem.add_runtime_dependency "logstash-core-event-java", "5.0.0-alpha3.snapshot2"
+  gem.add_runtime_dependency "logstash-core-event-java", "5.0.0.alpha3.snapshot4"
 
   gem.add_runtime_dependency "cabin", "~> 0.8.0" #(Apache 2.0 license)
   gem.add_runtime_dependency "pry", "~> 0.10.1"  #(Ruby license)

data/spec/api/spec_helper.rb

@@ -3,6 +3,7 @@ API_ROOT = File.expand_path(File.join(File.dirname(__FILE__), "..", "..", "lib",
 
 require "logstash/devutils/rspec/spec_helper"
 
+require "logstash/settings"
 require 'rack/test'
 require 'rspec'
 require "json"
@@ -37,26 +38,24 @@ class LogStashRunner
   attr_reader :config_str, :agent, :pipeline_settings
 
   def initialize
-    args = [
-      :logger => Cabin::Channel.get(LogStash),
-      :auto_reload => false,
-      :collect_metric => true,
-      :debug => false,
-      :node_name => "test_agent",
-      :web_api_http_port => rand(9600..9700)
-    ]
-
     @config_str   = "input { generator {count => 0} } output { }"
-    @agent = LogStash::DummyAgent.new(*args)
-    @pipeline_settings ||= { :pipeline_id => "main",
-                             :config_str => config_str,
-                            :pipeline_batch_size => 1,
-                            :flush_interval => 1,
-                            :pipeline_workers => 1 }
+    args = {
+      "config.reload.automatic" => false,
+      "metric.collect" => true,
+      "log.level" => "debug",
+      "node.name" => "test_agent",
+      "http.port" => rand(9600..9700),
+      "config.string" => @config_str,
+      "pipeline.batch.size" => 1,
+      "pipeline.workers" => 1
+    }
+    @settings = ::LogStash::SETTINGS.clone.merge(args)
+
+    @agent = LogStash::DummyAgent.new(@settings)
   end
 
   def start
-    agent.register_pipeline("main", pipeline_settings)
+    agent.register_pipeline("main", @settings)
     @runner = Thread.new(agent) do |_agent|
       _agent.execute
     end