logstash-core 6.0.0.alpha2-java → 6.0.0.beta1-java

data/lib/logstash/config/source/base.rb

@@ -1,8 +1,11 @@
 # encoding: utf-8
 module LogStash module Config module Source
   class Base
+    attr_reader :conflict_messages
+
     def initialize(settings)
       @settings = settings
+      @conflict_messages = []
     end
 
     def pipeline_configs
@@ -12,5 +15,77 @@ module LogStash module Config module Source
     def match?
       raise NotImplementedError, "`match?` must be implemented!"
     end
+
+    def config_conflict?
+      raise NotImplementedError, "`config_conflict?` must be implemented!"
+    end
+
+    def config_reload_automatic_setting
+      @settings.get_setting("config.reload.automatic")
+    end
+
+    def config_reload_automatic
+      config_reload_automatic_setting.value
+    end
+
+    def config_reload_automatic?
+      config_reload_automatic_setting.set?
+    end
+
+    def config_string_setting
+      @settings.get_setting("config.string")
+    end
+
+    def config_string
+      config_string_setting.value
+    end
+
+    def config_string?
+      !config_string.nil?
+    end
+
+    def config_path_setting
+      @settings.get_setting("path.config")
+    end
+
+    def config_path
+      config_path_setting.value
+    end
+
+    def config_path?
+      !(config_path.nil? || config_path.empty?)
+    end
+
+    def modules_cli_setting
+      @settings.get_setting("modules.cli")
+    end
+
+    def modules_cli
+      modules_cli_setting.value
+    end
+
+    def modules_cli?
+      !(modules_cli.nil? || modules_cli.empty?)
+    end
+
+    def modules_setting
+      @settings.get_setting("modules")
+    end
+
+    def modules
+      modules_setting.value
+    end
+
+    def modules?
+      !(modules.nil? || modules.empty?)
+    end
+
+    def both_module_configs?
+      modules_cli? && modules?
+    end
+
+    def modules_defined?
+      modules_cli? || modules?
+    end
   end
 end end end

data/lib/logstash/config/source/local.rb

@@ -18,8 +18,27 @@ module LogStash module Config module Source
   #
   class Local < Base
     class ConfigStringLoader
+      INPUT_BLOCK_RE = /input *{/
+      OUTPUT_BLOCK_RE = /output *{/
+      EMPTY_RE = /^\s*$/
+
       def self.read(config_string)
-        [org.logstash.common.SourceWithMetadata.new("string", "config_string", config_string)]
+        config_parts = [org.logstash.common.SourceWithMetadata.new("string", "config_string", 0, 0, config_string)]
+
+        # Make sure we have an input and at least 1 output
+        # if its not the case we will add stdin and stdout
+        # this is for backward compatibility reason
+        if !INPUT_BLOCK_RE.match(config_string)
+          config_parts << org.logstash.common.SourceWithMetadata.new(self.class.name, "default input", 0, 0, LogStash::Config::Defaults.input)
+
+        end
+
+        # include a default stdout output if no outputs given
+        if !OUTPUT_BLOCK_RE.match(config_string)
+          config_parts << org.logstash.common.SourceWithMetadata.new(self.class.name, "default output", 0, 0, LogStash::Config::Defaults.output)
+        end
+
+        config_parts
       end
     end
 
@@ -53,8 +72,8 @@ module LogStash module Config module Source
 
           config_string = ::File.read(file)
 
-          if valid_encoding?(config_string)
-            part = org.logstash.common.SourceWithMetadata.new("file", file, config_string)
+          if config_string.valid_encoding?
+            part = org.logstash.common.SourceWithMetadata.new("file", file, 0, 0, config_string)
             config_parts << part
           else
             encoding_issue_files << file
@@ -101,10 +120,6 @@ module LogStash module Config module Source
         all_files - get_matched_files
       end
 
-      def valid_encoding?(content)
-        content.ascii_only? && content.valid_encoding?
-      end
-
       def temporary_file?(filepath)
         filepath.match(TEMPORARY_FILE_RE)
       end
@@ -121,7 +136,7 @@ module LogStash module Config module Source
           # since we have fetching config we wont follow any redirection.
           case response.code.to_i
           when 200
-            [org.logstash.common.SourceWithMetadata.new(uri.scheme, uri.to_s, response.body)]
+            [org.logstash.common.SourceWithMetadata.new(uri.scheme, uri.to_s, 0, 0, response.body)]
           when 302
             raise LogStash::ConfigLoadingError, I18n.t("logstash.runner.configuration.fetch-failed", :path => uri.to_s, :message => "We don't follow redirection for remote configuration")
           when 404
@@ -139,17 +154,37 @@ module LogStash module Config module Source
 
     PIPELINE_ID = LogStash::SETTINGS.get("pipeline.id").to_sym
     HTTP_RE = /^http(s)?/
-    INPUT_BLOCK_RE = /input *{/
-    OUTPUT_BLOCK_RE = /output *{/
 
     def pipeline_configs
+      if config_conflict?
+        raise ConfigurationError, @conflict_messages.join(", ")
+      end
+      local_pipeline_configs
+    end
+
+    def match?
+      # see basic settings predicates and getters defined in the base class
+      (config_string? || config_path?) && !(modules_cli? || modules?) && !automatic_reload_with_config_string?
+    end
 
-      if config_path? && config_string?
-        raise ConfigurationError.new("Settings 'config.string' and 'path.config' can't be used simultaneously.")
-      elsif !config_path? && !config_string?
-        raise ConfigurationError.new("Either 'config.string' or 'path.config' must be set.")
+    def config_conflict?
+      @conflict_messages.clear
+
+      # Check if configuration auto-reload is used that -f is specified
+      if automatic_reload_with_config_string?
+        @conflict_messages << I18n.t("logstash.runner.reload-with-config-string")
+      end
+      # Check if both -f and -e are present
+      if config_string? && config_path?
+        @conflict_messages << I18n.t("logstash.runner.config-string-path-exclusive")
       end
 
+      @conflict_messages.any?
+    end
+
+    private
+
+    def local_pipeline_configs
       config_parts = if config_string?
         ConfigStringLoader.read(config_string)
       elsif local_config?
@@ -160,46 +195,13 @@ module LogStash module Config module Source
         []
       end
 
-      return if config_parts.empty?
-
-      add_missing_default_inputs_or_outputs(config_parts) if config_string?
+      return [] if config_parts.empty?
 
       [PipelineConfig.new(self.class, @settings.get("pipeline.id").to_sym, config_parts, @settings)]
     end
 
-    def match?
-      config_string? || config_path?
-    end
-
-    private
-    # Make sure we have an input and at least 1 output
-    # if its not the case we will add stdin and stdout
-    # this is for backward compatibility reason
-    def add_missing_default_inputs_or_outputs(config_parts)
-      if !config_parts.any? { |part| INPUT_BLOCK_RE.match(part.text) }
-        config_parts << org.logstash.common.SourceWithMetadata.new(self.class.name, "default input", LogStash::Config::Defaults.input)
-      end
-
-      # include a default stdout output if no outputs given
-      if !config_parts.any? { |part| OUTPUT_BLOCK_RE.match(part.text) }
-        config_parts << org.logstash.common.SourceWithMetadata.new(self.class.name, "default output", LogStash::Config::Defaults.output)
-      end
-    end
-
-    def config_string
-      @settings.get("config.string")
-    end
-
-    def config_string?
-      !config_string.nil?
-    end
-
-    def config_path
-      @settings.get("path.config")
-    end
-
-    def config_path?
-      !config_path.nil? && !config_path.empty?
+    def automatic_reload_with_config_string?
+      config_reload_automatic? && !config_path? && config_string?
     end
 
     def local_config?

data/lib/logstash/config/source/modules.rb

@@ -0,0 +1,55 @@
+# encoding: utf-8
+require "logstash/config/source/base"
+require "logstash/config/modules_common"
+require "logstash/config/pipeline_config"
+require "logstash/util/loggable"
+require "logstash/errors"
+
+module LogStash module Config module Source
+  class Modules < Base
+    include LogStash::Util::Loggable
+    def pipeline_configs
+      if config_conflict? # double check
+        raise ConfigurationError, @conflict_messages.join(", ")
+      end
+
+      pipelines = LogStash::Config::ModulesCommon.pipeline_configs(@settings)
+      pipelines.map do |hash|
+        PipelineConfig.new(self, hash["pipeline_id"].to_sym,
+          org.logstash.common.SourceWithMetadata.new("module", hash["alt_name"], 0, 0, hash["config_string"]),
+          hash["settings"])
+      end
+    end
+
+    def match?
+      # see basic settings predicates and getters defined in the base class
+      (modules_cli? || modules?) && !(config_string? || config_path?) && !automatic_reload_with_modules?
+    end
+
+    def config_conflict?
+      @conflict_messages.clear
+      # Make note that if modules are configured in both cli and logstash.yml that cli module
+      # settings will overwrite the logstash.yml modules settings
+      if modules_cli? && modules?
+        logger.info(I18n.t("logstash.runner.cli-module-override"))
+      end
+
+      if automatic_reload_with_modules?
+        @conflict_messages << I18n.t("logstash.runner.reload-with-modules")
+      end
+
+      # Check if config (-f or -e) and modules are configured
+      if (modules_cli? || modules?) && (config_string? || config_path?)
+        @conflict_messages << I18n.t("logstash.runner.config-module-exclusive")
+      end
+
+      @conflict_messages.any?
+    end
+
+    private
+
+    def automatic_reload_with_modules?
+      (modules_cli? || modules?) && config_reload_automatic?
+    end
+  end
+end end end

data/lib/logstash/config/source/multi_local.rb

@@ -10,6 +10,7 @@ module LogStash module Config module Source
     def initialize(settings)
       @original_settings = settings
       super(settings)
+      @match_warning_done = false
     end
 
     def pipeline_configs
@@ -18,28 +19,48 @@ module LogStash module Config module Source
         ::LogStash::PipelineSettings.from_settings(@original_settings.clone).merge(pipeline_settings)
       end
       detect_duplicate_pipelines(pipelines_settings)
-      pipelines_settings.map do |pipeline_settings|
+      pipeline_configs = pipelines_settings.map do |pipeline_settings|
         @settings = pipeline_settings
         # this relies on instance variable @settings and the parent class' pipeline_configs
         # method. The alternative is to refactor most of the Local source methods to accept
         # a settings object instead of relying on @settings.
-        super # create a PipelineConfig object based on @settings
+        local_pipeline_configs # create a PipelineConfig object based on @settings
       end.flatten
+      @settings = @original_settings
+      pipeline_configs
     end
 
     def match?
-      uses_config_string = @original_settings.get_setting("config.string").set?
-      uses_path_config = @original_settings.get_setting("path.config").set?
-      return true if !uses_config_string && !uses_path_config
-      if uses_path_config
-        logger.warn("Ignoring the 'pipelines.yml' file because 'path.config' (-f) is being used.")
-      elsif uses_config_string
-        logger.warn("Ignoring the 'pipelines.yml' file because 'config.string' (-e) is being used.")
+      detect_pipelines if !@detect_pipelines_called
+      # see basic settings predicates and getters defined in the base class
+      return !(invalid_pipelines_detected? || modules_cli? || modules? || config_string? || config_path?)
+    end
+
+    def invalid_pipelines_detected?
+      !@detected_marker || @detected_marker.is_a?(Class)
+    end
+
+    def config_conflict?
+      detect_pipelines if !@detect_pipelines_called
+      @conflict_messages.clear
+      # are there any auto-reload conflicts?
+      if !(modules_cli? || modules? || config_string? || config_path?)
+        if @detected_marker.nil?
+          @conflict_messages << I18n.t("logstash.runner.config-pipelines-failed-read", :path => pipelines_yaml_location)
+        elsif @detected_marker == false
+          @conflict_messages << I18n.t("logstash.runner.config-pipelines-empty", :path => pipelines_yaml_location)
+        elsif @detected_marker.is_a?(Class)
+          @conflict_messages << I18n.t("logstash.runner.config-pipelines-invalid", :invalid_class => @detected_marker, :path => pipelines_yaml_location)
+        end
+      else
+        do_warning? && logger.warn("Ignoring the 'pipelines.yml' file because modules or command line options are specified")
       end
-      false
+      @conflict_messages.any?
     end
 
     def retrieve_yaml_pipelines
+      # by now, either the config_conflict? or the match? should have ruled out any config problems
+      # but we don't rely on this, we can still get IO errors or
       result = read_pipelines_from_yaml(pipelines_yaml_location)
       case result
       when Array
@@ -68,5 +89,28 @@ module LogStash module Config module Source
         raise ConfigurationError.new("Pipelines YAML file contains duplicate pipeline ids: #{duplicate_ids.inspect}. Location: #{pipelines_yaml_location}")
       end
     end
+
+    def detect_pipelines
+      result = read_pipelines_from_yaml(pipelines_yaml_location) rescue nil
+      if result.is_a?(Array)
+        @detected_marker = true
+      elsif result.nil?
+        @detected_marker = nil
+      elsif !result
+        @detected_marker = false
+      else
+        @detected_marker = result.class
+      end
+      @detect_pipelines_called = true
+    end
+
+    private
+
+    def do_warning?
+      if !(done = true && @match_warning_done)
+        @match_warning_done = true
+      end
+      !done
+    end
   end
 end end end

data/lib/logstash/config/source_loader.rb

@@ -1,5 +1,6 @@
 # encoding: utf-8
 require "logstash/config/source/local"
+require "logstash/config/source/modules"
 require "logstash/config/source/multi_local"
 require "logstash/errors"
 require "thread"

data/lib/logstash/config/string_escape.rb

@@ -0,0 +1,27 @@
+
+
+module LogStash; module Config; module StringEscape
+  class << self
+    def process_escapes(input)
+      input.gsub(/\\./) do |value|
+        process(value)
+      end
+    end
+
+    private
+    def process(value)
+      case value[1]
+      when '"', "'", "\\"
+        value[1]
+      when "n"
+        "\n"
+      when "r"
+        "\r"
+      when "t"
+        "\t"
+      else
+        value
+      end
+    end
+  end
+end end end

data/lib/logstash/elasticsearch_client.rb

@@ -0,0 +1,142 @@
+# encoding: utf-8
+require "logstash/namespace"
+require "logstash/logging"
+require "elasticsearch"
+require "elasticsearch/transport/transport/http/manticore"
+
+module LogStash class ElasticsearchClient
+  include LogStash::Util::Loggable
+
+  class Response
+    # duplicated here from Elasticsearch::Transport::Transport::Response
+    # to create a normalised response across different client IMPL
+    attr_reader :status, :body, :headers
+    def initialize(status, body, headers={})
+      @status, @body, @headers = status, body, headers
+      @body = body.force_encoding('UTF-8') if body.respond_to?(:force_encoding)
+    end
+  end
+
+  def self.build(settings)
+    new(RubyClient.new(settings, logger))
+  end
+
+  class RubyClient
+    def initialize(settings, logger)
+      @settings = settings
+      @logger = logger
+      @client_args = client_args
+
+      ssl_options = {}
+
+      if @settings["var.elasticsearch.ssl.enabled"] == "true"
+        ssl_options[:verify] = @settings.fetch("var.elasticsearch.ssl.verification_mode", true)
+        ssl_options[:ca_file] = @settings.fetch("var.elasticsearch.ssl.certificate_authority", nil)
+        ssl_options[:client_cert] = @settings.fetch("var.elasticsearch.ssl.certificate", nil)
+        ssl_options[:client_key] = @settings.fetch("var.elasticsearch.ssl.key", nil)
+      end
+
+      @client_args[:ssl] = ssl_options
+
+      username = @settings["var.elasticsearch.username"]
+      password = @settings["var.elasticsearch.password"]
+      if username
+        @client_args[:transport_options] = { :headers => { "Authorization" => 'Basic ' + Base64.encode64( "#{username}:#{password}" ).chomp } }
+      end
+
+      @client = Elasticsearch::Client.new(@client_args)
+    end
+
+    def can_connect?
+      begin
+        head(SecureRandom.hex(32).prepend('_'))
+      rescue Elasticsearch::Transport::Transport::Errors::BadRequest
+        true
+      rescue Manticore::SocketException
+        false
+      end
+    end
+
+    def host_settings
+      @client_args[:hosts]
+    end
+
+    def delete(path)
+      begin
+        normalize_response(@client.perform_request('DELETE', path, {}, nil))
+      rescue Exception => e
+        if e.class.to_s =~ /NotFound/ || e.message =~ /Not\s*Found|404/i
+          Response.new(404, "", {})
+        else
+          raise e
+        end
+      end
+    end
+
+    def put(path, content)
+      normalize_response(@client.perform_request('PUT', path, {}, content))
+    end
+
+    def head(path)
+      begin
+        normalize_response(@client.perform_request('HEAD', path, {}, nil))
+      rescue Exception => e
+        if is_404_error?(e)
+          Response.new(404, "", {})
+        else
+          raise e
+        end
+      end
+    end
+
+    private
+
+    def is_404_error?(error)
+      error.class.to_s =~ /NotFound/ || error.message =~ /Not\s*Found|404/i
+    end
+
+    def normalize_response(response)
+      Response.new(response.status, response.body, response.headers)
+    end
+
+    def client_args
+      {
+        :transport_class => Elasticsearch::Transport::Transport::HTTP::Manticore,
+        :hosts => [*unpack_hosts],
+        # :logger => @logger, # silence the client logging
+      }
+    end
+
+    def unpack_hosts
+      setting = @settings.fetch("var.elasticsearch.hosts", "localhost:9200")
+      if setting.is_a?(String)
+        return setting.split(',').map(&:strip)
+      end
+      setting
+    end
+  end
+
+  def initialize(client)
+    @client = client
+  end
+
+  def delete(path)
+    @client.delete(path)
+  end
+
+  def put(path, content)
+    @client.put(path, content)
+  end
+
+  def head(path)
+    @client.head(path)
+  end
+
+  def can_connect?
+    @client.can_connect?
+  end
+
+  def host_settings
+    @client.host_settings
+  end
+end end