logstash-core 5.6.16-java → 6.0.0.alpha1-java

data/lib/logstash/compiler/treetop_monkeypatches.rb

@@ -0,0 +1,92 @@
+class Treetop::Runtime::SyntaxNode
+  def get_meta(key)
+    @ast_metadata ||= {}
+    return @ast_metadata[key] if @ast_metadata[key] 
+    return self.parent.get_meta(key) if self.parent
+    nil
+  end
+  
+  def set_meta(key, value)
+    @ast_metadata ||= {}
+    @ast_metadata[key] = value
+  end
+  def compile
+    return "" if elements.nil?
+    return elements.collect(&:compile).reject(&:empty?).join("")
+  end
+
+  # Traverse the syntax tree recursively.
+  # The order should respect the order of the configuration file as it is read
+  # and written by humans (and the order in which it is parsed).
+  def recurse(e, depth=0, &block)
+    r = block.call(e, depth)
+    e.elements.each { |e| recurse(e, depth + 1, &block) } if r && e.elements
+    nil
+  end
+
+  def recursive_inject(results=[], &block)
+    if !elements.nil?
+      elements.each do |element|
+        if block.call(element)
+          results << element
+        else
+          element.recursive_inject(results, &block)
+        end
+      end
+    end
+    return results
+  end
+
+  # When Treetop parses the configuration file
+  # it will generate a tree, the generated tree will contain
+  # a few `Empty` nodes to represent the actual space/tab or newline in the file.
+  # Some of theses node will point to our concrete class.
+  # To fetch a specific types of object we need to follow each branch
+  # and ignore the empty nodes.
+  def recursive_select(*klasses)
+    return recursive_inject { |e| klasses.any? {|k| e.is_a?(k)} }
+  end  
+
+  def recursive_inject_parent(results=[], &block)
+    if !parent.nil?
+      if block.call(parent)
+        results << parent
+      else
+        parent.recursive_inject_parent(results, &block)
+      end
+    end
+    return results
+  end
+
+  def recursive_select_parent(results=[], klass)
+    return recursive_inject_parent(results) { |e| e.is_a?(klass) }
+  end
+  
+  # Monkeypatch Treetop::Runtime::SyntaxNode's inspect method to skip
+  # any Whitespace or SyntaxNodes with no children.
+  def _inspect(indent="")
+    em = extension_modules
+    interesting_methods = methods-[em.last ? em.last.methods : nil]-self.class.instance_methods
+    im = interesting_methods.size > 0 ? " (#{interesting_methods.join(",")})" : ""
+    tv = text_value
+    tv = "...#{tv[-20..-1]}" if tv.size > 20
+
+    indent +
+    self.class.to_s.sub(/.*:/,'') +
+      em.map{|m| "+"+m.to_s.sub(/.*:/,'')}*"" +
+      " offset=#{interval.first}" +
+      ", #{tv.inspect}" +
+      im +
+      (elements && elements.size > 0 ?
+        ":" +
+          (elements.select { |e| !e.is_a?(LogStash::Config::AST::Whitespace) && e.elements && e.elements.size > 0 }||[]).map{|e|
+      begin
+        "\n"+e.inspect(indent+"  ")
+      rescue  # Defend against inspect not taking a parameter
+        "\n"+indent+" "+e.inspect
+      end
+          }.join("") :
+        ""
+      )
+  end
+end

data/lib/logstash/config/config_ast.rb

@@ -1,78 +1,10 @@
 # encoding: utf-8
 require 'logstash/errors'
-require "logstash/config/string_escape"
 require "treetop"
 
-class Treetop::Runtime::SyntaxNode
-
-  def compile
-    return "" if elements.nil?
-    return elements.collect(&:compile).reject(&:empty?).join("")
-  end
-
-  def get_meta(key)
-    @ast_metadata ||= {}
-    return @ast_metadata[key] if @ast_metadata[key]
-    return self.parent.get_meta(key) if self.parent
-    nil
-  end
-
-  def set_meta(key, value)
-    @ast_metadata ||= {}
-    @ast_metadata[key] = value
-  end
-
-  # Traverse the syntax tree recursively.
-  # The order should respect the order of the configuration file as it is read
-  # and written by humans (and the order in which it is parsed).
-  def recurse(e, depth=0, &block)
-    r = block.call(e, depth)
-    e.elements.each { |e| recurse(e, depth + 1, &block) } if r && e.elements
-    nil
-  end
-
-  def recursive_inject(results=[], &block)
-    if !elements.nil?
-      elements.each do |element|
-        if block.call(element)
-          results << element
-        else
-          element.recursive_inject(results, &block)
-        end
-      end
-    end
-    return results
-  end
-
-  # When Treetop parses the configuration file
-  # it will generate a tree, the generated tree will contain
-  # a few `Empty` nodes to represent the actual space/tab or newline in the file.
-  # Some of theses node will point to our concrete class.
-  # To fetch a specific types of object we need to follow each branch
-  # and ignore the empty nodes.
-  def recursive_select(klass)
-    return recursive_inject { |e| e.is_a?(klass) }
-  end
-
-  def recursive_inject_parent(results=[], &block)
-    if !parent.nil?
-      if block.call(parent)
-        results << parent
-      else
-        parent.recursive_inject_parent(results, &block)
-      end
-    end
-    return results
-  end
-
-  def recursive_select_parent(results=[], klass)
-    return recursive_inject_parent(results) { |e| e.is_a?(klass) }
-  end
-end
-
+require "logstash/compiler/treetop_monkeypatches"
 
 module LogStash; module Config; module AST
-  PROCESS_ESCAPE_SEQUENCES = :process_escape_sequences
 
   def self.deferred_conditionals=(val)
     @deferred_conditionals = val
@@ -105,11 +37,6 @@ module LogStash; module Config; module AST
   end
 
   class Config < Node
-    def process_escape_sequences=(val)
-      set_meta(PROCESS_ESCAPE_SEQUENCES, val)
-    end
-
-
     def compile
       LogStash::Config::AST.deferred_conditionals = []
       LogStash::Config::AST.deferred_conditionals_index = 0
@@ -141,9 +68,8 @@ module LogStash; module Config; module AST
         # of the output/filter function
         definitions << "define_singleton_method :#{type}_func do |event|"
         definitions << "  targeted_outputs = []" if type == "output"
-        definitions << "  events = event" if type == "filter"
-        definitions << "  @logger.debug? && @logger.debug(\"#{type} received\", \"event\" => event.to_hash)" if type == "output"
-        definitions << "  @logger.debug? && events.each { |e| @logger.debug(\"#{type} received\", \"event\" => e.to_hash)}" if type == "filter"
+        definitions << "  events = [event]" if type == "filter"
+        definitions << "  @logger.debug? && @logger.debug(\"#{type} received\", \"event\" => event.to_hash)"
 
         sections.select { |s| s.plugin_type.text_value == type }.each do |s|
           definitions << s.compile.split("\n", -1).map { |e| "  #{e}" }
@@ -353,11 +279,7 @@ module LogStash; module Config; module AST
   end
   class String < Value
     def compile
-      if get_meta(PROCESS_ESCAPE_SEQUENCES)
-        Unicode.wrap(LogStash::Config::StringEscape.process_escapes(text_value[1...-1]))
-      else
-        Unicode.wrap(text_value[1...-1])
-      end
+      return Unicode.wrap(text_value[1...-1])
     end
   end
   class RegExp < Value

data/lib/logstash/config/mixin.rb

@@ -33,21 +33,37 @@ LogStash::Environment.load_locale!
 # }
 #
 module LogStash::Config::Mixin
-  
-  include LogStash::Util::EnvironmentVariables
-  
   attr_accessor :config
   attr_accessor :original_params
 
   PLUGIN_VERSION_1_0_0 = LogStash::Util::PluginVersion.new(1, 0, 0)
   PLUGIN_VERSION_0_9_0 = LogStash::Util::PluginVersion.new(0, 9, 0)
-  
+
+  ENV_PLACEHOLDER_REGEX = /\$\{(?<name>\w+)(\:(?<default>[^}]*))?\}/
+
   # This method is called when someone does 'include LogStash::Config'
   def self.included(base)
     # Add the DSL methods to the 'base' given.
     base.extend(LogStash::Config::Mixin::DSL)
   end
-  
+
+  # Recursive method to replace environment variable references in parameters
+  def deep_replace(value)
+    if (value.is_a?(Hash))
+      value.each do |valueHashKey, valueHashValue|
+        value[valueHashKey.to_s] = deep_replace(valueHashValue)
+      end
+    else
+      if (value.is_a?(Array))
+        value.each_index do | valueArrayIndex|
+          value[valueArrayIndex] = deep_replace(value[valueArrayIndex])
+        end
+      else
+        return replace_env_placeholders(value)
+      end
+    end
+  end
+
   def config_init(params)
     # Validation will modify the values inside params if necessary.
     # For example: converting a string to a number, etc.
@@ -60,6 +76,30 @@ module LogStash::Config::Mixin
     # store the plugin type, turns LogStash::Inputs::Base into 'input'
     @plugin_type = self.class.ancestors.find { |a| a.name =~ /::Base$/ }.config_name
 
+    # warn about deprecated variable use
+    params.each do |name, value|
+      opts = self.class.get_config[name]
+      if opts && opts[:deprecated]
+        extra = opts[:deprecated].is_a?(String) ? opts[:deprecated] : ""
+        extra.gsub!("%PLUGIN%", self.class.config_name)
+        self.logger.warn("You are using a deprecated config setting " +
+                     "#{name.inspect} set in #{self.class.config_name}. " +
+                     "Deprecated settings will continue to work, " +
+                     "but are scheduled for removal from logstash " +
+                     "in the future. #{extra} If you have any questions " +
+                     "about this, please visit the #logstash channel " +
+                     "on freenode irc.", :name => name, :plugin => self)
+
+      end
+      if opts && opts[:obsolete]
+        extra = opts[:obsolete].is_a?(String) ? opts[:obsolete] : ""
+        extra.gsub!("%PLUGIN%", self.class.config_name)
+        raise LogStash::ConfigurationError,
+          I18n.t("logstash.runner.configuration.obsolete", :name => name,
+                 :plugin => self.class.config_name, :extra => extra)
+      end
+    end
+
     # Set defaults from 'config :foo, :default => somevalue'
     self.class.get_config.each do |name, opts|
       next if params.include?(name.to_s)
@@ -85,46 +125,17 @@ module LogStash::Config::Mixin
       params[name.to_s] = deep_replace(value)
     end
 
+
     if !self.class.validate(params)
       raise LogStash::ConfigurationError,
         I18n.t("logstash.runner.configuration.invalid_plugin_settings")
     end
 
-    # now that we know the parameters are valid, we can obfuscate the original copy
-    # of the parameters before storing them as an instance variable
-    self.class.secure_params!(original_params)
-    @original_params = original_params
-
-    # warn about deprecated variable use
-    original_params.each do |name, value|
-      opts = self.class.get_config[name]
-      if opts && opts[:deprecated]
-        extra = opts[:deprecated].is_a?(String) ? opts[:deprecated] : ""
-        extra.gsub!("%PLUGIN%", self.class.config_name)
-        self.logger.warn("You are using a deprecated config setting " +
-                     "#{name.inspect} set in #{self.class.config_name}. " +
-                     "Deprecated settings will continue to work, " +
-                     "but are scheduled for removal from logstash " +
-                     "in the future. #{extra} If you have any questions " +
-                     "about this, please visit the #logstash channel " +
-                     "on freenode irc.", :name => name, :plugin => self)
-
-      end
-
-      if opts && opts[:obsolete]
-        extra = opts[:obsolete].is_a?(String) ? opts[:obsolete] : ""
-        extra.gsub!("%PLUGIN%", self.class.config_name)
-        raise LogStash::ConfigurationError,
-          I18n.t("logstash.runner.configuration.obsolete", :name => name,
-                 :plugin => self.class.config_name, :extra => extra)
-      end
-    end
-
     # We remove any config options marked as obsolete,
     # no code should be associated to them and their values should not bleed
     # to the plugin context.
     #
-    # This need to be done after fetching the options from the parents class
+    # This need to be done after fetching the options from the parents classed
     params.reject! do |name, value|
       opts = self.class.get_config[name]
       opts.include?(:obsolete)
@@ -139,13 +150,36 @@ module LogStash::Config::Mixin
       instance_variable_set("@#{key}", value)
     end
 
+    # now that we know the parameters are valid, we can obfuscate the original copy
+    # of the parameters before storing them as an instance variable
+    self.class.secure_params!(original_params)
+    @original_params = original_params
+
     @config = params
   end # def config_init
 
-  module DSL
-
-    include LogStash::Util::EnvironmentVariables
+  # Replace all environment variable references in 'value' param by environment variable value and return updated value
+  # Process following patterns : $VAR, ${VAR}, ${VAR:defaultValue}
+  def replace_env_placeholders(value)
+    return value unless value.is_a?(String)
+
+    value.gsub(ENV_PLACEHOLDER_REGEX) do |placeholder|
+      # Note: Ruby docs claim[1] Regexp.last_match is thread-local and scoped to
+      # the call, so this should be thread-safe.
+      #
+      # [1] http://ruby-doc.org/core-2.1.1/Regexp.html#method-c-last_match
+      name = Regexp.last_match(:name)
+      default = Regexp.last_match(:default)
+
+      replacement = ENV.fetch(name, default)
+      if replacement.nil?
+        raise LogStash::ConfigurationError, "Cannot evaluate `#{placeholder}`. Environment variable `#{name}` is not set and there is no default value given."
+      end
+      replacement
+    end
+  end # def replace_env_placeholders
 
+  module DSL
     attr_accessor :flags
 
     # If name is given, set the name and return it.
@@ -391,8 +425,6 @@ module LogStash::Config::Mixin
       # (see LogStash::Inputs::File for example)
       result = nil
 
-      value = deep_replace(value)
-
       if validator.nil?
         return true, value
       elsif validator.is_a?(Array)

data/lib/logstash/config/pipeline_config.rb

@@ -0,0 +1,48 @@
+# encoding: utf-8
+require "digest"
+
+module LogStash module Config
+  class PipelineConfig
+    include LogStash::Util::Loggable
+
+    attr_reader :source, :pipeline_id, :config_parts, :settings, :read_at
+
+    def initialize(source, pipeline_id, config_parts, settings)
+      @source = source
+      @pipeline_id = pipeline_id
+      # We can't use Array() since config_parts may be a java object!
+      config_parts_array = config_parts.is_a?(Array) ? config_parts : [config_parts]
+      @config_parts = config_parts_array.sort_by { |config_part| [config_part.protocol.to_s, config_part.id] }
+      @settings = settings
+      @read_at = Time.now
+    end
+
+    def config_hash
+      @config_hash ||= Digest::SHA1.hexdigest(config_string)
+    end
+
+    def config_string
+      @config_string = config_parts.collect(&:text).join("\n")
+    end
+
+    def system?
+      @settings.get("pipeline.system")
+    end
+
+    def ==(other)
+      config_hash == other.config_hash && pipeline_id == other.pipeline_id
+    end
+
+    def display_debug_information
+      logger.debug("-------- Logstash Config ---------")
+      logger.debug("Config from source", :source => source, :pipeline_id => pipeline_id)
+
+      config_parts.each do |config_part|
+        logger.debug("Config string", :protocol => config_part.protocol, :id => config_part.id)
+        logger.debug("\n\n#{config_part.text}")
+      end
+      logger.debug("Merged config")
+      logger.debug("\n\n#{config_string}")
+    end
+  end
+end end

data/lib/logstash/config/source/base.rb

@@ -0,0 +1,16 @@
+# encoding: utf-8
+module LogStash module Config module Source
+  class Base
+    def initialize(settings)
+      @settings = settings
+    end
+
+    def pipeline_configs
+      raise NotImplementedError, "`#pipeline_configs` must be implemented!"
+    end
+
+    def match?
+      raise NotImplementedError, "`match?` must be implemented!"
+    end
+  end
+end end end

data/lib/logstash/config/source/local.rb

@@ -0,0 +1,215 @@
+# encoding: utf-8
+require "logstash/config/source/base"
+require "logstash/config/pipeline_config"
+require "logstash/util/loggable"
+require "logstash/errors"
+require "uri"
+
+module LogStash module Config module Source
+  # A locally defined configuration source
+  #
+  # Which can aggregate the following config options:
+  #  - settings.config_string: "input { stdin {} }"
+  #  - settings.config_path: /tmp/logstash/*.conf
+  #  - settings.config_path: http://localhost/myconfig.conf
+  #
+  #  All theses option will create a unique pipeline, generated parts will be
+  #  sorted alphabetically. Se `PipelineConfig` class for the sorting algorithm.
+  #
+  class Local < Base
+    class ConfigStringLoader
+      def self.read(config_string)
+        [org.logstash.common.SourceWithMetadata.new("string", "config_string", config_string)]
+      end
+    end
+
+    class ConfigPathLoader
+      include LogStash::Util::Loggable
+
+      TEMPORARY_FILE_RE = /~$/
+      LOCAL_FILE_URI = /^file:\/\//i
+
+      def initialize(path)
+        @path = normalize_path(path)
+      end
+
+      def read
+        config_parts = []
+        encoding_issue_files = []
+
+        get_files.each do |file|
+          next unless ::File.file?(file) # skip directory
+
+          logger.debug("Reading config file", :config_file => file)
+
+          if temporary_file?(file)
+            logger.warn("NOT reading config file because it is a temp file", :config_file => file)
+            next
+          end
+
+          config_string = ::File.read(file)
+
+          if valid_encoding?(config_string)
+            part = org.logstash.common.SourceWithMetadata.new("file", file, config_string)
+            config_parts << part
+          else
+            encoding_issue_files << file
+          end
+        end
+
+        if encoding_issue_files.any?
+          raise LogStash::ConfigLoadingError, "The following config files contains non-ascii characters but are not UTF-8 encoded #{encoding_issue_files}"
+        end
+
+        if config_parts.empty?
+          logger.info("No config files found in path", :path => path)
+        end
+
+        config_parts
+      end
+
+      def self.read(path)
+        ConfigPathLoader.new(path).read
+      end
+
+      private
+      def normalize_path(path)
+        path.gsub!(LOCAL_FILE_URI, "")
+        ::File.expand_path(path)
+      end
+
+      def get_files
+        Dir.glob(path).sort
+      end
+
+      def path
+        if ::File.directory?(@path)
+          ::File.join(@path, "*")
+        else
+          @path
+        end
+      end
+
+      def valid_encoding?(content)
+        content.ascii_only? && content.valid_encoding?
+      end
+
+      def temporary_file?(filepath)
+        filepath.match(TEMPORARY_FILE_RE)
+      end
+    end
+
+    class ConfigRemoteLoader
+      def self.read(uri)
+        uri = URI.parse(uri)
+
+        Net::HTTP.start(uri.host, uri.port, :use_ssl => uri.scheme == "https") do |http|
+          request = Net::HTTP::Get.new(uri.path)
+          response = http.request(request)
+
+          # since we have fetching config we wont follow any redirection.
+          case response.code.to_i
+          when 200
+            [org.logstash.common.SourceWithMetadata.new(uri.scheme, uri.to_s, response.body)]
+          when 302
+            raise LogStash::ConfigLoadingError, I18n.t("logstash.runner.configuration.fetch-failed", :path => uri.to_s, :message => "We don't follow redirection for remote configuration")
+          when 404
+            raise LogStash::ConfigLoadingError, I18n.t("logstash.runner.configuration.fetch-failed", :path => uri.to_s, :message => "File not found")
+          when 403
+            raise LogStash::ConfigLoadingError, I18n.t("logstash.runner.configuration.fetch-failed", :path => uri.to_s, :message => "Permission denied")
+          when 500
+            raise LogStash::ConfigLoadingError, I18n.t("logstash.runner.configuration.fetch-failed", :path => uri.to_s, :message => "500 error on remote host")
+          else
+            raise LogStash::ConfigLoadingError, I18n.t("logstash.runner.configuration.fetch-failed", :path => uri.to_s, :message => "code: #{response.code}, message: #{response.class.to_s}")
+          end
+        end
+      end
+    end
+
+    PIPELINE_ID = LogStash::SETTINGS.get("pipeline.id").to_sym
+    HTTP_RE = /^http(s)?/
+    INPUT_BLOCK_RE = /input *{/
+    OUTPUT_BLOCK_RE = /output *{/
+
+    def pipeline_configs
+      config_parts = []
+
+      config_parts.concat(ConfigStringLoader.read(config_string)) if config_string?
+      if local_config?
+        local_config_parts = ConfigPathLoader.read(config_path)
+        config_parts.concat(local_config_parts)
+      else
+        local_config_parts = []
+      end
+
+      config_parts.concat(ConfigRemoteLoader.read(config_path)) if remote_config?
+
+      return if config_parts.empty?
+      return if config_string? && config_string.strip.empty? && local_config? && local_config_parts.empty?
+
+      add_missing_default_inputs_or_outputs(config_parts)
+
+      [PipelineConfig.new(self.class, PIPELINE_ID, config_parts, @settings)]
+    end
+
+    def match?
+      config_string? || config_path?
+    end
+
+    private
+    # Make sure we have an input and at least 1 output
+    # if its not the case we will add stdin and stdout
+    # this is for backward compatibility reason
+    def add_missing_default_inputs_or_outputs(config_parts)
+      if !config_parts.any? { |part| INPUT_BLOCK_RE.match(part.text) }
+        config_parts << org.logstash.common.SourceWithMetadata.new(self.class.name, "default input", LogStash::Config::Defaults.input)
+      end
+
+      # include a default stdout output if no outputs given
+      if !config_parts.any? { |part| OUTPUT_BLOCK_RE.match(part.text) }
+        config_parts << org.logstash.common.SourceWithMetadata.new(self.class.name, "default output", LogStash::Config::Defaults.output)
+      end
+    end
+
+    def config_string
+      @settings.get("config.string")
+    end
+
+    def config_string?
+      !config_string.nil?
+    end
+
+    def config_path
+      @settings.get("path.config")
+    end
+
+    def config_path?
+      !config_path.nil? && !config_path.empty?
+    end
+
+    def local_config?
+      return false unless config_path?
+
+      begin
+        uri = URI.parse(config_path)
+        uri.scheme == "file" || uri.scheme.nil?
+      rescue URI::InvalidURIError
+        # fallback for windows.
+        # if the parsing of the file failed we assume we can reach it locally.
+        # some relative path on windows arent parsed correctly (.\logstash.conf)
+        true
+      end
+    end
+
+    def remote_config?
+      return false unless config_path?
+
+      begin
+        uri = URI.parse(config_path)
+        uri.scheme =~ HTTP_RE
+      rescue URI::InvalidURIError
+        false
+      end
+    end
+  end
+end end end