logstash-core 5.4.3-java → 5.5.0-java

data/lib/logstash/logging/logger.rb

@@ -7,6 +7,7 @@ module LogStash
     java_import org.apache.logging.log4j.LogManager
     java_import org.apache.logging.log4j.core.config.Configurator
     java_import org.apache.logging.log4j.core.config.DefaultConfiguration
+    java_import org.apache.logging.log4j.core.config.LoggerConfig
 
     class Logger
       @@config_mutex = Mutex.new
@@ -65,7 +66,7 @@ module LogStash
       end
 
       def self.configure_logging(level, path = LogManager::ROOT_LOGGER_NAME)
-        @@config_mutex.synchronize { Configurator.setLevel(path, Level.valueOf(level)) }
+        @@config_mutex.synchronize { set_level(level, path) }
       rescue Exception => e
         raise ArgumentError, "invalid level[#{level}] for logger[#{path}]"
       end
@@ -90,6 +91,30 @@ module LogStash
       def self.get_logging_context
         return @@logging_context
       end
+
+      # Clone of org.apache.logging.log4j.core.config.Configurator.setLevel(), but using initialized @@logging_context
+      def self.set_level(_level, path)
+        configuration = @@logging_context.getConfiguration()
+        level = Level.valueOf(_level)
+        if path.nil? || path.strip.empty?
+          root_logger = configuration.getRootLogger()
+          if root_logger.getLevel() != level
+            root_logger.setLevel(level)
+            @@logging_context.updateLoggers()
+          end
+        else
+          package_logger = configuration.getLoggerConfig(path)
+          if package_logger.name != path #no package logger found
+            configuration.addLogger(path, LoggerConfig.new(path, level, true))
+            @@logging_context.updateLoggers()
+          elsif package_logger.getLevel() != level
+            package_logger.setLevel(level)
+            @@logging_context.updateLoggers()
+          end
+        end
+      end
+
+      private_class_method :set_level
     end
 
     class SlowLogger

data/lib/logstash/modules/cli_parser.rb

@@ -0,0 +1,74 @@
+# encoding: utf-8
+require "logstash/namespace"
+require "logstash/logging"
+require "logstash/errors"
+
+module LogStash module Modules class CLIParser
+  include LogStash::Util::Loggable
+
+  attr_reader :output
+  def initialize(module_names, module_variables)
+    @output = []
+    # The #compact here catches instances when module_variables may be nil or
+    # [nil] and sets it to []
+    parse_it(module_names, Array(module_variables).compact)
+  end
+
+  def parse_modules(module_list)
+    parsed_modules = []
+    module_list.each do |module_value|
+      # Calling --modules but not filling it results in [nil], so skip that.
+      next if module_value.nil?
+      # Catch if --modules was launched empty but an option/flag (-something)
+      # follows immediately after
+      if module_value.start_with?('-')
+        raise LogStash::ConfigLoadingError, I18n.t("logstash.modules.configuration.modules-invalid-name", :module_name => module_value)
+      end
+      parsed_modules.concat module_value.split(',')
+    end
+    parsed_modules
+  end
+
+  def get_kv(module_name, unparsed)
+    # Ensure that there is at least 1 equals sign in our variable string
+    # Using String#partition to split on the first '='
+    # This hackery is to catch the possibility of an equals (`=`) sign
+    # in a passphrase, which might result in an incomplete key.  The
+    # portion before the first `=` should always be the key, leaving
+    # the rest to be the value
+    k, op, rest = unparsed.partition('=')
+    if rest.size.zero?
+      raise LogStash::ConfigLoadingError, I18n.t("logstash.modules.configuration.modules-variables-malformed", :rawvar => (module_name + '.' + unparsed))
+    end
+    return k.strip, rest.strip
+  end
+
+  def name_splitter(unparsed)
+    # It must have at least `modulename.something`
+    module_name, dot, rest = unparsed.partition('.')
+    if rest.count('.') >= 1
+      return module_name, rest
+    else
+      raise LogStash::ConfigLoadingError, I18n.t("logstash.modules.configuration.modules-variables-malformed", :rawvar => unparsed)
+    end
+  end
+
+  def parse_vars(module_name, vars_list)
+    module_hash = {"name" => module_name}
+    vars_list.each do |unparsed|
+      extracted_name, modvar = name_splitter(unparsed)
+      next if extracted_name != module_name
+      k, v = get_kv(extracted_name, modvar)
+      module_hash[k] = v
+    end
+    module_hash
+  end
+
+  def parse_it(module_list, module_variable_list)
+    if module_list.is_a?(Array)
+      parse_modules(module_list).each do |module_name|
+        @output << parse_vars(module_name, module_variable_list)
+      end
+    end
+  end
+end end end

data/lib/logstash/modules/elasticsearch_config.rb

@@ -0,0 +1,22 @@
+# encoding: utf-8
+require "logstash/namespace"
+require "logstash/logging"
+
+require_relative "elasticsearch_resource"
+
+module LogStash module Modules class ElasticsearchConfig
+  attr_reader :index_name
+
+  # We name it `modul` here because `module` has meaning in Ruby.
+  def initialize(modul, settings)
+    @directory = ::File.join(modul.directory, "elasticsearch")
+    @name = modul.module_name
+    @settings = settings
+    @full_path = ::File.join(@directory, "#{@name}.json")
+    @index_name = @settings.fetch("elasticsearch.template_path", "_template")
+  end
+
+  def resources
+    [ElasticsearchResource.new(@index_name, "not-used", @full_path)]
+  end
+end end end

data/lib/logstash/modules/elasticsearch_resource.rb

@@ -0,0 +1,10 @@
+# encoding: utf-8
+require "logstash/namespace"
+require_relative "resource_base"
+
+module LogStash module Modules class ElasticsearchResource
+  include ResourceBase
+  def import_path
+    base + "/" + content_id
+  end
+end end end

data/lib/logstash/modules/file_reader.rb

@@ -0,0 +1,36 @@
+# encoding: utf-8
+require "logstash/namespace"
+require "logstash/logging"
+require "logstash/json"
+
+module LogStash module Modules class FileReader
+  # stub these methods for testing
+  include LogStash::Util::Loggable
+
+  def self.read(path)
+    begin
+      ::File.read(path)
+    rescue => e
+      logger.error("Error when reading file from path", :path => path)
+      ""
+    end
+  end
+
+  def self.read_json(path)
+    json = read(path)
+    begin
+      LogStash::Json.load(json)
+    rescue => e
+      logger.error("Error when parsing json from path", :path => path)
+      return {}
+    end
+  end
+
+  def self.glob(path)
+    files = Dir.glob(path)
+    if files.empty?
+      logger.warn("No files found for glob", :pattern => path)
+    end
+    files
+  end
+end end end

data/lib/logstash/modules/importer.rb

@@ -0,0 +1,37 @@
+# encoding: utf-8
+require "logstash/namespace"
+require "logstash/logging"
+
+module LogStash module Modules class Importer
+  include LogStash::Util::Loggable
+
+  def initialize(client)
+    @client = client
+  end
+
+  def put(resource, overwrite = true)
+    path = resource.import_path
+    logger.info("Attempting PUT", :url_path => path, :file_path => resource.content_path)
+    if !overwrite && content_exists?(path)
+      logger.debug("Found existing Elasticsearch resource.", :resource => path)
+      return
+    end
+    put_overwrite(path, resource.content)
+  end
+
+  private
+
+  def put_overwrite(path, content)
+    if content_exists?(path)
+      response = @client.delete(path)
+    end
+    # hmmm, versioning?
+    @client.put(path, content).status == 201
+  end
+
+  def content_exists?(path)
+    response = @client.head(path)
+    response.status >= 200 && response.status <= 299
+  end
+
+end end end # class LogStash::Modules::Importer

data/lib/logstash/modules/kibana_base_resource.rb

@@ -0,0 +1,10 @@
+# encoding: utf-8
+require "logstash/namespace"
+require_relative "resource_base"
+
+module LogStash module Modules class KibanaBaseResource
+  include ResourceBase
+  def import_path
+    base
+  end
+end end end

data/lib/logstash/modules/kibana_config.rb

@@ -0,0 +1,104 @@
+# encoding: utf-8
+require "logstash/namespace"
+require "logstash/logging"
+
+require_relative "file_reader"
+require_relative "kibana_resource"
+require_relative "kibana_base_resource"
+
+module LogStash module Modules class KibanaConfig
+  include LogStash::Util::Loggable
+
+  ALLOWED_DIRECTORIES = ["search", "visualization"]
+
+  attr_reader :index_name
+
+  # We name it `modul` here because `module` has meaning in Ruby.
+  def initialize(modul, settings)
+    @directory = ::File.join(modul.directory, "kibana")
+    @name = modul.module_name
+    @settings = settings
+    @index_name = settings.fetch("dashboards.kibana_index", ".kibana")
+  end
+
+  def dashboards
+    # there can be more than one dashboard to load
+    filenames = FileReader.read_json(dynamic("dashboard"))
+    filenames.map do |filename|
+      KibanaResource.new(@index_name, "dashboard", dynamic("dashboard", filename))
+    end
+  end
+
+  def index_pattern
+    pattern_name = "#{@name}-*"
+    default_index_json = '{"defaultIndex": "#{pattern_name}"}'
+    default_index_content_id = @settings.fetch("index_pattern.kibana_version", "5.5.0")
+    [
+      KibanaResource.new(@index_name, "index-pattern", dynamic("index-pattern"),nil, pattern_name),
+      KibanaResource.new(@index_name, "config", nil, default_index_json, default_index_content_id)
+    ]
+  end
+
+  def resources
+    list = index_pattern
+    dashboards.each do |board|
+      extract_panels_into(board, list)
+    end
+    list.concat(extract_saved_searches(list))
+  end
+
+  private
+
+  def dynamic(dynamic_folder, filename = @name)
+    ::File.join(@directory, dynamic_folder, "#{filename}.json")
+  end
+
+  def extract_panels_into(dashboard, list)
+    list << dashboard
+
+    dash = FileReader.read_json(dashboard.content_path)
+
+    if !dash.is_a?(Hash)
+      logger.warn("Kibana dashboard JSON is not an Object", :module => @name)
+      return
+    end
+
+    panelsjson = dash["panelsJSON"]
+
+    if panelsjson.nil?
+      logger.info("No panelJSON key found in kibana dashboard", :module => @name)
+      return
+    end
+
+    begin
+      panels = LogStash::Json.load(panelsjson)
+    rescue => e
+      logger.error("JSON parse error when reading kibana panelsJSON", :module => @name)
+      return
+    end
+
+    panels.each do |panel|
+      panel_type = panel["type"]
+      if ALLOWED_DIRECTORIES.member?(panel_type)
+        list << KibanaResource.new(@index_name, panel_type, dynamic(panel_type, panel["id"]))
+      else
+        logger.warn("panelJSON contained unknown type", :type => panel_type)
+      end
+    end
+
+    def extract_saved_searches(list)
+      result = [] # must not add to list while iterating
+      list.each do |resource|
+        next unless resource.contains?("savedSearchId")
+        content = resource.content_as_object
+        next if content.nil?
+        saved_search = content["savedSearchId"]
+        next if saved_search.nil?
+        ss_resource = KibanaResource.new(@index_name, "search", dynamic("search", saved_search))
+        next if list.member?(ss_resource) || result.member?(ss_resource)
+        result << ss_resource
+      end
+      result
+    end
+  end
+end end end

data/lib/logstash/modules/kibana_resource.rb

@@ -0,0 +1,10 @@
+# encoding: utf-8
+require "logstash/namespace"
+require_relative "resource_base"
+
+module LogStash module Modules class KibanaResource
+  include ResourceBase
+  def import_path
+    base + "/" + content_type + "/" + content_id
+  end
+end end end

data/lib/logstash/modules/logstash_config.rb

@@ -0,0 +1,48 @@
+# encoding: utf-8
+require "logstash/namespace"
+require_relative "file_reader"
+
+module LogStash module Modules class LogStashConfig
+
+  # We name it `modul` here because `module` has meaning in Ruby.
+  def initialize(modul, settings)
+    @directory = ::File.join(modul.directory, "logstash")
+    @name = modul.module_name
+    @settings = settings
+  end
+
+  def template
+    ::File.join(@directory, "#{@name}.conf.erb")
+  end
+
+  def setting(value, default)
+    @settings.fetch(value, default)
+  end
+
+  def elasticsearch_output_config(type_string = nil)
+    hosts = setting("var.output.elasticsearch.hosts", "localhost:9200").split(',').map do |s|
+      '"' + s.strip + '"'
+    end.join(',')
+    index = "#{@name}-#{setting("var.output.elasticsearch.index_suffix", "%{+YYYY.MM.dd}")}"
+    password = "#{setting("var.output.elasticsearch.password", "changeme")}"
+    user = "#{setting("var.output.elasticsearch.user", "elastic")}"
+    document_type_line = type_string ? "document_type => #{type_string}" : ""
+    <<-CONF
+elasticsearch {
+hosts => [#{hosts}]
+index => "#{index}"
+password => "#{password}"
+user => "#{user}"
+manage_template => false
+#{document_type_line}
+}
+CONF
+  end
+
+  def config_string
+    # process the template and settings
+    # send back as a string
+    renderer = ERB.new(FileReader.read(template))
+    renderer.result(binding)
+  end
+end end end

data/lib/logstash/modules/resource_base.rb

@@ -0,0 +1,37 @@
+# encoding: utf-8
+require "logstash/namespace"
+require_relative "file_reader"
+
+module LogStash module Modules module ResourceBase
+  attr_reader :base, :content_type, :content_path, :content_id
+
+  def initialize(base, content_type, content_path, content = nil, content_id = nil)
+    @base, @content_type, @content_path = base, content_type, content_path
+    @content_id =  content_id || ::File.basename(@content_path, ".*")
+    @content = content
+  end
+
+  def content
+    @content ||= FileReader.read(@content_path)
+  end
+
+  def to_s
+    "#{base}, #{content_type}, #{content_path}, #{content_id}"
+  end
+
+  def contains?(text)
+    content.include?(text)
+  end
+
+  def content_as_object
+    LogStash::Json.load(content) rescue nil
+  end
+
+  def <=>(other)
+    to_s <=> other.to_s
+  end
+
+  def ==(other)
+    to_s == other.to_s
+  end
+end end end