logstash-core 5.6.16-java → 6.0.0.alpha1-java

data/lib/logstash/modules/kibana_dashboards.rb

@@ -1,36 +0,0 @@
-# encoding: utf-8
-require "logstash/namespace"
-require "logstash/logging"
-require_relative "kibana_base"
-
-module LogStash module Modules class KibanaDashboards < KibanaBase
-  include LogStash::Util::Loggable
-
-  attr_reader :import_path, :content
-
-  # content is a list of kibana file resources
-  def initialize(import_path, content)
-    @import_path, @content = import_path, content
-  end
-
-  def import(client)
-    # e.g. curl "http://localhost:5601/api/kibana/dashboards/import"
-    # extract and prepare all objects
-    objects = []
-    content.each do |resource|
-      hash = {
-        "id" => resource.content_id,
-        "type" => resource.content_type,
-        "version" => 1,
-        "attributes" => resource.content_as_object
-      }
-      objects << hash
-    end
-    body = {:version => client.version, :objects => objects}
-    response = client.post(import_path, body)
-    if response.failed?
-      logger.error("Attempted POST failed", :url_path => import_path, :response => response.body)
-    end
-    response
-  end
-end end end

data/lib/logstash/modules/kibana_importer.rb

@@ -1,17 +0,0 @@
-# encoding: utf-8
-require "logstash/namespace"
-require "logstash/logging"
-
-module LogStash module Modules class KibanaImporter
-  include LogStash::Util::Loggable
-
-  def initialize(client)
-    @client = client
-  end
-
-  def put(via_kibana)
-    path = via_kibana.import_path
-    logger.debug("Attempting POST", :url_path => path, :content => via_kibana.content)
-    via_kibana.import(@client)
-  end
-end end end

data/lib/logstash/modules/kibana_resource.rb

@@ -1,10 +0,0 @@
-# encoding: utf-8
-require "logstash/namespace"
-require_relative "resource_base"
-
-module LogStash module Modules class KibanaResource
-  include ResourceBase
-  def import_path
-    base + "/" + content_type + "/" + content_id
-  end
-end end end

data/lib/logstash/modules/kibana_settings.rb

@@ -1,40 +0,0 @@
-# encoding: utf-8
-require "logstash/namespace"
-require "logstash/logging"
-require_relative "kibana_base"
-
-module LogStash module Modules class KibanaSettings < KibanaBase
-  include LogStash::Util::Loggable
-
-  class Setting
-    attr_reader :name, :value
-    def initialize(name, value)
-      @name, @value = name, value
-    end
-  end
-
-  attr_reader :import_path, :content
-
-  # content is an array of Setting required for this module
-  def initialize(import_path, content)
-    @import_path, @content = import_path, content
-  end
-
-  def import(client)
-    # e.g. curl "http://localhost:5601/api/kibana/settings"
-    # 6.0.0-alpha3 -> {"settings":{"buildNum":{"userValue":15613},"defaultIndex":{"userValue":"arcsight-*"}}}
-    # 5.4 -> {"settings":{"defaultIndex":{"userValue":"cef-*"},"metrics:max_buckets":{"userValue":"600000"}}}
-    # array of Setting objects
-    # The POST api body { "changes": { "defaultIndex": "arcsight-*", "metrics:max_buckets": "400" } }
-    settings = {}
-    content.each do |setting|
-      settings[setting.name] = "#{setting.value}"
-    end
-    body = {"changes" => settings}
-    response = client.post(import_path, body)
-    if response.failed?
-      logger.error("Attempted POST failed", :url_path => import_path, :response => response.body)
-    end
-    response
-  end
-end end end

data/lib/logstash/modules/logstash_config.rb

@@ -1,120 +0,0 @@
-# encoding: utf-8
-require "logstash/namespace"
-require_relative "file_reader"
-require "logstash/settings"
-
-module LogStash module Modules class LogStashConfig
-  # We name it `modul` here because `module` has meaning in Ruby.
-  def initialize(modul, settings)
-    @directory = ::File.join(modul.directory, "logstash")
-    @name = modul.module_name
-    @settings = settings
-  end
-
-  def template
-    ::File.join(@directory, "#{@name}.conf.erb")
-  end
-
-  def configured_inputs(default = [], aliases = {})
-    name = "var.inputs"
-    values = get_setting(LogStash::Setting::SplittableStringArray.new(name, String, default))
-
-    aliases.each { |k,v| values << v if values.include?(k) }
-    aliases.invert.each { |k,v| values << v if values.include?(k) }
-    values.flatten.uniq
-  end
-
-  def alias_settings_keys!(aliases)
-    aliased_settings = alias_matching_keys(aliases, @settings)
-    @settings = alias_matching_keys(aliases.invert, aliased_settings)
-  end
-
-  def array_to_string(array)
-    "[#{array.collect { |i| "'#{i}'" }.join(", ")}]"
-  end
-
-  def csv_string(array)
-    "'#{array.join(',')}'"
-  end
-
-  def get_setting(setting_class)
-    raw_value = @settings[setting_class.name]
-    # If we dont check for NIL, the Settings class will try to coerce the value
-    # and most of the it will fails when a NIL value is explicitely set.
-    # This will be fixed once we wrap the plugins settings into a Settings class
-    setting_class.set(raw_value) unless raw_value.nil?
-    setting_class.value
-  end
-
-  def setting(name, default)
-    # by default we use the more permissive setting which is a `NullableString`
-    # This is fine because the end format of the logstash configuration is a string representation
-    # of the pipeline. There is a good reason why I think we should use the settings classes, we
-    # can `preprocess` a template and generate a configuration from the defined settings
-    # validate the values and replace them in the template.
-    case default
-      when String
-        get_setting(LogStash::Setting::NullableString.new(name, default.to_s))
-      when Numeric
-        get_setting(LogStash::Setting::Numeric.new(name, default))
-      else
-        get_setting(LogStash::Setting::NullableString.new(name, default.to_s))
-      end
-  end
-
-  def elasticsearch_output_config(type_string = nil)
-    hosts = array_to_string(get_setting(LogStash::Setting::SplittableStringArray.new("var.elasticsearch.hosts", String, ["localhost:9200"])))
-    index = "#{@name}-#{setting("var.elasticsearch.index_suffix", "%{+YYYY.MM.dd}")}"
-    user = @settings["var.elasticsearch.username"]
-    password = @settings["var.elasticsearch.password"]
-    lines = ["hosts => #{hosts}", "index => \"#{index}\""]
-    lines.push(user ? "user => \"#{user}\"" : nil)
-    lines.push(password ? "password => \"#{password}\"" : nil)
-    lines.push(type_string ? "document_type => #{type_string}" : nil)
-    lines.push("ssl => #{@settings.fetch('var.elasticsearch.ssl.enabled', false)}")
-    if cacert = @settings["var.elasticsearch.ssl.certificate_authority"]
-      lines.push("cacert => \"#{cacert}\"") if cacert
-    end
-    # NOTE: the first line should be indented in the conf.erb
-    <<-CONF
-elasticsearch {
-    #{lines.compact.join("\n    ")}
-    manage_template => false
-  }
-CONF
-  end
-
-  def config_string
-    # process the template and settings
-    # send back as a string
-    renderer = ERB.new(FileReader.read(template))
-    renderer.result(binding)
-  end
-
-  private
-  # For a first version we are copying the values of the original hash,
-  # this might become problematic if we users changes the values of the
-  # settings in the template, which could result in an inconsistent view of the original data
-  #
-  # For v1 of the feature I think its an OK compromise, v2 we have a more advanced hash that
-  # support alias.
-  def alias_matching_keys(aliases, target)
-    aliased_target = target.dup
-
-    aliases.each do |matching_key_prefix, new_key_prefix|
-      target.each do |k, v|
-        re = /^#{matching_key_prefix}\./
-
-        if k =~ re
-          alias_key = k.gsub(re, "#{new_key_prefix}.")
-
-          # If the user setup the same values twices with different values lets just halt.
-          raise "Cannot create an alias, the destination key has already a value set: original key: #{k}, alias key: #{alias_key}" if (!aliased_target[alias_key].nil? && aliased_target[alias_key] != v)
-          aliased_target[alias_key] = v unless v.nil?
-        end
-      end
-    end
-
-    aliased_target
-  end
-end end end

data/lib/logstash/modules/resource_base.rb

@@ -1,38 +0,0 @@
-# encoding: utf-8
-require "logstash/namespace"
-require "logstash/json"
-require_relative "file_reader"
-
-module LogStash module Modules module ResourceBase
-  attr_reader :base, :content_type, :content_path, :content_id
-
-  def initialize(base, content_type, content_path, content = nil, content_id = nil)
-    @base, @content_type, @content_path = base, content_type, content_path
-    @content_id =  content_id || ::File.basename(@content_path, ".*")
-    # content at this time will be a JSON string
-    @content = content
-    if !@content.nil?
-      @content_as_object = LogStash::Json.load(@content) rescue {}
-    end
-  end
-
-  def content
-    @content ||= FileReader.read(@content_path)
-  end
-
-  def to_s
-    "#{base}, #{content_type}, #{content_path}, #{content_id}"
-  end
-
-  def content_as_object
-    @content_as_object ||= FileReader.read_json(@content_path) rescue nil
-  end
-
-  def <=>(other)
-    to_s <=> other.to_s
-  end
-
-  def ==(other)
-    to_s == other.to_s
-  end
-end end end

data/lib/logstash/modules/scaffold.rb

@@ -1,52 +0,0 @@
-# encoding: utf-8
-require "logstash/namespace"
-require "logstash/logging"
-require "logstash/util/loggable"
-require "erb"
-
-require_relative "elasticsearch_config"
-require_relative "kibana_config"
-require_relative "logstash_config"
-
-module LogStash module Modules class Scaffold
-  include LogStash::Util::Loggable
-
-  attr_reader :directory, :module_name, :kibana_version_parts
-  attr_reader :kibana_configuration, :logstash_configuration, :elasticsearch_configuration
-
-  def initialize(name, directory)
-    @module_name = name
-    @directory = directory  # this is the 'configuration folder in the GEM root.'
-    @kibana_version_parts = "5.6.0".split('.') # this is backup in case kibana client fails to connect
-    logger.info("Initializing module", :module_name => name, :directory => directory)
-  end
-
-  def add_kibana_version(version_parts)
-    @kibana_version_parts = version_parts
-  end
-
-  def import(import_engine, kibana_import_engine)
-    @elasticsearch_configuration.resources.each do |resource|
-      import_engine.put(resource)
-    end
-    @kibana_configuration.resources.each do |resource|
-      kibana_import_engine.put(resource)
-    end
-  end
-
-  def with_settings(module_settings)
-    @logstash_configuration = LogStashConfig.new(self, module_settings)
-    @kibana_configuration = KibanaConfig.new(self, module_settings)
-    @elasticsearch_configuration = ElasticsearchConfig.new(self, module_settings)
-    self
-  end
-
-  def config_string()
-    # settings should be set earlier by the caller.
-    # settings should be the subset from the YAML file with a structure like
-    # {"name" => "plugin name", "k1" => "v1", "k2" => "v2"}, etc.
-    return nil if @logstash_configuration.nil?
-    @logstash_configuration.config_string
-  end
-end end end # class LogStash::Modules::Scaffold
-

data/lib/logstash/modules/settings_merger.rb

@@ -1,23 +0,0 @@
-# encoding: utf-8
-require "logstash/namespace"
-
-module LogStash module Modules class SettingsMerger
-  def self.merge(cli_settings, yml_settings)
-    # both args are arrays of hashes, e.g.
-    # [{"name"=>"mod1", "var.input.tcp.port"=>"3333"}, {"name"=>"mod2"}]
-    # [{"name"=>"mod1", "var.input.tcp.port"=>2222, "var.kibana.username"=>"rupert", "var.kibana.password"=>"fotherington"}, {"name"=>"mod3", "var.input.tcp.port"=>4445}]
-    merged = []
-    # union and group_by preserves order
-    # union will also coalesce identical hashes
-    union_of_settings = (cli_settings | yml_settings)
-    grouped_by_name = union_of_settings.group_by{|e| e["name"]}
-    grouped_by_name.each do |name, array|
-      if array.size == 2
-        merged << array.first.merge(array.last)
-      else
-        merged.concat(array)
-      end
-    end
-    merged
-  end
-end end end

data/lib/logstash/modules/util.rb

@@ -1,17 +0,0 @@
-# encoding: utf-8
-require_relative "scaffold"
-
-# This module function should be used when gems or
-# x-pack defines modules in their folder structures.
-module LogStash module Modules module Util
-  def self.register_local_modules(path)
-    modules_path = ::File.join(path, "modules")
-    ::Dir.foreach(modules_path) do |item|
-      # Ignore unix relative path ids
-      next if item == '.' or item == '..'
-      # Ignore non-directories
-      next if !::File.directory?(::File.join(modules_path, ::File::Separator, item))
-      LogStash::PLUGIN_REGISTRY.add(:modules, item, Scaffold.new(item, ::File.join(modules_path, item, "configuration")))
-    end
-  end
-end end end

data/lib/logstash/util/dead_letter_queue_manager.rb

@@ -1,61 +0,0 @@
-require 'logstash/environment'
-
-module LogStash; module Util
-  class PluginDeadLetterQueueWriter
-
-    attr_reader :plugin_id, :plugin_type, :inner_writer
-
-    def initialize(inner_writer, plugin_id, plugin_type)
-      @plugin_id = plugin_id
-      @plugin_type = plugin_type
-      @inner_writer = inner_writer
-    end
-
-    def write(logstash_event, reason)
-      if @inner_writer && @inner_writer.is_open
-        @inner_writer.writeEntry(logstash_event.to_java, @plugin_type, @plugin_id, reason)
-      end
-    end
-
-    def close
-      if @inner_writer && @inner_writer.is_open
-        @inner_writer.close
-      end
-    end
-  end
-
-  class DummyDeadLetterQueueWriter
-    # class uses to represent a writer when dead_letter_queue is disabled
-    def initialize
-    end
-
-    def write(logstash_event, reason)
-      # noop
-    end
-
-    def is_open
-      false
-    end
-
-    def close
-      # noop
-    end
-  end
-
-  class DeadLetterQueueFactory
-    java_import org.logstash.common.DeadLetterQueueFactory
-
-    def self.get(pipeline_id)
-      if LogStash::SETTINGS.get("dead_letter_queue.enable")
-        return DeadLetterQueueWriter.new(
-          DeadLetterQueueFactory.getWriter(pipeline_id, LogStash::SETTINGS.get("path.dead_letter_queue"), LogStash::SETTINGS.get('dead_letter_queue.max_bytes')))
-      else
-        return DeadLetterQueueWriter.new(nil)
-      end
-    end
-
-    def self.close(pipeline_id)
-      DeadLetterQueueFactory.close(pipeline_id)
-    end
-  end
-end end

data/lib/logstash/util/environment_variables.rb

@@ -1,43 +0,0 @@
-# encoding: utf-8
-module ::LogStash::Util::EnvironmentVariables
-
-  ENV_PLACEHOLDER_REGEX = /\${(?<name>[a-zA-Z_.][a-zA-Z0-9_.]*)(:(?<default>[^}]*))?}/
-
-  # Recursive method to replace environment variable references in parameters
-  def deep_replace(value)
-    if value.is_a?(Hash)
-      value.each do |valueHashKey, valueHashValue|
-        value[valueHashKey.to_s] = deep_replace(valueHashValue)
-      end
-    else
-      if value.is_a?(Array)
-        value.each_index do | valueArrayIndex|
-          value[valueArrayIndex] = deep_replace(value[valueArrayIndex])
-        end
-      else
-        return replace_env_placeholders(value)
-      end
-    end
-  end
-
-  # Replace all environment variable references in 'value' param by environment variable value and return updated value
-  # Process following patterns : $VAR, ${VAR}, ${VAR:defaultValue}
-  def replace_env_placeholders(value)
-    return value unless value.is_a?(String)
-
-    value.gsub(ENV_PLACEHOLDER_REGEX) do |placeholder|
-      # Note: Ruby docs claim[1] Regexp.last_match is thread-local and scoped to
-      # the call, so this should be thread-safe.
-      #
-      # [1] http://ruby-doc.org/core-2.1.1/Regexp.html#method-c-last_match
-      name = Regexp.last_match(:name)
-      default = Regexp.last_match(:default)
-
-      replacement = ENV.fetch(name, default)
-      if replacement.nil?
-        raise LogStash::ConfigurationError, "Cannot evaluate `#{placeholder}`. Environment variable `#{name}` is not set and there is no default value given."
-      end
-      replacement
-    end
-  end # def replace_env_placeholders
-end