logstash-core 5.6.16-java → 6.0.0.alpha1-java

data/lib/logstash/config/loader.rb

@@ -1,107 +0,0 @@
-require "logstash/config/defaults"
-
-module LogStash; module Config; class Loader
-  def initialize(logger)
-    @logger = logger
-    @config_debug = LogStash::SETTINGS.get_value("config.debug")
-  end
-
-  def format_config(config_path, config_string)
-    config_string = config_string.to_s
-    if config_path
-      # Append the config string.
-      # This allows users to provide both -f and -e flags. The combination
-      # is rare, but useful for debugging.
-      loaded_config = load_config(config_path)
-      if loaded_config.empty? && config_string.empty?
-        # If loaded config from `-f` is empty *and* if config string is empty we raise an error
-        fail(I18n.t("logstash.runner.configuration.file-not-found", :path => config_path))
-      end
-
-      # tell the user we are merging, otherwise it is very confusing
-      if !loaded_config.empty? && !config_string.empty?
-        @logger.info("Created final config by merging config string and config path", :path => config_path)
-      end
-
-      config_string = config_string + loaded_config
-    else
-      # include a default stdin input if no inputs given
-      if config_string !~ /input *{/
-        config_string += LogStash::Config::Defaults.input
-      end
-      # include a default stdout output if no outputs given
-      if config_string !~ /output *{/
-        config_string += LogStash::Config::Defaults.output
-      end
-    end
-    config_string
-  end
-
-  def load_config(path)
-    begin
-      uri = URI.parse(path)
-
-      case uri.scheme
-      when nil then
-        local_config(path)
-      when /http/ then
-        fetch_config(uri)
-      when "file" then
-        local_config(uri.path)
-      else
-        fail(I18n.t("logstash.runner.configuration.scheme-not-supported", :path => path))
-      end
-    rescue URI::InvalidURIError
-      # fallback for windows.
-      # if the parsing of the file failed we assume we can reach it locally.
-      # some relative path on windows arent parsed correctly (.\logstash.conf)
-      local_config(path)
-    end
-  end
-
-  def local_config(path)
-    path = ::File.expand_path(path)
-    path = ::File.join(path, "*") if ::File.directory?(path)
-
-    config = ""
-    if Dir.glob(path).length == 0
-      @logger.info("No config files found in path", :path => path)
-      return config
-    end
-
-    encoding_issue_files = []
-    Dir.glob(path).sort.each do |file|
-      next unless ::File.file?(file)
-      if file.match(/~$/)
-        @logger.debug("NOT reading config file because it is a temp file", :config_file => file)
-        next
-      end
-      @logger.debug("Reading config file", :config_file => file)
-      cfg = ::File.read(file)
-      if !cfg.ascii_only? && !cfg.valid_encoding?
-        encoding_issue_files << file
-      end
-      config << cfg + "\n"
-      if @config_debug
-        @logger.debug? && @logger.debug("\nThe following is the content of a file", :config_file => file.to_s)
-        @logger.debug? && @logger.debug("\n" + cfg + "\n\n")
-      end
-    end
-    if encoding_issue_files.any?
-      fail("The following config files contains non-ascii characters but are not UTF-8 encoded #{encoding_issue_files}")
-    end
-    if @config_debug
-      @logger.debug? && @logger.debug("\nThe following is the merged configuration")
-      @logger.debug? && @logger.debug("\n" + config + "\n\n")
-    end
-    return config
-  end # def load_config
-
-  def fetch_config(uri)
-    begin
-      Net::HTTP.get(uri) + "\n"
-    rescue Exception => e
-      fail(I18n.t("logstash.runner.configuration.fetch-failed", :path => uri.to_s, :message => e.message))
-    end
-  end
-end end end

data/lib/logstash/config/modules_common.rb

@@ -1,103 +0,0 @@
-# encoding: utf-8
-require "logstash/util/loggable"
-require "logstash/elasticsearch_client"
-require "logstash/modules/kibana_client"
-require "logstash/modules/elasticsearch_importer"
-require "logstash/modules/kibana_importer"
-require "logstash/modules/settings_merger"
-require "logstash/errors"
-
-module LogStash module Config
-  class ModulesCommon # extracted here for bwc with 5.x
-    include LogStash::Util::Loggable
-
-    MODULES_MAX_PIPELINES = 1
-
-    def self.pipeline_configs(settings)
-      pipelines = []
-      plugin_modules = LogStash::PLUGIN_REGISTRY.plugins_with_type(:modules)
-
-      cli_settings = settings.get("modules.cli")
-      yml_settings = settings.get("modules")
-
-      modules_array = if !(cli_settings.empty? && yml_settings.empty?)
-            LogStash::Modules::SettingsMerger.merge(cli_settings, yml_settings)
-          elsif cli_settings.empty?
-             yml_settings
-          else
-            cli_settings
-          end
-
-      if modules_array.empty?
-        # no specifed modules
-        return pipelines
-      end
-      logger.debug("Specified modules", :modules_array => modules_array.to_s)
-
-      module_names = modules_array.collect {|module_hash| module_hash["name"]}
-      if module_names.size > MODULES_MAX_PIPELINES
-        error_message = I18n.t("logstash.modules.configuration.modules-too-many-specified", :max => MODULES_MAX_PIPELINES, :specified_modules => module_names.join(', '))
-        raise LogStash::ConfigLoadingError, error_message
-      end
-
-      if module_names.length > module_names.uniq.length
-        duplicate_modules = module_names.group_by(&:to_s).select { |_,v| v.size > 1 }.keys
-        raise LogStash::ConfigLoadingError, I18n.t("logstash.modules.configuration.modules-must-be-unique", :duplicate_modules => duplicate_modules)
-      end
-
-      available_module_names = plugin_modules.map(&:module_name)
-      specified_and_available_names = module_names & available_module_names
-
-      if (specified_and_available_names).empty?
-        i18n_opts = {:specified_modules => module_names, :available_modules => available_module_names}
-        raise LogStash::ConfigLoadingError, I18n.t("logstash.modules.configuration.modules-unavailable", i18n_opts)
-      end
-
-      specified_and_available_names.each do |module_name|
-        connect_fail_args = {}
-        begin
-          module_settings = settings.clone
-
-          module_hash = modules_array.find {|m| m["name"] == module_name}
-          current_module = plugin_modules.find { |allmodules| allmodules.module_name == module_name }
-
-          alt_name = "module-#{module_name}"
-          pipeline_id = alt_name
-          module_settings.set("pipeline.id", pipeline_id)
-          current_module.with_settings(module_hash)
-          config_test = settings.get("config.test_and_exit")
-          modul_setup = settings.get("modules_setup")
-          # Only import data if it's not a config test and --setup is true
-          if !config_test && modul_setup
-            esclient = LogStash::ElasticsearchClient.build(module_hash)
-            kbnclient = LogStash::Modules::KibanaClient.new(module_hash)
-            esconnected = esclient.can_connect?
-            kbnconnected = kbnclient.can_connect?
-            if esconnected && kbnconnected
-              current_module.add_kibana_version(kbnclient.version_parts)
-              current_module.import(
-                  LogStash::Modules::ElasticsearchImporter.new(esclient),
-                  LogStash::Modules::KibanaImporter.new(kbnclient)
-                )
-            else
-              connect_fail_args[:module_name] = module_name
-              connect_fail_args[:elasticsearch_hosts] = esclient.host_settings
-              connect_fail_args[:kibana_hosts] = kbnclient.host_settings
-            end
-          end
-          config_string = current_module.config_string
-          pipelines << {"pipeline_id" => pipeline_id, "alt_name" => alt_name, "config_string" => config_string, "settings" => module_settings}
-        rescue => e
-          new_error = LogStash::ConfigLoadingError.new(I18n.t("logstash.modules.configuration.parse-failed", :error => e.message))
-          new_error.set_backtrace(e.backtrace)
-          raise new_error
-        end
-
-        if !connect_fail_args.empty?
-          raise LogStash::ConfigLoadingError, I18n.t("logstash.modules.configuration.elasticsearch_connection_failed", connect_fail_args)
-        end
-      end
-      pipelines
-    end
-  end
-end end

data/lib/logstash/config/source/modules.rb

@@ -1,55 +0,0 @@
-# encoding: utf-8
-require "logstash/config/source/base"
-require "logstash/config/modules_common"
-require "logstash/config/pipeline_config"
-require "logstash/util/loggable"
-require "logstash/errors"
-
-module LogStash module Config module Source
-  class Modules < Base
-    include LogStash::Util::Loggable
-    def pipeline_configs
-      if config_conflict? # double check
-        raise ConfigurationError, @conflict_messages.join(", ")
-      end
-
-      pipelines = LogStash::Config::ModulesCommon.pipeline_configs(@settings)
-      pipelines.map do |hash|
-        PipelineConfig.new(self, hash["pipeline_id"].to_sym,
-          org.logstash.common.SourceWithMetadata.new("module", hash["alt_name"], 0, 0, hash["config_string"]),
-          hash["settings"])
-      end
-    end
-
-    def match?
-      # see basic settings predicates and getters defined in the base class
-      (modules_cli? || modules?) && !(config_string? || config_path?) && !automatic_reload_with_modules?
-    end
-
-    def config_conflict?
-      @conflict_messages.clear
-      # Make note that if modules are configured in both cli and logstash.yml that cli module
-      # settings will overwrite the logstash.yml modules settings
-      if modules_cli? && modules?
-        logger.info(I18n.t("logstash.runner.cli-module-override"))
-      end
-
-      if automatic_reload_with_modules?
-        @conflict_messages << I18n.t("logstash.runner.reload-with-modules")
-      end
-
-      # Check if config (-f or -e) and modules are configured
-      if (modules_cli? || modules?) && (config_string? || config_path?)
-        @conflict_messages << I18n.t("logstash.runner.config-module-exclusive")
-      end
-
-      @conflict_messages.any?
-    end
-
-    private
-
-    def automatic_reload_with_modules?
-      (modules_cli? || modules?) && config_reload_automatic?
-    end
-  end
-end end end

data/lib/logstash/config/string_escape.rb

@@ -1,27 +0,0 @@
-
-
-module LogStash; module Config; module StringEscape
-  class << self
-    def process_escapes(input)
-      input.gsub(/\\./) do |value|
-        process(value)
-      end
-    end
-
-    private
-    def process(value)
-      case value[1]
-      when '"', "'", "\\"
-        value[1]
-      when "n"
-        "\n"
-      when "r"
-        "\r"
-      when "t"
-        "\t"
-      else
-        value
-      end
-    end
-  end
-end end end

data/lib/logstash/dependency_report.rb

@@ -1,131 +0,0 @@
-# encoding: utf-8
-Thread.abort_on_exception = true
-Encoding.default_external = Encoding::UTF_8
-$DEBUGLIST = (ENV["DEBUG"] || "").split(",")
-
-require "clamp"
-require "logstash/namespace"
-require "rubygems"
-require "jars/gemspec_artifacts"
-
-class LogStash::DependencyReport < Clamp::Command
-  option [ "--csv" ], "OUTPUT_PATH", "The path to write the dependency report in csv format.",
-    :required => true, :attribute_name => :output_path
-
-  def execute
-    require "csv"
-    CSV.open(output_path, "wb", :headers => [ "name", "version", "url", "license" ], :write_headers => true) do |csv|
-      puts "Finding gem dependencies"
-      gems.each { |d| csv << d }
-      puts "Finding java/jar dependencies"
-      jars.each { |d| csv << d }
-    end
-
-    # Copy in COPYING.csv which is a best-effort, hand-maintained file of dependency license information.
-    File.open(output_path, "a+") do |file|
-      extra = File.join(File.dirname(__FILE__), "..", "..", "..", "COPYING.csv")
-      file.write(IO.read(extra))
-    end
-    nil
-  end
-
-  def gems
-    # @mgreau requested `logstash-*` dependencies be removed from this list: 
-    # https://github.com/elastic/logstash/pull/8837#issuecomment-351859433
-    Gem::Specification.reject { |g| g.name =~ /^logstash-/ }.collect do |gem|
-      licenses = ("UNKNOWN" if gem.licenses.empty?) || (gem.licenses.map { |l| SPDX.map(l) }.join("|") if !gem.licenses.empty?)
-      [gem.name, gem.version.to_s, gem.homepage, licenses]
-    end
-  end
-
-  def jars
-    jars = []
-    # For any gems with jar dependencies,
-    #   Look at META-INF/MANIFEST.MF for any jars in each gem
-    #   Note any important details.
-    Gem::Specification.select { |g| g.requirements && g.requirements.any? { |r| r =~ /^jar / } }.collect do |gem|
-
-      # Where is the gem installed
-      root = gem.full_gem_path
-
-      Dir.glob(File.join(root, "**", "*.jar")).collect do |path|
-        jar = java.util.jar.JarFile.new(path)
-        manifest = jar.getManifest
-
-        pom_entries = jar.entries.select { |t| t.getName.start_with?("META-INF/maven/") && t.getName.end_with?("/pom.properties") }
-
-        # Some jar files have multiple maven pom.properties files. It is unclear how to know what is correct?
-        # TODO(sissel): Maybe we should use all pom.properties files? None of the pom.properties/pom.xml files have license information, though.
-        # TODO(sissel): In some cases, there are META-INF/COPYING and
-        #   META-INF/NOTICE.txt files? Can we use these somehow? There is no
-        #   common syntax for parsing these files, though...
-        pom_map = if pom_entries.count == 1
-          pom_in = jar.getInputStream(pom_entries.first)
-          pom_content = pom_in.available.times.collect { pom_in.read }.pack("C*")
-          # Split non-comment lines by `key=val` into a map { key => val }
-          Hash[pom_content.split(/\r?\n/).grep(/^[^#]/).map { |line| line.split("=", 2) }]
-        else
-          {}
-        end
-
-        next if manifest.nil?
-        # convert manifest attributes to a map w/ keys .to_s
-        # without this, the attribute keys will be `Object#inspect` values
-        # like #<Java::JavaUtilJar::Attributes::Name0xabcdef0>
-        attributes = Hash[manifest.getMainAttributes.map { |k,v| [k.to_s, v] }]
-
-        begin
-          # Prefer the maven/pom groupId when it is available.
-          artifact = pom_map.fetch("artifactId", attributes.fetch("Implementation-Title"))
-          group = pom_map.fetch("groupId", attributes.fetch("Implementation-Vendor-Id"))
-          jars << [
-            group + ":" + artifact,
-            attributes.fetch("Bundle-Version"),
-            attributes.fetch("Bundle-DocURL"),
-            SPDX.map(attributes.fetch("Bundle-License")),
-          ]
-        rescue KeyError => e
-          # The jar is missing a required manifest field, it may not have any useful manifest data.
-          # Ignore it and move on.
-        end
-      end
-    end
-    jars.uniq.sort
-  end
-
-  module SPDX
-    # This is a non-exhaustive, best effort list of licenses as they map to SPDX identifiers.
-    ALIASES = {
-      "Apache-2.0" => [
-        "Apache 2",
-        "apache-2.0",
-        "Apache 2.0",
-        "Apache License (2.0)",
-        "Apache License 2.0",
-        "https://www.apache.org/licenses/LICENSE-2.0.txt",
-        "http://www.apache.org/licenses/LICENSE-2.0.txt",
-      ],
-      "Artistic-2.0" => [
-        "Artistic 2.0"
-      ],
-      "BSD-2-Clause" => [
-        "2-clause BSDL",
-        "2-clause"
-      ],
-      "GPL-2.0" => [
-        "GPL-2"
-      ]
-    }
-
-    # Get a map of name => spdx
-    MAP_APACHE2 = Hash[ALIASES.map { |spdx,aliases| aliases.map { |value| [value, spdx] } }[0]]
-    MAP_ARTISTIC2 = Hash[ALIASES.map { |spdx,aliases| aliases.map { |value| [value, spdx] } }[1]]
-    MAP_BSD = Hash[ALIASES.map { |spdx,aliases| aliases.map { |value| [value, spdx] } }[2]]
-    MAP_GPL2 = Hash[ALIASES.map { |spdx,aliases| aliases.map { |value| [value, spdx] } }[3]]
-
-    module_function
-    def map(value)
-      MAP_APACHE2[value] ||  MAP_ARTISTIC2[value] || MAP_BSD[value] ||  MAP_GPL2[value] || value
-    end
-  end
-end

data/lib/logstash/dependency_report_runner.rb

@@ -1,17 +0,0 @@
-require_relative "../../../lib/bootstrap/environment"
-
-if $0 == __FILE__
-  begin
-    LogStash::Bundler.setup!({:without => [:build, :development]})
-  rescue => Bundler::GemfileNotFound
-    $stderr.puts("No Gemfile found. Maybe you need to run `rake artifact:tar`?")
-    raise
-  end
-
-  require "logstash/namespace"
-  require_relative "../../../lib/bootstrap/patches/jar_dependencies"
-  require "logstash/dependency_report"
-
-  exit_status = LogStash::DependencyReport.run 
-  exit(exit_status || 0)
-end

data/lib/logstash/elasticsearch_client.rb

@@ -1,142 +0,0 @@
-# encoding: utf-8
-require "logstash/namespace"
-require "logstash/logging"
-require "elasticsearch"
-require "elasticsearch/transport/transport/http/manticore"
-
-module LogStash class ElasticsearchClient
-  include LogStash::Util::Loggable
-
-  class Response
-    # duplicated here from Elasticsearch::Transport::Transport::Response
-    # to create a normalised response across different client IMPL
-    attr_reader :status, :body, :headers
-    def initialize(status, body, headers={})
-      @status, @body, @headers = status, body, headers
-      @body = body.force_encoding('UTF-8') if body.respond_to?(:force_encoding)
-    end
-  end
-
-  def self.build(settings)
-    new(RubyClient.new(settings, logger))
-  end
-
-  class RubyClient
-    def initialize(settings, logger)
-      @settings = settings
-      @logger = logger
-      @client_args = client_args
-
-      ssl_options = {}
-
-      if @settings["var.elasticsearch.ssl.enabled"] == "true"
-        ssl_options[:verify] = @settings.fetch("var.elasticsearch.ssl.verification_mode", true)
-        ssl_options[:ca_file] = @settings.fetch("var.elasticsearch.ssl.certificate_authority", nil)
-        ssl_options[:client_cert] = @settings.fetch("var.elasticsearch.ssl.certificate", nil)
-        ssl_options[:client_key] = @settings.fetch("var.elasticsearch.ssl.key", nil)
-      end
-
-      @client_args[:ssl] = ssl_options
-
-      username = @settings["var.elasticsearch.username"]
-      password = @settings["var.elasticsearch.password"]
-      if username
-        @client_args[:transport_options] = { :headers => { "Authorization" => 'Basic ' + Base64.encode64( "#{username}:#{password}" ).chomp } }
-      end
-
-      @client = Elasticsearch::Client.new(@client_args)
-    end
-
-    def can_connect?
-      begin
-        head(SecureRandom.hex(32).prepend('_'))
-      rescue Elasticsearch::Transport::Transport::Errors::BadRequest
-        true
-      rescue Manticore::SocketException
-        false
-      end
-    end
-
-    def host_settings
-      @client_args[:hosts]
-    end
-
-    def delete(path)
-      begin
-        normalize_response(@client.perform_request('DELETE', path, {}, nil))
-      rescue Exception => e
-        if is_404_error?(e)
-          Response.new(404, "", {})
-        else
-          raise e
-        end
-      end
-    end
-
-    def put(path, content)
-      normalize_response(@client.perform_request('PUT', path, {}, content))
-    end
-
-    def head(path)
-      begin
-        normalize_response(@client.perform_request('HEAD', path, {}, nil))
-      rescue Exception => e
-        if is_404_error?(e)
-          Response.new(404, "", {})
-        else
-          raise e
-        end
-      end
-    end
-
-    private
-
-    def is_404_error?(error)
-      error.class.to_s =~ /NotFound/ || error.message =~ /Not\s*Found|404/i
-    end
-
-    def normalize_response(response)
-      Response.new(response.status, response.body, response.headers)
-    end
-
-    def client_args
-      {
-        :transport_class => Elasticsearch::Transport::Transport::HTTP::Manticore,
-        :hosts => [*unpack_hosts],
-        # :logger => @logger, # silence the client logging
-      }
-    end
-
-    def unpack_hosts
-      setting = @settings.fetch("var.elasticsearch.hosts", "localhost:9200")
-      if setting.is_a?(String)
-        return setting.split(',').map(&:strip)
-      end
-      setting
-    end
-  end
-
-  def initialize(client)
-    @client = client
-  end
-
-  def delete(path)
-    @client.delete(path)
-  end
-
-  def put(path, content)
-    @client.put(path, content)
-  end
-
-  def head(path)
-    @client.head(path)
-  end
-
-  def can_connect?
-    @client.can_connect?
-  end
-
-  def host_settings
-    @client.host_settings
-  end
-end end