logstash-core 6.0.0.rc1-java → 6.0.0.rc2-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7f2f575a0b4346cfdb2ebd45a8f92c1f945af334576955f88db79b39e49d92aa
-  data.tar.gz: e6054d7cbf6db8a5e88554647b09f92b4781809f06f44f62e42987842e0b74be
+  metadata.gz: be7703ae1e02681bf68ba3dfd591f85886b15f271b2bce6cc17e49dddcd88c4c
+  data.tar.gz: 7c49c07b0af3dd06f605acb08243911e65f1be78108ee6fdb1cc4723dca694a6
 SHA512:
-  metadata.gz: 55addb91d39a5878c13ef3ea22e9e9d8dd8b543750e1248e6d1e6d165812169c7b2522a86640d85dcebeb949de6b48e9049f68c339bbe34798bba6694365e02c
-  data.tar.gz: 70d378ebbd7e89728069139a7a010f977ebf7cd2067cbb4ca2573e0cb481e94c4f69733a2af4622ad805f0ace5588642d7655c68381e56c3b161bfa14ef57f3e
+  metadata.gz: dc9b5bc1db75cfb1236aef3072ac38f2d943fb24aa752fd9a525bec60391e06f000f77d7fd8ac48d00744a0ff249c5edd6e7bec1c87ee7475d471999e4e4c032
+  data.tar.gz: e856579b5da5a494d8493e6863a96f6ced47e7442ce822bc94144a3dcf39ef0f5d3d5a5f9f99fafceb0ca1d070da0975ed6d33682b8191eb653c1dbc22227fed

data/gemspec_jars.rb

@@ -5,8 +5,8 @@
 gem.requirements << "jar org.apache.logging.log4j:log4j-slf4j-impl, 2.6.2"
 gem.requirements << "jar org.apache.logging.log4j:log4j-api, 2.6.2"
 gem.requirements << "jar org.apache.logging.log4j:log4j-core, 2.6.2"
-gem.requirements << "jar com.fasterxml.jackson.core:jackson-core, 2.7.3"
-gem.requirements << "jar com.fasterxml.jackson.core:jackson-databind, 2.7.3"
-gem.requirements << "jar com.fasterxml.jackson.core:jackson-annotations, 2.7.3"
-gem.requirements << "jar com.fasterxml.jackson.module:jackson-module-afterburner, 2.7.3"
-gem.requirements << "jar com.fasterxml.jackson.dataformat:jackson-dataformat-cbor, 2.7.3"
+gem.requirements << "jar com.fasterxml.jackson.core:jackson-core, 2.9.1"
+gem.requirements << "jar com.fasterxml.jackson.core:jackson-databind, 2.9.1"
+gem.requirements << "jar com.fasterxml.jackson.core:jackson-annotations, 2.9.1"
+gem.requirements << "jar com.fasterxml.jackson.module:jackson-module-afterburner, 2.9.1"
+gem.requirements << "jar com.fasterxml.jackson.dataformat:jackson-dataformat-cbor, 2.9.1"

data/lib/logstash-core/version.rb

@@ -2,7 +2,11 @@
 
 # The version of logstash core gem.
 #
-# Note to authors: this should not include dashes because 'gem' barfs if
-# you include a dash in the version string.
-
-LOGSTASH_CORE_VERSION = "6.0.0-rc1"
+# sourced from a copy of the master versions.yml file, see logstash-core/logstash-core.gemspec
+if !defined?(ALL_VERSIONS)
+  require 'yaml'
+  ALL_VERSIONS = YAML.load_file(File.expand_path("../../versions-gem-copy.yml", File.dirname(__FILE__)))
+end
+if !defined?(LOGSTASH_CORE_VERSION)
+  LOGSTASH_CORE_VERSION = ALL_VERSIONS.fetch("logstash-core")
+end

data/lib/logstash-core_jars.rb

@@ -2,25 +2,25 @@
 begin
   require 'jar_dependencies'
 rescue LoadError
-  require 'com/fasterxml/jackson/core/jackson-databind/2.7.3/jackson-databind-2.7.3.jar'
   require 'org/apache/logging/log4j/log4j-core/2.6.2/log4j-core-2.6.2.jar'
+  require 'com/fasterxml/jackson/core/jackson-databind/2.9.1/jackson-databind-2.9.1.jar'
   require 'org/apache/logging/log4j/log4j-api/2.6.2/log4j-api-2.6.2.jar'
   require 'org/slf4j/slf4j-api/1.7.21/slf4j-api-1.7.21.jar'
-  require 'com/fasterxml/jackson/core/jackson-annotations/2.7.3/jackson-annotations-2.7.3.jar'
+  require 'com/fasterxml/jackson/core/jackson-annotations/2.9.1/jackson-annotations-2.9.1.jar'
   require 'org/apache/logging/log4j/log4j-slf4j-impl/2.6.2/log4j-slf4j-impl-2.6.2.jar'
-  require 'com/fasterxml/jackson/module/jackson-module-afterburner/2.7.3/jackson-module-afterburner-2.7.3.jar'
-  require 'com/fasterxml/jackson/dataformat/jackson-dataformat-cbor/2.7.3/jackson-dataformat-cbor-2.7.3.jar'
-  require 'com/fasterxml/jackson/core/jackson-core/2.7.3/jackson-core-2.7.3.jar'
+  require 'com/fasterxml/jackson/module/jackson-module-afterburner/2.9.1/jackson-module-afterburner-2.9.1.jar'
+  require 'com/fasterxml/jackson/dataformat/jackson-dataformat-cbor/2.9.1/jackson-dataformat-cbor-2.9.1.jar'
+  require 'com/fasterxml/jackson/core/jackson-core/2.9.1/jackson-core-2.9.1.jar'
 end
 
 if defined? Jars
-  require_jar( 'com.fasterxml.jackson.core', 'jackson-databind', '2.7.3' )
   require_jar( 'org.apache.logging.log4j', 'log4j-core', '2.6.2' )
+  require_jar( 'com.fasterxml.jackson.core', 'jackson-databind', '2.9.1' )
   require_jar( 'org.apache.logging.log4j', 'log4j-api', '2.6.2' )
   require_jar( 'org.slf4j', 'slf4j-api', '1.7.21' )
-  require_jar( 'com.fasterxml.jackson.core', 'jackson-annotations', '2.7.3' )
+  require_jar( 'com.fasterxml.jackson.core', 'jackson-annotations', '2.9.1' )
   require_jar( 'org.apache.logging.log4j', 'log4j-slf4j-impl', '2.6.2' )
-  require_jar( 'com.fasterxml.jackson.module', 'jackson-module-afterburner', '2.7.3' )
-  require_jar( 'com.fasterxml.jackson.dataformat', 'jackson-dataformat-cbor', '2.7.3' )
-  require_jar( 'com.fasterxml.jackson.core', 'jackson-core', '2.7.3' )
+  require_jar( 'com.fasterxml.jackson.module', 'jackson-module-afterburner', '2.9.1' )
+  require_jar( 'com.fasterxml.jackson.dataformat', 'jackson-dataformat-cbor', '2.9.1' )
+  require_jar( 'com.fasterxml.jackson.core', 'jackson-core', '2.9.1' )
 end

data/lib/logstash/config/config_ast.rb

@@ -77,8 +77,9 @@ module LogStash; module Config; module AST
         # of the output/filter function
         definitions << "define_singleton_method :#{type}_func do |event|"
         definitions << "  targeted_outputs = []" if type == "output"
-        definitions << "  events = [event]" if type == "filter"
-        definitions << "  @logger.debug? && @logger.debug(\"#{type} received\", \"event\" => event.to_hash)"
+        definitions << "  events = event" if type == "filter"
+        definitions << "  @logger.debug? && @logger.debug(\"#{type} received\", \"event\" => event.to_hash)" if type == "output"
+        definitions << "  @logger.debug? && events.each { |e| @logger.debug(\"#{type} received\", \"event\" => e.to_hash)}" if type == "filter"
 
         sections.select { |s| s.plugin_type.text_value == type }.each do |s|
           definitions << s.compile.split("\n", -1).map { |e| "  #{e}" }

data/lib/logstash/config/modules_common.rb

@@ -65,6 +65,7 @@ module LogStash module Config
           pipeline_id = alt_name
           module_settings.set("pipeline.id", pipeline_id)
           LogStash::Modules::SettingsMerger.merge_cloud_settings(module_hash, module_settings)
+          LogStash::Modules::SettingsMerger.merge_kibana_auth!(module_hash)
           current_module.with_settings(module_hash)
           config_test = settings.get("config.test_and_exit")
           modul_setup = settings.get("modules_setup")

data/lib/logstash/instrument/periodic_poller/dlq.rb

@@ -10,10 +10,15 @@ module LogStash module Instrument module PeriodicPoller
     end
 
     def collect
-      _, pipeline = @agent.with_running_pipelines { |pipelines| pipelines.first }
-      unless pipeline.nil?
-        pipeline.collect_dlq_stats
+      pipelines = @agent.with_running_user_defined_pipelines {|pipelines| pipelines}
+      unless pipelines.nil?
+        pipelines.each {|_, pipeline|
+          unless pipeline.nil?
+            pipeline.collect_dlq_stats
+          end
+        }
       end
     end
   end
 end end end
+

data/lib/logstash/instrument/periodic_poller/pq.rb

@@ -11,9 +11,13 @@ module LogStash module Instrument module PeriodicPoller
     end
 
     def collect
-      pipeline_id, pipeline = @agent.with_running_pipelines {|pipelines| pipelines.first }
-      unless pipeline.nil?
-        pipeline.collect_stats
+      pipelines = @agent.with_running_user_defined_pipelines {|pipelines| pipelines}
+      unless pipelines.nil?
+        pipelines.each {|_, pipeline|
+          unless pipeline.nil?
+            pipeline.collect_stats
+          end
+        }
       end
     end
   end

data/lib/logstash/modules/kibana_client.rb

@@ -24,9 +24,11 @@ module LogStash module Modules class KibanaClient
     end
   end
 
-  attr_reader :version
+  SCHEME_REGEX = /^https?$/
 
-  def initialize(settings)
+  attr_reader :version, :endpoint
+
+  def initialize(settings, client = nil) # allow for test mock injection
     @settings = settings
 
     client_options = {
@@ -38,8 +40,8 @@ module LogStash module Modules class KibanaClient
     }
 
     ssl_options = {}
-
-    if @settings["var.kibana.ssl.enabled"] == "true"
+    ssl_enabled = @settings["var.kibana.ssl.enabled"] == "true"
+    if ssl_enabled
       ssl_options[:verify] = @settings.fetch("var.kibana.ssl.verification_mode", "strict").to_sym
       ssl_options[:ca_file] = @settings.fetch("var.kibana.ssl.certificate_authority", nil)
       ssl_options[:client_cert] = @settings.fetch("var.kibana.ssl.certificate", nil)
@@ -48,9 +50,34 @@ module LogStash module Modules class KibanaClient
 
     client_options[:ssl] = ssl_options
 
-    @client = Manticore::Client.new(client_options)
     @host = @settings.fetch("var.kibana.host", "localhost:5601")
-    @scheme = @settings.fetch("var.kibana.scheme", "http")
+    implicit_scheme, colon_slash_slash, host = @host.partition("://")
+    explicit_scheme = @settings["var.kibana.scheme"]
+    @scheme = "http"
+    if !colon_slash_slash.empty?
+      if !explicit_scheme.nil? && implicit_scheme != explicit_scheme
+        # both are set and not the same - error
+        msg = sprintf("Detected differing Kibana host schemes as sourced from var.kibana.host: '%s' and var.kibana.scheme: '%s'", implicit_scheme, explicit_scheme)
+        raise ArgumentError.new(msg)
+      end
+      @scheme = implicit_scheme
+      @host = host
+    elsif !explicit_scheme.nil?
+      @scheme = explicit_scheme
+    end
+
+    if SCHEME_REGEX.match(@scheme).nil?
+      msg = sprintf("Kibana host scheme given is invalid, given value: '%s' - acceptable values: 'http', 'https'", @scheme)
+      raise ArgumentError.new(msg)
+    end
+
+    if ssl_enabled && @scheme != "https"
+      @scheme = "https"
+    end
+
+    @endpoint = "#{@scheme}://#{@host}"
+
+    @client = client || Manticore::Client.new(client_options)
     @http_options = {:headers => {'Content-Type' => 'application/json'}}
     username = @settings["var.kibana.username"]
     if username
@@ -77,7 +104,7 @@ module LogStash module Modules class KibanaClient
   end
 
   def version_parts
-    @version.split(/\.|\-/)
+    @version.split(/[.-]/)
   end
 
   def host_settings
@@ -119,6 +146,6 @@ module LogStash module Modules class KibanaClient
   end
 
   def full_url(relative)
-    "#{@scheme}://#{@host}/#{relative}"
+    "#{@endpoint}/#{relative}"
   end
 end end end

data/lib/logstash/modules/settings_merger.rb

@@ -40,9 +40,10 @@ module LogStash module Modules module SettingsMerger
       settings_copy = LogStash::Util.deep_clone(module_settings)
     end
 
-    module_settings["var.kibana.scheme"] = "https"
+    module_settings["var.kibana.scheme"] = cloud_id.kibana_scheme
     module_settings["var.kibana.host"] = cloud_id.kibana_host
-    module_settings["var.elasticsearch.hosts"] = cloud_id.elasticsearch_host
+    # elasticsearch client does not use scheme, it URI parses the host setting
+    module_settings["var.elasticsearch.hosts"] = "#{cloud_id.elasticsearch_scheme}://#{cloud_id.elasticsearch_host}"
     unless cloud_auth.nil?
       module_settings["var.elasticsearch.username"] = cloud_auth.username
       module_settings["var.elasticsearch.password"] = cloud_auth.password
@@ -54,6 +55,11 @@ module LogStash module Modules module SettingsMerger
     end
   end
 
+  def merge_kibana_auth!(module_settings)
+    module_settings["var.kibana.username"] = module_settings["var.elasticsearch.username"] if module_settings["var.kibana.username"].nil?
+    module_settings["var.kibana.password"] = module_settings["var.elasticsearch.password"] if module_settings["var.kibana.password"].nil?
+  end
+
   def format_module_settings(settings_before, settings_after)
     output = []
     output << "-------- Module Settings ---------"

data/lib/logstash/pipeline.rb

@@ -498,13 +498,9 @@ module LogStash; class Pipeline < BasePipeline
   end
 
   def filter_batch(batch)
-    batch.each do |event|
-      return if @force_shutdown.true?
-
-      filter_func(event).each do |e|
-        #these are both original and generated events
-        batch.merge(e) unless e.cancelled?
-      end
+    filter_func(batch.to_a).each do |e|
+      #these are both original and generated events
+      batch.merge(e) unless e.cancelled?
     end
     @filter_queue_client.add_filtered_metrics(batch)
     @events_filtered.increment(batch.size)
@@ -662,7 +658,7 @@ module LogStash; class Pipeline < BasePipeline
   def filter(event, &block)
     maybe_setup_out_plugins
     # filter_func returns all filtered events, including cancelled ones
-    filter_func(event).each {|e| block.call(e)}
+    filter_func([event]).each {|e| block.call(e)}
   end
 
   # perform filters flush and yield flushed event to the passed block

data/lib/logstash/util/cloud_setting_id.rb

@@ -3,8 +3,26 @@ require "logstash/namespace"
 require "base64"
 
 module LogStash module Util class CloudSettingId
-  attr_reader :original, :decoded, :label, :elasticsearch_host, :kibana_host
 
+  def self.cloud_id_encode(*args)
+    Base64.urlsafe_encode64(args.join("$"))
+  end
+  DOT_SEPARATOR = "."
+  CLOUD_PORT = ":443"
+
+  attr_reader :original, :decoded, :label, :elasticsearch_host, :elasticsearch_scheme, :kibana_host, :kibana_scheme
+
+  # The constructor is expecting a 'cloud.id', a string in 2 variants.
+  # 1 part example: 'dXMtZWFzdC0xLmF3cy5mb3VuZC5pbyRub3RhcmVhbCRpZGVudGlmaWVy'
+  # 2 part example: 'foobar:dXMtZWFzdC0xLmF3cy5mb3VuZC5pbyRub3RhcmVhbCRpZGVudGlmaWVy'
+  # The two part variant has a 'label' prepended with a colon separator. The label is not encoded.
+  # The 1 part (or second section of the 2 part variant) is base64 encoded.
+  # The original string before encoding has three segments separated by a dollar sign.
+  # e.g. 'us-east-1.aws.found.io$notareal$identifier'
+  # The first segment is the cloud base url, e.g. 'us-east-1.aws.found.io'
+  # The second segment is the elasticsearch host identifier, e.g. 'notareal'
+  # The third segment is the kibana host identifier, e.g. 'identifier'
+  # The 'cloud.id' value decoded into the #attr_reader ivars.
   def initialize(value)
     return if value.nil?
 
@@ -12,27 +30,43 @@ module LogStash module Util class CloudSettingId
       raise ArgumentError.new("Cloud Id must be String. Received: #{value.class}")
     end
     @original = value
-    @label, sep, last = value.partition(":")
-    if last.empty?
+    @label, colon, encoded = @original.partition(":")
+    if encoded.empty?
       @decoded = Base64.urlsafe_decode64(@label) rescue ""
       @label = ""
     else
-      @decoded = Base64.urlsafe_decode64(last) rescue ""
+      @decoded = Base64.urlsafe_decode64(encoded) rescue ""
     end
+
+    @decoded = @decoded.encode(Encoding::UTF_8, :invalid => :replace, :undef => :replace)
+
     unless @decoded.count("$") == 2
-      raise ArgumentError.new("Cloud Id does not decode. Received: \"#{@original}\".")
+      raise ArgumentError.new("Cloud Id does not decode. You may need to enable Kibana in the Cloud UI. Received: \"#{@decoded}\".")
     end
-    parts = @decoded.split("$")
-    if parts.any?(&:empty?)
-      raise ArgumentError.new("Cloud Id, after decoding, is invalid. Format: '<part1>$<part2>$<part3>'. Received: \"#{@decoded}\".")
+
+    segments = @decoded.split("$")
+    if segments.any?(&:empty?)
+      raise ArgumentError.new("Cloud Id, after decoding, is invalid. Format: '<segment1>$<segment2>$<segment3>'. Received: \"#{@decoded}\".")
+    end
+    cloud_base = segments.shift
+    cloud_host = "#{DOT_SEPARATOR}#{cloud_base}#{CLOUD_PORT}"
+
+    @elasticsearch_host, @kibana_host = segments
+    if @elasticsearch_host == "undefined"
+      raise ArgumentError.new("Cloud Id, after decoding, elasticsearch segment is 'undefined', literally.")
+    end
+    @elasticsearch_scheme = "https"
+    @elasticsearch_host.concat(cloud_host)
+
+    if @kibana_host == "undefined"
+      raise ArgumentError.new("Cloud Id, after decoding, the kibana segment is 'undefined', literally. You may need to enable Kibana in the Cloud UI.")
     end
-    cloud_host, es_server, kb_server = parts
-    @elasticsearch_host = sprintf("%s.%s:443", es_server, cloud_host)
-    @kibana_host  = sprintf("%s.%s:443", kb_server, cloud_host)
+    @kibana_scheme = "https"
+    @kibana_host.concat(cloud_host)
   end
 
   def to_s
-    @original.to_s
+    @decoded.to_s
   end
 
   def inspect

data/lib/logstash/util/wrapped_acked_queue.rb

@@ -287,6 +287,12 @@ module LogStash; module Util
         # @cancelled[event] = true
       end
 
+      def to_a
+        events = []
+        each {|e| events << e}
+        events
+      end
+
       def each(&blk)
         # take care not to cause @originals or @generated to change during iteration
 

data/lib/logstash/util/wrapped_synchronous_queue.rb

@@ -218,6 +218,12 @@ module LogStash; module Util
         # @cancelled[event] = true
       end
 
+      def to_a
+        events = []
+        each {|e| events << e}
+        events
+      end
+
       def each(&blk)
         # take care not to cause @originals or @generated to change during iteration
         @iterating = true

data/lib/logstash/version.rb

@@ -2,13 +2,11 @@
 
 # The version of the logstash package (not the logstash-core gem version).
 #
-# Note to authors: this should not include dashes because 'gem' barfs if
-# you include a dash in the version string.
-
-# TODO: (colin) the logstash-core gem uses it's own version number in logstash-core/lib/logstash-core/version.rb
-#       there are some dependencies in logstash-core on the LOGSTASH_VERSION constant this is why
-#       the logstash version is currently defined here in logstash-core/lib/logstash/version.rb but
-#       eventually this file should be in the root logstash lib fir and dependencies in logstash-core should be
-#       fixed.
-
-LOGSTASH_VERSION = "6.0.0-rc1"
+# sourced from a copy of the master versions.yml file, see logstash-core/logstash-core.gemspec
+if !defined?(ALL_VERSIONS)
+  require 'yaml'
+  ALL_VERSIONS = YAML.load_file(File.expand_path("../../versions-gem-copy.yml", File.dirname(__FILE__)))
+end
+if !defined?(LOGSTASH_VERSION)
+  LOGSTASH_VERSION = ALL_VERSIONS.fetch("logstash")
+end

data/logstash-core.gemspec

@@ -1,6 +1,25 @@
 # -*- encoding: utf-8 -*-
 lib = File.expand_path('../lib', __FILE__)
 $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+
+project_versions_yaml_path = File.expand_path("../versions.yml", File.dirname(__FILE__))
+if File.exist?(project_versions_yaml_path)
+  # we need to copy the project level versions.yml into the gem root
+  # to be able to package it into the gems file structure
+  # as the require 'logstash-core/version' loads the yaml file from within the gem root.
+  #
+  # we ignore the copy in git and we overwrite an existing file
+  # each time we build the logstash-core gem
+  original_lines = IO.readlines(project_versions_yaml_path)
+  original_lines << ""
+  original_lines << "# This is a copy the project level versions.yml into this gem's root and it is created when the gemspec is evaluated."
+  gem_versions_yaml_path = File.expand_path("./versions-gem-copy.yml", File.dirname(__FILE__))
+  File.open(gem_versions_yaml_path, 'w') do |new_file|
+    # create or overwrite
+    new_file.puts(original_lines)
+  end
+end
+
 require 'logstash-core/version'
 
 Gem::Specification.new do |gem|
@@ -11,7 +30,10 @@ Gem::Specification.new do |gem|
   gem.homepage      = "http://www.elastic.co/guide/en/logstash/current/index.html"
   gem.license       = "Apache License (2.0)"
 
-  gem.files         = Dir.glob(["logstash-core.gemspec", "gemspec_jars.rb", "lib/**/*.rb", "spec/**/*.rb", "locales/*", "lib/logstash/api/init.ru", "lib/logstash-core/logstash-core.jar"])
+  gem.files         = Dir.glob(
+    %w(versions-gem-copy.yml logstash-core.gemspec gemspec_jars.rb lib/**/*.rb spec/**/*.rb locales/*
+    lib/logstash/api/init.ru lib/logstash-core/logstash-core.jar)
+  )
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.name          = "logstash-core"
   gem.require_paths = ["lib"]
@@ -46,7 +68,7 @@ Gem::Specification.new do |gem|
   gem.add_runtime_dependency "rubyzip", "~> 1.2.1"
   gem.add_runtime_dependency "thread_safe", "~> 0.3.5" #(Apache 2.0 license)
 
-  gem.add_runtime_dependency "jrjackson", "~> 0.4.2" #(Apache 2.0 license)
+  gem.add_runtime_dependency "jrjackson", "~> #{ALL_VERSIONS.fetch('jrjackson')}" #(Apache 2.0 license)
 
   gem.add_runtime_dependency "jar-dependencies"
   # as of Feb 3rd 2016, the ruby-maven gem is resolved to version 3.3.3 and that version

data/spec/logstash/modules/kibana_client_spec.rb

@@ -0,0 +1,60 @@
+# encoding: utf-8
+require "spec_helper"
+require "logstash/modules/kibana_client"
+module LogStash module Modules
+  KibanaTestResponse = Struct.new(:code, :body, :headers)
+  class KibanaTestClient
+    def http(method, endpoint, options)
+      self
+    end
+    def call
+      KibanaTestResponse.new(200, '{"version":{"number":"1.2.3","build_snapshot":false}}', {})
+    end
+  end
+  describe KibanaClient do
+    let(:settings) { Hash.new }
+    let(:test_client) { KibanaTestClient.new }
+    let(:kibana_host) { "https://foo.bar:4321" }
+    subject(:kibana_client) { described_class.new(settings, test_client) }
+
+    context "when supplied with conflicting scheme data" do
+      let(:settings) { {"var.kibana.scheme" => "http", "var.kibana.host" => kibana_host} }
+      it "a new instance will throw an error" do
+        expect{described_class.new(settings, test_client)}.to raise_error(ArgumentError, /Detected differing Kibana host schemes as sourced from var\.kibana\.host: 'https' and var\.kibana\.scheme: 'http'/)
+      end
+    end
+
+    context "when supplied with invalid schemes" do
+      ["httpd", "ftp", "telnet"].each do |uri_scheme|
+        it "a new instance will throw an error" do
+          re = /Kibana host scheme given is invalid, given value: '#{uri_scheme}' - acceptable values: 'http', 'https'/
+          expect{described_class.new({"var.kibana.scheme" => uri_scheme}, test_client)}.to raise_error(ArgumentError, re)
+        end
+      end
+    end
+
+    context "when supplied with the scheme in the host only" do
+      let(:settings) { {"var.kibana.host" => kibana_host} }
+      it "has a version and an endpoint" do
+        expect(kibana_client.version).to eq("1.2.3")
+        expect(kibana_client.endpoint).to eq("https://foo.bar:4321")
+      end
+    end
+
+    context "when supplied with the scheme in the scheme setting" do
+      let(:settings) { {"var.kibana.scheme" => "https", "var.kibana.host" => "foo.bar:4321"} }
+      it "has a version and an endpoint" do
+        expect(kibana_client.version).to eq("1.2.3")
+        expect(kibana_client.endpoint).to eq(kibana_host)
+      end
+    end
+
+    context "when supplied with a no scheme host setting and ssl is enabled" do
+      let(:settings) { {"var.kibana.ssl.enabled" => "true", "var.kibana.host" => "foo.bar:4321"} }
+      it "has a version and an endpoint" do
+        expect(kibana_client.version).to eq("1.2.3")
+        expect(kibana_client.endpoint).to eq(kibana_host)
+      end
+    end
+  end
+end end

data/spec/logstash/modules/settings_merger_spec.rb

@@ -28,6 +28,36 @@ describe LogStash::Modules::SettingsMerger do
     end
   end
 
+  describe "#merge_kibana_auth" do
+
+    before do
+      described_class.merge_kibana_auth!(mod_settings)
+    end
+
+    context 'only elasticsearch username and password is set' do
+      let(:mod_settings) { {"name"=>"mod1", "var.input.tcp.port"=>2222, "var.elasticsearch.username"=>"rupert", "var.elasticsearch.password"=>"fotherington" } }
+      it "sets kibana username and password" do
+        expect(mod_settings["var.elasticsearch.username"]).to eq("rupert")
+        expect(mod_settings["var.elasticsearch.password"]).to eq("fotherington")
+        expect(mod_settings["var.kibana.username"]).to eq("rupert")
+        expect(mod_settings["var.kibana.password"]).to eq("fotherington")
+      end
+    end
+
+    context 'elasticsearch and kibana usernames and passwords are set' do
+      let(:mod_settings) { {"name"=>"mod1", "var.input.tcp.port"=>2222, "var.elasticsearch.username"=>"rupert", "var.elasticsearch.password"=>"fotherington",
+                                                               "var.kibana.username"=>"davey", "var.kibana.password"=>"stott"} }
+
+      it "keeps existing kibana username and password" do
+        expect(mod_settings["var.elasticsearch.username"]).to eq("rupert")
+        expect(mod_settings["var.elasticsearch.password"]).to eq("fotherington")
+        expect(mod_settings["var.kibana.username"]).to eq("davey")
+        expect(mod_settings["var.kibana.password"]).to eq("stott")
+      end
+    end
+
+  end
+
   describe "#merge_cloud_settings" do
     let(:cloud_id) { LogStash::Util::CloudSettingId.new("label:dXMtZWFzdC0xLmF3cy5mb3VuZC5pbyRub3RhcmVhbCRpZGVudGlmaWVy") }
     let(:cloud_auth) { LogStash::Util::CloudSettingAuth.new("elastix:bigwhoppingfairytail") }
@@ -38,7 +68,7 @@ describe LogStash::Modules::SettingsMerger do
         {
           "var.kibana.scheme" => "https",
           "var.kibana.host" => "identifier.us-east-1.aws.found.io:443",
-          "var.elasticsearch.hosts" => "notareal.us-east-1.aws.found.io:443",
+          "var.elasticsearch.hosts" => "https://notareal.us-east-1.aws.found.io:443",
           "var.elasticsearch.username" => "elastix",
           "var.kibana.username" => "elastix"
         }
@@ -63,7 +93,7 @@ describe LogStash::Modules::SettingsMerger do
         {
           "var.kibana.scheme" => "https",
           "var.kibana.host" => "identifier.us-east-1.aws.found.io:443",
-          "var.elasticsearch.hosts" => "notareal.us-east-1.aws.found.io:443",
+          "var.elasticsearch.hosts" => "https://notareal.us-east-1.aws.found.io:443",
         }
       end
       let(:ls_settings) { SubstituteSettingsForRSpec.new({"cloud.id" => cloud_id}) }

data/spec/logstash/pipeline_spec.rb

@@ -254,6 +254,14 @@ describe LogStash::Pipeline do
           pipeline = mock_pipeline_from_string(test_config_with_filters, pipeline_settings_obj)
           pipeline.close
         end
+
+        it "should log each filtered event if config.debug is set to true" do
+          pipeline_settings_obj.set("config.debug", true)
+          pipeline = mock_pipeline_from_string(test_config_with_filters, pipeline_settings_obj)
+          expect(logger).to receive(:debug).with(/filter received/, anything)
+          pipeline.filter_func([LogStash::Event.new])
+          pipeline.close
+        end
       end
 
       context "when there is no command line -w N set" do
@@ -618,9 +626,9 @@ describe LogStash::Pipeline do
 
     it "should handle evaluating different config" do
       expect(pipeline1.output_func(LogStash::Event.new)).not_to include(nil)
-      expect(pipeline1.filter_func(LogStash::Event.new)).not_to include(nil)
+      expect(pipeline1.filter_func([LogStash::Event.new])).not_to include(nil)
       expect(pipeline2.output_func(LogStash::Event.new)).not_to include(nil)
-      expect(pipeline1.filter_func(LogStash::Event.new)).not_to include(nil)
+      expect(pipeline1.filter_func([LogStash::Event.new])).not_to include(nil)
     end
   end
 
@@ -700,9 +708,9 @@ describe LogStash::Pipeline do
       # in the current instance and was returning an array containing nil values for
       # the match.
       expect(pipeline1.output_func(LogStash::Event.new)).not_to include(nil)
-      expect(pipeline1.filter_func(LogStash::Event.new)).not_to include(nil)
+      expect(pipeline1.filter_func([LogStash::Event.new])).not_to include(nil)
       expect(pipeline2.output_func(LogStash::Event.new)).not_to include(nil)
-      expect(pipeline1.filter_func(LogStash::Event.new)).not_to include(nil)
+      expect(pipeline1.filter_func([LogStash::Event.new])).not_to include(nil)
     end
   end
 

data/spec/logstash/settings/modules_spec.rb

@@ -53,7 +53,7 @@ describe LogStash::Setting::Modules do
     context "when given a badly formatted encoded id" do
       it "should not raise an error" do
         encoded = Base64.urlsafe_encode64("foo$$bal")
-        expect { subject.set(encoded) }.to raise_error(ArgumentError, /Cloud Id, after decoding, is invalid. Format: '<part1>\$<part2>\$<part3>'/)
+        expect { subject.set(encoded) }.to raise_error(ArgumentError, "Cloud Id, after decoding, is invalid. Format: '<segment1>$<segment2>$<segment3>'. Received: \"foo$$bal\".")
       end
     end
 

data/spec/logstash/util/cloud_setting_id_spec.rb

@@ -0,0 +1,93 @@
+# encoding: utf-8
+require "spec_helper"
+require "logstash/util/cloud_setting_id"
+
+describe LogStash::Util::CloudSettingId do
+  let(:input) { "foobar:dXMtZWFzdC0xLmF3cy5mb3VuZC5pbyRub3RhcmVhbCRpZGVudGlmaWVy" }
+  subject { described_class.new(input) }
+
+  describe "when given unacceptable input" do
+    it "a nil input does not raise an exception" do
+      expect{described_class.new(nil)}.not_to raise_exception
+    end
+    it "when given a nil input, the accessors are all nil" do
+      cloud_id = described_class.new(nil)
+      expect(cloud_id.original).to be_nil
+      expect(cloud_id.decoded).to be_nil
+      expect(cloud_id.label).to be_nil
+      expect(cloud_id.elasticsearch_host).to be_nil
+      expect(cloud_id.kibana_host).to be_nil
+      expect(cloud_id.elasticsearch_scheme).to be_nil
+      expect(cloud_id.kibana_scheme).to be_nil
+    end
+
+    context "when a malformed value is given" do
+      let(:raw) {%w(first second)}
+      let(:input) { described_class.cloud_id_encode(*raw) }
+      it "raises an error" do
+        expect{subject}.to raise_exception(ArgumentError, "Cloud Id does not decode. You may need to enable Kibana in the Cloud UI. Received: \"#{raw[0]}$#{raw[1]}\".")
+      end
+    end
+
+    context "when at least one segment is empty" do
+      let(:raw) {["first", "", "third"]}
+      let(:input) { described_class.cloud_id_encode(*raw) }
+      it "raises an error" do
+        expect{subject}.to raise_exception(ArgumentError, "Cloud Id, after decoding, is invalid. Format: '<segment1>$<segment2>$<segment3>'. Received: \"#{raw[0]}$#{raw[1]}$#{raw[2]}\".")
+      end
+    end
+
+    context "when elasticsearch segment is undefined" do
+      let(:raw) {%w(us-east-1.aws.found.io undefined my-kibana)}
+      let(:input) { described_class.cloud_id_encode(*raw) }
+      it "raises an error" do
+        expect{subject}.to raise_exception(ArgumentError, "Cloud Id, after decoding, elasticsearch segment is 'undefined', literally.")
+      end
+    end
+
+    context "when kibana segment is undefined" do
+      let(:raw) {%w(us-east-1.aws.found.io my-elastic-cluster undefined)}
+      let(:input) { described_class.cloud_id_encode(*raw) }
+      it "raises an error" do
+        expect{subject}.to raise_exception(ArgumentError, "Cloud Id, after decoding, the kibana segment is 'undefined', literally. You may need to enable Kibana in the Cloud UI.")
+      end
+    end
+  end
+
+  describe "without a label" do
+    let(:input) { "dXMtZWFzdC0xLmF3cy5mb3VuZC5pbyRub3RhcmVhbCRpZGVudGlmaWVy" }
+    it "#label is empty" do
+      expect(subject.label).to be_empty
+    end
+    it "#decode is set" do
+      expect(subject.decoded).to eq("us-east-1.aws.found.io$notareal$identifier")
+    end
+  end
+
+  describe "when given acceptable input, the accessors:" do
+    it '#original has a value' do
+      expect(subject.original).to eq(input)
+    end
+    it '#decoded has a value' do
+      expect(subject.decoded).to eq("us-east-1.aws.found.io$notareal$identifier")
+    end
+    it '#label has a value' do
+      expect(subject.label).to eq("foobar")
+    end
+    it '#elasticsearch_host has a value' do
+      expect(subject.elasticsearch_host).to eq("notareal.us-east-1.aws.found.io:443")
+    end
+    it '#elasticsearch_scheme has a value' do
+      expect(subject.elasticsearch_scheme).to eq("https")
+    end
+    it '#kibana_host has a value' do
+      expect(subject.kibana_host).to eq("identifier.us-east-1.aws.found.io:443")
+    end
+    it '#kibana_scheme has a value' do
+      expect(subject.kibana_scheme).to eq("https")
+    end
+    it '#to_s has a value of #decoded' do
+      expect(subject.to_s).to eq(subject.decoded)
+    end
+  end
+end

data/versions-gem-copy.yml

@@ -0,0 +1,23 @@
+---
+logstash: 6.0.0-rc2
+logstash-core: 6.0.0-rc2
+logstash-core-plugin-api: 2.1.16
+jruby:
+  version: 9.1.13.0
+  sha1: 815bac27d5daa1459a4477d6d80584f007ce6a68
+
+# jruby-runtime-override, if specified, will override the jruby version installed in vendor/jruby for logstash runtime only,
+# not for the compile-time jars
+#
+#jruby-runtime-override:
+#  url: http://ci.jruby.org/snapshots/previous/jruby-bin-9.1.13.0-SNAPSHOT.tar.gz
+#  version: 9.1.13.0-SNAPSHOT
+#  sha1: IGNORE
+
+# Note: this file is copied to the root of logstash-core because its gemspec needs it when
+#       bundler evaluates the gemspec via bin/logstash
+# Ensure Jackson version here is kept in sync with version used by jrjackson gem
+jrjackson: 0.4.4
+jackson: 2.9.1
+
+# This is a copy the project level versions.yml into this gem's root and it is created when the gemspec is evaluated.

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-core
 version: !ruby/object:Gem::Version
-  version: 6.0.0.rc1
+  version: 6.0.0.rc2
 platform: java
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-09-25 00:00:00.000000000 Z
+date: 2017-10-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -251,7 +251,7 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.4.2
+        version: 0.4.4
   name: jrjackson
   prerelease: false
   type: :runtime
@@ -259,7 +259,7 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.4.2
+        version: 0.4.4
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
@@ -556,6 +556,7 @@ files:
 - spec/logstash/legacy_ruby_event_spec.rb
 - spec/logstash/legacy_ruby_timestamp_spec.rb
 - spec/logstash/modules/cli_parser_spec.rb
+- spec/logstash/modules/kibana_client_spec.rb
 - spec/logstash/modules/logstash_config_spec.rb
 - spec/logstash/modules/scaffold_spec.rb
 - spec/logstash/modules/settings_merger_spec.rb
@@ -593,6 +594,7 @@ files:
 - spec/logstash/util/buftok_spec.rb
 - spec/logstash/util/byte_value_spec.rb
 - spec/logstash/util/charset_spec.rb
+- spec/logstash/util/cloud_setting_id_spec.rb
 - spec/logstash/util/duration_formatter_spec.rb
 - spec/logstash/util/java_version_spec.rb
 - spec/logstash/util/plugin_version_spec.rb
@@ -609,6 +611,7 @@ files:
 - spec/support/mocks_classes.rb
 - spec/support/shared_contexts.rb
 - spec/support/shared_examples.rb
+- versions-gem-copy.yml
 homepage: http://www.elastic.co/guide/en/logstash/current/index.html
 licenses:
 - Apache License (2.0)
@@ -631,11 +634,11 @@ requirements:
 - jar org.apache.logging.log4j:log4j-slf4j-impl, 2.6.2
 - jar org.apache.logging.log4j:log4j-api, 2.6.2
 - jar org.apache.logging.log4j:log4j-core, 2.6.2
-- jar com.fasterxml.jackson.core:jackson-core, 2.7.3
-- jar com.fasterxml.jackson.core:jackson-databind, 2.7.3
-- jar com.fasterxml.jackson.core:jackson-annotations, 2.7.3
-- jar com.fasterxml.jackson.module:jackson-module-afterburner, 2.7.3
-- jar com.fasterxml.jackson.dataformat:jackson-dataformat-cbor, 2.7.3
+- jar com.fasterxml.jackson.core:jackson-core, 2.9.1
+- jar com.fasterxml.jackson.core:jackson-databind, 2.9.1
+- jar com.fasterxml.jackson.core:jackson-annotations, 2.9.1
+- jar com.fasterxml.jackson.module:jackson-module-afterburner, 2.9.1
+- jar com.fasterxml.jackson.dataformat:jackson-dataformat-cbor, 2.9.1
 rubyforge_project:
 rubygems_version: 2.6.13
 signing_key:
@@ -695,6 +698,7 @@ test_files:
 - spec/logstash/legacy_ruby_event_spec.rb
 - spec/logstash/legacy_ruby_timestamp_spec.rb
 - spec/logstash/modules/cli_parser_spec.rb
+- spec/logstash/modules/kibana_client_spec.rb
 - spec/logstash/modules/logstash_config_spec.rb
 - spec/logstash/modules/scaffold_spec.rb
 - spec/logstash/modules/settings_merger_spec.rb
@@ -732,6 +736,7 @@ test_files:
 - spec/logstash/util/buftok_spec.rb
 - spec/logstash/util/byte_value_spec.rb
 - spec/logstash/util/charset_spec.rb
+- spec/logstash/util/cloud_setting_id_spec.rb
 - spec/logstash/util/duration_formatter_spec.rb
 - spec/logstash/util/java_version_spec.rb
 - spec/logstash/util/plugin_version_spec.rb