logstash-core 6.0.0.beta2-java → 6.0.0-java

data/lib/logstash/util/modules_setting_array.rb

@@ -16,7 +16,7 @@ module LogStash module Util class ModulesSettingArray
     @original = value
     # wrap passwords
     @original.each do |hash|
-      hash.keys.select{|key| key.to_s.end_with?('password')}.each do |key|
+      hash.keys.select{|key| key.to_s.end_with?('password') && !hash[key].is_a?(LogStash::Util::Password)}.each do |key|
         hash[key] = LogStash::Util::Password.new(hash[key])
       end
     end

data/lib/logstash/util/password.rb

@@ -1,10 +1,9 @@
 # encoding: utf-8
 require "logstash/namespace"
-require "logstash/util"
 
 # This class exists to quietly wrap a password string so that, when printed or
 # logged, you don't accidentally print the password itself.
-class LogStash::Util::Password
+module LogStash module Util class Password
   attr_reader :value
 
   public
@@ -21,5 +20,4 @@ class LogStash::Util::Password
   def inspect
     return to_s
   end # def inspect
-end # class LogStash::Util::Password
-
+end end end # class LogStash::Util::Password

data/lib/logstash/util/wrapped_acked_queue.rb

@@ -287,6 +287,12 @@ module LogStash; module Util
         # @cancelled[event] = true
       end
 
+      def to_a
+        events = []
+        each {|e| events << e}
+        events
+      end
+
       def each(&blk)
         # take care not to cause @originals or @generated to change during iteration
 

data/lib/logstash/util/wrapped_synchronous_queue.rb

@@ -218,6 +218,12 @@ module LogStash; module Util
         # @cancelled[event] = true
       end
 
+      def to_a
+        events = []
+        each {|e| events << e}
+        events
+      end
+
       def each(&blk)
         # take care not to cause @originals or @generated to change during iteration
         @iterating = true

data/lib/logstash/version.rb

@@ -2,13 +2,11 @@
 
 # The version of the logstash package (not the logstash-core gem version).
 #
-# Note to authors: this should not include dashes because 'gem' barfs if
-# you include a dash in the version string.
-
-# TODO: (colin) the logstash-core gem uses it's own version number in logstash-core/lib/logstash-core/version.rb
-#       there are some dependencies in logstash-core on the LOGSTASH_VERSION constant this is why
-#       the logstash version is currently defined here in logstash-core/lib/logstash/version.rb but
-#       eventually this file should be in the root logstash lib fir and dependencies in logstash-core should be
-#       fixed.
-
-LOGSTASH_VERSION = "6.0.0-beta2"
+# sourced from a copy of the master versions.yml file, see logstash-core/logstash-core.gemspec
+if !defined?(ALL_VERSIONS)
+  require 'yaml'
+  ALL_VERSIONS = YAML.load_file(File.expand_path("../../versions-gem-copy.yml", File.dirname(__FILE__)))
+end
+if !defined?(LOGSTASH_VERSION)
+  LOGSTASH_VERSION = ALL_VERSIONS.fetch("logstash")
+end

data/logstash-core.gemspec

@@ -1,6 +1,25 @@
 # -*- encoding: utf-8 -*-
 lib = File.expand_path('../lib', __FILE__)
 $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+
+project_versions_yaml_path = File.expand_path("../versions.yml", File.dirname(__FILE__))
+if File.exist?(project_versions_yaml_path)
+  # we need to copy the project level versions.yml into the gem root
+  # to be able to package it into the gems file structure
+  # as the require 'logstash-core/version' loads the yaml file from within the gem root.
+  #
+  # we ignore the copy in git and we overwrite an existing file
+  # each time we build the logstash-core gem
+  original_lines = IO.readlines(project_versions_yaml_path)
+  original_lines << ""
+  original_lines << "# This is a copy the project level versions.yml into this gem's root and it is created when the gemspec is evaluated."
+  gem_versions_yaml_path = File.expand_path("./versions-gem-copy.yml", File.dirname(__FILE__))
+  File.open(gem_versions_yaml_path, 'w') do |new_file|
+    # create or overwrite
+    new_file.puts(original_lines)
+  end
+end
+
 require 'logstash-core/version'
 
 Gem::Specification.new do |gem|
@@ -11,7 +30,10 @@ Gem::Specification.new do |gem|
   gem.homepage      = "http://www.elastic.co/guide/en/logstash/current/index.html"
   gem.license       = "Apache License (2.0)"
 
-  gem.files         = Dir.glob(["logstash-core.gemspec", "gemspec_jars.rb", "lib/**/*.rb", "spec/**/*.rb", "locales/*", "lib/logstash/api/init.ru", "lib/logstash-core/logstash-core.jar"])
+  gem.files         = Dir.glob(
+    %w(versions-gem-copy.yml logstash-core.gemspec gemspec_jars.rb lib/**/*.rb spec/**/*.rb locales/*
+    lib/logstash/api/init.ru lib/logstash-core/logstash-core.jar)
+  )
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.name          = "logstash-core"
   gem.require_paths = ["lib"]
@@ -31,7 +53,7 @@ Gem::Specification.new do |gem|
 
   gem.add_runtime_dependency "sinatra", '~> 1.4', '>= 1.4.6'
   gem.add_runtime_dependency 'puma', '~> 2.16'
-  gem.add_runtime_dependency "jruby-openssl", "0.9.20" # >= 0.9.13 Required to support TLSv1.2
+  gem.add_runtime_dependency "jruby-openssl", ">= 0.9.20" # >= 0.9.13 Required to support TLSv1.2
   gem.add_runtime_dependency "chronic_duration", "0.10.6"
 
   # TODO(sissel): Treetop 1.5.x doesn't seem to work well, but I haven't
@@ -46,7 +68,7 @@ Gem::Specification.new do |gem|
   gem.add_runtime_dependency "rubyzip", "~> 1.2.1"
   gem.add_runtime_dependency "thread_safe", "~> 0.3.5" #(Apache 2.0 license)
 
-  gem.add_runtime_dependency "jrjackson", "~> 0.4.2" #(Apache 2.0 license)
+  gem.add_runtime_dependency "jrjackson", "~> #{ALL_VERSIONS.fetch('jrjackson')}" #(Apache 2.0 license)
 
   gem.add_runtime_dependency "jar-dependencies"
   # as of Feb 3rd 2016, the ruby-maven gem is resolved to version 3.3.3 and that version

data/spec/logstash/agent/converge_spec.rb

@@ -71,9 +71,16 @@ describe LogStash::Agent do
           TestSourceLoader.new(infinite_pipeline_config, system_pipeline_config)
         end
 
+        before(:each) do
+            @agent_task = start_agent(subject)
+        end
+
+        after(:each) do
+            @agent_task.stop!
+        end
+
         describe "#running_user_defined_pipelines" do
           it "returns the user defined pipelines" do
-            start_agent(subject)
             wait_for do
               subject.with_running_user_defined_pipelines {|pipelines| pipelines.keys }
             end.to eq([:main])
@@ -82,7 +89,6 @@ describe LogStash::Agent do
 
         describe "#running_user_defined_pipelines?" do
           it "returns true" do
-            start_agent(subject)
             wait_for do
               subject.running_user_defined_pipelines?
             end.to be_truthy
@@ -102,14 +108,19 @@ describe LogStash::Agent do
         context "and successfully load the config" do
           let(:agent_settings) { mock_settings("config.reload.automatic" => false) }
 
-          it "converge only once" do
-            agent_task = start_agent(subject)
+          before(:each) do
+            @agent_task = start_agent(subject)
+          end
 
+          after(:each) do
+            @agent_task.stop!
+          end
+
+          it "converge only once" do
             expect(source_loader.fetch_count).to eq(1)
             expect(subject).to have_running_pipeline?(pipeline_config)
 
             subject.shutdown
-            agent_task.stop!
           end
         end
 
@@ -137,11 +148,16 @@ describe LogStash::Agent do
             "config.reload.interval" =>  interval
           )
         end
+        before(:each) do
+          @agent_task = start_agent(subject)
+        end
+
+        after(:each) do
+          @agent_task.stop!
+        end
 
         context "and successfully load the config" do
           it "converges periodically the pipelines from the configs source" do
-            agent_task = start_agent(subject)
-
             sleep(2) # let the interval reload a few times
             expect(subject).to have_running_pipeline?(pipeline_config)
 
@@ -152,7 +168,6 @@ describe LogStash::Agent do
             end
 
             subject.shutdown
-            agent_task.stop!
           end
         end
 
@@ -162,14 +177,12 @@ describe LogStash::Agent do
           end
 
           it "it will keep trying to converge" do
-            agent_task = start_agent(subject)
 
             sleep(agent_settings.get("config.reload.interval") / 1_000_000_000.0 * 20) # let the interval reload a few times
             expect(subject.pipelines_count).to eq(0)
             expect(source_loader.fetch_count).to be > 1
 
             subject.shutdown
-            agent_task.stop!
           end
         end
       end

data/spec/logstash/agent_spec.rb

@@ -26,6 +26,7 @@ describe LogStash::Agent do
     sl
   end
   let(:logger) { double("logger") }
+  let(:timeout) {120} #seconds
 
   subject { LogStash::Agent.new(agent_settings, default_source_loader) }
 
@@ -33,7 +34,7 @@ describe LogStash::Agent do
     # This MUST run first, before `subject` is invoked to ensure clean state
     clear_data_dir
 
-    File.open(config_file, "w") { |f| f.puts config_file_txt }
+    File.open(config_file, "w") { |f| f.puts(config_file_txt) }
 
     agent_args.each do |key, value|
       agent_settings.set(key, value)
@@ -50,8 +51,9 @@ describe LogStash::Agent do
   after :each do
     subject.shutdown
     LogStash::SETTINGS.reset
-    File.unlink(config_file)
-    File.unlink(subject.id_path)
+
+    FileUtils.rm(config_file)
+    FileUtils.rm_rf(subject.id_path)
   end
 
   it "fallback to hostname when no name is provided" do
@@ -314,8 +316,10 @@ describe LogStash::Agent do
   context "metrics after config reloading" do
 
     let(:initial_generator_threshold) { 1000 }
-    let(:temporary_file) { Stud::Temporary.file.path }
-    let(:config_file_txt) { "input { generator { count => #{initial_generator_threshold*2} } } output { file { path => '#{temporary_file}'} }" }
+    let(:original_config_output) { Stud::Temporary.pathname }
+    let(:new_config_output) { Stud::Temporary.pathname }
+
+    let(:config_file_txt) { "input { generator { count => #{initial_generator_threshold*2} } } output { file { path => '#{original_config_output}'} }" }
 
     let(:agent_args) do
       {
@@ -326,14 +330,25 @@ describe LogStash::Agent do
 
     subject { described_class.new(agent_settings, default_source_loader) }
 
+    let(:agent_thread) do
+      # subject has to be called for the first time outside the thread because it could create a race condition
+      # with subsequent subject calls
+      s = subject
+      Thread.new { s.execute }
+    end
+
     before(:each) do
       @abort_on_exception = Thread.abort_on_exception
       Thread.abort_on_exception = true
 
-      @t = Thread.new { subject.execute }
+      agent_thread
 
       # wait for some events to reach the dummy_output
-      sleep(0.01) until IO.readlines(temporary_file).size > initial_generator_threshold
+      Timeout.timeout(timeout) do
+        # wait for file existence otherwise it will raise exception on Windows
+        sleep(0.1) until ::File.exist?(original_config_output)
+        sleep(0.1) until IO.readlines(original_config_output).size > initial_generator_threshold
+      end
 
       # write new config
       File.open(config_file, "w") { |f| f.write(new_config) }
@@ -341,10 +356,14 @@ describe LogStash::Agent do
 
     after :each do
       begin
+        Stud.stop!(agent_thread) rescue nil # it may be dead already
+        agent_thread.join
         subject.shutdown
-        Stud.stop!(@t) rescue nil # it may be dead already
-        @t.join
-        File.unlink(temporary_file)
+
+        FileUtils.rm(original_config_output)
+        FileUtils.rm(new_config_output) if File.exist?(new_config_output)
+      rescue
+          #don't care about errors here.
       ensure
         Thread.abort_on_exception = @abort_on_exception
       end
@@ -352,19 +371,20 @@ describe LogStash::Agent do
 
     context "when reloading a good config" do
       let(:new_config_generator_counter) { 500 }
-      let(:new_file) { Stud::Temporary.file.path }
-      let(:new_config) { "input { generator { count => #{new_config_generator_counter} } } output { file { path => '#{new_file}'} }" }
+      let(:new_config) { "input { generator { count => #{new_config_generator_counter} } } output { file { path => '#{new_config_output}'} }" }
 
       before :each do
         subject.converge_state_and_update
-        sleep(0.01) while ::File.read(new_file).chomp.empty?
+        Timeout.timeout(timeout) do
+          # wait for file existence otherwise it will raise exception on Windows
+          sleep(0.1) until ::File.exist?(new_config_output)
+          sleep(0.1) while ::File.read(new_config_output).chomp.empty?
+        end
         # ensure the converge_state_and_update method has updated metrics by
         # invoking the mutex
         subject.running_pipelines?
       end
 
-      after(:each) { File.unlink(new_file) }
-
       it "resets the pipeline metric collector" do
         snapshot = subject.metric.collector.snapshot_metric
         value = snapshot.metric_store.get_with_path("/stats/pipelines")[:stats][:pipelines][:main][:events][:in].value

data/spec/logstash/api/modules/node_stats_spec.rb

@@ -69,7 +69,7 @@ describe LogStash::Api::Modules::NodeStats do
       "cpu"=>{
         "total_in_millis"=>Numeric,
         "percent"=>Numeric,
-        "load_average" => { "1m" => Numeric }
+        # load_average is not supported on Windows, set it below
       }
     },
    "pipelines" => {
@@ -89,5 +89,9 @@ describe LogStash::Api::Modules::NodeStats do
    }
   }
 
+  unless LogStash::Environment.windows?
+    root_structure["process"]["cpu"]["load_average"] = { "1m" => Numeric }
+  end
+
   test_api_and_resources(root_structure)
 end

data/spec/logstash/compiler/compiler_spec.rb

@@ -193,6 +193,35 @@ describe LogStash::Compiler do
           expect(c_plugin).to ir_eql(j.iPlugin(INPUT, "generator", expected_plugin_args))
         end
       end
+
+      describe "a filter plugin that repeats a Hash directive" do
+        let(:source) { "input { } filter { #{plugin_source} } output { } " }
+        subject(:c_plugin) { compiled[:filter] }
+
+        let(:plugin_source) do
+          %q[
+              grok {
+                match => { "message" => "%{WORD:word}" }
+                match => { "examplefield" => "%{NUMBER:num}" }
+                break_on_match => false
+              }
+          ]
+        end
+
+        let(:expected_plugin_args) do
+          {
+            "match" => {
+              "message" => "%{WORD:word}",
+              "examplefield" => "%{NUMBER:num}"
+            },
+            "break_on_match" => "false"
+          }
+        end
+
+        it "should merge the contents of the individual directives" do
+          expect(c_plugin).to ir_eql(j.iPlugin(FILTER, "grok", expected_plugin_args))
+        end
+      end
     end
 
     context "inputs" do

data/spec/logstash/config/source/local_spec.rb

@@ -79,7 +79,7 @@ describe LogStash::Config::Source::Local::ConfigPathLoader do
 
         parts.each do |part|
           basename = ::File.basename(part.id)
-          file_path = ::File.join(directory, basename)
+          file_path = ::File.expand_path(::File.join(directory, basename))
           content = files[basename]
           expect(part).to be_a_source_with_metadata("file", file_path, content)
         end
@@ -99,7 +99,8 @@ describe LogStash::Config::Source::Local::ConfigPathLoader do
       end
 
       it "raises an exception" do
-        expect { subject.read(file_path) }.to raise_error LogStash::ConfigLoadingError, /#{file_path}/
+        # check against base name because on Windows long paths are shrinked in the exception message
+        expect { subject.read(file_path) }.to raise_error LogStash::ConfigLoadingError, /.+#{::File.basename(file_path)}/
       end
     end
 

data/spec/logstash/event_spec.rb

@@ -352,4 +352,61 @@ describe LogStash::Event do
       expect { subject.baz() }.to raise_error(NoMethodError, /undefined method `baz' for/)
     end
   end
+
+  describe "#clone" do
+    let(:fieldref) { "[@metadata][fancy]" }
+    let(:event1) { LogStash::Event.new("hello" => "world", "@metadata" => { "fancy" => "pants" }) }
+    let(:event2) { LogStash::Event.new("hello" => "world", "@metadata" => { "fancy" => {"fancy2" => "pants2"} }) }
+    let(:event3) { LogStash::Event.new("hello" => "world", "@metadata" => { "fancy" => {"fancy2" => {"fancy3" => "pants2"}} }) }
+    let(:event4) { LogStash::Event.new("hello" => "world", "@metadata" => { "fancy" => {"fancy2" => ["pants1", "pants2"]} }) }
+    let(:event5) { LogStash::Event.new("hello" => "world", "@metadata" => { "fancy" => "pants", "smarty" => "pants2" }) }
+
+    it "should clone metadata fields" do
+      cloned = event1.clone
+      expect(cloned.get(fieldref)).to eq("pants")
+      expect(cloned.to_hash_with_metadata).to include("@metadata")
+    end
+
+    it "should clone metadata fields with nested json" do
+      cloned = event2.clone
+      expect(cloned.get(fieldref)).to eq({"fancy2" => "pants2"})
+      expect(cloned.get("hello")).to eq("world")
+      expect(cloned.to_hash).not_to include("@metadata")
+      expect(cloned.to_hash_with_metadata).to include("@metadata")
+    end
+
+    it "should clone metadata fields with 2-level nested json" do
+      cloned = event3.clone
+      expect(cloned.get(fieldref)).to eq({"fancy2" => {"fancy3" => "pants2"}})
+      expect(cloned.to_hash).not_to include("@metadata")
+      expect(cloned.to_hash_with_metadata).to include("@metadata")
+    end
+
+    it "should clone metadata fields with nested json and array value" do
+      cloned = event4.clone
+      expect(cloned.get(fieldref)).to eq({"fancy2" => ["pants1", "pants2"]})
+      expect(cloned.to_hash_with_metadata).to include("@metadata")
+    end
+
+    it "should clone metadata fields with multiple keys" do
+      cloned = event5.clone
+      expect(cloned.get(fieldref)).to eq("pants")
+      expect(cloned.get("[@metadata][smarty]")).to eq("pants2")
+      expect(cloned.to_hash_with_metadata).to include("@metadata")
+    end
+
+    it "mutating cloned event should not affect the original event" do
+      cloned = event1.clone
+      cloned.set("hello", "foobar")
+      expect(cloned.get("hello")).to eq("foobar")
+      expect(event1.get("hello")).to eq("world")
+    end
+
+    it "mutating cloned event's metadata should not affect the original event metadata" do
+      cloned = event1.clone
+      cloned.set("[@metadata][fancy]", "foobar")
+      expect(cloned.get("[@metadata][fancy]")).to eq("foobar")
+      expect(event1.get("[@metadata][fancy]")).to eq("pants")
+    end
+  end
 end