logstash-core 2.3.3.snapshot1-java → 2.3.3.snapshot2-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 9080d8e89a2f372617f4919792ff47cdd16b6e46
-  data.tar.gz: d76d50b943f3450cd05b7bbfead1779a884dd287
+  metadata.gz: af008eb73fd26bbd55cc20cd8ff1d64fc9162689
+  data.tar.gz: c0720a21e07aba5d4142f41bfb7254e8be8d135a
 SHA512:
-  metadata.gz: 4aca0deea4c65a37239c9254b94e3801b636b66a7fb95a42dd71b67751861766b45bbc2863c1ec468d86ad4c05633c4fd3312cf061a9b5b9c2d49da557a81561
-  data.tar.gz: 0de14d2ef71f6029ff0bc89714273da4732fb0551528e49dd9f19f14c21cd8b4ad054fb3c662884951433ac2a210a5921e6bff62f0d3ba7785c4e0945dbfe358
+  metadata.gz: bba77ba08e638549e65a27b7787cca72ca80c92333774c2b9f483c1e622c043f08140334875f3096f063595313796eb037157e71acb8010872ba6d32f5ee7b90
+  data.tar.gz: d34568b2b7e15b021e65d050c5e1d4dd80a33e46ec2a2ed622f328998d4f6fbb596ad426ef9ec3408ef2401f94a675479dde64824ef36985cd54fd771c3053a3

data/lib/logstash-core/version.rb

@@ -5,4 +5,4 @@
 # Note to authors: this should not include dashes because 'gem' barfs if
 # you include a dash in the version string.
 
-LOGSTASH_CORE_VERSION = "2.3.3.snapshot1"
+LOGSTASH_CORE_VERSION = "2.3.3.snapshot2"

data/lib/logstash/agent.rb

@@ -59,7 +59,7 @@ class LogStash::Agent < Clamp::Command
   option "--debug", :flag, I18n.t("logstash.agent.flag.debug")
 
   option ["--debug-config"], :flag,
-         I18n.t("logstash.runner.flag.debug_config"),
+         I18n.t("logstash.agent.flag.debug_config"),
          :attribute_name => :debug_config, :default => false
 
   option ["-V", "--version"], :flag,
@@ -376,22 +376,6 @@ class LogStash::Agent < Clamp::Command
     end
   end
 
-  def create_pipeline(settings)
-    begin
-      config = fetch_config(settings)
-    rescue => e
-      @logger.error("failed to fetch pipeline configuration", :message => e.message)
-      return
-    end
-
-    begin
-      LogStash::Pipeline.new(config, settings)
-    rescue => e
-      @logger.error("fetched an invalid config", :config => config, :reason => e.message)
-      return
-    end
-  end
-
   def start_pipelines
     @pipelines.each { |id, _| start_pipeline(id) }
   end
@@ -429,6 +413,25 @@ class LogStash::Agent < Clamp::Command
     start_pipeline(pipeline_id)
   end
 
+  def create_pipeline(settings, config=nil)
+
+    if config.nil?
+      begin
+        config = fetch_config(settings)
+      rescue => e
+        @logger.error("failed to fetch pipeline configuration", :message => e.message)
+        return
+      end
+    end
+
+    begin
+      LogStash::Pipeline.new(config, settings)
+    rescue => e
+      @logger.error("fetched an invalid config", :config => config, :reason => e.message)
+      return
+    end
+  end
+
   def clean_state?
     @pipelines.empty?
   end
@@ -437,16 +440,21 @@ class LogStash::Agent < Clamp::Command
   # wrapped in @upgrade_mutex in the parent call `reload_state!`
   def reload_pipeline!(id)
     old_pipeline = @pipelines[id]
-    new_pipeline = create_pipeline(old_pipeline.original_settings)
+    new_config = fetch_config(old_pipeline.original_settings)
+    if old_pipeline.config_str == new_config
+      @logger.debug("no configuration change for pipeline",
+                    :pipeline => id, :config => new_config)
+      return
+    end
+
+    new_pipeline = create_pipeline(old_pipeline.original_settings, new_config)
     return if new_pipeline.nil?
 
-    if old_pipeline.config_str == new_pipeline.config_str
-      @logger.debug("no configuration change for pipeline",
-                    :pipeline => id, :config => old_pipeline.config_str)
-    elsif new_pipeline.non_reloadable_plugins.any?
+    if new_pipeline.non_reloadable_plugins.any?
       @logger.error(I18n.t("logstash.agent.non_reloadable_config_reload"),
                     :pipeline_id => id,
                     :plugins => new_pipeline.non_reloadable_plugins.map(&:class))
+      return
     else
       @logger.log("fetched new config for pipeline. upgrading..",
                    :pipeline => id, :config => new_pipeline.config_str)

data/lib/logstash/config/config_ast.rb

@@ -76,6 +76,14 @@ module LogStash; module Config; module AST
     @defered_conditionals_index = val
   end
 
+  def self.plugin_instance_index
+    @plugin_instance_index
+  end
+
+  def self.plugin_instance_index=(val)
+    @plugin_instance_index = val
+  end
+
   class Node < Treetop::Runtime::SyntaxNode
     def text_value_for_comments
       text_value.gsub(/[\r\n]/, " ")
@@ -86,6 +94,7 @@ module LogStash; module Config; module AST
     def compile
       LogStash::Config::AST.defered_conditionals = []
       LogStash::Config::AST.defered_conditionals_index = 0
+      LogStash::Config::AST.plugin_instance_index = 0
       code = []
 
       code << <<-CODE
@@ -94,6 +103,7 @@ module LogStash; module Config; module AST
         @outputs = []
         @periodic_flushers = []
         @shutdown_flushers = []
+        @generated_objects = {}
       CODE
 
       sections = recursive_select(LogStash::Config::AST::PluginSection)
@@ -137,7 +147,9 @@ module LogStash; module Config; module AST
   class PluginSection < Node
     # Global plugin numbering for the janky instance variable naming we use
     # like @filter_<name>_1
-    @@i = 0
+    def initialize(*args)
+      super(*args)
+    end
 
     # Generate ruby code to initialize all the plugins.
     def compile_initializer
@@ -147,31 +159,31 @@ module LogStash; module Config; module AST
 
 
         code << <<-CODE
-          #{name} = #{plugin.compile_initializer}
-          @#{plugin.plugin_type}s << #{name}
+          @generated_objects[:#{name}] = #{plugin.compile_initializer}
+          @#{plugin.plugin_type}s << @generated_objects[:#{name}]
         CODE
 
         # The flush method for this filter.
         if plugin.plugin_type == "filter"
 
           code << <<-CODE
-            #{name}_flush = lambda do |options, &block|
-              @logger.debug? && @logger.debug(\"Flushing\", :plugin => #{name})
+            @generated_objects[:#{name}_flush] = lambda do |options, &block|
+              @logger.debug? && @logger.debug(\"Flushing\", :plugin => @generated_objects[:#{name}])
 
-              events = #{name}.flush(options)
+              events = @generated_objects[:#{name}].flush(options)
 
               return if events.nil? || events.empty?
 
-              @logger.debug? && @logger.debug(\"Flushing\", :plugin => #{name}, :events => events)
+              @logger.debug? && @logger.debug(\"Flushing\", :plugin => @generated_objects[:#{name}], :events => events)
 
               #{plugin.compile_starting_here.gsub(/^/, "  ")}
 
               events.each{|e| block.call(e)}
             end
 
-            if #{name}.respond_to?(:flush)
-              @periodic_flushers << #{name}_flush if #{name}.periodic_flush
-              @shutdown_flushers << #{name}_flush
+            if @generated_objects[:#{name}].respond_to?(:flush)
+              @periodic_flushers << @generated_objects[:#{name}_flush] if @generated_objects[:#{name}].periodic_flush
+              @shutdown_flushers << @generated_objects[:#{name}_flush]
             end
           CODE
 
@@ -192,9 +204,10 @@ module LogStash; module Config; module AST
 
       plugins.each do |plugin|
         # Unique number for every plugin.
-        @@i += 1
+        LogStash::Config::AST.plugin_instance_index += 1
         # store things as ivars, like @filter_grok_3
-        var = "@#{plugin.plugin_type}_#{plugin.plugin_name}_#{@@i}"
+        var = :"#{plugin.plugin_type}_#{plugin.plugin_name}_#{LogStash::Config::AST.plugin_instance_index}"
+        # puts("var=#{var.inspect}")
         @variables[plugin] = var
       end
       return @variables
@@ -236,13 +249,13 @@ module LogStash; module Config; module AST
     def compile
       case plugin_type
       when "input"
-        return "start_input(#{variable_name})"
+        return "start_input(@generated_objects[:#{variable_name}])"
       when "filter"
         return <<-CODE
-          events = #{variable_name}.multi_filter(events)
+          events = @generated_objects[:#{variable_name}].multi_filter(events)
         CODE
       when "output"
-        return "targeted_outputs << #{variable_name}\n"
+        return "targeted_outputs << @generated_objects[:#{variable_name}]\n"
       when "codec"
         settings = attributes.recursive_select(Attribute).collect(&:compile).reject(&:empty?)
         attributes_code = "LogStash::Util.hash_merge_many(#{settings.map { |c| "{ #{c} }" }.join(", ")})"
@@ -391,7 +404,7 @@ module LogStash; module Config; module AST
       if type == "filter"
         i = LogStash::Config::AST.defered_conditionals_index += 1
         source = <<-CODE
-          define_singleton_method :cond_func_#{i} do |input_events|
+          @generated_objects[:cond_func_#{i}] = lambda do |input_events|
             result = []
             input_events.each do |event|
               events = [event]
@@ -405,7 +418,7 @@ module LogStash; module Config; module AST
         LogStash::Config::AST.defered_conditionals << source
 
         <<-CODE
-          events = cond_func_#{i}(events)
+          events = @generated_objects[:cond_func_#{i}].call(events)
         CODE
       else # Output
         <<-CODE

data/lib/logstash/config/loader.rb

@@ -55,7 +55,7 @@ module LogStash; module Config; class Loader
     path = ::File.join(path, "*") if ::File.directory?(path)
 
     if Dir.glob(path).length == 0
-      fail(I18n.t("logstash.runner.configuration.file-not-found", :path => path))
+      fail(I18n.t("logstash.agent.configuration.file-not-found", :path => path))
     end
 
     config = ""
@@ -91,7 +91,7 @@ module LogStash; module Config; class Loader
     begin
       Net::HTTP.get(uri) + "\n"
     rescue Exception => e
-      fail(I18n.t("logstash.runner.configuration.fetch-failed", :path => uri.to_s, :message => e.message))
+      fail(I18n.t("logstash.agent.configuration.fetch-failed", :path => uri.to_s, :message => e.message))
     end
   end
 end end end

data/lib/logstash/version.rb

@@ -11,4 +11,4 @@
 #       eventually this file should be in the root logstash lib fir and dependencies in logstash-core should be
 #       fixed.
 
-LOGSTASH_VERSION = "2.3.3.snapshot1"
+LOGSTASH_VERSION = "2.3.3.snapshot2"

data/locales/en.yml

@@ -78,8 +78,7 @@ en:
       non_reloadable_config_reload: >-
         Unable to reload configuration because it does not support dynamic reloading
       non_reloadable_config_register: |-
-        Logstash was not able to load configuration since it does not support
-        dynamic reloading and -r or --auto-reload flag was enabled
+        Logstash is not able to start since configuration auto reloading was enabled but the configuration contains plugins that don't support it. Quitting...
       configtest-flag-information: |-
         You may be interested in the '--configtest' flag which you can
         use to validate logstash's configuration before you choose
@@ -221,4 +220,8 @@ en:
           Force logstash to exit during shutdown even
           if there are still inflight events in memory.
           By default, logstash will refuse to quit until all
-          received events have been pushed to the outputs.
+          received events have been pushed to the outputs.
+        debug_config: |+
+          Print the compiled config ruby code out as a debug log (you must also have --debug enabled).
+          WARNING: This will include any 'password' options passed to plugin configs as plaintext, and may result
+          in plaintext passwords appearing in your logs!

data/logstash-core.gemspec

@@ -17,7 +17,7 @@ Gem::Specification.new do |gem|
   gem.require_paths = ["lib"]
   gem.version       = LOGSTASH_CORE_VERSION
 
-  gem.add_runtime_dependency "logstash-core-event", "2.3.3.snapshot1"
+  gem.add_runtime_dependency "logstash-core-event", "2.3.3.snapshot2"
 
   gem.add_runtime_dependency "cabin", "~> 0.8.0" #(Apache 2.0 license)
   gem.add_runtime_dependency "pry", "~> 0.10.1"  #(Ruby license)

data/spec/logstash/config/config_ast_spec.rb

@@ -144,6 +144,49 @@ describe LogStashConfigParser do
     end
   end
 
+  context "when using two plugin sections of the same type" do
+    let(:pipeline_klass) do
+      Class.new do
+        def initialize(config)
+          grammar = LogStashConfigParser.new
+          @config = grammar.parse(config)
+          @code = @config.compile
+          eval(@code)
+        end
+        def plugin(*args);end
+      end
+    end
+    context "(filters)" do
+      let(:config_string) {
+        "input { generator { } }
+         filter { filter1 { } }
+         filter { filter1 { } }
+         output { output1 { } }"
+      }
+
+
+      it "should create a pipeline with both sections" do
+        generated_objects = pipeline_klass.new(config_string).instance_variable_get("@generated_objects")
+        filters = generated_objects.keys.map(&:to_s).select {|obj_name| obj_name.match(/^filter.+?_\d+$/) }
+        expect(filters.size).to eq(2)
+      end
+    end
+
+    context "(filters)" do
+      let(:config_string) {
+        "input { generator { } }
+         output { output1 { } }
+         output { output1 { } }"
+      }
+
+
+      it "should create a pipeline with both sections" do
+        generated_objects = pipeline_klass.new(config_string).instance_variable_get("@generated_objects")
+        outputs = generated_objects.keys.map(&:to_s).select {|obj_name| obj_name.match(/^output.+?_\d+$/) }
+        expect(outputs.size).to eq(2)
+      end
+    end
+  end
   context "when creating two instances of the same configuration" do
 
     let(:config_string) {
@@ -169,10 +212,12 @@ describe LogStashConfigParser do
     end
 
     describe "generated conditional functionals" do
-      it "should be defined at instance level" do
+      it "should be created per instance" do
         instance_1 = pipeline_klass.new(config_string)
         instance_2 = pipeline_klass.new(config_string)
-        expect(instance_1.method(:cond_func_1).owner).to_not be(instance_2.method(:cond_func_1).owner)
+        generated_method_1 = instance_1.instance_variable_get("@generated_objects")[:cond_func_1]
+        generated_method_2 = instance_2.instance_variable_get("@generated_objects")[:cond_func_1]
+        expect(generated_method_1).to_not be(generated_method_2)
       end
     end
   end

data/spec/logstash/pipeline_spec.rb

@@ -452,4 +452,20 @@ describe LogStash::Pipeline do
       expect(pipeline1.filter_func(LogStash::Event.new)).not_to include(nil)
     end
   end
+
+  context "Pipeline object" do
+    before do
+      allow(LogStash::Plugin).to receive(:lookup).with("input", "generator").and_return(LogStash::Inputs::Generator)
+      allow(LogStash::Plugin).to receive(:lookup).with("codec", "plain").and_return(DummyCodec)
+      allow(LogStash::Plugin).to receive(:lookup).with("filter", "dummyfilter").and_return(DummyFilter)
+      allow(LogStash::Plugin).to receive(:lookup).with("output", "dummyoutput").and_return(DummyOutput)
+    end
+
+    let(:pipeline1) { LogStash::Pipeline.new("input { generator {} } filter { dummyfilter {} } output { dummyoutput {}}") }
+    let(:pipeline2) { LogStash::Pipeline.new("input { generator {} } filter { dummyfilter {} } output { dummyoutput {}}") }
+
+    it "should not add ivars" do
+       expect(pipeline1.instance_variables).to eq(pipeline2.instance_variables)
+    end
+  end
 end

data/spec/static/i18n_spec.rb

@@ -0,0 +1,25 @@
+# encoding: utf-8
+require "spec_helper"
+require "i18n"
+
+I18N_T_REGEX = Regexp.new('I18n.t.+?"(.+?)"')
+
+describe I18n do
+  context "when using en.yml" do
+    glob_path = File.join(LogStash::Environment::LOGSTASH_HOME, "logstash-*", "lib", "**", "*.rb")
+
+    Dir.glob(glob_path).each do |file_name|
+
+      context "in file \"#{file_name}\"" do
+        File.foreach(file_name) do |line|
+          next unless (match = line.match(I18N_T_REGEX))
+          line = $INPUT_LINE_NUMBER
+          key = match[1]
+          it "in line #{line} the \"#{key}\" key should exist" do
+            expect(I18n.exists?(key)).to be_truthy
+          end
+        end
+      end
+    end
+  end
+end

metadata

@@ -1,21 +1,21 @@
 --- !ruby/object:Gem::Specification
 name: logstash-core
 version: !ruby/object:Gem::Version
-  version: 2.3.3.snapshot1
+  version: 2.3.3.snapshot2
 platform: java
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-05-26 00:00:00.000000000 Z
+date: 2016-06-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 2.3.3.snapshot1
+        version: 2.3.3.snapshot2
   name: logstash-core-event
   prerelease: false
   type: :runtime
@@ -23,11 +23,11 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 2.3.3.snapshot1
+        version: 2.3.3.snapshot2
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 0.8.0
   name: cabin
@@ -35,13 +35,13 @@ dependencies:
   type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 0.8.0
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 0.10.1
   name: pry
@@ -49,13 +49,13 @@ dependencies:
   type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 0.10.1
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 0.0.19
   name: stud
@@ -63,13 +63,13 @@ dependencies:
   type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 0.0.19
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 0.6.5
   name: clamp
@@ -77,7 +77,7 @@ dependencies:
   type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 0.6.5
 - !ruby/object:Gem::Dependency
@@ -97,7 +97,7 @@ dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 0.8.3
   name: gems
@@ -105,7 +105,7 @@ dependencies:
   type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 0.8.3
 - !ruby/object:Gem::Dependency
@@ -139,7 +139,7 @@ dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - <
+    - - "<"
       - !ruby/object:Gem::Version
         version: 1.5.0
   name: treetop
@@ -147,7 +147,7 @@ dependencies:
   type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - <
+    - - "<"
       - !ruby/object:Gem::Version
         version: 1.5.0
 - !ruby/object:Gem::Dependency
@@ -167,7 +167,7 @@ dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 0.5.4
   name: minitar
@@ -175,13 +175,13 @@ dependencies:
   type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 0.5.4
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 1.1.7
   name: rubyzip
@@ -189,13 +189,13 @@ dependencies:
   type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 1.1.7
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 0.3.5
   name: thread_safe
@@ -203,13 +203,13 @@ dependencies:
   type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 0.3.5
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 0.3.7
   name: jrjackson
@@ -217,7 +217,7 @@ dependencies:
   type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 0.3.7
 description: The core components of logstash, the scalable log and event management tool
@@ -311,6 +311,7 @@ files:
 - spec/logstash/util/worker_threads_default_printer_spec.rb
 - spec/logstash/util/wrapped_synchronous_queue_spec.rb
 - spec/logstash/util_spec.rb
+- spec/static/i18n_spec.rb
 homepage: http://www.elastic.co/guide/en/logstash/current/index.html
 licenses:
 - Apache License (2.0)
@@ -321,17 +322,17 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>'
+  - - ">"
     - !ruby/object:Gem::Version
       version: 1.3.1
 requirements: []
 rubyforge_project:
-rubygems_version: 2.4.5
+rubygems_version: 2.4.8
 signing_key:
 specification_version: 4
 summary: logstash-core - The core components of logstash
@@ -365,3 +366,4 @@ test_files:
 - spec/logstash/util/worker_threads_default_printer_spec.rb
 - spec/logstash/util/wrapped_synchronous_queue_spec.rb
 - spec/logstash/util_spec.rb
+- spec/static/i18n_spec.rb