logstash-core 2.3.4.snapshot1-java → 2.4.0-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 54ad2dca97892c52abdd7681a54fe06463a5ec40
-  data.tar.gz: 9a6231f80dc1c611b6465cdd11e6a9c0a65ffa3e
+  metadata.gz: 3ce17d1073206a2d7cfde3dd8e13aba375a8d797
+  data.tar.gz: 966d4c0af56bb4b9fe56d37efec5812037b0c677
 SHA512:
-  metadata.gz: 15db28073a86f794a57487161e23701146dd94d71f38893b693026981d18bf6a30871fed36bd0ae15dc3d98b64302c1e051c44517dac7b973c906f76fbf14d74
-  data.tar.gz: f382a8b4505459643327c96ce0857d909fbeca6a3f9779427171d9c020b3d93ed57b9f68e2df7cad3a5b16cb3e1e5bb060ded2722b9d004b205dff011af19c86
+  metadata.gz: ccb3f5b9f0d2e3a01e2524f8f14f041bbece04ed8f31ee40780950438990dd149032b578735b0a2d6446e4f8ceca5eb691faa44399709104860dc8177aa22f91
+  data.tar.gz: acf3e571a22a304866669cf10964d1843c5d73904c2ddb0b49d19a4af942134fa030f79528d8aa83bc09e22dae6e76e5084af6ac2559f07023680d652b7389e0

data/lib/logstash-core/version.rb

@@ -5,4 +5,4 @@
 # Note to authors: this should not include dashes because 'gem' barfs if
 # you include a dash in the version string.
 
-LOGSTASH_CORE_VERSION = "2.3.4.snapshot1"
+LOGSTASH_CORE_VERSION = "2.4.0"

data/lib/logstash/agent.rb

@@ -91,6 +91,10 @@ class LogStash::Agent < Clamp::Command
     I18n.t("logstash.agent.flag.allow-env"),
     :attribute_name => :allow_env, :default => false
 
+  option ["--[no-]log-in-json"], :flag,
+    I18n.t("logstash.agent.flag.log-in-json"),
+    :default => false
+
   def initialize(*params)
     super(*params)
     @logger = Cabin::Channel.get(LogStash)
@@ -129,11 +133,11 @@ class LogStash::Agent < Clamp::Command
   # Emit a warning message.
   def warn(message)
     # For now, all warnings are fatal.
-    raise LogStash::ConfigurationError, message
+    signal_usage_error(message)
   end # def warn
 
   def fail(message)
-    raise LogStash::ConfigurationError, message
+    signal_usage_error(message)
   end # def fail
 
   # Run the agent. This method is invoked after clamp parses the
@@ -142,6 +146,7 @@ class LogStash::Agent < Clamp::Command
     require "logstash/pipeline"
     require "cabin" # gem 'cabin'
     require "logstash/plugin"
+    require "logstash/logging/json"
 
     LogStash::ShutdownWatcher.unsafe_shutdown = unsafe_shutdown?
     LogStash::ShutdownWatcher.logger = @logger
@@ -177,8 +182,7 @@ class LogStash::Agent < Clamp::Command
     end
 
     if config_test?
-      config_loader = LogStash::Config::Loader.new(@logger)
-      config_str = config_loader.format_config(config_path, config_string)
+      config_str = @config_loader.format_config(config_path, config_string)
       begin
         # currently the best strategy to validate the configuration
         # is creating a pipeline instance and checking for exceptions
@@ -198,6 +202,8 @@ class LogStash::Agent < Clamp::Command
                           :allow_env => allow_env?
                           }))
 
+    @thread = Thread.current # this var is implicilty used by Stud.stop?
+
     sigint_id = trap_sigint()
     sigterm_id = trap_sigterm()
     sighup_id = trap_sighup()
@@ -210,8 +216,6 @@ class LogStash::Agent < Clamp::Command
 
     return 1 if clean_state?
 
-    @thread = Thread.current # this var is implicilty used by Stud.stop?
-
     Stud.stoppable_sleep(reload_interval) # sleep before looping
 
     if auto_reload?
@@ -237,7 +241,10 @@ class LogStash::Agent < Clamp::Command
     end
     return 1
   rescue => e
-    @logger.unsubscribe(stdout_logs) if show_startup_errors
+    if show_startup_errors
+      @logger.terminal(e.message)
+      @logger.unsubscribe(stdout_logs)
+    end
     @logger.warn(I18n.t("oops"), :error => e, :class => e.class.name, :backtrace => e.backtrace)
     return 1
   ensure
@@ -294,11 +301,22 @@ class LogStash::Agent < Clamp::Command
 
       puts "Sending logstash logs to #{path}."
       @logger.unsubscribe(@logger_subscription) if @logger_subscription
-      @logger_subscription = @logger.subscribe(@log_fd)
+      if log_in_json?
+        @logger_subscription = @logger.subscribe(LogStash::Logging::JSON.new(@log_fd))
+        @logger.subscribe(LogStash::Logging::JSON.new(STDOUT), :level => :fatal)
+      else
+        @logger_subscription = @logger.subscribe(@log_fd)
+        @logger.subscribe(STDOUT, :level => :fatal)
+      end
     else
-      @logger.subscribe(STDOUT)
+      if log_in_json?
+        @logger.subscribe(LogStash::Logging::JSON.new(STDOUT))
+      else
+        @logger.subscribe(STDOUT)
+      end
     end
 
+
     # TODO(sissel): redirect stdout/stderr to the log as well
     # http://jira.codehaus.org/browse/JRUBY-7003
   end # def configure_logging
@@ -472,7 +490,7 @@ class LogStash::Agent < Clamp::Command
       begin
         pipeline.run
       rescue => e
-        @logger.error("Pipeline aborted due to error", :exception => e, :backtrace => e.backtrace)
+        @logger.error("Pipeline aborted due to error", :exception => e.class.name, :backtrace => e.backtrace)
       end
     end
     sleep 0.01 until pipeline.ready?

data/lib/logstash/codecs/base.rb

@@ -13,6 +13,7 @@ module LogStash::Codecs; class Base < LogStash::Plugin
     super
     config_init(@params)
     register if respond_to?(:register)
+    setup_multi_encode!
   end
 
   public
@@ -23,10 +24,37 @@ module LogStash::Codecs; class Base < LogStash::Plugin
   alias_method :<<, :decode
 
   public
+  # DEPRECATED: Prefer defining encode_sync or multi_encode
   def encode(event)
-    raise "#{self.class}#encode must be overidden"
+    encoded = multi_encode([event])
+    encoded.each {|event,data| @on_event.call(event,data) }
   end # def encode
 
+  public
+  # Relies on the codec being synchronous (which they all are!)
+  # We need a better long term design here, but this is an improvement
+  # over the current API for shared plugins
+  # It is best if the codec implements this directly
+  def multi_encode(events)
+    if @has_encode_sync              
+      events.map {|event| [event, self.encode_sync(event)]}
+    else
+      batch = Thread.current[:logstash_output_codec_batch] ||= []
+      batch.clear
+      
+      events.each {|event| self.encode(event) }
+      batch
+    end
+  end
+
+  def setup_multi_encode!
+    @has_encode_sync = self.methods.include?(:encode_sync)
+
+    on_event do |event, data|
+      Thread.current[:logstash_output_codec_batch] << [event, data]
+    end
+  end
+
   public
   def close; end;
 

data/lib/logstash/config/mixin.rb

@@ -3,6 +3,7 @@ require "logstash/namespace"
 require "logstash/config/registry"
 require "logstash/logging"
 require "logstash/util/password"
+require "logstash/util/safe_uri"
 require "logstash/version"
 require "logstash/environment"
 require "logstash/util/plugin_version"
@@ -333,58 +334,89 @@ module LogStash::Config::Mixin
       return true
     end # def validate_check_invalid_parameter_names
 
+    def validate_check_required_parameter(config_key, config_opts, k, v)
+      if config_key.is_a?(Regexp)
+        (k =~ config_key && v)
+      elsif config_key.is_a?(String)
+        k && v
+      end
+    end
+
     def validate_check_required_parameter_names(params)
       is_valid = true
 
       @config.each do |config_key, config|
         next unless config[:required]
 
-        if config_key.is_a?(Regexp)
-          next if params.keys.select { |k| k =~ config_key }.length > 0
-        elsif config_key.is_a?(String)
-          next if params.keys.member?(config_key)
+        value = params[config_key]
+
+        if config_key.is_a?(Regexp) && !params.keys.any? { |k| k =~ config_key }
+          is_valid = false
+        end
+        
+        if value.nil? || (config[:list] && Array(value).empty?)
+          @logger.error(I18n.t("logstash.agent.configuration.setting_missing",
+                               :setting => config_key, :plugin => @plugin_name,
+                               :type => @plugin_type))
+          is_valid = false
         end
-        @logger.error(I18n.t("logstash.agent.configuration.setting_missing",
-                             :setting => config_key, :plugin => @plugin_name,
-                             :type => @plugin_type))
-        is_valid = false
       end
 
       return is_valid
     end
 
+    def process_parameter_value(value, config_settings)
+      config_val = config_settings[:validate]
+      
+      if config_settings[:list]
+        value = Array(value) # coerce scalars to lists
+        # Empty lists are converted to nils
+        return true, nil if value.empty?
+          
+        validated_items = value.map {|v| validate_value(v, config_val)}
+        is_valid = validated_items.all? {|sr| sr[0] }
+        processed_value = validated_items.map {|sr| sr[1]}
+      else
+        is_valid, processed_value = validate_value(value, config_val)
+      end
+      
+      return [is_valid, processed_value]
+    end
+
     def validate_check_parameter_values(params)
       # Filter out parametrs that match regexp keys.
       # These are defined in plugins like this:
       #   config /foo.*/ => ... 
-      is_valid = true
+      all_params_valid = true
 
       params.each do |key, value|
         @config.keys.each do |config_key|
           next unless (config_key.is_a?(Regexp) && key =~ config_key) \
                       || (config_key.is_a?(String) && key == config_key)
-          config_val = @config[config_key][:validate]
-          #puts "  Key matches."
-          success, result = validate_value(value, config_val)
-          if success 
-            # Accept coerced value if success
+
+          config_settings = @config[config_key]          
+
+          is_valid, processed_value = process_parameter_value(value, config_settings)
+          
+          if is_valid
+            # Accept coerced value if valid
             # Used for converting values in the config to proper objects.
-            params[key] = result if !result.nil?
+            params[key] = processed_value
           else
             @logger.error(I18n.t("logstash.agent.configuration.setting_invalid",
                                  :plugin => @plugin_name, :type => @plugin_type,
                                  :setting => key, :value => value.inspect,
-                                 :value_type => config_val,
-                                 :note => result))
+                                 :value_type => config_settings[:validate],
+                                 :note => processed_value))
           end
-          #puts "Result: #{key} / #{result.inspect} / #{success}"
-          is_valid &&= success
+          
+          all_params_valid &&= is_valid
 
           break # done with this param key
         end # config.each
       end # params.each
 
-      return is_valid
+      return all_params_valid
     end # def validate_check_parameter_values
 
     def validator_find(key)
@@ -405,7 +437,7 @@ module LogStash::Config::Mixin
       result = nil
 
       if validator.nil?
-        return true
+        return true, value
       elsif validator.is_a?(Array)
         value = [*value]
         if value.size > 1
@@ -519,6 +551,12 @@ module LogStash::Config::Mixin
             end
 
             result = value.first.is_a?(::LogStash::Util::Password) ? value.first : ::LogStash::Util::Password.new(value.first)
+          when :uri
+            if value.size > 1
+              return false, "Expected uri (one value), got #{value.size} values?"
+            end
+            
+            result = value.first.is_a?(::LogStash::Util::SafeURI) ? value.first : ::LogStash::Util::SafeURI.new(value.first)
           when :path
             if value.size > 1 # Only 1 value wanted
               return false, "Expected path (one value), got #{value.size} values?"
@@ -554,8 +592,9 @@ module LogStash::Config::Mixin
 
     def secure_params!(params)
       params.each do |key, value|
-        if @config[key][:validate] == :password && !value.is_a?(::LogStash::Util::Password)
-          params[key] = ::LogStash::Util::Password.new(value)
+        if [:uri, :password].include? @config[key][:validate]
+          is_valid, processed_value = process_parameter_value(value, @config[key])
+          params[key] = processed_value
         end
       end
     end

data/lib/logstash/inputs/base.rb

@@ -78,7 +78,7 @@ class LogStash::Inputs::Base < LogStash::Plugin
 
   public
   def do_stop
-    @logger.debug("stopping", :plugin => self)
+    @logger.debug("stopping", :plugin => self.class.name)
     @stop_called.make_true
     stop
   end

data/lib/logstash/instrument/null_metric.rb

@@ -0,0 +1,45 @@
+# encoding: utf-8
+
+module LogStash module Instrument
+ # This class is used in the context when we disable the metric collection
+ # for specific plugin to replace the `NamespacedMetric` class with this one
+ # which doesn't produce any metric to the collector.
+ class NullMetric
+   attr_reader :namespace_name, :collector
+
+   def increment(key, value = 1)
+   end
+
+   def decrement(key, value = 1)
+   end
+
+   def gauge(key, value)
+   end
+
+   def report_time(key, duration)
+   end
+
+   # We have to manually redefine this method since it can return an
+   # object this object also has to be implemented as a NullObject
+   def time(key, &block)
+     if block_given?
+       yield
+     else
+       NullTimedExecution
+     end
+   end
+
+   def namespace(name)
+     self.class.new
+   end
+
+   private
+   # Null implementation of the internal timer class
+   #
+   # @see LogStash::Instrument::TimedExecution`
+   class NullTimedExecution
+     def self.stop
+     end
+   end
+ end
+end; end

data/lib/logstash/logging/json.rb

@@ -0,0 +1,21 @@
+# encoding: utf-8
+require "logstash/namespace"
+require "logstash/logging"
+require "logstash/json"
+
+module LogStash; class Logging; class JSON
+  def initialize(io)
+    raise ArgumentError, "Expected IO, got #{io.class.name}" unless io.is_a?(IO)
+
+    @io = io
+    @lock = Mutex.new
+  end
+
+  def <<(obj)
+    serialized = LogStash::Json.dump(obj)
+    @lock.synchronize do
+      @io.puts(serialized)
+      @io.flush
+    end
+  end
+end; end; end

data/lib/logstash/output_delegator.rb

@@ -74,7 +74,7 @@ module LogStash class OutputDelegator
     @workers << @klass.new(@config)
     @workers.first.register # Needed in case register calls `workers_not_supported`
 
-    @logger.debug("Will start workers for output", :worker_count => target_worker_count, :class => @klass)
+    @logger.debug("Will start workers for output", :worker_count => target_worker_count, :class => @klass.name)
 
     # Threadsafe versions don't need additional workers
     setup_additional_workers!(target_worker_count) unless @threadsafe
@@ -134,7 +134,7 @@ module LogStash class OutputDelegator
   end
 
   def do_close
-    @logger.debug("closing output delegator", :klass => @klass)
+    @logger.debug("closing output delegator", :klass => @klass.name)
 
     if @threadsafe
       @workers.each(&:do_close)
@@ -169,4 +169,4 @@ module LogStash class OutputDelegator
   private
   # Needed for testing, so private
   attr_reader :threadsafe_worker, :worker_queue
-end end
+end end

data/lib/logstash/outputs/base.rb

@@ -27,6 +27,22 @@ class LogStash::Outputs::Base < LogStash::Plugin
 
   attr_reader :worker_plugins, :available_workers, :workers, :worker_plugins, :workers_not_supported
 
+    # Set or return concurrency type
+  def self.concurrency(type=nil)
+    if type
+      @concurrency = type
+      
+      if type == :shared
+        declare_threadsafe!
+      elsif type == :single
+        declare_workers_not_supported!("This plugin only supports one worker!")
+      end
+      
+    else
+      @concurrency || :legacy # default is :legacyo
+    end
+  end
+
   def self.declare_threadsafe!
     declare_workers_not_supported!
     @threadsafe = true
@@ -65,6 +81,8 @@ class LogStash::Outputs::Base < LogStash::Plugin
     # If we're running with a single thread we must enforce single-threaded concurrency by default
     # Maybe in a future version we'll assume output plugins are threadsafe
     @single_worker_mutex = Mutex.new
+    
+    @receives_encoded = self.methods.include?(:multi_receive_encoded)
   end
 
   public
@@ -77,10 +95,23 @@ class LogStash::Outputs::Base < LogStash::Plugin
     raise "#{self.class}#receive must be overidden"
   end # def receive
 
+  public
+  def concurrency
+    self.class.concurrency
+  end
+
   public
   # To be overriden in implementations
   def multi_receive(events)
-    events.each {|event| receive(event) }
+    if @receives_encoded
+      self.multi_receive_encoded(codec.multi_encode(events))
+    else
+      events.each {|event| receive(event) }
+    end
+  end
+
+  def codec
+    params["codec"]
   end
 
   private