logstash-core 2.4.1-java → 5.0.0.alpha1-java

data/lib/logstash/api/lib/app/root.rb

@@ -0,0 +1,13 @@
+# encoding: utf-8
+require "app"
+
+module LogStash::Api
+  class Root < BaseApp
+
+    get "/" do
+      command = factory.build(:system_basic_info)
+      respond_with command.run
+    end
+
+  end
+end

data/lib/logstash/api/lib/app/service.rb

@@ -0,0 +1,61 @@
+# encoding: utf-8
+require "logstash/instrument/collector"
+require "logstash/util/loggable"
+
+class LogStash::Api::Service
+
+  include Singleton
+  include LogStash::Util::Loggable
+
+  def initialize
+    @snapshot_rotation_mutex = Mutex.new
+    @snapshot = nil
+    logger.debug("[api-service] start") if logger.debug?
+    LogStash::Instrument::Collector.instance.add_observer(self)
+  end
+
+  def stop
+    logger.debug("[api-service] stop") if logger.debug?
+    LogStash::Instrument::Collector.instance.delete_observer(self)
+  end
+
+  def agent
+    LogStash::Instrument::Collector.instance.agent
+  end
+
+  def started?
+    !@snapshot.nil? && has_counters?
+  end
+
+  def update(snapshot)
+    logger.debug("[api-service] snapshot received", :snapshot => snapshot) if logger.debug?
+    if @snapshot_rotation_mutex.try_lock
+      @snapshot = snapshot
+      @snapshot_rotation_mutex.unlock
+    end
+  end
+
+  def get(key)
+    metric_store = @snapshot.metric_store
+    if key == :jvm_memory_stats
+      data = metric_store.get_with_path("jvm/memory")[:jvm][:memory]
+    else
+      data = metric_store.get_with_path("stats/events")
+    end
+    LogStash::Json.dump(data)
+  end
+
+  private
+
+  def has_counters?
+    (["LogStash::Instrument::MetricType::Counter", "LogStash::Instrument::MetricType::Gauge"] - metric_types).empty?
+  end
+
+  def metric_types
+    types = []
+    @snapshot_rotation_mutex.synchronize do
+      types = @snapshot.metric_store.all.map { |t| t.class.to_s }
+    end
+    return types
+  end
+end

data/lib/logstash/api/lib/app/stats.rb

@@ -0,0 +1,56 @@
+# encoding: utf-8
+require "app"
+require "app/stats/events_command"
+require "app/stats/hotthreads_command"
+
+module LogStash::Api
+  class Stats < BaseApp
+
+    helpers AppHelpers
+
+
+    # Global _stats resource where all information is 
+    # retrieved and show
+    get "/" do
+      events_command = factory.build(:events_command)
+      memory_command = factory.build(:memory_command)
+      payload = {
+        :events => events_command.run,
+        :jvm => { :memory => memory_command.run }
+      }
+      respond_with payload
+    end
+
+    # Show all events stats information
+    # (for ingested, emitted, dropped)
+    # - #events since startup
+    # - #data (bytes) since startup
+    # - events/s
+    # - bytes/s
+    # - dropped events/s
+    # - events in the pipeline
+    get "/events" do
+      command = factory.build(:events_command)
+      respond_with({ :events => command.run })
+    end
+
+    # return hot threads information
+    get "/jvm/hot_threads" do
+      top_threads_count = params["threads"] || 3
+      ignore_idle_threads = params["ignore_idle_threads"] || true
+      options = {
+        :threads => top_threads_count.to_i,
+        :ignore_idle_threads => as_boolean(ignore_idle_threads)
+      }
+      command = factory.build(:hot_threads_command)
+      respond_with(command.run(options), :string)
+    end
+
+    # return hot threads information
+    get "/jvm/memory" do
+      command = factory.build(:memory_command)
+      respond_with({ :memory => command.run })
+    end
+
+  end
+end

data/lib/logstash/api/lib/helpers/app_helpers.rb

@@ -0,0 +1,23 @@
+# encoding: utf-8
+require "logstash/json"
+
+module LogStash::Api::AppHelpers
+
+  def respond_with(data, options={})
+    as     = options.fetch(:as, :json)
+    pretty = params.has_key?("pretty")
+    if as == :json
+      content_type "application/json"
+      LogStash::Json.dump(data, {:pretty => pretty})
+    else
+      content_type "text/plain"
+      data.to_s
+    end
+  end
+
+  def as_boolean(string)
+    return true   if string == true   || string =~ (/(true|t|yes|y|1)$/i)
+    return false  if string == false  || string.blank? || string =~ (/(false|f|no|n|0)$/i)
+    raise ArgumentError.new("invalid value for Boolean: \"#{string}\"")
+  end
+end

data/lib/logstash/codecs/base.rb

@@ -13,7 +13,6 @@ module LogStash::Codecs; class Base < LogStash::Plugin
     super
     config_init(@params)
     register if respond_to?(:register)
-    setup_multi_encode!
   end
 
   public
@@ -24,37 +23,10 @@ module LogStash::Codecs; class Base < LogStash::Plugin
   alias_method :<<, :decode
 
   public
-  # DEPRECATED: Prefer defining encode_sync or multi_encode
   def encode(event)
-    encoded = multi_encode([event])
-    encoded.each {|event,data| @on_event.call(event,data) }
+    raise "#{self.class}#encode must be overidden"
   end # def encode
 
-  public
-  # Relies on the codec being synchronous (which they all are!)
-  # We need a better long term design here, but this is an improvement
-  # over the current API for shared plugins
-  # It is best if the codec implements this directly
-  def multi_encode(events)
-    if @has_encode_sync              
-      events.map {|event| [event, self.encode_sync(event)]}
-    else
-      batch = Thread.current[:logstash_output_codec_batch] ||= []
-      batch.clear
-      
-      events.each {|event| self.encode(event) }
-      batch
-    end
-  end
-
-  def setup_multi_encode!
-    @has_encode_sync = self.methods.include?(:encode_sync)
-
-    on_event do |event, data|
-      Thread.current[:logstash_output_codec_batch] << [event, data]
-    end
-  end
-
   public
   def close; end;
 

data/lib/logstash/config/config_ast.rb

@@ -76,14 +76,6 @@ module LogStash; module Config; module AST
     @defered_conditionals_index = val
   end
 
-  def self.plugin_instance_index
-    @plugin_instance_index
-  end
-
-  def self.plugin_instance_index=(val)
-    @plugin_instance_index = val
-  end
-
   class Node < Treetop::Runtime::SyntaxNode
     def text_value_for_comments
       text_value.gsub(/[\r\n]/, " ")
@@ -94,7 +86,6 @@ module LogStash; module Config; module AST
     def compile
       LogStash::Config::AST.defered_conditionals = []
       LogStash::Config::AST.defered_conditionals_index = 0
-      LogStash::Config::AST.plugin_instance_index = 0
       code = []
 
       code << <<-CODE
@@ -103,7 +94,6 @@ module LogStash; module Config; module AST
         @outputs = []
         @periodic_flushers = []
         @shutdown_flushers = []
-        @generated_objects = {}
       CODE
 
       sections = recursive_select(LogStash::Config::AST::PluginSection)
@@ -147,9 +137,7 @@ module LogStash; module Config; module AST
   class PluginSection < Node
     # Global plugin numbering for the janky instance variable naming we use
     # like @filter_<name>_1
-    def initialize(*args)
-      super(*args)
-    end
+    @@i = 0
 
     # Generate ruby code to initialize all the plugins.
     def compile_initializer
@@ -159,31 +147,31 @@ module LogStash; module Config; module AST
 
 
         code << <<-CODE
-          @generated_objects[:#{name}] = #{plugin.compile_initializer}
-          @#{plugin.plugin_type}s << @generated_objects[:#{name}]
+          #{name} = #{plugin.compile_initializer}
+          @#{plugin.plugin_type}s << #{name}
         CODE
 
         # The flush method for this filter.
         if plugin.plugin_type == "filter"
 
           code << <<-CODE
-            @generated_objects[:#{name}_flush] = lambda do |options, &block|
-              @logger.debug? && @logger.debug(\"Flushing\", :plugin => @generated_objects[:#{name}])
+            #{name}_flush = lambda do |options, &block|
+              @logger.debug? && @logger.debug(\"Flushing\", :plugin => #{name})
 
-              events = @generated_objects[:#{name}].flush(options)
+              events = #{name}.flush(options)
 
               return if events.nil? || events.empty?
 
-              @logger.debug? && @logger.debug(\"Flushing\", :plugin => @generated_objects[:#{name}], :events => events)
+              @logger.debug? && @logger.debug(\"Flushing\", :plugin => #{name}, :events => events)
 
               #{plugin.compile_starting_here.gsub(/^/, "  ")}
 
               events.each{|e| block.call(e)}
             end
 
-            if @generated_objects[:#{name}].respond_to?(:flush)
-              @periodic_flushers << @generated_objects[:#{name}_flush] if @generated_objects[:#{name}].periodic_flush
-              @shutdown_flushers << @generated_objects[:#{name}_flush]
+            if #{name}.respond_to?(:flush)
+              @periodic_flushers << #{name}_flush if #{name}.periodic_flush
+              @shutdown_flushers << #{name}_flush
             end
           CODE
 
@@ -204,10 +192,9 @@ module LogStash; module Config; module AST
 
       plugins.each do |plugin|
         # Unique number for every plugin.
-        LogStash::Config::AST.plugin_instance_index += 1
+        @@i += 1
         # store things as ivars, like @filter_grok_3
-        var = :"#{plugin.plugin_type}_#{plugin.plugin_name}_#{LogStash::Config::AST.plugin_instance_index}"
-        # puts("var=#{var.inspect}")
+        var = "@#{plugin.plugin_type}_#{plugin.plugin_name}_#{@@i}"
         @variables[plugin] = var
       end
       return @variables
@@ -249,13 +236,13 @@ module LogStash; module Config; module AST
     def compile
       case plugin_type
       when "input"
-        return "start_input(@generated_objects[:#{variable_name}])"
+        return "start_input(#{variable_name})"
       when "filter"
         return <<-CODE
-          events = @generated_objects[:#{variable_name}].multi_filter(events)
+          events = #{variable_name}.multi_filter(events)
         CODE
       when "output"
-        return "targeted_outputs << @generated_objects[:#{variable_name}]\n"
+        return "targeted_outputs << #{variable_name}\n"
       when "codec"
         settings = attributes.recursive_select(Attribute).collect(&:compile).reject(&:empty?)
         attributes_code = "LogStash::Util.hash_merge_many(#{settings.map { |c| "{ #{c} }" }.join(", ")})"
@@ -357,7 +344,7 @@ module LogStash; module Config; module AST
 
       if duplicate_values.size > 0
         raise ConfigurationError.new(
-          I18n.t("logstash.agent.configuration.invalid_plugin_settings_duplicate_keys",
+          I18n.t("logstash.runner.configuration.invalid_plugin_settings_duplicate_keys",
                  :keys => duplicate_values.join(', '),
                  :line => input.line_of(interval.first),
                  :column => input.column_of(interval.first),
@@ -404,7 +391,7 @@ module LogStash; module Config; module AST
       if type == "filter"
         i = LogStash::Config::AST.defered_conditionals_index += 1
         source = <<-CODE
-          @generated_objects[:cond_func_#{i}] = lambda do |input_events|
+          define_singleton_method :cond_func_#{i} do |input_events|
             result = []
             input_events.each do |event|
               events = [event]
@@ -418,7 +405,7 @@ module LogStash; module Config; module AST
         LogStash::Config::AST.defered_conditionals << source
 
         <<-CODE
-          events = @generated_objects[:cond_func_#{i}].call(events)
+          events = cond_func_#{i}(events)
         CODE
       else # Output
         <<-CODE

data/lib/logstash/config/loader.rb

@@ -1,8 +1,6 @@
 require "logstash/config/defaults"
 
 module LogStash; module Config; class Loader
-  attr_accessor :debug_config
-
   def initialize(logger, debug_config=false)
     @logger = logger
     @debug_config = debug_config
@@ -40,7 +38,7 @@ module LogStash; module Config; class Loader
       when "file" then
         local_config(uri.path)
       else
-        fail(I18n.t("logstash.agent.configuration.scheme-not-supported", :path => path))
+        fail(I18n.t("logstash.runner.configuration.scheme-not-supported", :path => path))
       end
     rescue URI::InvalidURIError
       # fallback for windows.
@@ -55,7 +53,7 @@ module LogStash; module Config; class Loader
     path = ::File.join(path, "*") if ::File.directory?(path)
 
     if Dir.glob(path).length == 0
-      fail(I18n.t("logstash.agent.configuration.file-not-found", :path => path))
+      fail(I18n.t("logstash.runner.configuration.file-not-found", :path => path))
     end
 
     config = ""
@@ -91,7 +89,7 @@ module LogStash; module Config; class Loader
     begin
       Net::HTTP.get(uri) + "\n"
     rescue Exception => e
-      fail(I18n.t("logstash.agent.configuration.fetch-failed", :path => uri.to_s, :message => e.message))
+      fail(I18n.t("logstash.runner.configuration.fetch-failed", :path => uri.to_s, :message => e.message))
     end
   end
 end end end

data/lib/logstash/config/mixin.rb

@@ -3,7 +3,6 @@ require "logstash/namespace"
 require "logstash/config/registry"
 require "logstash/logging"
 require "logstash/util/password"
-require "logstash/util/safe_uri"
 require "logstash/version"
 require "logstash/environment"
 require "logstash/util/plugin_version"
@@ -133,7 +132,7 @@ module LogStash::Config::Mixin
 
     if !self.class.validate(params)
       raise LogStash::ConfigurationError,
-        I18n.t("logstash.agent.configuration.invalid_plugin_settings")
+        I18n.t("logstash.runner.configuration.invalid_plugin_settings")
     end
 
     # We remove any config options marked as obsolete,
@@ -334,89 +333,58 @@ module LogStash::Config::Mixin
       return true
     end # def validate_check_invalid_parameter_names
 
-    def validate_check_required_parameter(config_key, config_opts, k, v)
-      if config_key.is_a?(Regexp)
-        (k =~ config_key && v)
-      elsif config_key.is_a?(String)
-        k && v
-      end
-    end
-
     def validate_check_required_parameter_names(params)
       is_valid = true
 
       @config.each do |config_key, config|
         next unless config[:required]
 
-        value = params[config_key]
-
-        if config_key.is_a?(Regexp) && !params.keys.any? { |k| k =~ config_key }
-          is_valid = false
-        end
-        
-        if value.nil? || (config[:list] && Array(value).empty?)
-          @logger.error(I18n.t("logstash.agent.configuration.setting_missing",
-                               :setting => config_key, :plugin => @plugin_name,
-                               :type => @plugin_type))
-          is_valid = false
+        if config_key.is_a?(Regexp)
+          next if params.keys.select { |k| k =~ config_key }.length > 0
+        elsif config_key.is_a?(String)
+          next if params.keys.member?(config_key)
         end
+        @logger.error(I18n.t("logstash.runner.configuration.setting_missing",
+                             :setting => config_key, :plugin => @plugin_name,
+                             :type => @plugin_type))
+        is_valid = false
       end
 
       return is_valid
     end
 
-    def process_parameter_value(value, config_settings)
-      config_val = config_settings[:validate]
-      
-      if config_settings[:list]
-        value = Array(value) # coerce scalars to lists
-        # Empty lists are converted to nils
-        return true, nil if value.empty?
-          
-        validated_items = value.map {|v| validate_value(v, config_val)}
-        is_valid = validated_items.all? {|sr| sr[0] }
-        processed_value = validated_items.map {|sr| sr[1]}
-      else
-        is_valid, processed_value = validate_value(value, config_val)
-      end
-      
-      return [is_valid, processed_value]
-    end
-
     def validate_check_parameter_values(params)
       # Filter out parametrs that match regexp keys.
       # These are defined in plugins like this:
       #   config /foo.*/ => ... 
-      all_params_valid = true
+      is_valid = true
 
       params.each do |key, value|
         @config.keys.each do |config_key|
           next unless (config_key.is_a?(Regexp) && key =~ config_key) \
                       || (config_key.is_a?(String) && key == config_key)
-
-          config_settings = @config[config_key]          
-
-          is_valid, processed_value = process_parameter_value(value, config_settings)
-          
-          if is_valid
-            # Accept coerced value if valid
+          config_val = @config[config_key][:validate]
+          #puts "  Key matches."
+          success, result = validate_value(value, config_val)
+          if success 
+            # Accept coerced value if success
             # Used for converting values in the config to proper objects.
-            params[key] = processed_value
+            params[key] = result if !result.nil?
           else
-            @logger.error(I18n.t("logstash.agent.configuration.setting_invalid",
+            @logger.error(I18n.t("logstash.runner.configuration.setting_invalid",
                                  :plugin => @plugin_name, :type => @plugin_type,
                                  :setting => key, :value => value.inspect,
-                                 :value_type => config_settings[:validate],
-                                 :note => processed_value))
+                                 :value_type => config_val,
+                                 :note => result))
           end
-          
-          all_params_valid &&= is_valid
+          #puts "Result: #{key} / #{result.inspect} / #{success}"
+          is_valid &&= success
 
           break # done with this param key
         end # config.each
       end # params.each
 
-      return all_params_valid
+      return is_valid
     end # def validate_check_parameter_values
 
     def validator_find(key)
@@ -437,7 +405,7 @@ module LogStash::Config::Mixin
       result = nil
 
       if validator.nil?
-        return true, value
+        return true
       elsif validator.is_a?(Array)
         value = [*value]
         if value.size > 1
@@ -551,12 +519,6 @@ module LogStash::Config::Mixin
             end
 
             result = value.first.is_a?(::LogStash::Util::Password) ? value.first : ::LogStash::Util::Password.new(value.first)
-          when :uri
-            if value.size > 1
-              return false, "Expected uri (one value), got #{value.size} values?"
-            end
-            
-            result = value.first.is_a?(::LogStash::Util::SafeURI) ? value.first : ::LogStash::Util::SafeURI.new(value.first)
           when :path
             if value.size > 1 # Only 1 value wanted
               return false, "Expected path (one value), got #{value.size} values?"
@@ -592,9 +554,8 @@ module LogStash::Config::Mixin
 
     def secure_params!(params)
       params.each do |key, value|
-        if [:uri, :password].include? @config[key][:validate]
-          is_valid, processed_value = process_parameter_value(value, @config[key])
-          params[key] = processed_value
+        if @config[key][:validate] == :password && !value.is_a?(::LogStash::Util::Password)
+          params[key] = ::LogStash::Util::Password.new(value)
         end
       end
     end