logstash-core 5.0.0.alpha4.snapshot1-java → 5.0.0.alpha4.snapshot2-java

data/lib/logstash/api/modules/node_stats.rb

@@ -3,6 +3,11 @@ module LogStash
   module Api
     module Modules
       class NodeStats < ::LogStash::Api::Modules::Base
+        #set :environment, :test
+        #set :dump_errors, true
+        #set :raise_errors, true
+        #set :logging, Logger.new(STDERR)
+        
         
         before do
           @stats = factory.build(:stats)
@@ -14,7 +19,8 @@ module LogStash
           payload = {
             :events => events_payload,
             :jvm => jvm_payload,
-            :process => process_payload
+            :process => process_payload,
+            :mem => mem_payload
           }
 
           respond_with payload
@@ -40,6 +46,10 @@ module LogStash
           respond_with :process => process_payload
         end
 
+        get "/mem" do
+          respond_with :mem => mem_payload
+        end
+
         private
 
         def events_payload
@@ -53,6 +63,10 @@ module LogStash
         def process_payload
           @stats.process
         end
+
+        def mem_payload
+          @stats.memory
+        end
       end
     end
   end

data/lib/logstash/api/rack_app.rb

@@ -73,8 +73,9 @@ module LogStash
         end
       end
       
-      def self.app(logger, environment)
-        namespaces = rack_namespaces 
+      def self.app(logger, agent, environment)
+        namespaces = rack_namespaces(agent)
+
         Rack::Builder.new do
           # Custom logger object. Rack CommonLogger does not work with cabin
           use ApiLogger, logger
@@ -87,21 +88,23 @@ module LogStash
             use ApiErrorHandler, logger
           end
           
-          run LogStash::Api::Modules::Root
+          run LogStash::Api::Modules::Root.new(nil, agent)
           namespaces.each_pair do |namespace, app|
             map(namespace) do
-              run app
+              # Pass down a reference to the current agent
+              # This allow the API to have direct access to the collector
+              run app.new(nil, agent)
             end
           end
         end
       end
 
-      def self.rack_namespaces
+      def self.rack_namespaces(agent)
         {
           "/_node" => LogStash::Api::Modules::Node,
           "/_stats" => LogStash::Api::Modules::Stats,
           "/_node/stats" => LogStash::Api::Modules::NodeStats,
-          "/_plugins" => LogStash::Api::Modules::Plugins
+          "/_node/plugins" => LogStash::Api::Modules::Plugins
         }
       end
     end

data/lib/logstash/api/service.rb

@@ -5,68 +5,29 @@ require "logstash/util/loggable"
 module LogStash
   module Api
     class Service
-
-      include Singleton
       include LogStash::Util::Loggable
 
-      def initialize
-        @snapshot_rotation_mutex = Mutex.new
-        @snapshot = nil
-        logger.debug("[api-service] start") if logger.debug?
-        LogStash::Instrument::Collector.instance.add_observer(self)
-      end
+      attr_reader :agent
 
-      def stop
-        logger.debug("[api-service] stop") if logger.debug?
-        LogStash::Instrument::Collector.instance.delete_observer(self)
-      end
-
-      def agent
-        LogStash::Instrument::Collector.instance.agent
+      def initialize(agent)
+        @agent = agent
+        logger.debug("[api-service] start") if logger.debug?
       end
 
       def started?
-        !@snapshot.nil? && has_counters?        
-      end
-
-      def update(snapshot)
-        logger.debug("[api-service] snapshot received", :snapshot_time => snapshot.created_at) if logger.debug?
-
-        @snapshot_rotation_mutex.synchronize do
-          @snapshot = snapshot
-        end
+        true
       end
 
       def snapshot
-        @snapshot_rotation_mutex.synchronize { @snapshot }
+        agent.metric.collector.snapshot_metric
       end
 
       def get_shallow(*path)
         snapshot.metric_store.get_shallow(*path)
       end
 
-      def get(key)
-        metric_store = @snapshot_rotation_mutex.synchronize { @snapshot.metric_store }
-        if key == :jvm_memory_stats
-          data = metric_store.get_shallow(:jvm, :memory)
-        else
-          data = metric_store.get_with_path("stats/events")
-        end
-        LogStash::Json.dump(data)
-      end
-
-      private
-
-      def has_counters?
-        (["LogStash::Instrument::MetricType::Counter", "LogStash::Instrument::MetricType::Gauge"] - metric_types).empty?
-      end
-
-      def metric_types
-        types = []
-        @snapshot_rotation_mutex.synchronize do
-          types = @snapshot.metric_store.all.map { |t| t.class.to_s }
-        end
-        return types
+      def extract_metrics(path, *keys)
+        snapshot.metric_store.extract_metrics(path, *keys)
       end
     end
   end

data/lib/logstash/config/config_ast.rb

@@ -76,6 +76,14 @@ module LogStash; module Config; module AST
     @defered_conditionals_index = val
   end
 
+  def self.plugin_instance_index
+    @plugin_instance_index
+  end
+
+  def self.plugin_instance_index=(val)
+    @plugin_instance_index = val
+  end
+
   class Node < Treetop::Runtime::SyntaxNode
     def text_value_for_comments
       text_value.gsub(/[\r\n]/, " ")
@@ -86,6 +94,7 @@ module LogStash; module Config; module AST
     def compile
       LogStash::Config::AST.defered_conditionals = []
       LogStash::Config::AST.defered_conditionals_index = 0
+      LogStash::Config::AST.plugin_instance_index = 0
       code = []
 
       code << <<-CODE
@@ -140,7 +149,6 @@ module LogStash; module Config; module AST
     # like @filter_<name>_1
     def initialize(*args)
       super(*args)
-      @i = 0
     end
 
     # Generate ruby code to initialize all the plugins.
@@ -196,9 +204,9 @@ module LogStash; module Config; module AST
 
       plugins.each do |plugin|
         # Unique number for every plugin.
-        @i += 1
+        LogStash::Config::AST.plugin_instance_index += 1
         # store things as ivars, like @filter_grok_3
-        var = :"#{plugin.plugin_type}_#{plugin.plugin_name}_#{@i}"
+        var = :"#{plugin.plugin_type}_#{plugin.plugin_name}_#{LogStash::Config::AST.plugin_instance_index}"
         # puts("var=#{var.inspect}")
         @variables[plugin] = var
       end

data/lib/logstash/config/mixin.rb

@@ -4,6 +4,7 @@ require "logstash/config/registry"
 require "logstash/plugins/registry"
 require "logstash/logging"
 require "logstash/util/password"
+require "logstash/util/safe_uri"
 require "logstash/version"
 require "logstash/environment"
 require "logstash/util/plugin_version"
@@ -327,58 +328,88 @@ module LogStash::Config::Mixin
       return true
     end # def validate_check_invalid_parameter_names
 
+    def validate_check_required_parameter(config_key, config_opts, k, v)
+      if config_key.is_a?(Regexp)
+        (k =~ config_key && v)
+      elsif config_key.is_a?(String)
+        k && v
+      end
+    end
+
     def validate_check_required_parameter_names(params)
       is_valid = true
 
       @config.each do |config_key, config|
         next unless config[:required]
 
-        if config_key.is_a?(Regexp)
-          next if params.keys.select { |k| k =~ config_key }.length > 0
-        elsif config_key.is_a?(String)
-          next if params.keys.member?(config_key)
+        if config_key.is_a?(Regexp) && !params.keys.any? { |k| k =~ config_key }
+          is_valid = false
         end
-        @logger.error(I18n.t("logstash.runner.configuration.setting_missing",
-                             :setting => config_key, :plugin => @plugin_name,
-                             :type => @plugin_type))
-        is_valid = false
+
+        value = params[config_key]
+        if value.nil? || (config[:list] && Array(value).empty?)
+          @logger.error(I18n.t("logstash.runner.configuration.setting_missing",
+                               :setting => config_key, :plugin => @plugin_name,
+                               :type => @plugin_type))
+          is_valid = false
+        end        
       end
 
       return is_valid
     end
 
+    def process_parameter_value(value, config_settings)
+      config_val = config_settings[:validate]
+      
+      if config_settings[:list]
+        value = Array(value) # coerce scalars to lists
+        # Empty lists are converted to nils
+        return true, nil if value.empty?
+          
+        validated_items = value.map {|v| validate_value(v, config_val)}
+        is_valid = validated_items.all? {|sr| sr[0] }
+        processed_value = validated_items.map {|sr| sr[1]}
+      else
+        is_valid, processed_value = validate_value(value, config_val)
+      end
+      
+      return [is_valid, processed_value]
+    end
+
     def validate_check_parameter_values(params)
       # Filter out parametrs that match regexp keys.
       # These are defined in plugins like this:
       #   config /foo.*/ => ... 
-      is_valid = true
+      all_params_valid = true
 
       params.each do |key, value|
         @config.keys.each do |config_key|
           next unless (config_key.is_a?(Regexp) && key =~ config_key) \
                       || (config_key.is_a?(String) && key == config_key)
-          config_val = @config[config_key][:validate]
-          #puts "  Key matches."
-          success, result = validate_value(value, config_val)
-          if success 
-            # Accept coerced value if success
+
+          config_settings = @config[config_key]          
+
+          is_valid, processed_value = process_parameter_value(value, config_settings)
+          
+          if is_valid
+            # Accept coerced value if valid
             # Used for converting values in the config to proper objects.
-            params[key] = result if !result.nil?
+            params[key] = processed_value
           else
             @logger.error(I18n.t("logstash.runner.configuration.setting_invalid",
                                  :plugin => @plugin_name, :type => @plugin_type,
                                  :setting => key, :value => value.inspect,
-                                 :value_type => config_val,
-                                 :note => result))
+                                 :value_type => config_settings[:validate],
+                                 :note => processed_value))
           end
-          #puts "Result: #{key} / #{result.inspect} / #{success}"
-          is_valid &&= success
+          
+          all_params_valid &&= is_valid
 
           break # done with this param key
         end # config.each
       end # params.each
 
-      return is_valid
+      return all_params_valid
     end # def validate_check_parameter_values
 
     def validator_find(key)
@@ -513,6 +544,12 @@ module LogStash::Config::Mixin
             end
 
             result = value.first.is_a?(::LogStash::Util::Password) ? value.first : ::LogStash::Util::Password.new(value.first)
+          when :uri
+            if value.size > 1
+              return false, "Expected uri (one value), got #{value.size} values?"
+            end
+            
+            result = value.first.is_a?(::LogStash::Util::SafeURI) ? value.first : ::LogStash::Util::SafeURI.new(value.first)
           when :path
             if value.size > 1 # Only 1 value wanted
               return false, "Expected path (one value), got #{value.size} values?"
@@ -548,8 +585,9 @@ module LogStash::Config::Mixin
 
     def secure_params!(params)
       params.each do |key, value|
-        if @config[key][:validate] == :password && !value.is_a?(::LogStash::Util::Password)
-          params[key] = ::LogStash::Util::Password.new(value)
+        if [:uri, :password].include? @config[key][:validate]
+          is_valid, processed_value = process_parameter_value(value, @config[key])
+          params[key] = processed_value
         end
       end
     end

data/lib/logstash/inputs/metrics.rb

@@ -21,7 +21,7 @@ module LogStash module Inputs
       @queue = queue
 
       # we register to the collector after receiving the pipeline queue
-      LogStash::Instrument::Collector.instance.add_observer(self)
+      metric.collector.add_observer(self)
 
       # Keep this plugin thread alive,
       # until we shutdown the metric pipeline
@@ -30,7 +30,7 @@ module LogStash module Inputs
 
     def stop
       @logger.debug("Metrics input: stopped")
-      LogStash::Instrument::Collector.instance.delete_observer(self)
+      metric.collector.delete_observer(self)
     end
 
     def update(snapshot)

data/lib/logstash/instrument/collector.rb

@@ -8,7 +8,7 @@ require "singleton"
 require "thread"
 
 module LogStash module Instrument
-  # The Collector singleton is the single point of reference for all
+  # The Collector is the single point of reference for all
   # the metrics collection inside logstash, the metrics library will make
   # direct calls to this class.
   #
@@ -17,7 +17,6 @@ module LogStash module Instrument
   class Collector
     include LogStash::Util::Loggable
     include Observable
-    include Singleton
 
     SNAPSHOT_ROTATION_TIME_SECS = 1 # seconds
     SNAPSHOT_ROTATION_TIMEOUT_INTERVAL_SECS = 10 * 60 # seconds
@@ -59,10 +58,6 @@ module LogStash module Instrument
       end
     end
 
-    def clear
-      @metric_store = MetricStore.new
-    end
-
     # Monitor the `Concurrent::TimerTask` this update is triggered on every successful or not
     # run of the task, TimerTask implement Observable and the collector acts as
     # the observer and will keep track if something went wrong in the execution.
@@ -96,6 +91,10 @@ module LogStash module Instrument
       @snapshot_task.execute
     end
 
+    def stop
+      @snapshot_task.shutdown
+    end
+
     # Create a snapshot of the MetricStore and send it to to the registered observers
     # The observer will receive the following signature in the update methode.
     #

data/lib/logstash/instrument/metric.rb

@@ -13,7 +13,7 @@ module LogStash module Instrument
   class Metric
     attr_reader :collector
 
-    def initialize(collector = LogStash::Instrument::Collector.instance)
+    def initialize(collector)
       @collector = collector
     end
 

data/lib/logstash/instrument/metric_store.rb

@@ -110,6 +110,60 @@ module LogStash module Instrument
       key_paths.reduce(get(*key_paths)) {|acc, p| acc[p]}
     end
 
+
+    # Return a hash including the values of the keys given at the path given
+    # 
+    # Example Usage:
+    # extract_metrics(
+    #   [:jvm, :process],
+    #   :open_file_descriptors,
+    #   [:cpu, [:total_in_millis, :percent]]
+    #   [:pipelines, [:one, :two], :size]
+    # )
+    # 
+    # Returns:
+    # # From the jvm.process metrics namespace
+    # {
+    #   :open_file_descriptors => 123
+    #   :cpu => { :total_in_millis => 456, :percent => 789 }
+    #   :pipelines => {
+    #                   :one => {:size => 90210},
+    #                   :two => {:size => 8675309}
+    #                 }
+    # }
+    def extract_metrics(path, *keys)
+      keys.reduce({}) do |acc,k|
+        # Simplifiy 1-length keys
+        k = k.first if k.is_a?(Array) && k.size == 1
+
+        # If we have array values here we need to recurse
+        # There are two levels of looping here, one for the paths we might pass in
+        # one for the upcoming keys we might pass in
+        if k.is_a?(Array)
+          # We need to build up future executions to extract_metrics
+          # which means building up the path and keys arguments.
+          # We need a nested loop her to execute all permutations of these in case we hit
+          # something like [[:a,:b],[:c,:d]] which produces 4 different metrics
+          next_paths = Array(k.first)
+          next_keys = Array(k[1])
+          rest = k[2..-1]
+          next_paths.each do |next_path|
+            # If there already is a hash at this location use that so we don't overwrite it
+            np_hash = acc[next_path] || {}
+            
+            acc[next_path] = next_keys.reduce(np_hash) do |a,next_key|
+              a.merge! extract_metrics(path + [next_path], [next_key, *rest])
+            end
+          end
+        else # Scalar value
+          res = get_shallow(*path)[k]
+          acc[k] = res ? res.value : nil
+        end
+        
+        acc
+      end
+    end    
+
     # Return all the individuals Metric,
     # This call mimic a Enum's each if a block is provided
     #