logstash-core 5.0.0.alpha2.snapshot2-java → 5.0.0.alpha3-java

data/lib/logstash/api/modules/node.rb

@@ -0,0 +1,24 @@
+# encoding: utf-8
+module LogStash
+  module Api
+    module Modules
+      class Node < ::LogStash::Api::Modules::Base
+        # return hot threads information
+        get "/hot_threads" do
+          ignore_idle_threads = params["ignore_idle_threads"] || true
+
+          options = {
+            :ignore_idle_threads => as_boolean(ignore_idle_threads),
+            :human => params.has_key?("human")
+          }
+          options[:threads] = params["threads"].to_i if params.has_key?("threads")
+
+          stats = factory.build(:stats)
+          as    = options[:human] ? :string : :json
+          respond_with(stats.hot_threads(options), {:as => as})
+        end
+
+      end
+    end
+  end
+end

data/lib/logstash/api/modules/node_stats.rb

@@ -0,0 +1,59 @@
+# encoding: utf-8
+module LogStash
+  module Api
+    module Modules
+      class NodeStats < ::LogStash::Api::Modules::Base
+        
+        before do
+          @stats = factory.build(:stats)
+        end
+
+        # Global _stats resource where all information is
+        # retrieved and show
+        get "/" do          
+          payload = {
+            :events => events_payload,
+            :jvm => jvm_payload,
+            :process => process_payload
+          }
+
+          respond_with payload
+        end
+
+        # Show all events stats information
+        # (for ingested, emitted, dropped)
+        # - #events since startup
+        # - #data (bytes) since startup
+        # - events/s
+        # - bytes/s
+        # - dropped events/s
+        # - events in the pipeline
+        get "/events" do
+          respond_with({ :events => events_payload })
+        end
+
+        get "/jvm" do
+          respond_with :jvm => jvm_payload
+        end
+
+        get "/process" do
+          respond_with :process => process_payload
+        end
+
+        private
+
+        def events_payload
+          @stats.events
+        end
+
+        def jvm_payload
+          @stats.jvm
+        end
+
+        def process_payload
+          @stats.process
+        end
+      end
+    end
+  end
+end

data/lib/logstash/api/modules/plugins.rb

@@ -0,0 +1,15 @@
+# encoding: utf-8
+module LogStash
+  module Api
+    module Modules
+      class Plugins < ::LogStash::Api::Modules::Base
+
+        get "/" do
+          command = factory.build(:plugins_command)
+          respond_with(command.run())
+        end
+
+      end
+    end
+  end
+end

data/lib/logstash/api/modules/root.rb

@@ -0,0 +1,15 @@
+# encoding: utf-8
+module LogStash
+  module Api
+    module Modules
+      class Root < ::LogStash::Api::Modules::Base
+        
+        get "/" do
+          command = factory.build(:system_basic_info)
+          respond_with command.run
+        end
+        
+      end
+    end
+  end
+end

data/lib/logstash/api/modules/stats.rb

@@ -0,0 +1,63 @@
+# encoding: utf-8
+module LogStash
+  module Api
+    module Modules
+      class Stats < ::LogStash::Api::Modules::Base
+
+        def stats_command
+          factory.build(:stats)
+        end
+
+        # Global _stats resource where all information is 
+        # retrieved and show
+        get "/" do
+          payload = {
+            :events => stats_command.events,
+            :jvm => { :memory => stats_command.memory }
+          }
+          respond_with payload
+        end
+
+
+        # return hot threads information
+        get "/jvm" do
+          jvm_payload = {
+            :timestamp => stats_command.started_at,
+            :uptime_in_millis => stats_command.uptime,
+            :mem => stats_command.memory
+          }
+          respond_with({:jvm => jvm_payload})
+        end
+
+        # Show all events stats information
+        # (for ingested, emitted, dropped)
+        # - #events since startup
+        # - #data (bytes) since startup
+        # - events/s
+        # - bytes/s
+        # - dropped events/s
+        # - events in the pipeline
+        get "/events" do
+          respond_with({ :events => stats_command.events })
+        end
+
+        # return hot threads information
+        get "/jvm/hot_threads" do
+          top_threads_count = params["threads"] || 3
+          ignore_idle_threads = params["ignore_idle_threads"] || true
+          options = {
+            :threads => top_threads_count.to_i,
+            :ignore_idle_threads => as_boolean(ignore_idle_threads)
+          }
+
+          respond_with(stats_command.hot_threads(options))
+        end
+
+        # return hot threads information
+        get "/jvm/memory" do
+          respond_with({ :memory => stats_command.memory })
+        end
+      end
+    end
+  end
+end

data/lib/logstash/api/rack_app.rb

@@ -0,0 +1,33 @@
+require "logstash/api/modules/base"
+require "logstash/api/modules/node"
+require "logstash/api/modules/node_stats"
+require "logstash/api/modules/plugins"
+require "logstash/api/modules/root"
+require "logstash/api/modules/stats"
+
+module LogStash
+  module Api
+    module RackApp
+      def self.app
+        namespaces = rack_namespaces 
+        Rack::Builder.new do
+          run LogStash::Api::Modules::Root
+          namespaces.each_pair do |namespace, app|
+            map(namespace) do
+              run app
+            end
+          end
+        end
+      end
+
+      def self.rack_namespaces
+        {
+          "/_node" => LogStash::Api::Modules::Node,
+          "/_stats" => LogStash::Api::Modules::Stats,
+          "/_node/stats" => LogStash::Api::Modules::NodeStats,
+          "/_plugins" => LogStash::Api::Modules::Plugins
+        }
+      end
+    end
+  end
+end

data/lib/logstash/api/service.rb

@@ -0,0 +1,73 @@
+# encoding: utf-8
+require "logstash/instrument/collector"
+require "logstash/util/loggable"
+
+module LogStash
+  module Api
+    class Service
+
+      include Singleton
+      include LogStash::Util::Loggable
+
+      def initialize
+        @snapshot_rotation_mutex = Mutex.new
+        @snapshot = nil
+        logger.debug("[api-service] start") if logger.debug?
+        LogStash::Instrument::Collector.instance.add_observer(self)
+      end
+
+      def stop
+        logger.debug("[api-service] stop") if logger.debug?
+        LogStash::Instrument::Collector.instance.delete_observer(self)
+      end
+
+      def agent
+        LogStash::Instrument::Collector.instance.agent
+      end
+
+      def started?
+        !@snapshot.nil? && has_counters?
+      end
+
+      def update(snapshot)
+        logger.debug("[api-service] snapshot received", :snapshot_time => snapshot.created_at) if logger.debug?
+
+        @snapshot_rotation_mutex.synchronize do
+          @snapshot = snapshot
+        end
+      end
+
+      def snapshot
+        @snapshot_rotation_mutex.synchronize { @snapshot }
+      end
+
+      def get_shallow(*path)
+        snapshot.metric_store.get_shallow(*path)
+      end
+
+      def get(key)
+        metric_store = @snapshot_rotation_mutex.synchronize { @snapshot.metric_store }
+        if key == :jvm_memory_stats
+          data = metric_store.get_shallow(:jvm, :memory)
+        else
+          data = metric_store.get_with_path("stats/events")
+        end
+        LogStash::Json.dump(data)
+      end
+
+      private
+
+      def has_counters?
+        (["LogStash::Instrument::MetricType::Counter", "LogStash::Instrument::MetricType::Gauge"] - metric_types).empty?
+      end
+
+      def metric_types
+        types = []
+        @snapshot_rotation_mutex.synchronize do
+          types = @snapshot.metric_store.all.map { |t| t.class.to_s }
+        end
+        return types
+      end
+    end
+  end
+end

data/lib/logstash/config/config_ast.rb

@@ -94,6 +94,7 @@ module LogStash; module Config; module AST
         @outputs = []
         @periodic_flushers = []
         @shutdown_flushers = []
+        @generated_objects = {}
       CODE
 
       sections = recursive_select(LogStash::Config::AST::PluginSection)
@@ -137,7 +138,10 @@ module LogStash; module Config; module AST
   class PluginSection < Node
     # Global plugin numbering for the janky instance variable naming we use
     # like @filter_<name>_1
-    @@i = 0
+    def initialize(*args)
+      super(*args)
+      @i = 0
+    end
 
     # Generate ruby code to initialize all the plugins.
     def compile_initializer
@@ -147,31 +151,31 @@ module LogStash; module Config; module AST
 
 
         code << <<-CODE
-          #{name} = #{plugin.compile_initializer}
-          @#{plugin.plugin_type}s << #{name}
+          @generated_objects[:#{name}] = #{plugin.compile_initializer}
+          @#{plugin.plugin_type}s << @generated_objects[:#{name}]
         CODE
 
         # The flush method for this filter.
         if plugin.plugin_type == "filter"
 
           code << <<-CODE
-            #{name}_flush = lambda do |options, &block|
-              @logger.debug? && @logger.debug(\"Flushing\", :plugin => #{name})
+            @generated_objects[:#{name}_flush] = lambda do |options, &block|
+              @logger.debug? && @logger.debug(\"Flushing\", :plugin => @generated_objects[:#{name}])
 
-              events = #{name}.flush(options)
+              events = @generated_objects[:#{name}].flush(options)
 
               return if events.nil? || events.empty?
 
-              @logger.debug? && @logger.debug(\"Flushing\", :plugin => #{name}, :events => events)
+              @logger.debug? && @logger.debug(\"Flushing\", :plugin => @generated_objects[:#{name}], :events => events)
 
               #{plugin.compile_starting_here.gsub(/^/, "  ")}
 
               events.each{|e| block.call(e)}
             end
 
-            if #{name}.respond_to?(:flush)
-              @periodic_flushers << #{name}_flush if #{name}.periodic_flush
-              @shutdown_flushers << #{name}_flush
+            if @generated_objects[:#{name}].respond_to?(:flush)
+              @periodic_flushers << @generated_objects[:#{name}_flush] if @generated_objects[:#{name}].periodic_flush
+              @shutdown_flushers << @generated_objects[:#{name}_flush]
             end
           CODE
 
@@ -192,9 +196,10 @@ module LogStash; module Config; module AST
 
       plugins.each do |plugin|
         # Unique number for every plugin.
-        @@i += 1
+        @i += 1
         # store things as ivars, like @filter_grok_3
-        var = "@#{plugin.plugin_type}_#{plugin.plugin_name}_#{@@i}"
+        var = :"#{plugin.plugin_type}_#{plugin.plugin_name}_#{@i}"
+        # puts("var=#{var.inspect}")
         @variables[plugin] = var
       end
       return @variables
@@ -236,13 +241,13 @@ module LogStash; module Config; module AST
     def compile
       case plugin_type
       when "input"
-        return "start_input(#{variable_name})"
+        return "start_input(@generated_objects[:#{variable_name}])"
       when "filter"
         return <<-CODE
-          events = #{variable_name}.multi_filter(events)
+          events = @generated_objects[:#{variable_name}].multi_filter(events)
         CODE
       when "output"
-        return "targeted_outputs << #{variable_name}\n"
+        return "targeted_outputs << @generated_objects[:#{variable_name}]\n"
       when "codec"
         settings = attributes.recursive_select(Attribute).collect(&:compile).reject(&:empty?)
         attributes_code = "LogStash::Util.hash_merge_many(#{settings.map { |c| "{ #{c} }" }.join(", ")})"
@@ -391,7 +396,7 @@ module LogStash; module Config; module AST
       if type == "filter"
         i = LogStash::Config::AST.defered_conditionals_index += 1
         source = <<-CODE
-          define_singleton_method :cond_func_#{i} do |input_events|
+          @generated_objects[:cond_func_#{i}] = lambda do |input_events|
             result = []
             input_events.each do |event|
               events = [event]
@@ -405,7 +410,7 @@ module LogStash; module Config; module AST
         LogStash::Config::AST.defered_conditionals << source
 
         <<-CODE
-          events = cond_func_#{i}(events)
+          events = @generated_objects[:cond_func_#{i}].call(events)
         CODE
       else # Output
         <<-CODE
@@ -512,7 +517,7 @@ module LogStash; module Config; module AST
   end
   class Selector < RValue
     def compile
-      return "event[#{text_value.inspect}]"
+      return "event.get(#{text_value.inspect})"
     end
   end
   class SelectorElement < Node; end

data/lib/logstash/config/loader.rb

@@ -1,9 +1,9 @@
 require "logstash/config/defaults"
 
 module LogStash; module Config; class Loader
-  def initialize(logger, debug_config=false)
+  def initialize(logger)
     @logger = logger
-    @debug_config = debug_config
+    @config_debug = LogStash::SETTINGS.get_value("config.debug")
   end
 
   def format_config(config_path, config_string)
@@ -70,7 +70,7 @@ module LogStash; module Config; class Loader
         encoding_issue_files << file
       end
       config << cfg + "\n"
-      if @debug_config
+      if @config_debug
         @logger.debug? && @logger.debug("\nThe following is the content of a file", :config_file => file.to_s)
         @logger.debug? && @logger.debug("\n" + cfg + "\n\n")
       end
@@ -78,7 +78,7 @@ module LogStash; module Config; class Loader
     if encoding_issue_files.any?
       fail("The following config files contains non-ascii characters but are not UTF-8 encoded #{encoding_issue_files}")
     end
-    if @debug_config
+    if @config_debug
       @logger.debug? && @logger.debug("\nThe following is the merged configuration")
       @logger.debug? && @logger.debug("\n" + config + "\n\n")
     end

data/lib/logstash/config/mixin.rb

@@ -39,7 +39,6 @@ module LogStash::Config::Mixin
   PLUGIN_VERSION_1_0_0 = LogStash::Util::PluginVersion.new(1, 0, 0)
   PLUGIN_VERSION_0_9_0 = LogStash::Util::PluginVersion.new(0, 9, 0)
 
-  ALLOW_ENV_FLAG = "__ALLOW_ENV__"
   ENV_PLACEHOLDER_REGEX = /\$\{(?<name>\w+)(\:(?<default>[^}]*))?\}/
 
   # This method is called when someone does 'include LogStash::Config'
@@ -49,14 +48,6 @@ module LogStash::Config::Mixin
   end
 
   def config_init(params)
-    # HACK(talevy): https://github.com/elastic/logstash/issues/4958
-    # Currently, the regular plugins params argument is hijacked
-    # to pass along the `allow_env` configuration variable. This was done as to 
-    # not change the method signature of Plugin. This also makes it difficul to 
-    # reason about at the same time. ALLOW_ENV_FLAG is a special param that users 
-    # are now forbidden to set in their configuration definitions.
-    allow_env = params.delete(LogStash::Config::Mixin::ALLOW_ENV_FLAG) { false }
-
     # Validation will modify the values inside params if necessary.
     # For example: converting a string to a number, etc.
     
@@ -112,20 +103,18 @@ module LogStash::Config::Mixin
     end
 
     # Resolve environment variables references
-    if allow_env
-        params.each do |name, value|
-        if (value.is_a?(Hash))
-          value.each do |valueHashKey, valueHashValue|
-            value[valueHashKey.to_s] = replace_env_placeholders(valueHashValue)
+    params.each do |name, value|
+      if (value.is_a?(Hash))
+        value.each do |valueHashKey, valueHashValue|
+          value[valueHashKey.to_s] = replace_env_placeholders(valueHashValue)
+        end
+      else
+        if (value.is_a?(Array))
+          value.each_index do |valueArrayIndex|
+            value[valueArrayIndex] = replace_env_placeholders(value[valueArrayIndex])
           end
         else
-          if (value.is_a?(Array))
-            value.each_index do |valueArrayIndex|
-              value[valueArrayIndex] = replace_env_placeholders(value[valueArrayIndex])
-            end
-          else
-            params[name.to_s] = replace_env_placeholders(value)
-          end
+          params[name.to_s] = replace_env_placeholders(value)
         end
       end
     end