logstash-core 5.0.0.alpha4.snapshot3-java → 5.0.0.alpha5.snapshot1-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c90549a9a03e8d32ee0eb8146780ad53609abc84
-  data.tar.gz: 14e872af82cfa7cb8dbfc1b5bbce6e8fcd4e8031
+  metadata.gz: 31b35e84fcbac8fa8ccb7ac01aac834645c04084
+  data.tar.gz: 3c5d884562c45235f897279458b3e0ee8a840d4b
 SHA512:
-  metadata.gz: c666d73e6f87dd8de677888232066d807a2878105a40c023ad0a9d9f221eb9fea16ddf8694a3cd692c816ae92537c23a034e54162c71505990b53ba7a710b7d8
-  data.tar.gz: 45d360a8b9896e2ff744d0e84efce8126c33d5c4c6ce1d71a8fdbf89a61ee6940a2a80843cb7613d3eaa8baab677bf7c717009ac93a7c55b559678c840cc37f1
+  metadata.gz: 2355f8393a9f37222e45f7698fdb2b3cf10be0a52cfb132a3bbaf546dadf11e05f4c2b8f60a216409b5bed7b49eba79803270dbac561217a0e36ac107e2348b2
+  data.tar.gz: 5642f18d54cfb80d5b159575f1ba59b652ec2d57606a5609bda4644740367d7f82187718728c9ccc6854d944291855ec48f54d66b957e3e6e736c52d9c2b9339

data/lib/logstash/agent.rb

@@ -20,7 +20,7 @@ LogStash::Environment.load_locale!
 class LogStash::Agent
   STARTED_AT = Time.now.freeze
 
-  attr_reader :metric, :node_name, :pipelines, :settings
+  attr_reader :metric, :node_name, :pipelines, :settings, :webserver
   attr_accessor :logger
 
   # initialize method for LogStash::Agent
@@ -184,7 +184,11 @@ class LogStash::Agent
     begin
       LogStash::Pipeline.new(config, settings, metric)
     rescue => e
-      @logger.error("fetched an invalid config", :config => config, :reason => e.message)
+      if @logger.debug?
+        @logger.error("fetched an invalid config", :config => config, :reason => e.message, :backtrace => e.backtrace)
+      else
+        @logger.error("fetched an invalid config", :config => config, :reason => e.message)
+      end
       return
     end
   end

data/lib/logstash/api/app_helpers.rb

@@ -5,8 +5,15 @@ module LogStash::Api::AppHelpers
 
   def respond_with(data, options={})
     as     = options.fetch(:as, :json)
+    filter = options.fetch(:filter, "")
     pretty = params.has_key?("pretty")
+
     if as == :json
+      selected_fields = extract_fields(filter.to_s.strip)
+      data.select! { |k,v| selected_fields.include?(k) } unless selected_fields.empty?
+      unless options.include?(:exclude_default_metadata)
+        data = default_metadata.merge(data)
+      end
       content_type "application/json"
       LogStash::Json.dump(data, {:pretty => pretty})
     else
@@ -15,9 +22,17 @@ module LogStash::Api::AppHelpers
     end
   end
 
+  def extract_fields(filter_string)
+    (filter_string.empty? ? [] : filter_string.split(",").map { |s| s.strip.to_sym })
+  end
+
   def as_boolean(string)
     return true   if string == true   || string =~ (/(true|t|yes|y|1)$/i)
     return false  if string == false  || string.blank? || string =~ (/(false|f|no|n|0)$/i)
     raise ArgumentError.new("invalid value for Boolean: \"#{string}\"")
   end
+
+  def default_metadata
+    @factory.build(:default_metadata).all
+  end
 end

data/lib/logstash/api/command_factory.rb

@@ -4,6 +4,7 @@ require "logstash/api/commands/system/basicinfo_command"
 require "logstash/api/commands/system/plugins_command"
 require "logstash/api/commands/stats"
 require "logstash/api/commands/node"
+require "logstash/api/commands/default_metadata"
 
 
 module LogStash
@@ -17,7 +18,8 @@ module LogStash
           :system_basic_info => ::LogStash::Api::Commands::System::BasicInfo,
           :plugins_command => ::LogStash::Api::Commands::System::Plugins,
           :stats => ::LogStash::Api::Commands::Stats,
-          :node => ::LogStash::Api::Commands::Node
+          :node => ::LogStash::Api::Commands::Node,
+          :default_metadata => ::LogStash::Api::Commands::DefaultMetadata
         }
       end
 

data/lib/logstash/api/commands/base.rb

@@ -1,3 +1,5 @@
+# encoding: utf-8
+
 module LogStash
   module Api
     module Commands
@@ -8,14 +10,10 @@ module LogStash
           @service = service
         end
 
-        def hostname
-          service.agent.node_name
-        end
-
         def uptime
           service.agent.uptime
         end
-
+        
         def started_at
           (LogStash::Agent::STARTED_AT.to_f * 1000.0).to_i
         end

data/lib/logstash/api/commands/default_metadata.rb

@@ -0,0 +1,27 @@
+# encoding: utf-8
+
+require "logstash/api/commands/base"
+
+module LogStash
+  module Api
+    module Commands
+      class DefaultMetadata < Commands::Base
+        def all
+          {:host => host, :version => version, :http_address => http_address}
+        end
+        
+        def host
+          Socket.gethostname
+        end
+
+        def version
+          LOGSTASH_CORE_VERSION
+        end
+
+        def http_address
+          service.agent.webserver.address
+        end
+      end
+    end
+  end
+end

data/lib/logstash/api/commands/hot_threads_reporter.rb

@@ -0,0 +1,61 @@
+# encoding: utf-8
+
+class HotThreadsReport
+  HOT_THREADS_STACK_TRACES_SIZE_DEFAULT = 10.freeze
+
+  def initialize(cmd, options)
+    @cmd = cmd
+    filter = { :stacktrace_size => options.fetch(:stacktrace_size, HOT_THREADS_STACK_TRACES_SIZE_DEFAULT) }
+    jr_dump = JRMonitor.threads.generate(filter)
+    @thread_dump = ::LogStash::Util::ThreadDump.new(options.merge(:dump => jr_dump))
+  end
+
+  def to_s
+    hash = to_hash[:hot_threads]
+    report =  "#{I18n.t("logstash.web_api.hot_threads.title", :hostname => hash[:hostname], :time => hash[:time], :top_count => @thread_dump.top_count )} \n"
+    report << '=' * 80
+    report << "\n"
+    hash[:threads].each do |thread|
+      thread_report = ""
+      thread_report = "#{I18n.t("logstash.web_api.
+                                hot_threads.thread_title", :percent_of_cpu_time => thread[:percent_of_cpu_time], :thread_state => thread[:state], :thread_name => thread[:name])} \n"
+      thread_report = "#{thread[:percent_of_cpu_time]} % of of cpu usage by #{thread[:state]} thread named '#{thread[:name]}'\n"
+      thread_report << "#{thread[:path]}\n" if thread[:path]
+      thread[:traces].each do |trace|
+        thread_report << "\t#{trace}\n"
+      end
+      report << thread_report
+      report << '-' * 80
+      report << "\n"
+    end
+    report
+  end
+
+  def to_hash
+    hash = { :time => Time.now.iso8601, :busiest_threads => @thread_dump.top_count, :threads => [] }
+    @thread_dump.each do |thread_name, _hash|
+      thread_name, thread_path = _hash["thread.name"].split(": ")
+      thread = { :name => thread_name,
+                 :percent_of_cpu_time => cpu_time_as_percent(_hash),
+                 :state => _hash["thread.state"]
+      }
+      thread[:path] = thread_path if thread_path
+      traces = []
+      _hash["thread.stacktrace"].each do |trace|
+        traces << trace
+      end
+      thread[:traces] = traces unless traces.empty?
+      hash[:threads] << thread
+    end
+    { :hot_threads => hash }
+  end
+
+  def cpu_time_as_percent(hash)
+    (((cpu_time(hash) / @cmd.uptime * 1.0)*10000).to_i)/100.0
+  end
+
+  def cpu_time(hash)
+    hash["cpu.time"] / 1000000.0
+  end
+
+end

data/lib/logstash/api/commands/node.rb

@@ -1,17 +1,22 @@
+# encoding: utf-8
 require "logstash/api/commands/base"
+require_relative "hot_threads_reporter"
 
 module LogStash
   module Api
     module Commands
       class Node < Commands::Base
-        def all
-          {
+
+        def all(selected_fields=[])
+          payload = {
             :pipeline => pipeline,
             :os => os,
             :jvm => jvm
-          }           
+          }
+          payload.select! { |k,v| selected_fields.include?(k) } unless selected_fields.empty?
+          payload
         end
-        
+
         def pipeline
           extract_metrics(
             [:stats, :pipelines, :main, :config],
@@ -51,65 +56,6 @@ module LogStash
           HotThreadsReport.new(self, options)
         end
 
-        class HotThreadsReport
-          HOT_THREADS_STACK_TRACES_SIZE_DEFAULT = 10.freeze
-          
-          def initialize(cmd, options)
-            @cmd = cmd
-            filter = { :stacktrace_size => options.fetch(:stacktrace_size, HOT_THREADS_STACK_TRACES_SIZE_DEFAULT) }
-            jr_dump = JRMonitor.threads.generate(filter)
-            @thread_dump = ::LogStash::Util::ThreadDump.new(options.merge(:dump => jr_dump))
-          end
-          
-          def to_s
-            hash = to_hash
-            report =  "#{I18n.t("logstash.web_api.hot_threads.title", :hostname => hash[:hostname], :time => hash[:time], :top_count => @thread_dump.top_count )} \n"
-            report << '=' * 80
-            report << "\n"
-            hash[:threads].each do |thread|
-              thread_report = ""
-              thread_report = "#{I18n.t("logstash.web_api.
-                                hot_threads.thread_title", :percent_of_cpu_time => thread[:percent_of_cpu_time], :thread_state => thread[:state], :thread_name => thread[:name])} \n"
-              thread_report = "#{thread[:percent_of_cpu_time]} % of of cpu usage by #{thread[:state]} thread named '#{thread[:name]}'\n"
-              thread_report << "#{thread[:path]}\n" if thread[:path]
-              thread[:traces].each do |trace|
-                thread_report << "\t#{trace}\n"
-              end
-              report << thread_report
-              report << '-' * 80
-              report << "\n"
-            end
-            report
-          end
-
-          def to_hash
-            hash = { :hostname => @cmd.hostname, :time => Time.now.iso8601, :busiest_threads => @thread_dump.top_count, :threads => [] }
-            @thread_dump.each do |thread_name, _hash|
-              thread_name, thread_path = _hash["thread.name"].split(": ")
-              thread = { :name => thread_name,
-                         :percent_of_cpu_time => cpu_time_as_percent(_hash),
-                         :state => _hash["thread.state"]
-                       }
-              thread[:path] = thread_path if thread_path
-              traces = []
-              _hash["thread.stacktrace"].each do |trace|
-                traces << trace
-              end
-              thread[:traces] = traces unless traces.empty?
-              hash[:threads] << thread
-            end
-            hash
-          end
-
-          def cpu_time_as_percent(hash)
-            (((cpu_time(hash) / @cmd.uptime * 1.0)*10000).to_i)/100.0
-          end
-
-          def cpu_time(hash)
-            hash["cpu.time"] / 1000000.0
-          end
-
-        end
       end
     end
   end

data/lib/logstash/api/commands/stats.rb

@@ -1,5 +1,7 @@
+# encoding: utf-8
 require "logstash/api/commands/base"
 require 'logstash/util/thread_dump'
+require_relative "hot_threads_reporter"
 
 module LogStash
   module Api
@@ -11,7 +13,8 @@ module LogStash
               [:jvm, :threads],
               :count,
               :peak_count
-            )
+            ),
+            :mem => memory
           }
         end
 
@@ -60,65 +63,6 @@ module LogStash
           HotThreadsReport.new(self, options)
         end
 
-        class HotThreadsReport
-          HOT_THREADS_STACK_TRACES_SIZE_DEFAULT = 10.freeze
-          
-          def initialize(cmd, options)
-            @cmd = cmd
-            filter = { :stacktrace_size => options.fetch(:stacktrace_size, HOT_THREADS_STACK_TRACES_SIZE_DEFAULT) }
-            jr_dump = JRMonitor.threads.generate(filter)
-            @thread_dump = ::LogStash::Util::ThreadDump.new(options.merge(:dump => jr_dump))
-          end
-          
-          def to_s
-            hash = to_hash
-            report =  "#{I18n.t("logstash.web_api.hot_threads.title", :hostname => hash[:hostname], :time => hash[:time], :top_count => @thread_dump.top_count )} \n"
-            report << '=' * 80
-            report << "\n"
-            hash[:threads].each do |thread|
-              thread_report = ""
-              thread_report = "#{I18n.t("logstash.web_api.
-                                hot_threads.thread_title", :percent_of_cpu_time => thread[:percent_of_cpu_time], :thread_state => thread[:state], :thread_name => thread[:name])} \n"
-              thread_report = "#{thread[:percent_of_cpu_time]} % of of cpu usage by #{thread[:state]} thread named '#{thread[:name]}'\n"
-              thread_report << "#{thread[:path]}\n" if thread[:path]
-              thread[:traces].each do |trace|
-                thread_report << "\t#{trace}\n"
-              end
-              report << thread_report
-              report << '-' * 80
-              report << "\n"
-            end
-            report
-          end
-
-          def to_hash
-            hash = { :hostname => @cmd.hostname, :time => Time.now.iso8601, :busiest_threads => @thread_dump.top_count, :threads => [] }
-            @thread_dump.each do |thread_name, _hash|
-              thread_name, thread_path = _hash["thread.name"].split(": ")
-              thread = { :name => thread_name,
-                         :percent_of_cpu_time => cpu_time_as_percent(_hash),
-                         :state => _hash["thread.state"]
-                       }
-              thread[:path] = thread_path if thread_path
-              traces = []
-              _hash["thread.stacktrace"].each do |trace|
-                traces << trace
-              end
-              thread[:traces] = traces unless traces.empty?
-              hash[:threads] << thread
-            end
-            hash
-          end
-
-          def cpu_time_as_percent(hash)
-            (((cpu_time(hash) / @cmd.uptime * 1.0)*10000).to_i)/100.0
-          end
-
-          def cpu_time(hash)
-            hash["cpu.time"] / 1000000.0
-          end
-        end # class HotThreadsReport
-
         module PluginsStats
           module_function
 
@@ -138,7 +82,7 @@ module LogStash
 
             {
               :events => stats[:events],
-              :pipeline => {
+              :plugins => {
                 :inputs => plugin_stats(stats, :inputs),
                 :filters => plugin_stats(stats, :filters),
                 :outputs => plugin_stats(stats, :outputs)

data/lib/logstash/api/commands/system/basicinfo_command.rb

@@ -1,6 +1,7 @@
 # encoding: utf-8
 require 'logstash/api/commands/base'
 require "logstash/util/duration_formatter"
+require 'logstash/build'
 
 module LogStash
   module Api
@@ -9,12 +10,8 @@ module LogStash
         class BasicInfo < Commands::Base
 
           def run
-            {
-              "hostname" => hostname,
-              "version" => {
-                "number" => LOGSTASH_VERSION
-              }
-            }
+            # Just merge this stuff with the defaults
+            BUILD_INFO
           end
         end
       end

data/lib/logstash/api/modules/base.rb

@@ -1,3 +1,4 @@
+# encoding: utf-8
 require "logstash/api/app_helpers"
 require "logstash/api/command_factory"
 
@@ -15,7 +16,7 @@ module LogStash
         set :raise_errors, true
         set :show_exceptions, false
 
-        attr_reader :factory
+        attr_reader :factory, :agent
 
         include LogStash::Util::Loggable
 
@@ -23,6 +24,7 @@ module LogStash
 
         def initialize(app=nil, agent)
           super(app)
+          @agent = agent
           @factory = ::LogStash::Api::CommandFactory.new(LogStash::Api::Service.new(agent))
         end
 

data/lib/logstash/api/modules/node.rb

@@ -8,23 +8,7 @@ module LogStash
         def node
           factory.build(:node)
         end
-        
-        get "/" do
-          respond_with node.all
-        end
-
-        get "/os" do
-          respond_with :os => node.os
-        end
-
-        get "/jvm" do
-          respond_with :jvm => node.jvm          
-        end
 
-        get "/pipeline" do
-          respond_with :pipeline => node.pipeline
-        end
-        
         get "/hot_threads" do
           ignore_idle_threads = params["ignore_idle_threads"] || true
 
@@ -35,8 +19,14 @@ module LogStash
           options[:threads] = params["threads"].to_i if params.has_key?("threads")
 
           as = options[:human] ? :string : :json
-          respond_with({:hot_threads => node.hot_threads(options)}, {:as => as})
-        end       
+          respond_with(node.hot_threads(options), {:as => as})
+        end
+
+        get "/?:filter?" do
+          selected_fields = extract_fields(params["filter"].to_s.strip)
+          respond_with node.all(selected_fields)
+        end
+
       end
     end
   end

data/lib/logstash/api/modules/node_stats.rb

@@ -3,55 +3,19 @@ module LogStash
   module Api
     module Modules
       class NodeStats < ::LogStash::Api::Modules::Base
-        #set :environment, :test
-        #set :dump_errors, true
-        #set :raise_errors, true
-        #set :logging, Logger.new(STDERR)
-        
-        
+
         before do
           @stats = factory.build(:stats)
         end
 
-        # Global _stats resource where all information is
-        # retrieved and show
-        get "/" do          
+        get "/?:filter?" do
           payload = {
-            :events => events_payload,
             :jvm => jvm_payload,
             :process => process_payload,
-            :mem => mem_payload
+            :mem => mem_payload,
+            :pipeline => pipeline_payload
           }
-
-          respond_with payload
-        end
-
-        # Show all events stats information
-        # (for ingested, emitted, dropped)
-        # - #events since startup
-        # - #data (bytes) since startup
-        # - events/s
-        # - bytes/s
-        # - dropped events/s
-        # - events in the pipeline
-        get "/events" do
-          respond_with({ :events => events_payload })
-        end
-
-        get "/jvm" do
-          respond_with :jvm => jvm_payload
-        end
-
-        get "/process" do
-          respond_with :process => process_payload
-        end
-
-        get "/mem" do
-          respond_with :mem => mem_payload
-        end
-
-        get "/pipeline" do
-          respond_with :pipeline => pipeline_payload
+          respond_with(payload, {:filter => params["filter"]})
         end
 
         private

data/lib/logstash/api/modules/stats.rb

@@ -8,39 +8,6 @@ module LogStash
           factory.build(:stats)
         end
 
-        # Global _stats resource where all information is 
-        # retrieved and show
-        get "/" do
-          payload = {
-            :events => stats_command.events,
-            :jvm => { :memory => stats_command.memory }
-          }
-          respond_with payload
-        end
-
-
-        # return hot threads information
-        get "/jvm" do
-          jvm_payload = {
-            :timestamp => stats_command.started_at,
-            :uptime_in_millis => stats_command.uptime,
-            :mem => stats_command.memory
-          }
-          respond_with({:jvm => jvm_payload})
-        end
-
-        # Show all events stats information
-        # (for ingested, emitted, dropped)
-        # - #events since startup
-        # - #data (bytes) since startup
-        # - events/s
-        # - bytes/s
-        # - dropped events/s
-        # - events in the pipeline
-        get "/events" do
-          respond_with({ :events => stats_command.events })
-        end
-
         # return hot threads information
         get "/jvm/hot_threads" do
           top_threads_count = params["threads"] || 3
@@ -57,6 +24,19 @@ module LogStash
         get "/jvm/memory" do
           respond_with({ :memory => stats_command.memory })
         end
+
+        get "/?:filter?" do
+          payload = {
+            :events => stats_command.events,
+            :jvm => {
+              :timestamp => stats_command.started_at,
+              :uptime_in_millis => stats_command.uptime,
+              :memory => stats_command.memory
+            }
+          }
+          respond_with(payload, {:filter => params["filter"]})
+        end
+
       end
     end
   end

data/lib/logstash/build.rb

@@ -0,0 +1,6 @@
+# encoding: utf-8
+
+# DO NOT EDIT
+# this file acts as a placeholder for build information when executing
+# logstash in dev mode (outside of a package build)
+BUILD_INFO = {}

data/lib/logstash/environment.rb

@@ -3,12 +3,21 @@ require "logstash/errors"
 require "logstash/config/cpu_core_strategy"
 require "logstash/settings"
 require "socket"
+require "stud/temporary"
 
 module LogStash
+  # In the event that we're requiring this file without bootstrap/environment.rb
+  if !defined?(LogStash::Environment::LOGSTASH_HOME)
+    module Environment
+      LOGSTASH_HOME = Stud::Temporary.directory("logstash-home")
+      Dir.mkdir(::File.join(LOGSTASH_HOME, "data"))
+    end
+  end
 
   [
             Setting::String.new("node.name", Socket.gethostname),
             Setting::String.new("path.config", nil, false),
+ Setting::WritableDirectory.new("path.data", ::File.join(LogStash::Environment::LOGSTASH_HOME, "data")),
             Setting::String.new("config.string", nil, false),
            Setting::Boolean.new("config.test_and_exit", false),
            Setting::Boolean.new("config.reload.automatic", false),

data/lib/logstash/filter_delegator.rb

@@ -22,7 +22,7 @@ module LogStash
 
       # Scope the metrics to the plugin
       namespaced_metric = metric.namespace(@filter.plugin_unique_name.to_sym)
-      @filter.metric = metric
+      @filter.metric = namespaced_metric
 
       @metric_events = namespaced_metric.namespace(:events)
       namespaced_metric.gauge(:name, config_name)

data/lib/logstash/instrument/metric.rb

@@ -18,22 +18,22 @@ module LogStash module Instrument
     end
 
     def increment(namespace, key, value = 1)
-      validate_key!(key)
+      self.class.validate_key!(key)
       collector.push(namespace, key, :counter, :increment, value)
     end
 
     def decrement(namespace, key, value = 1)
-      validate_key!(key)
+      self.class.validate_key!(key)
       collector.push(namespace, key, :counter, :decrement, value)
     end
 
     def gauge(namespace, key, value)
-      validate_key!(key)
+      self.class.validate_key!(key)
       collector.push(namespace, key, :gauge, :set, value)
     end
 
     def time(namespace, key)
-      validate_key!(key)
+      self.class.validate_key!(key)
 
       if block_given?
         timer = TimedExecution.new(self, namespace, key)
@@ -46,6 +46,7 @@ module LogStash module Instrument
     end
 
     def report_time(namespace, key, duration)
+      self.class.validate_key!(key)
       collector.push(namespace, key, :counter, :increment, duration)
     end
 
@@ -69,11 +70,11 @@ module LogStash module Instrument
       NamespacedMetric.new(self, name)
     end
 
-    private
-    def validate_key!(key)
+    def self.validate_key!(key)
       raise MetricNoKeyProvided if key.nil? || key.empty?
     end
 
+    private
     # Allow to calculate the execution of a block of code.
     # This class support 2 differents syntax a block or the return of
     # the object itself, but in the later case the metric wont be recorded