logstash-core 5.6.16-java → 6.0.0.alpha1-java

data/lib/logstash/api/commands/default_metadata.rb

@@ -12,7 +12,7 @@ module LogStash
         end
 
         def host
-          @@host ||= Socket.gethostname
+          Socket.gethostname
         end
 
         def version

data/lib/logstash/api/commands/hot_threads_reporter.rb

@@ -1,5 +1,4 @@
 # encoding: utf-8
-java_import 'org.logstash.instrument.reports.ThreadsReport'
 
 class HotThreadsReport
   STRING_SEPARATOR_LENGTH = 80.freeze
@@ -8,7 +7,8 @@ class HotThreadsReport
   def initialize(cmd, options)
     @cmd = cmd
     filter = { :stacktrace_size => options.fetch(:stacktrace_size, HOT_THREADS_STACK_TRACES_SIZE_DEFAULT) }
-    @thread_dump = ::LogStash::Util::ThreadDump.new(options.merge(:dump => ThreadsReport.generate(filter)))
+    jr_dump = JRMonitor.threads.generate(filter)
+    @thread_dump = ::LogStash::Util::ThreadDump.new(options.merge(:dump => jr_dump))
   end
 
   def to_s
@@ -17,12 +17,7 @@ class HotThreadsReport
     report << '=' * STRING_SEPARATOR_LENGTH
     report << "\n"
     hash[:threads].each do |thread|
-      line_str = I18n.t("logstash.web_api.hot_threads.thread_title", 
-        :percent_of_cpu_time => thread[:percent_of_cpu_time], 
-        :thread_state => thread[:state], 
-        :thread_name => thread[:name],
-        :thread_id => thread[:thread_id])
-      thread_report = "#{line_str} \n"
+      thread_report = "#{I18n.t("logstash.web_api.hot_threads.thread_title", :percent_of_cpu_time => thread[:percent_of_cpu_time], :thread_state => thread[:state], :thread_name => thread[:name])} \n"
       thread_report << "#{thread[:path]}\n" if thread[:path]
       thread[:traces].each do |trace|
         thread_report << "\t#{trace}\n"
@@ -36,10 +31,9 @@ class HotThreadsReport
 
   def to_hash
     hash = { :time => Time.now.iso8601, :busiest_threads => @thread_dump.top_count, :threads => [] }
-    @thread_dump.each do |_hash|
+    @thread_dump.each do |thread_name, _hash|
       thread_name, thread_path = _hash["thread.name"].split(": ")
       thread = { :name => thread_name,
-                 :thread_id => _hash["thread.id"],
                  :percent_of_cpu_time => cpu_time_as_percent(_hash),
                  :state => _hash["thread.state"]
       }
@@ -48,7 +42,7 @@ class HotThreadsReport
       _hash["thread.stacktrace"].each do |trace|
         traces << trace
       end
-      thread[:traces] = traces
+      thread[:traces] = traces unless traces.empty?
       hash[:threads] << thread
     end
     { :hot_threads => hash }

data/lib/logstash/api/commands/node.rb

@@ -20,8 +20,8 @@ module LogStash
         def pipeline(pipeline_id = LogStash::SETTINGS.get("pipeline.id").to_sym)
           stats = extract_metrics(
             [:stats, :pipelines, pipeline_id, :config],
-            :workers, :batch_size, :batch_delay, :config_reload_automatic, :config_reload_interval, :dead_letter_queue_enabled, :dead_letter_queue_path
-          ).reject{|_, v|v.nil?}
+            :workers, :batch_size, :batch_delay, :config_reload_automatic, :config_reload_interval
+          )
           stats.merge(:id => pipeline_id)
         end
 
@@ -40,6 +40,7 @@ module LogStash
           {
             :pid =>  ManagementFactory.getRuntimeMXBean().getName().split("@").first.to_i,
             :version => java.lang.System.getProperty("java.version"),
+            :vm_name => java.lang.System.getProperty("java.vm.name"),
             :vm_version => java.lang.System.getProperty("java.version"),
             :vm_vendor => java.lang.System.getProperty("java.vendor"),
             :vm_name => java.lang.System.getProperty("java.vm.name"),

data/lib/logstash/api/commands/stats.rb

@@ -54,6 +54,7 @@ module LogStash
         def memory
           memory = service.get_shallow(:jvm, :memory)
           {
+            :heap_used_in_bytes => memory[:heap][:used_in_bytes],
             :heap_used_percent => memory[:heap][:used_percent],
             :heap_committed_in_bytes => memory[:heap][:committed_in_bytes],
             :heap_max_in_bytes => memory[:heap][:max_in_bytes],
@@ -107,8 +108,8 @@ module LogStash
               },
               :reloads => stats[:reloads],
               :queue => stats[:queue]
-            }.merge(stats[:dlq] ? {:dead_letter_queue => stats[:dlq]} : {})
-            end
+            }
+          end
         end # module PluginsStats
       end
     end

data/lib/logstash/bootstrap_check/bad_java.rb

@@ -0,0 +1,16 @@
+# encoding: utf-8
+require "logstash/util"
+require "logstash/util/java_version"
+require "logstash/errors"
+
+module LogStash module BootstrapCheck
+  class BadJava
+    def self.check(settings)
+      # Exit on bad java versions
+      LogStash::Util::JavaVersion.validate_java_version!
+    rescue => e
+      # Just rewrap the original exception
+      raise LogStash::BootstrapCheckError, e.message
+    end
+  end
+end end

data/lib/logstash/bootstrap_check/bad_ruby.rb

@@ -0,0 +1,12 @@
+# encoding: utf-8
+require "logstash/errors"
+
+module LogStash module BootstrapCheck
+  class BadRuby
+    def self.check(settings)
+      if RUBY_VERSION < "1.9.2"
+        raise LogStash::BootstrapCheckError, "Ruby 1.9.2 or later is required. (You are running: " + RUBY_VERSION + ")"
+      end
+    end
+  end
+end end

data/lib/logstash/bootstrap_check/default_config.rb

@@ -0,0 +1,17 @@
+# encoding: utf-8
+require "logstash/errors"
+
+module LogStash module BootstrapCheck
+  class DefaultConfig
+    def self.check(settings)
+      if settings.get("config.string").nil? && settings.get("path.config").nil?
+        raise LogStash::BootstrapCheckError, I18n.t("logstash.runner.missing-configuration")
+      end
+
+      if settings.get("config.reload.automatic") && settings.get("path.config").nil?
+        # there's nothing to reload
+        raise LogStash::BootstrapCheckError, I18n.t("logstash.runner.reload-without-config-path")
+      end
+    end
+  end
+end end

data/lib/logstash/compiler.rb

@@ -0,0 +1,38 @@
+require 'logstash/util/loggable'
+require 'logstash/compiler/lscl/lscl_grammar'
+
+java_import org.logstash.config.ir.Pipeline
+java_import org.logstash.config.ir.graph.Graph;
+java_import org.logstash.config.ir.graph.PluginVertex;
+
+module LogStash; class Compiler
+  include ::LogStash::Util::Loggable
+
+  def self.compile_pipeline(config_str, source_file=nil)
+    graph_sections = self.compile_graph(config_str, source_file)
+    pipeline = org.logstash.config.ir.Pipeline.new(
+      graph_sections[:input],
+      graph_sections[:filter],
+      graph_sections[:output]
+    )
+  end
+
+  def self.compile_ast(config_str, source_file=nil)
+    grammar = LogStashCompilerLSCLGrammarParser.new
+    config = grammar.parse(config_str)
+
+    if config.nil?
+      raise ConfigurationError, grammar.failure_reason
+    end
+
+    config
+  end
+
+  def self.compile_imperative(config_str, source_file=nil)
+    compile_ast(config_str, source_file).compile(source_file)
+  end
+
+  def self.compile_graph(config_str, source_file=nil)
+    Hash[compile_imperative(config_str, source_file).map {|section,icompiled| [section, icompiled.toGraph]}]
+  end
+end; end

data/lib/logstash/compiler/lscl.rb

@@ -0,0 +1,566 @@
+# encoding: utf-8
+require 'logstash/errors'
+require "treetop"
+require "logstash/compiler/treetop_monkeypatches"
+java_import org.logstash.config.ir.DSL
+java_import org.logstash.common.SourceWithMetadata
+
+module LogStashCompilerLSCLGrammar; module LogStash; module Compiler; module LSCL; module AST
+  # Helpers for parsing LSCL files
+  module Helpers
+    def source_meta
+      line, column = line_and_column
+      org.logstash.common.SourceWithMetadata.new(source_file, line, column, self.text_value)
+    end
+
+    def source_file=(value)
+      set_meta(:source_file, value)
+    end
+    
+    def source_file
+      get_meta(:source_file)
+    end
+
+    def compose(*statements)
+      compose_for(section_type.to_sym).call(source_meta, *statements)
+    end
+
+    def compose_for(section_sym)
+      if section_sym == :filter
+        jdsl.method(:iComposeSequence)
+      else
+        jdsl.method(:iComposeParallel)
+      end
+    end
+
+    def line_and_column
+      start = self.interval.first
+      [self.input.line_of(start), self.input.column_of(start)]
+    end
+
+    def empty_source_meta()
+      org.logstash.common.SourceWithMetadata.new()
+    end
+
+    def jdsl
+      org.logstash.config.ir.DSL
+    end
+
+    def self.jdsl
+      org.logstash.config.ir.DSL
+    end
+    
+    AND_METHOD = jdsl.method(:eAnd)
+    OR_METHOD = jdsl.method(:eOr)
+  end
+  
+  class Node < Treetop::Runtime::SyntaxNode
+    include Helpers
+    
+    def section_type
+      if recursive_select_parent(Plugin).any?
+        return "codec"
+      else
+        section = recursive_select_parent(PluginSection)
+        return section.first.plugin_type.text_value
+      end
+    end
+  end
+
+  class Config < Node
+    include Helpers
+    
+    def compile(source_file=nil)
+      # There is no way to move vars across nodes in treetop :(
+      self.source_file = source_file
+
+      sections = recursive_select(PluginSection)
+
+      section_map = {
+        :input  => [],
+        :filter => [],
+        :output => []
+      }
+
+      sections.each do |section|
+        section_name = section.plugin_type.text_value.to_sym
+        section_expr = section.expr
+        raise "Unknown section name #{section_name}!" if ![:input, :output, :filter].include?(section_name)
+        ::Array[section_expr].each do |se|
+          section_map[section_name].concat se
+        end
+      end
+
+      section_map.keys.each do |key|
+        section_map[key] = compose_for(key).call(empty_source_meta, *section_map[key])
+      end
+
+      section_map
+    end
+  end
+
+  class Comment < Node; end
+  class Whitespace < Node; end
+  
+  class PluginSection < Node
+    def expr
+      recursive_select(Branch, Plugin).map(&:expr)
+    end
+  end
+
+  class Plugins < Node; end
+  class Plugin < Node
+    def expr
+      jdsl.iPlugin(source_meta, plugin_type_enum, self.plugin_name, self.expr_attributes)
+    end
+
+    def plugin_type_enum
+      case section_type
+      when "input"
+        Java::OrgLogstashConfigIr::PluginDefinition::Type::INPUT
+      when "codec"
+        Java::OrgLogstashConfigIr::PluginDefinition::Type::CODEC
+      when "filter"
+        Java::OrgLogstashConfigIr::PluginDefinition::Type::FILTER
+      when "output"
+        Java::OrgLogstashConfigIr::PluginDefinition::Type::OUTPUT
+      end
+    end
+
+    def plugin_name
+      return name.text_value
+    end
+
+    def expr_attributes
+      # Turn attributes into a hash map
+      self.attributes.recursive_select(Attribute).map(&:expr).map {|k,v|
+        if v.java_kind_of?(Java::OrgLogstashConfigIrExpression::ValueExpression)
+          [k, v.get]
+        else
+          [k,v]
+        end
+      }.reduce({}) do |hash,kv|
+        k,v = kv
+        hash[k] = v
+        hash
+      end
+
+    end
+  end
+
+  class Name < Node
+    def expr
+      return text_value
+    end
+  end
+  
+  class Attribute < Node
+    def expr
+      [name.text_value, value.expr]
+    end
+  end
+  
+  class RValue < Node; end
+  class Value < RValue; end
+
+  class Bareword < Value
+    def expr
+      jdsl.eValue(source_meta, text_value)
+    end
+  end
+  
+  class String < Value
+    def expr
+      jdsl.e_value(source_meta, text_value[1...-1])
+    end
+  end
+  
+  class RegExp < Value
+    def expr
+      # Strip the slashes off
+      jdsl.eRegex(text_value[1..-2])
+    end
+  end
+  
+  class Number < Value
+    def expr
+      jdsl.eValue(source_meta, text_value.include?(".") ? 
+        text_value.to_f : 
+        text_value.to_i)
+    end
+  end
+  
+  class Array < Value
+    def expr
+      jdsl.eValue(source_meta, recursive_select(Value).map(&:expr).map(&:get))
+    end
+  end
+  
+  class Hash < Value
+    def validate!
+      duplicate_values = find_duplicate_keys
+
+      if duplicate_values.size > 0
+        raise ConfigurationError.new(
+          I18n.t("logstash.runner.configuration.invalid_plugin_settings_duplicate_keys",
+            :keys => duplicate_values.join(', '),
+            :line => input.line_of(interval.first),
+            :column => input.column_of(interval.first),
+            :byte => interval.first + 1,
+            :after => input[0..interval.first]
+          )
+        )
+      end
+    end
+
+    def find_duplicate_keys
+      values = recursive_select(HashEntry).collect { |hash_entry| hash_entry.name.text_value }
+      values.find_all { |v| values.count(v) > 1 }.uniq
+    end
+
+    def expr
+      validate!
+      ::Hash[recursive_select(HashEntry).map(&:expr)]
+    end
+  end
+
+  class HashEntries < Node; end
+
+  class HashEntry < Node
+    def expr
+      return [name.expr.get, value.expr.get()]
+    end
+  end
+
+  class Branch < Node
+    def expr
+      # Build this stuff as s-expressions for convenience at first
+      # This will turn if/elsif/else blocks into nested if/else trees
+
+      exprs = []
+      else_stack = [] # For turning if / elsif / else into nested ifs
+
+      self.recursive_select(Plugin, If, Elsif, Else).each do |node|
+        if node.is_a?(If)
+          exprs << :if
+          exprs << expr_cond(node)
+          exprs << expr_body(node)
+        elsif node.is_a?(Elsif)
+          condition = expr_cond(node)
+          body = expr_body(node)
+          else_stack << [:if, condition, body]
+        elsif node.is_a?(Else)
+          body = expr_body(node)
+          if else_stack.size >= 1
+            else_stack.last << body
+          else
+            exprs << body
+          end
+        end
+      end
+
+      else_stack.reverse.each_cons(2) do |cons|
+        later,earlier = cons
+        earlier << later
+      end
+      exprs << else_stack.first
+
+      # Then convert to the imperative java IR
+      javaify_sexpr(exprs)
+    end
+
+    def javaify_sexpr(sexpr)
+      return nil if sexpr.nil?
+
+      head, tail = sexpr.first
+      tail = sexpr[1..-1]
+
+      if head == :if
+        condition, t_branch, f_branch = tail
+
+        java_t_branch = t_branch && javaify_sexpr(t_branch)
+        java_f_branch = f_branch && javaify_sexpr(f_branch)
+
+        if java_t_branch || java_f_branch
+          jdsl.iIf(condition, java_t_branch || jdsl.noop, java_f_branch || jdsl.noop)
+        else
+          jdsl.noop()
+        end
+      elsif head == :compose
+        tail && tail.size > 0 ? compose(*tail) : jdsl.noop
+      else
+        raise "Unknown expression #{sexpr}!"
+      end
+    end
+
+    def expr_cond(node)
+      node.elements.find {|e| e.is_a?(Condition)}.expr
+    end
+
+    def expr_body(node)
+      [:compose, *node.recursive_select(Plugin, Branch).map(&:expr)]
+    end
+  end
+
+  # Branch covers all these
+  class BranchEntry < Node; end
+  class If < BranchEntry; end
+  class Elsif < BranchEntry; end
+  class Else < BranchEntry; end
+
+  class Condition < Node
+    include Helpers
+    
+    def expr
+      first_element = elements.first
+      rest_elements = elements.size > 1 ? elements[1].recursive_select(BooleanOperator, Expression, SelectorElement) : []
+
+      all_elements = [first_element, *rest_elements]
+
+      if all_elements.size == 1
+        elem = all_elements.first
+        if elem.is_a?(Selector)
+          eventValue = elem.recursive_select(SelectorElement).first.expr
+          jdsl.eTruthy(source_meta, eventValue)
+        elsif elem.is_a?(RegexpExpression)
+          elem.expr
+        else
+          join_conditions(all_elements)
+        end
+      else
+        join_conditions(all_elements)
+      end
+    end
+
+    def precedence(op)
+      #  Believe this is right for logstash?
+      case op
+      when AND_METHOD
+        2
+      when OR_METHOD
+        1
+      else
+        raise ArgumentError, "Unexpected operator #{op}"
+      end
+    end
+
+    # Converts an sexpr of :and or :or to the java imperative IR
+    def jconvert(sexpr)
+      raise "jconvert cannot handle nils!" if sexpr.nil?
+
+      if sexpr.java_kind_of?(Java::OrgLogstashConfigIrExpression::Expression)
+        return sexpr
+      end
+
+      op, left, right = sexpr
+
+      left_c = jconvert(left)
+      right_c = jconvert(right)
+
+      case op
+      when :and
+        return jdsl.eAnd(left, right);
+      when :or
+        return jdsl.eOr(left, right);
+      else
+        raise "Unknown op #{jop}"
+      end
+    end
+
+    def join_conditions(all_elements)
+      # Use Dijkstra's shunting yard algorithm
+      out = []
+      operators = []
+
+      all_elements.each do |e|
+        e_exp = e.expr
+
+        if e.is_a?(BooleanOperator)
+          if operators.last && precedence(operators.last) > precedence(e_exp)
+            out << operators.pop
+          end
+          operators << e_exp
+        else
+          out << e_exp
+        end
+      end
+      operators.reverse.each {|o| out << o}
+
+      stack = []
+      expr = []
+      out.each do |e|
+        if e.is_a?(Symbol)
+          rval, lval = stack.pop, stack.pop
+          stack << jconvert([e, lval, rval])
+        elsif e.nil?
+          raise "Nil expr encountered! This should not happen!"
+        else
+          stack << e
+        end
+      end
+
+      stack_to_expr(stack)
+    end
+
+    def stack_to_expr(stack)
+      raise "Got an empty stack! This should not happen!" if stack.empty?
+      stack = stack.reverse # We need to work the stack in reverse order
+
+      working_stack = []
+      while elem = stack.pop
+        if elem.is_a?(::Method)
+          right, left = working_stack.pop, working_stack.pop
+          working_stack << elem.call(left, right)
+        else
+          working_stack << elem
+        end
+      end
+
+      raise "Invariant violated! Stack size > 1" if working_stack.size > 1
+
+      working_stack.first
+    end
+  end
+
+  module Expression
+    def expr
+      # If we have a more specific type (like a Negative expression) use that
+      if defined?(super)
+        return super
+      end
+
+      exprs = self.recursive_select(Condition, Selector).map(&:expr)
+
+      raise "Exprs should only have one part!" if exprs.size != 1
+      exprs.first
+    end
+  end
+
+  module NegativeExpression
+    include Helpers
+
+    def expr
+      exprs = self.recursive_select(Condition, Selector).map(&:expr)
+      raise "Negative exprs should only have one part!" if exprs.size != 1
+      jdsl.eNot(source_meta, exprs.first)
+    end
+  end
+
+  module ComparisonExpression
+    include Helpers
+
+    def expr
+      lval, comparison_method, rval = self.recursive_select(Selector, Expression, ComparisonOperator, Number, String).map(&:expr)
+      comparison_method.call(source_meta, lval, rval)
+    end
+  end
+
+  module InExpression
+    include Helpers
+
+    def expr
+      item, list = recursive_select(RValue)
+      jdsl.eIn(source_meta, item.expr, list.expr)
+    end
+  end
+
+  module NotInExpression
+    include Helpers
+
+    def expr
+      item, list = recursive_select(RValue)
+      jdsl.eNot(source_meta, jdsl.eIn(item.expr, list.expr))
+    end
+  end
+
+  # Not implemented because no one uses this
+  class MethodCall < Node; end
+
+  class RegexpExpression < Node
+    def expr
+      selector, operator_method, regexp = recursive_select(
+        Selector, 
+        LogStash::Compiler::LSCL::AST::RegExpOperator, 
+        LogStash::Compiler::LSCL::AST::RegExp, 
+        LogStash::Compiler::LSCL::AST::String # Strings work as rvalues! :p
+      ).map(&:expr)
+
+      # Handle string rvalues, they just get turned into regexps
+      # Maybe we really shouldn't handle these anymore...
+      if regexp.class == org.logstash.config.ir.expression.ValueExpression
+        regexp = jdsl.eRegex(regexp.get)
+      end
+      
+      raise "Expected a selector in #{text_value}!" unless selector
+      raise "Expected a regexp in #{text_value}!" unless regexp
+
+      operator_method.call(source_meta, selector, regexp);
+    end
+  end
+
+  module BranchOrPlugin; end
+
+  module ComparisonOperator
+    include Helpers
+
+    def expr
+      case self.text_value
+      when "=="
+        jdsl.method(:eEq)
+      when "!="
+        jdsl.method(:eNeq)
+      when ">"
+        jdsl.method(:eGt)
+      when "<"
+        jdsl.method(:eLt)
+      when ">="
+        jdsl.method(:eGte)
+      when "<="
+        jdsl.method(:eLte)
+      else
+        raise "Unknown operator #{self.text_value}"
+      end
+    end
+  end
+  
+  module RegExpOperator
+    include Helpers
+    
+    def expr
+      if self.text_value == '!~'
+        jdsl.method(:eRegexNeq)
+      elsif self.text_value == '=~'
+        jdsl.method(:eRegexEq)
+      else
+        raise "Unknown regex operator #{self.text_value}"
+      end
+    end
+  end
+  
+  module BooleanOperator
+    include Helpers
+    
+    def expr
+      case self.text_value
+      when "and"
+        AND_METHOD
+      when "or"
+        OR_METHOD
+      else
+        raise "Unknown operator #{self.text_value}"
+      end
+    end
+  end
+  
+  class Selector < RValue
+    def expr
+      jdsl.eEventValue(source_meta, text_value)
+    end
+  end
+  
+  class SelectorElement < Node;
+    def expr
+      jdsl.eEventValue(source_meta, text_value)
+    end
+  end
+end; end; end; end; end;