logstash-core 1.5.0.rc2.snapshot-java → 1.5.0.rc3-java

data/lib/logstash/event.rb

@@ -16,6 +16,13 @@ require "logstash/json"
 class LogStash::ShutdownEvent; end
 class LogStash::FlushEvent; end
 
+module LogStash
+  FLUSH = LogStash::FlushEvent.new
+
+  # LogStash::SHUTDOWN is used by plugins
+  SHUTDOWN = LogStash::ShutdownEvent.new
+end
+
 # the logstash event object.
 #
 # An event is simply a tuple of (timestamp, data).
@@ -48,25 +55,26 @@ class LogStash::Event
   TIMESTAMP_FAILURE_TAG = "_timestampparsefailure"
   TIMESTAMP_FAILURE_FIELD = "_@timestamp"
 
+  METADATA = "@metadata".freeze
+  METADATA_BRACKETS = "[#{METADATA}]".freeze
+
   # Floats outside of these upper and lower bounds are forcibly converted
   # to scientific notation by Float#to_s
   MIN_FLOAT_BEFORE_SCI_NOT = 0.0001
   MAX_FLOAT_BEFORE_SCI_NOT = 1000000000000000.0
 
+  LOGGER = Cabin::Channel.get(LogStash)
+
   public
   def initialize(data = {})
-    @logger = Cabin::Channel.get(LogStash)
     @cancelled = false
     @data = data
     @accessors = LogStash::Util::Accessors.new(data)
     @data[VERSION] ||= VERSION_ONE
-    @data[TIMESTAMP] = init_timestamp(@data[TIMESTAMP])
+    ts = @data[TIMESTAMP]
+    @data[TIMESTAMP] = ts ? init_timestamp(ts) : LogStash::Timestamp.now
 
-    @metadata = if @data.include?("@metadata")
-      @data.delete("@metadata")
-    else
-      {}
-    end
+    @metadata = @data.delete(METADATA) || {}
     @metadata_accessors = LogStash::Util::Accessors.new(@metadata)
   end # def initialize
 
@@ -113,9 +121,6 @@ class LogStash::Event
     raise DeprecatedMethod
   end # def unix_timestamp
 
-  # field-related access
-  METADATA = "@metadata".freeze
-  METADATA_BRACKETS = "[#{METADATA}]".freeze
   public
   def [](fieldref)
     if fieldref.start_with?(METADATA_BRACKETS)
@@ -171,8 +176,14 @@ class LogStash::Event
   end
 
   public
-  def include?(key)
-    return !self[key].nil?
+  def include?(fieldref)
+    if fieldref.start_with?(METADATA_BRACKETS)
+      @metadata_accessors.include?(fieldref[METADATA_BRACKETS.length .. -1])
+    elsif fieldref == METADATA
+      true
+    else
+      @accessors.include?(fieldref)
+    end
   end # def include?
 
   # Append an event to this one.
@@ -224,6 +235,10 @@ class LogStash::Event
       # Take the inside of the %{ ... }
       key = tok[2 ... -1]
 
+      if key[0] == "+" && !@data.has_key?(TIMESTAMP)
+        raise LogStash::Error, "Unable to format \"#{key}\" in string \"#{format}\", #{TIMESTAMP} field not found"
+      end
+
       if key == "+%s"
         # Got %{+%s}, support for unix epoch time
         next @data[TIMESTAMP].to_i
@@ -263,12 +278,12 @@ class LogStash::Event
 
   def init_timestamp(o)
     begin
-      timestamp = o ? LogStash::Timestamp.coerce(o) : LogStash::Timestamp.now
+      timestamp = LogStash::Timestamp.coerce(o)
       return timestamp if timestamp
 
-      @logger.warn("Unrecognized #{TIMESTAMP} value, setting current time to #{TIMESTAMP}, original in #{TIMESTAMP_FAILURE_FIELD}field", :value => o.inspect)
+      LOGGER.warn("Unrecognized #{TIMESTAMP} value, setting current time to #{TIMESTAMP}, original in #{TIMESTAMP_FAILURE_FIELD}field", :value => o.inspect)
     rescue LogStash::TimestampParserError => e
-      @logger.warn("Error parsing #{TIMESTAMP} string, setting current time to #{TIMESTAMP}, original in #{TIMESTAMP_FAILURE_FIELD} field", :value => o.inspect, :exception => e.message)
+      LOGGER.warn("Error parsing #{TIMESTAMP} string, setting current time to #{TIMESTAMP}, original in #{TIMESTAMP_FAILURE_FIELD} field", :value => o.inspect, :exception => e.message)
     end
 
     @data["tags"] ||= []
@@ -280,11 +295,7 @@ class LogStash::Event
 
   public
   def to_hash_with_metadata
-    if @metadata.nil?
-      to_hash
-    else
-      to_hash.merge("@metadata" => @metadata)
-    end
+    @metadata.empty? ? to_hash : to_hash.merge(METADATA => @metadata)
   end
 
   public

data/lib/logstash/filters/base.rb

@@ -1,5 +1,6 @@
 # encoding: utf-8
 require "logstash/namespace"
+require "logstash/event"
 require "logstash/logging"
 require "logstash/plugin"
 require "logstash/config/mixin"

data/lib/logstash/gemfile.rb

@@ -1,3 +1,4 @@
+require "logstash/util"
 module LogStash
 
   class GemfileError < StandardError; end
@@ -17,6 +18,7 @@ module LogStash
 
     def load
       @gemset ||= DSL.parse(@io.read)
+      backup
       self
     end
 
@@ -51,6 +53,23 @@ module LogStash
     def remove(name)
       @gemset.remove_gem(name)
     end
+
+    def backup
+      @original_backup = @gemset.copy
+    end
+
+    def restore
+      @gemset = @original_backup
+    end
+
+    def restore!
+      restore
+      save
+    end
+
+    def locally_installed_gems
+      @gemset.gems.select { |gem| gem.options.include?(:path) }
+    end
   end
 
   class Gemset
@@ -101,7 +120,6 @@ module LogStash
     def copy
       Marshal.load(Marshal.dump(self))
     end
-
     private
 
     def sources_to_s

data/lib/logstash/java_integration.rb

@@ -6,27 +6,11 @@ require "java"
 # not test for is_a?(Array) or is_a?(Hash) and we do not want to include tests for
 # both classes everywhere. see LogStash::JSon.
 
-class Java::JavaUtil::ArrayList
-  # have ArrayList objects report is_a?(Array) == true
-  def is_a?(clazz)
-    return true if clazz == Array
-    super
-  end
-end
-
-class Java::JavaUtil::LinkedHashMap
-  # have LinkedHashMap objects report is_a?(Array) == true
-  def is_a?(clazz)
-    return true if clazz == Hash
-    super
-  end
-end
-
 class Array
   # enable class equivalence between Array and ArrayList
   # so that ArrayList will work with case o when Array ...
   def self.===(other)
-    return true if other.is_a?(Java::JavaUtil::ArrayList)
+    return true if other.is_a?(Java::JavaUtil::Collection)
     super
   end
 end
@@ -35,7 +19,77 @@ class Hash
   # enable class equivalence between Hash and LinkedHashMap
   # so that LinkedHashMap will work with case o when Hash ...
   def self.===(other)
-    return true if other.is_a?(Java::JavaUtil::LinkedHashMap)
+    return true if other.is_a?(Java::JavaUtil::Map)
     super
   end
 end
+
+# map_mixin to patch LinkedHashMap and HashMap. it must be done directly on the classes,
+# using a module mixin does not work, and injecting in the Map interface does not work either
+# but injecting in the class works.
+
+map_mixin = lambda do
+  # this is a temporary fix to solve a bug in JRuby where classes implementing the Map interface, like LinkedHashMap
+  # have a bug in the has_key? method that is implemented in the Enumerable module that is somehow mixed in the Map interface.
+  # this bug makes has_key? (and all its aliases) return false for a key that has a nil value.
+  # Only LinkedHashMap is patched here because patching the Map interface is not working.
+  # TODO find proper fix, and submit upstream
+  # releavant JRuby files:
+  # https://github.com/jruby/jruby/blob/master/core/src/main/ruby/jruby/java/java_ext/java.util.rb
+  # https://github.com/jruby/jruby/blob/master/core/src/main/java/org/jruby/java/proxies/MapJavaProxy.java
+  def has_key?(key)
+    self.containsKey(key)
+  end
+  alias_method :include?, :has_key?
+  alias_method :member?, :has_key?
+  alias_method :key?, :has_key?
+
+  # Java 8 Map implements a merge method with a different signature from
+  # the Ruby Hash#merge. see https://github.com/jruby/jruby/issues/1249
+  # this can be removed when fixed upstream
+  if ENV_JAVA['java.specification.version'] >= '1.8'
+    def merge(other)
+      dup.merge!(other)
+    end
+  end
+end
+
+Java::JavaUtil::LinkedHashMap.module_exec(&map_mixin)
+Java::JavaUtil::HashMap.module_exec(&map_mixin)
+
+module java::util::Map
+  # have Map objects like LinkedHashMap objects report is_a?(Array) == true
+  def is_a?(clazz)
+    return true if clazz == Hash
+    super
+  end
+end
+
+module java::util::Collection
+  # have Collections objects like ArrayList report is_a?(Array) == true
+  def is_a?(clazz)
+    return true if clazz == Array
+    super
+  end
+
+  # support the Ruby Array delete method on a Java Collection
+  def delete(o)
+    self.removeAll([o]) ? o : block_given? ? yield : nil
+  end
+
+  # support the Ruby intersection method on Java Collection
+  def &(other)
+    # transform self into a LinkedHashSet to remove duplicates and preserve order as defined by the Ruby Array intersection contract
+    duped = Java::JavaUtil::LinkedHashSet.new(self)
+    duped.retainAll(other)
+    duped
+  end
+
+  # support the Ruby union method on Java Collection
+  def |(other)
+    # transform self into a LinkedHashSet to remove duplicates and preserve order as defined by the Ruby Array union contract
+    duped = Java::JavaUtil::LinkedHashSet.new(self)
+    duped.addAll(other)
+    duped
+  end
+end

data/lib/logstash/namespace.rb

@@ -12,6 +12,4 @@ module LogStash
   module Util; end
   module PluginMixins; end
   module PluginManager; end
-
-  SHUTDOWN = :shutdown
 end # module LogStash

data/lib/logstash/outputs/base.rb

@@ -69,7 +69,7 @@ class LogStash::Outputs::Base < LogStash::Plugin
     else
       define_singleton_method(:handle, method(:handle_worker))
       @worker_queue = SizedQueue.new(20)
-      @worker_plugins = @workers.times.map { self.class.new(params.merge("workers" => 1, "codec" => @codec.clone)) }
+      @worker_plugins = @workers.times.map { self.class.new(@original_params.merge("workers" => 1)) }
       @worker_plugins.map.with_index do |plugin, i|
         Thread.new(original_params, @worker_queue) do |params, queue|
           LogStash::Util::set_thread_name(">#{self.class.config_name}.#{i}")

data/lib/logstash/patches/bundler.rb

@@ -0,0 +1,20 @@
+# Bundler monkey patches
+module ::Bundler
+  # Patch bundler to write a .lock file specific to the version of ruby.
+  # This keeps MRI/JRuby/RBX from conflicting over the Gemfile.lock updates
+  module SharedHelpers
+    def default_lockfile
+      ruby = "#{LogStash::Environment.ruby_engine}-#{LogStash::Environment.ruby_abi_version}"
+      Pathname.new("#{default_gemfile}.#{ruby}.lock")
+    end
+  end
+
+  # Add the Bundler.reset! method which has been added in master but is not in 1.7.9.
+  class << self
+    unless self.method_defined?("reset!")
+      def reset!
+        @definition = nil
+      end
+    end
+  end
+end

data/lib/logstash/patches/rubygems.rb

@@ -0,0 +1,37 @@
+# monkey patch RubyGems to silence ffi warnings:
+#
+# WARN: Unresolved specs during Gem::Specification.reset:
+#       ffi (>= 0)
+# WARN: Clearing out unresolved specs.
+# Please report a bug if this causes problems.
+#
+# see https://github.com/elasticsearch/logstash/issues/2556 and https://github.com/rubygems/rubygems/issues/1070
+#
+# this code is from Rubygems v2.1.9 in JRuby 1.7.17. Per tickets this issue should be solved at JRuby >= 1.7.20.
+#
+# this method implementation works for Rubygems version 2.1.0 and up, verified up to 2.4.6
+if ::Gem::Version.new(::Gem::VERSION) >= ::Gem::Version.new("2.1.0") && ::Gem::Version.new(::Gem::VERSION) < ::Gem::Version.new("2.5.0")
+  class ::Gem::Specification
+    def self.reset
+      @@dirs = nil
+      ::Gem.pre_reset_hooks.each { |hook| hook.call }
+      @@all = nil
+      @@stubs = nil
+      _clear_load_cache
+      unresolved = unresolved_deps
+      unless unresolved.empty?
+        unless (unresolved.size == 1 && unresolved["ffi"])
+          w = "W" + "ARN"
+          warn "#{w}: Unresolved specs during Gem::Specification.reset:"
+          unresolved.values.each do |dep|
+            warn "      #{dep}"
+          end
+          warn "#{w}: Clearing out unresolved specs."
+          warn "Please report a bug if this causes problems."
+        end
+        unresolved.clear
+      end
+      ::Gem.post_reset_hooks.each { |hook| hook.call }
+    end
+  end
+end

data/lib/logstash/pipeline.rb

@@ -11,8 +11,6 @@ require "logstash/outputs/base"
 
 class LogStash::Pipeline
 
-  FLUSH_EVENT = LogStash::FlushEvent.new
-
   def initialize(configstr)
     @logger = Cabin::Channel.get(LogStash)
     grammar = LogStashConfigParser.new
@@ -45,14 +43,19 @@ class LogStash::Pipeline
     @settings = {
       "filter-workers" => 1,
     }
+
+    @run_mutex = Mutex.new
+    @ready = false
+    @started = false
+    @input_threads = []
   end # def initialize
 
   def ready?
-    return @ready
+    @run_mutex.synchronize{@ready}
   end
 
   def started?
-    return @started
+    @run_mutex.synchronize{@started}
   end
 
   def configure(setting, value)
@@ -71,14 +74,14 @@ class LogStash::Pipeline
   end
 
   def run
-    @started = true
-    @input_threads = []
+    @run_mutex.synchronize{@started = true}
 
-    start_inputs
+    # synchronize @input_threads between run and shutdown
+    @run_mutex.synchronize{start_inputs}
     start_filters if filters?
     start_outputs
 
-    @ready = true
+    @run_mutex.synchronize{@ready = true}
 
     @logger.info("Pipeline started")
     @logger.terminal("Logstash startup completed")
@@ -113,7 +116,7 @@ class LogStash::Pipeline
 
   def shutdown_filters
     @flusher_lock.synchronize { @flusher_thread.kill }
-    @input_to_filter.push(LogStash::ShutdownEvent.new)
+    @input_to_filter.push(LogStash::SHUTDOWN)
   end
 
   def wait_filters
@@ -122,7 +125,7 @@ class LogStash::Pipeline
 
   def shutdown_outputs
     # nothing, filters will do this
-    @filter_to_output.push(LogStash::ShutdownEvent.new)
+    @filter_to_output.push(LogStash::SHUTDOWN)
   end
 
   def wait_outputs
@@ -134,7 +137,7 @@ class LogStash::Pipeline
     moreinputs = []
     @inputs.each do |input|
       if input.threadable && input.threads > 1
-        (input.threads-1).times do |i|
+        (input.threads - 1).times do |i|
           moreinputs << input.clone
         end
       end
@@ -154,7 +157,7 @@ class LogStash::Pipeline
     end
 
     @flusher_lock = Mutex.new
-    @flusher_thread = Thread.new { Stud.interval(5) { @flusher_lock.synchronize { @input_to_filter.push(FLUSH_EVENT) } } }
+    @flusher_thread = Thread.new { Stud.interval(5) { @flusher_lock.synchronize { @input_to_filter.push(LogStash::FLUSH) } } }
   end
 
   def start_outputs
@@ -173,7 +176,7 @@ class LogStash::Pipeline
     begin
       plugin.run(@input_to_filter)
     rescue LogStash::ShutdownSignal
-      return
+      # ignore and quit
     rescue => e
       if @logger.debug?
         @logger.error(I18n.t("logstash.pipeline.worker-error-debug",
@@ -185,14 +188,23 @@ class LogStash::Pipeline
                              :plugin => plugin.inspect, :error => e))
       end
       puts e.backtrace if @logger.debug?
-      plugin.teardown
+      # input teardown must be synchronized since is can be called concurrently by
+      # the input worker thread and from the pipeline thread shutdown method.
+      # this means that input teardown methods must support multiple calls.
+      @run_mutex.synchronize{plugin.teardown}
       sleep 1
       retry
     end
-  rescue LogStash::ShutdownSignal
-    # nothing
   ensure
-    plugin.teardown
+    begin
+      # input teardown must be synchronized since is can be called concurrently by
+      # the input worker thread and from the pipeline thread shutdown method.
+      # this means that input teardown methods must support multiple calls.
+      @run_mutex.synchronize{plugin.teardown}
+    rescue LogStash::ShutdownSignal
+      # teardown could receive the ShutdownSignal, retry it
+      retry
+    end
   end # def inputworker
 
   def filterworker
@@ -203,11 +215,8 @@ class LogStash::Pipeline
 
         case event
         when LogStash::Event
-          # use events array to guarantee ordering of origin vs created events
-          # where created events are emitted by filters like split or metrics
-          events = []
-          filter(event) { |newevent| events << newevent }
-          events.each { |event| @filter_to_output.push(event) }
+          # filter_func returns all filtered events, including cancelled ones
+          filter_func(event).each { |e| @filter_to_output.push(e) unless e.cancelled? }
         when LogStash::FlushEvent
           # handle filter flushing here so that non threadsafe filters (thus only running one filterworker)
           # don't have to deal with thread safety implementing the flush method
@@ -231,8 +240,8 @@ class LogStash::Pipeline
 
     while true
       event = @filter_to_output.pop
-      break if event.is_a?(LogStash::ShutdownEvent)
-      output(event)
+      break if event == LogStash::SHUTDOWN
+      output_func(event)
     end # while true
 
     @outputs.each do |output|
@@ -246,18 +255,30 @@ class LogStash::Pipeline
   def shutdown
     @input_threads.each do |thread|
       # Interrupt all inputs
-      @logger.info("Sending shutdown signal to input thread",
-                   :thread => thread)
-      thread.raise(LogStash::ShutdownSignal)
-      begin
-        thread.wakeup # in case it's in blocked IO or sleeping
-      rescue ThreadError
+      @logger.info("Sending shutdown signal to input thread", :thread => thread)
+
+      # synchronize both ShutdownSignal and teardown below. by synchronizing both
+      # we will avoid potentially sending a shutdown signal when the inputworker is
+      # executing the teardown method.
+      @run_mutex.synchronize do
+        thread.raise(LogStash::ShutdownSignal)
+        begin
+          thread.wakeup # in case it's in blocked IO or sleeping
+        rescue ThreadError
+        end
       end
+    end
 
-      # Sometimes an input is stuck in a blocking I/O
-      # so we need to tell it to teardown directly
-      @inputs.each do |input|
-        input.teardown
+    # sometimes an input is stuck in a blocking I/O so we need to tell it to teardown directly
+    @inputs.each do |input|
+      begin
+        # input teardown must be synchronized since is can be called concurrently by
+        # the input worker thread and from the pipeline thread shutdown method.
+        # this means that input teardown methods must support multiple calls.
+        @run_mutex.synchronize{input.teardown}
+      rescue LogStash::ShutdownSignal
+        # teardown could receive the ShutdownSignal, retry it
+        retry
       end
     end
 
@@ -271,12 +292,11 @@ class LogStash::Pipeline
     return klass.new(*args)
   end
 
+  # for backward compatibility in devutils for the rspec helpers, this method is not used
+  # in the pipeline anymore.
   def filter(event, &block)
-    @filter_func.call(event, &block)
-  end
-
-  def output(event)
-    @output_func.call(event)
+    # filter_func returns all filtered events, including cancelled ones
+    filter_func(event).each { |e| block.call(e) }
   end
 
   # perform filters flush and yeild flushed event to the passed block