logstash-core 5.0.2-java → 5.1.1.1-java

data/lib/logstash/settings.rb

@@ -1,10 +1,19 @@
 # encoding: utf-8
+require "logstash/util/loggable"
+require "fileutils"
+require "logstash/util/byte_value"
+require "logstash/util/time_value"
 
 module LogStash
   class Settings
 
     def initialize
       @settings = {}
+      # Theses settings were loaded from the yaml file
+      # but we didn't find any settings to validate them,
+      # lets keep them around until we do `validate_all` at that
+      # time universal plugins could have added new settings.
+      @transient_settings = {}
     end
 
     def register(setting)
@@ -48,8 +57,14 @@ module LogStash
     end
     alias_method :get, :get_value
 
-    def set_value(setting_name, value)
+    def set_value(setting_name, value, graceful = false)
       get_setting(setting_name).set(value)
+    rescue ArgumentError => e
+      if graceful
+        @transient_settings[setting_name] = value
+      else
+        raise e
+      end
     end
     alias_method :set, :set_value
 
@@ -61,8 +76,8 @@ module LogStash
       hash
     end
 
-    def merge(hash)
-      hash.each {|key, value| set_value(key, value) }
+    def merge(hash, graceful = false)
+      hash.each {|key, value| set_value(key, value, graceful) }
       self
     end
 
@@ -92,10 +107,13 @@ module LogStash
 
     def from_yaml(yaml_path)
       settings = read_yaml(::File.join(yaml_path, "logstash.yml"))
-      self.merge(flatten_hash(settings))
+      self.merge(flatten_hash(settings), true)
     end
 
     def validate_all
+      # lets merge the transient_settings again to see if new setting were added.
+      self.merge(@transient_settings)
+
       @settings.each do |name, setting|
         setting.validate_value
       end
@@ -118,6 +136,8 @@ module LogStash
   end
 
   class Setting
+    include LogStash::Util::Loggable
+
     attr_reader :name, :default
 
     def initialize(name, klass, default=nil, strict=true, &validator_proc)
@@ -389,17 +409,89 @@ module LogStash
 
     class WritableDirectory < Setting
       def initialize(name, default=nil, strict=false)
-        super(name, ::String, default, strict) do |path|
-          if ::File.directory?(path) && ::File.writable?(path)
-            true
-          else
-            raise ::ArgumentError.new("Path \"#{path}\" is not a directory or not writable.")
+        super(name, ::String, default, strict)
+      end
+      
+      def validate(path)
+        super(path)
+
+        if ::File.directory?(path)
+          if !::File.writable?(path)
+            raise ::ArgumentError.new("Path \"#{path}\" must be a writable directory. It is not writable.")
+          end
+        elsif ::File.symlink?(path)
+          # TODO(sissel): I'm OK if we relax this restriction. My experience
+          # is that it's usually easier and safer to just reject symlinks.
+          raise ::ArgumentError.new("Path \"#{path}\" must be a writable directory. It cannot be a symlink.")
+        elsif ::File.exist?(path)
+          raise ::ArgumentError.new("Path \"#{path}\" must be a writable directory. It is not a directory.")
+        else
+          parent = ::File.dirname(path)
+          if !::File.writable?(parent)
+            raise ::ArgumentError.new("Path \"#{path}\" does not exist and I cannot create it because the parent path \"#{parent}\" is not writable.")
+          end
+        end
+
+        # If we get here, the directory exists and is writable.
+        true
+      end
+
+      def value
+        super.tap do |path|
+          if !::File.directory?(path)
+            # Create the directory if it doesn't exist.
+            begin
+              logger.info("Creating directory", setting: name, path: path)
+              ::FileUtils.mkdir_p(path)
+            rescue => e
+              # TODO(sissel): Catch only specific exceptions?
+              raise ::ArgumentError.new("Path \"#{path}\" does not exist, and I failed trying to create it: #{e.class.name} - #{e}")
+            end
           end
         end
       end
     end
+
+    class Bytes < Coercible
+      def initialize(name, default=nil, strict=true)
+        super(name, ::Fixnum, default, strict=true) { |value| valid?(value) }
+      end
+
+      def valid?(value)
+        value.is_a?(Fixnum) && value >= 0
+      end
+
+      def coerce(value)
+        case value
+        when ::Numeric
+          value
+        when ::String
+          LogStash::Util::ByteValue.parse(value)
+        else
+          raise ArgumentError.new("Could not coerce '#{value}' into a bytes value")
+        end
+      end
+
+      def validate(value)
+        unless valid?(value)
+          raise ArgumentError.new("Invalid byte value \"#{value}\".")
+        end
+      end
+    end
+
+    class TimeValue < Coercible
+      def initialize(name, default, strict=true, &validator_proc)
+        super(name, ::Fixnum, default, strict, &validator_proc)
+      end
+
+      def coerce(value)
+        return value if value.is_a?(::Fixnum)
+        Util::TimeValue.from_value(value).to_nanos
+      end
+    end
   end
 
+
   SETTINGS = Settings.new
 end
 

data/lib/logstash/universal_plugin.rb

@@ -0,0 +1,13 @@
+# encoding: utf-8
+module LogStash
+  class UniversalPlugin
+    def initialize
+    end
+
+    def register_hooks(hookManager)
+    end
+
+    def additionals_settings(settings)
+    end
+  end
+end

data/lib/logstash/util/byte_value.rb

@@ -0,0 +1,60 @@
+# encoding: utf-8
+require "logstash/namespace"
+
+module LogStash; module Util; module ByteValue
+  module_function
+
+  B = 1
+  KB = B << 10
+  MB = B << 20
+  GB = B << 30
+  TB = B << 40
+  PB = B << 50
+
+  def parse(text)
+    if !text.is_a?(String)
+      raise ArgumentError, "ByteValue::parse takes a String, got a `#{text.class.name}`"
+    end
+    number = text.to_f
+    factor = multiplier(text)
+
+    (number * factor).to_i
+  end
+
+  def multiplier(text)
+    case text
+      when /(?:k|kb)$/ 
+        KB
+      when /(?:m|mb)$/
+        MB
+      when /(?:g|gb)$/
+        GB
+      when /(?:t|tb)$/
+        TB
+      when /(?:p|pb)$/
+        PB
+      when /(?:b)$/
+        B
+      else 
+        raise ArgumentError, "Unknown bytes value '#{text}'"
+    end
+  end
+
+  def human_readable(number)
+    value, unit = if number > PB
+      [number / PB, "pb"]
+    elsif number > TB
+      [number / TB, "tb"]
+    elsif number > GB
+      [number / GB, "gb"]
+    elsif number > MB
+      [number / MB, "mb"]
+    elsif number > KB
+      [number / KB, "kb"]
+    else
+      [number, "b"]
+    end
+
+    format("%.2d%s", value, unit)
+  end
+end end end

data/lib/logstash/util/loggable.rb

@@ -5,15 +5,27 @@ require "logstash/namespace"
 module LogStash module Util
   module Loggable
     def self.included(klass)
-      def klass.logger
+
+      def klass.log4j_name
         ruby_name = self.name || self.class.name || self.class.to_s
-        log4j_name = ruby_name.gsub('::', '.').downcase
+        ruby_name.gsub('::', '.').downcase
+      end
+
+      def klass.logger
         @logger ||= LogStash::Logging::Logger.new(log4j_name)
       end
 
+      def klass.slow_logger(warn_threshold, info_threshold, debug_threshold, trace_threshold)
+        @slow_logger ||= LogStash::Logging::SlowLogger.new(log4j_name, warn_threshold, info_threshold, debug_threshold, trace_threshold)
+      end
+
       def logger
         self.class.logger
       end
+
+      def slow_logger(warn_threshold, info_threshold, debug_threshold, trace_threshold)
+        self.class.slow_logger(warn_threshold, info_threshold, debug_threshold, trace_threshold)
+      end
     end
   end
 end; end

data/lib/logstash/util/safe_uri.rb

@@ -1,6 +1,7 @@
 # encoding: utf-8
 require "logstash/namespace"
 require "logstash/util"
+require "forwardable"
 
 # This class exists to quietly wrap a password string so that, when printed or
 # logged, you don't accidentally print the password itself.

data/lib/logstash/util/time_value.rb

@@ -0,0 +1,70 @@
+module LogStash
+  module Util
+    class TimeValue
+      def initialize(duration, time_unit)
+        @duration = duration
+        @time_unit = time_unit
+      end
+
+      def self.from_value(value)
+        if value.is_a?(TimeValue)
+          TimeValue.new(value.duration, value.time_unit)
+        elsif value.is_a?(::String)
+          normalized = value.downcase.strip
+          if normalized.end_with?("nanos")
+            TimeValue.new(parse(normalized, 5), :nanosecond)
+          elsif normalized.end_with?("micros")
+            TimeValue.new(parse(normalized, 6), :microsecond)
+          elsif normalized.end_with?("ms")
+            TimeValue.new(parse(normalized, 2), :millisecond)
+          elsif normalized.end_with?("s")
+            TimeValue.new(parse(normalized, 1), :second)
+          elsif normalized.end_with?("m")
+            TimeValue.new(parse(normalized, 1), :minute)
+          elsif normalized.end_with?("h")
+            TimeValue.new(parse(normalized, 1), :hour)
+          elsif normalized.end_with?("d")
+            TimeValue.new(parse(normalized, 1), :day)
+          elsif normalized =~ /^-0*1/
+            TimeValue.new(-1, :nanosecond)
+          else
+            raise ArgumentError.new("invalid time unit: \"#{value}\"")
+          end
+        else
+          raise ArgumentError.new("value is not a string: #{value} [#{value.class}]")
+        end
+      end
+
+      def to_nanos
+        case @time_unit
+        when :day
+          86400000000000 * @duration
+        when :hour
+          3600000000000 * @duration
+        when :minute
+          60000000000 * @duration
+        when :second
+          1000000000 * @duration
+        when :millisecond
+          1000000 * @duration
+        when :microsecond
+          1000 * @duration
+        when :nanosecond
+          @duration
+        end
+      end
+
+      def ==(other)
+        self.duration == other.duration and self.time_unit == other.time_unit
+      end
+
+      def self.parse(value, suffix)
+        Integer(value[0..(value.size - suffix - 1)].strip)
+      end
+
+      private_class_method :parse
+      attr_reader :duration
+      attr_reader :time_unit
+    end
+  end
+end

data/lib/logstash/util/wrapped_acked_queue.rb

@@ -0,0 +1,347 @@
+# encoding: utf-8
+
+require "logstash-core-queue-jruby/logstash-core-queue-jruby"
+require "concurrent"
+# This is an adapted copy of the wrapped_synchronous_queue file
+# ideally this should be moved to Java/JRuby
+
+module LogStash; module Util
+  # Some specialized constructors. The calling code *does* need to know what kind it creates but
+  # not the internal implementation e.g. LogStash::AckedMemoryQueue etc.
+  # Note the use of allocate - this is what new does before it calls initialize.
+  # Note that the new method has been made private this is because there is no
+  # default queue implementation.
+  # It would be expensive to create a persistent queue in the new method
+  # to then throw it away in favor of a memory based one directly after.
+  # Especially in terms of (mmap) memory allocation and proper close sequencing.
+
+  class WrappedAckedQueue
+    class QueueClosedError < ::StandardError; end
+    class NotImplementedError < ::StandardError; end
+
+    def self.create_memory_based(path, capacity, max_events, max_bytes)
+      self.allocate.with_queue(
+        LogStash::AckedMemoryQueue.new(path, capacity, max_events, max_bytes)
+      )
+    end
+
+    def self.create_file_based(path, capacity, max_events, checkpoint_max_writes, checkpoint_max_acks, checkpoint_max_interval, max_bytes)
+      self.allocate.with_queue(
+        LogStash::AckedQueue.new(path, capacity, max_events, checkpoint_max_writes, checkpoint_max_acks, checkpoint_max_interval, max_bytes)
+      )
+    end
+
+    private_class_method :new
+
+    def with_queue(queue)
+      @queue = queue
+      @queue.open
+      @closed = Concurrent::AtomicBoolean.new(false)
+      self
+    end
+
+    def closed?
+      @closed.true?
+    end
+
+    # Push an object to the queue if the queue is full
+    # it will block until the object can be added to the queue.
+    #
+    # @param [Object] Object to add to the queue
+    def push(obj)
+      check_closed("write")
+      @queue.write(obj)
+    end
+    alias_method(:<<, :push)
+
+    # TODO - fix doc for this noop method
+    # Offer an object to the queue, wait for the specified amount of time.
+    # If adding to the queue was successful it will return true, false otherwise.
+    #
+    # @param [Object] Object to add to the queue
+    # @param [Integer] Time in milliseconds to wait before giving up
+    # @return [Boolean] True if adding was successfull if not it return false
+    def offer(obj, timeout_ms)
+      raise NotImplementedError.new("The offer method is not implemented. There is no non blocking write operation yet.")
+    end
+
+    # Blocking
+    def take
+      check_closed("read a batch")
+      # TODO - determine better arbitrary timeout millis
+      @queue.read_batch(1, 200).get_elements.first
+    end
+
+    # Block for X millis
+    def poll(millis)
+      check_closed("read")
+      @queue.read_batch(1, millis).get_elements.first
+    end
+
+    def read_batch(size, wait)
+      check_closed("read a batch")
+      @queue.read_batch(size, wait)
+    end
+
+    def write_client
+      WriteClient.new(self)
+    end
+
+    def read_client()
+      ReadClient.new(self)
+    end
+
+    def check_closed(action)
+      if closed?
+        raise QueueClosedError.new("Attempted to #{action} on a closed AckedQueue")
+      end
+    end
+
+    def close
+      @queue.close
+      @closed.make_true
+    end
+
+    class ReadClient
+      # We generally only want one thread at a time able to access pop/take/poll operations
+      # from this queue. We also depend on this to be able to block consumers while we snapshot
+      # in-flight buffers
+
+      def initialize(queue, batch_size = 125, wait_for = 250)
+        @queue = queue
+        @mutex = Mutex.new
+        # Note that @inflight_batches as a central mechanism for tracking inflight
+        # batches will fail if we have multiple read clients in the pipeline.
+        @inflight_batches = {}
+        # allow the worker thread to report the execution time of the filter + output
+        @inflight_clocks = {}
+        @batch_size = batch_size
+        @wait_for = wait_for
+      end
+
+      def close
+        @queue.close
+      end
+
+      def set_batch_dimensions(batch_size, wait_for)
+        @batch_size = batch_size
+        @wait_for = wait_for
+      end
+
+      def set_events_metric(metric)
+        @event_metric = metric
+      end
+
+      def set_pipeline_metric(metric)
+        @pipeline_metric = metric
+      end
+
+      def inflight_batches
+        @mutex.synchronize do
+          yield(@inflight_batches)
+        end
+      end
+
+      def current_inflight_batch
+        @inflight_batches.fetch(Thread.current, [])
+      end
+
+      def take_batch
+        if @queue.closed?
+          raise QueueClosedError.new("Attempt to take a batch from a closed AckedQueue")
+        end
+        @mutex.synchronize do
+          batch = ReadBatch.new(@queue, @batch_size, @wait_for)
+          add_starting_metrics(batch)
+          set_current_thread_inflight_batch(batch)
+          start_clock
+          batch
+        end
+      end
+
+      def set_current_thread_inflight_batch(batch)
+        @inflight_batches[Thread.current] = batch
+      end
+
+      def close_batch(batch)
+        @mutex.synchronize do
+          batch.close
+          @inflight_batches.delete(Thread.current)
+          stop_clock
+        end
+      end
+
+      def start_clock
+        @inflight_clocks[Thread.current] = [
+        @event_metric.time(:duration_in_millis),
+        @pipeline_metric.time(:duration_in_millis)
+        ]
+      end
+
+      def stop_clock
+        @inflight_clocks[Thread.current].each(&:stop)
+        @inflight_clocks.delete(Thread.current)
+      end
+
+      def add_starting_metrics(batch)
+        return if @event_metric.nil? || @pipeline_metric.nil?
+        @event_metric.increment(:in, batch.starting_size)
+        @pipeline_metric.increment(:in, batch.starting_size)
+      end
+
+      def add_filtered_metrics(batch)
+        @event_metric.increment(:filtered, batch.filtered_size)
+        @pipeline_metric.increment(:filtered, batch.filtered_size)
+      end
+
+      def add_output_metrics(batch)
+        @event_metric.increment(:out, batch.filtered_size)
+        @pipeline_metric.increment(:out, batch.filtered_size)
+      end
+    end
+
+    class ReadBatch
+      def initialize(queue, size, wait)
+        @originals = Hash.new
+
+        # TODO: disabled for https://github.com/elastic/logstash/issues/6055 - will have to properly refactor
+        # @cancelled = Hash.new
+
+        @generated = Hash.new
+        @iterating_temp = Hash.new
+        @iterating = false # Atomic Boolean maybe? Although batches are not shared across threads
+        take_originals_from_queue(queue, size, wait) # this sets a reference to @acked_batch
+      end
+
+      def close
+        # this will ack the whole batch, regardless of whether some
+        # events were cancelled or failed
+        return if @acked_batch.nil?
+        @acked_batch.close
+      end
+
+      def merge(event)
+        return if event.nil? || @originals.key?(event)
+        # take care not to cause @generated to change during iteration
+        # @iterating_temp is merged after the iteration
+        if iterating?
+          @iterating_temp[event] = true
+        else
+          # the periodic flush could generate events outside of an each iteration
+          @generated[event] = true
+        end
+      end
+
+      def cancel(event)
+        # TODO: disabled for https://github.com/elastic/logstash/issues/6055 - will have to properly refactor
+        raise("cancel is unsupported")
+        # @cancelled[event] = true
+      end
+
+      def each(&blk)
+        # take care not to cause @originals or @generated to change during iteration
+
+        # below the checks for @cancelled.include?(e) have been replaced by e.cancelled?
+        # TODO: for https://github.com/elastic/logstash/issues/6055 = will have to properly refactor
+        @iterating = true
+        @originals.each do |e, _|
+          blk.call(e) unless e.cancelled?
+        end
+        @generated.each do |e, _|
+          blk.call(e) unless e.cancelled?
+        end
+        @iterating = false
+        update_generated
+      end
+
+      def size
+        filtered_size
+      end
+
+      def starting_size
+        @originals.size
+      end
+
+      def filtered_size
+        @originals.size + @generated.size
+      end
+
+      def cancelled_size
+        # TODO: disabled for https://github.com/elastic/logstash/issues/6055 = will have to properly refactor
+        raise("cancelled_size is unsupported ")
+        # @cancelled.size
+      end
+
+      def shutdown_signal_received?
+        false
+      end
+
+      def flush_signal_received?
+        false
+      end
+
+      private
+
+      def iterating?
+        @iterating
+      end
+
+      def update_generated
+        @generated.update(@iterating_temp)
+        @iterating_temp.clear
+      end
+
+      def take_originals_from_queue(queue, size, wait)
+        @acked_batch = queue.read_batch(size, wait)
+        return if @acked_batch.nil?
+        @acked_batch.get_elements.each do |e|
+          @originals[e] = true
+        end
+      end
+    end
+
+    class WriteClient
+      def initialize(queue)
+        @queue = queue
+      end
+
+      def get_new_batch
+        WriteBatch.new
+      end
+
+      def push(event)
+        if @queue.closed?
+          raise QueueClosedError.new("Attempted to write an event to a closed AckedQueue")
+        end
+        @queue.push(event)
+      end
+      alias_method(:<<, :push)
+
+      def push_batch(batch)
+        if @queue.closed?
+          raise QueueClosedError.new("Attempted to write a batch to a closed AckedQueue")
+        end
+        batch.each do |event|
+          push(event)
+        end
+      end
+    end
+
+    class WriteBatch
+      def initialize
+        @events = []
+      end
+
+      def push(event)
+        @events.push(event)
+      end
+      alias_method(:<<, :push)
+
+      def each(&blk)
+        @events.each do |e|
+          blk.call(e)
+        end
+      end
+    end
+  end
+end end