logstash-core 5.6.16-java → 6.0.0.alpha1-java

data/lib/logstash/settings.rb

@@ -2,14 +2,11 @@
 require "logstash/util/loggable"
 require "fileutils"
 require "logstash/util/byte_value"
-require "logstash/util/environment_variables"
 require "logstash/util/time_value"
 
 module LogStash
   class Settings
 
-    include LogStash::Util::EnvironmentVariables
-    
     def initialize
       @settings = {}
       # Theses settings were loaded from the yaml file
@@ -50,6 +47,7 @@ module LogStash
     def clone
       get_subset(".*")
     end
+    alias_method :dup, :clone
 
     def get_default(setting_name)
       get_setting(setting_name).default
@@ -110,10 +108,7 @@ module LogStash
 
     def from_yaml(yaml_path)
       settings = read_yaml(::File.join(yaml_path, "logstash.yml"))
-      self.merge(
-        deep_replace(flatten_hash(settings)),
-        true
-      )
+      self.merge(flatten_hash(settings), true)
       self
     end
     
@@ -534,27 +529,8 @@ module LogStash
         end
       end
     end
-
-    class SplittableStringArray < ArrayCoercible
-      DEFAULT_TOKEN = ","
-
-      def initialize(name, klass, default, strict=true, tokenizer = DEFAULT_TOKEN, &validator_proc)
-        @element_class = klass
-        @token = tokenizer
-        super(name, klass, default, strict, &validator_proc)
-      end
-
-      def coerce(value)
-        if value.is_a?(Array)
-          value
-        elsif value.nil?
-          []
-        else
-          value.split(@token).map(&:strip)
-        end
-      end
-    end
   end
 
+
   SETTINGS = Settings.new
 end

data/lib/logstash/state_resolver.rb

@@ -0,0 +1,41 @@
+# encoding: utf-8
+module LogStash
+  # In the beginning I was using this code as a method in the Agent class directly
+  # But with the plugins system I think we should be able to swap what kind of action would be run.
+  #
+  # Lets take the example of dynamic source, where the pipeline config and settings are located and
+  # managed outside of the machine.
+  class StateResolver
+    def initialize(metric)
+      @metric = metric
+    end
+
+    def resolve(pipelines, pipeline_configs)
+      actions = []
+
+      pipeline_configs.each do |pipeline_config|
+        pipeline = pipelines[pipeline_config.pipeline_id]
+
+        if pipeline.nil?
+          actions << LogStash::PipelineAction::Create.new(pipeline_config, @metric)
+        else
+          # TODO(ph): The pipeline should keep a reference to the original PipelineConfig
+          # and we could use straight comparison.
+          if pipeline_config.config_hash != pipeline.config_hash
+            actions << LogStash::PipelineAction::Reload.new(pipeline_config, @metric)
+          end
+        end
+      end
+
+      running_pipelines = pipeline_configs.collect(&:pipeline_id)
+
+      # If one of the running pipeline is not in the pipeline_configs, we assume that we need to
+      # stop it.
+      pipelines.keys
+        .select { |pipeline_id| !running_pipelines.include?(pipeline_id) }
+        .each { |pipeline_id| actions << LogStash::PipelineAction::Stop.new(pipeline_id) }
+
+      actions.sort # See logstash/pipeline_action.rb
+    end
+  end
+end

data/lib/logstash/util/java_version.rb

@@ -32,6 +32,12 @@ module LogStash::Util::JavaVersion
     }
   end
 
+  def self.validate_java_version!
+    if bad_java_version?(version)
+      raise "Java version 1.8.0 or later is required. (You are running: #{version})"
+    end
+  end
+
   # Determine if the given java version string is a bad version of java
   # If it is, return true, if it isn't return false.
   # Accepts nil, returning nil.

data/lib/logstash/util/safe_uri.rb

@@ -11,23 +11,21 @@ class LogStash::Util::SafeURI
   
   extend Forwardable
   
+  def_delegators :@uri, :coerce, :query=, :route_from, :port=, :default_port, :select, :normalize!, :absolute?, :registry=, :path, :password, :hostname, :merge, :normalize, :host, :component_ary, :userinfo=, :query, :set_opaque, :+, :merge!, :-, :password=, :parser, :port, :set_host, :set_path, :opaque=, :scheme, :fragment=, :set_query, :set_fragment, :userinfo, :hostname=, :set_port, :path=, :registry, :opaque, :route_to, :set_password, :hierarchical?, :set_user, :set_registry, :set_userinfo, :fragment, :component, :user=, :set_scheme, :absolute, :host=, :relative?, :scheme=, :user
   
   attr_reader :uri
-
+  
   public
-  def initialize(arg)
+  def initialize(arg)    
     @uri = case arg
            when String
              arg = "//#{arg}" if HOSTNAME_PORT_REGEX.match(arg)
-             java.net.URI.new(arg)
-           when java.net.URI
-             arg
+             URI.parse(arg)
            when URI
-             java.net.URI.new(arg.to_s)
+             arg
            else
-             raise ArgumentError, "Expected a string, java.net.URI, or URI, got a #{arg.class} creating a URL"
+             raise ArgumentError, "Expected a string or URI, got a #{arg.class} creating a URL"
            end
-    raise ArgumentError, "URI is not valid - host is not specified" if @uri.host.nil?
   end
 
   def to_s
@@ -39,11 +37,11 @@ class LogStash::Util::SafeURI
   end
 
   def sanitized
-    return uri unless password # nothing to sanitize here!
+    return uri unless uri.password # nothing to sanitize here!
     
-    user_info = user ? "#{user}:#{PASS_PLACEHOLDER}" : nil
-
-    make_uri(scheme, user_info, host, port, path, query, fragment)
+    safe = uri.clone
+    safe.password = PASS_PLACEHOLDER
+    safe
   end
 
   def ==(other)
@@ -51,142 +49,8 @@ class LogStash::Util::SafeURI
   end
 
   def clone
-    # No need to clone the URI, in java its immutable
-    self.class.new(uri)
-  end
-
-  def update(field, value)
-    new_scheme = scheme
-    new_user = user
-    new_password = password
-    new_host = host
-    new_port = port
-    new_path = path
-    new_query = query
-    new_fragment = fragment
-
-    case field 
-    when :scheme
-      new_scheme = value
-    when :user
-      new_user = value
-    when :password
-      new_password = value
-    when :host
-      new_host = value
-    when :port
-      new_port = value
-    when :path
-      new_path = value
-    when :query
-      new_query = value
-    when :fragment
-      new_fragment = value
-    end
-
-    user_info = new_user
-    if new_user && new_password
-      user_info += ":" + new_password
-    end
-
-    @uri = make_uri(new_scheme, user_info, new_host, new_port, new_path, new_query, new_fragment)
-  end
-
-  def user
-    if userinfo
-      userinfo.split(":")[0]
-    end
-  end
-
-  def user=(new_user)
-    update(:user, new_user)
-  end
-
-  def password
-    if userinfo
-      userinfo.split(":")[1]
-    end
-  end
-
-  def password=(new_password)
-    update(:password, new_password)
-  end
-
-  def hostname
-    # Alias from the ruby library
-    host
-  end
-
-  def host=(new_host)
-    update(:host, new_host)
-  end
-
-  def port
-    # In java this is an int
-    uri.port < 1 ? nil : uri.port
-  end
- 
-  def port=(new_port)
-    update(:port, new_port)
-  end
-
-  def path=(new_path)
-    update(:path, new_path)
-  end
-
-  def query=(new_query)
-    update(:query, new_query)
-  end
-
-  def fragment=(new_fragment)
-    update(:fragment, new_fragment)
-  end
-
-  # Same algorithm as Ruby's URI class uses
-  def normalize!
-    if path && path == ''
-      path = '/'
-    end
-    if scheme && scheme != scheme.downcase
-      scheme = self.scheme.downcase
-    end
-    if host && host != host.downcase
-      host = self.host.downcase
-    end
-  end
-
-  def normalize
-    d = self.dup
-    d.normalize!
-    d
-  end
-
-  def path
-    @uri.raw_path
-  end
-
-  def query
-    @uri.raw_query
-  end
-
-  def fragment
-    @uri.raw_fragment
-  end
-
-  def userinfo
-    @uri.raw_user_info
-  end
-
-  def_delegators :@uri, :absolute?, :scheme, :host
-
-  private
-
-  # Jruby doesn't guess the constructor correctly if there are some nil things in place
-  # hence, this method
-  def make_uri(scheme, user_info, host, port, path, query, fragment)
-    # It is lot legal to have a path not starting with a /
-    prefixed_path = path && path[0] != "/" ? "/#{path}" : path
-    java.net.URI.new(scheme, user_info, host, port || -1, prefixed_path, query, fragment)
+    cloned_uri = uri.clone
+    self.class.new(cloned_uri)
   end
 end
 

data/lib/logstash/util/thread_dump.rb

@@ -1,6 +1,4 @@
 # encoding: utf-8
-java_import 'org.logstash.instrument.reports.ThreadsReport'
-
 module LogStash
   module Util
     class ThreadDump
@@ -12,20 +10,19 @@ module LogStash
 
       def initialize(options={})
         @options   = options
-        @dump = options.fetch(:dump, ThreadsReport.generate({}))
+        @dump = options.fetch(:dump, JRMonitor.threads.generate({}))
         @top_count = options.fetch(:threads, THREADS_COUNT_DEFAULT)
         @ignore    = options.fetch(:ignore_idle_threads, IGNORE_IDLE_THREADS_DEFAULT)
       end
 
       def each(&block)
         i=0
-        dump.each do |hash|
-          thread_name = hash["thread.name"]
+        dump.each_pair do |thread_name, _hash|
           break if i >= top_count
           if ignore
-            next if idle_thread?(thread_name, hash)
+            next if idle_thread?(thread_name, _hash)
           end
-          block.call(hash)
+          block.call(thread_name, _hash)
           i += 1
         end
       end

data/lib/logstash/util/wrapped_acked_queue.rb

@@ -20,6 +20,12 @@ module LogStash; module Util
     class QueueClosedError < ::StandardError; end
     class NotImplementedError < ::StandardError; end
 
+    def self.create_memory_based(path, capacity, max_events, max_bytes)
+      self.allocate.with_queue(
+        LogStash::AckedMemoryQueue.new(path, capacity, max_events, max_bytes)
+      )
+    end
+
     def self.create_file_based(path, capacity, max_events, checkpoint_max_writes, checkpoint_max_acks, checkpoint_max_interval, max_bytes)
       self.allocate.with_queue(
         LogStash::AckedQueue.new(path, capacity, max_events, checkpoint_max_writes, checkpoint_max_acks, checkpoint_max_interval, max_bytes)
@@ -51,6 +57,24 @@ module LogStash; module Util
     end
     alias_method(:<<, :push)
 
+    # TODO - fix doc for this noop method
+    # Offer an object to the queue, wait for the specified amount of time.
+    # If adding to the queue was successful it will return true, false otherwise.
+    #
+    # @param [Object] Object to add to the queue
+    # @param [Integer] Time in milliseconds to wait before giving up
+    # @return [Boolean] True if adding was successful if not it return false
+    def offer(obj, timeout_ms)
+      raise NotImplementedError.new("The offer method is not implemented. There is no non blocking write operation yet.")
+    end
+
+    # Blocking
+    def take
+      check_closed("read a batch")
+      # TODO - determine better arbitrary timeout millis
+      @queue.read_batch(1, 200).get_elements.first
+    end
+
     # Block for X millis
     def poll(millis)
       check_closed("read")
@@ -76,10 +100,6 @@ module LogStash; module Util
       end
     end
 
-    def is_empty?
-      @queue.is_empty?
-    end
-
     def close
       @queue.close
       @closed.make_true
@@ -107,12 +127,7 @@ module LogStash; module Util
       end
 
       def empty?
-        @mutex.lock
-        begin
-          @queue.is_empty?
-        ensure
-          @mutex.unlock
-        end
+        @mutex.synchronize { @queue.is_fully_acked? }
       end
 
       def set_batch_dimensions(batch_size, wait_for)
@@ -137,11 +152,8 @@ module LogStash; module Util
       end
 
       def inflight_batches
-        @mutex.lock
-        begin
+        @mutex.synchronize do
           yield(@inflight_batches)
-        ensure
-          @mutex.unlock
         end
       end
 
@@ -161,24 +173,16 @@ module LogStash; module Util
         end
 
         batch = new_batch
-        @mutex.lock
-        begin
-          batch.read_next
-        ensure
-          @mutex.unlock
-        end
+        @mutex.synchronize { batch.read_next }
         start_metrics(batch)
         batch
       end
 
       def start_metrics(batch)
-        @mutex.lock
-        begin
+        @mutex.synchronize do
           # there seems to be concurrency issues with metrics, keep it in the mutex
           set_current_thread_inflight_batch(batch)
           start_clock
-        ensure
-          @mutex.unlock
         end
       end
 
@@ -187,30 +191,29 @@ module LogStash; module Util
       end
 
       def close_batch(batch)
-        @mutex.lock
-        begin
+        @mutex.synchronize do
           batch.close
+
           # there seems to be concurrency issues with metrics, keep it in the mutex
           @inflight_batches.delete(Thread.current)
           stop_clock(batch)
-        ensure
-          @mutex.unlock
         end
       end
 
       def start_clock
-        @inflight_clocks[Thread.current] = java.lang.System.current_time_millis
+        @inflight_clocks[Thread.current] = [
+          @event_metric.time(:duration_in_millis),
+          @pipeline_metric.time(:duration_in_millis)
+        ]
       end
 
       def stop_clock(batch)
         unless @inflight_clocks[Thread.current].nil?
           if batch.size > 0
-            # only stop (which also records) the metrics if the batch is non-empty.
+            # onl/y stop (which also records) the metrics if the batch is non-empty.
             # start_clock is now called at empty batch creation and an empty batch could
             # stay empty all the way down to the close_batch call.
-            time_taken = java.lang.System.current_time_millis - @inflight_clocks[Thread.current]
-            @event_metric.report_time(:duration_in_millis, time_taken)
-            @pipeline_metric.report_time(:duration_in_millis, time_taken)
+            @inflight_clocks[Thread.current].each(&:stop)
           end
           @inflight_clocks.delete(Thread.current)
         end
@@ -281,12 +284,6 @@ module LogStash; module Util
         # @cancelled[event] = true
       end
 
-      def to_a
-        events = []
-        each {|e| events << e}
-        events
-      end
-
       def each(&blk)
         # take care not to cause @originals or @generated to change during iteration