logstash-core 6.0.0.alpha2-java → 6.0.0.beta1-java

data/lib/logstash/settings.rb

@@ -537,8 +537,27 @@ module LogStash
         end
       end
     end
-  end
 
+    class SplittableStringArray < ArrayCoercible
+      DEFAULT_TOKEN = ","
+
+      def initialize(name, klass, default, strict=true, tokenizer = DEFAULT_TOKEN, &validator_proc)
+        @element_class = klass
+        @token = tokenizer
+        super(name, klass, default, strict, &validator_proc)
+      end
+
+      def coerce(value)
+        if value.is_a?(Array)
+          value
+        elsif value.nil?
+          []
+        else
+          value.split(@token).map(&:strip)
+        end
+      end
+    end
+  end
 
   SETTINGS = Settings.new
 end

data/lib/logstash/util/dead_letter_queue_manager.rb

@@ -48,7 +48,7 @@ module LogStash; module Util
     def self.get(pipeline_id)
       if LogStash::SETTINGS.get("dead_letter_queue.enable")
         return DeadLetterQueueWriter.new(
-          DeadLetterQueueFactory.getWriter(pipeline_id, LogStash::SETTINGS.get("path.dead_letter_queue")))
+          DeadLetterQueueFactory.getWriter(pipeline_id, LogStash::SETTINGS.get("path.dead_letter_queue"), LogStash::SETTINGS.get('dead_letter_queue.max_bytes')))
       else
         return DeadLetterQueueWriter.new(nil)
       end

data/lib/logstash/util/safe_uri.rb

@@ -11,20 +11,21 @@ class LogStash::Util::SafeURI
   
   extend Forwardable
   
-  def_delegators :@uri, :coerce, :query=, :route_from, :port=, :default_port, :select, :normalize!, :absolute?, :registry=, :path, :password, :hostname, :merge, :normalize, :host, :component_ary, :userinfo=, :query, :set_opaque, :+, :merge!, :-, :password=, :parser, :port, :set_host, :set_path, :opaque=, :scheme, :fragment=, :set_query, :set_fragment, :userinfo, :hostname=, :set_port, :path=, :registry, :opaque, :route_to, :set_password, :hierarchical?, :set_user, :set_registry, :set_userinfo, :fragment, :component, :user=, :set_scheme, :absolute, :host=, :relative?, :scheme=, :user
   
   attr_reader :uri
-  
+
   public
   def initialize(arg)    
     @uri = case arg
            when String
              arg = "//#{arg}" if HOSTNAME_PORT_REGEX.match(arg)
-             URI.parse(arg)
-           when URI
+             java.net.URI.new(arg)
+           when java.net.URI
              arg
+           when URI
+             java.net.URI.new(arg.to_s)
            else
-             raise ArgumentError, "Expected a string or URI, got a #{arg.class} creating a URL"
+             raise ArgumentError, "Expected a string, java.net.URI, or URI, got a #{arg.class} creating a URL"
            end
   end
 
@@ -37,11 +38,11 @@ class LogStash::Util::SafeURI
   end
 
   def sanitized
-    return uri unless uri.password # nothing to sanitize here!
+    return uri unless password # nothing to sanitize here!
     
-    safe = uri.clone
-    safe.password = PASS_PLACEHOLDER
-    safe
+    user_info = user ? "#{user}:#{PASS_PLACEHOLDER}" : nil
+
+    make_uri(scheme, user_info, host, port, path, query, fragment)
   end
 
   def ==(other)
@@ -49,8 +50,142 @@ class LogStash::Util::SafeURI
   end
 
   def clone
-    cloned_uri = uri.clone
-    self.class.new(cloned_uri)
+    # No need to clone the URI, in java its immutable
+    self.class.new(uri)
+  end
+
+  def update(field, value)
+    new_scheme = scheme
+    new_user = user
+    new_password = password
+    new_host = host
+    new_port = port
+    new_path = path
+    new_query = query
+    new_fragment = fragment
+
+    case field 
+    when :scheme
+      new_scheme = value
+    when :user
+      new_user = value
+    when :password
+      new_password = value
+    when :host
+      new_host = value
+    when :port
+      new_port = value
+    when :path
+      new_path = value
+    when :query
+      new_query = value
+    when :fragment
+      new_fragment = value
+    end
+
+    user_info = new_user
+    if new_user && new_password
+      user_info += ":" + new_password
+    end
+
+    @uri = make_uri(new_scheme, user_info, new_host, new_port, new_path, new_query, new_fragment)
+  end
+
+  def user
+    if userinfo
+      userinfo.split(":")[0]
+    end
+  end
+
+  def user=(new_user)
+    update(:user, new_user)
+  end
+
+  def password
+    if userinfo
+      userinfo.split(":")[1]
+    end
+  end
+
+  def password=(new_password)
+    update(:password, new_password)
+  end
+
+  def hostname
+    # Alias from the ruby library
+    host
+  end
+
+  def host=(new_host)
+    update(:host, new_host)
+  end
+
+  def port
+    # In java this is an int
+    uri.port < 1 ? nil : uri.port
+  end
+ 
+  def port=(new_port)
+    update(:port, new_port)
+  end
+
+  def path=(new_path)
+    update(:path, new_path)
+  end
+
+  def query=(new_query)
+    update(:query, new_query)
+  end
+
+  def fragment=(new_fragment)
+    update(:fragment, new_fragment)
+  end
+
+  # Same algorithm as Ruby's URI class uses
+  def normalize!
+    if path && path == ''
+      path = '/'
+    end
+    if scheme && scheme != scheme.downcase
+      scheme = self.scheme.downcase
+    end
+    if host && host != host.downcase
+      host = self.host.downcase
+    end
+  end
+
+  def normalize
+    d = self.dup
+    d.normalize!
+    d
+  end
+
+  def path
+    @uri.raw_path
+  end
+
+  def query
+    @uri.raw_query
+  end
+
+  def fragment
+    @uri.raw_fragment
+  end
+
+  def userinfo
+    @uri.raw_user_info
+  end
+
+  def_delegators :@uri, :absolute?, :scheme, :host
+
+  private
+
+  # Jruby doesn't guess the constructor correctly if there are some nil things in place
+  # hence, this method
+  def make_uri(scheme, user_info, host, port, path, query, fragment)
+    # It is lot legal to have a path not starting with a /
+    prefixed_path = path && path[0] != "/" ? "/#{path}" : path
+    java.net.URI.new(scheme, user_info, host, port || -1, prefixed_path, query, fragment)
   end
 end
 

data/lib/logstash/util/thread_dump.rb

@@ -19,12 +19,13 @@ module LogStash
 
       def each(&block)
         i=0
-        dump.each_pair do |thread_name, _hash|
+        dump.each do |hash|
+          thread_name = hash["thread.name"]
           break if i >= top_count
           if ignore
-            next if idle_thread?(thread_name, _hash)
+            next if idle_thread?(thread_name, hash)
           end
-          block.call(thread_name, _hash)
+          block.call(hash)
           i += 1
         end
       end

data/lib/logstash/util/wrapped_acked_queue.rb

@@ -57,24 +57,6 @@ module LogStash; module Util
     end
     alias_method(:<<, :push)
 
-    # TODO - fix doc for this noop method
-    # Offer an object to the queue, wait for the specified amount of time.
-    # If adding to the queue was successful it will return true, false otherwise.
-    #
-    # @param [Object] Object to add to the queue
-    # @param [Integer] Time in milliseconds to wait before giving up
-    # @return [Boolean] True if adding was successful if not it return false
-    def offer(obj, timeout_ms)
-      raise NotImplementedError.new("The offer method is not implemented. There is no non blocking write operation yet.")
-    end
-
-    # Blocking
-    def take
-      check_closed("read a batch")
-      # TODO - determine better arbitrary timeout millis
-      @queue.read_batch(1, 200).get_elements.first
-    end
-
     # Block for X millis
     def poll(millis)
       check_closed("read")
@@ -100,6 +82,10 @@ module LogStash; module Util
       end
     end
 
+    def is_empty?
+      @queue.is_empty?
+    end
+
     def close
       @queue.close
       @closed.make_true
@@ -127,7 +113,12 @@ module LogStash; module Util
       end
 
       def empty?
-        @mutex.synchronize { @queue.is_fully_acked? }
+        @mutex.lock
+        begin
+          @queue.is_empty?
+        ensure
+          @mutex.unlock
+        end
       end
 
       def set_batch_dimensions(batch_size, wait_for)
@@ -152,8 +143,11 @@ module LogStash; module Util
       end
 
       def inflight_batches
-        @mutex.synchronize do
+        @mutex.lock
+        begin
           yield(@inflight_batches)
+        ensure
+          @mutex.unlock
         end
       end
 
@@ -173,16 +167,24 @@ module LogStash; module Util
         end
 
         batch = new_batch
-        @mutex.synchronize { batch.read_next }
+        @mutex.lock
+        begin
+          batch.read_next
+        ensure
+          @mutex.unlock
+        end
         start_metrics(batch)
         batch
       end
 
       def start_metrics(batch)
-        @mutex.synchronize do
+        @mutex.lock
+        begin
           # there seems to be concurrency issues with metrics, keep it in the mutex
           set_current_thread_inflight_batch(batch)
           start_clock
+        ensure
+          @mutex.unlock
         end
       end
 
@@ -191,12 +193,14 @@ module LogStash; module Util
       end
 
       def close_batch(batch)
-        @mutex.synchronize do
+        @mutex.lock
+        begin
           batch.close
-
           # there seems to be concurrency issues with metrics, keep it in the mutex
           @inflight_batches.delete(Thread.current)
           stop_clock(batch)
+        ensure
+          @mutex.unlock
         end
       end
 

data/lib/logstash/util/wrapped_synchronous_queue.rb

@@ -18,21 +18,6 @@ module LogStash; module Util
     end
     alias_method(:<<, :push)
 
-    # Offer an object to the queue, wait for the specified amount of time.
-    # If adding to the queue was successful it wil return true, false otherwise.
-    #
-    # @param [Object] Object to add to the queue
-    # @param [Integer] Time in milliseconds to wait before giving up
-    # @return [Boolean] True if adding was successful if not it return false
-    def offer(obj, timeout_ms)
-      @queue.offer(obj, timeout_ms, TimeUnit::MILLISECONDS)
-    end
-
-    # Blocking
-    def take
-      @queue.take
-    end
-
     # Block for X millis
     def poll(millis)
       @queue.poll(millis, TimeUnit::MILLISECONDS)
@@ -98,8 +83,11 @@ module LogStash; module Util
       end
 
       def inflight_batches
-        @mutex.synchronize do
+        @mutex.lock
+        begin
           yield(@inflight_batches)
+        ensure
+          @mutex.unlock
         end
       end
 
@@ -115,16 +103,24 @@ module LogStash; module Util
 
       def read_batch
         batch = new_batch
-        @mutex.synchronize { batch.read_next }
+        @mutex.lock
+        begin
+          batch.read_next
+        ensure
+          @mutex.unlock
+        end
         start_metrics(batch)
         batch
       end
 
       def start_metrics(batch)
-        @mutex.synchronize do
-          # there seems to be concurrency issues with metrics, keep it in the mutex
+        @mutex.lock
+        # there seems to be concurrency issues with metrics, keep it in the mutex
+        begin
           set_current_thread_inflight_batch(batch)
           start_clock
+        ensure
+          @mutex.unlock
         end
       end
 
@@ -133,10 +129,13 @@ module LogStash; module Util
       end
 
       def close_batch(batch)
-        @mutex.synchronize do
+        @mutex.lock
+        begin
           # there seems to be concurrency issues with metrics, keep it in the mutex
           @inflight_batches.delete(Thread.current)
           stop_clock(batch)
+        ensure
+          @mutex.unlock
         end
       end
 

data/lib/logstash/version.rb

@@ -11,4 +11,4 @@
 #       eventually this file should be in the root logstash lib fir and dependencies in logstash-core should be
 #       fixed.
 
-LOGSTASH_VERSION = "6.0.0-alpha2"
+LOGSTASH_VERSION = "6.0.0-beta1"

data/locales/en.yml

@@ -81,14 +81,37 @@ en:
           ::: {%{hostname}}
           Hot threads at %{time}, busiestThreads=%{top_count}:
         thread_title: |-
-          %{percent_of_cpu_time} % of cpu usage, state: %{thread_state}, thread name: '%{thread_name}'
+          %{percent_of_cpu_time} % of cpu usage, state: %{thread_state}, thread name: '%{thread_name}', thread id: %{thread_id}
       logging:
         unrecognized_option: |-
           unrecognized option [%{option}]
+    modules:
+      configuration:
+        parse-failed: |-
+          Failed to parse the module configuration: [%{error}]
+        modules-must-be-unique: >-
+          Only a single instance of any module can be run at a time. Duplicate
+          modules: %{duplicate_modules}
+        modules-invalid-name: >-
+          Invalid module name: %{module_name}
+        modules-variables-malformed: >-
+          Failed to parse module variable %{rawvar}.  Must be in -M
+          "MODULE_NAME.KEY.SUBKEY=VALUE" format
+        modules-unavailable: >-
+          The modules specified are not available yet.
+          Specified modules: %{specified_modules}
+          Available modules: %{available_modules}
+        elasticsearch_connection_failed: >-
+          Failed to import module configurations to Elasticsearch and/or Kibana.
+          Module: %{module_name} has Elasticsearch hosts: %{elasticsearch_hosts} and Kibana hosts: %{kibana_hosts}
+        modules-too-many-specified: >-
+          Too many modules specified. Maximum allowed: %{max}, specified: %{specified_modules}
+
     runner:
       short-help: |-
         usage:
           bin/logstash -f CONFIG_PATH [-t] [-r] [] [-w COUNT] [-l LOG]
+          bin/logstash --modules MODULE_NAME [-M "MODULE_NAME.var.PLUGIN_TYPE.PLUGIN_NAME.VARIABLE_NAME=VALUE"] [-t] [-w COUNT] [-l LOG]
           bin/logstash -e CONFIG_STR [-t] [--log.level fatal|error|warn|info|debug|trace] [-w COUNT] [-l LOG]
           bin/logstash -i SHELL [--log.level fatal|error|warn|info|debug|trace]
           bin/logstash -V [--log.level fatal|error|warn|info|debug|trace]
@@ -100,6 +123,20 @@ en:
         the '-f yourlogstash.conf' flag?
       config-string-path-exclusive:
         Settings 'path.config' (-f) and 'config.string' (-e) can't be used simultaneously.
+      config-module-exclusive: >-
+        Settings 'path.config' (-f) or 'config.string' (-e) can't be used in conjunction with
+        (--modules) or the "modules:" block in the logstash.yml file.
+      reload-with-modules: >-
+        Configuration reloading can't be used with command-line or logstash.yml specified modules.
+      cli-module-override: >-
+        Both command-line and logstash.yml modules configurations detected.
+        Using command-line module configuration to override logstash.yml module configuration.
+      config-pipelines-failed-read: >-
+        Failed to read pipelines yaml file. Location: %{path}
+      config-pipelines-empty: >-
+        Pipelines YAML file is empty. Location: %{path}
+      config-pipelines-invalid: >-
+        Pipelines YAML file must contain an array of pipeline configs. Found "%{invalid_class}" in %{path}
       reload-without-config-path: >-
         Configuration reloading also requires passing a configuration path with '-f yourlogstash.conf'
       reload-with-config-string: >-
@@ -187,6 +224,24 @@ en:
           "%{default_output}"
           If you wish to use both defaults, please use
           the empty string for the '-e' flag.
+        modules: |+
+          Load Logstash modules.
+          Modules can be defined using multiple instances
+          '--modules module1 --modules module2',
+             or comma-separated syntax
+          '--modules=module1,module2'
+          Cannot be used in conjunction with '-e' or '-f'
+          Use of '--modules' will override modules declared
+          in the 'logstash.yml' file.
+        modules_variable: |+
+          Load variables for module template.
+          Multiple instances of '-M' or
+          '--modules.variable' are supported.
+          Ignored if '--modules' flag is not used.
+          Should be in the format of
+          '-M "MODULE_NAME.var.PLUGIN_TYPE.PLUGIN_NAME.VARIABLE_NAME=VALUE"'
+          as in
+          '-M "example.var.filter.mutate.fieldname=fieldvalue"'
         configtest: |+
           Check configuration for valid syntax and then exit.
         http_host: Web API binding host