logstash-core 6.0.0.alpha2-java → 6.0.0.beta1-java

data/lib/logstash/environment.rb

@@ -20,9 +20,12 @@ module LogStash
     Setting::NullableString.new("path.config", nil, false),
  Setting::WritableDirectory.new("path.data", ::File.join(LogStash::Environment::LOGSTASH_HOME, "data")),
     Setting::NullableString.new("config.string", nil, false),
+                    Setting.new("modules.cli", Array, []),
+                    Setting.new("modules", Array, []),
            Setting::Boolean.new("config.test_and_exit", false),
            Setting::Boolean.new("config.reload.automatic", false),
-           Setting::Numeric.new("config.reload.interval", 3), # in seconds
+           Setting::TimeValue.new("config.reload.interval", "3s"), # in seconds
+           Setting::Boolean.new("config.support_escapes", false),
            Setting::Boolean.new("metric.collect", true),
             Setting::String.new("pipeline.id", "main"),
            Setting::Boolean.new("pipeline.system", false),
@@ -51,6 +54,7 @@ module LogStash
             Setting::Numeric.new("queue.checkpoint.writes", 1024), # 0 is unlimited
             Setting::Numeric.new("queue.checkpoint.interval", 1000), # 0 is no time-based checkpointing
             Setting::Boolean.new("dead_letter_queue.enable", false),
+            Setting::Bytes.new("dead_letter_queue.max_bytes", "1024mb"),
             Setting::TimeValue.new("slowlog.threshold.warn", "-1"),
             Setting::TimeValue.new("slowlog.threshold.info", "-1"),
             Setting::TimeValue.new("slowlog.threshold.debug", "-1"),

data/lib/logstash/event.rb

@@ -5,7 +5,6 @@ require "logstash/json"
 require "jruby_event_ext"
 require "jruby_timestamp_ext"
 require "logstash/timestamp"
-require "logstash/string_interpolation"
 
 # transient pipeline events for normal in-flow signaling as opposed to
 # flow altering exceptions. for now having base classes is adequate and

data/lib/logstash/instrument/global_metrics.rb

@@ -0,0 +1,13 @@
+class GlobalMetrics
+  class Stats(metric)
+    @metric = metric
+  end
+
+  def initialize(metric)
+    @metric = metric
+
+    @pipeline_reloads = metric.namespace([:stats, :pipelines])
+  end
+
+
+end

data/lib/logstash/instrument/metric_store.rb

@@ -41,26 +41,28 @@ module LogStash module Instrument
     # @param [Symbol] The metric key
     # @return [Object] Return the new_value of the retrieve object in the tree
     def fetch_or_store(namespaces, key, default_value = nil)
-      provided_value =  block_given? ? yield(key) : default_value
 
       # We first check in the `@fast_lookup` store to see if we have already see that metrics before,
       # This give us a `o(1)` access, which is faster than searching through the structured
       # data store (Which is a `o(n)` operation where `n` is the number of element in the namespace and
-      # the value of the key). If the metric is already present in the `@fast_lookup`, the call to
-      # `#put_if_absent` will return the value. This value is send back directly to the caller.
+      # the value of the key). If the metric is already present in the `@fast_lookup`, then that value is sent
+      # back directly to the caller.
       #
-      # BUT. If the value is not present in the `@fast_lookup` the value will be inserted and
-      # `#puf_if_absent` will return nil. With this returned value of nil we assume that we don't
+      # BUT. If the value is not present in the `@fast_lookup` the value will be inserted and we assume that we don't
       # have it in the `@metric_store` for structured search so we add it there too.
-      if found_value = @fast_lookup.put_if_absent(namespaces.dup << key, provided_value)
-        return found_value
-      else
+
+      # array.hash as the key since it is faster then using the array itself, see #7772
+      fast_lookup_key = (namespaces.dup << key).hash
+      value = @fast_lookup.get(fast_lookup_key)
+      if value.nil?
+        value = block_given? ? yield(key) : default_value
+        @fast_lookup.put(fast_lookup_key, value)
         @structured_lookup_mutex.synchronize do
           # If we cannot find the value this mean we need to save it in the store.
-          fetch_or_store_namespaces(namespaces).fetch_or_store(key, provided_value)
+          fetch_or_store_namespaces(namespaces).fetch_or_store(key, value)
         end
-        return provided_value
       end
+      return value;
     end
 
     # This method allow to retrieve values for a specific path,
@@ -163,7 +165,7 @@ module LogStash module Instrument
     end    
 
     def has_metric?(*path)
-      @fast_lookup[path]
+      @fast_lookup[path.hash]
     end
 
     # Return all the individuals Metric,
@@ -185,8 +187,9 @@ module LogStash module Instrument
     def prune(path)
       key_paths = key_paths(path).map(&:to_sym)
       @structured_lookup_mutex.synchronize do
-        keys_to_delete = @fast_lookup.keys.select {|namespace| (key_paths - namespace[0..-2]).empty? }
-        keys_to_delete.each {|k| @fast_lookup.delete(k) }
+        fetch_or_store_namespaces(key_paths).each do |key, v|
+          @fast_lookup.delete((key_paths.dup << key).hash)
+        end
         delete_from_map(@store, key_paths)
       end
     end

data/lib/logstash/instrument/metric_type/counter.rb

@@ -1,29 +1,17 @@
-# encoding: utf-8
-require "logstash/instrument/metric_type/base"
-require "concurrent"
+#encoding: utf-8
+java_import org.logstash.instrument.metrics.counter.LongCounter
 
 module LogStash module Instrument module MetricType
-  class Counter < Base
-    def initialize(namespaces, key, value = 0)
-      super(namespaces, key)
+  class Counter < LongCounter
 
-      @counter = Concurrent::AtomicFixnum.new(value)
-    end
-
-    def increment(value = 1)
-      @counter.increment(value)
-    end
+    def initialize(namespaces, key)
+      super(namespaces, key.to_s)
 
-    def decrement(value = 1)
-      @counter.decrement(value)
     end
 
     def execute(action, value = 1)
-      @counter.send(action, value)
+      send(action, value)
     end
 
-    def value
-      @counter.value
-    end
   end
 end; end; end

data/lib/logstash/instrument/metric_type/gauge.rb

@@ -1,22 +1,16 @@
 # encoding: utf-8
-require "logstash/instrument/metric_type/base"
-require "concurrent/atomic_reference/mutex_atomic"
-require "logstash/json"
-
+java_import org.logstash.instrument.metrics.gauge.LazyDelegatingGauge
 module LogStash module Instrument module MetricType
-  class Gauge < Base
-    def initialize(namespaces, key)
-      super(namespaces, key)
+  class Gauge < LazyDelegatingGauge
 
-      @gauge = Concurrent::MutexAtomicReference.new()
+    def initialize(namespaces, key)
+      super(namespaces, key.to_s)
     end
 
     def execute(action, value = nil)
-      @gauge.set(value)
+      send(action, value)
     end
 
-    def value
-      @gauge.get
-    end
   end
 end; end; end
+

data/lib/logstash/instrument/periodic_poller/dlq.rb

@@ -0,0 +1,19 @@
+# encoding: utf-8
+require 'logstash/instrument/periodic_poller/base'
+
+module LogStash module Instrument module PeriodicPoller
+  class DeadLetterQueue < Base
+    def initialize(metric, agent, options = {})
+      super(metric, options)
+      @metric = metric
+      @agent = agent
+    end
+
+    def collect
+      _, pipeline = @agent.with_running_pipelines { |pipelines| pipelines.first }
+      unless pipeline.nil?
+        pipeline.collect_dlq_stats
+      end
+    end
+  end
+end end end

data/lib/logstash/instrument/periodic_pollers.rb

@@ -1,4 +1,5 @@
 # encoding: utf-8
+require "logstash/instrument/periodic_poller/dlq"
 require "logstash/instrument/periodic_poller/os"
 require "logstash/instrument/periodic_poller/jvm"
 require "logstash/instrument/periodic_poller/pq"
@@ -14,7 +15,8 @@ module LogStash module Instrument
       @metric = metric
       @periodic_pollers = [PeriodicPoller::Os.new(metric),
                            PeriodicPoller::JVM.new(metric),
-                           PeriodicPoller::PersistentQueue.new(metric, queue_type, pipelines)]
+                           PeriodicPoller::PersistentQueue.new(metric, queue_type, pipelines),
+                           PeriodicPoller::DeadLetterQueue.new(metric, pipelines)]
     end
 
     def start

data/lib/logstash/logging/logger.rb

@@ -7,10 +7,13 @@ module LogStash
     java_import org.apache.logging.log4j.LogManager
     java_import org.apache.logging.log4j.core.config.Configurator
     java_import org.apache.logging.log4j.core.config.DefaultConfiguration
+    java_import org.apache.logging.log4j.core.config.LoggerConfig
+    java_import org.logstash.log.LogstashLoggerContextFactory
+    java_import org.apache.logging.log4j.core.LoggerContext
+    java_import java.net.URI
 
     class Logger
       @@config_mutex = Mutex.new
-      @@logging_context = nil
 
       def initialize(name)
         @logger = LogManager.getLogger(name)
@@ -65,31 +68,57 @@ module LogStash
       end
 
       def self.configure_logging(level, path = LogManager::ROOT_LOGGER_NAME)
-        @@config_mutex.synchronize { Configurator.setLevel(path, Level.valueOf(level)) }
+        @@config_mutex.synchronize { set_level(level, path) }
       rescue Exception => e
         raise ArgumentError, "invalid level[#{level}] for logger[#{path}]"
       end
 
       def self.initialize(config_location)
         @@config_mutex.synchronize do
-          if @@logging_context.nil?
-            file_path = URI(config_location).path
-            if ::File.exists?(file_path)
-              logs_location = java.lang.System.getProperty("ls.logs")
-              puts "Sending Logstash's logs to #{logs_location} which is now configured via log4j2.properties"
-              @@logging_context = Configurator.initialize(nil, config_location)
-            else
-              # fall back to default config
-              puts "Could not find log4j2 configuration at path #{file_path}. Using default config which logs to console"
-              @@logging_context = Configurator.initialize(DefaultConfiguration.new)
-            end
+          config_location_uri = URI.create(config_location)
+          file_path = config_location_uri.path
+          if ::File.exists?(file_path)
+            logs_location = java.lang.System.getProperty("ls.logs")
+            puts "Sending Logstash's logs to #{logs_location} which is now configured via log4j2.properties"
+            #reconfigure the default context to use our log4j2.properties file
+            get_logging_context.setConfigLocation(URI.create(config_location))
+            #ensure everyone agrees which context to use for the LogManager
+            context_factory = LogstashLoggerContextFactory.new(get_logging_context)
+            LogManager.setFactory(context_factory)
+          else
+            # fall back to default config
+            puts "Could not find log4j2 configuration at path #{file_path}. Using default config which logs errors to the console"
           end
         end
       end
 
       def self.get_logging_context
-        return @@logging_context
+        return  LoggerContext.getContext(false)
+      end
+
+      # Clone of org.apache.logging.log4j.core.config.Configurator.setLevel(), but ensure the proper context is used
+      def self.set_level(_level, path)
+        configuration =  get_logging_context.getConfiguration()
+        level = Level.valueOf(_level)
+        if path.nil? || path.strip.empty?
+          root_logger = configuration.getRootLogger()
+          if root_logger.getLevel() != level
+            root_logger.setLevel(level)
+            get_logging_context.updateLoggers()
+          end
+        else
+          package_logger = configuration.getLoggerConfig(path)
+          if package_logger.name != path #no package logger found
+            configuration.addLogger(path, LoggerConfig.new(path, level, true))
+            get_logging_context.updateLoggers()
+          elsif package_logger.getLevel() != level
+            package_logger.setLevel(level)
+            get_logging_context.updateLoggers()
+          end
+        end
       end
+
+      private_class_method :set_level
     end
 
     class SlowLogger

data/lib/logstash/modules/cli_parser.rb

@@ -0,0 +1,74 @@
+# encoding: utf-8
+require "logstash/namespace"
+require "logstash/logging"
+require "logstash/errors"
+
+module LogStash module Modules class CLIParser
+  include LogStash::Util::Loggable
+
+  attr_reader :output
+  def initialize(module_names, module_variables)
+    @output = []
+    # The #compact here catches instances when module_variables may be nil or
+    # [nil] and sets it to []
+    parse_it(module_names, Array(module_variables).compact)
+  end
+
+  def parse_modules(module_list)
+    parsed_modules = []
+    module_list.each do |module_value|
+      # Calling --modules but not filling it results in [nil], so skip that.
+      next if module_value.nil?
+      # Catch if --modules was launched empty but an option/flag (-something)
+      # follows immediately after
+      if module_value.start_with?('-')
+        raise LogStash::ConfigLoadingError, I18n.t("logstash.modules.configuration.modules-invalid-name", :module_name => module_value)
+      end
+      parsed_modules.concat module_value.split(',')
+    end
+    parsed_modules
+  end
+
+  def get_kv(module_name, unparsed)
+    # Ensure that there is at least 1 equals sign in our variable string
+    # Using String#partition to split on the first '='
+    # This hackery is to catch the possibility of an equals (`=`) sign
+    # in a passphrase, which might result in an incomplete key.  The
+    # portion before the first `=` should always be the key, leaving
+    # the rest to be the value
+    k, op, rest = unparsed.partition('=')
+    if rest.size.zero?
+      raise LogStash::ConfigLoadingError, I18n.t("logstash.modules.configuration.modules-variables-malformed", :rawvar => (module_name + '.' + unparsed))
+    end
+    return k.strip, rest.strip
+  end
+
+  def name_splitter(unparsed)
+    # It must have at least `modulename.something`
+    module_name, dot, rest = unparsed.partition('.')
+    if rest.count('.') >= 1
+      return module_name, rest
+    else
+      raise LogStash::ConfigLoadingError, I18n.t("logstash.modules.configuration.modules-variables-malformed", :rawvar => unparsed)
+    end
+  end
+
+  def parse_vars(module_name, vars_list)
+    module_hash = {"name" => module_name}
+    vars_list.each do |unparsed|
+      extracted_name, modvar = name_splitter(unparsed)
+      next if extracted_name != module_name
+      k, v = get_kv(extracted_name, modvar)
+      module_hash[k] = v
+    end
+    module_hash
+  end
+
+  def parse_it(module_list, module_variable_list)
+    if module_list.is_a?(Array)
+      parse_modules(module_list).each do |module_name|
+        @output << parse_vars(module_name, module_variable_list)
+      end
+    end
+  end
+end end end

data/lib/logstash/modules/elasticsearch_config.rb

@@ -0,0 +1,22 @@
+# encoding: utf-8
+require "logstash/namespace"
+require "logstash/logging"
+
+require_relative "elasticsearch_resource"
+
+module LogStash module Modules class ElasticsearchConfig
+  attr_reader :index_name
+
+  # We name it `modul` here because `module` has meaning in Ruby.
+  def initialize(modul, settings)
+    @directory = ::File.join(modul.directory, "elasticsearch")
+    @name = modul.module_name
+    @settings = settings
+    @full_path = ::File.join(@directory, "#{@name}.json")
+    @index_name = @settings.fetch("elasticsearch.template_path", "_template")
+  end
+
+  def resources
+    [ElasticsearchResource.new(@index_name, "not-used", @full_path)]
+  end
+end end end

data/lib/logstash/modules/elasticsearch_importer.rb

@@ -0,0 +1,37 @@
+# encoding: utf-8
+require "logstash/namespace"
+require "logstash/logging"
+
+module LogStash module Modules class ElasticsearchImporter
+  include LogStash::Util::Loggable
+
+  def initialize(client)
+    @client = client
+  end
+
+  def put(resource, overwrite = true)
+    path = resource.import_path
+    logger.debug("Attempting PUT", :url_path => path, :file_path => resource.content_path)
+    if !overwrite && content_exists?(path)
+      logger.debug("Found existing Elasticsearch resource.", :resource => path)
+      return
+    end
+    put_overwrite(path, resource.content)
+  end
+
+  private
+
+  def put_overwrite(path, content)
+    if content_exists?(path)
+      response = @client.delete(path)
+    end
+    # hmmm, versioning?
+    @client.put(path, content).status == 201
+  end
+
+  def content_exists?(path)
+    response = @client.head(path)
+    response.status >= 200 && response.status < 300
+  end
+
+end end end # class LogStash::Modules::Importer

data/lib/logstash/modules/elasticsearch_resource.rb

@@ -0,0 +1,10 @@
+# encoding: utf-8
+require "logstash/namespace"
+require_relative "resource_base"
+
+module LogStash module Modules class ElasticsearchResource
+  include ResourceBase
+  def import_path
+    base + "/" + content_id
+  end
+end end end