logstash-core 6.1.4-java → 6.2.0-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b169d383bfb031f3f23268331a911211c1f040f27d7f68f9001500fd584ae386
-  data.tar.gz: d14b9d18184b1353cd4a7b3726b9d7f7f667d25f95f223c720a902657f3073d2
+  metadata.gz: 863042f74fef490f23b6393fd8f2e5e46ed2d41feefa150e3c6b5f32f6502b42
+  data.tar.gz: 33f0feed45740a7104a6e6c0a5f894ceb21a5916545971033288ca7604c65dd0
 SHA512:
-  metadata.gz: 768f160b92db5c33a94e33a6e73664bc46dcbdbf260b81c8419b076c67b02a75aa713cfe385641d56780bec9dc203cd248396c2c67334d13ffc802402191975d
-  data.tar.gz: ff7eafa5203c4e35597002e49bba9bc88787ec62ea89737c6242ea804a5d9f0ffefef0dce04f391727e52a220235a83601285da5359f9c4f0abb0820f08aa1c5
+  metadata.gz: d8969c92ee13601ac5e00ff2a06ee5f9c41d2d47b4739de30852a116f10a58a8bc3c9508bc42d972fcf4da5ecfcb59eddc597b8dcc7c7d5f40a9ed1b35783a8a
+  data.tar.gz: 3b3000cb8f55315c756e6651694313c73a746e92124582173cb23515317a2defcfd869269612f5d115891f41b916827978778ad393a0c1c6b066044211be55dc

data/lib/logstash-core/logstash-core.rb

@@ -2,36 +2,15 @@
 
 require "java"
 
-module LogStash
-end
-
-require "logstash-core_jars"
-
-# local dev setup
-alt_classdir = File.expand_path("../../../out/production/classes", __FILE__) #IntelliJ's gradle output as of 2017.02 https://youtrack.jetbrains.com/issue/IDEA-175172
-if File.directory?(alt_classdir)
-  classes_dir = alt_classdir
-  resources_dir = File.expand_path("../../../out/production/resources", __FILE__)
-else
-  classes_dir = File.expand_path("../../../build/classes/java/main", __FILE__)
-  resources_dir = File.expand_path("../../../build/resources/main", __FILE__)
-end
-
-
-
-if File.directory?(classes_dir) && File.directory?(resources_dir)
-  # if in local dev setup, add target to classpath
-  $CLASSPATH << classes_dir unless $CLASSPATH.include?(classes_dir)
-  $CLASSPATH << resources_dir unless $CLASSPATH.include?(resources_dir)
-else
-  # otherwise use included jar
-  begin
-    require "logstash-core/logstash-core.jar"
-  rescue Exception => e
-    raise("Error loading logstash-core/logstash-core.jar file, cause: #{e.message}")
+# This block is used to load Logstash's Java libraries when using a Ruby entrypoint and
+# LS_JARS_LOADED is not globally set.
+# Currently this happens when using the `bin/rspec` executable to invoke specs instead of the JUnit
+# wrapper.
+unless $LS_JARS_LOADED
+  jar_path = File.join(File.dirname(File.dirname(__FILE__)), "jars")
+  $:.unshift jar_path
+  Dir.glob(jar_path + '/*.jar') do |jar|
+    require File.basename(jar)
   end
+  java_import org.logstash.RubyUtil
 end
-
-# Load Logstash's Java-defined RubyClasses by classloading RubyUtil which sets them up in its
-# static constructor
-java_import org.logstash.RubyUtil

data/lib/logstash/agent.rb

@@ -81,7 +81,7 @@ class LogStash::Agent
 
   def execute
     @thread = Thread.current # this var is implicitly used by Stud.stop?
-    logger.debug("starting agent")
+    logger.debug("Starting agent")
 
     start_webserver
 
@@ -272,24 +272,6 @@ class LogStash::Agent
     end
   end
 
-  def close_pipeline(id)
-    with_pipelines do |pipelines|
-      pipeline = pipelines[id]
-      if pipeline
-        @logger.warn("closing pipeline", :id => id)
-        pipeline.close
-      end
-    end
-  end
-
-  def close_pipelines
-    with_pipelines do |pipelines|
-      pipelines.each  do |id, _|
-        close_pipeline(id)
-      end
-    end
-  end
-
   private
   def transition_to_stopped
     @running.make_false
@@ -310,12 +292,10 @@ class LogStash::Agent
   # for other tasks.
   #
   def converge_state(pipeline_actions)
-    logger.debug("Converging pipelines")
+    logger.debug("Converging pipelines state", :actions_count => pipeline_actions.size)
 
     converge_result = LogStash::ConvergeResult.new(pipeline_actions.size)
 
-    logger.debug("Needed actions to converge", :actions_count => pipeline_actions.size) unless pipeline_actions.empty?
-
     pipeline_actions.each do |action|
       # We execute every task we need to converge the current state of pipelines
       # for every task we will record the action result, that will help us
@@ -409,7 +389,7 @@ class LogStash::Agent
     @collector = LogStash::Instrument::Collector.new
 
     @metric = if collect_metrics?
-      @logger.debug("Agent: Configuring metric collection")
+      @logger.debug("Setting up metric collection")
       LogStash::Instrument::Metric.new(@collector)
     else
       LogStash::Instrument::NullMetric.new(@collector)

data/lib/logstash/api/modules/logging.rb

@@ -36,6 +36,17 @@ module LogStash
           end
         end
 
+        put "/reset" do
+          context = LogStash::Logging::Logger::get_logging_context
+          if context.nil?
+            status 500
+            respond_with({"error" => "Logstash loggers were not initialized properly"})
+          else
+            context.reconfigure
+            respond_with({"acknowledged" => true})
+          end
+        end
+
         get "/" do
           context = LogStash::Logging::Logger::get_logging_context
           if context.nil?

data/lib/logstash/config/source/multi_local.rb

@@ -31,9 +31,11 @@ module LogStash module Config module Source
     end
 
     def match?
+      if modules_cli? || modules? || config_string? || config_path?
+        return false
+      end
       detect_pipelines if !@detect_pipelines_called
-      # see basic settings predicates and getters defined in the base class
-      return !(invalid_pipelines_detected? || modules_cli? || modules? || config_string? || config_path?)
+      return !(invalid_pipelines_detected?)
     end
 
     def invalid_pipelines_detected?
@@ -41,10 +43,10 @@ module LogStash module Config module Source
     end
 
     def config_conflict?
-      detect_pipelines if !@detect_pipelines_called
       @conflict_messages.clear
       # are there any auto-reload conflicts?
       if !(modules_cli? || modules? || config_string? || config_path?)
+        detect_pipelines if !@detect_pipelines_called
         if @detected_marker.nil?
           @conflict_messages << I18n.t("logstash.runner.config-pipelines-failed-read", :path => pipelines_yaml_location)
         elsif @detected_marker == false

data/lib/logstash/environment.rb

@@ -1,4 +1,5 @@
 # encoding: utf-8
+require "logstash-core/logstash-core"
 require "logstash/errors"
 require "logstash/java_integration"
 require "logstash/config/cpu_core_strategy"
@@ -38,7 +39,7 @@ module LogStash
    Setting::PositiveInteger.new("pipeline.workers", LogStash::Config::CpuCoreStrategy.maximum),
    Setting::PositiveInteger.new("pipeline.output.workers", 1),
    Setting::PositiveInteger.new("pipeline.batch.size", 125),
-           Setting::Numeric.new("pipeline.batch.delay", 5), # in milliseconds
+           Setting::Numeric.new("pipeline.batch.delay", 50), # in milliseconds
            Setting::Boolean.new("pipeline.unsafe_shutdown", false),
            Setting::Boolean.new("pipeline.java_execution", false),
            Setting::Boolean.new("pipeline.reloadable", true),
@@ -54,7 +55,7 @@ module LogStash
             Setting::String.new("http.environment", "production"),
             Setting::String.new("queue.type", "memory", true, ["persisted", "memory", "memory_acked"]),
             Setting::Boolean.new("queue.drain", false),
-            Setting::Bytes.new("queue.page_capacity", "250mb"),
+            Setting::Bytes.new("queue.page_capacity", "64mb"),
             Setting::Bytes.new("queue.max_bytes", "1024mb"),
             Setting::Numeric.new("queue.max_events", 0), # 0 is unlimited
             Setting::Numeric.new("queue.checkpoint.acks", 1024), # 0 is unlimited
@@ -65,9 +66,14 @@ module LogStash
             Setting::TimeValue.new("slowlog.threshold.warn", "-1"),
             Setting::TimeValue.new("slowlog.threshold.info", "-1"),
             Setting::TimeValue.new("slowlog.threshold.debug", "-1"),
-            Setting::TimeValue.new("slowlog.threshold.trace", "-1")
+            Setting::TimeValue.new("slowlog.threshold.trace", "-1"),
+            Setting::String.new("keystore.classname", "org.logstash.secret.store.backend.JavaKeyStore"),
+            Setting::String.new("keystore.file", ::File.join(::File.join(LogStash::Environment::LOGSTASH_HOME, "config"), "logstash.keystore"), false) # will be populated on
+  # post_process
   ].each {|setting| SETTINGS.register(setting) }
 
+
+
   # Compute the default queue path based on `path.data`
   default_queue_file_path = ::File.join(SETTINGS.get("path.data"), "queue")
   SETTINGS.register Setting::WritableDirectory.new("path.queue", default_queue_file_path)
@@ -75,6 +81,7 @@ module LogStash
   default_dlq_file_path = ::File.join(SETTINGS.get("path.data"), "dead_letter_queue")
   SETTINGS.register Setting::WritableDirectory.new("path.dead_letter_queue", default_dlq_file_path)
 
+
   SETTINGS.on_post_process do |settings|
     # If the data path is overridden but the queue path isn't recompute the queue path
     # We need to do this at this stage because of the weird execution order

data/lib/logstash/event.rb

@@ -2,7 +2,6 @@
 
 require "logstash/namespace"
 require "logstash/json"
-require "logstash/timestamp"
 
 # transient pipeline events for normal in-flow signaling as opposed to
 # flow altering exceptions. for now having base classes is adequate and

data/lib/logstash/filter_delegator.rb

@@ -16,8 +16,7 @@ module LogStash
 
     attr_reader :id
 
-    def initialize(logger, klass, metric, execution_context, plugin_args)
-      @logger = logger
+    def initialize(klass, metric, execution_context, plugin_args)
       @klass = klass
       @id = plugin_args["id"]
       @filter = klass.new(plugin_args)

data/lib/logstash/inputs/base.rb

@@ -84,7 +84,7 @@ class LogStash::Inputs::Base < LogStash::Plugin
 
   public
   def do_stop
-    @logger.debug("stopping", :plugin => self.class.name)
+    @logger.debug("Stopping", :plugin => self.class.name)
     @stop_called.make_true
     stop
   end

data/lib/logstash/instrument/periodic_poller/base.rb

@@ -27,7 +27,7 @@ module LogStash module Instrument module PeriodicPoller
       if exception.is_a?(Concurrent::TimeoutError)
         # On a busy system this can happen, we just log it as a debug
         # event instead of an error, Some of the JVM calls can take a long time or block.
-        logger.debug("PeriodicPoller: Timeout exception",
+        logger.debug("Timeout exception",
                 :poller => self,
                 :result => result,
                 :polling_timeout => @options[:polling_timeout],
@@ -35,7 +35,7 @@ module LogStash module Instrument module PeriodicPoller
                 :exception => exception.class,
                 :executed_at => time)
       else
-        logger.error("PeriodicPoller: exception",
+        logger.error("Exception",
                 :poller => self,
                 :result => result,
                 :exception => exception.class,
@@ -50,7 +50,7 @@ module LogStash module Instrument module PeriodicPoller
     end
 
     def start
-      logger.debug("PeriodicPoller: Starting",
+      logger.debug("Starting",
                    :polling_interval => @options[:polling_interval],
                    :polling_timeout => @options[:polling_timeout]) if logger.debug?
 
@@ -59,7 +59,7 @@ module LogStash module Instrument module PeriodicPoller
     end
 
     def stop
-      logger.debug("PeriodicPoller: Stopping")
+      logger.debug("Stopping")
       @task.shutdown
     end
 

data/lib/logstash/instrument/periodic_poller/jvm.rb

@@ -20,8 +20,8 @@ java_import 'org.logstash.instrument.reports.ProcessReport'
 module LogStash module Instrument module PeriodicPoller
   class JVM < Base
     class GarbageCollectorName
-      YOUNG_GC_NAMES = Set.new(["Copy", "PS Scavenge", "ParNew", "G1 Young Generation"])
-      OLD_GC_NAMES = Set.new(["MarkSweepCompact", "PS MarkSweep", "ConcurrentMarkSweep", "G1 Old Generation"])
+      YOUNG_GC_NAMES = Set.new(["Copy", "PS Scavenge", "ParNew", "G1 Young Generation", "scavenge"])
+      OLD_GC_NAMES = Set.new(["MarkSweepCompact", "PS MarkSweep", "ConcurrentMarkSweep", "G1 Old Generation", "global"])
 
       YOUNG = :young
       OLD = :old
@@ -64,7 +64,9 @@ module LogStash module Instrument module PeriodicPoller
       garbage_collectors = ManagementFactory.getGarbageCollectorMXBeans()
 
       garbage_collectors.each do |collector|
-        name = GarbageCollectorName.get(collector.getName())
+        collector_name = collector.getName()
+        logger.debug("collector name", :name => collector_name)
+        name = GarbageCollectorName.get(collector_name)
         if name.nil?
           logger.error("Unknown garbage collector name", :name => name)
         else

data/lib/logstash/java_filter_delegator.rb

@@ -17,8 +17,7 @@ module LogStash
 
     attr_reader :id
 
-    def initialize(logger, klass, metric, execution_context, plugin_args)
-      @logger = logger
+    def initialize(klass, metric, execution_context, plugin_args)
       @klass = klass
       @id = plugin_args["id"]
       @filter = klass.new(plugin_args)

data/lib/logstash/java_pipeline.rb

@@ -57,7 +57,7 @@ module LogStash; class JavaBasePipeline
     @plugin_factory = LogStash::Plugins::PluginFactory.new(
       # use NullMetric if called in the BasePipeline context otherwise use the @metric value
       @lir, LogStash::Plugins::PluginMetricFactory.new(pipeline_id, @metric || Instrument::NullMetric.new),
-      @logger, LogStash::Plugins::ExecutionContextFactory.new(@agent, self, @dlq_writer),
+      LogStash::Plugins::ExecutionContextFactory.new(@agent, self, @dlq_writer),
       JavaFilterDelegator
     )
     @lir_execution = CompiledPipeline.new(@lir, @plugin_factory)
@@ -330,7 +330,10 @@ module LogStash; class JavaPipeline < JavaBasePipeline
   # @param plugins [Array[Plugin]] the list of plugins to register
   def register_plugins(plugins)
     registered = []
-    plugins.each { |plugin| registered << @lir_execution.registerPlugin(plugin) }
+    plugins.each do |plugin|
+      plugin.register
+      registered << plugin
+    end
   rescue => e
     registered.each(&:do_close)
     raise e
@@ -651,6 +654,7 @@ module LogStash; class JavaPipeline < JavaBasePipeline
     filtered_size = batch.filtered_size
     @filter_queue_client.add_output_metrics(filtered_size)
     @filter_queue_client.add_filtered_metrics(filtered_size)
+    @flushing.set(false) if flush
   rescue Exception => e
     # Plugins authors should manage their own exceptions in the plugin code
     # but if an exception is raised up to the worker thread they are considered

data/lib/logstash/modules/kibana_client.rb

@@ -2,7 +2,7 @@
 require "logstash/namespace"
 require "logstash/logging"
 require "logstash/json"
-require "manticore/client"
+require "manticore"
 
 module LogStash module Modules class KibanaClient
   include LogStash::Util::Loggable

data/lib/logstash/output_delegator.rb

@@ -7,8 +7,7 @@ require "logstash/output_delegator_strategies/legacy"
 module LogStash class OutputDelegator
   attr_reader :metric, :metric_events, :strategy, :namespaced_metric, :metric_events, :id
 
-  def initialize(logger, output_class, metric, execution_context, strategy_registry, plugin_args)
-    @logger = logger
+  def initialize(output_class, metric, execution_context, strategy_registry, plugin_args)
     @output_class = output_class
     @metric = metric
     @id = plugin_args["id"]
@@ -24,7 +23,7 @@ module LogStash class OutputDelegator
     @time_metric = @metric_events.counter(:duration_in_millis)
     @strategy = strategy_registry.
                   class_for(self.concurrency).
-                  new(@logger, @output_class, @namespaced_metric, execution_context, plugin_args)
+                  new(@output_class, @namespaced_metric, execution_context, plugin_args)
   end
 
   def config_name

data/lib/logstash/output_delegator_strategies/legacy.rb

@@ -1,8 +1,8 @@
 # Remove this in Logstash 6.0
 module LogStash module OutputDelegatorStrategies class Legacy
   attr_reader :worker_count, :workers
-  
-  def initialize(logger, klass, metric, execution_context, plugin_args)
+
+  def initialize(klass, metric, execution_context, plugin_args)
     @worker_count = (plugin_args["workers"] || 1).to_i
     @workers = @worker_count.times.map { klass.new(plugin_args) }
     @workers.each do |w|
@@ -12,11 +12,11 @@ module LogStash module OutputDelegatorStrategies class Legacy
     @worker_queue = SizedQueue.new(@worker_count)
     @workers.each {|w| @worker_queue << w}
   end
-  
+
   def register
     @workers.each(&:register)
   end
-  
+
   def multi_receive(events)
     worker = @worker_queue.pop
     worker.multi_receive(events)

data/lib/logstash/output_delegator_strategies/shared.rb

@@ -1,10 +1,10 @@
 module LogStash module OutputDelegatorStrategies class Shared
-  def initialize(logger, klass, metric, execution_context, plugin_args)
+  def initialize(klass, metric, execution_context, plugin_args)
     @output = klass.new(plugin_args)
     @output.metric = metric
     @output.execution_context = execution_context
   end
-  
+
   def register
     @output.register
   end
@@ -13,10 +13,10 @@ module LogStash module OutputDelegatorStrategies class Shared
     @output.multi_receive(events)
   end
 
-  def do_close    
+  def do_close
     @output.do_close
   end
 
-  ::LogStash::OutputDelegatorStrategyRegistry.instance.register(:shared, self)  
+  ::LogStash::OutputDelegatorStrategyRegistry.instance.register(:shared, self)
 end; end; end
 

data/lib/logstash/output_delegator_strategies/single.rb

@@ -1,5 +1,5 @@
 module LogStash module OutputDelegatorStrategies class Single
-  def initialize(logger, klass, metric, execution_context, plugin_args)
+  def initialize(klass, metric, execution_context, plugin_args)
     @output = klass.new(plugin_args)
     @output.metric = metric
     @output.execution_context = execution_context
@@ -9,7 +9,7 @@ module LogStash module OutputDelegatorStrategies class Single
   def register
     @output.register
   end
-  
+
   def multi_receive(events)
     @mutex.synchronize do
       @output.multi_receive(events)

data/lib/logstash/pipeline.rb

@@ -64,7 +64,7 @@ module LogStash; class BasePipeline
     @plugin_factory = LogStash::Plugins::PluginFactory.new(
       # use NullMetric if called in the BasePipeline context otherwise use the @metric value
       @lir, LogStash::Plugins::PluginMetricFactory.new(pipeline_id, @metric || Instrument::NullMetric.new),
-      @logger, LogStash::Plugins::ExecutionContextFactory.new(@agent, self, @dlq_writer),
+      LogStash::Plugins::ExecutionContextFactory.new(@agent, self, @dlq_writer),
       FilterDelegator
     )
     grammar = LogStashConfigParser.new
@@ -74,9 +74,7 @@ module LogStash; class BasePipeline
     parsed_config.process_escape_sequences = settings.get_value("config.support_escapes")
     config_code = parsed_config.compile
 
-    # config_code = BasePipeline.compileConfig(config_str)
-
-    if settings.get_value("config.debug") && @logger.debug?
+    if settings.get_value("config.debug")
       @logger.debug("Compiled pipeline code", default_logging_keys(:code => config_code))
     end
 
@@ -238,7 +236,10 @@ module LogStash; class Pipeline < BasePipeline
     collect_stats
     collect_dlq_stats
 
-    @logger.debug("Starting pipeline", default_logging_keys)
+    @logger.info("Starting pipeline", default_logging_keys(
+      "pipeline.workers" => @settings.get("pipeline.workers"),
+      "pipeline.batch.size" => @settings.get("pipeline.batch.size"),
+      "pipeline.batch.delay" => @settings.get("pipeline.batch.delay")))
 
     @finished_execution = Concurrent::AtomicBoolean.new(false)
 
@@ -249,14 +250,14 @@ module LogStash; class Pipeline < BasePipeline
         @finished_execution.make_true
       rescue => e
         close
-        logger.error("Pipeline aborted due to error", default_logging_keys(:exception => e, :backtrace => e.backtrace))
+        @logger.error("Pipeline aborted due to error", default_logging_keys(:exception => e, :backtrace => e.backtrace))
       end
     end
 
     status = wait_until_started
 
     if status
-      logger.debug("Pipeline started successfully", default_logging_keys(:pipeline_id => pipeline_id))
+      @logger.info("Pipeline started succesfully", default_logging_keys)
     end
 
     status
@@ -287,8 +288,6 @@ module LogStash; class Pipeline < BasePipeline
 
     start_workers
 
-    @logger.info("Pipeline started", "pipeline.id" => @pipeline_id)
-
     # Block until all inputs have stopped
     # Generally this happens if SIGINT is sent and `shutdown` is called from an external thread
 
@@ -297,14 +296,13 @@ module LogStash; class Pipeline < BasePipeline
     wait_inputs
     transition_to_stopped
 
-    @logger.debug("Input plugins stopped! Will shutdown filter/output workers.", default_logging_keys)
-
     shutdown_flusher
+    @logger.debug("Shutting down filter/output workers", default_logging_keys)
     shutdown_workers
 
     close
 
-    @logger.debug("Pipeline has been shutdown", default_logging_keys)
+    @logger.info("Pipeline has terminated", default_logging_keys)
 
     # exit code
     return 0
@@ -378,12 +376,6 @@ module LogStash; class Pipeline < BasePipeline
       config_metric.gauge(:dead_letter_queue_enabled, dlq_enabled?)
       config_metric.gauge(:dead_letter_queue_path, @dlq_writer.get_path.to_absolute_path.to_s) if dlq_enabled?
 
-
-      @logger.info("Starting pipeline", default_logging_keys(
-        "pipeline.workers" => pipeline_workers,
-        "pipeline.batch.size" => batch_size,
-        "pipeline.batch.delay" => batch_delay,
-        "pipeline.max_inflight" => max_inflight))
       if max_inflight > MAX_INFLIGHT_WARN_THRESHOLD
         @logger.warn("CAUTION: Recommended inflight events max exceeded! Logstash will run with up to #{max_inflight} events in memory in your current configuration. If your message sizes are large this may cause instability with the default heap size. Please consider setting a non-standard heap size, changing the batch size (currently #{batch_size}), or changing the number of pipeline workers (currently #{pipeline_workers})", default_logging_keys)
       end
@@ -565,19 +557,19 @@ module LogStash; class Pipeline < BasePipeline
     # stopped
     wait_for_workers
     clear_pipeline_metrics
-    @logger.info("Pipeline terminated", "pipeline.id" => @pipeline_id)
   end # def shutdown
 
   def wait_for_workers
-    @logger.debug("Closing inputs", default_logging_keys)
-    @worker_threads.map(&:join)
-    @logger.debug("Worker closed", default_logging_keys)
+    @worker_threads.each do |t|
+      t.join
+      @logger.debug("Worker terminated", default_logging_keys(:thread => t.inspect))
+    end
   end
 
   def stop_inputs
-    @logger.debug("Closing inputs", default_logging_keys)
+    @logger.debug("Stopping inputs", default_logging_keys)
     @inputs.each(&:do_stop)
-    @logger.debug("Closed inputs", default_logging_keys)
+    @logger.debug("Stopped inputs", default_logging_keys)
   end
 
   # After `shutdown` is called from an external thread this is called from the main thread to