logstash-core 5.5.3-java → 5.6.0-java

data/lib/logstash/runner.rb

@@ -21,20 +21,10 @@ require "logstash/patches/clamp"
 require "logstash/settings"
 require "logstash/version"
 require "logstash/plugins/registry"
+require "logstash/modules/util"
 
 java_import 'org.logstash.FileLockFactory'
 
-def register_local_modules(path)
-  modules_path = File.join(path, File::Separator, "modules")
-  Dir.foreach(modules_path) do |item|
-    # Ignore unix relative path ids
-    next if item == '.' or item == '..'
-    # Ignore non-directories
-    next if !File.directory?(File.join(modules_path, File::Separator, item))
-    LogStash::PLUGIN_REGISTRY.add(:modules, item, LogStash::Modules::Scaffold.new(item, File.join(modules_path, File::Separator, item, File::Separator, "configuration")))
-  end
-end
-
 class LogStash::Runner < Clamp::StrictCommand
   include LogStash::Util::Loggable
   # The `path.settings` and `path.logs` need to be defined in the runner instead of the `logstash-core/lib/logstash/environment.rb`
@@ -73,6 +63,11 @@ class LogStash::Runner < Clamp::StrictCommand
     :multivalued => true,
     :attribute_name => "modules_variable_list"
 
+  option ["--setup"], :flag,
+    I18n.t("logstash.runner.flag.modules_setup"),
+    :default => LogStash::SETTINGS.get_default("modules_setup"),
+    :attribute_name => "modules_setup"
+
   # Pipeline settings
   option ["-w", "--pipeline.workers"], "COUNT",
     I18n.t("logstash.runner.flag.pipeline-workers"),
@@ -226,7 +221,7 @@ class LogStash::Runner < Clamp::StrictCommand
     java.lang.System.setProperty("ls.log.level", setting("log.level"))
     unless java.lang.System.getProperty("log4j.configurationFile")
       log4j_config_location = ::File.join(setting("path.settings"), "log4j2.properties")
-      LogStash::Logging::Logger::initialize("file:///" + log4j_config_location)
+      LogStash::Logging::Logger::reconfigure("file:///" + log4j_config_location)
     end
     # override log level that may have been introduced from a custom log4j config file
     LogStash::Logging::Logger::configure_logging(setting("log.level"))
@@ -236,7 +231,7 @@ class LogStash::Runner < Clamp::StrictCommand
     end
 
     # Add local modules to the registry before everything else
-    register_local_modules(LogStash::Environment::LOGSTASH_HOME)
+    LogStash::Modules::Util.register_local_modules(LogStash::Environment::LOGSTASH_HOME)
 
     # We configure the registry and load any plugin that can register hooks
     # with logstash, this need to be done before any operation.

data/lib/logstash/settings.rb

@@ -534,8 +534,27 @@ module LogStash
         end
       end
     end
-  end
 
+    class SplittableStringArray < ArrayCoercible
+      DEFAULT_TOKEN = ","
+
+      def initialize(name, klass, default, strict=true, tokenizer = DEFAULT_TOKEN, &validator_proc)
+        @element_class = klass
+        @token = tokenizer
+        super(name, klass, default, strict, &validator_proc)
+      end
+
+      def coerce(value)
+        if value.is_a?(Array)
+          value
+        elsif value.nil?
+          []
+        else
+          value.split(@token).map(&:strip)
+        end
+      end
+    end
+  end
 
   SETTINGS = Settings.new
 end

data/lib/logstash/util/wrapped_acked_queue.rb

@@ -57,24 +57,6 @@ module LogStash; module Util
     end
     alias_method(:<<, :push)
 
-    # TODO - fix doc for this noop method
-    # Offer an object to the queue, wait for the specified amount of time.
-    # If adding to the queue was successful it will return true, false otherwise.
-    #
-    # @param [Object] Object to add to the queue
-    # @param [Integer] Time in milliseconds to wait before giving up
-    # @return [Boolean] True if adding was successful if not it return false
-    def offer(obj, timeout_ms)
-      raise NotImplementedError.new("The offer method is not implemented. There is no non blocking write operation yet.")
-    end
-
-    # Blocking
-    def take
-      check_closed("read a batch")
-      # TODO - determine better arbitrary timeout millis
-      @queue.read_batch(1, 200).get_elements.first
-    end
-
     # Block for X millis
     def poll(millis)
       check_closed("read")
@@ -223,19 +205,18 @@ module LogStash; module Util
       end
 
       def start_clock
-        @inflight_clocks[Thread.current] = [
-          @event_metric.time(:duration_in_millis),
-          @pipeline_metric.time(:duration_in_millis)
-        ]
+        @inflight_clocks[Thread.current] = java.lang.System.current_time_millis
       end
 
       def stop_clock(batch)
         unless @inflight_clocks[Thread.current].nil?
           if batch.size > 0
-            # onl/y stop (which also records) the metrics if the batch is non-empty.
+            # only stop (which also records) the metrics if the batch is non-empty.
             # start_clock is now called at empty batch creation and an empty batch could
             # stay empty all the way down to the close_batch call.
-            @inflight_clocks[Thread.current].each(&:stop)
+            time_taken = java.lang.System.current_time_millis - @inflight_clocks[Thread.current]
+            @event_metric.report_time(:duration_in_millis, time_taken)
+            @pipeline_metric.report_time(:duration_in_millis, time_taken)
           end
           @inflight_clocks.delete(Thread.current)
         end

data/lib/logstash/util/wrapped_synchronous_queue.rb

@@ -18,21 +18,6 @@ module LogStash; module Util
     end
     alias_method(:<<, :push)
 
-    # Offer an object to the queue, wait for the specified amount of time.
-    # If adding to the queue was successful it wil return true, false otherwise.
-    #
-    # @param [Object] Object to add to the queue
-    # @param [Integer] Time in milliseconds to wait before giving up
-    # @return [Boolean] True if adding was successful if not it return false
-    def offer(obj, timeout_ms)
-      @queue.offer(obj, timeout_ms, TimeUnit::MILLISECONDS)
-    end
-
-    # Blocking
-    def take
-      @queue.take
-    end
-
     # Block for X millis
     def poll(millis)
       @queue.poll(millis, TimeUnit::MILLISECONDS)
@@ -83,11 +68,17 @@ module LogStash; module Util
 
       def set_events_metric(metric)
         @event_metric = metric
+        @event_metric_out = @event_metric.counter(:out)
+        @event_metric_filtered = @event_metric.counter(:filtered)
+        @event_metric_time = @event_metric.counter(:duration_in_millis)
         define_initial_metrics_values(@event_metric)
       end
 
       def set_pipeline_metric(metric)
         @pipeline_metric = metric
+        @pipeline_metric_out = @pipeline_metric.counter(:out)
+        @pipeline_metric_filtered = @pipeline_metric.counter(:filtered)
+        @pipeline_metric_time = @pipeline_metric.counter(:duration_in_millis)
         define_initial_metrics_values(@pipeline_metric)
       end
 
@@ -155,10 +146,7 @@ module LogStash; module Util
       end
 
       def start_clock
-        @inflight_clocks[Thread.current] = [
-          @event_metric.time(:duration_in_millis),
-          @pipeline_metric.time(:duration_in_millis)
-        ]
+        @inflight_clocks[Thread.current] = java.lang.System.current_time_millis
       end
 
       def stop_clock(batch)
@@ -167,20 +155,22 @@ module LogStash; module Util
             # only stop (which also records) the metrics if the batch is non-empty.
             # start_clock is now called at empty batch creation and an empty batch could
             # stay empty all the way down to the close_batch call.
-            @inflight_clocks[Thread.current].each(&:stop)
+            time_taken = java.lang.System.current_time_millis - @inflight_clocks[Thread.current]
+            @event_metric_time.increment(time_taken)
+            @pipeline_metric_time.increment(time_taken)
           end
           @inflight_clocks.delete(Thread.current)
         end
       end
 
       def add_filtered_metrics(batch)
-        @event_metric.increment(:filtered, batch.filtered_size)
-        @pipeline_metric.increment(:filtered, batch.filtered_size)
+        @event_metric_filtered.increment(batch.filtered_size)
+        @pipeline_metric_filtered.increment(batch.filtered_size)
       end
 
       def add_output_metrics(batch)
-        @event_metric.increment(:out, batch.filtered_size)
-        @pipeline_metric.increment(:out, batch.filtered_size)
+        @event_metric_out.increment(batch.filtered_size)
+        @pipeline_metric_out.increment(batch.filtered_size)
       end
     end
 

data/lib/logstash/version.rb

@@ -11,4 +11,4 @@
 #       eventually this file should be in the root logstash lib fir and dependencies in logstash-core should be
 #       fixed.
 
-LOGSTASH_VERSION = "5.5.3"
+LOGSTASH_VERSION = "5.6.0"

data/locales/en.yml

@@ -100,8 +100,10 @@ en:
           Specified modules: %{specified_modules}
           Available modules: %{available_modules}
         elasticsearch_connection_failed: >-
-          Failed to import module configurations to Elasticsearch.
-          Module: %{module_name} has hosts: %{hosts}
+          Failed to import module configurations to Elasticsearch and/or Kibana.
+          Module: %{module_name} has Elasticsearch hosts: %{elasticsearch_hosts} and Kibana hosts: %{kibana_hosts}
+        modules-too-many-specified: >-
+          Too many modules specified. Maximum allowed: %{max}, specified: %{specified_modules}
 
     runner:
       short-help: |-
@@ -120,10 +122,11 @@ en:
       config-module-exclusive: >-
         Settings 'path.config' (-f) or 'config.string' (-e) can't be used in conjunction with
         (--modules) or the "modules:" block in the logstash.yml file.
+      reload-with-modules: >-
+        Configuration reloading can't be used with command-line or logstash.yml specified modules.
       cli-module-override: >-
         Both command-line and logstash.yml modules configurations detected.
-        Using command-line module configuration and ignoring logstash.yml module
-        configuration.
+        Using command-line module configuration to override logstash.yml module configuration.
       reload-without-config-path: >-
         Configuration reloading also requires passing a configuration path with '-f yourlogstash.conf'
       locked-data-path: >-
@@ -227,6 +230,10 @@ en:
           '-M "MODULE_NAME.var.PLUGIN_TYPE.PLUGIN_NAME.VARIABLE_NAME=VALUE"'
           as in
           '-M "example.var.filter.mutate.fieldname=fieldvalue"'
+        modules_setup: |+
+          Load index template into Elasticsearch, and saved searches, 
+          index-pattern, visualizations, and dashboards into Kibana when
+          running modules.
         configtest: |+
           Check configuration for valid syntax and then exit.
         http_host: Web API binding host

data/spec/logstash/agent_spec.rb

@@ -5,6 +5,7 @@ require "logstash/inputs/generator"
 require_relative "../support/mocks_classes"
 require "fileutils"
 require_relative "../support/helpers"
+require 'timeout'
 
 describe LogStash::Agent do
 
@@ -16,6 +17,7 @@ describe LogStash::Agent do
   let(:config_file) { Stud::Temporary.pathname }
   let(:config_file_txt) { "input { generator { count => 100000 } } output { }" }
   let(:logger) { double("logger") }
+  let(:timeout) {120} #seconds
 
   subject { LogStash::Agent.new(agent_settings) }
 
@@ -151,7 +153,9 @@ describe LogStash::Agent do
 
           it "does not try to reload the pipeline" do
             t = Thread.new { subject.execute }
-            sleep(0.01) until subject.running_pipelines? && subject.pipelines.values.first.running?
+            Timeout.timeout(timeout) do
+              sleep(0.01) until subject.running_pipelines? && subject.pipelines.values.first.running?
+            end
             expect(subject).to_not receive(:reload_pipeline!)
             File.open(config_file, "w") { |f| f.puts second_pipeline_config }
             subject.reload_state!
@@ -183,7 +187,9 @@ describe LogStash::Agent do
 
           it "tries to reload the pipeline" do
             t = Thread.new { subject.execute }
-            sleep(0.01) until subject.running_pipelines? && subject.pipelines.values.first.running?
+            Timeout.timeout(timeout) do
+              sleep(0.01) until subject.running_pipelines? && subject.pipelines.values.first.running?
+            end
             expect(subject).to receive(:reload_pipeline!).once.and_call_original
             File.open(config_file, "w") { |f| f.puts second_pipeline_config }
             subject.reload_state!
@@ -213,8 +219,9 @@ describe LogStash::Agent do
         it "should periodically reload_state" do
           allow(subject).to receive(:clean_state?).and_return(false)
           t = Thread.new { subject.execute }
-          sleep(0.01) until subject.running_pipelines? && subject.pipelines.values.first.running?
-
+          Timeout.timeout(timeout) do
+            sleep(0.01) until subject.running_pipelines? && subject.pipelines.values.first.running?
+          end
           expect(subject).to receive(:reload_state!).at_least(2).times
 
           sleep 1
@@ -451,7 +458,9 @@ describe LogStash::Agent do
       pipeline_thread
 
       # wait for some events to reach the dummy_output
-      sleep(0.1) until dummy_output.events_received > initial_generator_threshold
+      Timeout.timeout(timeout) do
+        sleep(0.1) until dummy_output.events_received > initial_generator_threshold
+      end
     end
 
     after :each do
@@ -459,6 +468,8 @@ describe LogStash::Agent do
         subject.shutdown
         Stud.stop!(pipeline_thread)
         pipeline_thread.join
+      rescue
+          #don't care about errors here.
       ensure
         Thread.abort_on_exception = @abort_on_exception
       end
@@ -477,7 +488,9 @@ describe LogStash::Agent do
         subject.send(:"reload_pipeline!", "main")
 
         # wait until pipeline restarts
-        sleep(0.01) until dummy_output2.events_received > 0
+        Timeout.timeout(timeout) do
+          sleep(0.01) until dummy_output2.events_received > 0
+        end
       end
 
       it "resets the pipeline metric collector" do

data/spec/logstash/api/modules/node_spec.rb

@@ -114,7 +114,8 @@ describe LogStash::Api::Modules::Node do
           "batch_size" => Numeric,
           "batch_delay" => Numeric,
           "config_reload_automatic" => Boolean,
-          "config_reload_interval" => Numeric
+          "config_reload_interval" => Numeric,
+          "dead_letter_queue_enabled" => Boolean
         },
         "os" => {
           "name" => String,

data/spec/logstash/config/config_ast_spec.rb

@@ -6,6 +6,8 @@ require "logstash/config/grammar"
 require "logstash/config/config_ast"
 
 describe LogStashConfigParser do
+  let(:settings) { mock_settings({}) }
+
   context '#parse' do
     context "valid configuration" do
       it "should permit single-quoted attribute names" do
@@ -77,7 +79,7 @@ describe LogStashConfigParser do
         }
       CONFIG
       subject { LogStashConfigParser.new }
-         
+
       it "should compile successfully" do
         result = subject.parse(config)
         expect(result).not_to(be_nil)
@@ -142,12 +144,50 @@ describe LogStashConfigParser do
         expect(config).to be_nil
       end
     end
+
+    context "when config.support_escapes" do
+      let(:parser) { LogStashConfigParser.new }
+
+      let(:processed_value)  { 'The computer says, "No"' }
+
+      let(:config) {
+        parser.parse(%q(
+          input {
+            foo {
+              bar => "The computer says, \"No\""
+            }
+          }
+        ))
+      }
+
+      let(:compiled_string) { eval(config.recursive_select(LogStash::Config::AST::String).first.compile) }
+
+      before do
+        config.process_escape_sequences = escapes
+      end
+
+      context "is enabled" do
+        let(:escapes) { true }
+
+        it "should process escape sequences" do
+          expect(compiled_string).to be == processed_value
+        end
+      end
+
+      context "is false" do
+        let(:escapes) { false }
+
+        it "should not process escape sequences" do
+          expect(compiled_string).not_to be == processed_value
+        end
+      end
+    end
   end
 
   context "when using two plugin sections of the same type" do
     let(:pipeline_klass) do
       Class.new do
-        def initialize(config)
+        def initialize(config, settings)
           grammar = LogStashConfigParser.new
           @config = grammar.parse(config)
           @code = @config.compile
@@ -166,7 +206,7 @@ describe LogStashConfigParser do
 
 
       it "should create a pipeline with both sections" do
-        generated_objects = pipeline_klass.new(config_string).instance_variable_get("@generated_objects")
+        generated_objects = pipeline_klass.new(config_string, settings).instance_variable_get("@generated_objects")
         filters = generated_objects.keys.map(&:to_s).select {|obj_name| obj_name.match(/^filter.+?_\d+$/) }
         expect(filters.size).to eq(2)
       end
@@ -181,14 +221,13 @@ describe LogStashConfigParser do
 
 
       it "should create a pipeline with both sections" do
-        generated_objects = pipeline_klass.new(config_string).instance_variable_get("@generated_objects")
+        generated_objects = pipeline_klass.new(config_string, settings).instance_variable_get("@generated_objects")
         outputs = generated_objects.keys.map(&:to_s).select {|obj_name| obj_name.match(/^output.+?_\d+$/) }
         expect(outputs.size).to eq(2)
       end
     end
   end
   context "when creating two instances of the same configuration" do
-
     let(:config_string) {
       "input { generator { } }
        filter {
@@ -201,7 +240,7 @@ describe LogStashConfigParser do
 
     let(:pipeline_klass) do
       Class.new do
-        def initialize(config)
+        def initialize(config, settings)
           grammar = LogStashConfigParser.new
           @config = grammar.parse(config)
           @code = @config.compile
@@ -213,8 +252,8 @@ describe LogStashConfigParser do
 
     describe "generated conditional functionals" do
       it "should be created per instance" do
-        instance_1 = pipeline_klass.new(config_string)
-        instance_2 = pipeline_klass.new(config_string)
+        instance_1 = pipeline_klass.new(config_string, settings)
+        instance_2 = pipeline_klass.new(config_string, settings)
         generated_method_1 = instance_1.instance_variable_get("@generated_objects")[:cond_func_1]
         generated_method_2 = instance_2.instance_variable_get("@generated_objects")[:cond_func_1]
         expect(generated_method_1).to_not be(generated_method_2)