logstash-core 6.0.0.beta1-java → 6.0.0.beta2-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 46f57cb18b555e501983b0d9d9ae61c3be63fd5925f444a4018adce705fe6b1f
-  data.tar.gz: 963cbd763704c8c84ca8bb5055412baa6b57d22d3e65ac03cd7f6b2434ec227a
+  metadata.gz: 5c353e53b9ba28be27a573084a75d1dc12d1257e2192057121d5e1c41de195bc
+  data.tar.gz: 47790a7e594369b8d2c3d1ebfcde2d7fef023aaaf3afb0b747bcc75124c35547
 SHA512:
-  metadata.gz: 93a8a8282730e5c5e687b5bf2bc48d0aab82fa5ec8398336114f3e4a3d73d6705dac0854827ab9bfd5c52173177c72db87da5e6bb9a1ca95b14a826c51ec2860
-  data.tar.gz: 61df6640d1d26a96b014c9eb7399982a9829465472f9f05f5b4882c4a8720a821c6e4eca9548286dcb907e792f32f91ab1ca90fdb3f9c96dfbdb1b45d9e0b6a0
+  metadata.gz: d7acc940476d31aca243c82ea0bc693324e8cdbbeeb7c8abbd06210cd5ad82f143499653b0952696a027ae0b60557518164409a36ff7a4b3ff2d6e4c536dcbb6
+  data.tar.gz: 8848ec115ba9abfa1f3065e1ee22a63b50e7172e4aea049ebd6a37193e6dd06ee8f4b257863ab625939416a75512dffeaf7f1ad1cd9ef88d8c16db344b1a62a8

data/lib/logstash-core/version.rb

@@ -5,4 +5,4 @@
 # Note to authors: this should not include dashes because 'gem' barfs if
 # you include a dash in the version string.
 
-LOGSTASH_CORE_VERSION = "6.0.0-beta1"
+LOGSTASH_CORE_VERSION = "6.0.0-beta2"

data/lib/logstash/agent.rb

@@ -190,13 +190,6 @@ class LogStash::Agent
     converge_result
   end
 
-  def force_shutdown!
-    stop_collecting_metrics
-    stop_webserver
-    transition_to_stopped
-    force_shutdown_pipelines!
-  end
-
   def id
     return @id if @id
 
@@ -427,15 +420,6 @@ class LogStash::Agent
     @collect_metric
   end
 
-  def force_shutdown_pipelines!
-    with_pipelines do |pipelines|
-      pipelines.each do |_, pipeline|
-        # TODO(ph): should it be his own action?
-        pipeline.force_shutdown!
-      end
-    end
-  end
-
   def shutdown_pipelines
     logger.debug("Shutting down all pipelines", :pipelines_count => pipelines_count)
 

data/lib/logstash/compiler/lscl.rb

@@ -2,6 +2,7 @@
 require 'logstash/errors'
 require "treetop"
 require "logstash/compiler/treetop_monkeypatches"
+require "logstash/compiler/lscl/helpers"
 require "logstash/config/string_escape"
 
 java_import org.logstash.config.ir.DSL
@@ -10,59 +11,7 @@ java_import org.logstash.common.SourceWithMetadata
 module LogStashCompilerLSCLGrammar; module LogStash; module Compiler; module LSCL; module AST
   PROCESS_ESCAPE_SEQUENCES = :process_escape_sequences
 
-  # Helpers for parsing LSCL files
-  module Helpers
-    def source_meta
-      line, column = line_and_column
-      org.logstash.common.SourceWithMetadata.new(base_protocol, base_id, line, column, self.text_value)
-    end
-
-    def base_source_with_metadata=(value)
-      set_meta(:base_source_with_metadata, value)
-    end
-    
-    def base_source_with_metadata
-      get_meta(:base_source_with_metadata)
-    end
-
-    def base_protocol
-      self.base_source_with_metadata.protocol
-    end
-
-    def base_id
-      self.base_source_with_metadata.id
-    end
-
-    def compose(*statements)
-      compose_for(section_type.to_sym).call(source_meta, *statements)
-    end
-
-    def compose_for(section_sym)
-      if section_sym == :filter
-        jdsl.method(:iComposeSequence)
-      else
-        jdsl.method(:iComposeParallel)
-      end
-    end
-
-    def line_and_column
-      start = self.interval.first
-      [self.input.line_of(start), self.input.column_of(start)]
-    end
-
-    def jdsl
-      org.logstash.config.ir.DSL
-    end
-
-    def self.jdsl
-      org.logstash.config.ir.DSL
-    end
-    
-    AND_METHOD = jdsl.method(:eAnd)
-    OR_METHOD = jdsl.method(:eOr)
-  end
-  
-  class Node < Treetop::Runtime::SyntaxNode
+    class Node < Treetop::Runtime::SyntaxNode
     include Helpers
     
     def section_type

data/lib/logstash/compiler/lscl/helpers.rb

@@ -0,0 +1,55 @@
+# encoding: utf-8
+
+module LogStashCompilerLSCLGrammar; module LogStash; module Compiler; module LSCL; module AST
+  # Helpers for parsing LSCL files
+  module Helpers
+    def source_meta
+      line, column = line_and_column
+      org.logstash.common.SourceWithMetadata.new(base_protocol, base_id, line, column, self.text_value)
+    end
+
+    def base_source_with_metadata=(value)
+      set_meta(:base_source_with_metadata, value)
+    end
+    
+    def base_source_with_metadata
+      get_meta(:base_source_with_metadata)
+    end
+
+    def base_protocol
+      self.base_source_with_metadata ? self.base_source_with_metadata.protocol : 'config_ast'
+    end
+
+    def base_id
+      self.base_source_with_metadata ? self.base_source_with_metadata.id : 'config_ast'
+    end
+
+    def compose(*statements)
+      compose_for(section_type.to_sym).call(source_meta, *statements)
+    end
+
+    def compose_for(section_sym)
+      if section_sym == :filter
+        jdsl.method(:iComposeSequence)
+      else
+        jdsl.method(:iComposeParallel)
+      end
+    end
+
+    def line_and_column
+      start = self.interval.first
+      [self.input.line_of(start), self.input.column_of(start)]
+    end
+
+    def jdsl
+      org.logstash.config.ir.DSL
+    end
+
+    def self.jdsl
+      org.logstash.config.ir.DSL
+    end
+    
+    AND_METHOD = jdsl.method(:eAnd)
+    OR_METHOD = jdsl.method(:eOr)
+  end
+end; end; end; end; end

data/lib/logstash/config/config_ast.rb

@@ -1,5 +1,6 @@
 # encoding: utf-8
 require 'logstash/errors'
+require "logstash/compiler/lscl/helpers"
 require "treetop"
 
 require "logstash/compiler/treetop_monkeypatches"
@@ -32,6 +33,8 @@ module LogStash; module Config; module AST
   end
 
   class Node < Treetop::Runtime::SyntaxNode
+    include LogStashCompilerLSCLGrammar::LogStash::Compiler::LSCL::AST::Helpers
+
     def text_value_for_comments
       text_value.gsub(/[\r\n]/, " ")
     end
@@ -189,12 +192,12 @@ module LogStash; module Config; module AST
       # If any parent is a Plugin, this must be a codec.
 
       if attributes.elements.nil?
-        return "plugin(#{plugin_type.inspect}, #{plugin_name.inspect})" << (plugin_type == "codec" ? "" : "\n")
+        return "plugin(#{plugin_type.inspect}, #{plugin_name.inspect}, #{source_meta.line}, #{source_meta.column})" << (plugin_type == "codec" ? "" : "\n")
       else
         settings = attributes.recursive_select(Attribute).collect(&:compile).reject(&:empty?)
 
         attributes_code = "LogStash::Util.hash_merge_many(#{settings.map { |c| "{ #{c} }" }.join(", ")})"
-        return "plugin(#{plugin_type.inspect}, #{plugin_name.inspect}, #{attributes_code})" << (plugin_type == "codec" ? "" : "\n")
+        return "plugin(#{plugin_type.inspect}, #{plugin_name.inspect}, #{source_meta.line}, #{source_meta.column}, #{attributes_code})" << (plugin_type == "codec" ? "" : "\n")
       end
     end
 
@@ -211,7 +214,7 @@ module LogStash; module Config; module AST
       when "codec"
         settings = attributes.recursive_select(Attribute).collect(&:compile).reject(&:empty?)
         attributes_code = "LogStash::Util.hash_merge_many(#{settings.map { |c| "{ #{c} }" }.join(", ")})"
-        return "plugin(#{plugin_type.inspect}, #{plugin_name.inspect}, #{attributes_code})"
+        return "plugin(#{plugin_type.inspect}, #{plugin_name.inspect}, #{source_meta.line}, #{source_meta.column}, #{attributes_code})"
       end
     end
 

data/lib/logstash/config/modules_common.rb

@@ -64,9 +64,12 @@ module LogStash module Config
           alt_name = "module-#{module_name}"
           pipeline_id = alt_name
           module_settings.set("pipeline.id", pipeline_id)
+          LogStash::Modules::SettingsMerger.merge_cloud_settings(module_hash, module_settings)
           current_module.with_settings(module_hash)
           config_test = settings.get("config.test_and_exit")
-          if !config_test
+          modul_setup = settings.get("modules_setup")
+          # Only import data if it's not a config test and --setup is true
+          if !config_test && modul_setup
             esclient = LogStash::ElasticsearchClient.build(module_hash)
             kbnclient = LogStash::Modules::KibanaClient.new(module_hash)
             esconnected = esclient.can_connect?

data/lib/logstash/elasticsearch_client.rb

@@ -39,8 +39,11 @@ module LogStash class ElasticsearchClient
       @client_args[:ssl] = ssl_options
 
       username = @settings["var.elasticsearch.username"]
-      password = @settings["var.elasticsearch.password"]
       if username
+        password = @settings["var.elasticsearch.password"]
+        if password.is_a?(LogStash::Util::Password)
+          password = password.value
+        end
         @client_args[:transport_options] = { :headers => { "Authorization" => 'Basic ' + Base64.encode64( "#{username}:#{password}" ).chomp } }
       end
 

data/lib/logstash/environment.rb

@@ -3,6 +3,9 @@ require "logstash/errors"
 require "logstash/java_integration"
 require "logstash/config/cpu_core_strategy"
 require "logstash/settings"
+require "logstash/util/cloud_setting_id"
+require "logstash/util/cloud_setting_auth"
+require "logstash/util/modules_setting_array"
 require "socket"
 require "stud/temporary"
 
@@ -20,8 +23,11 @@ module LogStash
     Setting::NullableString.new("path.config", nil, false),
  Setting::WritableDirectory.new("path.data", ::File.join(LogStash::Environment::LOGSTASH_HOME, "data")),
     Setting::NullableString.new("config.string", nil, false),
-                    Setting.new("modules.cli", Array, []),
-                    Setting.new("modules", Array, []),
+           Setting::Modules.new("modules.cli", LogStash::Util::ModulesSettingArray, []),
+           Setting::Modules.new("modules", LogStash::Util::ModulesSettingArray, []),
+           Setting::Modules.new("cloud.id", LogStash::Util::CloudSettingId),
+           Setting::Modules.new("cloud.auth",LogStash::Util::CloudSettingAuth),
+           Setting::Boolean.new("modules_setup", false),
            Setting::Boolean.new("config.test_and_exit", false),
            Setting::Boolean.new("config.reload.automatic", false),
            Setting::TimeValue.new("config.reload.interval", "3s"), # in seconds

data/lib/logstash/filter_delegator.rb

@@ -14,6 +14,8 @@ module LogStash
     ]
     def_delegators :@filter, *DELEGATED_METHODS
 
+    attr_reader :id
+
     def initialize(logger, klass, metric, execution_context, plugin_args)
       @logger = logger
       @klass = klass
@@ -26,6 +28,9 @@ module LogStash
       @filter.execution_context = execution_context
 
       @metric_events = namespaced_metric.namespace(:events)
+      @metric_events_in = @metric_events.counter(:in)
+      @metric_events_out = @metric_events.counter(:out)
+      @metric_events_time = @metric_events.counter(:duration_in_millis)
       namespaced_metric.gauge(:name, config_name)
 
       # Not all the filters will do bufferings
@@ -37,19 +42,19 @@ module LogStash
     end
 
     def multi_filter(events)
-      @metric_events.increment(:in, events.size)
+      @metric_events_in.increment(events.size)
 
-      clock = @metric_events.time(:duration_in_millis)
+      start_time = java.lang.System.current_time_millis
       new_events = @filter.multi_filter(events)
-      clock.stop
+      @metric_events_time.increment(java.lang.System.current_time_millis - start_time)
 
       # There is no guarantee in the context of filter
       # that EVENTS_INT == EVENTS_OUT, see the aggregates and
       # the split filter
       c = new_events.count { |event| !event.cancelled? }
-      @metric_events.increment(:out, c) if c > 0
 
-      return new_events
+      @metric_events_out.increment(c) if c > 0
+      new_events
     end
 
     private
@@ -61,7 +66,7 @@ module LogStash
 
         # Filter plugins that does buffering or spooling of events like the
         # `Logstash-filter-aggregates` can return `NIL` and will flush on the next flush ticks.
-        @metric_events.increment(:out, new_events.size) if new_events && new_events.size > 0
+        @metric_events_out.increment(new_events.size) if new_events && new_events.size > 0
         new_events
       end
     end

data/lib/logstash/instrument/collector.rb

@@ -33,11 +33,7 @@ module LogStash module Instrument
     #
     def push(namespaces_path, key, type, *metric_type_params)
       begin
-        metric = @metric_store.fetch_or_store(namespaces_path, key) do
-          LogStash::Instrument::MetricType.create(type, namespaces_path, key)
-        end
-
-        metric.execute(*metric_type_params)
+        get(namespaces_path, key, type).execute(*metric_type_params)
       rescue MetricStore::NamespacesExpectedError => e
         logger.error("Collector: Cannot record metric", :exception => e)
       rescue NameError => e
@@ -51,6 +47,12 @@ module LogStash module Instrument
       end
     end
 
+    def get(namespaces_path, key, type)
+      @metric_store.fetch_or_store(namespaces_path, key) do
+        LogStash::Instrument::MetricType.create(type, namespaces_path, key)
+      end
+    end
+
     # Snapshot the current Metric Store and return it immediately,
     # This is useful if you want to get access to the current metric store without
     # waiting for a periodic call.

data/lib/logstash/instrument/metric_store.rb

@@ -51,14 +51,12 @@ module LogStash module Instrument
       # BUT. If the value is not present in the `@fast_lookup` the value will be inserted and we assume that we don't
       # have it in the `@metric_store` for structured search so we add it there too.
 
-      # array.hash as the key since it is faster then using the array itself, see #7772
-      fast_lookup_key = (namespaces.dup << key).hash
-      value = @fast_lookup.get(fast_lookup_key)
+      value = @fast_lookup.get(namespaces.dup << key)
       if value.nil?
         value = block_given? ? yield(key) : default_value
-        @fast_lookup.put(fast_lookup_key, value)
+        @fast_lookup.put(namespaces.dup << key, value)
         @structured_lookup_mutex.synchronize do
-          # If we cannot find the value this mean we need to save it in the store.
+            # If we cannot find the value this mean we need to save it in the store.
           fetch_or_store_namespaces(namespaces).fetch_or_store(key, value)
         end
       end
@@ -165,7 +163,7 @@ module LogStash module Instrument
     end    
 
     def has_metric?(*path)
-      @fast_lookup[path.hash]
+      @fast_lookup[path]
     end
 
     # Return all the individuals Metric,
@@ -187,9 +185,8 @@ module LogStash module Instrument
     def prune(path)
       key_paths = key_paths(path).map(&:to_sym)
       @structured_lookup_mutex.synchronize do
-        fetch_or_store_namespaces(key_paths).each do |key, v|
-          @fast_lookup.delete((key_paths.dup << key).hash)
-        end
+        keys_to_delete = @fast_lookup.keys.select {|namespace| (key_paths - namespace[0..-2]).empty? }
+        keys_to_delete.each {|k| @fast_lookup.delete(k) }
         delete_from_map(@store, key_paths)
       end
     end

data/lib/logstash/instrument/namespaced_metric.rb

@@ -43,6 +43,10 @@ module LogStash module Instrument
     def collector
       @metric.collector
     end
+    
+    def counter(key)
+      collector.get(@namespace_name, key, :counter)
+    end
 
     def namespace(name)
       NamespacedMetric.new(metric, namespace_name + Array(name))

data/lib/logstash/instrument/namespaced_null_metric.rb

@@ -44,6 +44,10 @@ module LogStash module Instrument
       @metric.collector
     end
 
+    def counter(_)
+      ::LogStash::Instrument::NullMetric::NullGauge
+    end
+
     def namespace(name)
       NamespacedNullMetric.new(metric, namespace_name + Array(name))
     end

data/lib/logstash/instrument/null_metric.rb

@@ -39,6 +39,10 @@ module LogStash module Instrument
       end
     end
 
+    def counter(_)
+      NullGauge
+    end
+
     def namespace(name)
       raise MetricNoNamespaceProvided if name.nil? || name.empty?
       NamespacedNullMetric.new(self, name)
@@ -49,6 +53,12 @@ module LogStash module Instrument
     end
 
     private
+
+    class NullGauge
+      def self.increment(_)
+      end
+    end
+
     # Null implementation of the internal timer class
     #
     # @see LogStash::Instrument::TimedExecution`

data/lib/logstash/instrument/wrapped_write_client.rb

@@ -10,7 +10,12 @@ module LogStash module Instrument
       @events_metrics = metric.namespace([:stats, :events])
       @pipeline_metrics = metric.namespace([:stats, :pipelines, pipeline_id, :events])
       @plugin_events_metrics = metric.namespace([:stats, :pipelines, pipeline_id, :plugins, plugin_type, plugin.id.to_sym, :events])
-
+      @events_metrics_counter = @events_metrics.counter(:in)
+      @events_metrics_time = @events_metrics.counter(:queue_push_duration_in_millis)
+      @pipeline_metrics_counter = @pipeline_metrics.counter(:in)
+      @pipeline_metrics_time = @pipeline_metrics.counter(:queue_push_duration_in_millis)
+      @plugin_events_metrics_counter = @plugin_events_metrics.counter(:out)
+      @plugin_events_metrics_time = @plugin_events_metrics.counter(:queue_push_duration_in_millis)
       define_initial_metrics_values
     end
 
@@ -19,41 +24,45 @@ module LogStash module Instrument
     end
 
     def push(event)
-      record_metric { @write_client.push(event) }
+      increment_counters(1)
+      start_time = java.lang.System.current_time_millis
+      result = @write_client.push(event)
+      report_execution_time(start_time)
+      result
     end
+
     alias_method(:<<, :push)
 
     def push_batch(batch)
-      record_metric(batch.size) { @write_client.push_batch(batch) }
+      increment_counters(batch.size)
+      start_time = java.lang.System.current_time_millis
+      result = @write_client.push_batch(batch)
+      report_execution_time(start_time)
+      result
     end
 
     private
-    def record_metric(size = 1)
-      @events_metrics.increment(:in, size)
-      @pipeline_metrics.increment(:in, size)
-      @plugin_events_metrics.increment(:out, size)
-
-      clock = @events_metrics.time(:queue_push_duration_in_millis)
 
-      result = yield
-
-      # Reuse the same values for all the endpoints to make sure we don't have skew in times.
-      execution_time = clock.stop
-
-      @pipeline_metrics.report_time(:queue_push_duration_in_millis, execution_time)
-      @plugin_events_metrics.report_time(:queue_push_duration_in_millis, execution_time)
+    def increment_counters(size)
+      @events_metrics_counter.increment(size)
+      @pipeline_metrics_counter.increment(size)
+      @plugin_events_metrics_counter.increment(size)
+    end
 
-      result
+    def report_execution_time(start_time)
+      execution_time = java.lang.System.current_time_millis - start_time
+      @events_metrics_time.increment(execution_time)
+      @pipeline_metrics_time.increment(execution_time)
+      @plugin_events_metrics_time.increment(execution_time)
     end
 
     def define_initial_metrics_values
-      @events_metrics.increment(:in, 0)
-      @pipeline_metrics.increment(:in, 0)
-      @plugin_events_metrics.increment(:out, 0)
-
-      @events_metrics.report_time(:queue_push_duration_in_millis, 0)
-      @pipeline_metrics.report_time(:queue_push_duration_in_millis, 0)
-      @plugin_events_metrics.report_time(:queue_push_duration_in_millis, 0)
+      @events_metrics_counter.increment(0)
+      @pipeline_metrics_counter.increment(0)
+      @plugin_events_metrics_counter.increment(0)
+      @events_metrics_time.increment(0)
+      @pipeline_metrics_time.increment(0)
+      @plugin_events_metrics_time.increment(0)
     end
   end
 end end