logstash-core 5.0.0.alpha3.snapshot2-java → 5.0.0.alpha3.snapshot4-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 7405f32a6edf7da11bd24e6ea5c6258b4ce16a51
-  data.tar.gz: ac9e10aabeb05e5cb5c23146c5505bf28f3c0fe4
+  metadata.gz: e159407a5799becbef3c1eed0aa42ac18038ce20
+  data.tar.gz: 0308d3dc734c7615f12e194b49b4b86d6d69c2d5
 SHA512:
-  metadata.gz: 2253d47a0fad5651c01384756bc471477e316d034a39678e0c670122558d4cc18bae4e8341ca7dc6f9db28de5af1fd1a88c682d7f1d841238689b668d023af8b
-  data.tar.gz: dabcdc41ee124fae99062952d9485862239d9b5d86301149c2b9414b0bedf2bca65d7396e396e6ec3563f502c60d45703d783a123c0d06c9cbe45834f7d315d4
+  metadata.gz: 172062b5b43a2738e5350605b036dddcbe90e07741a53b385f48f85225bc59250ec5f6e324fb63600f7a21c3a99ba955e85160027aa5702ac566c4aff28006e3
+  data.tar.gz: 573884287eeda4cb355655cfad1398a8f02238e7c6860a6f66e53f5ac5f5150cb33601a6ffb716845b7c1e4b3e69cb078330fa83aa76e7897c46014599919b66

data/lib/logstash-core/version.rb

@@ -5,4 +5,4 @@
 # Note to authors: this should not include dashes because 'gem' barfs if
 # you include a dash in the version string.
 
-LOGSTASH_CORE_VERSION = "5.0.0-alpha3.snapshot2"
+LOGSTASH_CORE_VERSION = "5.0.0-alpha3.snapshot4"

data/lib/logstash/agent.rb

@@ -20,7 +20,8 @@ LogStash::Environment.load_locale!
 class LogStash::Agent
   STARTED_AT = Time.now.freeze
 
-  attr_reader :metric, :node_name, :pipelines, :logger
+  attr_reader :metric, :node_name, :pipelines, :settings
+  attr_accessor :logger
 
   # initialize method for LogStash::Agent
   # @param params [Hash] potential parameters are:
@@ -28,21 +29,23 @@ class LogStash::Agent
   #   :auto_reload [Boolean] - enable reloading of pipelines
   #   :reload_interval [Integer] - reload pipelines every X seconds
   #   :logger [Cabin::Channel] - logger instance
-  def initialize(params)
-    @logger = params[:logger]
-    @auto_reload = params[:auto_reload]
+  def initialize(settings = LogStash::SETTINGS)
+    @settings = settings
+    @logger = Cabin::Channel.get(LogStash)
+    @auto_reload = setting("config.reload.automatic")
 
     @pipelines = {}
-    @node_name = params[:node_name] || Socket.gethostname
-    @web_api_http_host = params[:web_api_http_host]
-    @web_api_http_port = params[:web_api_http_port]
+    @node_name = setting("node.name")
+    @http_host = setting("http.host")
+    @http_port = setting("http.port")
 
-    @config_loader = LogStash::Config::Loader.new(@logger, params[:debug_config])
-    @reload_interval = params[:reload_interval] || 3 # seconds
+    @config_loader = LogStash::Config::Loader.new(@logger)
+    @reload_interval = setting("config.reload.interval")
     @upgrade_mutex = Mutex.new
 
-    @collect_metric = params.fetch(:collect_metric, false)
-    setup_metric_collection
+    @collect_metric = setting("metric.collect")
+    @metric = create_metric_collector
+    @periodic_pollers = LogStash::Instrument::PeriodicPollers.new(metric)
   end
 
   def execute
@@ -74,9 +77,13 @@ class LogStash::Agent
   # @param pipeline_id [String] pipeline string identifier
   # @param settings [Hash] settings that will be passed when creating the pipeline.
   #   keys should be symbols such as :pipeline_workers and :pipeline_batch_delay
-  def register_pipeline(pipeline_id, settings)
-    pipeline = create_pipeline(settings.merge(:pipeline_id => pipeline_id, :metric => metric))
+  def register_pipeline(pipeline_id, settings = @settings)
+    pipeline_settings = settings.clone
+    pipeline_settings.set("pipeline.id", pipeline_id)
+
+    pipeline = create_pipeline(pipeline_settings)
     return unless pipeline.is_a?(LogStash::Pipeline)
+    pipeline.metric = @metric
     if @auto_reload && pipeline.non_reloadable_plugins.any?
       @logger.error(I18n.t("logstash.agent.non_reloadable_config_register"),
                     :pipeline_id => pipeline_id,
@@ -123,7 +130,7 @@ class LogStash::Agent
 
   private
   def start_webserver
-    options = {:http_host => @web_api_http_host, :http_port => @web_api_http_port }
+    options = {:http_host => @http_host, :http_port => @http_port }
     @webserver = LogStash::WebServer.new(@logger, options)
     Thread.new(@webserver) do |webserver|
       LogStash::Util.set_thread_name("Api Webserver")
@@ -149,28 +156,29 @@ class LogStash::Agent
     end
   end
 
-  def setup_metric_collection
+  def create_metric_collector
     if collect_metrics?
       @logger.debug("Agent: Configuring metric collection")
       LogStash::Instrument::Collector.instance.agent = self
-      @metric = LogStash::Instrument::Metric.new
+      LogStash::Instrument::Metric.new
     else
-      @metric = LogStash::Instrument::NullMetric.new
+      LogStash::Instrument::NullMetric.new
     end
-
-    @periodic_pollers = LogStash::Instrument::PeriodicPollers.new(metric)
   end
 
   def collect_metrics?
     @collect_metric
   end
 
-  def create_pipeline(settings)
-    begin
-      config = fetch_config(settings)
-    rescue => e
-      @logger.error("failed to fetch pipeline configuration", :message => e.message)
-      return
+  def create_pipeline(settings, config=nil)
+
+    if config.nil?
+      begin
+        config = fetch_config(settings)
+      rescue => e
+        @logger.error("failed to fetch pipeline configuration", :message => e.message)
+        return
+      end
     end
 
     begin
@@ -182,23 +190,29 @@ class LogStash::Agent
   end
 
   def fetch_config(settings)
-    @config_loader.format_config(settings[:config_path], settings[:config_string])
+    @config_loader.format_config(settings.get("path.config"), settings.get("config.string"))
   end
 
   # since this method modifies the @pipelines hash it is
   # wrapped in @upgrade_mutex in the parent call `reload_state!`
   def reload_pipeline!(id)
     old_pipeline = @pipelines[id]
-    new_pipeline = create_pipeline(old_pipeline.original_settings)
+    new_config = fetch_config(old_pipeline.settings)
+    if old_pipeline.config_str == new_config
+      @logger.debug("no configuration change for pipeline",
+                    :pipeline => id, :config => new_config)
+      return
+    end
+
+    new_pipeline = create_pipeline(old_pipeline.settings, new_config)
+
     return if new_pipeline.nil?
 
-    if old_pipeline.config_str == new_pipeline.config_str
-      @logger.debug("no configuration change for pipeline",
-                    :pipeline => id, :config => old_pipeline.config_str)
-    elsif new_pipeline.non_reloadable_plugins.any?
+    if new_pipeline.non_reloadable_plugins.any?
       @logger.error(I18n.t("logstash.agent.non_reloadable_config_reload"),
                     :pipeline_id => id,
                     :plugins => new_pipeline.non_reloadable_plugins.map(&:class))
+      return
     else
       @logger.warn("fetched new config for pipeline. upgrading..",
                    :pipeline => id, :config => new_pipeline.config_str)
@@ -262,4 +276,8 @@ class LogStash::Agent
   def reset_collector
     LogStash::Instrument::Collector.instance.clear
   end
+
+  def setting(key)
+    @settings.get(key)
+  end
 end # class LogStash::Agent

data/lib/logstash/api/init.ru

@@ -1,6 +1,6 @@
-ROOT = File.expand_path(File.dirname(__FILE__))
-$LOAD_PATH.unshift File.join(ROOT, 'lib')
-Dir.glob('lib/**').each{ |d| $LOAD_PATH.unshift(File.join(ROOT, d)) }
+api_root = File.expand_path(File.dirname(__FILE__))
+$LOAD_PATH.unshift File.join(api_root, 'lib')
+Dir.glob('lib/**').each{ |d| $LOAD_PATH.unshift(File.join(api_root, d)) }
 
 require 'sinatra'
 require 'app/root'

data/lib/logstash/api/lib/app/service.rb

@@ -28,7 +28,7 @@ class LogStash::Api::Service
   end
 
   def update(snapshot)
-    logger.debug("[api-service] snapshot received", :snapshot => snapshot) if logger.debug?
+    logger.debug("[api-service] snapshot received", :snapshot_time => snapshot.created_at) if logger.debug?
     if @snapshot_rotation_mutex.try_lock
       @snapshot = snapshot
       @snapshot_rotation_mutex.unlock

data/lib/logstash/config/config_ast.rb

@@ -94,6 +94,7 @@ module LogStash; module Config; module AST
         @outputs = []
         @periodic_flushers = []
         @shutdown_flushers = []
+        @generated_objects = {}
       CODE
 
       sections = recursive_select(LogStash::Config::AST::PluginSection)
@@ -137,7 +138,10 @@ module LogStash; module Config; module AST
   class PluginSection < Node
     # Global plugin numbering for the janky instance variable naming we use
     # like @filter_<name>_1
-    @@i = 0
+    def initialize(*args)
+      super(*args)
+      @i = 0
+    end
 
     # Generate ruby code to initialize all the plugins.
     def compile_initializer
@@ -147,31 +151,31 @@ module LogStash; module Config; module AST
 
 
         code << <<-CODE
-          #{name} = #{plugin.compile_initializer}
-          @#{plugin.plugin_type}s << #{name}
+          @generated_objects[:#{name}] = #{plugin.compile_initializer}
+          @#{plugin.plugin_type}s << @generated_objects[:#{name}]
         CODE
 
         # The flush method for this filter.
         if plugin.plugin_type == "filter"
 
           code << <<-CODE
-            #{name}_flush = lambda do |options, &block|
-              @logger.debug? && @logger.debug(\"Flushing\", :plugin => #{name})
+            @generated_objects[:#{name}_flush] = lambda do |options, &block|
+              @logger.debug? && @logger.debug(\"Flushing\", :plugin => @generated_objects[:#{name}])
 
-              events = #{name}.flush(options)
+              events = @generated_objects[:#{name}].flush(options)
 
               return if events.nil? || events.empty?
 
-              @logger.debug? && @logger.debug(\"Flushing\", :plugin => #{name}, :events => events)
+              @logger.debug? && @logger.debug(\"Flushing\", :plugin => @generated_objects[:#{name}], :events => events)
 
               #{plugin.compile_starting_here.gsub(/^/, "  ")}
 
               events.each{|e| block.call(e)}
             end
 
-            if #{name}.respond_to?(:flush)
-              @periodic_flushers << #{name}_flush if #{name}.periodic_flush
-              @shutdown_flushers << #{name}_flush
+            if @generated_objects[:#{name}].respond_to?(:flush)
+              @periodic_flushers << @generated_objects[:#{name}_flush] if @generated_objects[:#{name}].periodic_flush
+              @shutdown_flushers << @generated_objects[:#{name}_flush]
             end
           CODE
 
@@ -192,9 +196,10 @@ module LogStash; module Config; module AST
 
       plugins.each do |plugin|
         # Unique number for every plugin.
-        @@i += 1
+        @i += 1
         # store things as ivars, like @filter_grok_3
-        var = "@#{plugin.plugin_type}_#{plugin.plugin_name}_#{@@i}"
+        var = :"#{plugin.plugin_type}_#{plugin.plugin_name}_#{@i}"
+        # puts("var=#{var.inspect}")
         @variables[plugin] = var
       end
       return @variables
@@ -236,13 +241,13 @@ module LogStash; module Config; module AST
     def compile
       case plugin_type
       when "input"
-        return "start_input(#{variable_name})"
+        return "start_input(@generated_objects[:#{variable_name}])"
       when "filter"
         return <<-CODE
-          events = #{variable_name}.multi_filter(events)
+          events = @generated_objects[:#{variable_name}].multi_filter(events)
         CODE
       when "output"
-        return "targeted_outputs << #{variable_name}\n"
+        return "targeted_outputs << @generated_objects[:#{variable_name}]\n"
       when "codec"
         settings = attributes.recursive_select(Attribute).collect(&:compile).reject(&:empty?)
         attributes_code = "LogStash::Util.hash_merge_many(#{settings.map { |c| "{ #{c} }" }.join(", ")})"
@@ -391,7 +396,7 @@ module LogStash; module Config; module AST
       if type == "filter"
         i = LogStash::Config::AST.defered_conditionals_index += 1
         source = <<-CODE
-          define_singleton_method :cond_func_#{i} do |input_events|
+          @generated_objects[:cond_func_#{i}] = lambda do |input_events|
             result = []
             input_events.each do |event|
               events = [event]
@@ -405,7 +410,7 @@ module LogStash; module Config; module AST
         LogStash::Config::AST.defered_conditionals << source
 
         <<-CODE
-          events = cond_func_#{i}(events)
+          events = @generated_objects[:cond_func_#{i}].call(events)
         CODE
       else # Output
         <<-CODE
@@ -512,7 +517,7 @@ module LogStash; module Config; module AST
   end
   class Selector < RValue
     def compile
-      return "event[#{text_value.inspect}]"
+      return "event.get(#{text_value.inspect})"
     end
   end
   class SelectorElement < Node; end

data/lib/logstash/config/loader.rb

@@ -1,9 +1,9 @@
 require "logstash/config/defaults"
 
 module LogStash; module Config; class Loader
-  def initialize(logger, debug_config=false)
+  def initialize(logger)
     @logger = logger
-    @debug_config = debug_config
+    @config_debug = LogStash::SETTINGS.get_value("config.debug")
   end
 
   def format_config(config_path, config_string)
@@ -70,7 +70,7 @@ module LogStash; module Config; class Loader
         encoding_issue_files << file
       end
       config << cfg + "\n"
-      if @debug_config
+      if @config_debug
         @logger.debug? && @logger.debug("\nThe following is the content of a file", :config_file => file.to_s)
         @logger.debug? && @logger.debug("\n" + cfg + "\n\n")
       end
@@ -78,7 +78,7 @@ module LogStash; module Config; class Loader
     if encoding_issue_files.any?
       fail("The following config files contains non-ascii characters but are not UTF-8 encoded #{encoding_issue_files}")
     end
-    if @debug_config
+    if @config_debug
       @logger.debug? && @logger.debug("\nThe following is the merged configuration")
       @logger.debug? && @logger.debug("\n" + config + "\n\n")
     end

data/lib/logstash/config/mixin.rb

@@ -39,7 +39,6 @@ module LogStash::Config::Mixin
   PLUGIN_VERSION_1_0_0 = LogStash::Util::PluginVersion.new(1, 0, 0)
   PLUGIN_VERSION_0_9_0 = LogStash::Util::PluginVersion.new(0, 9, 0)
 
-  ALLOW_ENV_FLAG = "__ALLOW_ENV__"
   ENV_PLACEHOLDER_REGEX = /\$\{(?<name>\w+)(\:(?<default>[^}]*))?\}/
 
   # This method is called when someone does 'include LogStash::Config'
@@ -49,14 +48,6 @@ module LogStash::Config::Mixin
   end
 
   def config_init(params)
-    # HACK(talevy): https://github.com/elastic/logstash/issues/4958
-    # Currently, the regular plugins params argument is hijacked
-    # to pass along the `allow_env` configuration variable. This was done as to 
-    # not change the method signature of Plugin. This also makes it difficul to 
-    # reason about at the same time. ALLOW_ENV_FLAG is a special param that users 
-    # are now forbidden to set in their configuration definitions.
-    allow_env = params.delete(LogStash::Config::Mixin::ALLOW_ENV_FLAG) { false }
-
     # Validation will modify the values inside params if necessary.
     # For example: converting a string to a number, etc.
     
@@ -112,20 +103,18 @@ module LogStash::Config::Mixin
     end
 
     # Resolve environment variables references
-    if allow_env
-        params.each do |name, value|
-        if (value.is_a?(Hash))
-          value.each do |valueHashKey, valueHashValue|
-            value[valueHashKey.to_s] = replace_env_placeholders(valueHashValue)
+    params.each do |name, value|
+      if (value.is_a?(Hash))
+        value.each do |valueHashKey, valueHashValue|
+          value[valueHashKey.to_s] = replace_env_placeholders(valueHashValue)
+        end
+      else
+        if (value.is_a?(Array))
+          value.each_index do |valueArrayIndex|
+            value[valueArrayIndex] = replace_env_placeholders(value[valueArrayIndex])
           end
         else
-          if (value.is_a?(Array))
-            value.each_index do |valueArrayIndex|
-              value[valueArrayIndex] = replace_env_placeholders(value[valueArrayIndex])
-            end
-          else
-            params[name.to_s] = replace_env_placeholders(value)
-          end
+          params[name.to_s] = replace_env_placeholders(value)
         end
       end
     end

data/lib/logstash/environment.rb

@@ -1,7 +1,37 @@
 # encoding: utf-8
 require "logstash/errors"
+require "logstash/config/cpu_core_strategy"
+require "logstash/settings"
 
 module LogStash
+
+  [
+            Setting::String.new("node.name", Socket.gethostname),
+            Setting::String.new("path.config", nil, false),
+            Setting::String.new("config.string", nil, false),
+           Setting::Boolean.new("config.test_and_exit", false),
+           Setting::Boolean.new("config.reload.automatic", false),
+           Setting::Numeric.new("config.reload.interval", 3),
+           Setting::Boolean.new("metric.collect", true) {|v| v == true }, # metric collection cannot be disabled
+            Setting::String.new("path.settings", ::File.join(Environment::LOGSTASH_HOME, "config")),
+            Setting::String.new("pipeline.id", "main"),
+           Setting::Numeric.new("pipeline.workers", LogStash::Config::CpuCoreStrategy.maximum),
+           Setting::Numeric.new("pipeline.output.workers", 1),
+           Setting::Numeric.new("pipeline.batch.size", 125),
+           Setting::Numeric.new("pipeline.batch.delay", 5), # in milliseconds
+           Setting::Boolean.new("pipeline.unsafe_shutdown", false),
+                    Setting.new("path.plugins", Array, []),
+            Setting::String.new("interactive", nil, false),
+           Setting::Boolean.new("config.debug", false),
+            Setting::String.new("log.level", "warn", true, ["quiet", "verbose", "warn", "debug"]),
+           Setting::Boolean.new("version", false),
+           Setting::Boolean.new("help", false),
+            Setting::String.new("path.log", nil, false),
+            Setting::String.new("log.format", "plain", true, ["json", "plain"]),
+            Setting::String.new("http.host", "127.0.0.1"),
+              Setting::Port.new("http.port", 9600),
+  ].each {|setting| SETTINGS.register(setting) }
+
   module Environment
     extend self
 

data/lib/logstash/filters/base.rb

@@ -187,12 +187,12 @@ class LogStash::Filters::Base < LogStash::Plugin
     # this is important because a construct like event["tags"].delete(tag) will not work
     # in the current Java event implementation. see https://github.com/elastic/logstash/issues/4140
     @remove_tag.each do |tag|
-      tags = event["tags"]
+      tags = event.get("tags")
       break if tags.nil? || tags.empty?
       tag = event.sprintf(tag)
       @logger.debug? and @logger.debug("filters/#{self.class.name}: removing tag", :tag => tag)
       tags.delete(tag)
-      event["tags"] = tags
+      event.set("tags", tags)
     end
   end # def filter_matched
 

data/lib/logstash/inputs/base.rb

@@ -83,7 +83,7 @@ class LogStash::Inputs::Base < LogStash::Plugin
 
   public
   def do_stop
-    @logger.debug("stopping", :plugin => self)
+    @logger.debug("stopping", :plugin => self.class.name)
     @stop_called.make_true
     stop
   end
@@ -97,7 +97,7 @@ class LogStash::Inputs::Base < LogStash::Plugin
   protected
   def decorate(event)
     # Only set 'type' if not already set. This is backwards-compatible behavior
-    event["type"] = @type if @type && !event.include?("type")
+    event.set("type", @type) if @type && !event.include?("type")
 
     LogStash::Util::Decorators.add_fields(@add_field,event,"inputs/#{self.class.name}")
     LogStash::Util::Decorators.add_tags(@tags,event,"inputs/#{self.class.name}")