logstash-core 2.2.4.snapshot2-java → 2.3.0-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 52aed4fccb5c307e3a36a7303b85100121709b49
-  data.tar.gz: db411f7c6536b2838a4acde76fca939c27be2487
+  metadata.gz: 62ad94033e414179752f4aa8997e1453d707292c
+  data.tar.gz: 4d3dbe8a9096eed9fc2cf873b4115eb00d26adc3
 SHA512:
-  metadata.gz: 8bee266423a94552eafd7e277b764c1eed1e8ab237f7f3506fc919f5c885c467025cc515b7529f4546b8152dadeecaa64341a36a0484057b9d9c594c9b183e4d
-  data.tar.gz: f6dbf6b9511bd8c55632c61c617ca5b12fa9036caf1594f039bc361ae2c626189a78a7f973401df91c94387b915b8cbeef74a2c0167d104435003ea5f42b71fc
+  metadata.gz: 27671a2f16ee352e488edad4842c8ef8f563f7ccd1fbff9363aaac89c255960865536eacb3d825c74a60e820e9412406467e6270e3eae8444370fca983ab9f6a
+  data.tar.gz: 560cdb36d4971df6eb6a3f1b1359ac0599ce89887f68834762fb6ecd5e49e2f5e8e197236af2f1e1cb5b94c279aab1bf477c369572b5f90970e0570d7048c003

data/lib/logstash-core/version.rb

@@ -5,4 +5,4 @@
 # Note to authors: this should not include dashes because 'gem' barfs if
 # you include a dash in the version string.
 
-LOGSTASH_CORE_VERSION = "2.2.4.snapshot2"
+LOGSTASH_CORE_VERSION = "2.3.0"

data/lib/logstash/agent.rb

@@ -3,13 +3,16 @@ require "clamp" # gem 'clamp'
 require "logstash/environment"
 require "logstash/errors"
 require "logstash/config/cpu_core_strategy"
+require "stud/trap"
+require "logstash/config/loader"
 require "uri"
 require "net/http"
 require "logstash/pipeline"
-LogStash::Environment.load_locale!
 
 class LogStash::Agent < Clamp::Command
 
+  attr_reader :pipelines
+
   DEFAULT_INPUT = "input { stdin { type => stdin } }"
   DEFAULT_OUTPUT = "output { stdout { codec => rubydebug } }"
 
@@ -27,6 +30,7 @@ class LogStash::Agent < Clamp::Command
          :attribute_name => :pipeline_workers,
          :default => LogStash::Pipeline::DEFAULT_SETTINGS[:default_pipeline_workers]
 
+
   option ["-b", "--pipeline-batch-size"], "SIZE",
          I18n.t("logstash.agent.flag.pipeline-batch-size"),
          :attribute_name => :pipeline_batch_size,
@@ -54,10 +58,6 @@ class LogStash::Agent < Clamp::Command
   option "--verbose", :flag, I18n.t("logstash.agent.flag.verbose")
   option "--debug", :flag, I18n.t("logstash.agent.flag.debug")
 
-  option "--debug-config", :flag,
-    I18n.t("logstash.runner.flag.debug_config"),
-    :attribute_name => :debug_config, :default => false
-
   option ["-V", "--version"], :flag,
     I18n.t("logstash.agent.flag.version")
 
@@ -75,9 +75,21 @@ class LogStash::Agent < Clamp::Command
     :attribute_name => :unsafe_shutdown,
     :default => false
 
-  def initialize(*args)
-    super(*args)
-    @pipeline_settings ||= { :pipeline_id => "base" }
+  option ["-r", "--[no-]auto-reload"], :flag,
+    I18n.t("logstash.agent.flag.auto_reload"),
+    :attribute_name => :auto_reload, :default => false
+
+  option ["--reload-interval"], "RELOAD_INTERVAL",
+    I18n.t("logstash.agent.flag.reload_interval"),
+    :attribute_name => :reload_interval, :default => 3, &:to_i
+
+  def initialize(*params)
+    super(*params)
+    @logger = Cabin::Channel.get(LogStash)
+    @pipelines = {}
+    @pipeline_settings ||= { :pipeline_id => "main" }
+    @upgrade_mutex = Mutex.new
+    @config_loader = LogStash::Config::Loader.new(@logger)
   end
 
   def pipeline_workers=(pipeline_workers_value)
@@ -107,7 +119,6 @@ class LogStash::Agent < Clamp::Command
     raise LogStash::ConfigurationError, message
   end # def warn
 
-  # Emit a failure message and abort.
   def fail(message)
     raise LogStash::ConfigurationError, message
   end # def fail
@@ -118,7 +129,6 @@ class LogStash::Agent < Clamp::Command
     require "logstash/pipeline"
     require "cabin" # gem 'cabin'
     require "logstash/plugin"
-    @logger = Cabin::Channel.get(LogStash)
 
     LogStash::ShutdownWatcher.unsafe_shutdown = unsafe_shutdown?
     LogStash::ShutdownWatcher.logger = @logger
@@ -144,70 +154,63 @@ class LogStash::Agent < Clamp::Command
     end
 
     # You must specify a config_string or config_path
-    if @config_string.nil? && @config_path.nil?
-      fail(help + "\n" + I18n.t("logstash.agent.missing-configuration"))
+    if config_string.nil? && config_path.nil?
+      fail(I18n.t("logstash.agent.missing-configuration"))
     end
 
-    @config_string = @config_string.to_s
+    if auto_reload? && config_path.nil?
+      # there's nothing to reload
+      fail(I18n.t("logstash.agent.reload-without-config-path"))
+    end
 
-    if @config_path
-      # Append the config string.
-      # This allows users to provide both -f and -e flags. The combination
-      # is rare, but useful for debugging.
-      @config_string = @config_string + load_config(@config_path)
-    else
-      # include a default stdin input if no inputs given
-      if @config_string !~ /input *{/
-        @config_string += DEFAULT_INPUT
-      end
-      # include a default stdout output if no outputs given
-      if @config_string !~ /output *{/
-        @config_string += DEFAULT_OUTPUT
+    if config_test?
+      config_loader = LogStash::Config::Loader.new(@logger)
+      config_str = config_loader.format_config(config_path, config_string)
+      config_error = LogStash::Pipeline.config_valid?(config_str)
+      if config_error == true
+        @logger.terminal "Configuration OK"
+        return 0
+      else
+        @logger.fatal I18n.t("logstash.error", :error => config_error)
+        return 1
       end
     end
 
+    register_pipeline("main", @pipeline_settings.merge({
+                          :config_string => config_string,
+                          :config_path => config_path
+                          }))
 
-    begin
-      pipeline = LogStash::Pipeline.new(@config_string, @pipeline_settings)
-    rescue LoadError => e
-      fail("Configuration problem.")
-    end
+    sigint_id = trap_sigint()
+    sigterm_id = trap_sigterm()
+    sighup_id = trap_sighup()
 
-    # Make SIGINT shutdown the pipeline.
-    sigint_id = Stud::trap("INT") do
+    @logger.unsubscribe(stdout_logs) if show_startup_errors
 
-      if @interrupted_once
-        @logger.fatal(I18n.t("logstash.agent.forced_sigint"))
-        exit
-      else
-        @logger.warn(I18n.t("logstash.agent.sigint"))
-        Thread.new(@logger) {|logger| sleep 5; logger.warn(I18n.t("logstash.agent.slow_shutdown")) }
-        @interrupted_once = true
-        shutdown(pipeline)
-      end
-    end
+    @logger.info("starting agent")
 
-    # Make SIGTERM shutdown the pipeline.
-    sigterm_id = Stud::trap("TERM") do
-      @logger.warn(I18n.t("logstash.agent.sigterm"))
-      shutdown(pipeline)
-    end
+    start_pipelines
 
-    Stud::trap("HUP") do
-      @logger.info(I18n.t("logstash.agent.sighup"))
-      configure_logging(log_file)
-    end
+    return 1 if clean_state?
 
-    # Stop now if we are only asking for a config test.
-    if config_test?
-      @logger.terminal "Configuration OK"
-      return
+    @thread = Thread.current # this var is implicilty used by Stud.stop?
+
+    Stud.stoppable_sleep(reload_interval) # sleep before looping
+
+    if auto_reload?
+      Stud.interval(reload_interval) { reload_state! }
+    else
+      while !Stud.stop?
+        if clean_state? || running_pipelines?
+          sleep 0.5
+        else
+          break
+        end
+      end
     end
 
-    @logger.unsubscribe(stdout_logs) if show_startup_errors
+    shutdown
 
-    # TODO(sissel): Get pipeline completion status.
-    pipeline.run
     return 0
   rescue LogStash::ConfigurationError => e
     @logger.unsubscribe(stdout_logs) if show_startup_errors
@@ -224,51 +227,14 @@ class LogStash::Agent < Clamp::Command
     @log_fd.close if @log_fd
     Stud::untrap("INT", sigint_id) unless sigint_id.nil?
     Stud::untrap("TERM", sigterm_id) unless sigterm_id.nil?
+    Stud::untrap("HUP", sighup_id) unless sighup_id.nil?
   end # def execute
 
-  def shutdown(pipeline)
-    pipeline.shutdown do
-      ::LogStash::ShutdownWatcher.start(pipeline)
-    end
-  end
-
-  def show_version
-    show_version_logstash
-
-    if [:info, :debug].include?(verbosity?) || debug? || verbose?
-      show_version_ruby
-      show_version_java if LogStash::Environment.jruby?
-      show_gems if [:debug].include?(verbosity?) || debug?
-    end
-  end # def show_version
-
-  def show_version_logstash
-    require "logstash/version"
-    puts "logstash #{LOGSTASH_VERSION}"
-  end # def show_version_logstash
-
-  def show_version_ruby
-    puts RUBY_DESCRIPTION
-  end # def show_version_ruby
-
-  def show_version_java
-    properties = java.lang.System.getProperties
-    puts "java #{properties["java.version"]} (#{properties["java.vendor"]})"
-    puts "jvm #{properties["java.vm.name"]} / #{properties["java.vm.version"]}"
-  end # def show_version_java
-
-  def show_gems
-    require "rubygems"
-    Gem::Specification.each do |spec|
-      puts "gem #{spec.name} #{spec.version}"
-    end
-  end # def show_gems
 
   # Do any start-time configuration.
   #
   # Log file stuff, plugin path checking, etc.
   def configure
-    @pipeline_settings[:debug_config] = debug_config?
     configure_logging(log_file)
     configure_plugin_paths(plugin_paths)
   end # def configure
@@ -329,63 +295,205 @@ class LogStash::Agent < Clamp::Command
     end
   end
 
-  def load_config(path)
-    begin
-      uri = URI.parse(path)
-
-      case uri.scheme
-      when nil then
-        local_config(path)
-      when /http/ then
-        fetch_config(uri)
-      when "file" then
-        local_config(uri.path)
+  ## Signal Trapping ##
+  def trap_sigint
+    Stud::trap("INT") do
+      if @interrupted_once
+        @logger.fatal(I18n.t("logstash.agent.forced_sigint"))
+        exit
       else
-        fail(I18n.t("logstash.agent.configuration.scheme-not-supported", :path => path))
+        @logger.warn(I18n.t("logstash.agent.sigint"))
+        Thread.new(@logger) {|logger| sleep 5; logger.warn(I18n.t("logstash.agent.slow_shutdown")) }
+        @interrupted_once = true
+        Stud.stop!(@thread)
       end
-    rescue URI::InvalidURIError
-      # fallback for windows.
-      # if the parsing of the file failed we assume we can reach it locally.
-      # some relative path on windows arent parsed correctly (.\logstash.conf)
-      local_config(path)
     end
   end
 
-  def local_config(path)
-    path = File.expand_path(path)
-    path = File.join(path, "*") if File.directory?(path)
+  def trap_sigterm
+    Stud::trap("TERM") do
+      @logger.warn(I18n.t("logstash.agent.sigterm"))
+      Stud.stop!(@thread)
+    end
+  end
+
+  def trap_sighup
+    Stud::trap("HUP") do
+      @logger.warn(I18n.t("logstash.agent.sighup"))
+      reload_state!
+    end
+  end
 
-    if Dir.glob(path).length == 0
-      fail(I18n.t("logstash.agent.configuration.file-not-found", :path => path))
+  ## Pipeline CRUD ##
+  def shutdown(pipeline)
+    pipeline.shutdown do
+      ::LogStash::ShutdownWatcher.start(pipeline)
     end
+  end
+  #
+  # register_pipeline - adds a pipeline to the agent's state
+  # @param pipeline_id [String] pipeline string identifier
+  # @param settings [Hash] settings that will be passed when creating the pipeline.
+  #   keys should be symbols such as :pipeline_workers and :pipeline_batch_delay
+  def register_pipeline(pipeline_id, settings)
+    pipeline = create_pipeline(settings.merge(:pipeline_id => pipeline_id))
+    return unless pipeline.is_a?(LogStash::Pipeline)
+    if @auto_reload && pipeline.non_reloadable_plugins.any?
+      @logger.error(I18n.t("logstash.agent.non_reloadable_config_register"),
+                    :pipeline_id => pipeline_id,
+                    :plugins => pipeline.non_reloadable_plugins.map(&:class))
+      return
+    end
+    @pipelines[pipeline_id] = pipeline
+  end
 
-    config = ""
-    encoding_issue_files = []
-    Dir.glob(path).sort.each do |file|
-      next unless File.file?(file)
-      if file.match(/~$/)
-        @logger.debug("NOT reading config file because it is a temp file", :file => file)
-        next
-      end
-      @logger.debug("Reading config file", :file => file)
-      cfg = File.read(file)
-      if !cfg.ascii_only? && !cfg.valid_encoding?
-        encoding_issue_files << file
+  def reload_state!
+    @upgrade_mutex.synchronize do
+      @pipelines.each do |pipeline_id, _|
+        begin
+          reload_pipeline!(pipeline_id)
+        rescue => e
+          @logger.error(I18n.t("oops"), :error => e, :backtrace => e.backtrace)
+        end
       end
-      config << cfg + "\n"
     end
-    if (encoding_issue_files.any?)
-      fail("The following config files contains non-ascii characters but are not UTF-8 encoded #{encoding_issue_files}")
+  end
+
+  def create_pipeline(settings)
+    begin
+      config = fetch_config(settings)
+    rescue => e
+      @logger.error("failed to fetch pipeline configuration", :message => e.message)
+      return
     end
-    return config
-  end # def load_config
 
-  def fetch_config(uri)
     begin
-      Net::HTTP.get(uri) + "\n"
-    rescue Exception => e
-      fail(I18n.t("logstash.agent.configuration.fetch-failed", :path => uri.to_s, :message => e.message))
+      LogStash::Pipeline.new(config, settings)
+    rescue => e
+      @logger.error("fetched an invalid config", :config => config, :reason => e.message)
+      return
+    end
+  end
+
+  def start_pipelines
+    @pipelines.each { |id, _| start_pipeline(id) }
+  end
+
+  def shutdown
+    shutdown_pipelines
+  end
+
+  def shutdown_pipelines
+    @pipelines.each { |id, _| stop_pipeline(id) }
+  end
+
+  def stop_pipeline(id)
+    pipeline = @pipelines[id]
+    return unless pipeline
+    @logger.log("stopping pipeline", :id => id)
+    pipeline.shutdown { LogStash::ShutdownWatcher.start(pipeline) }
+    @pipelines[id].thread.join
+  end
+
+  def running_pipelines?
+    @upgrade_mutex.synchronize do
+      @pipelines.select {|pipeline_id, _| running_pipeline?(pipeline_id) }.any?
     end
   end
 
+  def running_pipeline?(pipeline_id)
+    thread = @pipelines[pipeline_id].thread
+    thread.is_a?(Thread) && thread.alive?
+  end
+
+  def upgrade_pipeline(pipeline_id, new_pipeline)
+    stop_pipeline(pipeline_id)
+    @pipelines[pipeline_id] = new_pipeline
+    start_pipeline(pipeline_id)
+  end
+
+  def clean_state?
+    @pipelines.empty?
+  end
+
+  # since this method modifies the @pipelines hash it is
+  # wrapped in @upgrade_mutex in the parent call `reload_state!`
+  def reload_pipeline!(id)
+    old_pipeline = @pipelines[id]
+    new_pipeline = create_pipeline(old_pipeline.original_settings)
+    return if new_pipeline.nil?
+
+    if old_pipeline.config_str == new_pipeline.config_str
+      @logger.debug("no configuration change for pipeline",
+                    :pipeline => id, :config => old_pipeline.config_str)
+    elsif new_pipeline.non_reloadable_plugins.any?
+      @logger.error(I18n.t("logstash.agent.non_reloadable_config_reload"),
+                    :pipeline_id => id,
+                    :plugins => new_pipeline.non_reloadable_plugins.map(&:class))
+    else
+      @logger.log("fetched new config for pipeline. upgrading..",
+                   :pipeline => id, :config => new_pipeline.config_str)
+      upgrade_pipeline(id, new_pipeline)
+    end
+  end
+
+  def start_pipeline(id)
+    pipeline = @pipelines[id]
+    return unless pipeline.is_a?(LogStash::Pipeline)
+    return if pipeline.ready?
+    @logger.info("starting pipeline", :id => id)
+    Thread.new do
+      LogStash::Util.set_thread_name("pipeline.#{id}")
+      begin
+        pipeline.run
+      rescue => e
+        @logger.error("Pipeline aborted due to error", :exception => e, :backtrace => e.backtrace)
+      end
+    end
+    sleep 0.01 until pipeline.ready?
+  end
+
+  ## Pipeline Aux methods ##
+  def fetch_config(settings)
+    @config_loader.format_config(settings[:config_path], settings[:config_string])
+  end
+
+  private
+  def node_uuid
+    @node_uuid ||= SecureRandom.uuid
+  end
+
+  ### Version actions ###
+  def show_version
+    show_version_logstash
+
+    if [:info, :debug].include?(verbosity?) || debug? || verbose?
+      show_version_ruby
+      show_version_java if LogStash::Environment.jruby?
+      show_gems if [:debug].include?(verbosity?) || debug?
+    end
+  end # def show_version
+
+  def show_version_logstash
+    require "logstash/version"
+    puts "logstash #{LOGSTASH_VERSION}"
+  end # def show_version_logstash
+
+  def show_version_ruby
+    puts RUBY_DESCRIPTION
+  end # def show_version_ruby
+
+  def show_version_java
+    properties = java.lang.System.getProperties
+    puts "java #{properties["java.version"]} (#{properties["java.vendor"]})"
+    puts "jvm #{properties["java.vm.name"]} / #{properties["java.vm.version"]}"
+  end # def show_version_java
+
+  def show_gems
+    require "rubygems"
+    Gem::Specification.each do |spec|
+      puts "gem #{spec.name} #{spec.version}"
+    end
+  end # def show_gems
+
 end # class LogStash::Agent