logstash-core 2.2.4.snapshot2-java → 2.3.0-java

data/lib/logstash/config/defaults.rb

@@ -6,6 +6,14 @@ module LogStash module Config module Defaults
 
   extend self
 
+  def input
+    "input { stdin { type => stdin } }"
+  end
+
+  def output
+    "output { stdout { codec => rubydebug } }"
+  end
+
   def cpu_cores
     Concurrent.processor_count
   end

data/lib/logstash/config/loader.rb

@@ -0,0 +1,90 @@
+require "logstash/config/defaults"
+
+module LogStash; module Config; class Loader
+  def initialize(logger)
+    @logger = logger
+  end
+
+  def format_config(config_path, config_string)
+    config_string = config_string.to_s
+    if config_path
+      # Append the config string.
+      # This allows users to provide both -f and -e flags. The combination
+      # is rare, but useful for debugging.
+      config_string = config_string + load_config(config_path)
+    else
+      # include a default stdin input if no inputs given
+      if config_string !~ /input *{/
+        config_string += LogStash::Config::Defaults.input
+      end
+      # include a default stdout output if no outputs given
+      if config_string !~ /output *{/
+        config_string += LogStash::Config::Defaults.output
+      end
+    end
+    config_string
+  end
+
+  def load_config(path)
+    begin
+      uri = URI.parse(path)
+
+      case uri.scheme
+      when nil then
+        local_config(path)
+      when /http/ then
+        fetch_config(uri)
+      when "file" then
+        local_config(uri.path)
+      else
+        fail(I18n.t("logstash.runner.configuration.scheme-not-supported", :path => path))
+      end
+    rescue URI::InvalidURIError
+      # fallback for windows.
+      # if the parsing of the file failed we assume we can reach it locally.
+      # some relative path on windows arent parsed correctly (.\logstash.conf)
+      local_config(path)
+    end
+  end
+
+  def local_config(path)
+    path = ::File.expand_path(path)
+    path = ::File.join(path, "*") if ::File.directory?(path)
+
+    if Dir.glob(path).length == 0
+      fail(I18n.t("logstash.runner.configuration.file-not-found", :path => path))
+    end
+
+    config = ""
+    encoding_issue_files = []
+    Dir.glob(path).sort.each do |file|
+      next unless ::File.file?(file)
+      if file.match(/~$/)
+        @logger.debug("NOT reading config file because it is a temp file", :config_file => file)
+        next
+      end
+      @logger.debug("Reading config file", :config_file => file)
+      cfg = ::File.read(file)
+      if !cfg.ascii_only? && !cfg.valid_encoding?
+        encoding_issue_files << file
+      end
+      config << cfg + "\n"
+      @logger.debug? && @logger.debug("\nThe following is the content of a file", :config_file => file.to_s)
+      @logger.debug? && @logger.debug("\n" + cfg + "\n\n")
+    end
+    if encoding_issue_files.any?
+      fail("The following config files contains non-ascii characters but are not UTF-8 encoded #{encoding_issue_files}")
+    end
+    @logger.debug? && @logger.debug("\nThe following is the merged configuration")
+    @logger.debug? && @logger.debug("\n" + config + "\n\n")
+    return config
+  end # def load_config
+
+  def fetch_config(uri)
+    begin
+      Net::HTTP.get(uri) + "\n"
+    rescue Exception => e
+      fail(I18n.t("logstash.runner.configuration.fetch-failed", :path => uri.to_s, :message => e.message))
+    end
+  end
+end end end

data/lib/logstash/config/mixin.rb

@@ -38,6 +38,8 @@ module LogStash::Config::Mixin
   PLUGIN_VERSION_1_0_0 = LogStash::Util::PluginVersion.new(1, 0, 0)
   PLUGIN_VERSION_0_9_0 = LogStash::Util::PluginVersion.new(0, 9, 0)
 
+  ENV_PLACEHOLDER_REGEX = /\$(?<name>\w+)|\$\{(?<name>\w+)(\:(?<default>[^}]*))?\}/
+
   # This method is called when someone does 'include LogStash::Config'
   def self.included(base)
     # Add the DSL methods to the 'base' given.
@@ -51,7 +53,7 @@ module LogStash::Config::Mixin
     # Keep a copy of the original config params so that we can later
     # differentiate between explicit configuration and implicit (default)
     # configuration.
-    original_params = params.clone
+    @original_params = params.clone
     
     # store the plugin type, turns LogStash::Inputs::Base into 'input'
     @plugin_type = self.class.ancestors.find { |a| a.name =~ /::Base$/ }.config_name
@@ -99,6 +101,23 @@ module LogStash::Config::Mixin
       end
     end
 
+    # Resolve environment variables references
+    params.each do |name, value|
+      if (value.is_a?(Hash))
+        value.each do |valueHashKey, valueHashValue|
+          value[valueHashKey.to_s] = replace_env_placeholders(valueHashValue)
+        end
+      else
+        if (value.is_a?(Array))
+          value.each_index do |valueArrayIndex|
+            value[valueArrayIndex] = replace_env_placeholders(value[valueArrayIndex])
+          end
+        else
+          params[name.to_s] = replace_env_placeholders(value)
+        end
+      end
+    end
+
     if !self.class.validate(params)
       raise LogStash::ConfigurationError,
         I18n.t("logstash.agent.configuration.invalid_plugin_settings")
@@ -123,14 +142,32 @@ module LogStash::Config::Mixin
       instance_variable_set("@#{key}", value)
     end
 
-    # now that we know the parameters are valid, we can obfuscate the original copy
-    # of the parameters before storing them as an instance variable
-    self.class.secure_params!(original_params)
-    @original_params = original_params
-
     @config = params
   end # def config_init
 
+  # Replace all environment variable references in 'value' param by environment variable value and return updated value
+  # Process following patterns : $VAR, ${VAR}, ${VAR:defaultValue}
+  def replace_env_placeholders(value)
+    return value unless value.is_a?(String)
+    #raise ArgumentError, "Cannot replace ENV placeholders on non-strings. Got #{value.class}" if !value.is_a?(String)
+
+    value.gsub(ENV_PLACEHOLDER_REGEX) do |placeholder|
+      # Note: Ruby docs claim[1] Regexp.last_match is thread-local and scoped to
+      # the call, so this should be thread-safe.
+      #
+      # [1] http://ruby-doc.org/core-2.1.1/Regexp.html#method-c-last_match
+      name = Regexp.last_match(:name)
+      default = Regexp.last_match(:default)
+
+      replacement = ENV.fetch(name, default)
+      if replacement.nil?
+        raise LogStash::ConfigurationError, "Cannot evaluate `#{placeholder}`. Environment variable `#{name}` is not set and there is no default value given."
+      end
+      @logger.info? && @logger.info("Evaluating environment variable placeholder", :placeholder => placeholder, :replacement => replacement)
+      replacement
+    end
+  end # def replace_env_placeholders
+
   module DSL
     attr_accessor :flags
 
@@ -152,7 +189,7 @@ module LogStash::Config::Mixin
     # inside the gemspec.
     def milestone(m = nil)
       @logger = Cabin::Channel.get(LogStash)
-      @logger.warn(I18n.t('logstash.plugin.deprecated_milestone', :plugin => config_name))
+      @logger.debug(I18n.t('logstash.plugin.deprecated_milestone', :plugin => config_name))
     end
 
     # Define a new configuration setting
@@ -500,14 +537,6 @@ module LogStash::Config::Mixin
       return true, result
     end # def validate_value
 
-    def secure_params!(params)
-      params.each do |key, value|
-        if @config[key][:validate] == :password && !value.is_a?(::LogStash::Util::Password)
-          params[key] = ::LogStash::Util::Password.new(value)
-        end
-      end
-    end
-
     def hash_or_array(value)
       if !value.is_a?(Hash)
         value = [*value] # coerce scalar to array if necessary

data/lib/logstash/output_delegator.rb

@@ -134,7 +134,7 @@ module LogStash class OutputDelegator
   end
 
   def do_close
-    @logger.debug("closing output delegator", :klass => @klass)
+    @logger.debug("closing output delegator", :klass => self)
 
     if @threadsafe
       @workers.each(&:do_close)

data/lib/logstash/pipeline.rb

@@ -17,21 +17,37 @@ require "logstash/pipeline_reporter"
 require "logstash/output_delegator"
 
 module LogStash; class Pipeline
-  attr_reader :inputs, :filters, :outputs, :worker_threads, :events_consumed, :events_filtered, :reporter, :pipeline_id, :logger
+  attr_reader :inputs, :filters, :outputs, :worker_threads, :events_consumed, :events_filtered, :reporter, :pipeline_id, :logger, :thread, :config_str, :original_settings
+
+  DEFAULT_OUTPUT_WORKERS = 1
 
   DEFAULT_SETTINGS = {
     :default_pipeline_workers => LogStash::Config::CpuCoreStrategy.maximum,
     :pipeline_batch_size => 125,
     :pipeline_batch_delay => 5, # in milliseconds
     :flush_interval => 5, # in seconds
-    :flush_timeout_interval => 60, # in seconds
-    :debug_config => false 
+    :flush_timeout_interval => 60 # in seconds
   }
   MAX_INFLIGHT_WARN_THRESHOLD = 10_000
 
+  RELOAD_INCOMPATIBLE_PLUGINS = [
+    "LogStash::Inputs::Stdin"
+  ]
+
+  def self.validate_config(config_str, settings = {})
+    begin
+      # There should be a better way to test this
+      self.new(config_str, settings)
+    rescue => e
+      e.message
+    end
+  end
+
   def initialize(config_str, settings = {})
-    @pipeline_id = settings[:pipeline_id] || self.object_id
+    @config_str = config_str
+    @original_settings = settings
     @logger = Cabin::Channel.get(LogStash)
+    @pipeline_id = settings[:pipeline_id] || self.object_id
     @settings = DEFAULT_SETTINGS.clone
     settings.each {|setting, value| configure(setting, value) }
     @reporter = LogStash::PipelineReporter.new(@logger, self)
@@ -53,9 +69,7 @@ module LogStash; class Pipeline
     code = @config.compile
     # The config code is hard to represent as a log message...
     # So just print it.
-    if @settings[:debug_config]
-      @logger.debug? && @logger.debug("Compiled pipeline code:\n#{code}")
-    end
+    @logger.debug? && @logger.debug("Compiled pipeline code:\n#{code}")
     begin
       eval(code)
     rescue => e
@@ -120,13 +134,12 @@ module LogStash; class Pipeline
   end
 
   def run
-    LogStash::Util.set_thread_name("[#{pipeline_id}]-pipeline-manager")
     @logger.terminal(LogStash::Util::DefaultsPrinter.print(@settings))
+    @thread = Thread.current
 
     start_workers
 
-    @logger.info("Pipeline started")
-    @logger.terminal("Logstash startup completed")
+    @logger.log("Pipeline #{@pipeline_id} started")
 
     # Block until all inputs have stopped
     # Generally this happens if SIGINT is sent and `shutdown` is called from an external thread
@@ -141,8 +154,7 @@ module LogStash; class Pipeline
     shutdown_flusher
     shutdown_workers
 
-    @logger.info("Pipeline shutdown complete.")
-    @logger.terminal("Logstash shutdown completed")
+    @logger.log("Pipeline #{@pipeline_id} has been shutdown")
 
     # exit code
     return 0
@@ -403,16 +415,12 @@ module LogStash; class Pipeline
     klass = LogStash::Plugin.lookup(plugin_type, name)
 
     if plugin_type == "output"
-      LogStash::OutputDelegator.new(@logger, klass, default_output_workers, *args)
+      LogStash::OutputDelegator.new(@logger, klass, DEFAULT_OUTPUT_WORKERS, *args)
     else
       klass.new(*args)
     end
   end
 
-  def default_output_workers
-    @settings[:pipeline_workers] || @settings[:default_pipeline_workers]
-  end
-
   # for backward compatibility in devutils for the rspec helpers, this method is not used
   # in the pipeline anymore.
   def filter(event, &block)
@@ -483,17 +491,10 @@ module LogStash; class Pipeline
       .each {|t| t.delete("status") }
   end
 
-  # Sometimes we log stuff that will dump the pipeline which may contain
-  # sensitive information (like the raw syntax tree which can contain passwords)
-  # We want to hide most of what's in here
-  def inspect
-    {
-      :pipeline_id => @pipeline_id,
-      :settings => @settings.inspect,
-      :ready => @ready,
-      :running => @running,
-      :flushing => @flushing
-    }
+  def non_reloadable_plugins
+    (inputs + filters + outputs).select do |plugin|
+      RELOAD_INCOMPATIBLE_PLUGINS.include?(plugin.class.name)
+    end
   end
 
 end end

data/lib/logstash/runner.rb

@@ -9,11 +9,15 @@ LogStash::Environment.load_locale!
 
 require "logstash/namespace"
 require "logstash/program"
+require "logstash/config/defaults"
 
 class LogStash::Runner
   include LogStash::Program
 
+  attr_reader :agent
+
   def main(args)
+
     require "logstash/util"
     require "logstash/util/java_version"
     require "stud/trap"
@@ -121,4 +125,5 @@ Available commands:
   def show_help(command)
     puts command.help
   end
+
 end # class LogStash::Runner

data/lib/logstash/shutdown_watcher.rb

@@ -34,8 +34,8 @@ module LogStash
     end
 
     def self.start(pipeline, cycle_period=CHECK_EVERY, report_every=REPORT_EVERY, abort_threshold=ABORT_AFTER)
-      watcher = self.new(pipeline, cycle_period, report_every, abort_threshold)
-      Thread.new(watcher) { |watcher| watcher.start }
+      controller = self.new(pipeline, cycle_period, report_every, abort_threshold)
+      Thread.new(controller) { |controller| controller.start }
     end
 
     def logger
@@ -47,6 +47,7 @@ module LogStash
       cycle_number = 0
       stalled_count = 0
       Stud.interval(@cycle_period) do
+        break unless @pipeline.thread.alive?
         @reports << pipeline_report_snapshot
         @reports.delete_at(0) if @reports.size > @report_every # expire old report
         if cycle_number == (@report_every - 1) # it's report time!

data/lib/logstash/special_agent.rb

@@ -0,0 +1,8 @@
+# encoding: utf-8
+require "logstash/agent"
+
+class LogStash::SpecialAgent < LogStash::Agent
+  def fetch_config(settings)
+    Net::HTTP.get(settings["remote.url"])
+  end
+end

data/lib/logstash/version.rb

@@ -11,4 +11,4 @@
 #       eventually this file should be in the root logstash lib fir and dependencies in logstash-core should be
 #       fixed.
 
-LOGSTASH_VERSION = "2.2.4.snapshot2"
+LOGSTASH_VERSION = "2.3.0"

data/locales/en.yml

@@ -59,18 +59,25 @@ en:
       missing-configuration: >-
         No configuration file was specified. Perhaps you forgot to provide
         the '-f yourlogstash.conf' flag?
+      reload-without-config-path: >-
+        Configuration reloading also requires passing a configuration path with '-f yourlogstash.conf'
       error: >-
         Error: %{error}
       sigint: >-
-        SIGINT received. Shutting down the pipeline.
+        SIGINT received. Shutting down the agent.
       sigterm: >-
-        SIGTERM received. Shutting down the pipeline.
+        SIGTERM received. Shutting down the agent.
       slow_shutdown: |-
         Received shutdown signal, but pipeline is still waiting for in-flight events
         to be processed. Sending another ^C will force quit Logstash, but this may cause
         data loss.
       forced_sigint: >-
         SIGINT received. Terminating immediately..
+      non_reloadable_config_reload: >-
+        Unable to reload configuration because it does not support dynamic reloading
+      non_reloadable_config_register: |-
+        Logstash was not able to load configuration since it does not support
+        dynamic reloading and -r or --auto-reload flag was enabled
       configtest-flag-information: |-
         You may be interested in the '--configtest' flag which you can
         use to validate logstash's configuration before you choose
@@ -163,6 +170,13 @@ en:
         pipeline-batch-delay: |+
           When creating pipeline batches, how long to wait while polling
           for the next event.
+        auto_reload: |+
+          Monitor configuration changes and reload
+          whenever it is changed.
+          NOTE: use SIGHUP to manually reload the config
+        reload_interval: |+
+          How frequently to poll the configuration location
+          for changes, in seconds.
         log: |+
           Write logstash internal logs to the given
           file. Without this flag, logstash will emit
@@ -193,10 +207,6 @@ en:
         debug: |+
           Most verbose logging. This causes 'debug'
           level logs to be emitted.
-        debug-config: |+
-          Print the compiled config ruby code out as a debug log (you must also have --debug enabled).
-          WARNING: This will include any 'password' options passed to plugin configs as plaintext, and may result
-          in plaintext passwords appearing in your logs!  
         unsafe_shutdown: |+
           Force logstash to exit during shutdown even
           if there are still inflight events in memory.