logstash-core 5.0.0.alpha3.snapshot2-java → 5.0.0.alpha3.snapshot4-java

data/lib/logstash/instrument/collector.rb

@@ -75,7 +75,7 @@ module LogStash module Instrument
       logger.error("Collector: Something went wrong went sending data to the observers",
                    :execution_time => time_of_execution,
                    :result => result,
-                   :exception => exception)
+                   :exception => exception.class.name)
     end
 
     # Snapshot the current Metric Store and return it immediately,

data/lib/logstash/logging/json.rb

@@ -0,0 +1,21 @@
+# encoding: utf-8
+require "logstash/namespace"
+require "logstash/logging"
+require "logstash/json"
+
+module LogStash; class Logging; class JSON
+  def initialize(io)
+    raise ArgumentError, "Expected IO, got #{io.class.name}" unless io.is_a?(IO)
+
+    @io = io
+    @lock = Mutex.new
+  end
+
+  def <<(obj)
+    serialized = LogStash::Json.dump(obj)
+    @lock.synchronize do
+      @io.puts(serialized)
+      @io.flush
+    end
+  end
+end; end; end

data/lib/logstash/output_delegator.rb

@@ -82,7 +82,7 @@ module LogStash class OutputDelegator
     @workers << @klass.new(@config)
     @workers.first.register # Needed in case register calls `workers_not_supported`
 
-    @logger.debug("Will start workers for output", :worker_count => target_worker_count, :class => @klass)
+    @logger.debug("Will start workers for output", :worker_count => target_worker_count, :class => @klass.name)
 
     # Threadsafe versions don't need additional workers
     setup_additional_workers!(target_worker_count) unless @threadsafe
@@ -147,7 +147,7 @@ module LogStash class OutputDelegator
   end
 
   def do_close
-    @logger.debug("closing output delegator", :klass => @klass)
+    @logger.debug("closing output delegator", :klass => @klass.name)
 
     if @threadsafe
       @workers.each(&:do_close)

data/lib/logstash/patches/clamp.rb

@@ -0,0 +1,69 @@
+require 'clamp'
+require 'logstash/environment'
+
+module Clamp
+  module Attribute
+    class Instance
+      def default_from_environment
+        # we don't want uncontrolled var injection from the environment
+        # since we're establishing that settings can be pulled from only three places:
+        # 1. default settings
+        # 2. yaml file
+        # 3. cli arguments
+      end
+    end
+  end
+
+  module Option
+
+    module StrictDeclaration
+
+      include Clamp::Attribute::Declaration
+
+      # Instead of letting Clamp set up accessors for the options
+      # weŕe going to tightly controlling them through
+      # LogStash::SETTINGS
+      def define_simple_writer_for(option, &block)
+        LogStash::SETTINGS.get(option.attribute_name)
+        define_method(option.write_method) do |value|
+          value = instance_exec(value, &block) if block
+          LogStash::SETTINGS.set_value(option.attribute_name, value)
+        end
+      end
+
+      def define_reader_for(option)
+        define_method(option.read_method) do
+          LogStash::SETTINGS.get_value(option.attribute_name)
+        end
+      end
+
+    end
+
+    class Definition
+      # Allow boolean flags to optionally receive a true/false argument
+      # to explicitly set them, i.e.
+      # --long.flag.name       => sets flag to true
+      # --long.flag.name true  => sets flag to true
+      # --long.flag.name false => sets flag to false
+      # --long.flag.name=true  => sets flag to true
+      # --long.flag.name=false => sets flag to false
+      def extract_value(switch, arguments)
+        if flag? && (arguments.first.nil? || arguments.first.match("^-"))
+          flag_value(switch)
+        else
+          arguments.shift
+        end
+      end
+    end
+  end
+
+  # Create a subclass of Clamp::Command that enforces the use of
+  # LogStash::SETTINGS for setting validation
+  class StrictCommand < Command
+    class << self
+      include ::Clamp::Option::StrictDeclaration
+    end
+  end
+end
+
+

data/lib/logstash/pipeline.rb

@@ -9,8 +9,6 @@ require "logstash/config/file"
 require "logstash/filters/base"
 require "logstash/inputs/base"
 require "logstash/outputs/base"
-require "logstash/config/cpu_core_strategy"
-require "logstash/util/defaults_printer"
 require "logstash/shutdown_watcher"
 require "logstash/util/wrapped_synchronous_queue"
 require "logstash/pipeline_reporter"
@@ -22,7 +20,7 @@ require "logstash/output_delegator"
 require "logstash/filter_delegator"
 
 module LogStash; class Pipeline
- attr_reader :inputs,
+  attr_reader :inputs,
     :filters,
     :outputs,
     :worker_threads,
@@ -30,39 +28,25 @@ module LogStash; class Pipeline
     :events_filtered,
     :reporter,
     :pipeline_id,
-    :metric,
     :logger,
     :started_at,
     :thread,
     :config_str,
-    :original_settings
-
-  DEFAULT_OUTPUT_WORKERS = 1
-
-  DEFAULT_SETTINGS = {
-    :default_pipeline_workers => LogStash::Config::CpuCoreStrategy.maximum,
-    :pipeline_batch_size => 125,
-    :pipeline_batch_delay => 5, # in milliseconds
-    :flush_interval => 5, # in seconds
-    :flush_timeout_interval => 60, # in seconds
-    :debug_config => false,
-    :allow_env => false
-  }
+    :settings
+  attr_accessor :metric
+
   MAX_INFLIGHT_WARN_THRESHOLD = 10_000
 
   RELOAD_INCOMPATIBLE_PLUGINS = [
     "LogStash::Inputs::Stdin"
   ]
 
-  def initialize(config_str, settings = {})
+  def initialize(config_str, settings = LogStash::SETTINGS)
     @config_str = config_str
-    @original_settings = settings
     @logger = Cabin::Channel.get(LogStash)
-    @pipeline_id = settings[:pipeline_id] || self.object_id
-    @settings = DEFAULT_SETTINGS.clone
-    settings.each {|setting, value| configure(setting, value) }
+    @settings = settings
+    @pipeline_id = @settings.get_value("pipeline.id") || self.object_id
     @reporter = LogStash::PipelineReporter.new(@logger, self)
-    @allow_env = settings[:allow_env]
 
     @inputs = nil
     @filters = nil
@@ -76,9 +60,11 @@ module LogStash; class Pipeline
     # We also do this to make the changes backward compatible with previous testing of the
     # pipeline.
     #
-    # This need to be configured before we evaluate the code to make
+    # This needs to be configured before we evaluate the code to make
     # sure the metric instance is correctly send to the plugin.
-    @metric = settings.fetch(:metric, Instrument::NullMetric.new)
+    # NOTE: It is the responsibility of the Agent to set this externally with a setter
+    # if there's an intent of this not being a NullMetric
+    @metric = Instrument::NullMetric.new
 
     grammar = LogStashConfigParser.new
     @config = grammar.parse(config_str)
@@ -94,7 +80,7 @@ module LogStash; class Pipeline
     # The config code is hard to represent as a log message...
     # So just print it.
 
-    if @settings[:debug_config] && logger.debug?
+    if @settings.get("config.debug") && logger.debug?
       logger.debug("Compiled pipeline code", :code => code)
     end
 
@@ -123,38 +109,29 @@ module LogStash; class Pipeline
     @ready.value
   end
 
-  def configure(setting, value)
-    @settings[setting] = value
-  end
-
   def safe_pipeline_worker_count
-    default = DEFAULT_SETTINGS[:default_pipeline_workers]
-    thread_count = @settings[:pipeline_workers] #override from args "-w 8" or config
+    default = @settings.get_default("pipeline.workers")
+    pipeline_workers = @settings.get("pipeline.workers") #override from args "-w 8" or config
     safe_filters, unsafe_filters = @filters.partition(&:threadsafe?)
+    plugins = unsafe_filters.collect { |f| f.config_name }
 
-    if unsafe_filters.any?
-      plugins = unsafe_filters.collect { |f| f.config_name }
-      case thread_count
-      when nil
-        # user did not specify a worker thread count
-        # warn if the default is multiple
-
-        if default > 1
-          @logger.warn("Defaulting pipeline worker threads to 1 because there are some filters that might not work with multiple worker threads",
-                       :count_was => default, :filters => plugins)
-        end
+    return pipeline_workers if unsafe_filters.empty?
 
-        1 # can't allow the default value to propagate if there are unsafe filters
-      when 0, 1
-        1
-      else
+    if @settings.set?("pipeline.workers")
+      if pipeline_workers > 1
         @logger.warn("Warning: Manual override - there are filters that might not work with multiple worker threads",
-                     :worker_threads => thread_count, :filters => plugins)
-        thread_count # allow user to force this even if there are unsafe filters
+                     :worker_threads => pipeline_workers, :filters => plugins)
       end
     else
-      thread_count || default
+      # user did not specify a worker thread count
+      # warn if the default is multiple
+      if default > 1
+        @logger.warn("Defaulting pipeline worker threads to 1 because there are some filters that might not work with multiple worker threads",
+                     :count_was => default, :filters => plugins)
+        return 1 # can't allow the default value to propagate if there are unsafe filters
+      end
     end
+    pipeline_workers
   end
 
   def filters?
@@ -164,9 +141,8 @@ module LogStash; class Pipeline
   def run
     @started_at = Time.now
 
-    LogStash::Util.set_thread_name("[#{pipeline_id}]-pipeline-manager")
-    @logger.terminal(LogStash::Util::DefaultsPrinter.print(@settings))
     @thread = Thread.current
+    LogStash::Util.set_thread_name("[#{pipeline_id}]-pipeline-manager")
 
     start_workers
 
@@ -217,15 +193,15 @@ module LogStash; class Pipeline
       @filters.each {|f| f.register }
 
       pipeline_workers = safe_pipeline_worker_count
-      batch_size = @settings[:pipeline_batch_size]
-      batch_delay = @settings[:pipeline_batch_delay]
+      batch_size = @settings.get("pipeline.batch.size")
+      batch_delay = @settings.get("pipeline.batch.delay")
       max_inflight = batch_size * pipeline_workers
       @logger.info("Starting pipeline",
-                   :id => self.pipeline_id,
-                   :pipeline_workers => pipeline_workers,
-                   :batch_size => batch_size,
-                   :batch_delay => batch_delay,
-                   :max_inflight => max_inflight)
+                   "id" => self.pipeline_id,
+                   "pipeline.workers" => pipeline_workers,
+                   "pipeline.batch.size" => batch_size,
+                   "pipeline.batch.delay" => batch_delay,
+                   "pipeline.max_inflight" => max_inflight)
       if max_inflight > MAX_INFLIGHT_WARN_THRESHOLD
         @logger.warn "CAUTION: Recommended inflight events max exceeded! Logstash will run with up to #{max_inflight} events in memory in your current configuration. If your message sizes are large this may cause instability with the default heap size. Please consider setting a non-standard heap size, changing the batch size (currently #{batch_size}), or changing the number of pipeline workers (currently #{pipeline_workers})"
       end
@@ -438,7 +414,7 @@ module LogStash; class Pipeline
   def shutdown_workers
     # Each worker thread will receive this exactly once!
     @worker_threads.each do |t|
-      @logger.debug("Pushing shutdown", :thread => t)
+      @logger.debug("Pushing shutdown", :thread => t.inspect)
       @input_queue.push(LogStash::SHUTDOWN)
     end
 
@@ -453,14 +429,13 @@ module LogStash; class Pipeline
 
   def plugin(plugin_type, name, *args)
     args << {} if args.empty?
-    args.first.merge!(LogStash::Config::Mixin::ALLOW_ENV_FLAG => @allow_env)
 
     pipeline_scoped_metric = metric.namespace([:stats, :pipelines, pipeline_id.to_s.to_sym, :plugins])
 
     klass = LogStash::Plugin.lookup(plugin_type, name)
 
     if plugin_type == "output"
-      LogStash::OutputDelegator.new(@logger, klass, DEFAULT_OUTPUT_WORKERS, pipeline_scoped_metric.namespace(:outputs), *args)
+      LogStash::OutputDelegator.new(@logger, klass, @settings.get("pipeline.output.workers"), pipeline_scoped_metric.namespace(:outputs), *args)
     elsif plugin_type == "filter"
       LogStash::FilterDelegator.new(@logger, klass, pipeline_scoped_metric.namespace(:filters), *args)
     else

data/lib/logstash/plugin.rb

@@ -71,7 +71,7 @@ class LogStash::Plugin
   # close is called during shutdown, after the plugin worker
   # main task terminates
   def do_close
-    @logger.debug("closing", :plugin => self)
+    @logger.debug("closing", :plugin => self.class.name)
     close
   end
 

data/lib/logstash/runner.rb

@@ -3,7 +3,8 @@ Thread.abort_on_exception = true
 Encoding.default_external = Encoding::UTF_8
 $DEBUGLIST = (ENV["DEBUG"] || "").split(",")
 
-require "clamp" # gem 'clamp'
+require "clamp"
+require "cabin"
 require "net/http"
 require "logstash/environment"
 
@@ -12,135 +13,137 @@ LogStash::Environment.load_locale!
 require "logstash/namespace"
 require "logstash/agent"
 require "logstash/config/defaults"
+require "logstash/shutdown_watcher"
+require "logstash/patches/clamp"
 
-class LogStash::Runner < Clamp::Command
+class LogStash::Runner < Clamp::StrictCommand
 
-  option ["-f", "--config"], "CONFIG_PATH",
+  # Node Settings
+  option ["-n", "--node.name"], "NAME",
+    I18n.t("logstash.runner.flag.node_name"),
+    :attribute_name => "node.name",
+    :default => LogStash::SETTINGS.get_default("node.name")
+
+  # Config Settings
+  option ["-f", "--path.config"], "CONFIG_PATH",
     I18n.t("logstash.runner.flag.config"),
-    :attribute_name => :config_path
+    :attribute_name => "path.config"
 
-  option "-e", "CONFIG_STRING",
+  option ["-e", "--config.string"], "CONFIG_STRING",
     I18n.t("logstash.runner.flag.config-string",
-           :default_input => LogStash::Config::Defaults.input,
-           :default_output => LogStash::Config::Defaults.output),
-    :default => nil, :attribute_name => :config_string
+      :default_input => LogStash::Config::Defaults.input,
+      :default_output => LogStash::Config::Defaults.output),
+    :default => LogStash::SETTINGS.get_default("config.string"),
+    :attribute_name => "config.string"
 
-  option ["-w", "--pipeline-workers"], "COUNT",
+  # Pipeline settings
+  option ["-w", "--pipeline.workers"], "COUNT",
     I18n.t("logstash.runner.flag.pipeline-workers"),
-    :attribute_name => :pipeline_workers,
-    :default => LogStash::Pipeline::DEFAULT_SETTINGS[:default_pipeline_workers]
+    :attribute_name => "pipeline.workers",
+    :default => LogStash::SETTINGS.get_default("pipeline.workers"), &:to_i
+
+  option ["-b", "--pipeline.batch.size"], "SIZE",
+    I18n.t("logstash.runner.flag.pipeline-batch-size"),
+    :attribute_name => "pipeline.batch.size",
+    :default => LogStash::SETTINGS.get_default("pipeline.batch.size"), &:to_i
 
-  option ["-b", "--pipeline-batch-size"], "SIZE",
-         I18n.t("logstash.runner.flag.pipeline-batch-size"),
-         :attribute_name => :pipeline_batch_size,
-         :default => LogStash::Pipeline::DEFAULT_SETTINGS[:pipeline_batch_size]
+  option ["-u", "--pipeline.batch.delay"], "DELAY_IN_MS",
+    I18n.t("logstash.runner.flag.pipeline-batch-delay"),
+    :attribute_name => "pipeline.batch.delay",
+    :default => LogStash::SETTINGS.get_default("pipeline.batch.delay"), &:to_i
 
-  option ["-u", "--pipeline-batch-delay"], "DELAY_IN_MS",
-         I18n.t("logstash.runner.flag.pipeline-batch-delay"),
-         :attribute_name => :pipeline_batch_delay,
-         :default => LogStash::Pipeline::DEFAULT_SETTINGS[:pipeline_batch_delay]
+  option ["--pipeline.unsafe_shutdown"], :flag,
+    I18n.t("logstash.runner.flag.unsafe_shutdown"),
+    :attribute_name => "pipeline.unsafe_shutdown",
+    :default => LogStash::SETTINGS.get_default("pipeline.unsafe_shutdown")
+
+  # Plugins Settings
+  option ["-p", "--path.plugins"] , "PATH",
+    I18n.t("logstash.runner.flag.pluginpath"),
+    :multivalued => true, :attribute_name => "path.plugins",
+    :default => LogStash::SETTINGS.get_default("path.plugins")
 
-  option ["-l", "--log"], "FILE",
+  # Logging Settings
+  option ["-l", "--path.log"], "FILE",
     I18n.t("logstash.runner.flag.log"),
-    :attribute_name => :log_file
+    :attribute_name => "path.log"
 
-  # Old support for the '-v' flag'
-  option "-v", :flag,
-    I18n.t("logstash.runner.flag.verbosity"),
-    :attribute_name => :verbosity, :multivalued => true
+  option "--log.level", "LEVEL", I18n.t("logstash.runner.flag.log_level"),
+    :default => LogStash::SETTINGS.get_default("log.level")
 
-  option "--quiet", :flag, I18n.t("logstash.runner.flag.quiet")
-  option "--verbose", :flag, I18n.t("logstash.runner.flag.verbose")
-  option "--debug", :flag, I18n.t("logstash.runner.flag.debug")
+  option "--config.debug", :flag,
+    I18n.t("logstash.runner.flag.config_debug"),
+    :default => LogStash::SETTINGS.get_default("config.debug"),
+    :attribute_name => "config.debug"
 
-  option ["--debug-config"], :flag,
-         I18n.t("logstash.runner.flag.debug_config"),
-         :attribute_name => :debug_config, :default => false
+  # Other settings
+  option ["-i", "--interactive"], "SHELL",
+    I18n.t("logstash.runner.flag.rubyshell"),
+    :attribute_name => "interactive"
 
   option ["-V", "--version"], :flag,
     I18n.t("logstash.runner.flag.version")
 
-  option ["-p", "--pluginpath"] , "PATH",
-    I18n.t("logstash.runner.flag.pluginpath"),
-    :multivalued => true,
-    :attribute_name => :plugin_paths
-
-  option ["-t", "--configtest"], :flag,
+  option ["-t", "--config.test_and_exit"], :flag,
     I18n.t("logstash.runner.flag.configtest"),
-    :attribute_name => :config_test
-
-  option "--[no-]allow-unsafe-shutdown", :flag,
-    I18n.t("logstash.runner.flag.unsafe_shutdown"),
-    :attribute_name => :unsafe_shutdown,
-    :default => false
+    :attribute_name => "config.test_and_exit",
+    :default => LogStash::SETTINGS.get_default("config.test_and_exit")
 
-  option ["-i", "--interactive"], "SHELL",
-    I18n.t("logstash.runner.flag.rubyshell"),
-    :attribute_name => :ruby_shell
-
-  option ["-n", "--node-name"], "NAME",
-    I18n.t("logstash.runner.flag.node_name"),
-    :attribute_name => :node_name
-
-  option ["-r", "--[no-]auto-reload"], :flag,
+  option ["-r", "--config.reload.automatic"], :flag,
     I18n.t("logstash.runner.flag.auto_reload"),
-    :attribute_name => :auto_reload, :default => false
+    :attribute_name => "config.reload.automatic",
+    :default => LogStash::SETTINGS.get_default("config.reload.automatic")
 
-  option ["--reload-interval"], "RELOAD_INTERVAL",
+  option ["--config.reload.interval"], "RELOAD_INTERVAL",
     I18n.t("logstash.runner.flag.reload_interval"),
-    :attribute_name => :reload_interval, :default => 3, &:to_i
-
-  option ["--http-host"], "WEB_API_HTTP_HOST",
-    I18n.t("logstash.web_api.flag.http_host"),
-    :attribute_name => :web_api_http_host, :default => "127.0.0.1"
+    :attribute_name => "config.reload.interval",
+    :default => LogStash::SETTINGS.get_default("config.reload.interval"), &:to_i
 
-  option ["--http-port"], "WEB_API_HTTP_PORT",
-    I18n.t("logstash.web_api.flag.http_port"),
-    :attribute_name => :web_api_http_port, :default => 9600
-
-  option ["--allow-env"], :flag,
-    I18n.t("logstash.runner.flag.allow-env"),
-    :attribute_name => :allow_env, :default => false
-
-  def pipeline_workers=(pipeline_workers_value)
-    @pipeline_settings[:pipeline_workers] = validate_positive_integer(pipeline_workers_value)
-  end
-
-  def pipeline_batch_size=(pipeline_batch_size_value)
-    @pipeline_settings[:pipeline_batch_size] = validate_positive_integer(pipeline_batch_size_value)
-  end
+  option ["--http.host"], "HTTP_HOST",
+    I18n.t("logstash.runner.flag.http_host"),
+    :attribute_name => "http.host",
+    :default => LogStash::SETTINGS.get_default("http.host")
 
-  def pipeline_batch_delay=(pipeline_batch_delay_value)
-    @pipeline_settings[:pipeline_batch_delay] = validate_positive_integer(pipeline_batch_delay_value)
-  end
+  option ["--http.port"], "HTTP_PORT",
+    I18n.t("logstash.runner.flag.http_port"),
+    :attribute_name => "http.port",
+    :default => LogStash::SETTINGS.get_default("http.port"), &:to_i
 
-  def validate_positive_integer(str_arg)
-    int_arg = str_arg.to_i
-    if str_arg !~ /^\d+$/ || int_arg < 1
-      raise ArgumentError, "Expected a positive integer, got '#{str_arg}'"
-    end
+  option ["--log.format"], "FORMAT",
+    I18n.t("logstash.runner.flag.log_format"),
+    :attribute_name => "log.format",
+    :default => LogStash::SETTINGS.get_default("log.format")
 
-    int_arg
-  end
+  option ["--path.settings"], "SETTINGS_DIR",
+    I18n.t("logstash.runner.flag.path_settings"),
+    :attribute_name => "path.settings",
+    :default => LogStash::SETTINGS.get_default("path.settings")
 
   attr_reader :agent
 
   def initialize(*args)
     @logger = Cabin::Channel.get(LogStash)
-    @pipeline_settings ||= { :pipeline_id => "main" }
+    @settings = LogStash::SETTINGS
     super(*args)
   end
 
+  def run(args)
+    settings_path = fetch_settings_path(args)
+    @settings.set("path.settings", settings_path) if settings_path
+    LogStash::SETTINGS.from_yaml(LogStash::SETTINGS.get("path.settings"))
+    super(*[args])
+  end
+
   def execute
     require "logstash/util"
     require "logstash/util/java_version"
     require "stud/task"
     require "cabin" # gem 'cabin'
-
+    require "logstash/logging/json"
 
     # Configure Logstash logging facility, this need to be done before everything else to
     # make sure the logger has the correct settings and the log level is correctly defined.
-    configure_logging(log_file)
+    configure_logging(setting("path.log"), setting("log.level"))
 
     LogStash::Util::set_thread_name(self.class.name)
 
@@ -149,33 +152,39 @@ class LogStash::Runner < Clamp::Command
       return 1
     end
 
-    # Print a warning to STDERR for bad java versions
-    LogStash::Util::JavaVersion.warn_on_bad_java_version
+    # Exit on bad java versions
+    java_version = LogStash::Util::JavaVersion.version
+    if LogStash::Util::JavaVersion.bad_java_version?(java_version)
+      $stderr.puts "Java version 1.8.0 or later is required. (You are running: #{java_version})"
+      return 1
+    end
 
-    LogStash::ShutdownWatcher.unsafe_shutdown = unsafe_shutdown?
+    LogStash::ShutdownWatcher.unsafe_shutdown = setting("pipeline.unsafe_shutdown")
     LogStash::ShutdownWatcher.logger = @logger
 
-    configure
+    configure_plugin_paths(setting("path.plugins"))
 
     if version?
       show_version
       return 0
     end
 
-    return start_shell(@ruby_shell, binding) if @ruby_shell
+    return start_shell(setting("interactive"), binding) if setting("interactive")
+
+    @settings.format_settings.each {|line| @logger.info(line) }
 
-    if config_string.nil? && config_path.nil?
+    if setting("config.string").nil? && setting("path.config").nil?
       fail(I18n.t("logstash.runner.missing-configuration"))
     end
 
-    if @auto_reload && config_path.nil?
+    if setting("config.reload.automatic") && setting("path.config").nil?
       # there's nothing to reload
       signal_usage_error(I18n.t("logstash.runner.reload-without-config-path"))
     end
 
-    if config_test?
-      config_loader = LogStash::Config::Loader.new(@logger, @debug_config)
-      config_str = config_loader.format_config(config_path, config_string)
+    if setting("config.test_and_exit")
+      config_loader = LogStash::Config::Loader.new(@logger)
+      config_str = config_loader.format_config(setting("path.config"), setting("config.string"))
       begin
         LogStash::Pipeline.new(config_str)
         @logger.terminal "Configuration OK"
@@ -186,22 +195,9 @@ class LogStash::Runner < Clamp::Command
       end
     end
 
-    @agent = create_agent(:logger => @logger,
-                          :auto_reload => @auto_reload,
-                          :reload_interval => @reload_interval,
-                          :collect_metric => true,
-                          :debug => debug?,
-                          :node_name => node_name,
-                          :debug_config => debug_config?,
-                          :web_api_http_host => @web_api_http_host,
-                          :web_api_http_port => @web_api_http_port)
-
-    @agent.register_pipeline("main", @pipeline_settings.merge({
-                          :config_string => config_string,
-                          :config_path => config_path,
-                          :debug_config => debug_config?,
-                          :allow_env => allow_env?
-                          }))
+    @agent = create_agent(@settings)
+
+    @agent.register_pipeline("main", @settings)
 
     # enable sigint/sigterm before starting the agent
     # to properly handle a stalled agent
@@ -236,10 +232,10 @@ class LogStash::Runner < Clamp::Command
   def show_version
     show_version_logstash
 
-    if [:info, :debug].include?(verbosity?) || debug? || verbose?
+    if @logger.debug? || @logger.info?
       show_version_ruby
       show_version_java if LogStash::Environment.jruby?
-      show_gems if [:debug].include?(verbosity?) || debug?
+      show_gems if @logger.debug?
     end
   end # def show_version
 
@@ -265,13 +261,6 @@ class LogStash::Runner < Clamp::Command
     end
   end # def show_gems
 
-  # Do any start-time configuration.
-  #
-  # Log file stuff, plugin path checking, etc.
-  def configure
-    configure_plugin_paths(plugin_paths)
-  end # def configure
-
   # add the given paths for ungemified/bare plugins lookups
   # @param paths [String, Array<String>] plugins path string or list of path strings to add
   def configure_plugin_paths(paths)
@@ -286,32 +275,21 @@ class LogStash::Runner < Clamp::Command
   end
 
   # Point logging at a specific path.
-  def configure_logging(path)
+  def configure_logging(path, level)
     @logger = Cabin::Channel.get(LogStash)
     # Set with the -v (or -vv...) flag
-    if quiet?
+    case level
+    when "quiet"
       @logger.level = :error
-    elsif verbose?
+    when "verbose"
       @logger.level = :info
-    elsif debug?
+    when "debug"
       @logger.level = :debug
     else
-      # Old support for the -v and -vv stuff.
-      if verbosity? && verbosity?.any?
-        # this is an array with length of how many times the flag is given
-        if verbosity?.length == 1
-          @logger.warn("The -v flag is deprecated and will be removed in a future release. You should use --verbose instead.")
-          @logger.level = :info
-        else
-          @logger.warn("The -vv flag is deprecated and will be removed in a future release. You should use --debug instead.")
-          @logger.level = :debug
-        end
-      else
-        @logger.level = :warn
-      end
+      @logger.level = :warn
     end
 
-    if log_file
+    if path
       # TODO(sissel): Implement file output/rotation in Cabin.
       # TODO(sissel): Catch exceptions, report sane errors.
       begin
@@ -322,15 +300,24 @@ class LogStash::Runner < Clamp::Command
                     :path => path, :error => e))
       end
 
-      @logger.subscribe(STDOUT, :level => :fatal)
-      @logger.subscribe(@log_fd)
+      if setting("log.format") == "json"
+        @logger.subscribe(LogStash::Logging::JSON.new(STDOUT), :level => :fatal)
+        @logger.subscribe(LogStash::Logging::JSON.new(@log_fd))
+      else
+        @logger.subscribe(STDOUT, :level => :fatal)
+        @logger.subscribe(@log_fd)
+      end
       @logger.terminal "Sending logstash logs to #{path}."
     else
-      @logger.subscribe(STDOUT)
+      if setting("log.format") == "json"
+        @logger.subscribe(LogStash::Logging::JSON.new(STDOUT))
+      else
+        @logger.subscribe(STDOUT)
+      end
     end
 
-    if debug_config? && @logger.level != :debug
-      @logger.warn("--debug-config was specified, but log level was not set to --debug! No config info will be logged.")
+    if setting("config.debug") && @logger.level != :debug
+      @logger.warn("--config.debug was specified, but log.level was not set to \'debug\'! No config info will be logged.")
     end
 
     # TODO(sissel): redirect stdout/stderr to the log as well
@@ -390,4 +377,26 @@ class LogStash::Runner < Clamp::Command
     end
   end
 
-end # class LogStash::Runner
+  def setting(key)
+    @settings.get_value(key)
+  end
+
+  # where can I find the logstash.yml file?
+  # 1. look for a "--path.settings path"
+  # 2. look for a "--path.settings=path"
+  # 3. check if the LS_SETTINGS_DIR environment variable is set
+  # 4. return nil if not found
+  def fetch_settings_path(cli_args)
+    if i=cli_args.find_index("--path.settings")
+      cli_args[i+1]
+    elsif settings_arg = cli_args.find {|v| v.match(/--path.settings=/) }
+      match = settings_arg.match(/--path.settings=(.*)/)
+      match[1]
+    elsif ENV['LS_SETTINGS_DIR']
+      ENV['LS_SETTINGS_DIR']
+    else
+      nil
+    end
+  end
+
+end