logstash-core 6.0.0.alpha1-java → 6.0.0.alpha2-java

data/lib/logstash/instrument/periodic_poller/pq.rb

@@ -11,7 +11,7 @@ module LogStash module Instrument module PeriodicPoller
     end
 
     def collect
-      pipeline_id, pipeline = @agent.running_pipelines.first
+      pipeline_id, pipeline = @agent.with_running_pipelines {|pipelines| pipelines.first }
       unless pipeline.nil?
         pipeline.collect_stats
       end

data/lib/logstash/outputs/base.rb

@@ -109,6 +109,8 @@ class LogStash::Outputs::Base < LogStash::Plugin
     super
     # There is no easy way to propage an instance variable into the codec, because the codec
     # are created at the class level
+    # TODO(talevy): Codecs should have their own execution_context, for now they will inherit their 
+    #               parent plugin's
     @codec.execution_context = context
     context
   end

data/lib/logstash/pipeline.rb

@@ -18,22 +18,28 @@ require "logstash/instrument/null_metric"
 require "logstash/instrument/namespaced_null_metric"
 require "logstash/instrument/collector"
 require "logstash/instrument/wrapped_write_client"
+require "logstash/util/dead_letter_queue_manager"
 require "logstash/output_delegator"
 require "logstash/filter_delegator"
 require "logstash/queue_factory"
 require "logstash/compiler"
 require "logstash/execution_context"
 
+java_import org.logstash.common.DeadLetterQueueFactory
+java_import org.logstash.common.io.DeadLetterQueueWriter
+
 module LogStash; class BasePipeline
   include LogStash::Util::Loggable
 
   attr_reader :settings, :config_str, :config_hash, :inputs, :filters, :outputs, :pipeline_id, :lir, :execution_context
+  attr_reader :pipeline_config
 
-  def initialize(config_str, settings = SETTINGS, namespaced_metric = nil, agent = nil)
+  def initialize(pipeline_config, namespaced_metric = nil, agent = nil)
     @logger = self.logger
 
-    @config_str = config_str
-    @settings = settings
+    @pipeline_config = pipeline_config
+    @config_str = pipeline_config.config_string
+    @settings = pipeline_config.settings
     @config_hash = Digest::SHA1.hexdigest(@config_str)
 
     @lir = compile_lir
@@ -42,14 +48,20 @@ module LogStash; class BasePipeline
     # a unique id when auto-generating plugin ids
     @plugin_counter ||= 0
 
-    @pipeline_id = settings.get_value("pipeline.id") || self.object_id
+    @pipeline_id = @settings.get_value("pipeline.id") || self.object_id
 
     # A list of plugins indexed by id
     @plugins_by_id = {}
     @inputs = nil
     @filters = nil
     @outputs = nil
-    @execution_context = LogStash::ExecutionContext.new(self, agent)
+    @agent = agent
+
+    if settings.get_value("dead_letter_queue.enable")
+      @dlq_writer = DeadLetterQueueFactory.getWriter(pipeline_id, settings.get_value("path.dead_letter_queue"))
+    else
+      @dlq_writer = LogStash::Util::DummyDeadLetterQueueWriter.new
+    end
 
     grammar = LogStashConfigParser.new
     parsed_config = grammar.parse(config_str)
@@ -73,7 +85,8 @@ module LogStash; class BasePipeline
   end
 
   def compile_lir
-    LogStash::Compiler.compile_pipeline(self.config_str)
+    source_with_metadata = org.logstash.common.SourceWithMetadata.new("str", "pipeline", self.config_str)
+    LogStash::Compiler.compile_sources(source_with_metadata)
   end
 
   def plugin(plugin_type, name, *args)
@@ -100,16 +113,18 @@ module LogStash; class BasePipeline
 
     klass = Plugin.lookup(plugin_type, name)
 
+    execution_context = ExecutionContext.new(self, @agent, id, klass.config_name, @dlq_writer)
+
     if plugin_type == "output"
-      OutputDelegator.new(@logger, klass, type_scoped_metric, @execution_context, OutputDelegatorStrategyRegistry.instance, args)
+      OutputDelegator.new(@logger, klass, type_scoped_metric, execution_context, OutputDelegatorStrategyRegistry.instance, args)
     elsif plugin_type == "filter"
-      FilterDelegator.new(@logger, klass, type_scoped_metric, @execution_context, args)
+      FilterDelegator.new(@logger, klass, type_scoped_metric, execution_context, args)
     else # input
       input_plugin = klass.new(args)
       scoped_metric = type_scoped_metric.namespace(id.to_sym)
       scoped_metric.gauge(:name, input_plugin.config_name)
       input_plugin.metric = scoped_metric
-      input_plugin.execution_context = @execution_context
+      input_plugin.execution_context = execution_context
       input_plugin
     end
   end
@@ -147,7 +162,8 @@ module LogStash; class Pipeline < BasePipeline
 
   MAX_INFLIGHT_WARN_THRESHOLD = 10_000
 
-  def initialize(config_str, settings = SETTINGS, namespaced_metric = nil, agent = nil)
+  def initialize(pipeline_config, namespaced_metric = nil, agent = nil)
+    @settings = pipeline_config.settings
     # This needs to be configured before we call super which will evaluate the code to make
     # sure the metric instance is correctly send to the plugins to make the namespace scoping work
     @metric = if namespaced_metric
@@ -180,6 +196,7 @@ module LogStash; class Pipeline < BasePipeline
     )
     @drain_queue =  @settings.get_value("queue.drain")
 
+
     @events_filtered = Concurrent::AtomicFixnum.new(0)
     @events_consumed = Concurrent::AtomicFixnum.new(0)
 
@@ -191,8 +208,6 @@ module LogStash; class Pipeline < BasePipeline
     @force_shutdown = Concurrent::AtomicBoolean.new(false)
   end # def initialize
 
-
-
   def ready?
     @ready.value
   end
@@ -230,7 +245,7 @@ module LogStash; class Pipeline < BasePipeline
     # this is useful in the context of pipeline reloading
     collect_stats
 
-    logger.debug("Starting pipeline", default_logging_keys)
+    @logger.debug("Starting pipeline", default_logging_keys)
 
     @finished_execution = Concurrent::AtomicBoolean.new(false)
 
@@ -279,7 +294,7 @@ module LogStash; class Pipeline < BasePipeline
 
     start_workers
 
-    @logger.info("Pipeline started", default_logging_keys)
+    @logger.info("Pipeline started", "pipeline.id" => @pipeline_id)
 
     # Block until all inputs have stopped
     # Generally this happens if SIGINT is sent and `shutdown` is called from an external thread
@@ -305,6 +320,7 @@ module LogStash; class Pipeline < BasePipeline
   def close
     @filter_queue_client.close
     @queue.close
+    @dlq_writer.close
   end
 
   def transition_to_running
@@ -559,6 +575,7 @@ module LogStash; class Pipeline < BasePipeline
     # stopped
     wait_for_workers
     clear_pipeline_metrics
+    @logger.info("Pipeline terminated", "pipeline.id" => @pipeline_id)
   end # def shutdown
 
   def force_shutdown!

data/lib/logstash/pipeline_action/create.rb

@@ -32,7 +32,7 @@ module LogStash module PipelineAction
     # The execute assume that the thread safety access of the pipeline
     # is managed by the caller.
     def execute(agent, pipelines)
-      pipeline = LogStash::Pipeline.new(@pipeline_config.config_string, @pipeline_config.settings, @metric, agent)
+      pipeline = LogStash::Pipeline.new(@pipeline_config, @metric, agent)
       
       status = pipeline.start # block until the pipeline is correctly started or crashed
 
@@ -42,6 +42,5 @@ module LogStash module PipelineAction
 
       LogStash::ConvergeResult::ActionResult.create(self, status)
     end
-
   end
 end end

data/lib/logstash/pipeline_action/reload.rb

@@ -27,7 +27,7 @@ module LogStash module PipelineAction
       end
 
       begin
-        pipeline_validator = LogStash::BasePipeline.new(@pipeline_config.config_string, @pipeline_config.settings)
+        pipeline_validator = LogStash::BasePipeline.new(@pipeline_config)
       rescue => e
         return LogStash::ConvergeResult::FailedAction.from_exception(e)
       end
@@ -36,6 +36,7 @@ module LogStash module PipelineAction
         return LogStash::ConvergeResult::FailedAction.new("Cannot reload pipeline, because the new pipeline is not reloadable")
       end
 
+      logger.info("Reloading pipeline", "pipeline.id" => pipeline_id)
       status = Stop.new(pipeline_id).execute(agent, pipelines)
 
       if status

data/lib/logstash/pipeline_settings.rb

@@ -0,0 +1,50 @@
+# encoding: utf-8
+require "logstash/settings"
+
+module LogStash
+  class PipelineSettings < Settings
+
+    # there are settings that the pipeline uses and can be changed per pipeline instance
+    SETTINGS_WHITE_LIST = [
+      "config.debug",
+      "config.reload.automatic",
+      "config.reload.interval",
+      "config.string",
+      "dead_letter_queue.enable",
+      "metric.collect",
+      "path.config",
+      "path.queue",
+      "pipeline.batch.delay",
+      "pipeline.batch.size",
+      "pipeline.id",
+      "pipeline.output.workers",
+      "pipeline.reloadable",
+      "pipeline.system",
+      "pipeline.workers",
+      "queue.checkpoint.acks",
+      "queue.checkpoint.interval",
+      "queue.checkpoint.writes",
+      "queue.drain",
+      "queue.max_bytes",
+      "queue.max_events",
+      "queue.page_capacity",
+      "queue.type",
+    ]
+
+    # register a set of settings that is used as the default set of pipelines settings
+    def self.from_settings(settings)
+      pipeline_settings = self.new
+      SETTINGS_WHITE_LIST.each do |setting|
+        pipeline_settings.register(settings.get_setting(setting).clone)
+      end
+      pipeline_settings
+    end
+
+    def register(setting)
+      unless SETTINGS_WHITE_LIST.include?(setting.name)
+        raise ArgumentError.new("Only pipeline related settings can be registed in a PipelineSettings object. Received \"#{setting.name}\". Allowed settings: #{SETTINGS_WHITE_LIST}")
+      end
+      super(setting)
+    end
+  end
+end

data/lib/logstash/plugin.rb

@@ -3,6 +3,7 @@ require "logstash/namespace"
 require "logstash/logging"
 require "logstash/config/mixin"
 require "logstash/instrument/null_metric"
+require "logstash/util/dead_letter_queue_manager"
 require "concurrent"
 require "securerandom"
 

data/lib/logstash/runner.rb

@@ -165,7 +165,7 @@ class LogStash::Runner < Clamp::StrictCommand
     I18n.t("logstash.runner.flag.quiet"),
     :new_flag => "log.level", :new_value => "error"
 
-  attr_reader :agent, :settings
+  attr_reader :agent, :settings, :source_loader
   attr_accessor :bootstrap_checks
 
   def initialize(*args)
@@ -173,7 +173,9 @@ class LogStash::Runner < Clamp::StrictCommand
     @bootstrap_checks = DEFAULT_BOOTSTRAP_CHECKS.dup
 
     # Default we check local sources: `-e`, `-f` and the logstash.yml options.
-    LogStash::Config::SOURCE_LOADER.add_source(LogStash::Config::Source::Local.new(@settings))
+    @source_loader = LogStash::Config::SourceLoader.new(@settings)
+    @source_loader.add_source(LogStash::Config::Source::Local.new(@settings))
+    @source_loader.add_source(LogStash::Config::Source::MultiLocal.new(@settings))
 
     super(*args)
   end
@@ -266,12 +268,12 @@ class LogStash::Runner < Clamp::StrictCommand
 
     if setting("config.test_and_exit")
       begin
-        results = LogStash::Config::SOURCE_LOADER.fetch
+        results = @source_loader.fetch
 
         # TODO(ph): make it better for multiple pipeline
         if results.success?
           results.response.each do |pipeline_config|
-            LogStash::BasePipeline.new(pipeline_config.config_string)
+            LogStash::BasePipeline.new(pipeline_config)
           end
           puts "Configuration OK"
           logger.info "Using config.test_and_exit mode. Config Validation Result: OK. Exiting Logstash"
@@ -289,7 +291,7 @@ class LogStash::Runner < Clamp::StrictCommand
     @data_path_lock = FileLockFactory.getDefault().obtainLock(setting("path.data"), ".lock");
 
     @dispatcher.fire(:before_agent)
-    @agent = create_agent(@settings, LogStash::Config::SOURCE_LOADER)
+    @agent = create_agent(@settings, @source_loader)
     @dispatcher.fire(:after_agent)
 
     # enable sigint/sigterm before starting the agent

data/lib/logstash/settings.rb

@@ -2,11 +2,14 @@
 require "logstash/util/loggable"
 require "fileutils"
 require "logstash/util/byte_value"
+require "logstash/util/environment_variables"
 require "logstash/util/time_value"
 
 module LogStash
   class Settings
 
+    include LogStash::Util::EnvironmentVariables
+    
     def initialize
       @settings = {}
       # Theses settings were loaded from the yaml file
@@ -106,9 +109,9 @@ module LogStash
       @settings.values.each(&:reset)
     end
 
-    def from_yaml(yaml_path)
-      settings = read_yaml(::File.join(yaml_path, "logstash.yml"))
-      self.merge(flatten_hash(settings), true)
+    def from_yaml(yaml_path, file_name="logstash.yml")
+      settings = read_yaml(::File.join(yaml_path, file_name))
+      self.merge(deep_replace(flatten_hash(settings)), true)
       self
     end
     
@@ -134,6 +137,11 @@ module LogStash
       end
     end
 
+    def ==(other)
+      return false unless other.kind_of?(::LogStash::Settings)
+      self.to_hash == other.to_hash
+    end
+
     private
     def read_yaml(path)
       YAML.safe_load(IO.read(path)) || {}

data/lib/logstash/shutdown_watcher.rb

@@ -1,4 +1,6 @@
 # encoding: utf-8
+require "concurrent/atomic/atomic_fixnum"
+require "concurrent/atomic/atomic_boolean"
 
 module LogStash
   class ShutdownWatcher
@@ -16,6 +18,8 @@ module LogStash
       @report_every = report_every
       @abort_threshold = abort_threshold
       @reports = []
+      @attempts_count = Concurrent::AtomicFixnum.new(0)
+      @running = Concurrent::AtomicBoolean.new(false)
     end
 
     def self.unsafe_shutdown=(boolean)
@@ -35,11 +39,26 @@ module LogStash
       self.class.logger
     end
 
+    def attempts_count
+      @attempts_count.value
+    end
+
+    def stop!
+      @running.make_false
+    end
+
+    def stopped?
+      @running.false?
+    end
+
     def start
       sleep(@cycle_period)
       cycle_number = 0
       stalled_count = 0
+      running!
       Stud.interval(@cycle_period) do
+        @attempts_count.increment
+        break if stopped?
         break unless @pipeline.thread.alive?
         @reports << pipeline_report_snapshot
         @reports.delete_at(0) if @reports.size > @report_every # expire old report
@@ -61,6 +80,8 @@ module LogStash
         end
         cycle_number = (cycle_number + 1) % @report_every
       end
+    ensure
+      stop!
     end
 
     def pipeline_report_snapshot
@@ -90,5 +111,10 @@ module LogStash
     def force_exit
       exit(-1)
     end
+
+    private
+    def running!
+      @running.make_true
+    end
   end
 end

data/lib/logstash/state_resolver.rb

@@ -19,9 +19,7 @@ module LogStash
         if pipeline.nil?
           actions << LogStash::PipelineAction::Create.new(pipeline_config, @metric)
         else
-          # TODO(ph): The pipeline should keep a reference to the original PipelineConfig
-          # and we could use straight comparison.
-          if pipeline_config.config_hash != pipeline.config_hash
+          if pipeline_config != pipeline.pipeline_config
             actions << LogStash::PipelineAction::Reload.new(pipeline_config, @metric)
           end
         end

data/lib/logstash/util/dead_letter_queue_manager.rb

@@ -0,0 +1,61 @@
+require 'logstash/environment'
+
+module LogStash; module Util
+  class PluginDeadLetterQueueWriter
+
+    attr_reader :plugin_id, :plugin_type, :inner_writer
+
+    def initialize(inner_writer, plugin_id, plugin_type)
+      @plugin_id = plugin_id
+      @plugin_type = plugin_type
+      @inner_writer = inner_writer
+    end
+
+    def write(logstash_event, reason)
+      if @inner_writer && @inner_writer.is_open
+        @inner_writer.writeEntry(logstash_event.to_java, @plugin_type, @plugin_id, reason)
+      end
+    end
+
+    def close
+      if @inner_writer && @inner_writer.is_open
+        @inner_writer.close
+      end
+    end
+  end
+
+  class DummyDeadLetterQueueWriter
+    # class uses to represent a writer when dead_letter_queue is disabled
+    def initialize
+    end
+
+    def write(logstash_event, reason)
+      # noop
+    end
+
+    def is_open
+      false
+    end
+
+    def close
+      # noop
+    end
+  end
+
+  class DeadLetterQueueFactory
+    java_import org.logstash.common.DeadLetterQueueFactory
+
+    def self.get(pipeline_id)
+      if LogStash::SETTINGS.get("dead_letter_queue.enable")
+        return DeadLetterQueueWriter.new(
+          DeadLetterQueueFactory.getWriter(pipeline_id, LogStash::SETTINGS.get("path.dead_letter_queue")))
+      else
+        return DeadLetterQueueWriter.new(nil)
+      end
+    end
+
+    def self.close(pipeline_id)
+      DeadLetterQueueFactory.close(pipeline_id)
+    end
+  end
+end end