logstash-core 6.0.0.alpha2-java → 6.0.0.beta1-java

data/lib/logstash/modules/resource_base.rb

@@ -0,0 +1,38 @@
+# encoding: utf-8
+require "logstash/namespace"
+require "logstash/json"
+require_relative "file_reader"
+
+module LogStash module Modules module ResourceBase
+  attr_reader :base, :content_type, :content_path, :content_id
+
+  def initialize(base, content_type, content_path, content = nil, content_id = nil)
+    @base, @content_type, @content_path = base, content_type, content_path
+    @content_id =  content_id || ::File.basename(@content_path, ".*")
+    # content at this time will be a JSON string
+    @content = content
+    if !@content.nil?
+      @content_as_object = LogStash::Json.load(@content) rescue {}
+    end
+  end
+
+  def content
+    @content ||= FileReader.read(@content_path)
+  end
+
+  def to_s
+    "#{base}, #{content_type}, #{content_path}, #{content_id}"
+  end
+
+  def content_as_object
+    @content_as_object ||= FileReader.read_json(@content_path) rescue nil
+  end
+
+  def <=>(other)
+    to_s <=> other.to_s
+  end
+
+  def ==(other)
+    to_s == other.to_s
+  end
+end end end

data/lib/logstash/modules/scaffold.rb

@@ -0,0 +1,50 @@
+# encoding: utf-8
+require "logstash/namespace"
+require "logstash/logging"
+require "erb"
+
+require_relative "elasticsearch_config"
+require_relative "kibana_config"
+require_relative "logstash_config"
+
+module LogStash module Modules class Scaffold
+  include LogStash::Util::Loggable
+
+  attr_reader :directory, :module_name, :kibana_version_parts
+  attr_reader :kibana_configuration, :logstash_configuration, :elasticsearch_configuration
+
+  def initialize(name, directory)
+    @module_name = name
+    @directory = directory  # this is the 'configuration folder in the GEM root.'
+    @kibana_version_parts = "6.0.0".split('.') # this is backup in case kibana client fails to connect
+  end
+
+  def add_kibana_version(version_parts)
+    @kibana_version_parts = version_parts
+  end
+
+  def import(import_engine, kibana_import_engine)
+    @elasticsearch_configuration.resources.each do |resource|
+      import_engine.put(resource)
+    end
+    @kibana_configuration.resources.each do |resource|
+      kibana_import_engine.put(resource)
+    end
+  end
+
+  def with_settings(module_settings)
+    @logstash_configuration = LogStashConfig.new(self, module_settings)
+    @kibana_configuration = KibanaConfig.new(self, module_settings)
+    @elasticsearch_configuration = ElasticsearchConfig.new(self, module_settings)
+    self
+  end
+
+  def config_string()
+    # settings should be set earlier by the caller.
+    # settings should be the subset from the YAML file with a structure like
+    # {"name" => "plugin name", "k1" => "v1", "k2" => "v2"}, etc.
+    return nil if @logstash_configuration.nil?
+    @logstash_configuration.config_string
+  end
+end end end # class LogStash::Modules::Scaffold
+

data/lib/logstash/modules/settings_merger.rb

@@ -0,0 +1,23 @@
+# encoding: utf-8
+require "logstash/namespace"
+
+module LogStash module Modules class SettingsMerger
+  def self.merge(cli_settings, yml_settings)
+    # both args are arrays of hashes, e.g.
+    # [{"name"=>"mod1", "var.input.tcp.port"=>"3333"}, {"name"=>"mod2"}]
+    # [{"name"=>"mod1", "var.input.tcp.port"=>2222, "var.kibana.username"=>"rupert", "var.kibana.password"=>"fotherington"}, {"name"=>"mod3", "var.input.tcp.port"=>4445}]
+    merged = []
+    # union and group_by preserves order
+    # union will also coalesce identical hashes
+    union_of_settings = (cli_settings | yml_settings)
+    grouped_by_name = union_of_settings.group_by{|e| e["name"]}
+    grouped_by_name.each do |name, array|
+      if array.size == 2
+        merged << array.first.merge(array.last)
+      else
+        merged.concat(array)
+      end
+    end
+    merged
+  end
+end end end

data/lib/logstash/modules/util.rb

@@ -0,0 +1,17 @@
+# encoding: utf-8
+require_relative "scaffold"
+
+# This module function should be used when gems or
+# x-pack defines modules in their folder structures.
+module LogStash module Modules module Util
+  def self.register_local_modules(path)
+    modules_path = ::File.join(path, "modules")
+    ::Dir.foreach(modules_path) do |item|
+      # Ignore unix relative path ids
+      next if item == '.' or item == '..'
+      # Ignore non-directories
+      next if !::File.directory?(::File.join(modules_path, ::File::Separator, item))
+      LogStash::PLUGIN_REGISTRY.add(:modules, item, Scaffold.new(item, ::File.join(modules_path, item, "configuration")))
+    end
+  end
+end end end

data/lib/logstash/namespace.rb

@@ -11,4 +11,5 @@ module LogStash
   module PluginMixins; end
   module PluginManager; end
   module Api; end
+  module Modules; end
 end # module LogStash

data/lib/logstash/pipeline.rb

@@ -24,19 +24,23 @@ require "logstash/filter_delegator"
 require "logstash/queue_factory"
 require "logstash/compiler"
 require "logstash/execution_context"
+require "securerandom"
 
 java_import org.logstash.common.DeadLetterQueueFactory
+java_import org.logstash.common.SourceWithMetadata
 java_import org.logstash.common.io.DeadLetterQueueWriter
 
 module LogStash; class BasePipeline
   include LogStash::Util::Loggable
 
-  attr_reader :settings, :config_str, :config_hash, :inputs, :filters, :outputs, :pipeline_id, :lir, :execution_context
+  attr_reader :settings, :config_str, :config_hash, :inputs, :filters, :outputs, :pipeline_id, :lir, :execution_context, :ephemeral_id
   attr_reader :pipeline_config
 
   def initialize(pipeline_config, namespaced_metric = nil, agent = nil)
     @logger = self.logger
 
+    @ephemeral_id = SecureRandom.uuid
+
     @pipeline_config = pipeline_config
     @config_str = pipeline_config.config_string
     @settings = pipeline_config.settings
@@ -57,16 +61,13 @@ module LogStash; class BasePipeline
     @outputs = nil
     @agent = agent
 
-    if settings.get_value("dead_letter_queue.enable")
-      @dlq_writer = DeadLetterQueueFactory.getWriter(pipeline_id, settings.get_value("path.dead_letter_queue"))
-    else
-      @dlq_writer = LogStash::Util::DummyDeadLetterQueueWriter.new
-    end
+    @dlq_writer = dlq_writer
 
     grammar = LogStashConfigParser.new
     parsed_config = grammar.parse(config_str)
     raise(ConfigurationError, grammar.failure_reason) if parsed_config.nil?
 
+    parsed_config.process_escape_sequences = settings.get_value("config.support_escapes")
     config_code = parsed_config.compile
 
     # config_code = BasePipeline.compileConfig(config_str)
@@ -84,9 +85,26 @@ module LogStash; class BasePipeline
     end
   end
 
+  def dlq_writer
+    if settings.get_value("dead_letter_queue.enable")
+      @dlq_writer = DeadLetterQueueFactory.getWriter(pipeline_id, settings.get_value("path.dead_letter_queue"), settings.get_value("dead_letter_queue.max_bytes"))
+    else
+      @dlq_writer = LogStash::Util::DummyDeadLetterQueueWriter.new
+    end
+  end
+
+  def close_dlq_writer
+    @dlq_writer.close
+    if settings.get_value("dead_letter_queue.enable")
+      DeadLetterQueueFactory.release(pipeline_id)
+    end
+  end
+
   def compile_lir
-    source_with_metadata = org.logstash.common.SourceWithMetadata.new("str", "pipeline", self.config_str)
-    LogStash::Compiler.compile_sources(source_with_metadata)
+    sources_with_metadata = [
+      SourceWithMetadata.new("str", "pipeline", 0, 0, self.config_str)
+    ]
+    LogStash::Compiler.compile_sources(sources_with_metadata, @settings)
   end
 
   def plugin(plugin_type, name, *args)
@@ -144,6 +162,12 @@ module LogStash; class BasePipeline
   def non_reloadable_plugins
     (inputs + filters + outputs).select { |plugin| !plugin.reloadable? }
   end
+
+  private
+
+  def default_logging_keys(other_keys = {})
+    { :pipeline_id => pipeline_id }.merge(other_keys)
+  end
 end; end
 
 module LogStash; class Pipeline < BasePipeline
@@ -172,6 +196,7 @@ module LogStash; class Pipeline < BasePipeline
       Instrument::NullMetric.new
     end
 
+    @ephemeral_id = SecureRandom.uuid
     @settings = settings
     @reporter = PipelineReporter.new(@logger, self)
     @worker_threads = []
@@ -187,7 +212,7 @@ module LogStash; class Pipeline < BasePipeline
 
     @input_queue_client = @queue.write_client
     @filter_queue_client = @queue.read_client
-    @signal_queue = Queue.new
+    @signal_queue = java.util.concurrent.LinkedBlockingQueue.new
     # Note that @inflight_batches as a central mechanism for tracking inflight
     # batches will fail if we have multiple read clients here.
     @filter_queue_client.set_events_metric(metric.namespace([:stats, :events]))
@@ -244,6 +269,7 @@ module LogStash; class Pipeline < BasePipeline
     # Since we start lets assume that the metric namespace is cleared
     # this is useful in the context of pipeline reloading
     collect_stats
+    collect_dlq_stats
 
     @logger.debug("Starting pipeline", default_logging_keys)
 
@@ -320,7 +346,7 @@ module LogStash; class Pipeline < BasePipeline
   def close
     @filter_queue_client.close
     @queue.close
-    @dlq_writer.close
+    close_dlq_writer
   end
 
   def transition_to_running
@@ -382,6 +408,9 @@ module LogStash; class Pipeline < BasePipeline
       config_metric.gauge(:batch_delay, batch_delay)
       config_metric.gauge(:config_reload_automatic, @settings.get("config.reload.automatic"))
       config_metric.gauge(:config_reload_interval, @settings.get("config.reload.interval"))
+      config_metric.gauge(:dead_letter_queue_enabled, dlq_enabled?)
+      config_metric.gauge(:dead_letter_queue_path, @dlq_writer.get_path.to_absolute_path.to_s) if dlq_enabled?
+
 
       @logger.info("Starting pipeline", default_logging_keys(
         "pipeline.workers" => pipeline_workers,
@@ -415,6 +444,10 @@ module LogStash; class Pipeline < BasePipeline
     end
   end
 
+  def dlq_enabled?
+    @settings.get("dead_letter_queue.enable")
+  end
+
   # Main body of what a worker thread does
   # Repeatedly takes batches off the queue, filters, then outputs them
   def worker_loop(batch_size, batch_delay)
@@ -423,19 +456,22 @@ module LogStash; class Pipeline < BasePipeline
     @filter_queue_client.set_batch_dimensions(batch_size, batch_delay)
 
     while true
-      signal = @signal_queue.empty? ? NO_SIGNAL : @signal_queue.pop
+      signal = @signal_queue.poll || NO_SIGNAL
       shutdown_requested |= signal.shutdown? # latch on shutdown signal
 
       batch = @filter_queue_client.read_batch # metrics are started in read_batch
-      @events_consumed.increment(batch.size)
-      filter_batch(batch)
-      flush_filters_to_batch(batch, :final => false) if signal.flush?
-      output_batch(batch)
-      break if @force_shutdown.true? # Do not ack the current batch
-      @filter_queue_client.close_batch(batch)
+      if (batch.size > 0)
+        @events_consumed.increment(batch.size)
+        filter_batch(batch)
+        flush_filters_to_batch(batch, :final => false) if signal.flush?
+        output_batch(batch)
+        unless @force_shutdown.true? # ack the current batch
+          @filter_queue_client.close_batch(batch)
+        end
+      end
 
       # keep break at end of loop, after the read_batch operation, some pipeline specs rely on this "final read_batch" before shutdown.
-      break if shutdown_requested && !draining_queue?
+      break if (shutdown_requested && !draining_queue?) || @force_shutdown.true?
     end
 
     # we are shutting down, queue is drained if it was required, now  perform a final flush.
@@ -601,7 +637,7 @@ module LogStash; class Pipeline < BasePipeline
     # Each worker thread will receive this exactly once!
     @worker_threads.each do |t|
       @logger.debug("Pushing shutdown", default_logging_keys(:thread => t.inspect))
-      @signal_queue.push(SHUTDOWN)
+      @signal_queue.put(SHUTDOWN)
     end
 
     @worker_threads.each do |t|
@@ -652,7 +688,7 @@ module LogStash; class Pipeline < BasePipeline
   def flush
     if @flushing.compare_and_set(false, true)
       @logger.debug? && @logger.debug("Pushing flush onto pipeline", default_logging_keys)
-      @signal_queue.push(FLUSH)
+      @signal_queue.put(FLUSH)
     end
   end
 
@@ -695,10 +731,16 @@ module LogStash; class Pipeline < BasePipeline
       .each {|t| t.delete("status") }
   end
 
+  def collect_dlq_stats
+    if dlq_enabled?
+      dlq_metric = @metric.namespace([:stats, :pipelines, pipeline_id.to_s.to_sym, :dlq])
+      dlq_metric.gauge(:queue_size_in_bytes, @dlq_writer.get_current_queue_size)
+    end
+  end
+
   def collect_stats
     pipeline_metric = @metric.namespace([:stats, :pipelines, pipeline_id.to_s.to_sym, :queue])
     pipeline_metric.gauge(:type, settings.get("queue.type"))
-
     if @queue.is_a?(LogStash::Util::WrappedAckedQueue) && @queue.queue.is_a?(LogStash::AckedQueue)
       queue = @queue.queue
       dir_path = queue.dir_path
@@ -750,12 +792,9 @@ module LogStash; class Pipeline < BasePipeline
   private
 
   def default_logging_keys(other_keys = {})
-    default_options = if thread
-                        { :pipeline_id => pipeline_id, :thread => thread.inspect }
-                      else
-                        { :pipeline_id => pipeline_id }
-                      end
-    default_options.merge(other_keys)
+    keys = super
+    keys[:thread] = thread.inspect if thread
+    keys
   end
 
   def draining_queue?

data/lib/logstash/pipeline_settings.rb

@@ -7,6 +7,7 @@ module LogStash
     # there are settings that the pipeline uses and can be changed per pipeline instance
     SETTINGS_WHITE_LIST = [
       "config.debug",
+      "config.support_escapes",
       "config.reload.automatic",
       "config.reload.interval",
       "config.string",

data/lib/logstash/plugins/registry.rb

@@ -3,6 +3,7 @@ require "rubygems/package"
 require "logstash/util/loggable"
 require "logstash/plugin"
 require "logstash/plugins/hooks_registry"
+require "logstash/modules/scaffold"
 
 module LogStash module Plugins
   class Registry

data/lib/logstash/runner.rb

@@ -9,6 +9,7 @@ require "net/http"
 require "logstash/namespace"
 require "logstash-core/logstash-core"
 require "logstash/environment"
+require "logstash/modules/cli_parser"
 
 LogStash::Environment.load_locale!
 
@@ -19,6 +20,7 @@ require "logstash/patches/clamp"
 require "logstash/settings"
 require "logstash/version"
 require "logstash/plugins/registry"
+require "logstash/modules/util"
 require "logstash/bootstrap_check/default_config"
 require "logstash/bootstrap_check/bad_java"
 require "logstash/bootstrap_check/bad_ruby"
@@ -61,6 +63,17 @@ class LogStash::Runner < Clamp::StrictCommand
     :default => LogStash::SETTINGS.get_default("config.string"),
     :attribute_name => "config.string"
 
+  # Module settings
+  option ["--modules"], "MODULES",
+    I18n.t("logstash.runner.flag.modules"),
+    :multivalued => true,
+    :attribute_name => "modules_list"
+
+  option ["-M", "--modules.variable"], "MODULES_VARIABLE",
+    I18n.t("logstash.runner.flag.modules_variable"),
+    :multivalued => true,
+    :attribute_name => "modules_variable_list"
+
   # Pipeline settings
   option ["-w", "--pipeline.workers"], "COUNT",
     I18n.t("logstash.runner.flag.pipeline-workers"),
@@ -175,6 +188,7 @@ class LogStash::Runner < Clamp::StrictCommand
     # Default we check local sources: `-e`, `-f` and the logstash.yml options.
     @source_loader = LogStash::Config::SourceLoader.new(@settings)
     @source_loader.add_source(LogStash::Config::Source::Local.new(@settings))
+    @source_loader.add_source(LogStash::Config::Source::Modules.new(@settings))
     @source_loader.add_source(LogStash::Config::Source::MultiLocal.new(@settings))
 
     super(*args)
@@ -237,6 +251,9 @@ class LogStash::Runner < Clamp::StrictCommand
       return 0
     end
 
+    # Add local modules to the registry before everything else
+    LogStash::Modules::Util.register_local_modules(LogStash::Environment::LOGSTASH_HOME)
+
     # We configure the registry and load any plugin that can register hooks
     # with logstash, this need to be done before any operation.
     LogStash::PLUGIN_REGISTRY.setup!
@@ -249,6 +266,10 @@ class LogStash::Runner < Clamp::StrictCommand
 
     return start_shell(setting("interactive"), binding) if setting("interactive")
 
+    module_parser = LogStash::Modules::CLIParser.new(@modules_list, @modules_variable_list)
+    # Now populate Setting for modules.list with our parsed array.
+    @settings.set("modules.cli", module_parser.output)
+
     begin
       @bootstrap_checks.each { |bootstrap| bootstrap.check(@settings) }
     rescue LogStash::BootstrapCheckError => e
@@ -266,6 +287,29 @@ class LogStash::Runner < Clamp::StrictCommand
 
     @settings.format_settings.each {|line| logger.debug(line) }
 
+    # Check for absence of any configuration
+    # not bulletproof because we don't know yet if there
+    # are no pipelines from pipelines.yml
+    sources_without_conflict = []
+    unmatched_sources_conflict_messages = []
+    @source_loader.sources do |source|
+      if source.config_conflict?
+        if source.conflict_messages.any?
+          unmatched_sources_conflict_messages << source.conflict_messages.join(", ")
+        end
+      else
+        sources_without_conflict << source
+      end
+    end
+    if unmatched_sources_conflict_messages.any?
+      # i18n should be done at the sources side
+      signal_usage_error(unmatched_sources_conflict_messages.join(" "))
+      return 1
+    elsif sources_without_conflict.empty?
+      signal_usage_error(I18n.t("logstash.runner.missing-configuration"))
+      return 1
+    end
+
     if setting("config.test_and_exit")
       begin
         results = @source_loader.fetch
@@ -288,7 +332,7 @@ class LogStash::Runner < Clamp::StrictCommand
     end
 
     # lock path.data before starting the agent
-    @data_path_lock = FileLockFactory.getDefault().obtainLock(setting("path.data"), ".lock");
+    @data_path_lock = FileLockFactory.obtainLock(setting("path.data"), ".lock");
 
     @dispatcher.fire(:before_agent)
     @agent = create_agent(@settings, @source_loader)
@@ -333,7 +377,7 @@ class LogStash::Runner < Clamp::StrictCommand
     Stud::untrap("INT", sigint_id) unless sigint_id.nil?
     Stud::untrap("TERM", sigterm_id) unless sigterm_id.nil?
     Stud::untrap("HUP", sighup_id) unless sighup_id.nil?
-    FileLockFactory.getDefault().releaseLock(@data_path_lock) if @data_path_lock
+    FileLockFactory.releaseLock(@data_path_lock) if @data_path_lock
     @log_fd.close if @log_fd
   end # def self.main
 
@@ -456,7 +500,7 @@ class LogStash::Runner < Clamp::StrictCommand
       nil
     end
   end
-  
+
   # is the user asking for CLI help subcommand?
   def cli_help?(args)
     # I know, double negative