logstash-core 6.0.0.alpha1-java → 6.0.0.alpha2-java

data/lib/logstash/compiler.rb

@@ -1,38 +1,56 @@
 require 'logstash/util/loggable'
 require 'logstash/compiler/lscl/lscl_grammar'
 
-java_import org.logstash.config.ir.Pipeline
+java_import org.logstash.config.ir.PipelineIR
 java_import org.logstash.config.ir.graph.Graph;
 java_import org.logstash.config.ir.graph.PluginVertex;
 
 module LogStash; class Compiler
   include ::LogStash::Util::Loggable
 
-  def self.compile_pipeline(config_str, source_file=nil)
-    graph_sections = self.compile_graph(config_str, source_file)
-    pipeline = org.logstash.config.ir.Pipeline.new(
-      graph_sections[:input],
-      graph_sections[:filter],
-      graph_sections[:output]
-    )
+  def self.compile_sources(*sources_with_metadata)
+    graph_sections = sources_with_metadata.map do |swm|
+      self.compile_graph(swm)
+    end
+
+    input_graph = org.logstash.config.ir.graph.Graph.combine(*graph_sections.map {|s| s[:input] }).graph
+    output_graph = org.logstash.config.ir.graph.Graph.combine(*graph_sections.map {|s| s[:output] }).graph
+
+    filter_graph = graph_sections.reduce(nil) do |acc, s| 
+      filter_section = s[:filter]
+
+      if acc.nil? 
+        filter_section
+      else
+        acc.chain(filter_section)
+      end
+    end
+
+    original_source = sources_with_metadata.map(&:text).join("\n")
+
+    org.logstash.config.ir.PipelineIR.new(input_graph, filter_graph, output_graph, original_source)
   end
 
-  def self.compile_ast(config_str, source_file=nil)
+  def self.compile_ast(source_with_metadata)
+    if !source_with_metadata.is_a?(org.logstash.common.SourceWithMetadata)
+      raise ArgumentError, "Expected 'org.logstash.common.SourceWithMetadata', got #{source_with_metadata.class}"
+    end
+
     grammar = LogStashCompilerLSCLGrammarParser.new
-    config = grammar.parse(config_str)
+    config = grammar.parse(source_with_metadata.text)
 
     if config.nil?
       raise ConfigurationError, grammar.failure_reason
     end
 
-    config
+    config.compile(source_with_metadata)
   end
 
-  def self.compile_imperative(config_str, source_file=nil)
-    compile_ast(config_str, source_file).compile(source_file)
+  def self.compile_imperative(source_with_metadata)
+    compile_ast(source_with_metadata)
   end
 
-  def self.compile_graph(config_str, source_file=nil)
-    Hash[compile_imperative(config_str, source_file).map {|section,icompiled| [section, icompiled.toGraph]}]
+  def self.compile_graph(source_with_metadata)
+    Hash[compile_imperative(source_with_metadata).map {|section,icompiled| [section, icompiled.toGraph]}]
   end
 end; end

data/lib/logstash/compiler/lscl.rb

@@ -10,15 +10,23 @@ module LogStashCompilerLSCLGrammar; module LogStash; module Compiler; module LSC
   module Helpers
     def source_meta
       line, column = line_and_column
-      org.logstash.common.SourceWithMetadata.new(source_file, line, column, self.text_value)
+      org.logstash.common.SourceWithMetadata.new(base_protocol, base_id, line, column, self.text_value)
     end
 
-    def source_file=(value)
-      set_meta(:source_file, value)
+    def base_source_with_metadata=(value)
+      set_meta(:base_source_with_metadata, value)
     end
     
-    def source_file
-      get_meta(:source_file)
+    def base_source_with_metadata
+      get_meta(:base_source_with_metadata)
+    end
+
+    def base_protocol
+      self.base_source_with_metadata.protocol
+    end
+
+    def base_id
+      self.base_source_with_metadata.id
     end
 
     def compose(*statements)
@@ -39,7 +47,7 @@ module LogStashCompilerLSCLGrammar; module LogStash; module Compiler; module LSC
     end
 
     def empty_source_meta()
-      org.logstash.common.SourceWithMetadata.new()
+      org.logstash.common.SourceWithMetadata.new(base_protocol, base_id, nil)
     end
 
     def jdsl
@@ -70,9 +78,9 @@ module LogStashCompilerLSCLGrammar; module LogStash; module Compiler; module LSC
   class Config < Node
     include Helpers
     
-    def compile(source_file=nil)
+    def compile(base_source_with_metadata=nil)
       # There is no way to move vars across nodes in treetop :(
-      self.source_file = source_file
+      self.base_source_with_metadata = base_source_with_metadata
 
       sections = recursive_select(PluginSection)
 

data/lib/logstash/config/mixin.rb

@@ -33,37 +33,21 @@ LogStash::Environment.load_locale!
 # }
 #
 module LogStash::Config::Mixin
+  
+  include LogStash::Util::EnvironmentVariables
+  
   attr_accessor :config
   attr_accessor :original_params
 
   PLUGIN_VERSION_1_0_0 = LogStash::Util::PluginVersion.new(1, 0, 0)
   PLUGIN_VERSION_0_9_0 = LogStash::Util::PluginVersion.new(0, 9, 0)
-
-  ENV_PLACEHOLDER_REGEX = /\$\{(?<name>\w+)(\:(?<default>[^}]*))?\}/
-
+  
   # This method is called when someone does 'include LogStash::Config'
   def self.included(base)
     # Add the DSL methods to the 'base' given.
     base.extend(LogStash::Config::Mixin::DSL)
   end
-
-  # Recursive method to replace environment variable references in parameters
-  def deep_replace(value)
-    if (value.is_a?(Hash))
-      value.each do |valueHashKey, valueHashValue|
-        value[valueHashKey.to_s] = deep_replace(valueHashValue)
-      end
-    else
-      if (value.is_a?(Array))
-        value.each_index do | valueArrayIndex|
-          value[valueArrayIndex] = deep_replace(value[valueArrayIndex])
-        end
-      else
-        return replace_env_placeholders(value)
-      end
-    end
-  end
-
+  
   def config_init(params)
     # Validation will modify the values inside params if necessary.
     # For example: converting a string to a number, etc.
@@ -158,27 +142,6 @@ module LogStash::Config::Mixin
     @config = params
   end # def config_init
 
-  # Replace all environment variable references in 'value' param by environment variable value and return updated value
-  # Process following patterns : $VAR, ${VAR}, ${VAR:defaultValue}
-  def replace_env_placeholders(value)
-    return value unless value.is_a?(String)
-
-    value.gsub(ENV_PLACEHOLDER_REGEX) do |placeholder|
-      # Note: Ruby docs claim[1] Regexp.last_match is thread-local and scoped to
-      # the call, so this should be thread-safe.
-      #
-      # [1] http://ruby-doc.org/core-2.1.1/Regexp.html#method-c-last_match
-      name = Regexp.last_match(:name)
-      default = Regexp.last_match(:default)
-
-      replacement = ENV.fetch(name, default)
-      if replacement.nil?
-        raise LogStash::ConfigurationError, "Cannot evaluate `#{placeholder}`. Environment variable `#{name}` is not set and there is no default value given."
-      end
-      replacement
-    end
-  end # def replace_env_placeholders
-
   module DSL
     attr_accessor :flags
 

data/lib/logstash/config/pipeline_config.rb

@@ -30,7 +30,7 @@ module LogStash module Config
     end
 
     def ==(other)
-      config_hash == other.config_hash && pipeline_id == other.pipeline_id
+      config_hash == other.config_hash && pipeline_id == other.pipeline_id && settings == other.settings
     end
 
     def display_debug_information

data/lib/logstash/config/source/local.rb

@@ -37,7 +37,11 @@ module LogStash module Config module Source
         config_parts = []
         encoding_issue_files = []
 
-        get_files.each do |file|
+        if logger.debug?
+          logger.debug("Skipping the following files while reading config since they don't match the specified glob pattern", :files => get_unmatched_files)
+        end
+
+        get_matched_files.each do |file|
           next unless ::File.file?(file) # skip directory
 
           logger.debug("Reading config file", :config_file => file)
@@ -78,7 +82,7 @@ module LogStash module Config module Source
         ::File.expand_path(path)
       end
 
-      def get_files
+      def get_matched_files
         Dir.glob(path).sort
       end
 
@@ -90,6 +94,13 @@ module LogStash module Config module Source
         end
       end
 
+      def get_unmatched_files
+        # transform "/var/lib/*.conf" => /var/lib/*
+        t = ::File.split(@path)
+        all_files = Dir.glob(::File.join(t.first, "*")).sort
+        all_files - get_matched_files
+      end
+
       def valid_encoding?(content)
         content.ascii_only? && content.valid_encoding?
       end
@@ -132,24 +143,28 @@ module LogStash module Config module Source
     OUTPUT_BLOCK_RE = /output *{/
 
     def pipeline_configs
-      config_parts = []
 
-      config_parts.concat(ConfigStringLoader.read(config_string)) if config_string?
-      if local_config?
-        local_config_parts = ConfigPathLoader.read(config_path)
-        config_parts.concat(local_config_parts)
-      else
-        local_config_parts = []
+      if config_path? && config_string?
+        raise ConfigurationError.new("Settings 'config.string' and 'path.config' can't be used simultaneously.")
+      elsif !config_path? && !config_string?
+        raise ConfigurationError.new("Either 'config.string' or 'path.config' must be set.")
       end
 
-      config_parts.concat(ConfigRemoteLoader.read(config_path)) if remote_config?
+      config_parts = if config_string?
+        ConfigStringLoader.read(config_string)
+      elsif local_config?
+        ConfigPathLoader.read(config_path)
+      elsif remote_config?
+        ConfigRemoteLoader.read(config_path)
+      else
+        []
+      end
 
       return if config_parts.empty?
-      return if config_string? && config_string.strip.empty? && local_config? && local_config_parts.empty?
 
-      add_missing_default_inputs_or_outputs(config_parts)
+      add_missing_default_inputs_or_outputs(config_parts) if config_string?
 
-      [PipelineConfig.new(self.class, PIPELINE_ID, config_parts, @settings)]
+      [PipelineConfig.new(self.class, @settings.get("pipeline.id").to_sym, config_parts, @settings)]
     end
 
     def match?

data/lib/logstash/config/source/multi_local.rb

@@ -0,0 +1,72 @@
+# encoding: utf-8
+require "logstash/config/source/local"
+require "logstash/util/loggable"
+require "logstash/pipeline_settings"
+
+module LogStash module Config module Source
+  class MultiLocal < Local
+    include LogStash::Util::Loggable
+
+    def initialize(settings)
+      @original_settings = settings
+      super(settings)
+    end
+
+    def pipeline_configs
+      pipelines = retrieve_yaml_pipelines()
+      pipelines_settings = pipelines.map do |pipeline_settings|
+        ::LogStash::PipelineSettings.from_settings(@original_settings.clone).merge(pipeline_settings)
+      end
+      detect_duplicate_pipelines(pipelines_settings)
+      pipelines_settings.map do |pipeline_settings|
+        @settings = pipeline_settings
+        # this relies on instance variable @settings and the parent class' pipeline_configs
+        # method. The alternative is to refactor most of the Local source methods to accept
+        # a settings object instead of relying on @settings.
+        super # create a PipelineConfig object based on @settings
+      end.flatten
+    end
+
+    def match?
+      uses_config_string = @original_settings.get_setting("config.string").set?
+      uses_path_config = @original_settings.get_setting("path.config").set?
+      return true if !uses_config_string && !uses_path_config
+      if uses_path_config
+        logger.warn("Ignoring the 'pipelines.yml' file because 'path.config' (-f) is being used.")
+      elsif uses_config_string
+        logger.warn("Ignoring the 'pipelines.yml' file because 'config.string' (-e) is being used.")
+      end
+      false
+    end
+
+    def retrieve_yaml_pipelines
+      result = read_pipelines_from_yaml(pipelines_yaml_location)
+      case result
+      when Array
+        result
+      when false
+        raise ConfigurationError.new("Pipelines YAML file is empty. Path: #{pipelines_yaml_location}")
+      else
+        raise ConfigurationError.new("Pipelines YAML file must contain an array of pipeline configs. Found \"#{result.class}\" in #{pipelines_yaml_location}")
+      end
+    end
+
+    def read_pipelines_from_yaml(yaml_location)
+      logger.debug("Reading pipeline configurations from YAML", :location => pipelines_yaml_location)
+      ::YAML.load(IO.read(yaml_location))
+    rescue => e
+      raise ConfigurationError.new("Failed to read pipelines yaml file. Location: #{yaml_location}, Exception: #{e.inspect}")
+    end
+
+    def pipelines_yaml_location
+      ::File.join(@original_settings.get("path.settings"), "pipelines.yml")
+    end
+
+    def detect_duplicate_pipelines(pipelines)
+      duplicate_ids = pipelines.group_by {|pipeline| pipeline.get("pipeline.id") }.select {|k, v| v.size > 1 }.map {|k, v| k}
+      if duplicate_ids.any?
+        raise ConfigurationError.new("Pipelines YAML file contains duplicate pipeline ids: #{duplicate_ids.inspect}. Location: #{pipelines_yaml_location}")
+      end
+    end
+  end
+end end end

data/lib/logstash/config/source_loader.rb

@@ -1,5 +1,6 @@
 # encoding: utf-8
 require "logstash/config/source/local"
+require "logstash/config/source/multi_local"
 require "logstash/errors"
 require "thread"
 require "set"
@@ -120,6 +121,4 @@ module LogStash module Config
         .select { |group, pipeline_configs| pipeline_configs.size > 1 }
     end
   end
-
-  SOURCE_LOADER = SourceLoader.new
 end end

data/lib/logstash/environment.rb

@@ -50,6 +50,7 @@ module LogStash
             Setting::Numeric.new("queue.checkpoint.acks", 1024), # 0 is unlimited
             Setting::Numeric.new("queue.checkpoint.writes", 1024), # 0 is unlimited
             Setting::Numeric.new("queue.checkpoint.interval", 1000), # 0 is no time-based checkpointing
+            Setting::Boolean.new("dead_letter_queue.enable", false),
             Setting::TimeValue.new("slowlog.threshold.warn", "-1"),
             Setting::TimeValue.new("slowlog.threshold.info", "-1"),
             Setting::TimeValue.new("slowlog.threshold.debug", "-1"),
@@ -59,13 +60,21 @@ module LogStash
   # Compute the default queue path based on `path.data`
   default_queue_file_path = ::File.join(SETTINGS.get("path.data"), "queue")
   SETTINGS.register Setting::WritableDirectory.new("path.queue", default_queue_file_path)
-  
+  # Compute the default dead_letter_queue path based on `path.data`
+  default_dlq_file_path = ::File.join(SETTINGS.get("path.data"), "dead_letter_queue")
+  SETTINGS.register Setting::WritableDirectory.new("path.dead_letter_queue", default_dlq_file_path)
+
   SETTINGS.on_post_process do |settings|
     # If the data path is overridden but the queue path isn't recompute the queue path
     # We need to do this at this stage because of the weird execution order
     # our monkey-patched Clamp follows
-    if settings.set?("path.data") && !settings.set?("path.queue")
-      settings.set_value("path.queue", ::File.join(settings.get("path.data"), "queue"))
+    if settings.set?("path.data")
+      if !settings.set?("path.queue")
+        settings.set_value("path.queue", ::File.join(settings.get("path.data"), "queue"))
+      end
+      if !settings.set?("path.dead_letter_queue")
+        settings.set_value("path.dead_letter_queue", ::File.join(settings.get("path.data"), "dead_letter_queue"))
+      end
     end
   end
 

data/lib/logstash/execution_context.rb

@@ -1,13 +1,17 @@
 # encoding: utf-8
+require "logstash/util/dead_letter_queue_manager"
 module LogStash
   class ExecutionContext
-    attr_reader :pipeline, :agent
+    attr_reader :pipeline, :agent, :dlq_writer
 
-    def initialize(pipeline, agent)
+    def initialize(pipeline, agent, plugin_id, plugin_type, dlq_writer)
       @pipeline = pipeline
       @agent = agent
+      @plugin_id = plugin_id
+      @plugin_type = plugin_type
+      @dlq_writer = LogStash::Util::PluginDeadLetterQueueWriter.new(dlq_writer, @plugin_id, @plugin_type)
     end
-    
+
     def pipeline_id
       @pipeline.pipeline_id
     end

data/lib/logstash/inputs/base.rb

@@ -105,6 +105,8 @@ class LogStash::Inputs::Base < LogStash::Plugin
     super
     # There is no easy way to propage an instance variable into the codec, because the codec
     # are created at the class level
+    # TODO(talevy): Codecs should have their own execution_context, for now they will inherit their 
+    #               parent plugin's
     @codec.execution_context = context
     context
   end

data/lib/logstash/instrument/metric_type.rb

@@ -1,13 +1,11 @@
 # encoding: utf-8
 require "logstash/instrument/metric_type/counter"
-require "logstash/instrument/metric_type/mean"
 require "logstash/instrument/metric_type/gauge"
 
 module LogStash module Instrument
   module MetricType
     METRIC_TYPE_LIST = {
       :counter => LogStash::Instrument::MetricType::Counter,
-      :mean => LogStash::Instrument::MetricType::Mean,
       :gauge => LogStash::Instrument::MetricType::Gauge
     }.freeze
 

data/lib/logstash/instrument/periodic_poller/jvm.rb

@@ -2,18 +2,19 @@
 require "logstash/instrument/periodic_poller/base"
 require "logstash/instrument/periodic_poller/load_average"
 require "logstash/environment"
-require "jrmonitor"
 require "set"
 
+java_import 'com.sun.management.UnixOperatingSystemMXBean'
 java_import 'java.lang.management.ManagementFactory'
 java_import 'java.lang.management.OperatingSystemMXBean'
 java_import 'java.lang.management.GarbageCollectorMXBean'
 java_import 'java.lang.management.RuntimeMXBean'
-java_import 'com.sun.management.UnixOperatingSystemMXBean'
 java_import 'javax.management.MBeanServer'
 java_import 'javax.management.ObjectName'
 java_import 'javax.management.AttributeList'
 java_import 'javax.naming.directory.Attribute'
+java_import 'org.logstash.instrument.reports.MemoryReport'
+java_import 'org.logstash.instrument.reports.ProcessReport'
 
 
 module LogStash module Instrument module PeriodicPoller
@@ -50,7 +51,7 @@ module LogStash module Instrument module PeriodicPoller
     end
 
     def collect
-      raw = JRMonitor.memory.generate
+      raw = MemoryReport.generate
       collect_jvm_metrics(raw)
       collect_pools_metrics(raw)
       collect_threads_metrics
@@ -81,11 +82,10 @@ module LogStash module Instrument module PeriodicPoller
     end
 
     def collect_process_metrics
-      process_metrics = JRMonitor.process.generate
+      process_metrics = ProcessReport.generate
 
       path = [:jvm, :process]
 
-
       open_fds = process_metrics["open_file_descriptors"]
       if @peak_open_fds.nil? || open_fds > @peak_open_fds
         @peak_open_fds = open_fds