fluentd 0.10.45 → 0.10.46

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: df6457fef8673bd624e1baa875292fd83ee71f56
-  data.tar.gz: 829ece938ca14738720a69d9642c3f24859a595a
+  metadata.gz: f5c5c7236589cb3aa5fadf040dbee965523f6ef0
+  data.tar.gz: 392f590233abe73edab6c425556de56ab7ddd6f3
 SHA512:
-  metadata.gz: ffe86b09fb5aef434fc97fda76ba8697d48aadd8954cea28285cdb419447800e46b1df868faa549185ba7d058dae7058bec0338b87a2330eebbb8bd5e5c76539
-  data.tar.gz: 3949343e2b31f9eef4e6edc045e5e84f58adc4db9546ea30966116ef9457897dd3f291bca75260407197a6952682f69cef8da8862f49841c7582695725419737
+  metadata.gz: ea036af28812b5a21fafe9731fe93959f85c2529e88feff2347d1ebfb0724c54a93a65bd738371cc924247e43ef9c6f997dd88792893fbb94c2fbdfc52fa362d
+  data.tar.gz: 6087c6298709c8f0e32f79830c8950ab5378225c8a68ecf8ae236accdf2e08b670e51dbd7c08ac2e4397a36d704e4219a4585ceebb67ab974428cdce578e806d

data/.travis.yml

@@ -4,12 +4,13 @@ rvm:
   - 1.9.3
   - 2.0.0
   - 2.1.0
+  - 2.1.1
+  - ruby-head
   - rbx
 
 branches:
   only:
     - master
-    - v11
 
 gemfile:
   - Gemfile
@@ -19,4 +20,5 @@ script: bundle exec rake
 
 matrix:
   allow_failures:
+    - rvm: ruby-head
     - rvm: rbx

data/ChangeLog

@@ -1,3 +1,16 @@
+Release 0.10.46 - 2014/04/21
+
+* in_tail: Fix typo in rescue for dumping backtrace
+* in_forward: Add blocking_timeout option to avoid Cool.io loop stuck in shutdown
+* out_forward: Add phi_failure_detector option to disable "The Phi accrual failure detector"
+* buf_memory: Add flush_at_shutdown option to discard buffered event at shutdown
+* parser: Use Engine.now when time_key is missing in ValuesParser
+* parser: TextParser loads fluent/plugin/parser_xxx.rb automatically when 'xxx' format not found
+* Fix match cache expiration
+* Add new v1 configuration format and --use-v1-config option to enable v1 configuration format
+* Add config_section to Configurable to support nested parameters in plugins
+* Relax http_parser.rb gem version to support 0.6 series
+
 Release 0.10.45 - 2014/03/28
 
 * in_tail: Merge in_tail_ex and in_tail_multiline features

data/Rakefile

@@ -4,12 +4,15 @@ require "bundler/gem_tasks"
 require 'fileutils'
 require 'rake/testtask'
 require 'rake/clean'
+require 'rspec/core'
+require 'rspec/core/rake_task'
 
-task :test => [:base_test]
+task :test => [:base_test, :spec]
 
+desc 'Run test_unit based test'
 Rake::TestTask.new(:base_test) do |t|
   t.libs << "test"
-  t.test_files = (Dir["test/*.rb"] + Dir["test/plugin/*.rb"] - ["helper.rb"]).sort
+  t.test_files = (Dir["test/test_*.rb"] + Dir["test/plugin/test_*.rb"] - ["helper.rb"]).sort
   t.verbose = true
   #t.warning = true
 end
@@ -20,4 +23,17 @@ task :parallel_test do
   FileUtils.rm_rf('./test/tmp')
 end
 
+desc 'Run rspec based test'
+RSpec::Core::RakeTask.new(:spec) do |t|
+  t.rspec_opts = %w[-c -f progress -r ./spec/spec_helper.rb]
+  t.pattern = 'spec/**/*_spec.rb'
+  t.verbose = true
+end
+
+desc 'Run rspec with simplecov'
+task :coverage do |t|
+  ENV['SIMPLE_COV'] = '1'
+  Rake::Task["spec"].invoke
+end
+
 task :default => [:test, :build]

data/fluentd.gemspec

@@ -22,12 +22,14 @@ Gem::Specification.new do |gem|
   gem.add_runtime_dependency("json", [">= 1.4.3"])
   gem.add_runtime_dependency("yajl-ruby", ["~> 1.0"])
   gem.add_runtime_dependency("cool.io", [">= 1.1.1", "!= 1.2.0", "< 2.0.0"])
-  gem.add_runtime_dependency("http_parser.rb", ["~> 0.5.1"])
+  gem.add_runtime_dependency("http_parser.rb", [">= 0.5.1", "< 0.7.0"])
   gem.add_runtime_dependency("sigdump", ["~> 0.2.2"])
 
   gem.add_development_dependency("rake", [">= 0.9.2"])
   gem.add_development_dependency("flexmock")
   gem.add_development_dependency("parallel_tests", [">= 0.15.3"])
+  gem.add_development_dependency("rspec", ["~> 2.13"])
+  gem.add_development_dependency("simplecov", ["~> 0.6.4"])
   gem.add_development_dependency("rr", [">= 1.0.0"])
   gem.add_development_dependency("timecop", [">= 0.3.0"])
 end

data/lib/fluent/command/fluentd.rb

@@ -37,6 +37,7 @@ opts = {
   :chgroup => nil,
   :suppress_interval => 0,
   :suppress_repeated_stacktrace => false,
+  :use_v1_config => false,
 }
 
 op.on('-s', "--setup [DIR=#{File.dirname(Fluent::DEFAULT_CONFIG_PATH)}]", "install sample configuration file to the directory") {|s|
@@ -91,6 +92,10 @@ op.on('--suppress-repeated-stacktrace', "suppress repeated stacktrace", TrueClas
   opts[:suppress_repeated_stacktrace] = b
 }
 
+op.on('--use-v1-config', "Use v1 configuration format", TrueClass) {|b|
+  opts[:use_v1_config] = b
+}
+
 op.on('-v', '--verbose', "increase verbose level (-v: debug, -vv: trace)", TrueClass) {|b|
   if b
     opts[:log_level] = [opts[:log_level] - 1, Fluent::Log::LEVEL_TRACE].max

data/lib/fluent/config.rb

@@ -16,351 +16,36 @@
 #    limitations under the License.
 #
 module Fluent
-  class ConfigError < StandardError
-  end
-
-  class ConfigParseError < ConfigError
-  end
-
+  require 'fluent/config/error'
+  require 'fluent/config/element'
 
   module Config
-    class Element < Hash
-      def initialize(name, arg, attrs, elements, unused=nil)
-        @name = name
-        @arg = arg
-        @elements = elements
-        super()
-        attrs.each {|k,v|
-          self[k] = v
-        }
-        @unused = unused || attrs.keys
-      end
-
-      attr_accessor :name, :arg, :elements, :unused
-
-      def add_element(name, arg='')
-        e = Element.new(name, arg, {}, [])
-        @elements << e
-        e
-      end
-
-      def +(o)
-        Element.new(@name.dup, @arg.dup, o.merge(self), @elements+o.elements, (@unused+o.unused).uniq)
-      end
-
-      def has_key?(key)
-        @unused.delete(key)
-        super
-      end
-
-      def [](key)
-        @unused.delete(key)
-        super
-      end
-
-      def check_not_fetched(&block)
-        each_key {|key|
-          if @unused.include?(key)
-            block.call(key, self)
-          end
-        }
-        @elements.each {|e|
-          e.check_not_fetched(&block)
-        }
-      end
-
-      def to_s(nest = 0)
-        indent = "  "*nest
-        nindent = "  "*(nest+1)
-        out = ""
-        if @arg.empty?
-          out << "#{indent}<#{@name}>\n"
-        else
-          out << "#{indent}<#{@name} #{@arg}>\n"
-        end
-        each_pair {|k,v|
-          out << "#{nindent}#{k} #{v}\n"
-        }
-        @elements.each {|e|
-          out << e.to_s(nest+1)
-        }
-        out << "#{indent}</#{@name}>\n"
-        out
-      end
-    end
-
-    def self.read(path)
-      Parser.read(path)
-    end
-
-    def self.parse(str, fname, basepath=Dir.pwd)
-      Parser.parse(str, fname, basepath)
-    end
-
-    def self.new(name='')
-      Element.new('', '', {}, [])
-    end
-
-    def self.size_value(str)
-      case str.to_s
-      when /([0-9]+)k/i
-        $~[1].to_i * 1024
-      when /([0-9]+)m/i
-        $~[1].to_i * (1024**2)
-      when /([0-9]+)g/i
-        $~[1].to_i * (1024**3)
-      when /([0-9]+)t/i
-        $~[1].to_i * (1024**4)
-      else
-        str.to_i
-      end
-    end
-
-    def self.time_value(str)
-      case str.to_s
-      when /([0-9]+)s/
-        $~[1].to_i
-      when /([0-9]+)m/
-        $~[1].to_i * 60
-      when /([0-9]+)h/
-        $~[1].to_i * 60*60
-      when /([0-9]+)d/
-        $~[1].to_i * 24*60*60
+    def self.read(path, v1_config = false)
+      if v1_config
+        require 'fluent/config/v1_parser'
+        V1Parser.read(path)
       else
-        str.to_f
+        require 'fluent/config/parser'
+        Parser.read(path)
       end
     end
 
-    def self.bool_value(str)
-      case str.to_s
-      when 'true', 'yes'
-        true
-      when 'false', 'no'
-        false
+    def self.parse(str, fname, basepath = Dir.pwd, v1_config = false)
+      if v1_config
+        require 'fluent/config/v1_parser'
+        V1Parser.parse(str, fname, basepath, Kernel.binding)
       else
-        nil
-      end
-    end
-
-    private
-    class Parser
-      def self.read(path)
-        path = File.expand_path(path)
-        File.open(path) {|io|
-          parse(io, File.basename(path), File.dirname(path))
-        }
-      end
-
-      def self.parse(io, fname, basepath=Dir.pwd)
-        attrs, elems = Parser.new(basepath, io.each_line, fname).parse!(true)
-        Element.new('ROOT', '', attrs, elems)
-      end
-
-      def initialize(basepath, iterator, fname, i=0)
-        @basepath = basepath
-        @iterator = iterator
-        @i = i
-        @fname = fname
-      end
-
-      def parse!(allow_include, elem_name=nil, attrs={}, elems=[])
-        while line = @iterator.next
-          line.force_encoding('UTF-8')
-          @i += 1
-          line.lstrip!
-          line.gsub!(/\s*(?:\#.*)?$/,'')
-          if line.empty?
-            next
-          elsif m = /^\<include\s*(.*)\s*\/\>$/.match(line)
-            value = m[1].strip
-            process_include(attrs, elems, value, allow_include)
-          elsif m = /^\<([a-zA-Z0-9_]+)\s*(.+?)?\>$/.match(line)
-            e_name = m[1]
-            e_arg = m[2] || ""
-            e_attrs, e_elems = parse!(false, e_name)
-            elems << Element.new(e_name, e_arg, e_attrs, e_elems)
-          elsif line == "</#{elem_name}>"
-            break
-          elsif m = /^([a-zA-Z0-9_]+)\s*(.*)$/.match(line)
-            key = m[1]
-            value = m[2]
-            if allow_include && key == 'include'
-              process_include(attrs, elems, value)
-            else
-              attrs[key] = value
-            end
-            next
-          else
-            raise ConfigParseError, "parse error at #{@fname} line #{@i}"
-          end
-        end
-
-        return attrs, elems
-      rescue StopIteration
-        return attrs, elems
+        require 'fluent/config/parser'
+        Parser.parse(str, fname, basepath)
       end
-
-      def process_include(attrs, elems, uri, allow_include = true)
-        u = URI.parse(uri)
-        if u.scheme == 'file' || u.path == uri  # file path
-          path = u.path
-          if path[0] != ?/
-            pattern = File.expand_path("#{@basepath}/#{path}")
-          else
-            pattern = path
-          end
-
-          Dir.glob(pattern).sort.each {|path|
-            basepath = File.dirname(path)
-            fname = File.basename(path)
-            File.open(path) {|f|
-              Parser.new(basepath, f.each_line, fname).parse!(allow_include, nil, attrs, elems)
-            }
-          }
-
-        else
-          basepath = '/'
-          fname = path
-          require 'open-uri'
-          open(uri) {|f|
-            Parser.new(basepath, f.each_line, fname).parse!(allow_include, nil, attrs, elems)
-          }
-        end
-
-      rescue SystemCallError
-        raise ConfigParseError, "include error at #{@fname} line #{@i}: #{$!.to_s}"
-      end
-    end
-  end
-
-
-  module Configurable
-    attr_reader :config
-
-    def self.included(mod)
-      mod.extend(ClassMethods)
     end
 
-    def initialize
-      self.class.config_defaults.each_pair {|name,defval|
-        varname = :"@#{name}"
-        instance_variable_set(varname, defval)
-      }
-    end
-
-    def configure(conf)
-      @config = conf
-
-      self.class.config_params.each_pair {|name,(block,opts)|
-        varname = :"@#{name}"
-        if val = conf[name.to_s]
-          val = self.instance_exec(val, opts, name, &block)
-          instance_variable_set(varname, val)
-        end
-        unless instance_variable_defined?(varname)
-          $log.error "config error in:\n#{conf}"
-          raise ConfigError, "'#{name}' parameter is required"
-        end
-      }
-    end
-
-    module ClassMethods
-      def config_param(name, *args, &block)
-        name = name.to_sym
-
-        opts = {}
-        args.each {|a|
-          if a.is_a?(Symbol)
-            opts[:type] = a
-          elsif a.is_a?(Hash)
-            opts.merge!(a)
-          else
-            raise ArgumentError, "wrong number of arguments (#{1+args.length} for #{block ? 2 : 3})"
-          end
-        }
-
-        type = opts[:type]
-        if block && type
-          raise ArgumentError, "wrong number of arguments (#{1+args.length} for #{block ? 2 : 3})"
-        end
-
-        block ||= case type
-                  when :string, nil
-                    Proc.new {|val| val }
-                  when :integer
-                    Proc.new {|val| val.to_i }
-                  when :float
-                    Proc.new {|val| val.to_f }
-                  when :size
-                    Proc.new {|val| Config.size_value(val) }
-                  when :bool
-                    Proc.new {|val| Config.bool_value(val) }
-                  when :time
-                    Proc.new {|val| Config.time_value(val) }
-                  else
-                    raise ArgumentError, "unknown config_param type `#{type}'"
-                  end
-
-        params = config_params_set
-        params.delete(name)
-        params[name] = [block, opts]
-
-        if opts.has_key?(:default)
-          config_set_default(name, opts[:default])
-        end
-
-        attr_accessor name
-      end
-
-      def config_set_default(name, defval)
-        name = name.to_sym
-
-        defaults = config_defaults_set
-        defaults.delete(name)
-        defaults[name] = defval
-
-        nil
-      end
-
-      def config_params
-        singleton_value(:_config_params)
-      end
-
-      def config_defaults
-        singleton_value(:_config_defaults)
-      end
-
-      private
-      def config_params_set
-        singleton_value_set(:_config_params)
-      end
-
-      def config_defaults_set
-        singleton_value_set(:_config_defaults)
-      end
-
-      def singleton_value_set(name)
-        if methods(false).include?(name)
-          __send__(name)
-        else
-          val = {}
-          define_singleton_method(name) { val }
-          val
-        end
-      end
-
-      def singleton_value(name)
-        val = {}
-        ancestors.reverse_each {|c|
-          if c.methods(false).include?(name)
-            val.merge!(c.__send__(name))
-          end
-        }
-        val
-      end
+    def self.new(name = '')
+      Element.new(name, '', {}, [])
     end
   end
 
+  require 'fluent/configurable'
 
   module PluginId
     def configure(conf)
@@ -380,4 +65,3 @@ module Fluent
     end
   end
 end
-