fluentd 0.10.45 → 0.10.46

data/spec/config/configure_proxy_spec.rb

@@ -0,0 +1,96 @@
+require 'fluent/config/configure_proxy'
+
+describe Fluent::Config::ConfigureProxy do
+  context 'to generate a instance' do
+    describe '#initialize' do
+      it 'has default values' do
+        proxy = Fluent::Config::ConfigureProxy.new('section')
+        expect(proxy.name).to eql(:section)
+
+        proxy = Fluent::Config::ConfigureProxy.new(:section)
+        expect(proxy.name).to eql(:section)
+        expect(proxy.param_name).to eql(:section)
+        expect(proxy.required).to be_nil
+        expect(proxy.required?).to be_false
+        expect(proxy.multi).to be_nil
+        expect(proxy.multi?).to be_true
+      end
+
+      it 'can specify param_name/required/multi with optional arguments' do
+        proxy = Fluent::Config::ConfigureProxy.new(:section, param_name: 'sections', required: false, multi: true)
+        expect(proxy.name).to eql(:section)
+        expect(proxy.param_name).to eql(:sections)
+        expect(proxy.required).to be_false
+        expect(proxy.required?).to be_false
+        expect(proxy.multi).to be_true
+        expect(proxy.multi?).to be_true
+
+        proxy = Fluent::Config::ConfigureProxy.new(:section, param_name: :sections, required: true, multi: false)
+        expect(proxy.name).to eql(:section)
+        expect(proxy.param_name).to eql(:sections)
+        expect(proxy.required).to be_true
+        expect(proxy.required?).to be_true
+        expect(proxy.multi).to be_false
+        expect(proxy.multi?).to be_false
+      end
+    end
+
+    describe '#merge' do
+      it 'generate a new instance which values are overwritten by the argument object' do
+        proxy = p1 = Fluent::Config::ConfigureProxy.new(:section)
+        expect(proxy.name).to eql(:section)
+        expect(proxy.param_name).to eql(:section)
+        expect(proxy.required).to be_nil
+        expect(proxy.required?).to be_false
+        expect(proxy.multi).to be_nil
+        expect(proxy.multi?).to be_true
+
+        p2 = Fluent::Config::ConfigureProxy.new(:section, param_name: :sections, required: true, multi: false)
+        proxy = p1.merge(p2)
+        expect(proxy.name).to eql(:section)
+        expect(proxy.param_name).to eql(:sections)
+        expect(proxy.required).to be_true
+        expect(proxy.required?).to be_true
+        expect(proxy.multi).to be_false
+        expect(proxy.multi?).to be_false
+      end
+
+      it 'does not overwrite with argument object without any specifications of required/multi' do
+        p1 = Fluent::Config::ConfigureProxy.new(:section1)
+        p2 = Fluent::Config::ConfigureProxy.new(:section2, param_name: :sections, required: true, multi: false)
+        p3 = Fluent::Config::ConfigureProxy.new(:section3)
+        proxy = p1.merge(p2).merge(p3)
+        expect(proxy.name).to eql(:section3)
+        expect(proxy.param_name).to eql(:section3)
+        expect(proxy.required).to be_true
+        expect(proxy.required?).to be_true
+        expect(proxy.multi).to be_false
+        expect(proxy.multi?).to be_false
+      end
+    end
+
+    describe '#config_param / #config_set_default / #config_argument' do
+      it 'does not permit config_set_default for param w/ :default option' do
+        proxy = Fluent::Config::ConfigureProxy.new(:section)
+        proxy.config_param(:name, :string, default: "name1")
+        expect{ proxy.config_set_default(:name, "name2") }.to raise_error(ArgumentError)
+      end
+
+      it 'does not permit default value specification twice' do
+        proxy = Fluent::Config::ConfigureProxy.new(:section)
+        proxy.config_param(:name, :string)
+        proxy.config_set_default(:name, "name1")
+        expect{ proxy.config_set_default(:name, "name2") }.to raise_error(ArgumentError)
+      end
+
+      it 'does not permit default value specification twice, even on config_argument' do
+        proxy = Fluent::Config::ConfigureProxy.new(:section)
+        proxy.config_param(:name, :string)
+        proxy.config_set_default(:name, "name1")
+
+        proxy.config_argument(:name)
+        expect{ proxy.config_argument(:name, default: "name2") }.to raise_error(ArgumentError)
+      end
+    end
+  end
+end

data/spec/config/dsl_spec.rb

@@ -0,0 +1,239 @@
+require_relative "./helper"
+
+require 'fluent/config/element'
+require "fluent/config/dsl"
+
+DSL_CONFIG_EXAMPLE = %q[
+worker {
+  hostname = "myhostname"
+
+  (0..9).each { |i|
+    source {
+      type :tail
+      path "/var/log/httpd/access.part#{i}.log"
+
+      filter ('bar.**') {
+        type :hoge
+        val1 "moge"
+        val2 ["foo", "bar", "baz"]
+        val3 10
+        id :hoge
+
+        subsection {
+          foo "bar"
+        }
+        subsection {
+          foo "baz"
+        }
+      }
+
+      filter ('foo.**') {
+        type "pass"
+      }
+
+      match ('{foo,bar}.**') {
+        type "file"
+        path "/var/log/httpd/access.#{hostname}.#{i}.log"
+      }
+    }
+  }
+}
+]
+
+DSL_CONFIG_EXAMPLE_WITHOUT_WORKER = %q[
+hostname = "myhostname"
+
+source {
+  type :tail
+  path "/var/log/httpd/access.part.log"
+
+  element {
+    name "foo"
+  }
+
+  match ('{foo,bar}.**') {
+    type "file"
+    path "/var/log/httpd/access.full.log"
+  }
+}
+]
+
+DSL_CONFIG_RETURNS_NON_ELEMENT = %q[
+worker {
+}
+[]
+]
+
+DSL_CONFIG_WRONG_SYNTAX1 = %q[
+match
+]
+DSL_CONFIG_WRONG_SYNTAX2 = %q[
+match('aa','bb'){
+  type :null
+}
+]
+DSL_CONFIG_WRONG_SYNTAX3 = %q[
+match('aa','bb')
+]
+
+describe Fluent::Config::DSL::Parser do
+  include_context 'config_helper'
+
+  context 'with worker tag on top level' do
+    root = nil
+
+    describe '.parse' do
+      it 'makes root element' do
+        root = Fluent::Config::DSL::Parser.parse(DSL_CONFIG_EXAMPLE, 'dsl_config.rb')
+
+        expect(root.name).to eql('ROOT')
+        expect(root.arg).to be_empty
+        expect(root.keys.size).to eql(0)
+      end
+
+      it 'makes worker element for worker tag' do
+        expect(root.elements.size).to eql(1)
+
+        worker = root.elements.first
+
+        expect(worker.name).to eql('worker')
+        expect(worker.arg).to be_empty
+        expect(worker.keys.size).to eql(0)
+        expect(worker.elements.size).to eql(10)
+      end
+
+      it 'makes subsections for blocks, with variable substitution' do
+        ele4 = root.elements.first.elements[4]
+
+        expect(ele4.name).to eql('source')
+        expect(ele4.arg).to be_empty
+        expect(ele4.keys.size).to eql(2)
+        expect(ele4['type']).to eql('tail')
+        expect(ele4['path']).to eql("/var/log/httpd/access.part4.log")
+      end
+
+      it 'makes user-defined sections with blocks' do
+        filter0 = root.elements.first.elements[4].elements.first
+
+        expect(filter0.name).to eql('filter')
+        expect(filter0.arg).to eql('bar.**')
+        expect(filter0['type']).to eql('hoge')
+        expect(filter0['val1']).to eql('moge')
+        expect(filter0['val2']).to eql(JSON.dump(['foo', 'bar', 'baz']))
+        expect(filter0['val3']).to eql('10')
+        expect(filter0['id']).to eql('hoge')
+
+        expect(filter0.elements.size).to eql(2)
+        expect(filter0.elements[0].name).to eql('subsection')
+        expect(filter0.elements[0]['foo']).to eql('bar')
+        expect(filter0.elements[1].name).to eql('subsection')
+        expect(filter0.elements[1]['foo']).to eql('baz')
+      end
+
+      it 'makes values with user-assigned variable substitutions' do
+        match0 = root.elements.first.elements[4].elements.last
+
+        expect(match0.name).to eql('match')
+        expect(match0.arg).to eql('{foo,bar}.**')
+        expect(match0['type']).to eql('file')
+        expect(match0['path']).to eql('/var/log/httpd/access.myhostname.4.log')
+      end
+    end
+  end
+
+  context 'without worker tag on top level' do
+    root = nil
+
+    describe '.parse' do
+      it 'makes root element' do
+        root = Fluent::Config::DSL::Parser.parse(DSL_CONFIG_EXAMPLE_WITHOUT_WORKER, 'dsl_config_without_worker.rb')
+
+        expect(root.name).to eql('ROOT')
+        expect(root.arg).to be_empty
+        expect(root.keys.size).to eql(0)
+      end
+
+      it 'does not make worker element implicitly because DSL configuration does not support v10 compat mode' do
+        expect(root.elements.size).to eql(1)
+        expect(root.elements.first.name).to eql('source')
+        expect(root.elements.find{|e| e.name == 'worker'}).to be_false
+      end
+    end
+  end
+
+  context 'with configuration that returns non element on top' do
+    describe '.parse' do
+      it 'does not crash' do
+        root = Fluent::Config::DSL::Parser.parse(DSL_CONFIG_RETURNS_NON_ELEMENT, 'dsl_config_returns_non_element.rb')
+      end
+    end
+  end
+
+  context 'with configuration with wrong arguments for specific elements' do
+    describe '.parse' do
+      it 'raises ArgumentError correctly' do
+        expect{ Fluent::Config::DSL::Parser.parse(DSL_CONFIG_WRONG_SYNTAX1, 'dsl_config_wrong_syntax1') }.to raise_error(ArgumentError)
+        expect{ Fluent::Config::DSL::Parser.parse(DSL_CONFIG_WRONG_SYNTAX2, 'dsl_config_wrong_syntax1') }.to raise_error(ArgumentError)
+        expect{ Fluent::Config::DSL::Parser.parse(DSL_CONFIG_WRONG_SYNTAX3, 'dsl_config_wrong_syntax1') }.to raise_error(ArgumentError)
+      end
+    end
+  end
+
+  context 'with ruby keyword, that provides ruby Kernel module features' do
+    describe '.parse' do
+      it 'can get result of Kernel.open() by ruby.open()' do
+        uname_string = `uname -a`
+        root = Fluent::Config::DSL::Parser.parse(<<DSL)
+worker {
+  uname_str = ruby.open('|uname -a'){|out| out.read}
+  source {
+    uname uname_str
+  }
+}
+DSL
+        worker = root.elements.first
+        expect(worker.name).to eql('worker')
+        source = worker.elements.first
+        expect(source.name).to eql('source')
+        expect(source.keys.size).to eql(1)
+        expect(source['uname']).to eql(uname_string)
+      end
+
+      it 'accepts ruby keyword with block, which allow to use methods included from ::Kernel' do
+        root = Fluent::Config::DSL::Parser.parse(<<DSL)
+worker {
+  ruby_version = ruby {
+    require 'erb'
+    ERB.new('<%= RUBY_VERSION %> from erb').result
+  }
+  source {
+    version ruby_version
+  }
+}
+DSL
+        worker = root.elements.first
+        expect(worker.name).to eql('worker')
+        source = worker.elements.first
+        expect(source.name).to eql('source')
+        expect(source.keys.size).to eql(1)
+        expect(source['version']).to eql("#{RUBY_VERSION} from erb")
+      end
+
+      it 'raises NoMethodError when configuration DSL elements are written in ruby block' do
+        conf = <<DSL
+worker {
+  ruby {
+    source {
+      type "tail"
+    }
+  }
+  source {
+    uname uname_str
+  }
+}
+DSL
+        expect{ Fluent::Config::DSL::Parser.parse(conf) }.to raise_error(NoMethodError)
+      end
+    end
+  end
+end

data/spec/config/helper.rb

@@ -0,0 +1,50 @@
+shared_context 'config_helper' do
+  RSpec::Matchers.define :be_parsed_as do |obj|
+    match do |text|
+      v = parse_text(text)
+      if obj.is_a?(Float)
+        v.is_a?(Float) && (v == obj || (v.nan? && obj.nan?) || (v - obj).abs < 0.000001)
+      else
+        v == obj
+      end
+    end
+
+    failure_message_for_should do |text|
+      msg = parse_text(text).inspect rescue 'failed'
+      "expected that #{text.inspect} would be a parsed as #{obj.inspect} but got #{msg}"
+    end
+  end
+
+  RSpec::Matchers.define :be_parsed_as_json do |obj|
+    match do |text|
+      v = JSON.parse(parse_text(text))
+      v == obj
+    end
+
+    failure_message_for_should do |text|
+      msg = parse_text(text).inspect rescue 'failed'
+      "expected that #{text.inspect} would be a parsed as #{obj.inspect} but got #{msg}"
+    end
+  end
+
+  RSpec::Matchers.define :be_parse_error do |obj|
+    match do |text|
+      begin
+        parse_text(text)
+        false
+      rescue Fluent::ConfigParseError
+        true
+      end
+    end
+
+    failure_message_for_should do |text|
+      begin
+        msg = parse_text(text).inspect
+      rescue
+        msg = $!.inspect
+      end
+      "expected that #{text.inspect} would cause a parse error but got #{msg}"
+    end
+  end
+end
+

data/spec/config/literal_parser_spec.rb

@@ -0,0 +1,190 @@
+require "config/helper"
+require "fluent/config/error"
+require "fluent/config/literal_parser"
+require "fluent/config/v1_parser"
+
+describe Fluent::Config::LiteralParser do
+  include_context 'config_helper'
+
+  TestLiteralParserContext = Struct.new(:v1, :v2, :v3)
+
+  let(:v1) { :test }
+  let(:v2) { true }
+  let(:v3) { nil }
+
+  let(:eval_context) { TestLiteralParserContext.new(v1, v2, v3) }
+
+  def parse_text(text)
+    basepath = File.expand_path(File.dirname(__FILE__)+'/../../')
+    ss = StringScanner.new(text)
+    parser = Fluent::Config::V1Parser.new(ss, basepath, "(test)", eval_context)
+    parser.parse_literal
+  end
+
+  describe 'boolean parsing' do
+    it { 'true'.should be_parsed_as("true") }
+    it { 'false'.should be_parsed_as("false") }
+    it { 'trueX'.should be_parsed_as("trueX") }
+    it { 'falseX'.should be_parsed_as("falseX") }
+  end
+
+  describe 'integer parsing' do
+    it { '0'.should be_parsed_as("0") }
+    it { '1'.should be_parsed_as("1") }
+    it { '10'.should be_parsed_as("10") }
+    it { '-1'.should be_parsed_as("-1") }
+    it { '-10'.should be_parsed_as("-10") }
+    it { '0 '.should be_parsed_as("0") }
+    it { ' -1 '.should be_parsed_as("-1") }
+    # string
+    it { '01'.should be_parsed_as("01") }
+    it { '00'.should be_parsed_as("00") }
+    it { '-01'.should be_parsed_as("-01") }
+    it { '-00'.should be_parsed_as("-00") }
+    it { '0x61'.should be_parsed_as("0x61") }
+    it { '0s'.should be_parsed_as("0s") }
+  end
+
+  describe 'float parsing' do
+    it { '1.1'.should be_parsed_as("1.1") }
+    it { '0.1'.should be_parsed_as("0.1") }
+    it { '0.0'.should be_parsed_as("0.0") }
+    it { '-1.1'.should be_parsed_as("-1.1") }
+    it { '-0.1'.should be_parsed_as("-0.1") }
+    it { '1.10'.should be_parsed_as("1.10") }
+    # string
+    it { '12e8'.should be_parsed_as("12e8") }
+    it { '12.1e7'.should be_parsed_as("12.1e7") }
+    it { '-12e8'.should be_parsed_as("-12e8") }
+    it { '-12.1e7'.should be_parsed_as("-12.1e7") }
+    it { '.0'.should be_parsed_as(".0") }
+    it { '.1'.should be_parsed_as(".1") }
+    it { '0.'.should be_parsed_as("0.") }
+    it { '1.'.should be_parsed_as("1.") }
+    it { '.0a'.should be_parsed_as(".0a") }
+    it { '1.a'.should be_parsed_as("1.a") }
+    it { '0@'.should be_parsed_as("0@") }
+  end
+
+  describe 'float keywords parsing' do
+    it { 'NaN'.should be_parsed_as("NaN") }
+    it { 'Infinity'.should be_parsed_as("Infinity") }
+    it { '-Infinity'.should be_parsed_as("-Infinity") }
+    it { 'NaNX'.should be_parsed_as("NaNX") }
+    it { 'InfinityX'.should be_parsed_as("InfinityX") }
+    it { '-InfinityX'.should be_parsed_as("-InfinityX") }
+  end
+
+  describe 'quoted string' do
+    it { '""'.should be_parsed_as("") }
+    it { '"text"'.should be_parsed_as("text") }
+    it { '"\\""'.should be_parsed_as("\"") }
+    it { '"\\t"'.should be_parsed_as("\t") }
+    it { '"\\n"'.should be_parsed_as("\n") }
+    it { '"\\r\\n"'.should be_parsed_as("\r\n") }
+    it { '"\\f\\b"'.should be_parsed_as("\f\b") }
+    it { '"\\.t"'.should be_parsed_as(".t") }
+    it { '"\\$t"'.should be_parsed_as("$t") }
+    it { '"\\#t"'.should be_parsed_as("#t") }
+    it { '"\\z"'.should be_parse_error }  # unknown escaped character
+    it { '"\\0"'.should be_parse_error }  # unknown escaped character
+    it { '"\\1"'.should be_parse_error }  # unknown escaped character
+    it { '"t'.should be_parse_error }  # non-terminated quoted character
+    it { 't"'.should be_parsed_as('t"') }
+    it { '"."'.should be_parsed_as('.') }
+    it { '"*"'.should be_parsed_as('*') }
+    it { '"@"'.should be_parsed_as('@') }
+    it { '"\\#{test}"'.should be_parsed_as("\#{test}") }
+    it { '"$"'.should be_parsed_as('$') }
+    it { '"$t"'.should be_parsed_as('$t') }
+    it { '"$}"'.should be_parsed_as('$}') }
+  end
+
+  describe 'nonquoted string parsing' do
+    # empty
+    it { ''.should be_parsed_as(nil) }
+
+    it { 't'.should be_parsed_as('t') }
+    it { 'T'.should be_parsed_as('T') }
+    it { '_'.should be_parsed_as('_') }
+    it { 'T1'.should be_parsed_as('T1') }
+    it { '_2'.should be_parsed_as('_2') }
+    it { 't0'.should be_parsed_as('t0') }
+    it { 't@'.should be_parsed_as('t@') }
+    it { 't-'.should be_parsed_as('t-') }
+    it { 't.'.should be_parsed_as('t.') }
+    it { 't+'.should be_parsed_as('t+') }
+    it { 't/'.should be_parsed_as('t/') }
+    it { 't='.should be_parsed_as('t=') }
+    it { 't,'.should be_parsed_as('t,') }
+    it { '0t'.should be_parsed_as("0t") }
+    it { '@1t'.should be_parsed_as('@1t') }
+    it { '-1t'.should be_parsed_as('-1t') }
+    it { '.1t'.should be_parsed_as('.1t') }
+    it { ',1t'.should be_parsed_as(',1t') }
+    it { '.t'.should be_parsed_as('.t') }
+    it { '*t'.should be_parsed_as('*t') }
+    it { '@t'.should be_parsed_as('@t') }
+    it { '$t'.should be_parsed_as('$t') }
+    it { '{t'.should be_parse_error }  # '{' begins map
+    it { 't{'.should be_parsed_as('t{') }
+    it { '}t'.should be_parsed_as('}t') }
+    it { '[t'.should be_parse_error }  # '[' begins array
+    it { 't['.should be_parsed_as('t[') }
+    it { ']t'.should be_parsed_as(']t') }
+    it { '$t'.should be_parsed_as('$t') }
+    it { 't:'.should be_parsed_as('t:') }
+    it { 't;'.should be_parsed_as('t;') }
+    it { 't?'.should be_parsed_as('t?') }
+    it { 't^'.should be_parsed_as('t^') }
+    it { 't`'.should be_parsed_as('t`') }
+    it { 't~'.should be_parsed_as('t~') }
+    it { 't|'.should be_parsed_as('t|') }
+    it { 't>'.should be_parsed_as('t>') }
+    it { 't<'.should be_parsed_as('t<') }
+    it { 't('.should be_parsed_as('t(') }
+    it { 't['.should be_parsed_as('t[') }
+  end
+
+  describe 'embedded ruby code parsing' do
+    it { '"#{v1}"'.should be_parsed_as("#{v1}") }
+    it { '"#{v2}"'.should be_parsed_as("#{v2}") }
+    it { '"#{v3}"'.should be_parsed_as("#{v3}") }
+    it { '"#{1+1}"'.should be_parsed_as("2") }
+    it { '"#{}"'.should be_parsed_as("") }
+    it { '"t#{v1}"'.should be_parsed_as("t#{v1}") }
+    it { '"t#{v1}t"'.should be_parsed_as("t#{v1}t") }
+    it { '"#{"}"}"'.should be_parsed_as("}") }
+    it { '"#{#}"'.should be_parse_error }  # error in embedded ruby code
+    it { "\"\#{\n=begin\n}\"".should be_parse_error }  # error in embedded ruby code
+  end
+
+  describe 'array parsing' do
+    it { '[]'.should be_parsed_as_json([]) }
+    it { '[1]'.should be_parsed_as_json([1]) }
+    it { '[1,2]'.should be_parsed_as_json([1,2]) }
+    it { '[1, 2]'.should be_parsed_as_json([1,2]) }
+    it { '[ 1 , 2 ]'.should be_parsed_as_json([1,2]) }
+    it { '[1,2,]'.should be_parse_error } # TODO: Need trailing commas support?
+    it { "[\n1\n,\n2\n]".should be_parsed_as_json([1,2]) }
+    it { '["a"]'.should be_parsed_as_json(["a"]) }
+    it { '["a","b"]'.should be_parsed_as_json(["a","b"]) }
+    it { '[ "a" , "b" ]'.should be_parsed_as_json(["a","b"]) }
+    it { "[\n\"a\"\n,\n\"b\"\n]".should be_parsed_as_json(["a","b"]) }
+    it { '["ab","cd"]'.should be_parsed_as_json(["ab","cd"]) }
+  end
+
+  describe 'map parsing' do
+    it { '{}'.should be_parsed_as_json({}) }
+    it { '{"a":1}'.should be_parsed_as_json({"a"=>1}) }
+    it { '{"a":1,"b":2}'.should be_parsed_as_json({"a"=>1,"b"=>2}) }
+    it { '{ "a" : 1 , "b" : 2 }'.should be_parsed_as_json({"a"=>1,"b"=>2}) }
+    it { '{"a":1,"b":2,}'.should be_parse_error } # TODO: Need trailing commas support?
+    it { "{\n\"a\"\n:\n1\n,\n\"b\"\n:\n2\n}".should be_parsed_as_json({"a"=>1,"b"=>2}) }
+    it { '{"a":"b"}'.should be_parsed_as_json({"a"=>"b"}) }
+    it { '{"a":"b","c":"d"}'.should be_parsed_as_json({"a"=>"b","c"=>"d"}) }
+    it { '{ "a" : "b" , "c" : "d" }'.should be_parsed_as_json({"a"=>"b","c"=>"d"}) }
+    it { "{\n\"a\"\n:\n\"b\"\n,\n\"c\"\n:\n\"d\"\n}".should be_parsed_as_json({"a"=>"b","c"=>"d"}) }
+  end
+end
+