fluentd 0.10.62 → 0.12.0.pre.1

data/lib/fluent/plugin/in_dummy.rb

@@ -1,103 +0,0 @@
-#
-# Fluentd
-#
-#    Licensed under the Apache License, Version 2.0 (the "License");
-#    you may not use this file except in compliance with the License.
-#    You may obtain a copy of the License at
-#
-#        http://www.apache.org/licenses/LICENSE-2.0
-#
-#    Unless required by applicable law or agreed to in writing, software
-#    distributed under the License is distributed on an "AS IS" BASIS,
-#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#    See the License for the specific language governing permissions and
-#    limitations under the License.
-#
-
-module Fluent
-  class DummyInput < Input
-    Fluent::Plugin.register_input('dummy', self)
-
-    BIN_NUM = 10
-
-    config_param :tag, :string
-    config_param :rate, :integer, :default => 1
-    config_param :auto_increment_key, :string, :default => nil
-    config_param :dummy, :default => [{"message"=>"dummy"}] do |val|
-      begin
-        parsed = JSON.parse(val)
-      rescue JSON::ParserError => e
-        # Fluent::ConfigParseError, "got incomplete JSON" will be raised
-        # at literal_parser.rb with --use-v1-config, but I had to
-        # take care at here for the case of --use-v0-config.
-        raise Fluent::ConfigError, "#{e.class}: #{e.message}"
-      end
-      dummy = parsed.is_a?(Array) ? parsed : [parsed]
-      dummy.each_with_index do |e, i|
-        raise Fluent::ConfigError, "#{i}th element of dummy, #{e}, is not a hash" unless e.is_a?(Hash)
-      end
-      dummy
-    end
-
-    def configure(conf)
-      super
-
-      @increment_value = 0
-      @dummy_index = 0
-    end
-
-    def start
-      super
-      @running = true
-      @thread = Thread.new(&method(:run))
-    end
-
-    def shutdown
-      @running = false
-      @thread.join
-    end
-
-    def run
-      batch_num    = (@rate / BIN_NUM).to_i
-      residual_num = (@rate % BIN_NUM)
-      while @running
-        current_time = Time.now.to_i
-        BIN_NUM.times do
-          break unless (@running && Time.now.to_i <= current_time)
-          wait(0.1) { emit(batch_num) }
-        end
-        emit(residual_num)
-        # wait for next second
-        while @running && Time.now.to_i <= current_time
-          sleep 0.01
-        end
-      end
-    end
-
-    def emit(num)
-      num.times { router.emit(@tag, Fluent::Engine.now, generate()) }
-    end
-
-    def generate
-      d = @dummy[@dummy_index]
-      unless d
-        @dummy_index = 0
-        d = @dummy[0]
-      end
-      @dummy_index += 1
-      if @auto_increment_key
-        d = d.dup
-        d[@auto_increment_key] = @increment_value
-        @increment_value += 1
-      end
-      d
-    end
-
-    def wait(time)
-      start_time = Time.now
-      yield
-      sleep_time = time - (Time.now - start_time)
-      sleep sleep_time if sleep_time > 0
-    end
-  end
-end

data/lib/fluent/timezone.rb

@@ -1,131 +0,0 @@
-#
-# Fluentd
-#
-#    Licensed under the Apache License, Version 2.0 (the "License");
-#    you may not use this file except in compliance with the License.
-#    You may obtain a copy of the License at
-#
-#        http://www.apache.org/licenses/LICENSE-2.0
-#
-#    Unless required by applicable law or agreed to in writing, software
-#    distributed under the License is distributed on an "AS IS" BASIS,
-#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#    See the License for the specific language governing permissions and
-#    limitations under the License.
-#
-
-require 'tzinfo'
-
-module Fluent
-  class Timezone
-    # [+-]HH:MM, [+-]HHMM, [+-]HH
-    NUMERIC_PATTERN = %r{\A[+-]\d\d(:?\d\d)?\z}
-
-    # Region/Zone, Region/Zone/Zone
-    NAME_PATTERN = %r{\A[^/]+/[^/]+(/[^/]+)?\z}
-
-    # Validate the format of the specified timezone.
-    #
-    # Valid formats are as follows. Note that timezone abbreviations
-    # such as PST and JST are not supported intentionally.
-    #
-    #   1. [+-]HH:MM         (e.g. "+09:00")
-    #   2. [+-]HHMM          (e.g. "+0900")
-    #   3. [+-]HH            (e.g. "+09")
-    #   4. Region/Zone       (e.g. "Asia/Tokyo")
-    #   5. Region/Zone/Zone  (e.g. "America/Argentina/Buenos_Aires")
-    #
-    # In the 4th and 5th cases, it is checked whether the specified
-    # timezone exists in the timezone database.
-    #
-    # When the given timezone is valid, true is returned. Otherwise,
-    # false is returned. When nil is given, false is returned.
-    def self.validate(timezone)
-      # If the specified timezone is nil.
-      if timezone.nil?
-        # Invalid.
-        return false
-      end
-
-      # [+-]HH:MM, [+-]HHMM, [+-]HH
-      if NUMERIC_PATTERN === timezone
-        # Valid. It can be parsed by Time.zone_offset method.
-        return true
-      end
-
-      # Region/Zone, Region/Zone/Zone
-      if NAME_PATTERN === timezone
-        begin
-          # Get a Timezone instance for the specified timezone.
-          TZInfo::Timezone.get(timezone)
-        rescue
-          # Invalid. The string does not exist in the timezone database.
-          return false
-        else
-          # Valid. The string was found in the timezone database.
-          return true
-        end
-      else
-        # Invalid. Timezone abbreviations are not supported.
-        return false
-      end
-    end
-
-    # Validate the format of the specified timezone.
-    #
-    # The implementation of this method calls validate(timezone) method
-    # to check whether the given timezone is valid. When invalid, this
-    # method raises a ConfigError.
-    def self.validate!(timezone)
-      unless validate(timezone)
-        raise ConfigError, "Unsupported timezone '#{timezone}'"
-      end
-    end
-
-    # Create a formatter for a timezone and optionally a format.
-    #
-    # An Proc object is returned. If the given timezone is invalid,
-    # nil is returned.
-    def self.formatter(timezone, format = nil)
-      if timezone.nil?
-        return nil
-      end
-
-      # [+-]HH:MM, [+-]HHMM, [+-]HH
-      if NUMERIC_PATTERN === timezone
-        offset = Time.zone_offset(timezone)
-
-        if format
-          return Proc.new {|time|
-            Time.at(time).localtime(offset).strftime(format)
-          }
-        else
-          return Proc.new {|time|
-            Time.at(time).localtime(offset).iso8601
-          }
-        end
-      end
-
-      # Region/Zone, Region/Zone/Zone
-      if NAME_PATTERN === timezone
-        begin
-          tz = TZInfo::Timezone.get(timezone)
-        rescue
-          return nil
-        end
-
-        if format
-          return Proc.new {|time|
-            Time.at(time).localtime(tz.period_for_utc(time).utc_total_offset).strftime(format)
-          }
-        else
-          return Proc.new {|time|
-            Time.at(time).localtime(tz.period_for_utc(time).utc_total_offset).iso8601
-          }
-        end
-      end
-
-      return nil
-    end
-  end
-end

data/test/config/assertions.rb

@@ -1,42 +0,0 @@
-require 'test/unit/assertions'
-
-module Test::Unit::Assertions
-  def assert_text_parsed_as(expected, actual)
-    msg = parse_text(actual).inspect rescue 'failed'
-    msg = "expected that #{actual.inspect} would be a parsed as #{expected.inspect} but got #{msg}"
-    assert_block(msg) {
-      v = parse_text(actual)
-      if expected.is_a?(Float)
-        v.is_a?(Float) && (v == obj || (v.nan? && obj.nan?) || (v - obj).abs < 0.000001)
-      else
-        v == expected
-      end
-    }
-  end
-
-  def assert_text_parsed_as_json(expected, actual)
-    msg = parse_text(actual).inspect rescue 'failed'
-    msg = "expected that #{actual.inspect} would be a parsed as #{expected.inspect} but got #{msg}"
-    assert_block(msg) {
-      v = JSON.parse(parse_text(actual))
-      v == expected
-    }
-  end
-
-  def assert_parse_error(actual)
-    msg = begin
-            parse_text(actual).inspect
-          rescue => e
-            e.inspect
-          end
-    msg = "expected that #{actual.inspect} would cause a parse error but got #{msg}"
-    assert_block(msg) {
-      begin
-        parse_text(actual)
-        false
-      rescue Fluent::ConfigParseError
-        true
-      end
-    }
-  end
-end

data/test/config/test_config_parser.rb

@@ -1,389 +0,0 @@
-require_relative '../helper'
-require_relative "assertions"
-require "json"
-require "fluent/config/error"
-require "fluent/config/basic_parser"
-require "fluent/config/literal_parser"
-require "fluent/config/v1_parser"
-
-module Fluent::Config
-  module V1TestHelper
-    def root(*elements)
-      if elements.first.is_a?(Fluent::Config::Element)
-        attrs = {}
-      else
-        attrs = elements.shift || {}
-      end
-      Fluent::Config::Element.new('ROOT', '', attrs, elements)
-    end
-
-    def e(name, arg='', attrs={}, elements=[])
-      Fluent::Config::Element.new(name, arg, attrs, elements)
-    end
-  end
-
-  class TestV1Parser < ::Test::Unit::TestCase
-    def read_config(path)
-      path = File.expand_path(path)
-      data = File.read(path)
-      Fluent::Config::V1Parser.parse(data, File.basename(path), File.dirname(path))
-    end
-
-    def parse_text(text)
-      basepath = File.expand_path(File.dirname(__FILE__) + '/../../')
-      Fluent::Config::V1Parser.parse(text, '(test)', basepath, nil)
-    end
-
-    include V1TestHelper
-    extend V1TestHelper
-
-    sub_test_case 'attribute parsing' do
-      test "parses attributes" do
-        assert_text_parsed_as(e('ROOT', '', {"k1"=>"v1", "k2"=>"v2"}), %[
-          k1 v1
-          k2 v2
-        ])
-      end
-
-      test "allows attribute without value" do
-        assert_text_parsed_as(e('ROOT', '', {"k1"=>"", "k2"=>"v2"}), %[
-          k1
-          k2 v2
-        ])
-      end
-
-      test "parses attribute key always string" do
-        assert_text_parsed_as(e('ROOT', '', {"1" => "1"}), "1 1")
-      end
-
-      data("_.%$!,"     => "_.%$!,",
-           "/=~-~@\`:?" => "/=~-~@\`:?",
-           "()*{}.[]"   => "()*{}.[]")
-      test "parses a value with symbols" do |v|
-        assert_text_parsed_as(e('ROOT', '', {"k" => v}), "k #{v}")
-      end
-
-      test "ignores spacing around value" do
-        assert_text_parsed_as(e('ROOT', '', {"k1" => "a"}), "  k1     a    ")
-      end
-
-      test "allows spaces in value" do
-        assert_text_parsed_as(e('ROOT', '', {"k1" => "a  b  c"}), "k1 a  b  c")
-      end
-
-      sub_test_case 'non-quoted string' do
-        test "remains text starting with '#'" do
-          assert_text_parsed_as(e('ROOT', '', {"k1" => "#not_comment"}), "  k1 #not_comment")
-        end
-
-        test "remains text just after '#'" do
-          assert_text_parsed_as(e('ROOT', '', {"k1" => "a#not_comment"}), "  k1 a#not_comment")
-        end
-
-        test "remove text after ` #` (comment)" do
-          assert_text_parsed_as(e('ROOT', '', {"k1" => "a"}), "  k1 a #comment")
-        end
-
-        test "does not require escaping backslash" do
-          assert_text_parsed_as(e('ROOT', '', {"k1" => "\\\\"}), "  k1 \\\\")
-          assert_text_parsed_as(e('ROOT', '', {"k1" => "\\"}), "  k1 \\")
-        end
-
-        test "remains backslash in front of a normal character" do
-          assert_text_parsed_as(e('ROOT', '', {"k1" => '\['}), "  k1 \\[")
-        end
-
-        test "does not accept escape characters" do
-          assert_text_parsed_as(e('ROOT', '', {"k1" => '\n'}), "  k1 \\n")
-        end
-      end
-
-      sub_test_case 'double quoted string' do
-        test "allows # in value" do
-          assert_text_parsed_as(e('ROOT', '', {"k1" => "a#comment"}), '  k1 "a#comment"')
-        end
-
-        test "rejects characters after double quoted string" do
-          assert_parse_error('  k1 "a" 1')
-        end
-
-        test "requires escaping backslash" do
-          assert_text_parsed_as(e('ROOT', '', {"k1" => "\\"}), '  k1 "\\\\"')
-          assert_parse_error('  k1 "\\"')
-        end
-
-        test "requires escaping double quote" do
-          assert_text_parsed_as(e('ROOT', '', {"k1" => '"'}), '  k1 "\\""')
-          assert_parse_error('  k1 """')
-        end
-
-        test "removes backslash in front of a normal character" do
-          assert_text_parsed_as(e('ROOT', '', {"k1" => '['}), '  k1 "\\["')
-        end
-
-        test "accepts escape characters" do
-          assert_text_parsed_as(e('ROOT', '', {"k1" => "\n"}), '  k1 "\\n"')
-        end
-      end
-
-      sub_test_case 'single quoted string' do
-        test "allows # in value" do
-          assert_text_parsed_as(e('ROOT', '', {"k1" => "a#comment"}), "  k1 'a#comment'")
-        end
-
-        test "rejects characters after single quoted string" do
-          assert_parse_error("  k1 'a' 1")
-        end
-
-        test "requires escaping backslash" do
-          assert_text_parsed_as(e('ROOT', '', {"k1" => "\\"}), "  k1 '\\\\'")
-          assert_parse_error("  k1 '\\'")
-        end
-
-        test "requires escaping single quote" do
-          assert_text_parsed_as(e('ROOT', '', {"k1" => "'"}), "  k1 '\\''")
-          assert_parse_error("  k1 '''")
-        end
-
-        test "remains backslash in front of a normal character" do
-          assert_text_parsed_as(e('ROOT', '', {"k1" => '\\['}), "  k1 '\\['")
-        end
-
-        test "does not accept escape characters" do
-          assert_text_parsed_as(e('ROOT', '', {"k1" => "\\n"}), "  k1 '\\n'")
-        end
-      end
-
-      data(
-        "in match" => %[
-          <match>
-            @k v
-          </match>
-        ],
-        "in source" => %[
-          <source>
-            @k v
-          </source>
-        ],
-        "in filter" => %[
-          <filter>
-            @k v
-          </filter>
-        ],
-        "in top-level" => '  @k v '
-        )
-      def test_rejects_at_prefix_in_the_parameter_name(data)
-        assert_parse_error(data)
-      end
-
-      data(
-        "in nested" => %[
-          <match>
-            <record>
-              @k v
-            </record>
-          </match>
-        ]
-        )
-      def test_not_reject_at_prefix_in_the_parameter_name(data)
-        assert_nothing_raised { parse_text(data) }
-      end
-    end
-
-    sub_test_case 'element parsing' do
-      data(
-        'root' => [root, ""],
-        "accepts empty element" => [root(e("test")), %[
-          <test>
-          </test>
-        ]],
-        "accepts argument and attributes" => [root(e("test", 'var', {'key'=>"val"})), %[
-          <test var>
-            key val
-          </test>
-        ]],
-        "accepts nested elements" => [root(
-          e("test", 'var', {'key'=>'1'}, [
-            e('nested1'),
-            e('nested2')
-          ])), %[
-          <test var>
-            key 1
-            <nested1>
-            </nested1>
-            <nested2>
-            </nested2>
-          </test>
-        ]],
-        "accepts multiline json values" => [root(e("test", 'var', {'key'=>"[\"a\",\"b\",\"c\",\"d\"]"})), %[
-          <test var>
-            key ["a",
-"b", "c",
-"d"]
-          </test>
-        ]],
-        "parses empty element argument to nil" => [root(e("test", '')), %[
-          <test >
-          </test>
-        ]],
-        "ignores spacing around element argument" => [root(e("test", "a")), %[
-          <test    a    >
-          </test>
-        ]],
-        "accepts spacing inside element argument (for multiple tags)" => [root(e("test", "a.** b.**")), %[
-          <test    a.** b.** >
-          </test>
-        ]])
-      def test_parse_element(data)
-        expected, target = data
-        assert_text_parsed_as(expected, target)
-      end
-
-      [
-        "**",
-        "*.*",
-        "1",
-        "_.%$!",
-        "/",
-        "()*{}.[]",
-      ].each do |arg|
-        test "parses symbol element argument:#{arg}" do
-          assert_text_parsed_as(root(e("test", arg)), %[
-            <test #{arg}>
-            </test>
-          ])
-        end
-      end
-
-      data(
-        "considers comments in element argument" => %[
-          <test #a>
-          </test>
-        ],
-        "requires line_end after begin tag" => %[
-          <test></test>
-        ],
-        "requires line_end after end tag" => %[
-          <test>
-          </test><test>
-          </test>
-        ])
-      def test_parse_error(data)
-        assert_parse_error(data)
-      end
-    end
-
-    # port from test_config.rb
-    sub_test_case '@include parsing' do
-      TMP_DIR = File.dirname(__FILE__) + "/tmp/v1_config#{ENV['TEST_ENV_NUMBER']}"
-
-      def write_config(path, data)
-        FileUtils.mkdir_p(File.dirname(path))
-        File.open(path, "w") { |f| f.write data }
-      end
-
-      def prepare_config
-        write_config "#{TMP_DIR}/config_test_1.conf", %[
-        k1 root_config
-        include dir/config_test_2.conf  #
-        @include #{TMP_DIR}/config_test_4.conf
-        include file://#{TMP_DIR}/config_test_5.conf
-        @include config.d/*.conf
-      ]
-        write_config "#{TMP_DIR}/dir/config_test_2.conf", %[
-        k2 relative_path_include
-        @include ../config_test_3.conf
-      ]
-        write_config "#{TMP_DIR}/config_test_3.conf", %[
-        k3 relative_include_in_included_file
-      ]
-        write_config "#{TMP_DIR}/config_test_4.conf", %[
-        k4 absolute_path_include
-      ]
-        write_config "#{TMP_DIR}/config_test_5.conf", %[
-        k5 uri_include
-      ]
-        write_config "#{TMP_DIR}/config.d/config_test_6.conf", %[
-        k6 wildcard_include_1
-        <elem1 name>
-          include normal_parameter
-        </elem1>
-      ]
-        write_config "#{TMP_DIR}/config.d/config_test_7.conf", %[
-        k7 wildcard_include_2
-      ]
-        write_config "#{TMP_DIR}/config.d/config_test_8.conf", %[
-        <elem2 name>
-          @include ../dir/config_test_9.conf
-        </elem2>
-      ]
-        write_config "#{TMP_DIR}/dir/config_test_9.conf", %[
-        k9 embeded
-        <elem3 name>
-          nested nested_value
-          include hoge
-        </elem3>
-      ]
-        write_config "#{TMP_DIR}/config.d/00_config_test_8.conf", %[
-        k8 wildcard_include_3
-        <elem4 name>
-          include normal_parameter
-        </elem4>
-      ]
-      end
-
-      test 'parses @include / include correctly' do
-        prepare_config
-        c = read_config("#{TMP_DIR}/config_test_1.conf")
-        assert_equal('root_config', c['k1'])
-        assert_equal('relative_path_include', c['k2'])
-        assert_equal('relative_include_in_included_file', c['k3'])
-        assert_equal('absolute_path_include', c['k4'])
-        assert_equal('uri_include', c['k5'])
-        assert_equal('wildcard_include_1', c['k6'])
-        assert_equal('wildcard_include_2', c['k7'])
-        assert_equal('wildcard_include_3', c['k8'])
-        assert_equal([
-            'k1',
-            'k2',
-            'k3',
-            'k4',
-            'k5',
-            'k8', # Because of the file name this comes first.
-            'k6',
-            'k7',
-          ], c.keys)
-
-        elem1 = c.elements.find { |e| e.name == 'elem1' }
-        assert(elem1)
-        assert_equal('name', elem1.arg)
-        assert_equal('normal_parameter', elem1['include'])
-
-        elem2 = c.elements.find { |e| e.name == 'elem2' }
-        assert(elem2)
-        assert_equal('name', elem2.arg)
-        assert_equal('embeded', elem2['k9'])
-        assert_not_include(elem2, 'include')
-
-        elem3 = elem2.elements.find { |e| e.name == 'elem3' }
-        assert(elem3)
-        assert_equal('nested_value', elem3['nested'])
-        assert_equal('hoge', elem3['include'])
-      end
-
-      # TODO: Add uri based include spec
-    end
-
-    sub_test_case '#to_s' do
-      test 'parses dumpped configuration' do
-        original = %q!a\\\n\r\f\b'"z!
-        expected = %q!a\\\n\r\f\b'"z!
-
-        conf = parse_text(%[k1 #{original}])
-        assert_equal(expected, conf['k1']) # escape check
-        conf2 = parse_text(conf.to_s) # use dumpped configuration to check unescape
-        assert_equal(expected, conf2.elements.first['k1'])
-      end
-    end
-  end
-end