fluentd 0.14.6 → 0.14.7

data/test/test_time_parser.rb

@@ -0,0 +1,211 @@
+require_relative 'helper'
+require 'fluent/test'
+require 'fluent/time'
+
+class TimeParserTest < ::Test::Unit::TestCase
+  def setup
+    Fluent::Test.setup
+  end
+
+  def test_call_with_parse
+    parser = Fluent::TimeParser.new
+
+    assert(parser.parse('2013-09-18 12:00:00 +0900').is_a?(Fluent::EventTime))
+
+    time = event_time('2013-09-18 12:00:00 +0900')
+    assert_equal(time, parser.parse('2013-09-18 12:00:00 +0900'))
+  end
+
+  def test_parse_with_strptime
+    parser = Fluent::TimeParser.new('%d/%b/%Y:%H:%M:%S %z')
+
+    assert(parser.parse('28/Feb/2013:12:00:00 +0900').is_a?(Fluent::EventTime))
+
+    time = event_time('28/Feb/2013:12:00:00 +0900', format: '%d/%b/%Y:%H:%M:%S %z')
+    assert_equal(time, parser.parse('28/Feb/2013:12:00:00 +0900'))
+  end
+
+  def test_parse_nsec_with_strptime
+    parser = Fluent::TimeParser.new('%d/%b/%Y:%H:%M:%S:%N %z')
+
+    assert(parser.parse('28/Feb/2013:12:00:00:123456789 +0900').is_a?(Fluent::EventTime))
+
+    time = event_time('28/Feb/2013:12:00:00:123456789 +0900', format: '%d/%b/%Y:%H:%M:%S:%N %z')
+    assert_equal_event_time(time, parser.parse('28/Feb/2013:12:00:00:123456789 +0900'))
+  end
+
+  def test_parse_with_invalid_argument
+    parser = Fluent::TimeParser.new
+
+    [[], {}, nil, true, 10000, //, ->{}, '', :symbol].each { |v|
+      assert_raise Fluent::TimeParser::TimeParseError do
+        parser.parse(v)
+      end
+    }
+  end
+
+  def test_parse_time_in_localtime
+    time = with_timezone("UTC+02") do
+      parser = Fluent::TimeParser.new("%Y-%m-%d %H:%M:%S.%N", true)
+      parser.parse("2016-09-02 18:42:31.123456789")
+    end
+    assert_equal_event_time(time, event_time("2016-09-02 18:42:31.123456789 -02:00", format: '%Y-%m-%d %H:%M:%S.%N %z'))
+  end
+
+  def test_parse_time_in_utc
+    time = with_timezone("UTC-09") do
+      parser = Fluent::TimeParser.new("%Y-%m-%d %H:%M:%S.%N", false)
+      parser.parse("2016-09-02 18:42:31.123456789")
+    end
+    assert_equal_event_time(time, event_time("2016-09-02 18:42:31.123456789 UTC", format: '%Y-%m-%d %H:%M:%S.%N %z'))
+  end
+
+  def test_parse_string_with_expected_timezone
+    time = with_timezone("UTC-09") do
+      parser = Fluent::TimeParser.new("%Y-%m-%d %H:%M:%S.%N", nil, "-07:00")
+      parser.parse("2016-09-02 18:42:31.123456789")
+    end
+    assert_equal_event_time(time, event_time("2016-09-02 18:42:31.123456789 -07:00", format: '%Y-%m-%d %H:%M:%S.%N %z'))
+  end
+
+  sub_test_case 'TimeMixin::Parser' do
+    class DummyForTimeParser
+      include Fluent::Configurable
+      include Fluent::TimeMixin::Parser
+    end
+
+    test 'provides configuration parameters for TimeParser with default values for localtime' do
+      time = with_timezone("UTC+07") do
+        i = DummyForTimeParser.new
+        i.configure(config_element('parse'))
+
+        assert_nil   i.time_format
+        assert_true  i.localtime
+        assert_false i.utc
+        assert_nil   i.timezone
+
+        parser = i.time_parser_create
+        # time_format unspecified
+        # localtime
+        parser.parse("2016-09-02 18:42:31.012345678")
+      end
+      assert_equal_event_time(event_time("2016-09-02 18:42:31.012345678 -07:00", format: '%Y-%m-%d %H:%M:%S.%N %z'), time)
+    end
+
+    test 'provides configuration parameters for TimeParser, configurable for any time format' do
+      time = with_timezone("UTC+07") do
+        i = DummyForTimeParser.new
+        i.configure(config_element('parse', '', {'time_format' => '%m/%d/%Y %H-%M-%S %N'}))
+        parser = i.time_parser_create
+        # time_format specified
+        # localtime
+        parser.parse("09/02/2016 18-42-31 012345678")
+      end
+      assert_equal_event_time(event_time("2016-09-02 18:42:31.012345678 -07:00", format: '%Y-%m-%d %H:%M:%S.%N %z'), time)
+    end
+
+    test 'provides configuration parameters for TimeParser, configurable for UTC by localtime=false' do
+      time = with_timezone("UTC+07") do
+        i = DummyForTimeParser.new
+        i.configure(config_element('parse', '', {'time_format' => '%m/%d/%Y %H-%M-%S %N', 'localtime' => 'false'}))
+        parser = i.time_parser_create
+        # time_format specified
+        # utc
+        parser.parse("09/02/2016 18-42-31 012345678")
+      end
+      assert_equal_event_time(event_time("2016-09-02 18:42:31.012345678 UTC", format: '%Y-%m-%d %H:%M:%S.%N %z'), time)
+    end
+
+    test 'provides configuration parameters for TimeParser, configurable for UTC by utc=true' do
+      time = with_timezone("UTC+07") do
+        i = DummyForTimeParser.new
+        i.configure(config_element('parse', '', {'time_format' => '%m/%d/%Y %H-%M-%S %N', 'utc' => 'true'}))
+        parser = i.time_parser_create
+        # time_format specified
+        # utc
+        parser.parse("09/02/2016 18-42-31 012345678")
+      end
+      assert_equal_event_time(event_time("2016-09-02 18:42:31.012345678 UTC", format: '%Y-%m-%d %H:%M:%S.%N %z'), time)
+    end
+
+    test 'provides configuration parameters for TimeParser, configurable for any timezone' do
+      time = with_timezone("UTC+07") do
+        i = DummyForTimeParser.new
+        i.configure(config_element('parse', '', {'time_format' => '%m/%d/%Y %H-%M-%S %N', 'timezone' => '-01:00'}))
+        parser = i.time_parser_create
+        # time_format specified
+        # -01:00
+        parser.parse("09/02/2016 18-42-31 012345678")
+      end
+      assert_equal_event_time(event_time("2016-09-02 18:42:31.012345678 -01:00", format: '%Y-%m-%d %H:%M:%S.%N %z'), time)
+    end
+
+    test 'specifying timezone without time format raises configuration error' do
+      assert_raise Fluent::ConfigError.new("specifying timezone requires time format") do
+        i = DummyForTimeParser.new
+        i.configure(config_element('parse', '', {'utc' => 'true'}))
+        i.time_parser_create
+      end
+      assert_raise Fluent::ConfigError.new("specifying timezone requires time format") do
+        i = DummyForTimeParser.new
+        i.configure(config_element('parse', '', {'localtime' => 'false'}))
+        i.time_parser_create
+      end
+      assert_raise Fluent::ConfigError.new("specifying timezone requires time format") do
+        i = DummyForTimeParser.new
+        i.configure(config_element('parse', '', {'timezone' => '-0700'}))
+        i.time_parser_create
+      end
+    end
+
+    test '#time_parser_create returns TimeParser with specified time format and timezone' do
+      time = with_timezone("UTC-09") do
+        i = DummyForTimeParser.new
+        i.configure(config_element('parse', '', {'time_format' => '%m/%d/%Y %H-%M-%S %N'}))
+        assert_equal '%m/%d/%Y %H-%M-%S %N', i.time_format
+        assert_true i.localtime
+        parser = i.time_parser_create(format: '%Y-%m-%d %H:%M:%S.%N %z')
+        parser.parse("2016-09-05 17:59:38.987654321 -03:00")
+      end
+      assert_equal_event_time(event_time("2016-09-05 17:59:38.987654321 -03:00", format: '%Y-%m-%d %H:%M:%S.%N %z'), time)
+    end
+
+    test '#time_parser_create returns TimeParser with localtime when specified it forcedly besides any configuration parameters' do
+      time = with_timezone("UTC-09") do
+        i = DummyForTimeParser.new
+        i.configure(config_element('parse', '', {'time_format' => '%m/%d/%Y %H-%M-%S', 'utc' => 'true'}))
+        assert_equal '%m/%d/%Y %H-%M-%S', i.time_format
+        assert_true i.utc
+        parser = i.time_parser_create(format: '%Y-%m-%d %H:%M:%S.%N', force_localtime: true)
+        parser.parse("2016-09-05 17:59:38.987654321")
+      end
+      assert_equal_event_time(event_time("2016-09-05 17:59:38.987654321 +09:00", format: '%Y-%m-%d %H:%M:%S.%N %z'), time)
+
+      time = with_timezone("UTC-09") do
+        i = DummyForTimeParser.new
+        i.configure(config_element('parse', '', {'time_format' => '%m/%d/%Y %H-%M-%S', 'timezone' => '+0000'}))
+        assert_equal '%m/%d/%Y %H-%M-%S', i.time_format
+        assert_equal '+0000', i.timezone
+        parser = i.time_parser_create(format: '%Y-%m-%d %H:%M:%S.%N', force_localtime: true)
+        parser.parse("2016-09-05 17:59:38.987654321")
+      end
+      assert_equal_event_time(event_time("2016-09-05 17:59:38.987654321 +09:00", format: '%Y-%m-%d %H:%M:%S.%N %z'), time)
+    end
+
+    test '#time_parser_create returns NumericTimeParser to parse time as unixtime when time_type unixtime specified' do
+      i = DummyForTimeParser.new
+      i.configure(config_element('parse', '', {'time_type' => 'unixtime'}))
+      parser = i.time_parser_create
+      time = event_time("2016-10-03 20:08:30.123456789 +0100", format: '%Y-%m-%d %H:%M:%S.%N %z')
+      assert_equal_event_time(Fluent::EventTime.new(time.to_i), parser.parse("#{time.sec}"))
+    end
+
+    test '#time_parser_create returns NumericTimeParser to parse time as float when time_type float specified' do
+      i = DummyForTimeParser.new
+      i.configure(config_element('parse', '', {'time_type' => 'float'}))
+      parser = i.time_parser_create
+      time = event_time("2016-10-03 20:08:30.123456789 +0100", format: '%Y-%m-%d %H:%M:%S.%N %z')
+      assert_equal_event_time(time, parser.parse("#{time.sec}.#{time.nsec}"))
+    end
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluentd
 version: !ruby/object:Gem::Version
-  version: 0.14.6
+  version: 0.14.7
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-09-07 00:00:00.000000000 Z
+date: 2016-10-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: msgpack
@@ -279,6 +279,7 @@ description: Fluentd is an open source data collector designed to scale and simp
 email:
 - frsyuki@gmail.com
 executables:
+- fluent-binlog-reader
 - fluent-cat
 - fluent-debug
 - fluent-gem
@@ -298,12 +299,14 @@ files:
 - Rakefile
 - Vagrantfile
 - appveyor.yml
+- bin/fluent-binlog-reader
 - bin/fluent-cat
 - bin/fluent-debug
 - bin/fluent-gem
 - bin/fluentd
 - example/copy_roundrobin.conf
 - example/filter_stdout.conf
+- example/in_dummy_with_compression.conf
 - example/in_forward.conf
 - example/in_forward_client.conf
 - example/in_forward_shared_key.conf
@@ -329,6 +332,7 @@ files:
 - fluent.conf
 - fluentd.gemspec
 - lib/fluent/agent.rb
+- lib/fluent/command/binlog_reader.rb
 - lib/fluent/command/bundler_injection.rb
 - lib/fluent/command/cat.rb
 - lib/fluent/command/debug.rb
@@ -392,6 +396,7 @@ files:
 - lib/fluent/plugin/buffer/chunk.rb
 - lib/fluent/plugin/buffer/file_chunk.rb
 - lib/fluent/plugin/buffer/memory_chunk.rb
+- lib/fluent/plugin/compressable.rb
 - lib/fluent/plugin/exec_util.rb
 - lib/fluent/plugin/file_util.rb
 - lib/fluent/plugin/file_wrapper.rb
@@ -458,6 +463,7 @@ files:
 - lib/fluent/plugin_helper/compat_parameters.rb
 - lib/fluent/plugin_helper/event_emitter.rb
 - lib/fluent/plugin_helper/event_loop.rb
+- lib/fluent/plugin_helper/extract.rb
 - lib/fluent/plugin_helper/formatter.rb
 - lib/fluent/plugin_helper/inject.rb
 - lib/fluent/plugin_helper/parser.rb
@@ -498,6 +504,7 @@ files:
 - lib/fluent/unique_id.rb
 - lib/fluent/version.rb
 - lib/fluent/winsvc.rb
+- test/command/test_binlog_reader.rb
 - test/compat/test_calls_super.rb
 - test/compat/test_parser.rb
 - test/config/assertions.rb
@@ -525,6 +532,7 @@ files:
 - test/plugin/test_buffer_chunk.rb
 - test/plugin/test_buffer_file_chunk.rb
 - test/plugin/test_buffer_memory_chunk.rb
+- test/plugin/test_compressable.rb
 - test/plugin/test_file_util.rb
 - test/plugin/test_filter.rb
 - test/plugin/test_filter_grep.rb
@@ -564,6 +572,7 @@ files:
 - test/plugin/test_out_stream.rb
 - test/plugin/test_output.rb
 - test/plugin/test_output_as_buffered.rb
+- test/plugin/test_output_as_buffered_compress.rb
 - test/plugin/test_output_as_buffered_overflow.rb
 - test/plugin/test_output_as_buffered_retries.rb
 - test/plugin/test_output_as_buffered_secondary.rb
@@ -581,7 +590,6 @@ files:
 - test/plugin/test_parser_none.rb
 - test/plugin/test_parser_regexp.rb
 - test/plugin/test_parser_syslog.rb
-- test/plugin/test_parser_time.rb
 - test/plugin/test_parser_tsv.rb
 - test/plugin/test_storage.rb
 - test/plugin/test_storage_local.rb
@@ -590,6 +598,7 @@ files:
 - test/plugin_helper/test_compat_parameters.rb
 - test/plugin_helper/test_event_emitter.rb
 - test/plugin_helper/test_event_loop.rb
+- test/plugin_helper/test_extract.rb
 - test/plugin_helper/test_formatter.rb
 - test/plugin_helper/test_inject.rb
 - test/plugin_helper/test_parser.rb
@@ -598,6 +607,8 @@ files:
 - test/plugin_helper/test_thread.rb
 - test/plugin_helper/test_timer.rb
 - test/scripts/exec_script.rb
+- test/scripts/fluent/plugin/formatter1/formatter_test1.rb
+- test/scripts/fluent/plugin/formatter2/formatter_test2.rb
 - test/scripts/fluent/plugin/formatter_known.rb
 - test/scripts/fluent/plugin/out_test.rb
 - test/scripts/fluent/plugin/out_test2.rb
@@ -620,6 +631,7 @@ files:
 - test/test_root_agent.rb
 - test/test_supervisor.rb
 - test/test_time_formatter.rb
+- test/test_time_parser.rb
 - test/test_unique_id.rb
 homepage: http://fluentd.org/
 licenses:
@@ -646,6 +658,7 @@ signing_key:
 specification_version: 4
 summary: Fluentd event collector
 test_files:
+- test/command/test_binlog_reader.rb
 - test/compat/test_calls_super.rb
 - test/compat/test_parser.rb
 - test/config/assertions.rb
@@ -673,6 +686,7 @@ test_files:
 - test/plugin/test_buffer_chunk.rb
 - test/plugin/test_buffer_file_chunk.rb
 - test/plugin/test_buffer_memory_chunk.rb
+- test/plugin/test_compressable.rb
 - test/plugin/test_file_util.rb
 - test/plugin/test_filter.rb
 - test/plugin/test_filter_grep.rb
@@ -712,6 +726,7 @@ test_files:
 - test/plugin/test_out_stream.rb
 - test/plugin/test_output.rb
 - test/plugin/test_output_as_buffered.rb
+- test/plugin/test_output_as_buffered_compress.rb
 - test/plugin/test_output_as_buffered_overflow.rb
 - test/plugin/test_output_as_buffered_retries.rb
 - test/plugin/test_output_as_buffered_secondary.rb
@@ -729,7 +744,6 @@ test_files:
 - test/plugin/test_parser_none.rb
 - test/plugin/test_parser_regexp.rb
 - test/plugin/test_parser_syslog.rb
-- test/plugin/test_parser_time.rb
 - test/plugin/test_parser_tsv.rb
 - test/plugin/test_storage.rb
 - test/plugin/test_storage_local.rb
@@ -738,6 +752,7 @@ test_files:
 - test/plugin_helper/test_compat_parameters.rb
 - test/plugin_helper/test_event_emitter.rb
 - test/plugin_helper/test_event_loop.rb
+- test/plugin_helper/test_extract.rb
 - test/plugin_helper/test_formatter.rb
 - test/plugin_helper/test_inject.rb
 - test/plugin_helper/test_parser.rb
@@ -746,6 +761,8 @@ test_files:
 - test/plugin_helper/test_thread.rb
 - test/plugin_helper/test_timer.rb
 - test/scripts/exec_script.rb
+- test/scripts/fluent/plugin/formatter1/formatter_test1.rb
+- test/scripts/fluent/plugin/formatter2/formatter_test2.rb
 - test/scripts/fluent/plugin/formatter_known.rb
 - test/scripts/fluent/plugin/out_test.rb
 - test/scripts/fluent/plugin/out_test2.rb
@@ -768,4 +785,6 @@ test_files:
 - test/test_root_agent.rb
 - test/test_supervisor.rb
 - test/test_time_formatter.rb
+- test/test_time_parser.rb
 - test/test_unique_id.rb
+has_rdoc: false

data/test/plugin/test_parser_time.rb

@@ -1,46 +0,0 @@
-require_relative '../helper'
-require 'fluent/test/driver/parser'
-require 'fluent/plugin/parser'
-
-class TimeParserTest < ::Test::Unit::TestCase
-  def setup
-    Fluent::Test.setup
-  end
-
-  def test_call_with_parse
-    parser = Fluent::Test::Driver::Parser.new(Fluent::Plugin::Parser::TimeParser.new(nil))
-
-    assert(parser.instance.parse('2013-09-18 12:00:00 +0900').is_a?(Fluent::EventTime))
-
-    time = event_time('2013-09-18 12:00:00 +0900')
-    assert_equal(time, parser.instance.parse('2013-09-18 12:00:00 +0900'))
-  end
-
-  def test_parse_with_strptime
-    parser = Fluent::Test::Driver::Parser.new(Fluent::Plugin::Parser::TimeParser.new('%d/%b/%Y:%H:%M:%S %z'))
-
-    assert(parser.instance.parse('28/Feb/2013:12:00:00 +0900').is_a?(Fluent::EventTime))
-
-    time = event_time('28/Feb/2013:12:00:00 +0900', format: '%d/%b/%Y:%H:%M:%S %z')
-    assert_equal(time, parser.instance.parse('28/Feb/2013:12:00:00 +0900'))
-  end
-
-  def test_parse_nsec_with_strptime
-    parser = Fluent::Test::Driver::Parser.new(Fluent::Plugin::Parser::TimeParser.new('%d/%b/%Y:%H:%M:%S:%N %z'))
-
-    assert(parser.instance.parse('28/Feb/2013:12:00:00:123456789 +0900').is_a?(Fluent::EventTime))
-
-    time = event_time('28/Feb/2013:12:00:00:123456789 +0900', format: '%d/%b/%Y:%H:%M:%S:%N %z')
-    assert_equal_event_time(time, parser.instance.parse('28/Feb/2013:12:00:00:123456789 +0900'))
-  end
-
-  def test_parse_with_invalid_argument
-    parser = Fluent::Test::Driver::Parser.new(Fluent::Plugin::Parser::TimeParser.new(nil))
-
-    [[], {}, nil, true, 10000, //, ->{}, '', :symbol].each { |v|
-      assert_raise Fluent::ParserError do
-        parser.instance.parse(v)
-      end
-    }
-  end
-end