fluentd 1.6.3 → 1.7.0

data/test/plugin/test_parser_csv.rb

@@ -100,4 +100,87 @@ class CSVParserTest < ::Test::Unit::TestCase
       assert_equal record['c'], '789'
     end
   end
+
+  sub_test_case 'parser' do
+    data('normal' => 'normal',
+         'fast' => 'fast')
+    def test_compatibility_between_normal_and_fast_parser(param)
+      d = create_driver(
+        'keys' => 'time,key1,key2,key3,key4,key5',
+        'time_key' => 'time',
+        'time_format' => "%d/%b/%Y:%H:%M:%S %z",
+        'keep_time_key' => 'false',
+        'parser_type' => param
+      )
+
+      # non quoted
+      text = '28/Feb/2013:12:00:00 +0900,value1,value2,value3,value4,value5'
+      expected = {'key1' => 'value1', 'key2' => 'value2', 'key3' => "value3",
+                  'key4' => 'value4', 'key5' => "value5"}
+      d.instance.parse(text) do |time, record|
+        assert_equal(event_time("28/Feb/2013:12:00:00 +0900", format: '%d/%b/%Y:%H:%M:%S %z'), time)
+        assert_equal expected, record
+      end
+
+      # quoted
+      text = '28/Feb/2013:12:00:00 +0900,"value1","val,ue2","va,lu,e3","val ue4",""'
+      expected = {'key1' => 'value1', 'key2' => 'val,ue2', 'key3' => "va,lu,e3",
+                  'key4' => 'val ue4', 'key5' => ""}
+      d.instance.parse(text) do |time, record|
+        assert_equal(event_time("28/Feb/2013:12:00:00 +0900", format: '%d/%b/%Y:%H:%M:%S %z'), time)
+        assert_equal expected, record
+      end
+
+      # mixed
+      text = '28/Feb/2013:12:00:00 +0900,message,"mes,sage","me,ssa,ge",mess age,""'
+      expected = {'key1' => 'message', 'key2' => 'mes,sage', 'key3' => "me,ssa,ge",
+                  'key4' => 'mess age', 'key5' => ""}
+      d.instance.parse(text) do |time, record|
+        assert_equal(event_time("28/Feb/2013:12:00:00 +0900", format: '%d/%b/%Y:%H:%M:%S %z'), time)
+        assert_equal expected, record
+      end
+
+      # escaped
+      text = '28/Feb/2013:12:00:00 +0900,"message","mes""sage","""message""",,""""""'
+      expected = {'key1' => 'message', 'key2' => 'mes"sage', 'key3' => '"message"',
+                  'key4' => nil, 'key5' => '""'}
+      d.instance.parse(text) do |time, record|
+        assert_equal(event_time("28/Feb/2013:12:00:00 +0900", format: '%d/%b/%Y:%H:%M:%S %z'), time)
+        assert_equal expected, record
+      end
+    end
+
+    def test_incompatibility_between_normal_and_fast_parser
+      normal = create_driver(
+        'keys' => 'key1,key2',
+        'parser_type' => 'normal'
+      )
+      fast = create_driver(
+        'keys' => 'key1,key2',
+        'parser_type' => 'fast'
+      )
+
+      # unexpected quote position
+      text = 'a"b,"a"""c"'
+      assert_raise(CSV::MalformedCSVError) {
+        normal.instance.parse(text) { |t, r| }
+      } 
+      assert_nothing_raised {
+        # generate broken record
+        fast.instance.parse(text) { |t, r| }
+      }
+
+      # incorrect the number of column
+      text = 'a,b,c'
+      expected = {"key1" => 'a', "key2" => 'b'}
+      normal.instance.parse(text) { |t, r|
+        assert_equal expected, r
+      }
+      fast.instance.parse(text) { |t, r|
+        assert_not_equal expected, r
+      }
+
+      # And more...
+    end
+  end
 end

data/test/plugin_helper/test_record_accessor.rb

@@ -141,7 +141,7 @@ class RecordAccessorHelperTest < Test::Unit::TestCase
 
     data("missing ']'" => "$['key1'",
          "missing array index with dot" => "$.hello[]",
-         "missing array index with braket" => "$['hello'][]",
+         "missing array index with bracket" => "$['hello'][]",
          "whitespace char included key in dot notation" => "$.key[0].ke y",
          "more chars" => "$.key1[0]foo",
          "empty keys with dot" => "$.",

data/test/test_time_formatter.rb

@@ -3,9 +3,8 @@ require 'fluent/test'
 require 'fluent/time'
 
 class TimeFormatterTest < ::Test::Unit::TestCase
-  def setup
-    @time = Time.new(2014, 9, 27, 0, 0, 0, 0).to_i
-    @fmt  = "%Y%m%d %H%M%z"  # YYYYMMDD HHMM[+-]HHMM
+  setup do
+    @fmt ="%Y%m%d %H%M%z"  # YYYYMMDD HHMM[+-]HHMM
   end
 
   def format(format, localtime, timezone)
@@ -13,166 +12,186 @@ class TimeFormatterTest < ::Test::Unit::TestCase
     formatter.format(@time)
   end
 
-  def test_default_utc_nil
-    assert_equal("2014-09-27T00:00:00Z", format(nil, false, nil))
-  end
+  module TestLists
+    def test_default_utc_nil
+      assert_equal("2014-09-27T00:00:00Z", format(nil, false, nil))
+    end
 
-  def test_default_utc_pHH_MM
-    assert_equal("2014-09-27T01:30:00+01:30", format(nil, false, "+01:30"))
-  end
+    def test_default_utc_pHH_MM
+      assert_equal("2014-09-27T01:30:00+01:30", format(nil, false, "+01:30"))
+    end
 
-  def test_default_utc_nHH_MM
-    assert_equal("2014-09-26T22:30:00-01:30", format(nil, false, "-01:30"))
-  end
+    def test_default_utc_nHH_MM
+      assert_equal("2014-09-26T22:30:00-01:30", format(nil, false, "-01:30"))
+    end
 
-  def test_default_utc_pHHMM
-    assert_equal("2014-09-27T02:30:00+02:30", format(nil, false, "+0230"))
-  end
+    def test_default_utc_pHHMM
+      assert_equal("2014-09-27T02:30:00+02:30", format(nil, false, "+0230"))
+    end
 
-  def test_default_utc_nHHMM
-    assert_equal("2014-09-26T21:30:00-02:30", format(nil, false, "-0230"))
-  end
+    def test_default_utc_nHHMM
+      assert_equal("2014-09-26T21:30:00-02:30", format(nil, false, "-0230"))
+    end
 
-  def test_default_utc_pHH
-    assert_equal("2014-09-27T03:00:00+03:00", format(nil, false, "+03"))
-  end
+    def test_default_utc_pHH
+      assert_equal("2014-09-27T03:00:00+03:00", format(nil, false, "+03"))
+    end
 
-  def test_default_utc_nHH
-    assert_equal("2014-09-26T21:00:00-03:00", format(nil, false, "-03"))
-  end
+    def test_default_utc_nHH
+      assert_equal("2014-09-26T21:00:00-03:00", format(nil, false, "-03"))
+    end
 
-  def test_default_utc_timezone_1
-    # Asia/Tokyo (+09:00) does not have daylight saving time.
-    assert_equal("2014-09-27T09:00:00+09:00", format(nil, false, "Asia/Tokyo"))
-  end
+    def test_default_utc_timezone_1
+      # Asia/Tokyo (+09:00) does not have daylight saving time.
+      assert_equal("2014-09-27T09:00:00+09:00", format(nil, false, "Asia/Tokyo"))
+    end
 
-  def test_default_utc_timezone_2
-    # Pacific/Honolulu (-10:00) does not have daylight saving time.
-    assert_equal("2014-09-26T14:00:00-10:00", format(nil, false, "Pacific/Honolulu"))
-  end
+    def test_default_utc_timezone_2
+      # Pacific/Honolulu (-10:00) does not have daylight saving time.
+      assert_equal("2014-09-26T14:00:00-10:00", format(nil, false, "Pacific/Honolulu"))
+    end
 
-  def test_default_utc_timezone_3
-    # America/Argentina/Buenos_Aires (-03:00) does not have daylight saving time.
-    assert_equal("2014-09-26T21:00:00-03:00", format(nil, false, "America/Argentina/Buenos_Aires"))
-  end
+    def test_default_utc_timezone_3
+      # America/Argentina/Buenos_Aires (-03:00) does not have daylight saving time.
+      assert_equal("2014-09-26T21:00:00-03:00", format(nil, false, "America/Argentina/Buenos_Aires"))
+    end
 
-  def test_default_utc_timezone_4
-    # Europe/Paris has daylight saving time. Its UTC offset is +01:00 and its
-    # UTC offset in DST is +02:00. In September, Europe/Paris is in DST.
-    assert_equal("2014-09-27T02:00:00+02:00", format(nil, false, "Europe/Paris"))
-  end
+    def test_default_utc_timezone_4
+      # Europe/Paris has daylight saving time. Its UTC offset is +01:00 and its
+      # UTC offset in DST is +02:00. In September, Europe/Paris is in DST.
+      assert_equal("2014-09-27T02:00:00+02:00", format(nil, false, "Europe/Paris"))
+    end
 
-  def test_default_utc_timezone_5
-    # Europe/Paris has daylight saving time. Its UTC offset is +01:00 and its
-    # UTC offset in DST is +02:00. In January, Europe/Paris is not in DST.
-    @time = Time.new(2014, 1, 24, 0, 0, 0, 0).to_i
-    assert_equal("2014-01-24T01:00:00+01:00", format(nil, false, "Europe/Paris"))
-  end
+    def test_default_utc_timezone_5
+      # Europe/Paris has daylight saving time. Its UTC offset is +01:00 and its
+      # UTC offset in DST is +02:00. In January, Europe/Paris is not in DST.
+      @time = Time.new(2014, 1, 24, 0, 0, 0, 0).to_i
+      assert_equal("2014-01-24T01:00:00+01:00", format(nil, false, "Europe/Paris"))
+    end
 
-  def test_default_utc_invalid
-    assert_equal("2014-09-27T00:00:00Z", format(nil, false, "Invalid"))
-  end
+    def test_default_utc_invalid
+      assert_equal("2014-09-27T00:00:00Z", format(nil, false, "Invalid"))
+    end
 
-  def test_default_localtime_nil_1
-    with_timezone("UTC-04") do
-      assert_equal("2014-09-27T04:00:00+04:00", format(nil, true, nil))
+    def test_default_localtime_nil_1
+      with_timezone("UTC-04") do
+        assert_equal("2014-09-27T04:00:00+04:00", format(nil, true, nil))
+      end
     end
-  end
 
-  def test_default_localtime_nil_2
-    with_timezone("UTC+05") do
-      assert_equal("2014-09-26T19:00:00-05:00", format(nil, true, nil))
+    def test_default_localtime_nil_2
+      with_timezone("UTC+05") do
+        assert_equal("2014-09-26T19:00:00-05:00", format(nil, true, nil))
+      end
     end
-  end
 
-  def test_default_localtime_timezone
-    # 'timezone' takes precedence over 'localtime'.
-    with_timezone("UTC-06") do
-      assert_equal("2014-09-27T07:00:00+07:00", format(nil, true, "+07"))
+    def test_default_localtime_timezone
+      # 'timezone' takes precedence over 'localtime'.
+      with_timezone("UTC-06") do
+        assert_equal("2014-09-27T07:00:00+07:00", format(nil, true, "+07"))
+      end
     end
-  end
 
-  def test_specific_utc_nil
-    assert_equal("20140927 0000+0000", format(@fmt, false, nil))
-  end
+    def test_specific_utc_nil
+      assert_equal("20140927 0000+0000", format(@fmt, false, nil))
+    end
 
-  def test_specific_utc_pHH_MM
-    assert_equal("20140927 0830+0830", format(@fmt, false, "+08:30"))
-  end
+    def test_specific_utc_pHH_MM
+      assert_equal("20140927 0830+0830", format(@fmt, false, "+08:30"))
+    end
 
-  def test_specific_utc_nHH_MM
-    assert_equal("20140926 1430-0930", format(@fmt, false, "-09:30"))
-  end
+    def test_specific_utc_nHH_MM
+      assert_equal("20140926 1430-0930", format(@fmt, false, "-09:30"))
+    end
 
-  def test_specific_utc_pHHMM
-    assert_equal("20140927 1030+1030", format(@fmt, false, "+1030"))
-  end
+    def test_specific_utc_pHHMM
+      assert_equal("20140927 1030+1030", format(@fmt, false, "+1030"))
+    end
 
-  def test_specific_utc_nHHMM
-    assert_equal("20140926 1230-1130", format(@fmt, false, "-1130"))
-  end
+    def test_specific_utc_nHHMM
+      assert_equal("20140926 1230-1130", format(@fmt, false, "-1130"))
+    end
 
-  def test_specific_utc_pHH
-    assert_equal("20140927 1200+1200", format(@fmt, false, "+12"))
-  end
+    def test_specific_utc_pHH
+      assert_equal("20140927 1200+1200", format(@fmt, false, "+12"))
+    end
 
-  def test_specific_utc_nHH
-    assert_equal("20140926 1100-1300", format(@fmt, false, "-13"))
-  end
+    def test_specific_utc_nHH
+      assert_equal("20140926 1100-1300", format(@fmt, false, "-13"))
+    end
 
-  def test_specific_utc_timezone_1
-    # Europe/Moscow (+04:00) does not have daylight saving time.
-    assert_equal("20140927 0400+0400", format(@fmt, false, "Europe/Moscow"))
-  end
+    def test_specific_utc_timezone_1
+      # Europe/Moscow (+04:00) does not have daylight saving time.
+      assert_equal("20140927 0400+0400", format(@fmt, false, "Europe/Moscow"))
+    end
 
-  def test_specific_utc_timezone_2
-    # Pacific/Galapagos (-06:00) does not have daylight saving time.
-    assert_equal("20140926 1800-0600", format(@fmt, false, "Pacific/Galapagos"))
-  end
+    def test_specific_utc_timezone_2
+      # Pacific/Galapagos (-06:00) does not have daylight saving time.
+      assert_equal("20140926 1800-0600", format(@fmt, false, "Pacific/Galapagos"))
+    end
 
-  def test_specific_utc_timezone_3
-    # America/Argentina/Buenos_Aires (-03:00) does not have daylight saving time.
-    assert_equal("20140926 2100-0300", format(@fmt, false, "America/Argentina/Buenos_Aires"))
-  end
+    def test_specific_utc_timezone_3
+      # America/Argentina/Buenos_Aires (-03:00) does not have daylight saving time.
+      assert_equal("20140926 2100-0300", format(@fmt, false, "America/Argentina/Buenos_Aires"))
+    end
 
-  def test_specific_utc_timezone_4
-    # America/Los_Angeles has daylight saving time. Its UTC offset is -08:00 and its
-    # UTC offset in DST is -07:00. In September, America/Los_Angeles is in DST.
-    assert_equal("20140926 1700-0700", format(@fmt, false, "America/Los_Angeles"))
-  end
+    def test_specific_utc_timezone_4
+      # America/Los_Angeles has daylight saving time. Its UTC offset is -08:00 and its
+      # UTC offset in DST is -07:00. In September, America/Los_Angeles is in DST.
+      assert_equal("20140926 1700-0700", format(@fmt, false, "America/Los_Angeles"))
+    end
 
-  def test_specific_utc_timezone_5
-    # America/Los_Angeles has daylight saving time. Its UTC offset is -08:00 and its
-    # UTC offset in DST is -07:00. In January, America/Los_Angeles is not in DST.
-    @time = Time.new(2014, 1, 24, 0, 0, 0, 0).to_i
-    assert_equal("20140123 1600-0800", format(@fmt, false, "America/Los_Angeles"))
-  end
+    def test_specific_utc_timezone_5
+      # America/Los_Angeles has daylight saving time. Its UTC offset is -08:00 and its
+      # UTC offset in DST is -07:00. In January, America/Los_Angeles is not in DST.
+      @time = Time.new(2014, 1, 24, 0, 0, 0, 0).to_i
+      assert_equal("20140123 1600-0800", format(@fmt, false, "America/Los_Angeles"))
+    end
 
-  def test_specific_utc_invalid
-    assert_equal("20140927 0000+0000", format(@fmt, false, "Invalid"))
-  end
+    def test_specific_utc_invalid
+      assert_equal("20140927 0000+0000", format(@fmt, false, "Invalid"))
+    end
+
+    def test_specific_localtime_nil_1
+      with_timezone("UTC-07") do
+        assert_equal("20140927 0700+0700", format(@fmt, true, nil))
+      end
+    end
+
+    def test_specific_localtime_nil_2
+      with_timezone("UTC+08") do
+        assert_equal("20140926 1600-0800", format(@fmt, true, nil))
+      end
+    end
 
-  def test_specific_localtime_nil_1
-    with_timezone("UTC-07") do
-      assert_equal("20140927 0700+0700", format(@fmt, true, nil))
+    def test_specific_localtime_timezone
+      # 'timezone' takes precedence over 'localtime'.
+      with_timezone("UTC-09") do
+        assert_equal("20140926 1400-1000", format(@fmt, true, "-10"))
+      end
     end
   end
 
-  def test_specific_localtime_nil_2
-    with_timezone("UTC+08") do
-      assert_equal("20140926 1600-0800", format(@fmt, true, nil))
+  sub_test_case 'Fluent::EventTime time' do
+    setup do
+      @time = Fluent::EventTime.from_time(Time.new(2014, 9, 27, 0, 0, 0, 0))
     end
+
+    include TestLists
   end
 
-  def test_specific_localtime_timezone
-    # 'timezone' takes precedence over 'localtime'.
-    with_timezone("UTC-09") do
-      assert_equal("20140926 1400-1000", format(@fmt, true, "-10"))
+  # for v0.12 compatibility
+  sub_test_case 'Integer time' do
+    setup do
+      @time = Time.new(2014, 9, 27, 0, 0, 0, 0).to_i
     end
+
+    include TestLists
   end
 
   def test_format_with_subsec
-    time = Fluent::EventTime.new(@time)
+    time = Time.new(2014, 9, 27, 0, 0, 0, 0).to_i
+    time = Fluent::EventTime.new(time)
     formatter = Fluent::TimeFormatter.new("%Y%m%d %H%M.%N", false, nil)
     assert_equal("20140927 0000.000000000", formatter.format(time))
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluentd
 version: !ruby/object:Gem::Version
-  version: 1.6.3
+  version: 1.7.0
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-07-29 00:00:00.000000000 Z
+date: 2019-08-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: msgpack
@@ -124,14 +124,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: '2.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: '2.0'
 - !ruby/object:Gem::Dependency
   name: tzinfo-data
   requirement: !ruby/object:Gem::Requirement
@@ -328,6 +328,7 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
+- ".drone.yml"
 - ".github/ISSUE_TEMPLATE.md"
 - ".github/ISSUE_TEMPLATE/bug_report.md"
 - ".github/ISSUE_TEMPLATE/feature_request.md"
@@ -469,10 +470,12 @@ files:
 - lib/fluent/plugin/bare_output.rb
 - lib/fluent/plugin/base.rb
 - lib/fluent/plugin/buf_file.rb
+- lib/fluent/plugin/buf_file_single.rb
 - lib/fluent/plugin/buf_memory.rb
 - lib/fluent/plugin/buffer.rb
 - lib/fluent/plugin/buffer/chunk.rb
 - lib/fluent/plugin/buffer/file_chunk.rb
+- lib/fluent/plugin/buffer/file_single_chunk.rb
 - lib/fluent/plugin/buffer/memory_chunk.rb
 - lib/fluent/plugin/compressable.rb
 - lib/fluent/plugin/exec_util.rb
@@ -513,6 +516,14 @@ files:
 - lib/fluent/plugin/out_exec_filter.rb
 - lib/fluent/plugin/out_file.rb
 - lib/fluent/plugin/out_forward.rb
+- lib/fluent/plugin/out_forward/ack_handler.rb
+- lib/fluent/plugin/out_forward/connection_manager.rb
+- lib/fluent/plugin/out_forward/error.rb
+- lib/fluent/plugin/out_forward/failure_detector.rb
+- lib/fluent/plugin/out_forward/handshake_protocol.rb
+- lib/fluent/plugin/out_forward/load_balancer.rb
+- lib/fluent/plugin/out_forward/socket_cache.rb
+- lib/fluent/plugin/out_http.rb
 - lib/fluent/plugin/out_null.rb
 - lib/fluent/plugin/out_relabel.rb
 - lib/fluent/plugin/out_roundrobin.rb
@@ -643,6 +654,7 @@ files:
 - test/counter/test_validator.rb
 - test/helper.rb
 - test/helpers/fuzzy_assert.rb
+- test/helpers/process_extenstion.rb
 - test/plugin/data/2010/01/20100102-030405.log
 - test/plugin/data/2010/01/20100102-030406.log
 - test/plugin/data/2010/01/20100102.log
@@ -650,13 +662,20 @@ files:
 - test/plugin/data/log/foo/bar.log
 - test/plugin/data/log/foo/bar2
 - test/plugin/data/log/test.log
+- test/plugin/out_forward/test_ack_handler.rb
+- test/plugin/out_forward/test_connection_manager.rb
+- test/plugin/out_forward/test_handshake_protocol.rb
+- test/plugin/out_forward/test_load_balancer.rb
+- test/plugin/out_forward/test_socket_cache.rb
 - test/plugin/test_bare_output.rb
 - test/plugin/test_base.rb
 - test/plugin/test_buf_file.rb
+- test/plugin/test_buf_file_single.rb
 - test/plugin/test_buf_memory.rb
 - test/plugin/test_buffer.rb
 - test/plugin/test_buffer_chunk.rb
 - test/plugin/test_buffer_file_chunk.rb
+- test/plugin/test_buffer_file_single_chunk.rb
 - test/plugin/test_buffer_memory_chunk.rb
 - test/plugin/test_compressable.rb
 - test/plugin/test_file_util.rb
@@ -694,6 +713,7 @@ files:
 - test/plugin/test_out_exec_filter.rb
 - test/plugin/test_out_file.rb
 - test/plugin/test_out_forward.rb
+- test/plugin/test_out_http.rb
 - test/plugin/test_out_null.rb
 - test/plugin/test_out_relabel.rb
 - test/plugin/test_out_roundrobin.rb
@@ -827,6 +847,7 @@ test_files:
 - test/counter/test_validator.rb
 - test/helper.rb
 - test/helpers/fuzzy_assert.rb
+- test/helpers/process_extenstion.rb
 - test/plugin/data/2010/01/20100102-030405.log
 - test/plugin/data/2010/01/20100102-030406.log
 - test/plugin/data/2010/01/20100102.log
@@ -834,13 +855,20 @@ test_files:
 - test/plugin/data/log/foo/bar.log
 - test/plugin/data/log/foo/bar2
 - test/plugin/data/log/test.log
+- test/plugin/out_forward/test_ack_handler.rb
+- test/plugin/out_forward/test_connection_manager.rb
+- test/plugin/out_forward/test_handshake_protocol.rb
+- test/plugin/out_forward/test_load_balancer.rb
+- test/plugin/out_forward/test_socket_cache.rb
 - test/plugin/test_bare_output.rb
 - test/plugin/test_base.rb
 - test/plugin/test_buf_file.rb
+- test/plugin/test_buf_file_single.rb
 - test/plugin/test_buf_memory.rb
 - test/plugin/test_buffer.rb
 - test/plugin/test_buffer_chunk.rb
 - test/plugin/test_buffer_file_chunk.rb
+- test/plugin/test_buffer_file_single_chunk.rb
 - test/plugin/test_buffer_memory_chunk.rb
 - test/plugin/test_compressable.rb
 - test/plugin/test_file_util.rb
@@ -878,6 +906,7 @@ test_files:
 - test/plugin/test_out_exec_filter.rb
 - test/plugin/test_out_file.rb
 - test/plugin/test_out_forward.rb
+- test/plugin/test_out_http.rb
 - test/plugin/test_out_null.rb
 - test/plugin/test_out_relabel.rb
 - test/plugin/test_out_roundrobin.rb