fluentd 0.14.22 → 0.14.23.rc1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 216e64dfd14f91902147d2450a8ea3c10f8900fe
-  data.tar.gz: 107c33b29d5e6a7f18d823795a8b174978b364d7
+  metadata.gz: 6e963518d8ad90373d793c9af9275cc45720d45e
+  data.tar.gz: 43875ee26f0702c7c23514d5b38ef8b163e0e6d8
 SHA512:
-  metadata.gz: 2f34dc55b3dab852af483ae93f2d95fcfbc0dd9c9daa32f407f767c1d2c4670b4ff7508883e6e4d04d8eed54c6d73852f2c1a08973304198d243db2c6645c068
-  data.tar.gz: e0ccbe9c85711e66da262611cbac23763d8712c4250f64e08e8b2634b8e678bc5eebb7f67df461125ee5b01632ca8e86b03a57b211d8b760f0092f8606d3be8c
+  metadata.gz: ec41c99be159e167051b78446a57e89d7627b7f940d10f4c5bc012224c5e7756fd4cbeccd4e4e59fc2d96204e57851f85c5e089896a4fd0acbe914bfd82e3158
+  data.tar.gz: 8b2f5f28e11ed7342f816f1e35e5fa9c4de984aad1cbf24c36c273ddc90f4982b06386bdebd31a8259b2dd924d965b84bc194e28cbc6bb6436a71d62109c0cb1

data/lib/fluent/plugin/buffer.rb

@@ -62,6 +62,74 @@ module Fluent
         def empty?
           timekey.nil? && tag.nil? && variables.nil?
         end
+
+        def cmp_variables(v1, v2)
+          if v1.nil? && v2.nil?
+            return 0
+          elsif v1.nil? # v2 is non-nil
+            return -1
+          elsif v2.nil? # v1 is non-nil
+            return 1
+          end
+          # both of v1 and v2 are non-nil
+          v1_sorted_keys = v1.keys.sort
+          v2_sorted_keys = v2.keys.sort
+          if v1_sorted_keys != v2_sorted_keys
+            if v1_sorted_keys.size == v2_sorted_keys.size
+              v1_sorted_keys <=> v2_sorted_keys
+            else
+              v1_sorted_keys.size <=> v2_sorted_keys.size
+            end
+          else
+            v1_sorted_keys.each do |k|
+              a = v1[k]
+              b = v2[k]
+              if a && b && a != b
+                return a <=> b
+              elsif a && b || (!a && !b) # same value (including both are nil)
+                next
+              elsif a # b is nil
+                return 1
+              else # a is nil (but b is non-nil)
+                return -1
+              end
+            end
+
+            0
+          end
+        end
+
+        def <=>(o)
+          timekey2 = o.timekey
+          tag2 = o.tag
+          variables2 = o.variables
+          if (!!timekey ^ !!timekey2) || (!!tag ^ !!tag2) || (!!variables ^ !!variables2)
+            # One has value in a field, but another doesn't have value in same field
+            # This case occurs very rarely
+            if timekey == timekey2 # including the case of nil == nil
+              if tag == tag2
+                cmp_variables(variables, variables2)
+              elsif tag.nil?
+                -1
+              elsif tag2.nil?
+                1
+              else
+                tag <=> tag2
+              end
+            elsif timekey.nil?
+              -1
+            elsif timekey2.nil?
+              1
+            else
+              timekey <=> timekey2
+            end
+          else
+            # objects have values in same field pairs (comparison with non-nil and nil doesn't occur here)
+            (timekey <=> timekey2 || 0).nonzero? || # if `a <=> b` is nil, then both are nil
+              (tag <=> tag2 || 0).nonzero? ||
+              cmp_variables(variables, variables2)
+          end
+        end
       end
 
       # for tests
@@ -203,7 +271,9 @@ module Fluent
         chunks_to_enqueue = []
 
         begin
-          metadata_and_data.each do |metadata, data|
+          # sort metadata to get lock of chunks in same order with other threads
+          metadata_and_data.keys.sort.each do |metadata|
+            data = metadata_and_data[metadata]
             write_once(metadata, data, format: format, size: size) do |chunk, adding_bytesize|
               chunk.mon_enter # add lock to prevent to be committed/rollbacked from other threads
               operated_chunks << chunk

data/lib/fluent/plugin/in_tcp.rb

@@ -55,13 +55,12 @@ module Fluent::Plugin
     def start
       super
 
-      @buffer = ''
       server_create(:in_tcp_server, @port, bind: @bind) do |data, conn|
-        @buffer << data
+        conn.buffer << data
         begin
           pos = 0
-          while i = @buffer.index(@delimiter, pos)
-            msg = @buffer[pos...i]
+          while i = conn.buffer.index(@delimiter, pos)
+            msg = conn.buffer[pos...i]
             pos = i + @delimiter.length
 
             @parser.parse(msg) do |time, record|
@@ -77,7 +76,7 @@ module Fluent::Plugin
               router.emit(tag, time, record)
             end
           end
-          @buffer.slice!(0, pos) if pos > 0
+          conn.buffer.slice!(0, pos) if pos > 0
         end
       end
     end

data/lib/fluent/plugin_helper/server.rb

@@ -425,9 +425,12 @@ module Fluent
       end
 
       class TCPCallbackSocket < CallbackSocket
+        attr_accessor :buffer
+
         def initialize(sock)
           super("tcp", sock, [:data, :write_complete, :close])
           @peeraddr = @sock.peeraddr
+          @buffer = ''
         end
 
         def write(data)

data/lib/fluent/version.rb

@@ -16,6 +16,6 @@
 
 module Fluent
 
-  VERSION = '0.14.22'
+  VERSION = '0.14.23.rc1'
 
 end

data/test/plugin/test_buffer.rb

@@ -851,7 +851,8 @@ class BufferTest < Test::Unit::TestCase
       assert{ @p.stage[@dm1].size == 2 }
       assert !@p.stage[@dm1].rollbacked
 
-      @p.stage[@dm1].failing = true
+      meta_list = [@dm0, @dm1, @dm2, @dm3].sort
+      @p.stage[meta_list.last].failing = true
 
       assert_raise(FluentPluginBufferTest::DummyMemoryChunkError) do
         @p.write({ @dm2 => [row], @dm3 => [row], @dm0 => [row, row, row], @dm1 => [row, row] })

data/test/plugin/test_metadata.rb

@@ -0,0 +1,89 @@
+require_relative '../helper'
+require 'fluent/plugin/buffer'
+
+class BufferMetadataTest < Test::Unit::TestCase
+
+  def meta(timekey=nil, tag=nil, variables=nil)
+    Fluent::Plugin::Buffer::Metadata.new(timekey, tag, variables)
+  end
+
+  setup do
+    Fluent::Test.setup
+  end
+
+  sub_test_case 'about metadata' do
+    test 'comparison of variables should be stable' do
+      m = meta(nil, nil, nil)
+      # different sets of keys
+      assert_equal(-1, m.cmp_variables({}, {a: 1}))
+      assert_equal(1, m.cmp_variables({a: 1}, {}))
+      assert_equal(1, m.cmp_variables({c: 1}, {a: 1}))
+      assert_equal(-1, m.cmp_variables({a: 1}, {a: 1, b: 2}))
+      assert_equal(1, m.cmp_variables({a: 1, c: 1}, {a: 1, b: 2}))
+      assert_equal(1, m.cmp_variables({a: 1, b: 0, c: 1}, {a: 1, b: 2}))
+      # same set of keys
+      assert_equal(-1, m.cmp_variables({a: 1}, {a: 2}))
+      assert_equal(-1, m.cmp_variables({a: 1, b: 0}, {a: 1, b: 1}))
+      assert_equal(-1, m.cmp_variables({a: 1, b: 1, c: 100}, {a: 1, b: 1, c: 200}))
+      assert_equal(-1, m.cmp_variables({b: 1, c: 100, a: 1}, {a: 1, b: 1, c: 200})) # comparison sorts keys
+      assert_equal(-1, m.cmp_variables({a: nil}, {a: 1}))
+      assert_equal(-1, m.cmp_variables({a: 1, b: nil}, {a: 1, b: 1}))
+    end
+
+    test 'comparison of metadata should be stable' do
+      n = Time.now.to_i
+
+      assert_equal(0, meta(nil, nil, nil) <=> meta(nil, nil, nil))
+      assert_equal(0, meta(n, nil, nil) <=> meta(n, nil, nil))
+      assert_equal(0, meta(nil, "t1", nil) <=> meta(nil, "t1", nil))
+      assert_equal(0, meta(nil, nil, {}) <=> meta(nil, nil, {}))
+      assert_equal(0, meta(nil, nil, {a: "1"}) <=> meta(nil, nil, {a: "1"}))
+      assert_equal(0, meta(n, nil, {}) <=> meta(n, nil, {}))
+      assert_equal(0, meta(n, "t1", {}) <=> meta(n, "t1", {}))
+      assert_equal(0, meta(n, "t1", {a: "x", b: 10}) <=> meta(n, "t1", {a: "x", b: 10}))
+
+      # timekey is 1st comparison key
+      assert_equal(-1, meta(n - 300, nil, nil) <=> meta(n - 270, nil, nil))
+      assert_equal(1, meta(n + 1, "a", nil) <=> meta(n - 1, "b", nil))
+      assert_equal(-1, meta(n - 1, nil, {a: 100}) <=> meta(n + 1, nil, {}))
+
+      # tag is 2nd
+      assert_equal(-1, meta(nil, "a", {}) <=> meta(nil, "b", {}))
+      assert_equal(-1, meta(n, "a", {}) <=> meta(n, "b", {}))
+      assert_equal(1, meta(nil, "x", {a: 1}) <=> meta(nil, "t", {}))
+      assert_equal(1, meta(n, "x", {a: 1}) <=> meta(n, "t", {}))
+      assert_equal(1, meta(nil, "x", {a: 1}) <=> meta(nil, "t", {a: 1}))
+      assert_equal(1, meta(n, "x", {a: 1}) <=> meta(n, "t", {a: 2}))
+      assert_equal(1, meta(n, "x", {a: 1}) <=> meta(n, "t", {a: 10, b: 1}))
+
+      # variables is the last
+      assert_equal(-1, meta(nil, nil, {}) <=> meta(nil, nil, {a: 1}))
+      assert_equal(-1, meta(n, "t", {}) <=> meta(n, "t", {a: 1}))
+      assert_equal(1, meta(n, "t", {a: 1}) <=> meta(n, "t", {}))
+      assert_equal(-1, meta(n, "t", {a: 1}) <=> meta(n, "t", {a: 2}))
+      assert_equal(-1, meta(n, "t", {a: 1}) <=> meta(n, "t", {a: 1, b: 1}))
+      assert_equal(1, meta(nil, nil, {b: 1}) <=> meta(nil, nil, {a: 1}))
+      assert_equal(1, meta(n, "t", {b: 1}) <=> meta(n, "t", {a: 1}))
+    end
+
+    test 'metadata can be sorted' do
+      n = Time.now.to_i
+      m0 = meta(nil, nil, nil)
+      m1 = meta(n - 1, nil, nil)
+      m2 = meta(n - 1, "a", nil)
+      m3 = meta(n - 1, "a", {a: 1})
+      m4 = meta(n - 1, "a", {a: 100})
+      m5 = meta(n - 1, "a", {a: 100, b: 1})
+      m6 = meta(n - 1, "aa", nil)
+      m7 = meta(n - 1, "aa", {a: 1})
+      m8 = meta(n - 1, "b", nil)
+      m9 = meta(n, nil, nil)
+      m10 = meta(n + 1, nil, {a: 1})
+      expected = [m0, m1, m2, m3, m4, m5, m6, m7, m8, m9, m10].freeze
+      ary = expected.dup
+      100.times do
+        assert_equal expected, ary.shuffle.sort
+      end
+    end
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluentd
 version: !ruby/object:Gem::Version
-  version: 0.14.22
+  version: 0.14.23.rc1
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-11-02 00:00:00.000000000 Z
+date: 2017-11-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: msgpack
@@ -639,6 +639,7 @@ files:
 - test/plugin/test_in_udp.rb
 - test/plugin/test_in_unix.rb
 - test/plugin/test_input.rb
+- test/plugin/test_metadata.rb
 - test/plugin/test_multi_output.rb
 - test/plugin/test_out_copy.rb
 - test/plugin/test_out_exec.rb
@@ -735,9 +736,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '2.1'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
 rubyforge_project: 
 rubygems_version: 2.6.13
@@ -806,6 +807,7 @@ test_files:
 - test/plugin/test_in_udp.rb
 - test/plugin/test_in_unix.rb
 - test/plugin/test_input.rb
+- test/plugin/test_metadata.rb
 - test/plugin/test_multi_output.rb
 - test/plugin/test_out_copy.rb
 - test/plugin/test_out_exec.rb