fluent-logger 0.8.0 → 0.8.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: eb07188f881f645087ed46e08dcb033abd3e78f3
-  data.tar.gz: c5e7715e87ebf4bb7ca5f5a0afb6297d4772011b
+SHA256:
+  metadata.gz: 7be5ae39c39ebb27bdeeaa66b1d103cb6e7d8f7d6f0f5073ff5320c333345c37
+  data.tar.gz: 6f226c613086aff26729786e088a2923552fd02a8e28ebe88654d1dda068b98f
 SHA512:
-  metadata.gz: 44b031f5063e83cac44e704f8a3fe95d1fa8bdf732f68dad096bd6da73b41c31c480412c00c1fe006f9b2bd6285e626f6ab6bc3096115ac559c3961ffd3c9a59
-  data.tar.gz: d3929bbf74cc3e77fc76149f98a0cd001f34213a715c807290cdc7bdc88df35d4d11d07d963a6a765ad144552ee99fb8cb191d7dce8f28e91e4e4c5c053ef37c
+  metadata.gz: 53da0192645010cd5fe3ddc739ff4a6c5784ceb5ddbb8216b25ccd562a176606abfeeb5a9e164d0734e53467b69de749075136c07aebfa02e1da9babfb3f5e1b
+  data.tar.gz: db4c2283dc26641cdf4ac727ebb8815fa0e9ce3c0672b0e5bb98030aa83538005d653547551fa1ac954e591013ff1aaefbf1556905e5e4445479492e5b07b6a0

data/ChangeLog

@@ -1,3 +1,12 @@
+Release 0.8.2 - 2019/08/21
+
+  * Block timeout during IO#write to avoid writing invalid bytes
+  * Fluent::Logger::EventTime#to_json returns String
+
+Release 0.8.1 - 2019/05/30
+
+  * Improve non-blocking write handling
+
 Release 0.8.0 - 2019/01/25
 
   * Add use_nonblock and wait_writeable parameters
@@ -130,4 +139,3 @@ Release 0.2.0 - 2011/08/05
 Release 0.1.0 - 2011/08/04
 
   * First release
-

data/README.md

@@ -4,6 +4,22 @@
 
 A structured event logger
 
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'fluent-logger'
+```
+
+And then execute:
+
+    $ bundle install
+
+Or install it yourself as:
+
+    $ gem install fluent-logger
+
 ## Examples
 
 ### Simple
@@ -27,6 +43,7 @@ require 'fluent-logger'
 
 log = Fluent::Logger::FluentLogger.new(nil, :socket_path => "/tmp/fluent.sock")
 unless log.post("myapp.access", {"agent" => "foo"})
+  # Passed records are stored into logger's internal buffer so don't re-post same event.
   p log.last_error # You can get last error object via last_error method
 end
 
@@ -49,10 +66,17 @@ log.post("access", {"agent" => "foo"})
 require 'fluent-logger'
 
 log = Fluent::Logger::FluentLogger.new(nil, :host => 'localhost', :port => 24224, :use_nonblock => true, :wait_writeable => false)
+# When wait_writeable is false
 begin
   log.post("myapp.access", {"agent" => "foo"})
 rescue IO::EAGAINWaitWritable => e
   # wait code for avoding "Resource temporarily unavailable"
+  # Passed records are stored into logger's internal buffer so don't re-post same event.
+end
+
+# When wait_writeable is true
+unless log.post("myapp.access", {"agent" => "foo"})
+  # same as other example
 end
 
 # output: myapp.access {"agent":"foo"}
@@ -88,11 +112,11 @@ Use nano second event time instead of epoch. See also "Tips" section.
 
 #### use_nonblock (Bool)
 
-Use nonblocking write(`IO#write_nonblock`) instead of normal write(`IO#write`). If `Logger#post` stuck on your environment, specify `true`.
+Use nonblocking write(`IO#write_nonblock`) instead of normal write(`IO#write`). If `Logger#post` stuck on your environment, specify `true`.  Default: `false`
 
 #### wait_writeable (Bool)
 
-If `true`, `Logger#post` raises an error when nonblocking write gets `EAGAIN`.
+If `false`, `Logger#post` raises an error when nonblocking write gets `EAGAIN` (i.e. `use_nonblock` must be `true`, otherwise this will have no effect).  Default: `true`
 
 #### buffer_overflow_handler (Proc)
 

data/lib/fluent/logger/fluent_logger.rb

@@ -15,6 +15,7 @@
 #    See the License for the specific language governing permissions and
 #    limitations under the License.
 #
+require 'timeout'
 require 'msgpack'
 require 'socket'
 require 'monitor'
@@ -44,7 +45,7 @@ module Fluent
       end
 
       def to_json(*args)
-        @sec
+        @sec.to_s
       end
     end
 
@@ -126,7 +127,6 @@ module Fluent
       end
 
       attr_accessor :limit, :logger, :log_reconnect_error_threshold
-      attr_reader :last_error
 
       def last_error
         @last_error[Thread.current.object_id]
@@ -230,7 +230,11 @@ module Fluent
           end
 
           begin
-            send_data(@pending)
+            written = send_data(@pending)
+            if @pending.bytesize != written
+              raise "Actual written data size(#{written} bytes) is different from the received data size(#{@pending.bytesize} bytes)."
+            end
+
             @pending = nil
             true
           rescue => e
@@ -255,9 +259,13 @@ module Fluent
           connect!
         end
         if @use_nonblock
-          @con.write_nonblock data
+          send_data_nonblock(data)
         else
-          @con.write data
+          _, ws = IO.select([], [@con])
+          Thread.handle_interrupt(::Timeout::Error => :never) do
+            # block timeout error during IO#write
+            ws.first.write(data)
+          end
         end
         #while true
         #  puts "sending #{data.length} bytes"
@@ -272,7 +280,19 @@ module Fluent
         #  end
         #  data = data[n..-1]
         #end
-        true
+      end
+
+      def send_data_nonblock(data)
+        written = @con.write_nonblock(data)
+        remaining = data.bytesize - written
+
+        while remaining > 0
+          len = @con.write_nonblock(data.byteslice(written, remaining))
+          remaining -= len
+          written += len
+        end
+
+        written
       end
 
       def connect!

data/lib/fluent/logger/version.rb

@@ -1,5 +1,5 @@
 module Fluent
   module Logger
-    VERSION = '0.8.0'
+    VERSION = '0.8.2'
   end
 end

data/spec/fluent_logger_spec.rb

@@ -58,6 +58,57 @@ describe Fluent::Logger::FluentLogger do
       @serverengine.shutdown
     end
 
+    describe('testing interaction of use_nonblock and wait_writeable') do
+      before(:example) do
+        allow_any_instance_of(TCPSocket).to receive(:write_nonblock).and_raise(IO::EAGAINWaitWritable)
+        allow_any_instance_of(TCPSocket).to receive(:write) { |_, buf| buf.size }
+      end
+
+      context('use_nonblock is false') do
+        let(:block_config) { logger_config.merge(use_nonblock: false) }
+
+        it('post returns true when wait_writeable is false') {
+          cfg = block_config.merge(wait_writeable: false)
+          l = Fluent::Logger::FluentLogger.new('logger-test', cfg)
+          expect(l.post('hello', foo: 'bar')).to eq true
+        }
+
+        it('post returns true when wait_writeable is true') {
+          cfg = block_config.merge(wait_writeable: true)
+          l = Fluent::Logger::FluentLogger.new('logger-test', cfg)
+          expect(l.post('hello', {foo: 'bar'})).to eq true
+        }
+      end
+
+      context('use_nonblock is true') do
+        let(:nonblock_config) { logger_config.merge(use_nonblock: true) }
+
+        it('post raises IO::EAGAINWaitWritable when wait_writeable is false') {
+          cfg = nonblock_config.merge(wait_writeable: false)
+          l = Fluent::Logger::FluentLogger.new('logger-test', cfg)
+          expect { l.post('hello', foo: 'bar') }.to raise_error(IO::EAGAINWaitWritable)
+        }
+
+        it('post returns false when wait_writeable is true') {
+          cfg = nonblock_config.merge(wait_writeable: true)
+          l = Fluent::Logger::FluentLogger.new('logger-test', cfg)
+          expect(l.post('hello', {foo: 'bar'})).to eq false
+        }
+
+        context 'when write_nonblock returns the size less than received data' do
+          before do
+            allow_any_instance_of(TCPSocket).to receive(:write_nonblock).and_return(1) # write 1 bytes per call
+          end
+
+          it 'buffering data and flush at closed time' do
+            logger = Fluent::Logger::FluentLogger.new('logger-test', nonblock_config)
+            expect(logger.post('hello', foo: 'bar')).to eq(true)
+            expect(logger.pending_bytesize).to eq(0)
+          end
+        end
+      end
+    end
+
     context('Post by CUI') do
       it('post') {
         args = %W(-h localhost -p #{fluentd.port} -t logger-test.tag -v a=b -v foo=bar)
@@ -86,6 +137,16 @@ describe Fluent::Logger::FluentLogger do
         }
       end
 
+      context 'when the message has object which does not have #to_msgpack method' do
+        it 'success with nanosecond' do
+          expect(logger_with_nanosec.pending_bytesize).to eq(0)
+          expect(logger_with_nanosec.post('tag', 'a' => Errno::ETIMEDOUT)).to eq(true)
+          fluentd.wait_transfer
+          expect(fluentd.queue.last).to eq(['logger-test.tag', { 'a' => 'Errno::ETIMEDOUT' }])
+          expect(logger_with_nanosec.pending_bytesize).to eq(0)
+        end
+      end
+
       it ('close after post') {
         expect(logger).to be_connect
         logger.close

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-logger
 version: !ruby/object:Gem::Version
-  version: 0.8.0
+  version: 0.8.2
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-01-25 00:00:00.000000000 Z
+date: 2019-08-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: msgpack
@@ -161,8 +161,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.6.14.1
+rubygems_version: 3.0.3
 signing_key: 
 specification_version: 4
 summary: fluent logger for ruby