fluent-logger 0.5.0 → 0.5.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: d93795042f622581efc248bfb4458418a56487c6
-  data.tar.gz: 96e4b0d775212bb6094dea72de1625a9fc80e38e
+  metadata.gz: a3cc43a68a8c73c0232044f58faae75f84eb2fc3
+  data.tar.gz: 43ed7af93c1e1e0976a813b89daedba55f88ff66
 SHA512:
-  metadata.gz: 54f9d02fd16d99b4bc2f2be8ad73b66b43386c1f01022b1f1c4c671ef22df9a8ca0cf53c1fa06cd5a13423952d19e5a65dc8688718a98fd60fbfce68ec0eca65
-  data.tar.gz: e7172605473f80a5ea13f910a1de12c663070d123c9e242f896715c59ec5180e5b137b6d1a172ddaf5c2860957a83e1eb7b69a01880facf01282d1b1396dad79
+  metadata.gz: 607a3d6c81390b225efba3a1fff812f233d53ff6e7521aa1b95fbf0cfdc052f55961c908455b477b2511dda86c9830bdbabb372c0952589fd37390949414127e
+  data.tar.gz: a7f32f72c5411e264463a451f830228f7a15d943445ec0b5058ab1c3aee811252dc879822c3f7f7adba94a790c74be6970235ff05503d3bb9b931f3e090477ac

data/.gitignore

@@ -1,3 +1,4 @@
+.bundle
 Gemfile.lock
 pkg/*
 coverage/*

data/.travis.yml

@@ -1,18 +1,12 @@
 rvm:
- - 1.8.7
- - 1.9.2
  - 1.9.3
  - 2.0.0
  - 2.1
  - 2.2
- - ree
  - rbx
 
 script: bundle exec rake spec
 
 matrix:
   allow_failures:
-    - rvm: 1.8.7
-    - rvm: 1.9.2
-    - rvm: ree
     - rvm: rbx

data/ChangeLog

@@ -1,3 +1,9 @@
+Release 0.5.1 - 2015/11/24
+
+  * Allow injection of buffer overflow handler
+  * tag_prefix now default to nil
+  * LoggerBase.open follows README
+
 Release 0.5.0 - 2015/01/25
 
   * Use json instead of yajl. If you pass an invalid string to post, you may get 'invalid byte sequence' error.

data/README.md

@@ -1,4 +1,7 @@
 # Fluent logger
+
+[![Build Status](https://travis-ci.org/fluent/fluent-logger-ruby.svg?branch=master)](https://travis-ci.org/fluent/fluent-logger-ruby)
+
 A structured event logger
 
 ## Examples
@@ -53,6 +56,32 @@ Fluent::Logger::ConsoleLogger.open(io)
 Fluent::Logger::NullLogger.open
 ```
 
+## Buffer overflow
+
+You can inject your own custom proc to handle buffer overflow in the event of connection failure. This will mitigate the loss of data instead of simply throwing data away.
+
+Your proc must accept a single argument, which will be the internal buffer of messages from the logger. A typical use-case for this would be writing to disk or possibly writing to Redis.
+
+##### Example
+```
+class BufferOverflowHandler
+  attr_accessor :buffer
+
+  def flush(messages)
+    @buffer ||= []
+    MessagePack::Unpacker.new.feed_each(messages) do |msg|
+      @buffer << msg
+    end
+  end
+end
+
+handler = Proc.new { |messages| BufferOverflowHandler.new.flush(messages) }
+
+Fluent::Logger::FluentLogger.new(nil,
+  :host => 'localhost', :port => 24224,
+  :buffer_overflow_handler => handler)
+```
+
 |name|description|
 |---|---|
 |Web site|http://fluentd.org/|

data/VERSION

@@ -1 +1 @@
-0.5.0
+0.5.1

data/lib/fluent/logger/fluent_logger.rb

@@ -34,7 +34,7 @@ module Fluent
         r / RECONNECT_WAIT_INCR_RATE
       }
 
-      def initialize(tag_prefix = '', *args)
+      def initialize(tag_prefix = nil, *args)
         super()
 
         options = {
@@ -62,6 +62,8 @@ module Fluent
         @limit = options[:buffer_limit] || BUFFER_LIMIT
         @log_reconnect_error_threshold = options[:log_reconnect_error_threshold] ||  RECONNECT_WAIT_MAX_COUNT
 
+        @buffer_overflow_handler = options[:buffer_overflow_handler]
+
         if logger = options[:logger]
           @logger = logger
         else
@@ -107,6 +109,7 @@ module Fluent
             rescue => e
               set_last_error(e)
               @logger.error("FluentLogger: Can't send logs to #{@host}:#{@port}: #{$!}")
+              call_buffer_overflow_handler(@pending)
             end
           end
           @con.close if connect?
@@ -168,6 +171,7 @@ module Fluent
             set_last_error(e)
             if @pending.bytesize > @limit
               @logger.error("FluentLogger: Can't send logs to #{@host}:#{@port}: #{$!}")
+              call_buffer_overflow_handler(@pending)
               @pending = nil
             end
             @con.close if connect?
@@ -217,6 +221,14 @@ module Fluent
         raise e
       end
 
+      def call_buffer_overflow_handler(pending)
+        if @buffer_overflow_handler
+          @buffer_overflow_handler.call(pending)
+        end
+      rescue Exception => e
+        @logger.error("FluentLogger: Can't call buffer overflow handler: #{$!}")
+      end
+
       def log_reconnect_error
         @logger.error("FluentLogger: Can't connect to #{@host}:#{@port}(#{@connect_error_history.size} retried): #{$!}")
       end

data/lib/fluent/logger/logger_base.rb

@@ -19,7 +19,7 @@ module Fluent
   module Logger
     class LoggerBase
       def self.open(*args, &block)
-        Fluent::Logger.open(*args, &block)
+        Fluent::Logger.open(self, *args, &block)
       end
 
       def post(tag, map)

data/lib/fluent/logger/version.rb

@@ -1,7 +1,7 @@
 module Fluent
 module Logger
 
-VERSION = '0.5.0'
+VERSION = '0.5.1'
 
 end
 end

data/spec/fluent_logger_spec.rb

@@ -10,7 +10,7 @@ require 'stringio'
 require 'fluent/logger/fluent_logger/cui'
 require 'plugin/out_test'
 
-$log = Fluent::Log.new(StringIO.new) # XXX should remove $log from fluentd 
+$log = Fluent::Log.new(StringIO.new) # XXX should remove $log from fluentd
 
 describe Fluent::Logger::FluentLogger do
   WAIT = ENV['WAIT'] ? ENV['WAIT'].to_f : 0.1
@@ -32,19 +32,26 @@ describe Fluent::Logger::FluentLogger do
     @logger_io = StringIO.new
     logger = ::Logger.new(@logger_io)
     Fluent::Logger::FluentLogger.new('logger-test', {
-      :host   => 'localhost', 
+      :host   => 'localhost',
       :port   => fluentd_port,
       :logger => logger,
+      :buffer_overflow_handler => buffer_overflow_handler
     })
   }
 
+  let(:buffer_overflow_handler) { nil }
+
   let(:logger_io) {
     @logger_io
   }
 
   let(:output) {
     sleep 0.0001 # next tick
-    Fluent::Engine.match('logger-test').output
+    if Fluent::Engine.respond_to?(:match)
+      Fluent::Engine.match('logger-test').output
+    else
+      Fluent::Engine.root_agent.event_router.match('logger-test')
+    end
   }
 
   let(:queue) {
@@ -95,33 +102,33 @@ EOF
         args = %W(-h localhost -p #{fluentd_port} -t logger-test.tag -v a=b -v foo=bar)
         Fluent::Logger::FluentLogger::CUI.post(args)
         wait_transfer
-        queue.last.should == ['logger-test.tag', {'a' => 'b', 'foo' => 'bar'}]
+        expect(queue.last).to eq ['logger-test.tag', {'a' => 'b', 'foo' => 'bar'}]
       }
     end
 
     context('post') do
-      it ('success') { 
+      it ('success') {
         expect(logger.post('tag', {'a' => 'b'})).to be true
         wait_transfer
-        queue.last.should == ['logger-test.tag', {'a' => 'b'}]
+        expect(queue.last).to eq ['logger-test.tag', {'a' => 'b'}]
       }
 
       it ('close after post') {
-        logger.should be_connect
+        expect(logger).to be_connect
         logger.close
-        logger.should_not be_connect
+        expect(logger).not_to be_connect
 
         logger.post('tag', {'b' => 'c'})
-        logger.should be_connect
+        expect(logger).to be_connect
         wait_transfer
-        queue.last.should == ['logger-test.tag', {'b' => 'c'}]
+        expect(queue.last).to eq ['logger-test.tag', {'b' => 'c'}]
       }
 
       it ('large data') {
         data = {'a' => ('b' * 1000000)}
         logger.post('tag', data)
         wait_transfer
-        queue.last.should == ['logger-test.tag', data]
+        expect(queue.last).to eq ['logger-test.tag', data]
       }
 
       it ('msgpack unsupport data') {
@@ -133,9 +140,9 @@ EOF
         logger.post('tag', data)
         wait_transfer
         logger_data = queue.last.last
-        logger_data['time'].should == '2008-09-01 10:05:00 UTC'
-        logger_data['proc'].should be
-        logger_data['object'].should be
+        expect(logger_data['time']).to eq '2008-09-01 10:05:00 UTC'
+        expect(logger_data['proc']).to be_truthy
+        expect(logger_data['object']).to be_truthy
       }
 
       it ('msgpack and JSON unsupport data') {
@@ -147,7 +154,7 @@ EOF
         }
         logger.post('tag', data)
         wait_transfer
-        queue.last.should be_nil
+        expect(queue.last).to be_nil
         logger_io.rewind
         logger_io.read =~ /FluentLogger: Can't convert to msgpack:/
       }
@@ -167,66 +174,105 @@ EOF
 
     context "initializer" do
       it "backward compatible" do
-        port = fluentd_port
-        fluent_logger = Fluent::Logger::FluentLogger.new('logger-test', 'localhost', port)
-        fluent_logger.instance_eval {
-          @host.should == 'localhost'
-          @port.should == port
-        }
+        fluent_logger = Fluent::Logger::FluentLogger.new('logger-test', 'localhost', fluentd_port)
+        host, port = fluent_logger.instance_eval { [@host, @port] }
+        expect(host).to eq 'localhost'
+        expect(port).to eq fluentd_port
       end
 
       it "hash argument" do
-        port = fluentd_port
         fluent_logger = Fluent::Logger::FluentLogger.new('logger-test', {
           :host => 'localhost',
-          :port => port
+          :port => fluentd_port
         })
-        fluent_logger.instance_eval {
-          @host.should == 'localhost'
-          @port.should == port
-        }
+
+        host, port = fluent_logger.instance_eval { [@host, @port] }
+        expect(host).to eq 'localhost'
+        expect(port).to eq fluentd_port
       end
     end
   end
-  
+
   context "not running fluentd" do
     context('fluent logger interface') do
       it ('post & close') {
         expect(logger.post('tag', {'a' => 'b'})).to be false
         wait_transfer  # even if wait
-        queue.last.should be_nil
+        expect(queue.last).to be_nil
         logger.close
         logger_io.rewind
         log = logger_io.read
-        log.should =~ /Failed to connect/
-        log.should =~ /Can't send logs to/
+        expect(log).to match /Failed to connect/
+        expect(log).to match /Can't send logs to/
       }
 
       it ('post limit over') do
         logger.limit = 100
         logger.post('tag', {'a' => 'b'})
         wait_transfer  # even if wait
-        queue.last.should be_nil
+        expect(queue.last).to be_nil
 
         logger_io.rewind
-        logger_io.read.should_not =~ /Can't send logs to/
+        expect(logger_io.read).not_to match /Can't send logs to/
 
         logger.post('tag', {'a' => ('c' * 1000)})
         logger_io.rewind
-        logger_io.read.should =~ /Can't send logs to/
+        expect(logger_io.read).to match /Can't send logs to/
       end
 
       it ('log connect error once') do
-        Fluent::Logger::FluentLogger.any_instance.stub(:suppress_sec).and_return(-1)
+        allow_any_instance_of(Fluent::Logger::FluentLogger).to receive(:suppress_sec).and_return(-1)
         logger.log_reconnect_error_threshold = 1
-        Fluent::Logger::FluentLogger.any_instance.should_receive(:log_reconnect_error).once.and_call_original
+        expect_any_instance_of(Fluent::Logger::FluentLogger).to receive(:log_reconnect_error).once.and_call_original
 
         logger.post('tag', {'a' => 'b'})
         wait_transfer  # even if wait
         logger.post('tag', {'a' => 'b'})
         wait_transfer  # even if wait
         logger_io.rewind
-        logger_io.read.should =~ /Failed to connect/
+        expect(logger_io.read).to match /Failed to connect/
+      end
+    end
+
+    context "when a buffer overflow handler is provided" do
+      class BufferOverflowHandler
+        attr_accessor :buffer
+
+        def flush(messages)
+          @buffer ||= []
+          MessagePack::Unpacker.new.feed_each(messages) do |msg|
+            @buffer << msg
+          end
+        end
+      end
+
+      let(:handler) { BufferOverflowHandler.new }
+      let(:buffer_overflow_handler) { Proc.new { |messages| handler.flush(messages) } }
+
+      it ('post limit over') do
+        logger.limit = 100
+        event_1 = {'a' => 'b'}
+        logger.post('tag', event_1)
+        wait_transfer  # even if wait
+        queue.last.should be_nil
+
+        logger_io.rewind
+        logger_io.read.should_not =~ /Can't send logs to/
+
+        event_2 = {'a' => ('c' * 1000)}
+        logger.post('tag', event_2)
+        logger_io.rewind
+        logger_io.read.should =~ /Can't send logs to/
+
+        buffer = handler.buffer
+
+        buffer[0][0].should == 'logger-test.tag'
+        buffer[0][1].should.to_s =~ /\d{10}/
+        buffer[0][2].should == event_1
+
+        buffer[1][0].should == 'logger-test.tag'
+        buffer[1][1].should.to_s =~ /\d{10}/
+        buffer[1][2].should == event_2
       end
     end
   end

data/spec/logger_base_spec.rb

@@ -3,9 +3,21 @@ require 'spec_helper'
 
 describe Fluent::Logger::LoggerBase do
   context "subclass" do
-    subject { Class.new(Fluent::Logger::LoggerBase) }
-    its(:open) {
-      should be_kind_of(Fluent::Logger::LoggerBase)
-    }
+    let(:subclass) { Class.new(Fluent::Logger::LoggerBase) }
+    let(:other_subclass) { Class.new(Fluent::Logger::LoggerBase) }
+
+    describe ".open" do
+      subject(:open) { subclass.open }
+
+      it { should be_kind_of(Fluent::Logger::LoggerBase) }
+
+      it "changes Fluent::Logger.default" do
+        subclass.open
+        expect(Fluent::Logger.default).to be_kind_of(subclass)
+
+        other_subclass.open
+        expect(Fluent::Logger.default).to be_kind_of(other_subclass)
+      end
+    end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-logger
 version: !ruby/object:Gem::Version
-  version: 0.5.0
+  version: 0.5.1
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-01-25 00:00:00.000000000 Z
+date: 2015-11-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: msgpack
@@ -181,7 +181,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.5
+rubygems_version: 2.4.5.1
 signing_key: 
 specification_version: 4
 summary: fluent logger for ruby