fluent-logger 0.7.1 → 0.9.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 138aa1f6b2e893e052fa4aab33e38ffe50a001da
-  data.tar.gz: d6fb45180e4d79317df370ffd14ef8acb659ffab
+SHA256:
+  metadata.gz: 00161d7e89f54b0c1627bb5b0ce36a7689405b84da319db5c4052e585c09f6a7
+  data.tar.gz: fc7229efbe7fb40412fe22080aa974156f49f7d5e4257c0071d82eb7359cca72
 SHA512:
-  metadata.gz: 4b402b52625c8f1676eaece98d86e5aad61fda1d05c0623f926caf9074afdf589efca44851aefdead592212addc4124874ea27903f3861b867ca7336f65f5f47
-  data.tar.gz: 921a3538d1e758502cf9986d2857c5f41e676c5dc5af6a72c57ac08d29e4889c0dbc9e786ef0d855230a29a965efcc0654c1c7e85da0fcbfa71d847c9ab76124
+  metadata.gz: 1066053b93cf4173b29e063659d7f80bb42b77a116b85b2a18f3d43cf19c54538359336b4b120d0ff43b65cc58110ae8a3357a52f417a0379ea09c306e27b808
+  data.tar.gz: 8c7839a41b615ccf34ebcd656c1719ba99245c39abe9981969b8cb7d1a9cf57ad972dfb8276ebffe4c8c132d95d2a1dfc1db5b354bfc720b4a7c569722688a33

data/.travis.yml

@@ -1,15 +1,12 @@
 rvm:
- - 1.9.3
- - 2.0.0
- - 2.1
- - 2.2
- - 2.3.1
- - 2.4.0
+ - 2.4.10
+ - 2.5
+ - 2.6
+ - 2.7
  - ruby-head
 
 gemfile:
  - Gemfile
- - Gemfile.v0.12
 
 before_install: gem update bundler
 script: bundle exec rake spec
@@ -19,16 +16,3 @@ sudo: false
 matrix:
   allow_failures:
     - rvm: ruby-head
-  exclude:
-    - rvm: 1.9.3
-      gemfile: Gemfile
-    - rvm: 2.0.0
-      gemfile: Gemfile
-    - rvm: 2.2
-      gemfile: Gemfile.v0.12
-    - rvm: 2.3.1
-      gemfile: Gemfile.v0.12
-    - rvm: 2.4.0
-      gemfile: Gemfile.v0.12
-    - rvm: ruby-head
-      gemfile: Gemfile.v0.12

data/ChangeLog

@@ -1,3 +1,24 @@
+Release 0.9.0 - 2020/09/04
+
+  * FluentLogger supports TLS
+
+Release 0.8.2 - 2019/08/21
+
+  * Block timeout during IO#write to avoid writing invalid bytes
+  * Fluent::Logger::EventTime#to_json returns String
+
+Release 0.8.1 - 2019/05/30
+
+  * Improve non-blocking write handling
+
+Release 0.8.0 - 2019/01/25
+
+  * Add use_nonblock and wait_writeable parameters
+
+Release 0.7.2 - 2018/01/04
+
+  * Fix EventTime for nanosecond_precision
+
 Release 0.7.1 - 2017/04/19
 
   * Fix packer clear bug for non-msgpackable data
@@ -122,4 +143,3 @@ Release 0.2.0 - 2011/08/05
 Release 0.1.0 - 2011/08/04
 
   * First release
-

data/README.md

@@ -4,6 +4,22 @@
 
 A structured event logger
 
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'fluent-logger'
+```
+
+And then execute:
+
+    $ bundle install
+
+Or install it yourself as:
+
+    $ gem install fluent-logger
+
 ## Examples
 
 ### Simple
@@ -11,6 +27,7 @@ A structured event logger
 ```ruby
 require 'fluent-logger'
 
+# API: FluentLogger.new(tag_prefix, options)
 log = Fluent::Logger::FluentLogger.new(nil, :host => 'localhost', :port => 24224)
 unless log.post("myapp.access", {"agent" => "foo"})
   p log.last_error # You can get last error object via last_error method
@@ -26,32 +43,128 @@ require 'fluent-logger'
 
 log = Fluent::Logger::FluentLogger.new(nil, :socket_path => "/tmp/fluent.sock")
 unless log.post("myapp.access", {"agent" => "foo"})
+  # Passed records are stored into logger's internal buffer so don't re-post same event.
   p log.last_error # You can get last error object via last_error method
 end
 
 # output: myapp.access {"agent":"foo"}
 ```
 
-### Singleton
+### Tag prefix
 ```ruby
 require 'fluent-logger'
 
-Fluent::Logger::FluentLogger.open(nil, :host => 'localhost', :port => 24224)
-Fluent::Logger.post("myapp.access", {"agent" => "foo"})
+log = Fluent::Logger::FluentLogger.new('myapp', :host => 'localhost', :port => 24224)
+log.post("access", {"agent" => "foo"})
 
 # output: myapp.access {"agent":"foo"}
 ```
 
-### Tag prefix
+### Nonblocking write
+
 ```ruby
 require 'fluent-logger'
 
-log = Fluent::Logger::FluentLogger.new('myapp', :host => 'localhost', :port => 24224)
-log.post("access", {"agent" => "foo"})
+log = Fluent::Logger::FluentLogger.new(nil, :host => 'localhost', :port => 24224, :use_nonblock => true, :wait_writeable => false)
+# When wait_writeable is false
+begin
+  log.post("myapp.access", {"agent" => "foo"})
+rescue IO::EAGAINWaitWritable => e
+  # wait code for avoding "Resource temporarily unavailable"
+  # Passed records are stored into logger's internal buffer so don't re-post same event.
+end
+
+# When wait_writeable is true
+unless log.post("myapp.access", {"agent" => "foo"})
+  # same as other example
+end
 
 # output: myapp.access {"agent":"foo"}
 ```
 
+### TLS setting
+
+```ruby
+require 'fluent-logger'
+
+tls_opts = {
+  :ca   => '/path/to/cacert.pem',
+  :cert => '/path/to/client-cert.pem',
+  :key  => '/path/to/client-key.pem',
+  :key_passphrase => 'test'
+}
+log = Fluent::Logger::FluentLogger.new(nil, :host => 'localhost', :port => 24224, :tls_options => tls_opts)
+```
+
+`in_forward` config example:
+
+```
+<source>
+  @type forward
+  <transport tcp>
+    version TLS1_2
+    ca_path /path/to/cacert.pem
+    cert_path /path/to/server-cert.pem
+    private_key_path /path/to/server-key.pem
+    private_key_passphrase test
+    client_cert_auth true
+  </transport>
+</source>
+```
+
+### Singleton
+```ruby
+require 'fluent-logger'
+
+Fluent::Logger::FluentLogger.open(nil, :host => 'localhost', :port => 24224)
+Fluent::Logger.post("myapp.access", {"agent" => "foo"})
+
+# output: myapp.access {"agent":"foo"}
+```
+
+### Logger options
+
+#### host (String)
+
+fluentd instance host
+
+#### port (Integer)
+
+fluentd instance port
+
+#### socket_path (String)
+
+If specified, fluentd uses unix domain socket instead of TCP.
+
+#### nanosecond_precision (Bool)
+
+Use nano second event time instead of epoch. See also "Tips" section.
+
+#### use_nonblock (Bool)
+
+Use nonblocking write(`IO#write_nonblock`) instead of normal write(`IO#write`). If `Logger#post` stuck on your environment, specify `true`.  Default: `false`
+
+#### wait_writeable (Bool)
+
+If `false`, `Logger#post` raises an error when nonblocking write gets `EAGAIN` (i.e. `use_nonblock` must be `true`, otherwise this will have no effect).  Default: `true`
+
+#### buffer_overflow_handler (Proc)
+
+Pass callback for handling buffer overflow with pending data. See "Buffer overflow" section.
+
+#### tls_options (Hash)
+
+Pass TLS related options.
+
+- use_default_ca: Set `true` if you want to use default CA
+- ca: CA file path
+- cert: Certificate file path
+- key: Private key file path
+- key_passphrase: Private key passphrase
+- version: TLS version. Default is `OpenSSL::SSL::TLS1_2_VERSION`
+- ciphers: The list of cipher suites. Default is `ALL:!aNULL:!eNULL:!SSLv2`
+- insecure: Set `true` when `in_forward` uses `insecure true`
+
 ### Standard ::Logger compatible interface
 
 #### Example1

data/fluent-logger.gemspec

@@ -1,20 +1,10 @@
 # encoding: utf-8
 $:.push File.expand_path('../lib', __FILE__)
+require 'fluent/logger/version'
 
 Gem::Specification.new do |gem|
   version_file = "lib/fluent/logger/version.rb"
-  version = File.read("VERSION").strip
-  File.open(version_file, "w") {|f|
-    f.write <<EOF
-module Fluent
-module Logger
-
-VERSION = '#{version}'
-
-end
-end
-EOF
-  }
+  version = Fluent::Logger::VERSION
 
   gem.name        = %q{fluent-logger}
   gem.version     = version
@@ -29,6 +19,7 @@ EOF
   gem.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
   gem.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
   gem.require_paths = ['lib']
+  gem.license       = "Apache-2.0"
 
   gem.add_dependency "msgpack", ">= 1.0.0", "< 2"
   gem.add_development_dependency 'rake', '>= 0.9.2'

data/lib/fluent/logger/fluent_logger.rb

@@ -15,8 +15,10 @@
 #    See the License for the specific language governing permissions and
 #    limitations under the License.
 #
+require 'timeout'
 require 'msgpack'
 require 'socket'
+require 'openssl'
 require 'monitor'
 require 'logger'
 require 'json'
@@ -26,16 +28,17 @@ module Fluent
     class EventTime
       TYPE = 0
 
-      def initialize(raw_time)
-        @time = raw_time
+      def initialize(sec, nsec = 0)
+        @sec = sec
+        @nsec = nsec
       end
 
       def to_msgpack(io = nil)
-        @time.to_i.to_msgpack(io)
+        @sec.to_msgpack(io)
       end
 
       def to_msgpack_ext
-        [@time.to_i, @time.nsec].pack('NN')
+        [@sec, @nsec].pack('NN')
       end
 
       def self.from_msgpack_ext(data)
@@ -43,7 +46,7 @@ module Fluent
       end
 
       def to_json(*args)
-        @time.to_i
+        @sec.to_s
       end
     end
 
@@ -63,7 +66,8 @@ module Fluent
 
         options = {
           :host => 'localhost',
-          :port => 24224
+          :port => 24224,
+          :use_nonblock => false
         }
 
         case args.first
@@ -80,6 +84,8 @@ module Fluent
         @port = options[:port]
         @socket_path = options[:socket_path]
         @nanosecond_precision = options[:nanosecond_precision]
+        @use_nonblock = options[:use_nonblock]
+        @tls_options = options[:tls_options]
 
         @factory = MessagePack::Factory.new
         if @nanosecond_precision
@@ -95,8 +101,8 @@ module Fluent
         @log_reconnect_error_threshold = options[:log_reconnect_error_threshold] || RECONNECT_WAIT_MAX_COUNT
 
         @buffer_overflow_handler = options[:buffer_overflow_handler]
-        if logger = options[:logger]
-          @logger = logger
+        if options[:logger]
+          @logger = options[:logger]
         else
           @logger = ::Logger.new(STDERR)
           if options[:debug]
@@ -106,6 +112,9 @@ module Fluent
           end
         end
 
+        @wait_writeable = true
+        @wait_writeable = options[:wait_writeable] if options.key?(:wait_writeable)
+
         @last_error = {}
 
         begin
@@ -120,7 +129,6 @@ module Fluent
       end
 
       attr_accessor :limit, :logger, :log_reconnect_error_threshold
-      attr_reader :last_error
 
       def last_error
         @last_error[Thread.current.object_id]
@@ -130,7 +138,7 @@ module Fluent
         @logger.debug { "event: #{tag} #{map.to_json}" rescue nil } if @logger.debug?
         tag = "#{@tag_prefix}.#{tag}" if @tag_prefix
         if @nanosecond_precision && time.is_a?(Time)
-          write [tag, EventTime.new(time), map]
+          write [tag, EventTime.new(time.to_i, time.nsec), map]
         else
           write [tag, time.to_i, map]
         end
@@ -163,6 +171,33 @@ module Fluent
           @con = UNIXSocket.new(@socket_path)
         else
           @con = TCPSocket.new(@host, @port)
+          if @tls_options
+            context = OpenSSL::SSL::SSLContext.new
+            if @tls_options[:insecure]
+              context.verify_mode = OpenSSL::SSL::VERIFY_NONE
+            else
+              context.set_params({})
+              context.verify_mode = OpenSSL::SSL::VERIFY_PEER
+              cert_store = OpenSSL::X509::Store.new
+              if @tls_options[:use_default_ca]
+                cert_store.set_default_paths
+              end
+              if @tls_options[:ca]
+                cert_store.add_file(@tls_options[:ca])
+              end
+
+              context.cert = OpenSSL::X509::Certificate.new(File.read(@tls_options[:cert])) if @tls_options[:cert]
+              context.key = OpenSSL::PKey::read(File.read(@tls_options[:key]), @tls_options[:key_passphrase]) if @tls_options[:key]
+              context.ciphers = @tls_options[:ciphers] || "ALL:!aNULL:!eNULL:!SSLv2".freeze
+              context.cert_store = cert_store
+            end
+            set_tls_version(context)
+
+            @con = OpenSSL::SSL::SSLSocket.new(@con, context)
+            @con.sync_close = true
+            @con.connect
+          end
+          @con
         end
       end
 
@@ -180,6 +215,16 @@ module Fluent
 
       private
 
+      def set_tls_version(context)
+        if context.respond_to?(:min_version=)
+          ver = @tls_options[:version] || OpenSSL::SSL::TLS1_2_VERSION
+          context.min_version = ver
+          context.max_version = ver
+        else
+          context.ssl_version = @tls_options[:version] || :'TLSv1_2'
+        end
+      end
+
       def to_msgpack(msg)
         @mon.synchronize {
           res = begin
@@ -224,10 +269,17 @@ module Fluent
           end
 
           begin
-            send_data(@pending)
+            written = send_data(@pending)
+            if @pending.bytesize != written
+              raise "Actual written data size(#{written} bytes) is different from the received data size(#{@pending.bytesize} bytes)."
+            end
+
             @pending = nil
             true
           rescue => e
+            unless wait_writeable?(e)
+              raise e
+            end
             set_last_error(e)
             if pending_bytesize > @limit
               @logger.error("FluentLogger: Can't send logs to #{connection_string}: #{$!}")
@@ -245,7 +297,15 @@ module Fluent
         unless connect?
           connect!
         end
-        @con.write data
+        if @use_nonblock
+          send_data_nonblock(data)
+        else
+          _, ws = IO.select([], [@con])
+          Thread.handle_interrupt(::Timeout::Error => :never) do
+            # block timeout error during IO#write
+            ws.first.write(data)
+          end
+        end
         #while true
         #  puts "sending #{data.length} bytes"
         #  if data.length > 32*1024
@@ -259,7 +319,19 @@ module Fluent
         #  end
         #  data = data[n..-1]
         #end
-        true
+      end
+
+      def send_data_nonblock(data)
+        written = @con.write_nonblock(data)
+        remaining = data.bytesize - written
+
+        while remaining > 0
+          len = @con.write_nonblock(data.byteslice(written, remaining))
+          remaining -= len
+          written += len
+        end
+
+        written
       end
 
       def connect!
@@ -297,6 +369,14 @@ module Fluent
         # TODO: Check non GVL env
         @last_error[Thread.current.object_id] = e
       end
+
+      def wait_writeable?(e)
+        if e.instance_of?(IO::EAGAINWaitWritable)
+          @wait_writeable
+        else
+          true
+        end
+      end
     end
   end
 end

data/lib/fluent/logger/version.rb

@@ -1,7 +1,5 @@
 module Fluent
-module Logger
-
-VERSION = '0.7.1'
-
-end
+  module Logger
+    VERSION = '0.9.0'
+  end
 end

data/spec/fluent_logger_spec.rb

@@ -58,6 +58,57 @@ describe Fluent::Logger::FluentLogger do
       @serverengine.shutdown
     end
 
+    describe('testing interaction of use_nonblock and wait_writeable') do
+      before(:example) do
+        allow_any_instance_of(TCPSocket).to receive(:write_nonblock).and_raise(IO::EAGAINWaitWritable)
+        allow_any_instance_of(TCPSocket).to receive(:write) { |_, buf| buf.size }
+      end
+
+      context('use_nonblock is false') do
+        let(:block_config) { logger_config.merge(use_nonblock: false) }
+
+        it('post returns true when wait_writeable is false') {
+          cfg = block_config.merge(wait_writeable: false)
+          l = Fluent::Logger::FluentLogger.new('logger-test', cfg)
+          expect(l.post('hello', foo: 'bar')).to eq true
+        }
+
+        it('post returns true when wait_writeable is true') {
+          cfg = block_config.merge(wait_writeable: true)
+          l = Fluent::Logger::FluentLogger.new('logger-test', cfg)
+          expect(l.post('hello', {foo: 'bar'})).to eq true
+        }
+      end
+
+      context('use_nonblock is true') do
+        let(:nonblock_config) { logger_config.merge(use_nonblock: true) }
+
+        it('post raises IO::EAGAINWaitWritable when wait_writeable is false') {
+          cfg = nonblock_config.merge(wait_writeable: false)
+          l = Fluent::Logger::FluentLogger.new('logger-test', cfg)
+          expect { l.post('hello', foo: 'bar') }.to raise_error(IO::EAGAINWaitWritable)
+        }
+
+        it('post returns false when wait_writeable is true') {
+          cfg = nonblock_config.merge(wait_writeable: true)
+          l = Fluent::Logger::FluentLogger.new('logger-test', cfg)
+          expect(l.post('hello', {foo: 'bar'})).to eq false
+        }
+
+        context 'when write_nonblock returns the size less than received data' do
+          before do
+            allow_any_instance_of(TCPSocket).to receive(:write_nonblock).and_return(1) # write 1 bytes per call
+          end
+
+          it 'buffering data and flush at closed time' do
+            logger = Fluent::Logger::FluentLogger.new('logger-test', nonblock_config)
+            expect(logger.post('hello', foo: 'bar')).to eq(true)
+            expect(logger.pending_bytesize).to eq(0)
+          end
+        end
+      end
+    end
+
     context('Post by CUI') do
       it('post') {
         args = %W(-h localhost -p #{fluentd.port} -t logger-test.tag -v a=b -v foo=bar)
@@ -86,6 +137,16 @@ describe Fluent::Logger::FluentLogger do
         }
       end
 
+      context 'when the message has object which does not have #to_msgpack method' do
+        it 'success with nanosecond' do
+          expect(logger_with_nanosec.pending_bytesize).to eq(0)
+          expect(logger_with_nanosec.post('tag', 'a' => Errno::ETIMEDOUT)).to eq(true)
+          fluentd.wait_transfer
+          expect(fluentd.queue.last).to eq(['logger-test.tag', { 'a' => 'Errno::ETIMEDOUT' }])
+          expect(logger_with_nanosec.pending_bytesize).to eq(0)
+        end
+      end
+
       it ('close after post') {
         expect(logger).to be_connect
         logger.close
@@ -312,4 +373,51 @@ describe Fluent::Logger::FluentLogger do
       end
     end
   end
+
+  context "running fluentd with TLS" do
+    before(:all) do
+      @serverengine = DummyServerengine.new
+      @serverengine.startup
+    end
+
+    before(:each) do
+      fluentd.startup(true)
+    end
+
+    after(:each) do
+      fluentd.shutdown
+    end
+
+    after(:all) do
+      @serverengine.shutdown
+    end
+
+    let(:logger_config) {
+      {
+        :host   => 'localhost',
+        :port   => fluentd.port,
+        :logger => internal_logger,
+        :buffer_overflow_handler => buffer_overflow_handler,
+        :tls_options => {:insecure => true}
+      }
+    }
+
+    context('post') do
+      it ('success') {
+        expect(logger.pending_bytesize).to eq 0
+        expect(logger.post('tag', {'a' => 'b'})).to be true
+        fluentd.wait_transfer
+        expect(fluentd.queue.last).to eq ['logger-test.tag', {'a' => 'b'}]
+        expect(logger.pending_bytesize).to eq 0
+      }
+
+      it ('success with nanosecond') {
+        expect(logger_with_nanosec.pending_bytesize).to eq 0
+        expect(logger_with_nanosec.post('tag', {'a' => 'b'})).to be true
+        fluentd.wait_transfer
+        expect(fluentd.queue.last).to eq ['logger-test.tag', {'a' => 'b'}]
+        expect(fluentd.output.emits.first[1]).to be_a_kind_of(Fluent::EventTime)
+      }
+    end
+  end
 end

data/spec/support/dummy_fluentd.rb

@@ -51,8 +51,22 @@ class DummyFluentd
     queue
   end
 
-  def startup
-    config = Fluent::Config.parse(<<EOF, '(logger-spec)', '(logger-spec-dir)', true)
+  def startup(with_tls = false)
+    if with_tls
+      config = Fluent::Config.parse(<<EOF, '(logger-spec)', '(logger-spec-dir)', true)
+<source>
+  type forward
+  port #{port}
+  <transport tls>
+    insecure true
+  </transport>
+</source>
+<match logger-test.**>
+  type test
+</match>
+EOF
+    else
+      config = Fluent::Config.parse(<<EOF, '(logger-spec)', '(logger-spec-dir)', true)
 <source>
   type forward
   port #{port}
@@ -61,6 +75,8 @@ class DummyFluentd
   type test
 </match>
 EOF
+    end
+
     Fluent::Test.setup
     Fluent::Engine.run_configure(config)
     @coolio_default_loop = nil

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-logger
 version: !ruby/object:Gem::Version
-  version: 0.7.1
+  version: 0.9.0
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-04-18 00:00:00.000000000 Z
+date: 2020-09-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: msgpack
@@ -117,7 +117,6 @@ files:
 - Gemfile.v0.12
 - README.md
 - Rakefile
-- VERSION
 - bin/fluent-post
 - fluent-logger.gemspec
 - lib/fluent-logger.rb
@@ -144,7 +143,8 @@ files:
 - spec/support/timecop.rb
 - spec/test_logger_spec.rb
 homepage: https://github.com/fluent/fluent-logger-ruby
-licenses: []
+licenses:
+- Apache-2.0
 metadata: {}
 post_install_message: 
 rdoc_options: []
@@ -161,8 +161,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.6.11
+rubygems_version: 3.1.2
 signing_key: 
 specification_version: 4
 summary: fluent logger for ruby

data/VERSION

@@ -1 +0,0 @@
-0.7.1