fluent-logger 0.8.0 → 0.9.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: eb07188f881f645087ed46e08dcb033abd3e78f3
-  data.tar.gz: c5e7715e87ebf4bb7ca5f5a0afb6297d4772011b
+SHA256:
+  metadata.gz: 00161d7e89f54b0c1627bb5b0ce36a7689405b84da319db5c4052e585c09f6a7
+  data.tar.gz: fc7229efbe7fb40412fe22080aa974156f49f7d5e4257c0071d82eb7359cca72
 SHA512:
-  metadata.gz: 44b031f5063e83cac44e704f8a3fe95d1fa8bdf732f68dad096bd6da73b41c31c480412c00c1fe006f9b2bd6285e626f6ab6bc3096115ac559c3961ffd3c9a59
-  data.tar.gz: d3929bbf74cc3e77fc76149f98a0cd001f34213a715c807290cdc7bdc88df35d4d11d07d963a6a765ad144552ee99fb8cb191d7dce8f28e91e4e4c5c053ef37c
+  metadata.gz: 1066053b93cf4173b29e063659d7f80bb42b77a116b85b2a18f3d43cf19c54538359336b4b120d0ff43b65cc58110ae8a3357a52f417a0379ea09c306e27b808
+  data.tar.gz: 8c7839a41b615ccf34ebcd656c1719ba99245c39abe9981969b8cb7d1a9cf57ad972dfb8276ebffe4c8c132d95d2a1dfc1db5b354bfc720b4a7c569722688a33

data/.travis.yml

@@ -1,15 +1,12 @@
 rvm:
- - 2.1
- - 2.2
- - 2.3.7
- - 2.4.5
- - 2.5.3
- - 2.6.0
+ - 2.4.10
+ - 2.5
+ - 2.6
+ - 2.7
  - ruby-head
 
 gemfile:
  - Gemfile
- - Gemfile.v0.12
 
 before_install: gem update bundler
 script: bundle exec rake spec
@@ -19,16 +16,3 @@ sudo: false
 matrix:
   allow_failures:
     - rvm: ruby-head
-  exclude:
-    - rvm: 2.2
-      gemfile: Gemfile.v0.12
-    - rvm: 2.3.7
-      gemfile: Gemfile.v0.12
-    - rvm: 2.4.5
-      gemfile: Gemfile.v0.12
-    - rvm: 2.5.3
-      gemfile: Gemfile.v0.12
-    - rvm: 2.6.0
-      gemfile: Gemfile.v0.12
-    - rvm: ruby-head
-      gemfile: Gemfile.v0.12

data/ChangeLog

@@ -1,3 +1,16 @@
+Release 0.9.0 - 2020/09/04
+
+  * FluentLogger supports TLS
+
+Release 0.8.2 - 2019/08/21
+
+  * Block timeout during IO#write to avoid writing invalid bytes
+  * Fluent::Logger::EventTime#to_json returns String
+
+Release 0.8.1 - 2019/05/30
+
+  * Improve non-blocking write handling
+
 Release 0.8.0 - 2019/01/25
 
   * Add use_nonblock and wait_writeable parameters
@@ -130,4 +143,3 @@ Release 0.2.0 - 2011/08/05
 Release 0.1.0 - 2011/08/04
 
   * First release
-

data/README.md

@@ -4,6 +4,22 @@
 
 A structured event logger
 
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'fluent-logger'
+```
+
+And then execute:
+
+    $ bundle install
+
+Or install it yourself as:
+
+    $ gem install fluent-logger
+
 ## Examples
 
 ### Simple
@@ -27,6 +43,7 @@ require 'fluent-logger'
 
 log = Fluent::Logger::FluentLogger.new(nil, :socket_path => "/tmp/fluent.sock")
 unless log.post("myapp.access", {"agent" => "foo"})
+  # Passed records are stored into logger's internal buffer so don't re-post same event.
   p log.last_error # You can get last error object via last_error method
 end
 
@@ -49,15 +66,52 @@ log.post("access", {"agent" => "foo"})
 require 'fluent-logger'
 
 log = Fluent::Logger::FluentLogger.new(nil, :host => 'localhost', :port => 24224, :use_nonblock => true, :wait_writeable => false)
+# When wait_writeable is false
 begin
   log.post("myapp.access", {"agent" => "foo"})
 rescue IO::EAGAINWaitWritable => e
   # wait code for avoding "Resource temporarily unavailable"
+  # Passed records are stored into logger's internal buffer so don't re-post same event.
+end
+
+# When wait_writeable is true
+unless log.post("myapp.access", {"agent" => "foo"})
+  # same as other example
 end
 
 # output: myapp.access {"agent":"foo"}
 ```
 
+### TLS setting
+
+```ruby
+require 'fluent-logger'
+
+tls_opts = {
+  :ca   => '/path/to/cacert.pem',
+  :cert => '/path/to/client-cert.pem',
+  :key  => '/path/to/client-key.pem',
+  :key_passphrase => 'test'
+}
+log = Fluent::Logger::FluentLogger.new(nil, :host => 'localhost', :port => 24224, :tls_options => tls_opts)
+```
+
+`in_forward` config example:
+
+```
+<source>
+  @type forward
+  <transport tcp>
+    version TLS1_2
+    ca_path /path/to/cacert.pem
+    cert_path /path/to/server-cert.pem
+    private_key_path /path/to/server-key.pem
+    private_key_passphrase test
+    client_cert_auth true
+  </transport>
+</source>
+```
+
 ### Singleton
 ```ruby
 require 'fluent-logger'
@@ -88,16 +142,29 @@ Use nano second event time instead of epoch. See also "Tips" section.
 
 #### use_nonblock (Bool)
 
-Use nonblocking write(`IO#write_nonblock`) instead of normal write(`IO#write`). If `Logger#post` stuck on your environment, specify `true`.
+Use nonblocking write(`IO#write_nonblock`) instead of normal write(`IO#write`). If `Logger#post` stuck on your environment, specify `true`.  Default: `false`
 
 #### wait_writeable (Bool)
 
-If `true`, `Logger#post` raises an error when nonblocking write gets `EAGAIN`.
+If `false`, `Logger#post` raises an error when nonblocking write gets `EAGAIN` (i.e. `use_nonblock` must be `true`, otherwise this will have no effect).  Default: `true`
 
 #### buffer_overflow_handler (Proc)
 
 Pass callback for handling buffer overflow with pending data. See "Buffer overflow" section.
 
+#### tls_options (Hash)
+
+Pass TLS related options.
+
+- use_default_ca: Set `true` if you want to use default CA
+- ca: CA file path
+- cert: Certificate file path
+- key: Private key file path
+- key_passphrase: Private key passphrase
+- version: TLS version. Default is `OpenSSL::SSL::TLS1_2_VERSION`
+- ciphers: The list of cipher suites. Default is `ALL:!aNULL:!eNULL:!SSLv2`
+- insecure: Set `true` when `in_forward` uses `insecure true`
+
 ### Standard ::Logger compatible interface
 
 #### Example1

data/lib/fluent/logger/fluent_logger.rb

@@ -15,8 +15,10 @@
 #    See the License for the specific language governing permissions and
 #    limitations under the License.
 #
+require 'timeout'
 require 'msgpack'
 require 'socket'
+require 'openssl'
 require 'monitor'
 require 'logger'
 require 'json'
@@ -44,7 +46,7 @@ module Fluent
       end
 
       def to_json(*args)
-        @sec
+        @sec.to_s
       end
     end
 
@@ -83,6 +85,7 @@ module Fluent
         @socket_path = options[:socket_path]
         @nanosecond_precision = options[:nanosecond_precision]
         @use_nonblock = options[:use_nonblock]
+        @tls_options = options[:tls_options]
 
         @factory = MessagePack::Factory.new
         if @nanosecond_precision
@@ -126,7 +129,6 @@ module Fluent
       end
 
       attr_accessor :limit, :logger, :log_reconnect_error_threshold
-      attr_reader :last_error
 
       def last_error
         @last_error[Thread.current.object_id]
@@ -169,6 +171,33 @@ module Fluent
           @con = UNIXSocket.new(@socket_path)
         else
           @con = TCPSocket.new(@host, @port)
+          if @tls_options
+            context = OpenSSL::SSL::SSLContext.new
+            if @tls_options[:insecure]
+              context.verify_mode = OpenSSL::SSL::VERIFY_NONE
+            else
+              context.set_params({})
+              context.verify_mode = OpenSSL::SSL::VERIFY_PEER
+              cert_store = OpenSSL::X509::Store.new
+              if @tls_options[:use_default_ca]
+                cert_store.set_default_paths
+              end
+              if @tls_options[:ca]
+                cert_store.add_file(@tls_options[:ca])
+              end
+
+              context.cert = OpenSSL::X509::Certificate.new(File.read(@tls_options[:cert])) if @tls_options[:cert]
+              context.key = OpenSSL::PKey::read(File.read(@tls_options[:key]), @tls_options[:key_passphrase]) if @tls_options[:key]
+              context.ciphers = @tls_options[:ciphers] || "ALL:!aNULL:!eNULL:!SSLv2".freeze
+              context.cert_store = cert_store
+            end
+            set_tls_version(context)
+
+            @con = OpenSSL::SSL::SSLSocket.new(@con, context)
+            @con.sync_close = true
+            @con.connect
+          end
+          @con
         end
       end
 
@@ -186,6 +215,16 @@ module Fluent
 
       private
 
+      def set_tls_version(context)
+        if context.respond_to?(:min_version=)
+          ver = @tls_options[:version] || OpenSSL::SSL::TLS1_2_VERSION
+          context.min_version = ver
+          context.max_version = ver
+        else
+          context.ssl_version = @tls_options[:version] || :'TLSv1_2'
+        end
+      end
+
       def to_msgpack(msg)
         @mon.synchronize {
           res = begin
@@ -230,7 +269,11 @@ module Fluent
           end
 
           begin
-            send_data(@pending)
+            written = send_data(@pending)
+            if @pending.bytesize != written
+              raise "Actual written data size(#{written} bytes) is different from the received data size(#{@pending.bytesize} bytes)."
+            end
+
             @pending = nil
             true
           rescue => e
@@ -255,9 +298,13 @@ module Fluent
           connect!
         end
         if @use_nonblock
-          @con.write_nonblock data
+          send_data_nonblock(data)
         else
-          @con.write data
+          _, ws = IO.select([], [@con])
+          Thread.handle_interrupt(::Timeout::Error => :never) do
+            # block timeout error during IO#write
+            ws.first.write(data)
+          end
         end
         #while true
         #  puts "sending #{data.length} bytes"
@@ -272,7 +319,19 @@ module Fluent
         #  end
         #  data = data[n..-1]
         #end
-        true
+      end
+
+      def send_data_nonblock(data)
+        written = @con.write_nonblock(data)
+        remaining = data.bytesize - written
+
+        while remaining > 0
+          len = @con.write_nonblock(data.byteslice(written, remaining))
+          remaining -= len
+          written += len
+        end
+
+        written
       end
 
       def connect!

data/lib/fluent/logger/version.rb

@@ -1,5 +1,5 @@
 module Fluent
   module Logger
-    VERSION = '0.8.0'
+    VERSION = '0.9.0'
   end
 end

data/spec/fluent_logger_spec.rb

@@ -58,6 +58,57 @@ describe Fluent::Logger::FluentLogger do
       @serverengine.shutdown
     end
 
+    describe('testing interaction of use_nonblock and wait_writeable') do
+      before(:example) do
+        allow_any_instance_of(TCPSocket).to receive(:write_nonblock).and_raise(IO::EAGAINWaitWritable)
+        allow_any_instance_of(TCPSocket).to receive(:write) { |_, buf| buf.size }
+      end
+
+      context('use_nonblock is false') do
+        let(:block_config) { logger_config.merge(use_nonblock: false) }
+
+        it('post returns true when wait_writeable is false') {
+          cfg = block_config.merge(wait_writeable: false)
+          l = Fluent::Logger::FluentLogger.new('logger-test', cfg)
+          expect(l.post('hello', foo: 'bar')).to eq true
+        }
+
+        it('post returns true when wait_writeable is true') {
+          cfg = block_config.merge(wait_writeable: true)
+          l = Fluent::Logger::FluentLogger.new('logger-test', cfg)
+          expect(l.post('hello', {foo: 'bar'})).to eq true
+        }
+      end
+
+      context('use_nonblock is true') do
+        let(:nonblock_config) { logger_config.merge(use_nonblock: true) }
+
+        it('post raises IO::EAGAINWaitWritable when wait_writeable is false') {
+          cfg = nonblock_config.merge(wait_writeable: false)
+          l = Fluent::Logger::FluentLogger.new('logger-test', cfg)
+          expect { l.post('hello', foo: 'bar') }.to raise_error(IO::EAGAINWaitWritable)
+        }
+
+        it('post returns false when wait_writeable is true') {
+          cfg = nonblock_config.merge(wait_writeable: true)
+          l = Fluent::Logger::FluentLogger.new('logger-test', cfg)
+          expect(l.post('hello', {foo: 'bar'})).to eq false
+        }
+
+        context 'when write_nonblock returns the size less than received data' do
+          before do
+            allow_any_instance_of(TCPSocket).to receive(:write_nonblock).and_return(1) # write 1 bytes per call
+          end
+
+          it 'buffering data and flush at closed time' do
+            logger = Fluent::Logger::FluentLogger.new('logger-test', nonblock_config)
+            expect(logger.post('hello', foo: 'bar')).to eq(true)
+            expect(logger.pending_bytesize).to eq(0)
+          end
+        end
+      end
+    end
+
     context('Post by CUI') do
       it('post') {
         args = %W(-h localhost -p #{fluentd.port} -t logger-test.tag -v a=b -v foo=bar)
@@ -86,6 +137,16 @@ describe Fluent::Logger::FluentLogger do
         }
       end
 
+      context 'when the message has object which does not have #to_msgpack method' do
+        it 'success with nanosecond' do
+          expect(logger_with_nanosec.pending_bytesize).to eq(0)
+          expect(logger_with_nanosec.post('tag', 'a' => Errno::ETIMEDOUT)).to eq(true)
+          fluentd.wait_transfer
+          expect(fluentd.queue.last).to eq(['logger-test.tag', { 'a' => 'Errno::ETIMEDOUT' }])
+          expect(logger_with_nanosec.pending_bytesize).to eq(0)
+        end
+      end
+
       it ('close after post') {
         expect(logger).to be_connect
         logger.close
@@ -312,4 +373,51 @@ describe Fluent::Logger::FluentLogger do
       end
     end
   end
+
+  context "running fluentd with TLS" do
+    before(:all) do
+      @serverengine = DummyServerengine.new
+      @serverengine.startup
+    end
+
+    before(:each) do
+      fluentd.startup(true)
+    end
+
+    after(:each) do
+      fluentd.shutdown
+    end
+
+    after(:all) do
+      @serverengine.shutdown
+    end
+
+    let(:logger_config) {
+      {
+        :host   => 'localhost',
+        :port   => fluentd.port,
+        :logger => internal_logger,
+        :buffer_overflow_handler => buffer_overflow_handler,
+        :tls_options => {:insecure => true}
+      }
+    }
+
+    context('post') do
+      it ('success') {
+        expect(logger.pending_bytesize).to eq 0
+        expect(logger.post('tag', {'a' => 'b'})).to be true
+        fluentd.wait_transfer
+        expect(fluentd.queue.last).to eq ['logger-test.tag', {'a' => 'b'}]
+        expect(logger.pending_bytesize).to eq 0
+      }
+
+      it ('success with nanosecond') {
+        expect(logger_with_nanosec.pending_bytesize).to eq 0
+        expect(logger_with_nanosec.post('tag', {'a' => 'b'})).to be true
+        fluentd.wait_transfer
+        expect(fluentd.queue.last).to eq ['logger-test.tag', {'a' => 'b'}]
+        expect(fluentd.output.emits.first[1]).to be_a_kind_of(Fluent::EventTime)
+      }
+    end
+  end
 end

data/spec/support/dummy_fluentd.rb

@@ -51,8 +51,22 @@ class DummyFluentd
     queue
   end
 
-  def startup
-    config = Fluent::Config.parse(<<EOF, '(logger-spec)', '(logger-spec-dir)', true)
+  def startup(with_tls = false)
+    if with_tls
+      config = Fluent::Config.parse(<<EOF, '(logger-spec)', '(logger-spec-dir)', true)
+<source>
+  type forward
+  port #{port}
+  <transport tls>
+    insecure true
+  </transport>
+</source>
+<match logger-test.**>
+  type test
+</match>
+EOF
+    else
+      config = Fluent::Config.parse(<<EOF, '(logger-spec)', '(logger-spec-dir)', true)
 <source>
   type forward
   port #{port}
@@ -61,6 +75,8 @@ class DummyFluentd
   type test
 </match>
 EOF
+    end
+
     Fluent::Test.setup
     Fluent::Engine.run_configure(config)
     @coolio_default_loop = nil

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-logger
 version: !ruby/object:Gem::Version
-  version: 0.8.0
+  version: 0.9.0
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-01-25 00:00:00.000000000 Z
+date: 2020-09-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: msgpack
@@ -161,8 +161,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.6.14.1
+rubygems_version: 3.1.2
 signing_key: 
 specification_version: 4
 summary: fluent logger for ruby