fluent-plugin-syslog-tls 1.0.0 → 1.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 194b7c338bc48c3b97b41d51f5258157f0bf4fd1
-  data.tar.gz: 340359e27afeac0a38eb705cfac9cdf5c123cd1e
+  metadata.gz: 3d451a9db525bf51ce92a8fb6ba314e07ef4fde7
+  data.tar.gz: 578fa7e673a40b1a469bb7eb70681f0cfe102878
 SHA512:
-  metadata.gz: 66915ee3952bcc0eacd004e7b56bb3d7521bb2748c7921b0321dbac1ab09c8fcf356355e3b600287e13251cd545209cacd95edb6f579bfe382cba0309b88d14b
-  data.tar.gz: 8b2e05659598a9dc390350841455c9d003afa520e984de48f30d32974ea014c4b36e21c583edf60e3932cc9a4d0830f66cb32935f9d57d79ee5bb90e7533f847
+  metadata.gz: e8f97eb7e0028c5a41735c8c474128b0ec6181aaac527e5772c984a22ee9ba4e5e424cd1aceae806bebe5f42b4768e3647d8d3d07a44e4c0db14b630e7660a55
+  data.tar.gz: d8121c0078122038002ccb99c7888a125656e3728d2a75588408ccbd55e23563d5d5ce1ac6a7945ee68c8933c78549e401af02e2b76d7791360d0bd263123ee9

data/CHANGELOG.md

@@ -0,0 +1,17 @@
+#### 1.1.0
+
+* Renamed `cert` and `key` to `client_cert` and `client_key` respectively.
+* Change to short timeouts on network calls so logging doesn't go dead for extended periods.
+* Added `idle_timeout` to force upstream reconnection after a period of time with no traffic for a particular tag. Useful for low-traffic senders. Not recommended for high-traffic.
+* Added `ca_cert` to validate the remote certificate. Defaults to 'system' which uses the system certificate store.
+
+
+#### 1.0.0
+
+* Standard fluent formatting plugins are supported. Json output remains the default.
+* `token` (Structured Data in syslog terms) is now optional, for syslog hosts that don't require it.
+* Message payload in the syslog packet no longer duplicates Time or includes Tag by default.
+
+
+#### < 1.0.0
+From [Fluent::Plugin::SumologicCloudSyslog](https://github.com/acquia/fluent-plugin-sumologic-cloud-syslog)

data/README.md

@@ -17,6 +17,8 @@ or
 $ td-agent-gem install fluent-plugin-syslog-tls
 ```
 
+Note: `fluent-plugin-syslog-tls` is compatible with Fluent 0.14.
+
 
 ## Configuration
 ---
@@ -48,11 +50,7 @@ For more configuration options see [configuration docs](docs/configuration.md)
 
 ## Origin/History
 
-This plugin is derived from [Fluent::Plugin::SumologicCloudSyslog](https://github.com/acquia/fluent-plugin-sumologic-cloud-syslog). Changes from the original:
-
-* Standard fluent formatting plugins are supported. Json output remains the default.
-* `token` (Structured Data in syslog terms) is now optional, for syslog hosts that don't require it.
-* Message payload in the syslog packet no longer duplicates Time or includes Tag by default.
+This plugin is derived from [Fluent::Plugin::SumologicCloudSyslog](https://github.com/acquia/fluent-plugin-sumologic-cloud-syslog). Changes are in the [Changelog](CHANGELOG.md).
 
 
 ## License

data/docs/configuration.md

@@ -15,15 +15,27 @@ Host represents DNS name of endpoint where should be data sent. Example: `syslog
 
 Example: `6514`
 
+### idle_timeout
+
+If a given tag has gone this many seconds between log messages, disconnect and reconnect before sending logs. Useful in low-traffic logging situations with remote hosts that disconnect after a period of time. Disabled by default. Example: `600`
+
+### ca_cert
+
+Whether and how to verify the server's TLS certificate. Examples:
+* ca_cert system - Default; use the system CA certificate store (which must then be configured correctly)
+* ca_cert false - Disable verification; not recommended
+* ca_cert /path/to/file - A path+filename to a single CA file
+* ca_cert /path/to/dir/ - A directory of CA files (in format that OpenSSL can parse); must end with /
+
 ### token
 
 Some services require a token to identify the account. Example: `ABABABABABABA@99999`. Not required for Papertrail.
 
-### cert
+### client_cert
 
 Optionally path to client certificate for TLS connection. Example: `/path/to/crt/file.crt`
 
-### key
+### client_key
 
 Optionally path to client private key for TLS connection. Example: `/path/to/key/file.key`
 
@@ -72,6 +84,7 @@ Optionally record key where to get msgid from the record. If not provided nil va
   @type syslog_tls
   host logs1.papertrailapp.com
   port 12345
+  idle_timeout 720
 
   hostname static-hostname
   facility SYSLOG
@@ -87,7 +100,7 @@ Optionally record key where to get msgid from the record. If not provided nil va
   msgid_key ...
 
   # Fluent's standard formatting options are supported. Default is 'json'.
-  # Example: For Docker logs sent to Papertrail, sending only the log text:
+  # Example: For Docker logs sent to Papertrail, send only the log text:
   format single_value
   message_key log
 </match>
@@ -99,8 +112,8 @@ Optionally record key where to get msgid from the record. If not provided nil va
   host syslog.collection.us1.sumologic.com
   port 6514
   token [token]@[iana-id]
-  cert /path/to/cert/file.crt
-  key /path/to/key/file.key
+  client_cert /path/to/cert/file.crt
+  client_key /path/to/key/file.key
 
   hostname static-hostname
   facility SYSLOG

data/fluent-plugin-syslog-tls.gemspec

@@ -30,9 +30,9 @@ Gem::Specification.new do |s|
   s.executables   = s.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
   s.test_files    = s.files.grep(%r{^(test|spec|features)/})
   s.require_paths = ['lib']
-  s.required_ruby_version = '>= 2.0.0'
+  s.required_ruby_version = '>= 2.3.0'
 
-  s.add_runtime_dependency 'fluentd', '~> 0.12'
+  s.add_runtime_dependency 'fluentd', '~> 0.14.0'
   s.add_runtime_dependency 'fluent-mixin-config-placeholders', '~> 0.3'
 
   s.add_development_dependency 'minitest', '~> 5.8'

data/lib/fluent/plugin/out_syslog_tls.rb

@@ -29,11 +29,13 @@ module Fluent
 
     config_param :host, :string
     config_param :port, :integer
-    config_param :token, :string, :default => nil
-    config_param :cert, :string, :default => nil
-    config_param :key, :string, :default => nil
-    config_param :hostname, :string, :default => nil
-    config_param :facility, :string, :default => 'LOCAL0'
+    config_param :idle_timeout, :integer, default: nil
+    config_param :ca_cert, :string, default: 'system'
+    config_param :token, :string, default: nil
+    config_param :client_cert, :string, default: nil
+    config_param :client_key, :string, default: nil
+    config_param :hostname, :string, default: nil
+    config_param :facility, :string, default: 'LOCAL0'
 
     # Allow to map keys from record to syslog message headers
     SYSLOG_HEADERS = [
@@ -41,7 +43,7 @@ module Fluent
     ]
 
     SYSLOG_HEADERS.each do |key_name|
-      config_param "#{key_name}_key".to_sym, :string, :default => nil
+      config_param "#{key_name}_key".to_sym, :string, default: nil
     end
 
     config_section :format do
@@ -99,7 +101,7 @@ module Fluent
     end
 
     def new_logger(tag)
-      transport = ::SyslogTls::SSLTransport.new(host, port, cert: cert, key: key, max_retries: 3)
+      transport = ::SyslogTls::SSLTransport.new(host, port, idle_timeout: idle_timeout, ca_cert: ca_cert, client_cert: client_cert, client_key: client_key, max_retries: 3)
       logger = ::SyslogTls::Logger.new(transport, token)
       logger.facility(facility)
       logger.hostname(hostname)

data/lib/syslog_tls/ssl_transport.rb

@@ -1,4 +1,5 @@
 # Copyright 2016 Acquia, Inc.
+# Copyright 2016 t.e.morgan.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -18,17 +19,23 @@ require 'openssl'
 module SyslogTls
   # Supports SSL connection to remote host
   class SSLTransport
+    CONNECT_TIMEOUT = 10
+    # READ_TIMEOUT    = 5
+    WRITE_TIMEOUT   = 5
+
     attr_accessor :socket
 
-    attr_reader :host, :port, :cert, :key, :ssl_version
+    attr_reader :host, :port, :idle_timeout, :ca_cert, :client_cert, :client_key, :ssl_version
 
     attr_writer :retries
 
-    def initialize(host, port, cert: nil, key: nil, ssl_version: :TLSv1_2, max_retries: 1)
+    def initialize(host, port, idle_timeout: nil, ca_cert: 'system', client_cert: nil, client_key: nil, ssl_version: :TLSv1_2, max_retries: 1)
       @host = host
       @port = port
-      @cert = cert
-      @key = key
+      @idle_timeout = idle_timeout
+      @ca_cert = ca_cert
+      @client_cert = client_cert
+      @client_key = client_key
       @ssl_version = ssl_version
       @retries = max_retries
       connect
@@ -36,28 +43,96 @@ module SyslogTls
 
     def connect
       @socket = get_ssl_connection
-      @socket.connect
+      begin
+        begin
+          @socket.connect_nonblock
+        rescue Errno::EAGAIN, Errno::EWOULDBLOCK, IO::WaitReadable
+          select_with_timeout(@socket, :connect_read) && retry
+        rescue IO::WaitWritable
+          select_with_timeout(@socket, :connect_write) && retry
+        end
+      rescue Errno::ETIMEDOUT
+        raise 'Socket timeout during connect'
+      end
+      @last_write = Time.now if idle_timeout
+    end
+
+    def get_tcp_connection
+      tcp = nil
+
+      family = Socket::Constants::AF_UNSPEC
+      sock_type = Socket::Constants::SOCK_STREAM
+      addr_info = Socket.getaddrinfo(host, port, family, sock_type, nil, nil, false).first
+      _, port, _, address, family, sock_type = addr_info
+
+      begin
+        sock_addr = Socket.sockaddr_in(port, address)
+        tcp = Socket.new(family, sock_type, 0)
+        tcp.setsockopt(Socket::SOL_SOCKET, Socket::Constants::SO_REUSEADDR, true)
+        tcp.setsockopt(Socket::SOL_SOCKET, Socket::Constants::SO_REUSEPORT, true)
+        tcp.connect_nonblock(sock_addr)
+      rescue Errno::EINPROGRESS
+        select_with_timeout(tcp, :connect_write)
+        begin
+          tcp.connect_nonblock(sock_addr)
+        rescue Errno::EISCONN
+          # all good
+        rescue SystemCallError
+          tcp.close rescue nil
+          raise
+        end
+      rescue SystemCallError
+        tcp.close rescue nil
+        raise
+      end
+
+      tcp.setsockopt(Socket::IPPROTO_TCP, Socket::TCP_NODELAY, true)
+      tcp
     end
 
     def get_ssl_connection
-      tcp = TCPSocket.new(host, port)
+      tcp = get_tcp_connection
 
       ctx = OpenSSL::SSL::SSLContext.new
-      ctx.set_params(verify_mode: OpenSSL::SSL::VERIFY_PEER)
+      ctx.verify_mode = OpenSSL::SSL::VERIFY_PEER
       ctx.ssl_version = ssl_version
 
-      ctx.cert = OpenSSL::X509::Certificate.new(File.open(cert)) if cert
-      ctx.key = OpenSSL::PKey::RSA.new(File.open(key)) if key
-      OpenSSL::SSL::SSLSocket.new(tcp, ctx)
+      case ca_cert
+      when true, 'true', 'system'
+        # use system certs, same as openssl cli
+        ctx.cert_store = OpenSSL::X509::Store.new
+        ctx.cert_store.set_default_paths
+      when false, 'false'
+        ctx.verify_mode = OpenSSL::SSL::VERIFY_NONE
+      when %r{/$} # ends in /
+        ctx.ca_path = ca_cert
+      when String
+        ctx.ca_file = ca_cert
+      end
+
+      ctx.cert = OpenSSL::X509::Certificate.new(File.read(client_cert)) if client_cert
+      ctx.key = OpenSSL::PKey::read(File.read(client_key)) if client_key
+      socket = OpenSSL::SSL::SSLSocket.new(tcp, ctx)
+      socket.sync_close = true
+      socket
     end
 
     # Allow to retry on failed writes
     def write(s)
+      if idle_timeout
+        if (t=Time.now) > @last_write + idle_timeout
+          @socket.close rescue nil
+          connect
+        else
+          @last_write = t
+        end
+      end
       begin
         retry_id ||= 0
-        @socket.send(:write, s)
+        do_write(s)
       rescue => e
         if (retry_id += 1) < @retries
+          @socket.close rescue nil
           connect
           retry
         else
@@ -66,6 +141,41 @@ module SyslogTls
       end
     end
 
+    def do_write(data)
+      data.force_encoding('BINARY') # so we can break in the middle of multi-byte characters
+      loop do
+        sent = 0
+        begin
+          sent = @socket.write_nonblock(data)
+        rescue OpenSSL::SSL::SSLError, Errno::EAGAIN, Errno::EWOULDBLOCK, IO::WaitWritable => e
+          if e.is_a?(OpenSSL::SSL::SSLError) && e.message !~ /write would block/
+            raise e
+          else
+            select_with_timeout(@socket, :write) && retry
+          end
+        end
+
+        break if sent >= data.size
+        data = data[sent, data.size]
+      end
+    end
+
+    def select_with_timeout(tcp, type)
+      o = case type
+      when :connect_read
+        args = [[tcp], nil, nil, CONNECT_TIMEOUT]
+      when :connect_write
+        args = [nil, [tcp], nil, CONNECT_TIMEOUT]
+      # when :read
+      #   args = [[tcp], nil, nil, READ_TIMEOUT]
+      when :write
+        args = [nil, [tcp], nil, WRITE_TIMEOUT]
+      else
+        raise "Unknown select type #{type}"
+      end
+      IO.select(*args) || raise("Socket timeout during #{type}")
+    end
+
     # Forward any methods directly to SSLSocket
     def method_missing(method_sym, *arguments, &block)
       @socket.send(method_sym, *arguments, &block)

data/lib/syslog_tls/version.rb

@@ -14,5 +14,5 @@
 # limitations under the License.
 
 module SyslogTls
-  VERSION = '1.0.0'
+  VERSION = '1.1.0'
 end

data/test/fluent/test_out_syslog_tls.rb

@@ -52,16 +52,16 @@ class SyslogTlsOutputTest < Test::Unit::TestCase
     config = %{
       host   syslog.collection.us1.sumologic.com
       port   6514
-      cert
-      key
+      client_cert
+      client_key
       token  1234567890
     }
     instance = driver('test', config).instance
 
     assert_equal 'syslog.collection.us1.sumologic.com', instance.host
     assert_equal '6514', instance.port
-    assert_equal '', instance.cert
-    assert_equal '', instance.key
+    assert_equal '', instance.client_cert
+    assert_equal '', instance.client_key
     assert_equal '1234567890', instance.token
   end
 
@@ -69,8 +69,8 @@ class SyslogTlsOutputTest < Test::Unit::TestCase
     config = %{
       host   syslog.collection.us1.sumologic.com
       port   6514
-      cert
-      key
+      client_cert
+      client_key
     }
     instance = driver('test', config).instance
 
@@ -91,8 +91,8 @@ class SyslogTlsOutputTest < Test::Unit::TestCase
     config = %{
       host   syslog.collection.us1.sumologic.com
       port   6514
-      cert
-      key
+      client_cert
+      client_key
       token  1234567890
       hostname_key hostname
       procid_key procid
@@ -118,8 +118,8 @@ class SyslogTlsOutputTest < Test::Unit::TestCase
     config = %{
       host   syslog.collection.us1.sumologic.com
       port   6514
-      cert
-      key
+      client_cert
+      client_key
       token  1234567890
       severity_key severity
     }
@@ -142,11 +142,11 @@ class SyslogTlsOutputTest < Test::Unit::TestCase
     config = %{
       host   syslog.collection.us1.sumologic.com
       port   6514
-      cert
-      key
+      client_cert
+      client_key
       token  1234567890
       format out_file
-      utc    true
+      localtime false
     }
     instance = driver('test', config).instance
 
@@ -178,8 +178,8 @@ class SyslogTlsOutputTest < Test::Unit::TestCase
     config = %{
       host   localhost
       port   #{server.addr[1]}
-      cert
-      key
+      client_cert
+      client_key
       token  1234567890
       hostname_key hostname
       procid_key procid

data/test/syslog_tls/test_ssl_transport.rb

@@ -37,10 +37,10 @@ class SSLTransportTest < Test::Unit::TestCase
 
   def test_retry
     client = Object.new
-    def client.connect
+    def client.connect_nonblock
       true
     end
-    def client.write(s)
+    def client.write_nonblock(s)
       raise "Test"
     end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-syslog-tls
 version: !ruby/object:Gem::Version
-  version: 1.0.0
+  version: 1.1.0
 platform: ruby
 authors:
 - thomas morgan
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-11-04 00:00:00.000000000 Z
+date: 2016-11-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -16,14 +16,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.12'
+        version: 0.14.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.12'
+        version: 0.14.0
 - !ruby/object:Gem::Dependency
   name: fluent-mixin-config-placeholders
   requirement: !ruby/object:Gem::Requirement
@@ -132,6 +132,7 @@ files:
 - ".coveralls.yml"
 - ".gitignore"
 - ".travis.yml"
+- CHANGELOG.md
 - Gemfile
 - LICENSE
 - README.md
@@ -164,7 +165,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 2.0.0
+      version: 2.3.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
@@ -172,7 +173,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.5.1
+rubygems_version: 2.5.2
 signing_key: 
 specification_version: 4
 summary: Fluent Syslog TLS output plugin