fluent-plugin-secure-forward 0.2.6 → 0.3.0

data/lib/fluent/plugin/in_secure_forward.rb

@@ -8,6 +8,7 @@ module Fluent
 end
 
 require_relative 'input_session'
+require_relative './secure_forward/cert_util'
 
 module Fluent
   class SecureForwardInput < Input
@@ -15,36 +16,44 @@ module Fluent
 
     Fluent::Plugin.register_input('secure_forward', self)
 
+    config_param :secure, :bool # if secure, cert_path or ca_cert_path required
+
     config_param :self_hostname, :string
     include Fluent::Mixin::ConfigPlaceholders
 
     config_param :shared_key, :string
 
-    config_param :bind, :string, :default => '0.0.0.0'
-    config_param :port, :integer, :default => DEFAULT_SECURE_LISTEN_PORT
-    config_param :allow_keepalive, :bool, :default => true #TODO: implement
+    config_param :bind, :string, default: '0.0.0.0'
+    config_param :port, :integer, default: DEFAULT_SECURE_LISTEN_PORT
+    config_param :allow_keepalive, :bool, default: true #TODO: implement
+
+    config_param :allow_anonymous_source, :bool, default: true
+    config_param :authentication, :bool, default: false
 
-    config_param :allow_anonymous_source, :bool, :default => true
-    config_param :authentication, :bool, :default => false
+    config_param :ssl_version, :string, default: 'TLSv1_2'
+    config_param :ssl_ciphers, :string, default: nil
 
-    ## meaningless for security...? not implemented yet
-    # config_param :dns_reverse_lookup_check, :bool, :default => false
+    # Cert signed by public CA
+    config_param :cert_path, :string, default: nil
+    config_param :private_key_path, :string, default: nil
+    config_param :private_key_passphrase, :string, default: nil
 
-    config_param :cert_auto_generate, :bool, :default => false
-    config_param :generate_private_key_length, :integer, :default => 2048
+    # Cert automatically generated and signed by private CA
+    config_param :ca_cert_path, :string, default: nil
+    config_param :ca_private_key_path, :string, default: nil
+    config_param :ca_private_key_passphrase, :string, default: nil
 
-    config_param :generate_cert_country, :string, :default => 'US'
-    config_param :generate_cert_state, :string, :default => 'CA'
-    config_param :generate_cert_locality, :string, :default => 'Mountain View'
-    config_param :generate_cert_common_name, :string, :default => nil
+    # Otherwise: Cert automatically generated and signed by itself (for without any verification)
 
-    config_param :cert_file_path, :string, :default => nil
-    config_param :private_key_file, :string, :default => nil
-    config_param :private_key_passphrase, :string, :default => nil
+    config_param :generate_private_key_length, :integer, default: 2048
+    config_param :generate_cert_country, :string, default: 'US'
+    config_param :generate_cert_state, :string, default: 'CA'
+    config_param :generate_cert_locality, :string, default: 'Mountain View'
+    config_param :generate_cert_common_name, :string, default: nil
 
-    config_param :read_length, :size, :default => 8*1024*1024 # 8MB
-    config_param :read_interval_msec, :integer, :default => 50 # 50ms
-    config_param :socket_interval_msec, :integer, :default => 200 # 200ms
+    config_param :read_length, :size, default: 8*1024*1024 # 8MB
+    config_param :read_interval_msec, :integer, default: 50 # 50ms
+    config_param :socket_interval_msec, :integer, default: 200 # 200ms
 
     attr_reader :read_interval, :socket_interval
 
@@ -80,8 +89,19 @@ module Fluent
     def configure(conf)
       super
 
-      unless @cert_auto_generate || @cert_file_path
-        raise Fluent::ConfigError, "One of 'cert_auto_generate' or 'cert_file_path' must be specified"
+      if @secure
+        unless @cert_path || @ca_cert_path
+          raise Fluent::ConfigError, "cert_path or ca_cert_path required for secure communication"
+        end
+        if @cert_path
+          raise Fluent::ConfigError, "private_key_path required" unless @private_key_path
+          raise Fluent::ConfigError, "private_key_passphrase required" unless @private_key_passphrase
+        else # @ca_cert_path
+          raise Fluent::ConfigError, "ca_private_key_path required" unless @ca_private_key_path
+          raise Fluent::ConfigError, "ca_private_key_passphrase required" unless @ca_private_key_passphrase
+        end
+      else
+        log.warn "'insecure' mode has vulnerability for man-in-the-middle attacks for clients (output plugins)."
       end
 
       @read_interval = @read_interval_msec / 1000.0
@@ -117,7 +137,10 @@ module Fluent
       end
 
       @generate_cert_common_name ||= @self_hostname
+
+      # To check whether certificates are successfully generated/loaded at startup time
       self.certificate
+
       true
     end
 
@@ -127,6 +150,7 @@ module Fluent
       @sessions = []
       @sock = nil
       @listener = Thread.new(&method(:run))
+      @listener.abort_on_exception
     end
 
     def shutdown
@@ -147,44 +171,59 @@ module Fluent
     def certificate
       return @cert, @key if @cert && @key
 
-      if @cert_auto_generate
-        key = OpenSSL::PKey::RSA.generate(@generate_private_key_length)
-
-        digest = OpenSSL::Digest::SHA1.new
-        issuer = subject = OpenSSL::X509::Name.new
-        subject.add_entry('C', @generate_cert_country)
-        subject.add_entry('ST', @generate_cert_state)
-        subject.add_entry('L', @generate_cert_locality)
-        subject.add_entry('CN', @generate_cert_common_name)
-
-        cer = OpenSSL::X509::Certificate.new
-        cer.not_before = Time.at(0)
-        cer.not_after = Time.at(0)
-        cer.public_key = key
-        cer.serial = 1
-        cer.issuer = issuer
-        cer.subject  = subject
-        cer.sign(key, digest)
-
-        @cert = cer
-        @key = key
-        return @cert, @key
+      if @cert_path
+        @key = OpenSSL::PKey::RSA.new(File.read(@private_key_path), @private_key_passphrase)
+        @cert = OpenSSL::X509::Certificate.new(File.read(@cert_path))
+      elsif @ca_cert_path
+        opts = {
+          ca_cert_path: @ca_cert_path,
+          ca_key_path: @ca_private_key_path,
+          ca_key_passphrase: @ca_private_key_passphrase,
+          private_key_length: @generate_private_key_length,
+          country: @generate_cert_country,
+          state: @generate_cert_state,
+          locality: @generate_cert_locality,
+          common_name: @generate_cert_common_name,
+        }
+        @cert, @key = Fluent::SecureForward::CertUtil.generate_server_pair(opts)
+      else
+        opts = {
+          private_key_length: @generate_private_key_length,
+          country: @generate_cert_country,
+          state: @generate_cert_state,
+          locality: @generate_cert_locality,
+          common_name: @generate_cert_common_name,
+        }
+        @cert, @key = Fluent::SecureForward::CertUtil.generate_self_signed_server_pair(opts)
       end
-
-      @cert = OpenSSL::X509::Certificate.new(File.read(@cert_file_path))
-      @key = OpenSSL::PKey::RSA.new(File.read(@private_key_file), @private_key_passphrase)
+      return @cert, @key
     end
 
     def run # sslsocket server thread
       log.trace "setup for ssl sessions"
       cert, key = self.certificate
-      ctx = OpenSSL::SSL::SSLContext.new
+
+      ctx = OpenSSL::SSL::SSLContext.new(@ssl_version)
+      if @secure
+        # inject OpenSSL::SSL::SSLContext::DEFAULT_PARAMS
+        # https://bugs.ruby-lang.org/issues/9424
+        ctx.set_params({})
+
+        if @ssl_ciphers
+          ctx.ciphers = @ssl_ciphers
+        else
+          ### follow httpclient configuration by nahi
+          # OpenSSL 0.9.8 default: "ALL:!ADH:!LOW:!EXP:!MD5:+SSLv2:@STRENGTH"
+          ctx.ciphers = "ALL:!aNULL:!eNULL:!SSLv2" # OpenSSL >1.0.0 default
+        end
+      end
+
       ctx.cert = cert
       ctx.key = key
 
-      log.trace "start to listen", :bind => @bind, :port => @port
+      log.trace "start to listen", bind: @bind, port: @port
       server = TCPServer.new(@bind, @port)
-      log.trace "starting SSL server", :bind => @bind, :port => @port
+      log.trace "starting SSL server", bind: @bind, port: @port
       @sock = OpenSSL::SSL::SSLServer.new(server, ctx)
       @sock.start_immediately = false
       begin
@@ -196,7 +235,7 @@ module Fluent
 
             # cleanup closed session instance
             @sessions.delete_if(&:closed?)
-            log.trace "session instances:", :all => @sessions.size, :closed => @sessions.select(&:closed?).size
+            log.trace "session instances:", all: @sessions.size, closed: @sessions.select(&:closed?).size
           end
         end
       rescue OpenSSL::SSL::SSLError => e

data/lib/fluent/plugin/input_session.rb

@@ -151,7 +151,7 @@ class Fluent::SecureForwardInput::Session
     begin
       @socket.accept
     rescue OpenSSL::SSL::SSLError => e
-      log.debug "failed to establish ssl session"
+      log.debug "failed to establish ssl session", error_class: e.class, error: e
       self.shutdown
       return
     end
@@ -195,7 +195,7 @@ class Fluent::SecureForwardInput::Session
   rescue Errno::ECONNRESET => e
     # disconnected from client
   rescue => e
-    log.warn "unexpected error in in_secure_forward", :error_class => e.class, :error => e
+    log.warn "unexpected error in in_secure_forward", error_class: e.class, error: e
   ensure
     self.shutdown
   end

data/lib/fluent/plugin/out_secure_forward.rb

@@ -15,26 +15,31 @@ module Fluent
 
     Fluent::Plugin.register_output('secure_forward', self)
 
+    config_param :secure, :bool
+
     config_param :self_hostname, :string
     include Fluent::Mixin::ConfigPlaceholders
 
     config_param :shared_key, :string
 
-    config_param :keepalive, :time, :default => nil # nil/0 means disable keepalive expiration
+    config_param :keepalive, :time, default: nil # nil/0 means disable keepalive expiration
 
-    config_param :send_timeout, :time, :default => 60
+    config_param :send_timeout, :time, default: 60
     # config_param :hard_timeout, :time, :default => 60
     # config_param :expire_dns_cache, :time, :default => 0 # 0 means disable cache
 
-    config_param :allow_self_signed_certificate, :bool, :default => true
-    config_param :ca_file_path, :string, :default => nil
+    config_param :ca_cert_path, :string, default: nil
+
+    config_param :enable_strict_verification, :bool, default: nil # FQDN check with hostlabel
+    config_param :ssl_version, :string, default: 'TLSv1_2'
+    config_param :ssl_ciphers, :string, default: nil
 
-    config_param :read_length, :size, :default => 512 # 512bytes
-    config_param :read_interval_msec, :integer, :default => 50 # 50ms
-    config_param :socket_interval_msec, :integer, :default => 200 # 200ms
+    config_param :read_length, :size, default: 512 # 512bytes
+    config_param :read_interval_msec, :integer, default: 50 # 50ms
+    config_param :socket_interval_msec, :integer, default: 200 # 200ms
 
-    config_param :reconnect_interval, :time, :default => 5
-    config_param :established_timeout, :time, :default => 10
+    config_param :reconnect_interval, :time, default: 5
+    config_param :established_timeout, :time, default: 10
 
     attr_reader :read_interval, :socket_interval
 
@@ -68,8 +73,20 @@ module Fluent
     def configure(conf)
       super
 
-      unless @allow_self_signed_certificate
-        raise Fluent::ConfigError, "not tested yet!"
+      if @secure
+        if @ca_cert_path
+          raise Fluent::ConfigError, "CA cert file not found nor readable at '#{@ca_cert_path}'" unless File.readable?(@ca_cert_path)
+          begin
+            OpenSSL::X509::Certificate.new File.read(@ca_cert_path)
+          rescue OpenSSL::X509::CertificateError => e
+            raise Fluent::ConfigError, "failed to load CA cert file"
+          end
+        else
+          raise Fluent::ConfigError, "FQDN verification required for certificates issued from public CA" unless @enable_strict_verification
+          log.info "secure connection with valid certificates issued from public CA"
+        end
+      else
+        log.warn "'insecure' mode has vulnerability for man-in-the-middle attacks."
       end
 
       @read_interval = @read_interval_msec / 1000.0
@@ -89,7 +106,7 @@ module Fluent
       @next_node = 0
       @mutex = Mutex.new
 
-      @hostname_resolver = Resolve::Hostname.new(:system_resolver => true)
+      @hostname_resolver = Resolve::Hostname.new(system_resolver: true)
 
       true
     end
@@ -122,7 +139,7 @@ module Fluent
       OpenSSL::Random.seed(SecureRandom.random_bytes(16))
       log.debug "start to connect target nodes"
       @nodes.each do |node|
-        log.debug "connecting node", :host => node.host, :port => node.port
+        log.debug "connecting node", host: node.host, port: node.port
         node.start
       end
       @nodewatcher = Thread.new(&method(:node_watcher))
@@ -153,13 +170,13 @@ module Fluent
           end
 
           node = @nodes[i]
-          log.debug "reconnecting to node", :host => node.host, :port => node.port, :expire => node.expire, :expired => node.expired?, :detached => node.detached?
+          log.debug "reconnecting to node", host: node.host, port: node.port, expire: node.expire, expired: node.expired?, detached: node.detached?
 
           renewed = node.dup
           renewed.start
 
           Thread.pass # to connection thread
-          reconnectings[i] = { :conn => renewed, :at => Time.now, :reason => reason }
+          reconnectings[i] = { conn: renewed, at: Time.now, reason: reason }
         end
 
         (0...nodes_size).each do |i|
@@ -191,7 +208,7 @@ module Fluent
 
           # not connected yet, and timeout
           timeout_conn = reconnectings[i][:conn]
-          log.debug "SSL connection is not established until timemout", :host => timeout_conn.host, :port => timeout_conn.port, :timeout => @established_timeout
+          log.debug "SSL connection is not established until timemout", host: timeout_conn.host, port: timeout_conn.port, timeout: @established_timeout
           reconnectings[i] = nil
           timeout_conn.detach! if timeout_conn # connection object doesn't raise any exceptions
         end
@@ -215,13 +232,13 @@ module Fluent
       unless node
         raise "no one nodes with valid ssl session"
       end
-      log.trace "selected node", :host => node.host, :port => node.port, :standby => node.standby
+      log.trace "selected node", host: node.host, port: node.port, standby: node.standby
 
       begin
         send_data(node, tag, es)
         node.release!
       rescue Errno::EPIPE, IOError, OpenSSL::SSL::SSLError => e
-        log.warn "Failed to send messages to #{node.host}, parging.", :error_class => e.class, :error => e
+        log.warn "Failed to send messages to #{node.host}, parging.", error_class: e.class, error: e
         node.release!
         node.detach!
 

data/lib/fluent/plugin/output_node.rb

@@ -60,8 +60,6 @@ class Fluent::SecureForwardOutput::Node
 
   def start
     @thread = Thread.new(&method(:connect))
-    ## If you want to check code bug, turn this line enable
-    # @thread.abort_on_exception = true
   end
 
   def detach!
@@ -165,6 +163,10 @@ class Fluent::SecureForwardOutput::Node
       return false, 'authentication failed: ' + reason
     end
 
+    if hostname == @sender.self_hostname
+      return false, 'same hostname between input and output: invalid configuration'
+    end
+
     clientside = Digest::SHA512.new.update(@shared_key_salt).update(hostname).update(@shared_key).hexdigest
     unless shared_key_hexdigest == clientside
       return false, 'shared key mismatch'
@@ -204,19 +206,20 @@ class Fluent::SecureForwardOutput::Node
       log.info "connection established to #{@host}" if @first_session
       @state = :established
       @expire = Time.now + @keepalive if @keepalive && @keepalive > 0
-      log.debug "connection established", :host => @host, :port => @port, :expire => @expire
+      log.debug "connection established", host: @host, port: @port, expire: @expire
     end
   end
 
   def connect
+    Thread.current.abort_on_exception = true
     log.debug "starting client"
 
     addr = @sender.hostname_resolver.getaddress(@host)
-    log.debug "create tcp socket to node", :host => @host, :address => addr, :port => @port
+    log.debug "create tcp socket to node", host: @host, address: addr, port: @port
     begin
       sock = TCPSocket.new(addr, @port)
     rescue => e
-      log.warn "failed to connect for secure-forward", :error_class => e.class, :error => e, :host => @host, :address => addr, :port => @port
+      log.warn "failed to connect for secure-forward", error_class: e.class, error: e, host: @host, address: addr, port: @port
       @state = :failed
       return
     end
@@ -228,45 +231,65 @@ class Fluent::SecureForwardOutput::Node
     opt = [@sender.send_timeout.to_i, 0].pack('L!L!')  # struct timeval
     sock.setsockopt(Socket::SOL_SOCKET, Socket::SO_SNDTIMEO, opt)
 
-    # TODO: SSLContext constructer parameter (SSL/TLS protocol version)
     log.trace "initializing SSL contexts"
-    context = OpenSSL::SSL::SSLContext.new
-    # TODO: context.ca_file = (ca_file_path)
-    # TODO: context.ciphers = (SSL Shared key chiper protocols)
 
-    log.debug "trying to connect ssl session", :host => @host, :address => addr, :port => @port
+    context = OpenSSL::SSL::SSLContext.new(@sender.ssl_version)
+
+    log.trace "setting SSL verification options"
+
+    if @sender.secure
+      # inject OpenSSL::SSL::SSLContext::DEFAULT_PARAMS
+      # https://bugs.ruby-lang.org/issues/9424
+      context.set_params({})
+
+      if @sender.ssl_ciphers
+        context.ciphers = @sender.ssl_ciphers
+      else
+        ### follow httpclient configuration by nahi
+        # OpenSSL 0.9.8 default: "ALL:!ADH:!LOW:!EXP:!MD5:+SSLv2:@STRENGTH"
+        context.ciphers = "ALL:!aNULL:!eNULL:!SSLv2" # OpenSSL >1.0.0 default
+      end
+
+      log.trace "set verify_mode VERIFY_PEER"
+      context.verify_mode = OpenSSL::SSL::VERIFY_PEER
+      if @sender.ca_cert_path
+        log.trace "set to use private CA", path: @sender.ca_cert_path
+        context.ca_file = @sender.ca_cert_path
+      end
+    end
+
+    log.debug "trying to connect ssl session", host: @host, address: addr, port: @port
     begin
       sslsession = OpenSSL::SSL::SSLSocket.new(sock, context)
+      log.trace "connecting...", host: @host, address: addr, port: @port
+      sslsession.connect
     rescue => e
-      log.warn "failed to establish SSL connection", :host => @host, :address => addr, :port => @port
-    end
-
-    unless sslsession.connect
-      log.debug "failed to connect", :host => @host, :address => addr, :port => @port
+      log.warn "failed to establish SSL connection", error_class: e.class, error: e, host: @host, address: addr, port: @port
       @state = :failed
       return
     end
-    log.debug "ssl session connected", :host => @host, :port => @port
+
+    log.debug "ssl session connected", host: @host, port: @port
 
     begin
-      unless @sender.allow_self_signed_certificate
-        log.debug "checking peer's certificate", :subject => sslsession.peer_cert.subject
+      if @sender.enable_strict_verification
+        log.debug "checking peer's certificate", subject: sslsession.peer_cert.subject
         sslsession.post_connection_check(@hostlabel)
         verify = sslsession.verify_result
         if verify != OpenSSL::X509::V_OK
           err_name = Fluent::SecureForwardOutput::OpenSSLUtil.verify_result_name(verify)
-          log.warn "failed to verify certification while connecting host #{@host} as #{@hostlabel} (but not raised, why?)"
-          log.warn "verify_result: #{err_name}"
-          raise RuntimeError, "failed to verify certification while connecting host #{@host} as #{@hostlabel}"
+          log.warn "BUG: failed to verify certification while connecting host #{@host} as #{@hostlabel} (but not raised, why?)"
+          log.warn "BUG: verify_result: #{err_name}"
+          raise RuntimeError, "BUG: failed to verify certification and to handle it correctly while connecting host #{@host} as #{@hostlabel}"
         end
       end
     rescue OpenSSL::SSL::SSLError => e
-      log.warn "failed to verify certification while connecting ssl session", :host => @host, :hostlabel => @hostlabel
+      log.warn "failed to verify certification while connecting ssl session", host: @host, hostlabel: @hostlabel
       self.shutdown
       raise
     end
 
-    log.debug "ssl sessison connected", :host => @host, :port => @port
+    log.debug "ssl session connected", host: @host, port: @port
     @socket = sock
     @sslsession = sslsession
 

data/lib/fluent/plugin/secure_forward/cert_util.rb

@@ -0,0 +1,85 @@
+require 'openssl'
+
+module Fluent
+  module SecureForward
+    module CertUtil
+      def self.generate_ca_pair(opts={})
+        key = OpenSSL::PKey::RSA.generate(opts[:private_key_length])
+
+        issuer = subject = OpenSSL::X509::Name.new
+        subject.add_entry('C', opts[:cert_country])
+        subject.add_entry('ST', opts[:cert_state])
+        subject.add_entry('L', opts[:cert_locality])
+        subject.add_entry('CN', opts[:cert_common_name])
+
+        digest = OpenSSL::Digest::SHA1.new
+
+        cert = OpenSSL::X509::Certificate.new
+        cert.not_before = Time.at(0)
+        cert.not_after = Time.now + 5 * 365 * 86400 # 5 years after
+        cert.public_key = key
+        cert.serial = 1
+        cert.issuer = issuer
+        cert.subject = subject
+        cert.add_extension OpenSSL::X509::Extension.new('basicConstraints', OpenSSL::ASN1.Sequence([OpenSSL::ASN1::Boolean(true)]))
+        cert.sign(key, digest)
+
+        return cert, key
+      end
+
+      def self.generate_server_pair(opts={})
+        key = OpenSSL::PKey::RSA.generate(opts[:private_key_length])
+
+        ca_key = OpenSSL::PKey::RSA.new(File.read(opts[:ca_key_path]), opts[:ca_key_passphrase])
+        ca_cert = OpenSSL::X509::Certificate.new(File.read(opts[:ca_cert_path]))
+        issuer = ca_cert.issuer
+
+        subject = OpenSSL::X509::Name.new
+        subject.add_entry('C', opts[:country])
+        subject.add_entry('ST', opts[:state])
+        subject.add_entry('L', opts[:locality])
+        subject.add_entry('CN', opts[:common_name])
+
+        digest = OpenSSL::Digest::SHA1.new
+
+        cert = OpenSSL::X509::Certificate.new
+        cert.not_before = Time.at(0)
+        cert.not_after = Time.now + 5 * 365 * 86400 # 5 years after
+        cert.public_key = key
+        cert.serial = 2
+        cert.issuer = issuer
+        cert.subject = subject
+
+        cert.add_extension OpenSSL::X509::Extension.new('basicConstraints', OpenSSL::ASN1.Sequence([OpenSSL::ASN1::Boolean(false)]))
+        cert.add_extension OpenSSL::X509::Extension.new('nsCertType', 'server')
+
+        cert.sign ca_key, digest
+
+        return cert, key
+      end
+
+      def self.generate_self_signed_server_pair(opts={})
+        key = OpenSSL::PKey::RSA.generate(opts[:private_key_length])
+
+        issuer = subject = OpenSSL::X509::Name.new
+        subject.add_entry('C', opts[:country])
+        subject.add_entry('ST', opts[:state])
+        subject.add_entry('L', opts[:locality])
+        subject.add_entry('CN', opts[:common_name])
+
+        digest = OpenSSL::Digest::SHA1.new
+
+        cert = OpenSSL::X509::Certificate.new
+        cert.not_before = Time.at(0)
+        cert.not_after = Time.now + 5 * 365 * 86400 # 5 years after
+        cert.public_key = key
+        cert.serial = 1
+        cert.issuer = issuer
+        cert.subject  = subject
+        cert.sign(key, digest)
+
+        return cert, key
+      end
+    end
+  end
+end