fluent-plugin-secure-forward-addproxy 0.3.3dev2

data/lib/fluent/plugin/out_secure_forward.rb

@@ -0,0 +1,280 @@
+# -*- coding: utf-8 -*-
+
+require 'fluent/mixin/config_placeholders'
+
+module Fluent
+  class SecureForwardOutput < ObjectBufferedOutput
+  end
+end
+
+require_relative 'output_node'
+
+module Fluent
+  class SecureForwardOutput < ObjectBufferedOutput
+    DEFAULT_SECURE_CONNECT_PORT = 24284
+
+    Fluent::Plugin.register_output('secure_forward', self)
+
+    config_param :secure, :bool
+
+    config_param :self_hostname, :string
+    include Fluent::Mixin::ConfigPlaceholders
+
+    config_param :shared_key, :string
+
+    config_param :keepalive, :time, default: nil # nil/0 means disable keepalive expiration
+
+    config_param :send_timeout, :time, default: 60
+    # config_param :hard_timeout, :time, :default => 60
+    # config_param :expire_dns_cache, :time, :default => 0 # 0 means disable cache
+
+    config_param :ca_cert_path, :string, default: nil
+
+    config_param :enable_strict_verification, :bool, default: nil # FQDN check with hostlabel
+    config_param :ssl_version, :string, default: 'TLSv1_2'
+    config_param :ssl_ciphers, :string, default: nil
+
+    config_param :read_length, :size, default: 512 # 512bytes
+    config_param :read_interval_msec, :integer, default: 50 # 50ms
+    config_param :socket_interval_msec, :integer, default: 200 # 200ms
+
+    config_param :reconnect_interval, :time, default: 5
+    config_param :established_timeout, :time, default: 10
+
+    config_param :proxy_uri, :string, default: nil
+
+    attr_reader :read_interval, :socket_interval
+
+    config_section :server, param_name: :servers do
+      config_param :host, :string
+      config_param :hostlabel, :string, default: nil
+      config_param :port, :integer, default: DEFAULT_SECURE_CONNECT_PORT
+      config_param :shared_key, :string, default: nil
+      config_param :username, :string, default: ''
+      config_param :password, :string, default: ''
+      config_param :standby, :bool, default: false
+      config_param :proxy_uri, :string, default: nil
+    end
+    attr_reader :nodes
+
+    attr_reader :hostname_resolver
+
+    def initialize
+      super
+      require 'socket'
+      require 'openssl'
+      require 'digest'
+      require 'resolve/hostname'
+      require 'securerandom'
+    end
+
+    # Define `log` method for v0.10.42 or earlier
+    unless method_defined?(:log)
+      define_method("log") { $log }
+    end
+
+    def configure(conf)
+      super
+
+      if @secure
+        if @ca_cert_path
+          raise Fluent::ConfigError, "CA cert file not found nor readable at '#{@ca_cert_path}'" unless File.readable?(@ca_cert_path)
+          begin
+            OpenSSL::X509::Certificate.new File.read(@ca_cert_path)
+          rescue OpenSSL::X509::CertificateError => e
+            raise Fluent::ConfigError, "failed to load CA cert file"
+          end
+        else
+          raise Fluent::ConfigError, "FQDN verification required for certificates issued from public CA" unless @enable_strict_verification
+          log.info "secure connection with valid certificates issued from public CA"
+        end
+      else
+        log.warn "'insecure' mode has vulnerability for man-in-the-middle attacks."
+      end
+
+      @read_interval = @read_interval_msec / 1000.0
+      @socket_interval = @socket_interval_msec / 1000.0
+
+      @nodes = []
+      @servers.each do |server|
+        node = Node.new(self, server)
+        node.first_session = true
+        @nodes.push node
+      end
+
+      if @num_threads > @nodes.select{|n| not n.standby}.size
+        log.warn "Too many num_threads for secure-forward: threads should be smaller or equal to non standby servers"
+      end
+
+      @next_node = 0
+      @mutex = Mutex.new
+
+      @hostname_resolver = Resolve::Hostname.new(system_resolver: true)
+
+      true
+    end
+
+    def select_node(permit_standby=false)
+      tries = 0
+      nodes = @nodes.size
+      @mutex.synchronize {
+        n = nil
+        while tries <= nodes
+          n = @nodes[@next_node]
+          @next_node += 1
+          @next_node = 0 if @next_node >= nodes
+
+          if n && n.established? && (! n.tained?) && (! n.detached?) && (!n.standby || permit_standby)
+            n.tain!
+            return n
+          end
+
+          tries += 1
+        end
+        nil
+      }
+    end
+
+    def start
+      super
+
+      log.debug "starting secure-forward"
+      OpenSSL::Random.seed(SecureRandom.random_bytes(16))
+      log.debug "start to connect target nodes"
+      @nodes.each do |node|
+        log.debug "connecting node", host: node.host, port: node.port
+        node.start
+      end
+      @nodewatcher = Thread.new(&method(:node_watcher))
+      @nodewatcher.abort_on_exception = true
+    end
+
+    def node_watcher
+      reconnectings = Array.new(@nodes.size)
+      nodes_size = @nodes.size
+
+      loop do
+        sleep @reconnect_interval
+
+        log.trace "in node health watcher"
+
+        (0...nodes_size).each do |i|
+          log.trace "node health watcher for #{@nodes[i].host}"
+
+          next if @nodes[i].established? && ! @nodes[i].expired? && ! @nodes[i].detached?
+
+          next if reconnectings[i]
+
+          reason = :expired
+
+          unless @nodes[i].established?
+            log.warn "dead connection found: #{@nodes[i].host}, reconnecting..."
+            reason = :dead
+          end
+
+          node = @nodes[i]
+          log.debug "reconnecting to node", host: node.host, port: node.port, expire: node.expire, expired: node.expired?, detached: node.detached?
+
+          renewed = node.dup
+          renewed.start
+
+          Thread.pass # to connection thread
+          reconnectings[i] = { conn: renewed, at: Time.now, reason: reason }
+        end
+
+        (0...nodes_size).each do |i|
+          next unless reconnectings[i]
+
+          log.trace "checking reconnecting node #{reconnectings[i][:conn].host}"
+
+          if reconnectings[i][:conn].established?
+            log.debug "connection established for reconnecting node"
+
+            oldconn = @nodes[i]
+            @nodes[i] = reconnectings[i][:conn]
+
+            if reconnectings[i][:reason] == :dead
+              log.warn "recovered connection to dead node: #{nodes[i].host}"
+            end
+
+            log.trace "old connection shutting down"
+            oldconn.detach! if oldconn # connection object doesn't raise any exceptions
+            log.trace "old connection shutted down"
+
+            reconnectings[i] = nil
+            next
+          end
+
+          # not connected yet
+
+          next if reconnectings[i][:at] + @established_timeout > Time.now
+
+          # not connected yet, and timeout
+          timeout_conn = reconnectings[i][:conn]
+          log.debug "SSL connection is not established until timemout", host: timeout_conn.host, port: timeout_conn.port, timeout: @established_timeout
+          reconnectings[i] = nil
+          timeout_conn.detach! if timeout_conn # connection object doesn't raise any exceptions
+        end
+      end
+    end
+
+    def shutdown
+      super
+
+      @nodewatcher.kill
+      @nodewatcher.join
+
+      @nodes.each do |node|
+        node.detach!
+        node.join
+      end
+    end
+
+    def write_objects(tag, es)
+      node = select_node || select_node(true)
+      unless node
+        raise "no one nodes with valid ssl session"
+      end
+      log.trace "selected node", host: node.host, port: node.port, standby: node.standby
+
+      begin
+        send_data(node, tag, es)
+        node.release!
+      rescue Errno::EPIPE, IOError, OpenSSL::SSL::SSLError => e
+        log.warn "Failed to send messages to #{node.host}, parging.", error_class: e.class, error: e
+        node.release!
+        node.detach!
+
+        raise # to retry #write_objects
+      end
+    end
+
+    # MessagePack FixArray length = 2
+    FORWARD_HEADER = [0x92].pack('C')
+
+    # to forward messages
+    def send_data(node, tag, es)
+      ssl = node.sslsession
+      # beginArray(2)
+      ssl.write FORWARD_HEADER
+
+      # writeRaw(tag)
+      ssl.write tag.to_msgpack
+
+      # beginRaw(size)
+      sz = es.size
+      #  # FixRaw
+      #  ssl.write [0xa0 | sz].pack('C')
+      #elsif sz < 65536
+      #  # raw 16
+      #  ssl.write [0xda, sz].pack('Cn')
+      #else
+      # raw 32
+      ssl.write [0xdb, sz].pack('CN')
+      #end
+
+      # writeRawBody(packed_es)
+      es.write_to(ssl)
+    end
+  end
+end

data/lib/fluent/plugin/output_node.rb

@@ -0,0 +1,348 @@
+# require 'msgpack'
+# require 'socket'
+# require 'openssl'
+# require 'digest'
+# require 'resolve/hostname'
+
+require_relative 'openssl_util'
+
+class Fluent::SecureForwardOutput::Node
+  attr_accessor :host, :port, :hostlabel, :shared_key, :username, :password, :standby
+
+  attr_accessor :authentication, :keepalive
+  attr_accessor :socket, :sslsession, :unpacker, :shared_key_salt, :state
+
+  attr_accessor :first_session, :detach
+
+  attr_reader :expire
+
+  def initialize(sender, conf)
+    @sender = sender
+    @shared_key = conf.shared_key || sender.shared_key
+
+    @host = conf.host
+    @port = conf.port
+    @hostlabel = conf.hostlabel || conf.host
+    @username = conf.username
+    @password = conf.password
+    @standby = conf.standby
+
+    @proxy_uri = conf.proxy_uri
+
+    @keepalive = sender.keepalive
+
+    @authentication = nil
+
+    @writing = false
+
+    @expire = nil
+    @first_session = false
+    @detach = false
+
+    @socket = nil
+    @sslsession = nil
+    @unpacker = MessagePack::Unpacker.new
+
+    @shared_key_salt = generate_salt
+    @state = :helo
+    @thread = nil
+  end
+
+  def log
+    @sender.log
+  end
+
+  def dup
+    renewed = self.class.new(
+      @sender,
+      Fluent::Config::Section.new({host: @host, port: @port, hostlabel: @hostlabel, username: @username, password: @password, shared_key: @shared_key, standby: @standby, proxy_uri: @proxy_uri})
+    )
+    renewed
+  end
+
+  def start
+    @thread = Thread.new(&method(:connect))
+  end
+
+  def detach!
+    @detach = true
+  end
+
+  def detached?
+    @detach
+  end
+
+  def tain!
+    raise RuntimeError, "BUG: taining detached node" if @detach
+    @writing = true
+  end
+
+  def tained?
+    @writing
+  end
+
+  def release!
+    @writing = false
+  end
+
+  def shutdown
+    log.debug "shutting down node #{@host}"
+    @state = :closed
+
+    if @thread == Thread.current
+      @sslsession.close if @sslsession
+      @socket.close if @socket
+      @thread.kill
+    else
+      if @thread
+        @thread.kill
+        @thread.join
+      end
+      @sslsession.close if @sslsession
+      @socket.close if @socket
+    end
+  rescue => e
+    log.debug "error on node shutdown #{e.class}:#{e.message}"
+  end
+
+  def join
+    @thread && @thread.join
+  end
+
+  def established?
+    @state == :established
+  end
+
+  def expired?
+    if @keepalive.nil? || @keepalive == 0
+      false
+    else
+      @expire && @expire < Time.now
+    end
+  end
+
+  def generate_salt
+    OpenSSL::Random.random_bytes(16)
+  end
+
+  def check_helo(message)
+    log.debug "checking helo"
+    # ['HELO', options(hash)]
+    unless message.size == 2 && message[0] == 'HELO'
+      return false
+    end
+    opts = message[1]
+    @shared_key_nonce = opts['nonce'] || '' # make shared_key_check failed (instead of error) if protocol version mismatch exist
+    @authentication = opts['auth']
+    @allow_keepalive = opts['keepalive']
+    true
+  end
+
+  def generate_ping
+    log.debug "generating ping"
+    # ['PING', self_hostname, sharedkey\_salt, sha512\_hex(sharedkey\_salt + self_hostname + nonce + shared_key),
+    #  username || '', sha512\_hex(auth\_salt + username + password) || '']
+    shared_key_hexdigest = Digest::SHA512.new.update(@shared_key_salt).update(@sender.self_hostname).update(@shared_key_nonce).update(@shared_key).hexdigest
+    ping = ['PING', @sender.self_hostname, @shared_key_salt, shared_key_hexdigest]
+    if @authentication != ''
+      password_hexdigest = Digest::SHA512.new.update(@authentication).update(@username).update(@password).hexdigest
+      ping.push(@username, password_hexdigest)
+    else
+      ping.push('','')
+    end
+    ping
+  end
+
+  def check_pong(message)
+    log.debug "checking pong"
+    # ['PONG', bool(authentication result), 'reason if authentication failed',
+    #  self_hostname, sha512\_hex(salt + self_hostname + nonce + sharedkey)]
+    unless message.size == 5 && message[0] == 'PONG'
+      return false, 'invalid format for PONG message'
+    end
+    pong, auth_result, reason, hostname, shared_key_hexdigest = message
+
+    unless auth_result
+      return false, 'authentication failed: ' + reason
+    end
+
+    if hostname == @sender.self_hostname
+      return false, 'same hostname between input and output: invalid configuration'
+    end
+
+    clientside = Digest::SHA512.new.update(@shared_key_salt).update(hostname).update(@shared_key_nonce).update(@shared_key).hexdigest
+    unless shared_key_hexdigest == clientside
+      return false, 'shared key mismatch'
+    end
+
+    return true, nil
+  end
+
+  def send_data(data)
+    @sslsession.write data.to_msgpack
+  end
+
+  def on_read(data)
+    log.debug "on_read"
+    if self.established?
+      #TODO: ACK
+      log.warn "unknown packets arrived..."
+      return
+    end
+
+    case @state
+    when :helo
+      unless check_helo(data)
+        log.warn "received invalid helo message from #{@host}"
+        self.shutdown
+        return
+      end
+      send_data generate_ping()
+      @state = :pingpong
+    when :pingpong
+      success, reason = check_pong(data)
+      unless success
+        log.warn "connection refused to #{@host}:" + reason
+        self.shutdown
+        return
+      end
+      log.info "connection established to #{@host}" if @first_session
+      @state = :established
+      @expire = Time.now + @keepalive if @keepalive && @keepalive > 0
+      log.debug "connection established", host: @host, port: @port, expire: @expire
+    end
+  end
+
+  def connect
+    Thread.current.abort_on_exception = true
+    log.debug "starting client"
+
+    addr = @sender.hostname_resolver.getaddress(@host)
+    log.debug "create tcp socket to node", host: @host, address: addr, port: @port
+
+    begin
+    if @proxy_uri.nil? then
+      sock = TCPSocket.new(addr, @port)
+    else
+      proxy = Proxifier::Proxy(@proxy_uri)
+      sock = proxy.open(addr, @port)
+    end
+    rescue => e
+      log.warn "failed to connect for secure-forward", error_class: e.class, error: e, host: @host, address: addr, port: @port
+      @state = :failed
+      return
+    end
+
+    log.trace "changing socket options"
+    opt = [1, @sender.send_timeout.to_i].pack('I!I!')  # { int l_onoff; int l_linger; }
+    sock.setsockopt(Socket::SOL_SOCKET, Socket::SO_LINGER, opt)
+
+    opt = [@sender.send_timeout.to_i, 0].pack('L!L!')  # struct timeval
+    sock.setsockopt(Socket::SOL_SOCKET, Socket::SO_SNDTIMEO, opt)
+
+    log.trace "initializing SSL contexts"
+
+    context = OpenSSL::SSL::SSLContext.new(@sender.ssl_version)
+
+    log.trace "setting SSL verification options"
+
+    if @sender.secure
+      # inject OpenSSL::SSL::SSLContext::DEFAULT_PARAMS
+      # https://bugs.ruby-lang.org/issues/9424
+      context.set_params({})
+
+      if @sender.ssl_ciphers
+        context.ciphers = @sender.ssl_ciphers
+      else
+        ### follow httpclient configuration by nahi
+        # OpenSSL 0.9.8 default: "ALL:!ADH:!LOW:!EXP:!MD5:+SSLv2:@STRENGTH"
+        context.ciphers = "ALL:!aNULL:!eNULL:!SSLv2" # OpenSSL >1.0.0 default
+      end
+
+      log.trace "set verify_mode VERIFY_PEER"
+      context.verify_mode = OpenSSL::SSL::VERIFY_PEER
+      if @sender.enable_strict_verification
+        context.cert_store = OpenSSL::X509::Store.new
+        begin
+          context.cert_store.set_default_paths
+        rescue OpenSSL::X509::StoreError => e
+          log.warn "faild to load system default certificates", error: e
+        end
+      end
+      if @sender.ca_cert_path
+        log.trace "set to use private CA", path: @sender.ca_cert_path
+        context.ca_file = @sender.ca_cert_path
+      end
+    end
+
+    log.debug "trying to connect ssl session", host: @host, address: addr, port: @port
+    begin
+      sslsession = OpenSSL::SSL::SSLSocket.new(sock, context)
+      log.trace "connecting...", host: @host, address: addr, port: @port
+      sslsession.connect
+    rescue => e
+      log.warn "failed to establish SSL connection", error_class: e.class, error: e, host: @host, address: addr, port: @port
+      @state = :failed
+      return
+    end
+
+    log.debug "ssl session connected", host: @host, port: @port
+
+    begin
+      if @sender.enable_strict_verification
+        log.debug "checking peer's certificate", subject: sslsession.peer_cert.subject
+        sslsession.post_connection_check(@hostlabel)
+        verify = sslsession.verify_result
+        if verify != OpenSSL::X509::V_OK
+          err_name = Fluent::SecureForwardOutput::OpenSSLUtil.verify_result_name(verify)
+          log.warn "BUG: failed to verify certification while connecting host #{@host} as #{@hostlabel} (but not raised, why?)"
+          log.warn "BUG: verify_result: #{err_name}"
+          raise RuntimeError, "BUG: failed to verify certification and to handle it correctly while connecting host #{@host} as #{@hostlabel}"
+        end
+      end
+    rescue OpenSSL::SSL::SSLError => e
+      log.warn "failed to verify certification while connecting ssl session", host: @host, hostlabel: @hostlabel
+      self.shutdown
+      raise
+    end
+
+    log.debug "ssl session connected", host: @host, port: @port
+    @socket = sock
+    @sslsession = sslsession
+
+    buf = ''
+    read_length = @sender.read_length
+    read_interval = @sender.read_interval
+    socket_interval = @sender.socket_interval
+
+    loop do
+      break if @detach
+
+      begin
+        while @sslsession.read_nonblock(read_length, buf)
+          if buf == ''
+            sleep read_interval
+            next
+          end
+          @unpacker.feed_each(buf, &method(:on_read))
+          buf = ''
+        end
+      rescue OpenSSL::SSL::SSLError
+        # to wait i/o restart
+        sleep socket_interval
+      rescue SystemCallError => e
+        log.warn "disconnected by Error", error_class: e.class, error: e, host: @host, port: @port
+        break
+      rescue EOFError
+        log.warn "disconnected", host: @host, port: @port
+        break
+      end
+    end
+    while @writing
+      break if @detach
+
+      sleep read_interval
+    end
+    self.shutdown
+  end
+end

data/lib/fluent/plugin/secure/forward/addproxy/version.rb

@@ -0,0 +1,11 @@
+module Fluent
+  module Plugin
+    module Secure
+      module Forward
+        module Addproxy
+          VERSION = "0.1.0"
+        end
+      end
+    end
+  end
+end

data/lib/fluent/plugin/secure/forward/addproxy.rb

@@ -0,0 +1,13 @@
+require "fluent/plugin/secure/forward/addproxy/version"
+
+module Fluent
+  module Plugin
+    module Secure
+      module Forward
+        module Addproxy
+          # Your code goes here...
+        end
+      end
+    end
+  end
+end

data/lib/fluent/plugin/secure/forward/v033dev2/addproxy/version.rb

@@ -0,0 +1,13 @@
+module Fluent
+  module Plugin
+    module Secure
+      module Forward
+        module V033dev2
+          module Addproxy
+            VERSION = "0.1.0"
+          end
+        end
+      end
+    end
+  end
+end

data/lib/fluent/plugin/secure/forward/v033dev2/addproxy.rb

@@ -0,0 +1,15 @@
+require "fluent/plugin/secure/forward/v033dev2/addproxy/version"
+
+module Fluent
+  module Plugin
+    module Secure
+      module Forward
+        module V033dev2
+          module Addproxy
+            # Your code goes here...
+          end
+        end
+      end
+    end
+  end
+end