fluent-plugin-syslog-tls-with-backoff-test 2.1.0

data/lib/syslog_tls/protocol.rb

@@ -0,0 +1,145 @@
+# Copyright 2016 Acquia, Inc.
+# Copyright 2016 t.e.morgan.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'date'
+
+require_relative 'facility'
+require_relative 'severity'
+
+# Syslog protocol https://tools.ietf.org/html/rfc5424
+module SyslogTls
+  # RFC defined nil value
+  NIL_VALUE = '-'
+
+  # All headers by specification wrapped in single object
+  class Header
+    attr_accessor :version, :hostname, :app_name, :procid, :msgid
+    attr_reader :facility, :severity, :timestamp
+
+    FACILITIES = {}
+    SEVERITIES = {}
+
+    Facility.setup_constants FACILITIES
+    Severity.setup_constants SEVERITIES
+
+    def initialize
+      @timestamp = Time.now
+      @severity = 'INFO'
+      @facility = 'LOCAL0'
+      @version = 1
+      @hostname = NIL_VALUE
+      @app_name = NIL_VALUE
+      @procid = NIL_VALUE
+      @msgid = NIL_VALUE
+    end
+
+    def timestamp=(val)
+      raise ArgumentError.new("Must provide Time object value instead: #{val.inspect}") unless val.is_a?(Time)
+      @timestamp = val
+    end
+
+    def facility=(val)
+      raise ArgumentError.new("Invalid facility value: #{val.inspect}") unless FACILITIES.key?(val)
+      @facility = val
+    end
+
+    def severity=(val)
+      raise ArgumentError.new("Invalid severity value: #{val.inspect}") unless SEVERITIES.key?(val)
+      @severity = val
+    end
+
+    # Priority value is calculated by first multiplying the Facility
+    # number by 8 and then adding the numerical value of the Severity.
+    def pri
+      FACILITIES[facility] * 8 + SEVERITIES[severity]
+    end
+
+    def assemble
+      [
+        "<#{pri}>#{version}",
+        timestamp.to_datetime.rfc3339,
+        hostname,
+        app_name,
+        procid,
+        msgid
+      ].join(' ')
+    end
+
+    def to_s
+      assemble
+    end
+  end
+
+  # Structured data field
+  class StructuredData
+    attr_accessor :id, :data
+
+    def initialize(id)
+      @id = id
+      @data = {}
+    end
+
+    # Format data structured data to
+    # [id k="v" ...]
+    def assemble
+      return NIL_VALUE unless id
+      parts = [id]
+      data.each do |k, v|
+        # Characters ", ] and \ must be escaped to prevent any parsing errors
+        v = v.gsub(/(\"|\]|\\)/) { |match| '\\' + match }
+        parts << "#{k}=\"#{v}\""
+      end
+      "[#{parts.join(' ')}]"
+    end
+
+    def to_s
+      assemble
+    end
+  end
+
+  # Message represents full message that can be sent to syslog
+  class Message
+    attr_accessor :structured_data, :msg
+    attr_writer :header
+
+    def initialize
+      @msg = ''
+      @structured_data = []
+    end
+
+    def header
+      @header ||= Header.new
+    end
+
+    def assemble
+      # Start with header
+      out = [header.to_s]
+      # Add all structured data
+      if structured_data.length > 0
+        out << structured_data.map(&:to_s).join('')
+      else
+        out << NIL_VALUE
+      end
+      # Add message
+      out << msg if msg.length > 0
+      # Message must end with new line delimiter
+      out.join(' ') + "\n"
+    end
+
+    def to_s
+      assemble
+    end
+  end
+end

data/lib/syslog_tls/severity.rb

@@ -0,0 +1,30 @@
+# Copyright 2016 Acquia, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require_relative 'lookup_from_const'
+
+module SyslogTls
+  module Severity
+    extend LookupFromConst
+    EMERG  = PANIC   = 0
+    ALERT            = 1
+    CRIT             = 2
+    ERR    = ERROR   = 3
+    WARN   = WARNING = 4
+    NOTICE           = 5
+    INFO             = 6
+    DEBUG            = 7
+    NONE             = 10
+  end
+end

data/lib/syslog_tls/ssl_transport.rb

@@ -0,0 +1,208 @@
+# Copyright 2016 Acquia, Inc.
+# Copyright 2016-2023 t.e.morgan.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'socket'
+require 'openssl'
+require 'pp'
+
+module SyslogTls
+  # Supports SSL connection to remote host
+  class SSLTransport
+    CONNECT_TIMEOUT = 10
+    # READ_TIMEOUT    = 5
+    WRITE_TIMEOUT   = 5
+
+    attr_accessor :socket
+
+    attr_reader :host, :port, :idle_timeout, :ca_cert, :client_cert, :client_key, :verify_cert_name, :ssl_version
+
+    attr_writer :retries
+
+    def initialize(host, port, idle_timeout: nil, ca_cert: 'system', client_cert: nil, client_key: nil, verify_cert_name: true, ssl_version: :TLS1_2, max_retries: 1)
+      @host = host
+      @port = port
+      @idle_timeout = idle_timeout
+      @ca_cert = ca_cert
+      @client_cert = client_cert
+      @client_key = client_key
+      @verify_cert_name = verify_cert_name
+      @ssl_version = ssl_version
+      @retries = max_retries
+      connect
+    end
+
+    def connect
+      timwnow = Time.now
+      @socket = get_ssl_connection
+      begin
+        begin
+          @socket.connect_nonblock
+        rescue Errno::EAGAIN, Errno::EWOULDBLOCK, IO::WaitReadable
+          select_with_timeout(@socket, :connect_read) && retry
+        rescue IO::WaitWritable
+          select_with_timeout(@socket, :connect_write) && retry
+        end
+      rescue Errno::ETIMEDOUT
+        raise 'Socket timeout during connect'
+      end
+      @last_write = Time.now if idle_timeout
+    end
+
+    def get_tcp_connection
+      tcp = nil
+
+      family = Socket::Constants::AF_UNSPEC
+      sock_type = Socket::Constants::SOCK_STREAM
+      addr_info = Socket.getaddrinfo(host, port, family, sock_type, nil, nil, false).first
+      _, port, _, address, family, sock_type = addr_info
+
+      begin
+        sock_addr = Socket.sockaddr_in(port, address)
+        tcp = Socket.new(family, sock_type, 0)
+        tcp.setsockopt(Socket::SOL_SOCKET, Socket::Constants::SO_REUSEADDR, true)
+        tcp.setsockopt(Socket::SOL_SOCKET, Socket::Constants::SO_REUSEPORT, true)
+        tcp.connect_nonblock(sock_addr)
+      rescue Errno::EINPROGRESS
+        select_with_timeout(tcp, :connect_write)
+        begin
+          tcp.connect_nonblock(sock_addr)
+        rescue Errno::EISCONN
+          # all good
+        rescue SystemCallError
+          tcp.close rescue nil
+          raise
+        end
+      rescue SystemCallError
+        tcp.close rescue nil
+        raise
+      end
+
+      tcp.setsockopt(Socket::IPPROTO_TCP, Socket::TCP_NODELAY, true)
+      tcp
+    end
+
+    def get_ssl_connection
+      tcp = get_tcp_connection
+
+      ctx = OpenSSL::SSL::SSLContext.new
+      ctx.verify_mode = OpenSSL::SSL::VERIFY_PEER
+      ctx.min_version = ssl_version
+
+      ctx.verify_hostname = verify_cert_name != false
+
+      case ca_cert
+      when true, 'true', 'system'
+        # use system certs, same as openssl cli
+        ctx.cert_store = OpenSSL::X509::Store.new
+        ctx.cert_store.set_default_paths
+      when false, 'false'
+        ctx.verify_hostname = false
+        ctx.verify_mode = OpenSSL::SSL::VERIFY_NONE
+      when %r{/$} # ends in /
+        ctx.ca_path = ca_cert
+      when String
+        ctx.ca_file = ca_cert
+      end
+
+      ctx.cert = OpenSSL::X509::Certificate.new(File.read(client_cert)) if client_cert
+      ctx.key = OpenSSL::PKey::read(File.read(client_key)) if client_key
+      socket = OpenSSL::SSL::SSLSocket.new(tcp, ctx)
+      socket.hostname = host
+      socket.sync_close = true
+      socket
+    end
+
+    # Allow to retry on failed writes
+    def write(s)
+      if idle_timeout
+        if (t=Time.now) > @last_write + idle_timeout
+          @socket.close rescue nil
+          connect
+        else
+          @last_write = t
+        end
+      end
+      begin
+        retry_id ||= 0
+        do_write(s)
+      rescue => e
+        if (retry_id += 1) < @retries
+          @socket.close rescue nil
+          connect
+          retry
+        else
+          raise e
+        end
+      end
+    end
+
+    def do_write(data)
+      data.force_encoding('BINARY') # so we can break in the middle of multi-byte characters
+      loop do
+        sent = 0
+        begin
+          sent = @socket.write_nonblock(data)
+        rescue OpenSSL::SSL::SSLError, Errno::EAGAIN, Errno::EWOULDBLOCK, IO::WaitWritable => e
+          if e.is_a?(OpenSSL::SSL::SSLError) && e.message !~ /write would block/
+            raise e
+          else
+            select_with_timeout(@socket, :write) && retry
+          end
+        end
+
+        break if sent >= data.size
+        data = data[sent, data.size]
+      end
+    end
+
+    def select_with_timeout(tcp, type)
+      host_ip_port = host + ":" + port.to_s
+      case type
+      when :connect_read
+        args = [[tcp], nil, nil, CONNECT_TIMEOUT]
+      when :connect_write
+        args = [nil, [tcp], nil, CONNECT_TIMEOUT]
+      # when :read
+      #   args = [[tcp], nil, nil, READ_TIMEOUT]
+      when :write
+        args = [nil, [tcp], nil, WRITE_TIMEOUT]
+      else
+        raise "Unknown select type #{type}"
+      end
+      if type.to_s == "connect_write"
+        if can_write(host_ip_port) == 1
+          io_select_return = IO.select(*args)
+          ready_sockets, _, _ = io_select_return
+          if !ready_sockets.empty?
+            reset_tries(host_ip_port)
+            io_select_return
+          else
+            increase_retry(host_ip_port)
+            io_select_return || raise("Socket timeout during #{type}")
+          end
+        # else
+        #   raise("Failed to write #{type}")
+        end
+      else
+        IO.select(*args) || raise("Socket timeout during #{type}")
+      end
+    end
+
+    # Forward any methods directly to SSLSocket
+    def method_missing(method_sym, *arguments, &block)
+      @socket.send(method_sym, *arguments, &block)
+    end
+  end
+end

data/lib/syslog_tls/version.rb

@@ -0,0 +1,18 @@
+# Copyright 2016 Acquia, Inc.
+# Copyright 2016-2019 t.e.morgan.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module SyslogTls
+  VERSION = '2.1.0'
+end

data/test/fluent/test_out_syslog_tls.rb

@@ -0,0 +1,192 @@
+# Copyright 2016 Acquia, Inc.
+# Copyright 2016-2019 t.e.morgan.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'helper'
+require 'ssl'
+require 'date'
+require 'minitest/mock'
+require 'fluent/plugin/out_syslog_tls'
+require 'fluent/test/driver/output'
+
+class SyslogTlsOutputTest < Test::Unit::TestCase
+  include SSLTestHelper
+
+  def setup
+    Fluent::Test.setup
+    @driver = nil
+  end
+
+  def driver(conf='')
+    @driver ||= Fluent::Test::Driver::Output.new(Fluent::Plugin::SyslogTlsOutput).configure(conf)
+  end
+
+  def sample_record
+    {
+      "app_name" => "app",
+      "hostname" => "host",
+      "procid" => $$,
+      "msgid" => 1000,
+      "message" => "MESSAGE",
+      "severity" => "PANIC",
+    }
+  end
+
+  def mock_logger(token='TOKEN')
+    io = StringIO.new
+    io.set_encoding('utf-8')
+    ::SyslogTls::Logger.new(io, token)
+  end
+
+  def test_configure
+    config = %{
+      host   syslog.collection.us1.sumologic.com
+      port   6514
+      client_cert
+      client_key
+      verify_cert_name true
+      token  1234567890
+    }
+    instance = driver(config).instance
+
+    assert_equal 'syslog.collection.us1.sumologic.com', instance.host
+    assert_equal '6514', instance.port
+    assert_equal '', instance.client_cert
+    assert_equal '', instance.client_key
+    assert_equal true, instance.verify_cert_name
+    assert_equal '1234567890', instance.token
+  end
+
+  def test_default_emit
+    config = %{
+      host   syslog.collection.us1.sumologic.com
+      port   6514
+      client_cert
+      client_key
+    }
+    instance = driver(config).instance
+
+    time = Time.now
+    record = sample_record
+    logger = mock_logger(instance.token)
+
+    instance.stub(:new_logger, logger) do
+      instance.process('test', {time.to_i => record})
+    end
+
+    assert_equal "<134>1 #{time.to_datetime.rfc3339} - - - - - #{record.to_json.to_s}\n\n", logger.transport.string
+  end
+
+  def test_message_headers_mapping
+    config = %{
+      host   syslog.collection.us1.sumologic.com
+      port   6514
+      client_cert
+      client_key
+      token  1234567890
+      hostname_key hostname
+      procid_key procid
+      app_name_key app_name
+      msgid_key msgid
+    }
+    instance = driver(config).instance
+
+    time = Time.now
+    record = sample_record
+    logger = mock_logger
+
+    instance.stub(:new_logger, logger) do
+      instance.process('test', {time.to_i => record})
+    end
+
+    assert_true logger.transport.string.start_with?("<134>1 #{time.to_datetime.rfc3339} host app #{$$} 1000 [TOKEN]")
+  end
+
+  def test_message_severity_mapping
+    config = %{
+      host   syslog.collection.us1.sumologic.com
+      port   6514
+      client_cert
+      client_key
+      token  1234567890
+      severity_key severity
+    }
+    instance = driver(config).instance
+
+    time = Time.now
+    record = sample_record
+    logger = mock_logger
+
+    instance.stub(:new_logger, logger) do
+      instance.process('test', {time.to_i => record})
+    end
+
+    assert_true logger.transport.string.start_with?("<128>1")
+  end
+
+  def test_formatter
+    config = %{
+      host   syslog.collection.us1.sumologic.com
+      port   6514
+      client_cert
+      client_key
+      token  1234567890
+      format out_file
+      localtime false
+    }
+    instance = driver(config).instance
+
+    time = Time.now
+    record = sample_record
+    logger = mock_logger
+
+    instance.stub(:new_logger, logger) do
+      instance.process('test', {time.to_i => record})
+    end
+
+    formatted_time = time.dup.utc.strftime("%Y-%m-%dT%H:%M:%SZ")
+    assert_equal "<134>1 #{time.to_datetime.rfc3339} - - - - [TOKEN] #{formatted_time}\ttest\t#{record.to_json.to_s}\n\n", logger.transport.string
+  end
+
+  def test_ssl
+    time = Time.now
+    record = sample_record
+
+    server = ssl_server
+    st = Thread.new {
+        client = server.accept
+        assert_equal "<134>1 #{time.to_datetime.rfc3339} host app #{$$} 1000 [1234567890] #{record.to_json.to_s}\n", client.gets
+        client.close
+    }
+
+    config = %{
+      host   localhost
+      port   #{server.addr[1]}
+      client_cert
+      client_key
+      token  1234567890
+      hostname_key hostname
+      procid_key procid
+      app_name_key app_name
+      msgid_key msgid
+    }
+    instance = driver(config).instance
+
+    SyslogTls::SSLTransport.stub_any_instance(:get_ssl_connection, ssl_client) do
+      instance.process('test', {time.to_i => record})
+    end
+
+    st.join
+  end
+end

data/test/helper.rb

@@ -0,0 +1,25 @@
+# Copyright 2016 Acquia, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'simplecov'
+
+SimpleCov.start
+
+require 'test/unit'
+require 'fluent/test'
+require 'minitest/pride'
+require 'minitest/stub_any_instance'
+
+require 'webmock/test_unit'
+WebMock.disable_net_connect!

data/test/ssl.rb

@@ -0,0 +1,53 @@
+require 'socket'
+require 'openssl'
+
+module SSLTestHelper
+  def ssl_server(min_version: nil, max_version: nil)
+    @ssl_server ||= begin
+      tcp_server = TCPServer.new("localhost", 33000 + Random.rand(1000))
+      ssl_context = OpenSSL::SSL::SSLContext.new
+      ssl_context.cert = certificate
+      ssl_context.key = rsa_key
+      ssl_context.min_version = min_version if min_version
+      ssl_context.max_version = max_version if max_version
+      OpenSSL::SSL::SSLServer.new(tcp_server, ssl_context)
+    end
+  end
+
+  def ssl_client
+    tcp = TCPSocket.new("localhost", ssl_server.addr[1])
+    ctx = OpenSSL::SSL::SSLContext.new
+    ctx.set_params(verify_mode: OpenSSL::SSL::VERIFY_NONE)
+    ctx.cert = certificate
+    ctx.key = rsa_key
+    OpenSSL::SSL::SSLSocket.new(tcp, ctx)
+  end
+
+  def rsa_key
+    @rsa_key ||= OpenSSL::PKey::RSA.new(2048)
+  end
+
+  def certificate
+    @cert ||= begin
+      cert = OpenSSL::X509::Certificate.new
+      cert.subject = cert.issuer = OpenSSL::X509::Name.parse("/C=BE/O=Test/OU=Test/CN=Test")
+      cert.not_before = Time.now
+      cert.not_after = Time.now + 365 * 24 * 60 * 60
+      cert.public_key = rsa_key.public_key
+      cert.serial = 0x0
+      cert.version = 2
+
+      ef = OpenSSL::X509::ExtensionFactory.new
+      ef.subject_certificate = cert
+      ef.issuer_certificate = cert
+      cert.extensions = [
+        ef.create_extension("basicConstraints","CA:TRUE", true),
+        ef.create_extension("subjectKeyIdentifier", "hash"),
+        # ef.create_extension("keyUsage", "cRLSign,keyCertSign", true),
+      ]
+      cert.add_extension ef.create_extension("authorityKeyIdentifier", "keyid:always,issuer:always")
+      cert.sign(rsa_key, OpenSSL::Digest::SHA1.new)
+      cert
+    end
+  end
+end