fluent-plugin-syslog-gobi-tls 2.1.1

data/docs/configuration.md

@@ -0,0 +1,139 @@
+# Configuration
+
+This plugin allows to configure all syslog message attributes.
+
+## Static configuration
+
+Static configuration allow to define connection details, facility and hostname that will
+apply to all messages.
+
+### host
+
+Host represents DNS name of endpoint where should be data sent. Example: `syslog.collection.us1.sumologic.com` or `logs1.papertrailapp.com`
+
+### port
+
+Example: `6514`
+
+### idle_timeout
+
+If a given tag has gone this many seconds between log messages, disconnect and reconnect before sending logs. Useful in low-traffic logging situations with remote hosts that disconnect after a period of time. Disabled by default. Example: `600`
+
+### ca_cert
+
+Whether and how to verify the server's TLS certificate signing chain. Examples:
+* ca_cert system - Default; use the system CA certificate store (which must then be configured correctly)
+* ca_cert false - Disable verification; not recommended
+* ca_cert /path/to/file - A path+filename to a single CA file
+* ca_cert /path/to/dir/ - A directory of CA files (in format that OpenSSL can parse); must end with /
+
+### verify_cert_name
+
+Whether to verify that the server's cert matches `host`. Enabled by default (except when `ca_cert false`). Recommended; helps prevent MitM attacks. Example: `true`
+
+### token
+
+Some services require a token to identify the account. Example: `ABABABABABABA@99999`. Not required for Papertrail.
+
+### client_cert
+
+Optionally path to client certificate for TLS connection. Example: `/path/to/crt/file.crt`
+
+### client_key
+
+Optionally path to client private key for TLS connection. Example: `/path/to/key/file.key`
+
+### hostname
+
+Default hostname that is going to be sent in syslog message. Example: `ip-10-0-0-10`
+
+### facility
+
+Default syslog facility for all records. Example: `LOCAL0`
+
+## Per message configuration
+
+Its possible to configure the plugin to extract various parts of syslog message header
+from processed record itself. That allows to dynamically set app_name, procid, msgid from
+records if single match is being used to log messages from multiple sources.
+
+### severity_key
+
+Optionally record key where to get severity from the record. If not provided default `INFO` will be used.
+
+### facility_key
+
+Optionally record key where to get syslog facility from the record. If not provided default `LOCAL0` will be used.
+
+### hostname_key
+
+Optionally record key where to get hostname from the record. If not provided hostname is determined by system hostname.
+
+### app_name_key
+
+Optionally record key where to get app_name from the record. If not provided nil value will be sent.
+
+### procid_key
+
+Optionally record key where to get procid from the record. If not provided nil value will be sent.
+
+### msgid_key
+
+Optionally record key where to get msgid from the record. If not provided nil value will be sent.
+
+## Example
+
+```
+<match>
+  @type syslog_tls
+  host logs1.papertrailapp.com
+  port 12345
+  idle_timeout 720
+
+  hostname static-hostname
+  facility SYSLOG
+
+  # You can configure syslog headers to be picked from actual message
+  # processed by plugin. If key is not provided '-' value will be sent
+  # which is NIL by syslog specification.
+  severity_key RECORD_SEVERITY_KEY
+  facility_key RECORD_FACILITY_KEY
+  hostname_key ...
+  app_name_key ...
+  procid_key ...
+  msgid_key ...
+
+  # Fluent's standard formatting options are supported. Default is 'json'.
+  # Example: For Docker logs sent to Papertrail, send only the log text:
+  format single_value
+  message_key log
+</match>
+```
+
+```
+<match>
+  @type syslog_tls
+  host syslog.collection.us1.sumologic.com
+  port 6514
+  token [token]@[iana-id]
+  client_cert /path/to/cert/file.crt
+  client_key /path/to/key/file.key
+  verify_cert_name true
+
+  hostname static-hostname
+  facility SYSLOG
+
+  # You can configure syslog headers to be picked from actual message
+  # processed by plugin. If key is not provided '-' value will be sent
+  # which is NIL by syslog specification.
+  severity_key RECORD_SEVERITY_KEY
+  facility_key RECORD_FACILITY_KEY
+  hostname_key ...
+  app_name_key ...
+  procid_key ...
+  msgid_key ...
+
+  # Fluent's standard formatting options are supported. Default is 'json'.
+  format json
+</match>
+```

data/fluent-plugin-syslog-tls.gemspec

@@ -0,0 +1,43 @@
+# Copyright 2016 Acquia, Inc.
+# Copyright 2016-2023 t.e.morgan.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'syslog_tls/version'
+
+Gem::Specification.new do |s|
+  s.name          = 'fluent-plugin-syslog-gobi-tls'
+  s.version       = '2.1.1'
+  s.summary       = %q{Fluent Syslog TLS output plugin}
+  s.authors       = ['thomas morgan']
+  s.email         = ['tm@iprog.com']
+  s.description   = %q{Syslog TLS output plugin with formatting support, for Fluentd}
+  s.homepage      = 'https://github.com/zarqman/fluent-plugin-syslog-tls'
+  s.license       = 'Apache v2'
+  s.files         = `git ls-files`.split($/)
+  s.executables   = s.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  s.test_files    = s.files.grep(%r{^(test|spec|features)/})
+  s.require_paths = ['lib']
+  s.required_ruby_version = '>= 2.5'
+
+  s.add_runtime_dependency 'fluentd', [">= 0.14.0", "< 2"]
+
+  s.add_development_dependency 'minitest', '~> 5.8'
+  s.add_development_dependency 'minitest-stub_any_instance', '~> 1.0.0'
+  s.add_development_dependency 'rake'
+  s.add_development_dependency 'test-unit', '~> 3.1'
+  s.add_development_dependency 'webmock', '~> 3.0'
+  s.add_development_dependency 'simplecov', '~> 0.11'
+end

data/lib/fluent/plugin/out_syslog_tls.rb

@@ -0,0 +1,150 @@
+# Copyright 2016 Acquia, Inc.
+# Copyright 2016-2019 t.e.morgan.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'socket'
+require 'syslog_tls/logger'
+require 'fluent/plugin/output'
+
+module Fluent::Plugin
+  class SyslogTlsOutput < Output
+    Fluent::Plugin.register_output('syslog_tls', self)
+
+    helpers :inject, :formatter, :compat_parameters
+
+    DEFAULT_FORMAT_TYPE = 'json'
+
+    config_param :host, :string
+    config_param :port, :integer
+    config_param :idle_timeout, :integer, default: nil
+    config_param :ca_cert, :string, default: 'system'
+    config_param :verify_cert_name, :bool, default: true
+    config_param :token, :string, default: nil
+    config_param :client_cert, :string, default: nil
+    config_param :client_key, :string, default: nil
+    config_param :hostname, :string, default: nil
+    config_param :facility, :string, default: 'LOCAL0'
+
+    # Allow to map keys from record to syslog message headers
+    SYSLOG_HEADERS = [
+      :severity, :facility, :hostname, :app_name, :procid, :msgid
+    ]
+
+    SYSLOG_HEADERS.each do |key_name|
+      config_param "#{key_name}_key".to_sym, :string, default: nil
+    end
+
+    config_section :format do
+      config_set_default :@type, DEFAULT_FORMAT_TYPE
+    end
+
+    attr_accessor :formatter
+
+
+    def initialize
+      super
+      @loggers = {}
+    end
+
+    def shutdown
+      @loggers.values.each(&:close)
+      super
+    end
+
+    # This method is called before starting.
+    def configure(conf)
+      if conf['output_type'] && !conf['format']
+        conf['format'] = conf['output_type']
+      end
+      compat_parameters_convert(conf, :inject, :formatter)
+
+      super
+      @host = conf['host']
+      @port = conf['port']
+      @token = conf['token']
+      @hostname = conf['hostname'] || Socket.gethostname.split('.').first
+
+      # Determine mapping of record keys to syslog keys
+      @mappings = {}
+      SYSLOG_HEADERS.each do |key_name|
+        conf_key = "#{key_name}_key"
+        @mappings[key_name] = conf[conf_key] if conf.key?(conf_key)
+      end
+
+      @formatter = formatter_create(conf: conf.elements('format').first, default_type: DEFAULT_FORMAT_TYPE)
+    end
+
+    # Get logger for given tag
+    def logger(tag)
+      # Try to reuse existing logger
+      @loggers[tag] ||= new_logger(tag)
+
+      # Create new logger if old one is closed
+      if @loggers[tag].closed?
+        @loggers[tag] = new_logger(tag)
+      end
+
+      @loggers[tag]
+    end
+
+    def new_logger(tag)
+      transport = ::SyslogTls::SSLTransport.new(host, port,
+        idle_timeout: idle_timeout,
+        ca_cert: ca_cert,
+        client_cert: client_cert,
+        client_key: client_key,
+        verify_cert_name: verify_cert_name,
+        max_retries: 3,
+      )
+      logger = ::SyslogTls::Logger.new(transport, token)
+      logger.facility(facility)
+      logger.hostname(hostname)
+      logger.app_name(tag)
+      logger
+    end
+
+    def format(tag, time, record)
+      record = inject_values_to_record(tag, time, record)
+      @formatter.format(tag, time, record)
+    end
+
+    def process(tag, es)
+      es.each do |time, record|
+        record.each_pair do |_, v|
+          v.force_encoding('utf-8') if v.is_a?(String)
+        end
+
+        # Check if severity has been provided in record otherwise use INFO
+        # by default.
+        severity = if @mappings.key?(:severity)
+                     record[@mappings[:severity]] || 'INFO'
+                   else
+                     'INFO'
+                   end
+
+        # Send message to Syslog
+        begin
+          logger(tag).log(severity, format(tag, time, record), time: Time.at(time)) do |header|
+            # Map syslog headers from record
+            @mappings.each do |name, record_key|
+              header.send("#{name}=", record[record_key]) unless record[record_key].nil?
+            end
+          end
+        rescue => e
+          log.error e.to_s
+        end
+      end
+    end
+  end
+end

data/lib/syslog_tls/facility.rb

@@ -0,0 +1,46 @@
+# Copyright 2016 Acquia, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require_relative 'lookup_from_const'
+
+module SyslogTls
+  module Facility
+    extend LookupFromConst
+    KERN     =  0
+    USER     =  1
+    MAIL     =  2
+    DAEMON   =  3
+    AUTH     =  4
+    SYSLOG   =  5
+    LPR      =  6
+    NEWS     =  7
+    UUCP     =  8
+    CRON     =  9
+    AUTHPRIV = 10
+    FTP      = 11
+    NTP      = 12
+    SECURITY = 13
+    CONSOLE  = 14
+    RAS      = 15
+    LOCAL0   = 16
+    LOCAL1   = 17
+    LOCAL2   = 18
+    LOCAL3   = 19
+    LOCAL4   = 20
+    LOCAL5   = 21
+    LOCAL6   = 22
+    LOCAL7   = 23
+    NONE     = SYSLOG
+  end
+end

data/lib/syslog_tls/logger.rb

@@ -0,0 +1,82 @@
+# Copyright 2016 Acquia, Inc.
+# Copyright 2016 t.e.morgan.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require_relative 'protocol'
+require_relative 'ssl_transport'
+
+module SyslogTls
+  class Logger
+    attr_reader :token
+    attr_accessor :transport
+
+    # Logger accepts transport which should implement IO methods
+    # close, closed? and write
+    def initialize(transport, token=nil)
+      @transport = transport
+      @default_header = SyslogTls::Header.new
+      @default_structured_data = SyslogTls::StructuredData.new(token)
+    end
+
+    # Sets default facility for each message
+    def facility(val)
+      @default_header.facility = val
+    end
+
+    # Sets default hostname for each message
+    def hostname(val)
+      @default_header.hostname = val
+    end
+
+    # Sets default app_name for each message
+    def app_name(val)
+      @default_header.app_name = val
+    end
+
+    # Sets default procid for message
+    def procid(val)
+      @default_header.procid = val
+    end
+
+    # Check if IO is closed
+    def closed?
+      transport && transport.closed?
+    end
+
+    def close
+      transport.close
+    end
+
+    # Send log message with severity to syslog
+    def log(severity, message, time: nil)
+      time ||= Time.now
+
+      m = SyslogTls::Message.new
+
+      # Include authentication header
+      m.structured_data << @default_structured_data
+
+      # Adjust header with current timestamp and severity
+      m.header = @default_header.dup
+      m.header.severity = severity
+      m.header.timestamp = time
+
+      yield m.header if block_given?
+
+      m.msg = message
+
+      transport.write(m.to_s)
+    end
+  end
+end

data/lib/syslog_tls/lookup_from_const.rb

@@ -0,0 +1,36 @@
+# Copyright 2016 Acquia, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module SyslogTls
+  module LookupFromConst
+    def setup_constants(dst)
+      constants.each do |pri|
+        cval = const_get pri
+
+        dst[pri] = cval
+        dst[pri.downcase] = cval
+
+        dst[:"LOG_#{pri.to_s}"] = cval
+        dst[:"LOG_#{pri.downcase.to_s}"] = cval
+        const_set :"LOG_#{pri.to_s}", cval
+
+        dst[pri.to_s] = cval
+        dst[pri.downcase.to_s] = cval
+
+        dst[cval] = cval
+        dst[cval.to_s] = cval
+      end
+    end
+  end
+end

data/lib/syslog_tls/protocol.rb

@@ -0,0 +1,145 @@
+# Copyright 2016 Acquia, Inc.
+# Copyright 2016 t.e.morgan.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'date'
+
+require_relative 'facility'
+require_relative 'severity'
+
+# Syslog protocol https://tools.ietf.org/html/rfc5424
+module SyslogTls
+  # RFC defined nil value
+  NIL_VALUE = '-'
+
+  # All headers by specification wrapped in single object
+  class Header
+    attr_accessor :version, :hostname, :app_name, :procid, :msgid
+    attr_reader :facility, :severity, :timestamp
+
+    FACILITIES = {}
+    SEVERITIES = {}
+
+    Facility.setup_constants FACILITIES
+    Severity.setup_constants SEVERITIES
+
+    def initialize
+      @timestamp = Time.now
+      @severity = 'INFO'
+      @facility = 'LOCAL0'
+      @version = 1
+      @hostname = NIL_VALUE
+      @app_name = NIL_VALUE
+      @procid = NIL_VALUE
+      @msgid = NIL_VALUE
+    end
+
+    def timestamp=(val)
+      raise ArgumentError.new("Must provide Time object value instead: #{val.inspect}") unless val.is_a?(Time)
+      @timestamp = val
+    end
+
+    def facility=(val)
+      raise ArgumentError.new("Invalid facility value: #{val.inspect}") unless FACILITIES.key?(val)
+      @facility = val
+    end
+
+    def severity=(val)
+      raise ArgumentError.new("Invalid severity value: #{val.inspect}") unless SEVERITIES.key?(val)
+      @severity = val
+    end
+
+    # Priority value is calculated by first multiplying the Facility
+    # number by 8 and then adding the numerical value of the Severity.
+    def pri
+      FACILITIES[facility] * 8 + SEVERITIES[severity]
+    end
+
+    def assemble
+      [
+        "<#{pri}>#{version}",
+        timestamp.to_datetime.rfc3339,
+        hostname,
+        app_name,
+        procid,
+        msgid
+      ].join(' ')
+    end
+
+    def to_s
+      assemble
+    end
+  end
+
+  # Structured data field
+  class StructuredData
+    attr_accessor :id, :data
+
+    def initialize(id)
+      @id = id
+      @data = {}
+    end
+
+    # Format data structured data to
+    # [id k="v" ...]
+    def assemble
+      return NIL_VALUE unless id
+      parts = [id]
+      data.each do |k, v|
+        # Characters ", ] and \ must be escaped to prevent any parsing errors
+        v = v.gsub(/(\"|\]|\\)/) { |match| '\\' + match }
+        parts << "#{k}=\"#{v}\""
+      end
+      "[#{parts.join(' ')}]"
+    end
+
+    def to_s
+      assemble
+    end
+  end
+
+  # Message represents full message that can be sent to syslog
+  class Message
+    attr_accessor :structured_data, :msg
+    attr_writer :header
+
+    def initialize
+      @msg = ''
+      @structured_data = []
+    end
+
+    def header
+      @header ||= Header.new
+    end
+
+    def assemble
+      # Start with header
+      out = [header.to_s]
+      # Add all structured data
+      if structured_data.length > 0
+        out << structured_data.map(&:to_s).join('')
+      else
+        out << NIL_VALUE
+      end
+      # Add message
+      out << msg if msg.length > 0
+      # Message must end with new line delimiter
+      out.join(' ') + "\n"
+    end
+
+    def to_s
+      assemble
+    end
+  end
+end

data/lib/syslog_tls/severity.rb

@@ -0,0 +1,30 @@
+# Copyright 2016 Acquia, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require_relative 'lookup_from_const'
+
+module SyslogTls
+  module Severity
+    extend LookupFromConst
+    EMERG  = PANIC   = 0
+    ALERT            = 1
+    CRIT             = 2
+    ERR    = ERROR   = 3
+    WARN   = WARNING = 4
+    NOTICE           = 5
+    INFO             = 6
+    DEBUG            = 7
+    NONE             = 10
+  end
+end