fluent-plugin-remote-syslog-cg 1.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: e279be2ffb7abea3d88ceb3b3043882ccc49dce5
+  data.tar.gz: a0d594be3a4bd02c59cf6ddc4dc10b59dc98e09d
+SHA512:
+  metadata.gz: 082b8195907ee98995ff34e4b757ee66b01204258c29c316286025e59fdf18ab5ec08f1487a6e3396669df5b09b31d62a718b6cb1e8ad5f4dd8f9f2cf322c28f
+  data.tar.gz: 22e0cfc1c2e35d48355e4a00b93459fef6e8e70694cb571407f13452e1f1c76bb548dc92276de181674769d51449aa1c8b19726e7440c9a5468556c66fcb49cf

data/.gitignore

@@ -0,0 +1,3 @@
+.idea
+*.gem
+*.lock

data/Gemfile

@@ -0,0 +1,3 @@
+source "http://rubygems.org"
+
+gemspec

data/README.md

@@ -0,0 +1,41 @@
+fluent-plugin-remote-syslog-cg
+===========================
+
+fluentd plugin for streaming logs out to a remote syslog server or syslog SaaS service (like Papertrail)
+
+#Available Plugins:
+* out_syslog: registers itself as "syslog-cg", and is the non-buffered implementation, communicating through UDP
+* out_syslog_buffered: registers itself as "syslog_buffered-cg", and is the buffered implementation, communicating through TCP
+
+#Plugin Settings:
+Both plugins have the same configuration options:
+
+* remote_syslog: fqdn or ip of the remote syslog instance
+* port: the port, where the remote syslog instance is listening
+* hostname: hostname to be set for syslog messages
+* remove_tag_prefix: remove tag prefix for tag placeholder. 
+* tag_key: use the field specified in tag_key from record to set the syslog key
+* facility: Syslog log facility
+* severity: Syslog log severity
+* use_record: Use severity and facility from record if available
+* payload_key: Use the field specified in payload_key from record to set payload
+* max_size: the max size of message to send defaults to 4096
+
+#Configuration example:
+```
+<match site.*>
+  type syslog_buffered-cg
+  remote_syslog your.syslog.host
+  port 25
+  hostname ${hostname}
+  facility local0
+  severity debug
+</match>
+```
+
+
+Contributors:
+
+* CompareGroup
+* Andrea Spoldi 
+* [deathowl](http://github.com/deathowl)

data/Rakefile

@@ -0,0 +1,11 @@
+#!/usr/bin/env rake
+require "bundler/gem_tasks"
+
+require 'rake/testtask'
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.pattern = 'test/**/test_*.rb'
+  test.verbose = true
+end
+
+task :default => :test

data/fluent-plugin-remote-syslog-cg.gemspec

@@ -0,0 +1,23 @@
+# encoding: utf-8
+$:.push File.expand_path('../lib', __FILE__)
+
+Gem::Specification.new do |gem|
+  gem.name        = "fluent-plugin-remote-syslog-cg"
+  gem.description = "Output plugin for streaming logs out to a remote syslog"
+  gem.homepage    = "https://stash.comparegroup.eu/projects/IAC/repos/fluent-plugin-remote-syslog-cg/"
+  gem.summary     = gem.description
+  gem.version     = "1.0.0"
+  gem.authors     = ["CompareGroup"]
+  gem.email       = "itops-box@comparegroup.eu"
+  gem.has_rdoc    = false
+  gem.license     = 'MIT'
+  gem.files       = `git ls-files`.split("\n")
+  gem.test_files  = `git ls-files -- {test,spec,features}/*`.split("\n")
+  gem.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  gem.require_paths = ['lib']
+
+  gem.add_dependency "fluentd", "~> 0.10.45"
+  gem.add_dependency "fluent-mixin-config-placeholders", "~> 0.2.0"
+  gem.add_dependency "syslog_protocol"
+  gem.add_development_dependency "rake", ">= 0.9.2"
+end

data/lib/fluentd/plugin/out_syslog-cg.rb

@@ -0,0 +1,99 @@
+require 'fluent/mixin/config_placeholders'
+module Fluent
+
+  class SyslogOutputCG < Fluent::Output
+    # First, register the plugin. NAME is the name of this plugin
+    # and identifies the plugin in the configuration file.
+    # From configuration, fluentd knows only @type NAME information.
+    # So we write a same NAME both lib/fluent/plugin/<TYPE>_<NAME>.rb and
+    # Fluent::Plugin.register_input('NAME', self) in the plugin development article.
+    Fluent::Plugin.register_output('syslog-cg', self)
+
+    # This method is called before starting.
+
+    config_param :remote_syslog, :string, :default => nil
+    config_param :port, :integer, :default => 25
+    config_param :hostname, :string, :default => ""
+    config_param :remove_tag_prefix, :string, :default => nil
+    config_param :tag_key, :string, :default => nil
+    config_param :facility, :string, :default => 'user'
+    config_param :severity, :string, :default => 'debug'
+    config_param :use_record, :string, :default => nil
+    config_param :payload_key, :string, :default => 'message'
+    config_param :max_size, :integer, :default => 4096
+
+
+    def initialize
+      super
+      require 'socket'
+      require 'syslog_protocol'
+    end
+
+    def configure(conf)
+      super
+      if not conf['remote_syslog']
+        raise Fluent::ConfigError.new("remote syslog required")
+      end
+      @socket = UDPSocket.new
+      @packet = SyslogProtocol::Packet.new
+      if remove_tag_prefix = conf['remove_tag_prefix']
+          @remove_tag_prefix = Regexp.new('^' + Regexp.escape(remove_tag_prefix))
+      end
+      @facilty = conf['facility']
+      @severity = conf['severity']
+      @use_record = conf['use_record']
+      @payload_key = conf['payload_key']
+      if not @payload_key
+        @payload_key = "message"
+      end
+      if conf['max_size']
+        @max_size = conf['max_size'].to_i
+      end
+    end
+
+
+    # This method is called when starting.
+    def start
+      super
+    end
+
+    # This method is called when shutting down.
+    def shutdown
+      super
+    end
+
+    # This method is called when an event reaches Fluentd.
+    # 'es' is a Fluent::EventStream object that includes multiple events.
+    # You can use 'es.each {|time,record| ... }' to retrieve events.
+    # 'chain' is an object that manages transactions. Call 'chain.next' at
+    # appropriate points and rollback if it raises an exception.
+    def emit(tag, es, chain)
+      tag = tag.sub(@remove_tag_prefix, '') if @remove_tag_prefix
+      chain.next
+      es.each {|time,record|
+        @packet.hostname = hostname
+        if @use_record
+          @packet.facility = record['facility'] || @facilty
+          @packet.severity = record['severity'] || @severity
+        else
+          @packet.facility = @facilty
+          @packet.severity = @severity
+        end
+        if record['time']
+          time = Time.parse(record['time'])
+        else
+          time = Time.now
+        end
+        @packet.time = time
+        @packet.tag      = if tag_key
+                              record[tag_key][0..31].gsub(/[\[\]]/,'') # tag is trimmed to 32 chars for syslog_protocol gem compatibility
+                           else
+                              tag[0..31] # tag is trimmed to 32 chars for syslog_protocol gem compatibility
+                           end
+        packet = @packet.dup
+        packet.content = record[@payload_key]
+        @socket.send(packet.assemble(@max_size), 0, @remote_syslog, @port)
+    }
+    end
+  end
+end

data/lib/fluentd/plugin/out_syslog_buffered-cg.rb

@@ -0,0 +1,142 @@
+require 'fluent/mixin/config_placeholders'
+module Fluent
+  class SyslogBufferedOutputCG < Fluent::BufferedOutput
+    # First, register the plugin. NAME is the name of this plugin
+    # and identifies the plugin in the configuration file.
+    # From configuration, fluentd knows only @type NAME information.
+    # So we write a same NAME both lib/fluent/plugin/<TYPE>_<NAME>.rb and
+    # Fluent::Plugin.register_input('NAME', self) in the plugin development article.
+    Fluent::Plugin.register_output('syslog_buffered-cg', self)
+
+    # This method is called before starting.
+
+    config_param :remote_syslog, :string, :default => ""
+    config_param :port, :integer, :default => 25
+    config_param :hostname, :string, :default => ""
+    config_param :remove_tag_prefix, :string, :default => nil
+    config_param :tag_key, :string, :default => nil
+    config_param :facility, :string, :default => 'user'
+    config_param :severity, :string, :default => 'debug'
+    config_param :use_record, :string, :default => nil
+    config_param :payload_key, :string, :default => 'message'
+    config_param :max_size, :integer, :default => 4096
+
+
+    def initialize
+      super
+      require 'socket'
+      require 'syslog_protocol'
+      require 'timeout'
+    end
+
+    def configure(conf)
+      super
+      if not conf['remote_syslog']
+        raise Fluent::ConfigError.new("remote syslog required")
+      end
+        @socket = create_tcp_socket(conf['remote_syslog'], conf['port'])
+      @packet = SyslogProtocol::Packet.new
+      if remove_tag_prefix = conf['remove_tag_prefix']
+        @remove_tag_prefix = Regexp.new('^' + Regexp.escape(remove_tag_prefix))
+      end
+      @facilty = conf['facility']
+      @severity = conf['severity']
+      @use_record = conf['use_record']
+      @payload_key = conf['payload_key']
+      if not @payload_key
+        @payload_key = "message"
+      end
+      if conf['max_size']
+        @max_size = conf['max_size'].to_i
+      end
+    end
+
+    def format(tag, time, record)
+      [tag, time, record].to_msgpack
+    end
+
+    def create_tcp_socket(host, port)
+      begin
+        Timeout.timeout(10) do
+          begin
+            socket = TCPSocket.new(host, port)
+          rescue Errno::ENETUNREACH
+            retry
+          end
+        end
+        socket = TCPSocket.new(host, port)
+        secs = Integer(1)
+        usecs = Integer((1 - secs) * 1_000_000)
+        optval = [secs, usecs].pack("l_2")
+        socket.setsockopt Socket::SOL_SOCKET, Socket::SO_SNDTIMEO, optval
+      rescue SocketError, Errno::ECONNREFUSED, Errno::ECONNRESET, Errno::EPIPE, Timeout::Error, OpenSSL::SSL::SSLError, Timeout::Error => e
+        log.warn "out:syslog: failed to open tcp socket  #{@remote_syslog}:#{@port} :#{e}"
+        socket = nil
+      end
+      socket
+    end
+
+    # This method is called when starting.
+    def start
+      super
+    end
+
+    # This method is called when shutting down.
+    def shutdown
+      super
+    end
+
+
+    def write(chunk)
+      chunk.msgpack_each {|(tag,time,record)|
+            send_to_syslog(tag, time, record)
+          }
+    end
+
+    def send_to_syslog(tag, time, record)
+      tag = tag.sub(@remove_tag_prefix, '') if @remove_tag_prefix
+      @packet.hostname = hostname
+      if @use_record
+        @packet.facility = record['facility'] || @facilty
+        @packet.severity = record['severity'] || @severity
+      else
+        @packet.facility = @facilty
+        @packet.severity = @severity
+      end
+      if record['time']
+        time = Time.parse(record['time'])
+      else
+        time = Time.now
+      end
+      @packet.time = time
+      @packet.tag      = if tag_key
+                           record[tag_key][0..31].gsub(/[\[\]]/,'') # tag is trimmed to 32 chars for syslog_protocol gem compatibility
+                         else
+                           tag[0..31] # tag is trimmed to 32 chars for syslog_protocol gem compatibility
+                         end
+      packet = @packet.dup
+      packet.content = record[@payload_key]
+      begin
+        if not @socket
+          @socket = create_tcp_socket(@remote_syslog, @port)
+        end
+        if @socket
+          begin
+            @socket.write packet.assemble(@max_size) + "\n"
+            @socket.flush
+          rescue SocketError, Errno::ECONNREFUSED, Errno::ECONNRESET, Errno::EPIPE, Timeout::Error, OpenSSL::SSL::SSLError => e
+            log.warn "out:syslog: connection error by #{@remote_syslog}:#{@port} :#{e}"
+            @socket = nil
+            raise #{e}
+          end
+        else
+          log.warn "out:syslog: Socket connection couldn't be reestablished"
+          raise #{e}
+        end
+      end
+    end
+
+
+  end
+end
+

data/test/helper.rb

@@ -0,0 +1,30 @@
+require 'rubygems'
+require 'bundler'
+begin
+  Bundler.setup(:default, :development)
+rescue Bundler::BundlerError => e
+  $stderr.puts e.message
+  $stderr.puts "Run `bundle install` to install missing gems"
+  exit e.status_code
+end
+require 'test/unit'
+
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+require 'fluent/test'
+unless ENV.has_key?('VERBOSE')
+  nulllogger = Object.new
+  nulllogger.instance_eval {|obj|
+    def method_missing(method, *args)
+      # pass
+    end
+  }
+  $log = nulllogger
+end
+
+require 'fluentd/plugin/out_syslog-cg'
+require 'fluentd/plugin/out_syslog_buffered-cg'
+
+
+class Test::Unit::TestCase
+end

data/test/plugin/test_out_syslog.rb

@@ -0,0 +1,138 @@
+require 'helper'
+
+class SyslogOutputTest < Test::Unit::TestCase
+  def setup
+    Fluent::Test.setup
+  end
+
+  CONFIG = %[
+    remote_syslog 127.0.0.1
+    port    25
+    hostname      testhost
+    remove_tag_prefix        test
+    severity  debug
+    facility user
+    payload_key message
+  ]
+
+  def create_driver(conf=CONFIG,tag='test')
+    Fluent::Test::OutputTestDriver.new(Fluent::SyslogOutputCG, tag).configure(conf)
+  end
+
+  def test_configure
+    assert_raise(Fluent::ConfigError) {
+      d = create_driver('')
+    }
+    assert_raise(Fluent::ConfigError) {
+      d = create_driver %[
+      hostname      testhost
+      remove_tag_prefix        test
+      ]
+    }
+    assert_nothing_raised {
+      d = create_driver %[
+        remote_syslog 127.0.0.1
+      ]
+    }
+    assert_nothing_raised {
+      d = create_driver %[
+        remote_syslog 127.0.0.1
+        port    639
+      ]
+    }
+    assert_nothing_raised {
+      d = create_driver %[
+        remote_syslog 127.0.0.1
+        port    25
+        hostname      deathstar
+      ]
+    }
+    assert_nothing_raised {
+      d = create_driver %[
+       remote_syslog 127.0.0.1
+        port    25
+        hostname      testhost
+        remove_tag_prefix        test123
+      ]
+    }
+    assert_nothing_raised {
+      d = create_driver %[
+        remote_syslog 127.0.0.1
+        port    25
+        hostname      testhost
+        remove_tag_prefix        test
+        tag_key   tagtag
+        severity  debug
+      ]
+    }
+    assert_nothing_raised {
+      d = create_driver %[
+        remote_syslog 127.0.0.1
+        port    25
+        hostname      testhost
+        remove_tag_prefix        test
+        tag_key   tagtag
+        severity  debug
+        facility user
+      ]
+    }
+    assert_nothing_raised {
+      d = create_driver %[
+        remote_syslog 127.0.0.1
+        port    25
+        hostname      testhost
+        remove_tag_prefix        test
+        tag_key   tagtag
+        severity  debug
+        facility user
+        payload_key message
+      ]
+    }
+    d = create_driver %[
+        remote_syslog 127.0.0.1
+        port    25
+        hostname      testhost
+        remove_tag_prefix        test
+        tag_key   tagtag
+        severity  debug
+        facility user
+        payload_key message
+    ]
+    assert_equal 25, d.instance.port
+    assert_equal "127.0.0.1", d.instance.remote_syslog
+    assert_equal "testhost", d.instance.hostname
+    assert_equal Regexp.new('^' + Regexp.escape("test")), d.instance.remove_tag_prefix
+    assert_equal "tagtag", d.instance.tag_key
+    assert_equal "debug", d.instance.severity
+    assert_equal "user", d.instance.facility
+    assert_equal "message", d.instance.payload_key
+
+  end
+  def test_emit
+    d1 = create_driver(CONFIG, 'test.in')
+    d1.run do
+      d1.emit({'message' => 'asd asd'})
+      d1.emit({'message' => 'dsa xasd'})
+      d1.emit({'message' => 'ddd ddddd'})
+      d1.emit({'message' => '7sssss8 ssssdasd'})
+      d1.emit({'message' => 'aaassddffg asdasdasfasf'})
+    end
+    assert_equal 0, d1.emits.size
+
+  end
+
+  def test_emit_with_time_and_without_time
+    d1 = create_driver(CONFIG, 'test.in')
+    d1.run do
+      d1.emit({'message' => 'asd asd', 'time' => '2007-01-31 12:22:26'})
+      d1.emit({'message' => 'dsa xasd'})
+      d1.emit({'message' => 'ddd ddddd', 'time' => '2007-03-01 12:22:26'})
+      d1.emit({'message' => '7sssss8 ssssdasd', 'time' => '2011-03-01 12:22:26'})
+      d1.emit({'message' => 'aaassddffg asdasdasfasf', 'time' => '2016-03-01 12:22:26'})
+    end
+    assert_equal 0, d1.emits.size
+
+  end
+
+
+end

metadata

@@ -0,0 +1,110 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-remote-syslog-cg
+version: !ruby/object:Gem::Version
+  version: 1.0.0
+platform: ruby
+authors:
+- CompareGroup
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2018-07-18 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.10.45
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.10.45
+- !ruby/object:Gem::Dependency
+  name: fluent-mixin-config-placeholders
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.2.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.2.0
+- !ruby/object:Gem::Dependency
+  name: syslog_protocol
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.9.2
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.9.2
+description: Output plugin for streaming logs out to a remote syslog
+email: itops-box@comparegroup.eu
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- Gemfile
+- README.md
+- Rakefile
+- fluent-plugin-remote-syslog-cg.gemspec
+- lib/fluentd/plugin/out_syslog-cg.rb
+- lib/fluentd/plugin/out_syslog_buffered-cg.rb
+- test/helper.rb
+- test/plugin/test_out_syslog.rb
+homepage: https://stash.comparegroup.eu/projects/IAC/repos/fluent-plugin-remote-syslog-cg/
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.6.14
+signing_key: 
+specification_version: 4
+summary: Output plugin for streaming logs out to a remote syslog
+test_files:
+- test/helper.rb
+- test/plugin/test_out_syslog.rb