fluent-plugin-remote_syslog_multiline 1.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 180fbce858c4fbb8cf9216ab3ad30aded4ab7c5a
+  data.tar.gz: f9312dcaf78a7cb1f04a2fa8908077bf80d53b77
+SHA512:
+  metadata.gz: f32b13a75d118bfa356ea8176f5383b2c1bdf7eb3bd48693ca2f64cd7529b698d3c829c29043f0eee406556de10236c4038857beb4f2243482f68c883308e385
+  data.tar.gz: e50fb00dc990de3ab7a0e33be31d1297fa576fc3406c3cca4a5cb7c5971b2aee1ef94bac9bb4d10eaad4204c6f6d324349cf8e865fda5a5b1e357dfd3413922b

data/.gitignore

@@ -0,0 +1,14 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+*.bundle
+*.so
+*.o
+*.a
+mkmf.log

data/.travis.yml

@@ -0,0 +1,10 @@
+language: ruby
+sudo: false
+cache: bundler
+rvm:
+  - 2.1.10
+  - 2.2.5
+  - 2.3.1
+  - 2.4.1
+before_install:
+  - gem update bundler

data/CHANGELOG.md

@@ -0,0 +1,35 @@
+## 1.0.0
+
+* Support fluentd-0.14 or later features [#24](https://github.com/dlackty/fluent-plugin-remote_syslog/pull/24)
+* Support TCP and TLS protocol [#24](https://github.com/dlackty/fluent-plugin-remote_syslog/pull/24)
+* Not support fluentd-0.12.0 from this version
+
+## 0.3.3
+
+* Allow override "severity" or "facility" from records [#19](https://github.com/dlackty/fluent-plugin-remote_syslog/pull/19)
+* Change default port to 514 [#15](https://github.com/dlackty/fluent-plugin-remote_syslog/pull/15)
+
+## 0.3.2
+
+* Rewrite plugin to make rewrite tag function work properly [#4](https://github.com/dlackty/fluent-plugin-remote_syslog/pull/4)
+
+## 0.3.1
+
+* Fix errors in last release [#3](https://github.com/dlackty/fluent-plugin-remote_syslog/pull/3)
+
+## 0.3.0 (yanked)
+
+* Integrate with `Fluent::Mixin::RewriteTagName` [#2](https://github.com/dlackty/fluent-plugin-remote_syslog/pull/2)
+
+## 0.2.1
+
+* Fix encoding issue
+
+## 0.2.0
+
+* Integrate with `Fluent::Mixin::PlainTextFormatter`
+* **BREAKING**: Remove `key_name` config, use `output_data_type` instead
+
+## 0.1.0
+
+* Initial release

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in fluent-plugin-remote_syslog.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2014 Richard Lee
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,83 @@
+# fluent-plugin-remote_syslog
+
+[![Build Status](https://travis-ci.org/dlackty/fluent-plugin-remote_syslog.svg?branch=master)](https://travis-ci.org/dlackty/fluent-plugin-remote_syslog)
+
+[Fluentd](http://fluentd.org) plugin for output to remote syslog serivce (e.g. [Papertrail](http://papertrailapp.com/))
+
+## Requirements
+
+| fluent-plugin-remote_syslog | fluentd                 | ruby   |
+| -------------------         | ---------               | ------ |
+| >= 1.0.0                    | >= v0.14.0 or >= v1.0.0 | >= 2.1 |
+| < 1.0.0                     | >= v0.12.0              | >= 1.9 |
+
+## Installation
+
+```bash
+ fluent-gem install fluent-plugin-remote_syslog
+```
+
+## Usage
+
+```
+<match foo.bar>
+  @type remote_syslog
+  host example.com
+  port 514
+  severity debug
+  program fluentd
+  hostname ${tag[1]}
+
+  <buffer tag>
+  </buffer>
+
+  <format>
+    @type single_value
+    message_key msg
+  </format>
+</match>
+```
+
+## Configuration
+
+| name              | type                             | placeholder support | description                                           |
+| --------------    | -------                          | -----------         | ---------------------------------                     |
+| hostname          | string                           | support             | departure of log                                      |
+| host              | string                           | support             | syslog target host                                    |
+| port              | integer (default: `514`)         |                     | syslog target port                                    |
+| host_with_port    | string                           | support             | parameter for <host>:<port> style                     |
+| facility          | string (default: `"user"`)       | support             | syslog facility                                       |
+| severity          | string (default: `"notice"`      | support             | syslog severity                                       |
+| program           | string (default: `"fluentd"`     | support             | syslog program name                                   |
+| protocol          | enum (udp, tcp) (default: `udp`) |                     | transfer protocol                                     |
+| tls               | bool (default: false)            |                     | use TLS (tcp only)                                    |
+| ca_file           | string                           |                     | ca_file path (tls mode only)                          |
+| verify_mode       | integer                          |                     | SSL verification mode (tls mode only)                 |
+| packet_size       | integer (default: `1024`)        |                     | size limitation for syslog packet                     |
+| timeout           | integer                          |                     | TCP transfer timeout. if value is 0, wait forever     |
+| timeout_exception | bool (default: `false`)          |                     | if value is true, raise exception by transfer timeout |
+| keep_alive        | bool (default: `false`)          |                     | use TCP keep alive                                    |
+| keep_alive_idle   | integer                          |                     | set TCP keep alive idle time                          |
+| keep_alive_cnt    | integer                          |                     | set TCP keep alive probe count                        |
+| keep_alive_intvl  | integer                          |                     | set TCP keep alive probe interval                     |
+
+### Common Configuration
+
+#### Buffer Section
+
+| name                        | default  |
+| --------------              | -------  |
+| flush_mode                  | interval |
+| flush_interval              | 5        |
+| flush_thread_interval       | 0.5      |
+| flush_thread_burst_interval | 0.5      |
+
+#### Format Section
+
+| name           | default |
+| -------------- | ------- |
+| @type          | ltsv    |
+
+## License
+
+Copyright (c) 2014-2017 Richard Lee. See LICENSE for details.

data/Rakefile

@@ -0,0 +1,10 @@
+require "bundler/gem_tasks"
+
+require 'rake/testtask'
+Rake::TestTask.new(:test) do |test|
+  test.libs << "lib" << "test"
+  test.pattern = "test/plugin/*.rb"
+  test.verbose = true
+end
+
+task :default => :test

data/VERSION

@@ -0,0 +1 @@
+1.0.0

data/fluent-plugin-remote_syslog.gemspec

@@ -0,0 +1,25 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+
+Gem::Specification.new do |spec|
+  spec.name          = "fluent-plugin-remote_syslog_multiline"
+  spec.version       = File.read("VERSION").strip
+  spec.authors       = ["Lawrence Li","Richard Lee"]
+  spec.email         = ["lawrleegle@gmail.com"]
+  spec.summary       = %q{Fluentd output plugin for remote syslog}
+  spec.description   = spec.description
+  spec.homepage      = "https://github.com/gitlawr/fluent-plugin-remote_syslog"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "test-unit"
+  spec.add_development_dependency "test-unit-rr"
+
+  spec.add_runtime_dependency "fluentd"
+  spec.add_runtime_dependency "remote_syslog_sender", ">= 1.1.1"
+end

data/lib/fluent/plugin/out_remote_syslog.rb

@@ -0,0 +1,153 @@
+require "remote_syslog_sender"
+
+module Fluent
+  module Plugin
+    class RemoteSyslogOutput < Output
+      Fluent::Plugin.register_output("remote_syslog", self)
+
+      helpers :formatter, :inject
+
+      config_param :hostname, :string, :default => ""
+
+      config_param :host, :string, :default => nil
+      config_param :port, :integer, :default => 514
+      config_param :host_with_port, :string, :default => nil
+
+      config_param :facility, :string, :default => "user"
+      config_param :severity, :string, :default => "notice"
+      config_param :program, :string, :default => "fluentd"
+
+      config_param :protocol, :enum, list: [:udp, :tcp], :default => :udp
+      config_param :tls, :bool, :default => false
+      config_param :ca_file, :string, :default => nil
+      config_param :verify_mode, :integer, default: nil
+      config_param :packet_size, :size, default: 1024
+      config_param :timeout, :time, default: nil
+      config_param :timeout_exception, :bool, default: false
+
+      config_param :keep_alive, :bool, :default => false
+      config_param :keep_alive_idle, :integer, :default => nil
+      config_param :keep_alive_cnt, :integer, :default => nil
+      config_param :keep_alive_intvl, :integer, :default => nil
+
+      config_section :buffer do
+        config_set_default :flush_mode, :interval
+        config_set_default :flush_interval, 5
+        config_set_default :flush_thread_interval, 0.5
+        config_set_default :flush_thread_burst_interval, 0.5
+      end
+
+      config_section :format do
+        config_set_default :@type, 'ltsv'
+      end
+
+      def initialize
+        super
+      end
+
+      def configure(conf)
+        super
+        if @host.nil? && @host_with_port.nil?
+          raise ConfigError, "host or host_with_port is required"
+        end
+
+        @formatter = formatter_create
+        unless @formatter.formatter_type == :text_per_line
+          raise ConfigError, "formatter_type must be text_per_line formatter"
+        end
+
+        validate_target = "host=#{@host}/host_with_port=#{@host_with_port}/hostname=#{@hostname}/facility=#{@facility}/severity=#{@severity}/program=#{@program}"
+        placeholder_validate!(:remote_syslog, validate_target)
+        @senders = []
+      end
+
+      def multi_workers_ready?
+        true
+      end
+
+      def close
+        super
+        @senders.each { |s| s.close if s }
+        @senders.clear
+      end
+
+      def format(tag, time, record)
+        r = inject_values_to_record(tag, time, record)
+        @formatter.format(tag, time, r)
+      end
+
+      def write(chunk)
+        return if chunk.empty?
+
+        host = extract_placeholders(@host, chunk.metadata)
+        port = @port
+
+        if @host_with_port
+          host, port = extract_placeholders(@host_with_port, chunk.metadata).split(":")
+        end
+
+        host_with_port = "#{host}:#{port}"
+
+        Thread.current[host_with_port] ||= create_sender(host, port)
+        sender = Thread.current[host_with_port]
+
+        facility = extract_placeholders(@facility, chunk.metadata)
+        severity = extract_placeholders(@severity, chunk.metadata)
+        program = extract_placeholders(@program, chunk.metadata)
+        hostname = extract_placeholders(@hostname, chunk.metadata)
+
+        packet_options = {facility: facility, severity: severity, program: program}
+        packet_options[:hostname] = hostname unless hostname.empty?
+
+        begin
+          chunk.open do |io|
+            msg = io.read
+            sender.transmit(msg.chomp!, packet_options)
+          end
+        rescue
+          if Thread.current[host_with_port]
+            Thread.current[host_with_port].close
+            @senders.delete(Thread.current[host_with_port])
+            Thread.current[host_with_port] = nil
+          end
+          raise
+        end
+      end
+
+      private
+
+      def create_sender(host, port)
+        if @protocol == :tcp
+          options = {
+            tls: @tls,
+            whinyerrors: true,
+            packet_size: @packet_size,
+            timeout: @timeout,
+            timeout_exception: @timeout_exception,
+            keep_alive: @keep_alive,
+            keep_alive_idle: @keep_alive_idle,
+            keep_alive_cnt: @keep_alive_cnt,
+            keep_alive_intvl: @keep_alive_intvl,
+            program: @program,
+          }
+          options[:ca_file] = @ca_file if @ca_file
+          options[:verify_mode] = @verify_mode if @verify_mode
+          sender = RemoteSyslogSender::TcpSender.new(
+            host,
+            port,
+            options
+          )
+        else
+          sender = RemoteSyslogSender::UdpSender.new(
+            host,
+            port,
+            whinyerrors: true,
+            program: @program,
+          )
+        end
+        @senders << sender
+        sender
+      end
+    end
+  end
+end

data/test/plugin/out_remote_syslog.rb

@@ -0,0 +1,84 @@
+require "test_helper"
+require "fluent/plugin/out_remote_syslog"
+
+class RemoteSyslogOutputTest < Test::Unit::TestCase
+  def setup
+    Fluent::Test.setup
+  end
+
+  def create_driver(conf = CONFIG)
+    Fluent::Test::Driver::Output.new(Fluent::Plugin::RemoteSyslogOutput).configure(conf)
+  end
+
+  def test_configure
+    d = create_driver %[
+      @type remote_syslog
+      hostname foo.com
+      host example.com
+      port 5566
+      severity debug
+      program minitest
+    ]
+
+    loggers = d.instance.instance_variable_get(:@senders)
+    assert_equal loggers, []
+
+    assert_equal "example.com", d.instance.instance_variable_get(:@host)
+    assert_equal 5566, d.instance.instance_variable_get(:@port)
+    assert_equal "debug", d.instance.instance_variable_get(:@severity)
+  end
+
+  def test_write
+    d = create_driver %[
+      @type remote_syslog
+      hostname foo.com
+      host example.com
+      port 5566
+      severity debug
+      program minitest
+
+      <format>
+        @type single_value
+        message_key message
+      </format>
+    ]
+
+    mock.proxy(RemoteSyslogSender::UdpSender).new("example.com", 5566, whinyerrors: true, program: "minitest") do |sender|
+      mock.proxy(sender).transmit("foo",  facility: "user", severity: "debug", program: "minitest", hostname: "foo.com")
+    end
+
+    d.run do
+      d.feed("tag", Fluent::EventTime.now, {"message" => "foo"})
+    end
+  end
+
+  def test_write_tcp
+    d = create_driver %[
+      @type remote_syslog
+      hostname foo.com
+      host example.com
+      port 5566
+      severity debug
+      program minitest
+
+      protocol tcp
+
+      <format>
+        @type single_value
+        message_key message
+      </format>
+    ]
+
+    any_instance_of(RemoteSyslogSender::TcpSender) do |klass|
+      mock(klass).connect
+    end
+
+    mock.proxy(RemoteSyslogSender::TcpSender).new("example.com", 5566, whinyerrors: true, program: "minitest", tls: false, packet_size: 1024, timeout: nil, timeout_exception: false, keep_alive: false, keep_alive_cnt: nil, keep_alive_idle: nil, keep_alive_intvl: nil) do |sender|
+      mock(sender).transmit("foo",  facility: "user", severity: "debug", program: "minitest", hostname: "foo.com")
+    end
+
+    d.run do
+      d.feed("tag", Fluent::EventTime.now, {"message" => "foo"})
+    end
+  end
+end

data/test/test_helper.rb

@@ -0,0 +1,10 @@
+require 'bundler/setup'
+require 'test/unit'
+
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), "..", "lib"))
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+
+require "fluent/test"
+require 'fluent/test/driver/output'
+
+require 'test/unit/rr'

metadata

@@ -0,0 +1,129 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-remote_syslog_multiline
+version: !ruby/object:Gem::Version
+  version: 1.0.0
+platform: ruby
+authors:
+- Lawrence Li
+- Richard Lee
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2020-06-16 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: test-unit
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: test-unit-rr
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: remote_syslog_sender
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.1.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.1.1
+description: ''
+email:
+- lawrleegle@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".travis.yml"
+- CHANGELOG.md
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- VERSION
+- fluent-plugin-remote_syslog.gemspec
+- lib/fluent/plugin/out_remote_syslog.rb
+- test/plugin/out_remote_syslog.rb
+- test/test_helper.rb
+homepage: https://github.com/gitlawr/fluent-plugin-remote_syslog
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.6.11
+signing_key: 
+specification_version: 4
+summary: Fluentd output plugin for remote syslog
+test_files:
+- test/plugin/out_remote_syslog.rb
+- test/test_helper.rb