fluent-plugin-remote_syslog 0.3.3 → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 5cfd7384704b75fef6c379506ad00bd2556dfcb6
-  data.tar.gz: 9df4ff109259582a6ce45684ee26e1297b3fe8f4
+  metadata.gz: 5b7602139f0aea954fe6d7fbfe8ea83aedc6a218
+  data.tar.gz: 041633a7947a2d028a91a91cc4b8fb60d1255990
 SHA512:
-  metadata.gz: 698c42dcefb20f4866e89d09cf1f5601cac77d34c4e4748d05b35c8f7846d6d47cd53fbcaffafa821f5c76e74f15442c4c827cb53edc7af02ae86383de95f6d3
-  data.tar.gz: 15b23fc23fc097e8be9fdf4e9ff4028754e5c600041c512b68b5db820f9de4b07edf937d34c0c3fd2aba8959a36737b8600bbfd6bc1799b66d123f46be63dffc
+  metadata.gz: e6582e74ecb339c7d39a862ca5122b935271940d8c8322ec214aada1adcb2d13f79a9a11ef8571fa87e9028d27b1c5f3c0f88d74d21dcdc2076fac1323eabb74
+  data.tar.gz: 443286f184920a6f0a35bab20e58fa6b1c837ee2246d044ff66f2c3ad93e31d62741ab8a145535e77badbfa13c7d7bffab1c65c8e000f74ea4b248876255292b

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+## 1.0.0
+
+* Support fluentd-0.14 or later features [#24](https://github.com/dlackty/fluent-plugin-remote_syslog/pull/24)
+* Support TCP and TLS protocol [#24](https://github.com/dlackty/fluent-plugin-remote_syslog/pull/24)
+* Not support fluentd-0.12.0 from this version
+
 ## 0.3.3
 
 * Allow override "severity" or "facility" from records [#19](https://github.com/dlackty/fluent-plugin-remote_syslog/pull/19)

data/Gemfile

@@ -1,4 +1,6 @@
 source 'https://rubygems.org'
 
+gem 'remote_syslog_sender', github: "reproio/remote_syslog_logger"
+
 # Specify your gem's dependencies in fluent-plugin-remote_syslog.gemspec
 gemspec

data/README.md

@@ -4,6 +4,13 @@
 
 [Fluentd](http://fluentd.org) plugin for output to remote syslog serivce (e.g. [Papertrail](http://papertrailapp.com/))
 
+## Requirements
+
+| fluent-plugin-remote_syslog | fluentd                 | ruby   |
+| -------------------         | ---------               | ------ |
+| >= 1.0.0                    | >= v0.14.0 or >= v1.0.0 | >= 2.1 |
+| < 1.0.0                     | >= v0.12.0              | >= 1.9 |
+
 ## Installation
 
 ```bash
@@ -13,16 +20,63 @@
 ## Usage
 
 ```
-<match foo>
-  type remote_syslog
+<match foo.bar>
+  @type remote_syslog
   host example.com
   port 514
   severity debug
-  tag fluentd
+  program fluentd
+  hostname ${tag[1]}
+
+  <buffer tag>
+  </buffer>
+
+  <format>
+    @type single_value
+    message_key msg
+  </format>
 </match>
 ```
 
-This plugin makes use of [Fluent::Mixin::PlainTextFormatter](https://github.com/tagomoris/fluent-mixin-plaintextformatter) and [Fluent::Mixin::RewriteTagName](https://github.com/y-ken/fluent-mixin-rewrite-tag-name), please check out their documentations for more configuration options.
+## Configuration
+
+| name              | type                             | placeholder support | description                                           |
+| --------------    | -------                          | -----------         | ---------------------------------                     |
+| hostname          | string                           | support             | departure of log                                      |
+| host              | string                           | support             | syslog target host                                    |
+| port              | integer (default: `514`)         |                     | syslog target port                                    |
+| host_with_port    | string                           | support             | parameter for <host>:<port> style                     |
+| facility          | string (default: `"user"`)       | support             | syslog facility                                       |
+| severity          | string (default: `"notice"`      | support             | syslog severity                                       |
+| program           | string (default: `"fluentd"`     | support             | syslog program name                                   |
+| protocol          | enum (udp, tcp) (default: `udp`) |                     | transfer protocol                                     |
+| tls               | bool (default: false)            |                     | use TLS (tcp only)                                    |
+| ca_file           | string                           |                     | ca_file path (tls mode only)                          |
+| verify_mode       | integer                          |                     | SSL verification mode (tls mode only)                 |
+| packet_size       | integer (default: `1024`)        |                     | size limitation for syslog packet                     |
+| timeout           | integer                          |                     | TCP transfer timeout. if value is 0, wait forever     |
+| timeout_exception | bool (default: `false`)          |                     | if value is true, raise exception by transfer timeout |
+| keep_alive        | bool (default: `false`)          |                     | use TCP keep alive                                    |
+| keep_alive_idle   | integer                          |                     | set TCP keep alive idle time                          |
+| keep_alive_cnt    | integer                          |                     | set TCP keep alive probe count                        |
+| keep_alive_intvl  | integer                          |                     | set TCP keep alive probe interval                     |
+
+### Common Configuration
+
+#### Buffer Section
+
+| name                        | default  |
+| --------------              | -------  |
+| flush_mode                  | interval |
+| flush_interval              | 5        |
+| flush_thread_interval       | 0.5      |
+| flush_thread_burst_interval | 0.5      |
+
+#### Format Section
+
+| name           | default |
+| -------------- | ------- |
+| @type          | ltsv    |
 
 ## License
 

data/VERSION

@@ -1 +1 @@
-0.3.3
+1.0.0

data/fluent-plugin-remote_syslog.gemspec

@@ -17,12 +17,9 @@ Gem::Specification.new do |spec|
   spec.require_paths = ["lib"]
 
   spec.add_development_dependency "rake", "~> 10.0"
-  spec.add_development_dependency "test-unit-minitest"
-  spec.add_development_dependency "minitest"
+  spec.add_development_dependency "test-unit"
+  spec.add_development_dependency "test-unit-rr"
 
   spec.add_runtime_dependency "fluentd"
-  spec.add_runtime_dependency "fluent-mixin-plaintextformatter"
-  spec.add_runtime_dependency "remote_syslog_logger", "~> 1.0.0"
-  spec.add_runtime_dependency "fluent-mixin-config-placeholders"
-  spec.add_runtime_dependency "fluent-mixin-rewrite-tag-name"
+  spec.add_runtime_dependency "remote_syslog_sender", ">= 1.1.1"
 end

data/lib/fluent/plugin/out_remote_syslog.rb

@@ -1,54 +1,153 @@
-require "fluent/mixin/config_placeholders"
-require "fluent/mixin/plaintextformatter"
-require 'fluent/mixin/rewrite_tag_name'
+require "remote_syslog_sender"
 
 module Fluent
-  class RemoteSyslogOutput < Fluent::Output
-    Fluent::Plugin.register_output("remote_syslog", self)
+  module Plugin
+    class RemoteSyslogOutput < Output
+      Fluent::Plugin.register_output("remote_syslog", self)
 
-    config_param :hostname, :string, :default => ""
+      helpers :formatter, :inject
 
-    include Fluent::Mixin::PlainTextFormatter
-    include Fluent::Mixin::ConfigPlaceholders
-    include Fluent::HandleTagNameMixin
-    include Fluent::Mixin::RewriteTagName
+      config_param :hostname, :string, :default => ""
 
-    config_param :host, :string
-    config_param :port, :integer, :default => 514
+      config_param :host, :string, :default => nil
+      config_param :port, :integer, :default => 514
+      config_param :host_with_port, :string, :default => nil
 
-    config_param :facility, :string, :default => "user"
-    config_param :severity, :string, :default => "notice"
-    config_param :tag, :string, :default => "fluentd"
+      config_param :facility, :string, :default => "user"
+      config_param :severity, :string, :default => "notice"
+      config_param :program, :string, :default => "fluentd"
 
-    def initialize
-      super
-      require "remote_syslog_logger"
-      @loggers = {}
-    end
+      config_param :protocol, :enum, list: [:udp, :tcp], :default => :udp
+      config_param :tls, :bool, :default => false
+      config_param :ca_file, :string, :default => nil
+      config_param :verify_mode, :integer, default: nil
+      config_param :packet_size, :size, default: 1024
+      config_param :timeout, :time, default: nil
+      config_param :timeout_exception, :bool, default: false
 
-    def shutdown
-      super
-      @loggers.values.each(&:close)
-    end
+      config_param :keep_alive, :bool, :default => false
+      config_param :keep_alive_idle, :integer, :default => nil
+      config_param :keep_alive_cnt, :integer, :default => nil
+      config_param :keep_alive_intvl, :integer, :default => nil
+
+      config_section :buffer do
+        config_set_default :flush_mode, :interval
+        config_set_default :flush_interval, 5
+        config_set_default :flush_thread_interval, 0.5
+        config_set_default :flush_thread_burst_interval, 0.5
+      end
+
+      config_section :format do
+        config_set_default :@type, 'ltsv'
+      end
+
+      def initialize
+        super
+      end
+
+      def configure(conf)
+        super
+        if @host.nil? && @host_with_port.nil?
+          raise ConfigError, "host or host_with_port is required"
+        end
+
+        @formatter = formatter_create
+        unless @formatter.formatter_type == :text_per_line
+          raise ConfigError, "formatter_type must be text_per_line formatter"
+        end
+
+        validate_target = "host=#{@host}/host_with_port=#{@host_with_port}/hostname=#{@hostname}/facility=#{@facility}/severity=#{@severity}/program=#{@program}"
+        placeholder_validate!(:remote_syslog, validate_target)
+        @senders = []
+      end
+
+      def multi_workers_ready?
+        true
+      end
 
-    def emit(tag, es, chain)
-      chain.next
-      es.each do |time, record|
-        record.each_pair do |k, v|
-          if v.is_a?(String)
-            v.force_encoding("utf-8")
+      def close
+        super
+        @senders.each { |s| s.close if s }
+        @senders.clear
+      end
+
+      def format(tag, time, record)
+        r = inject_values_to_record(tag, time, record)
+        @formatter.format(tag, time, r)
+      end
+
+      def write(chunk)
+        return if chunk.empty?
+
+        host = extract_placeholders(@host, chunk.metadata)
+        port = @port
+
+        if @host_with_port
+          host, port = extract_placeholders(@host_with_port, chunk.metadata).split(":")
+        end
+
+        host_with_port = "#{host}:#{port}"
+
+        Thread.current[host_with_port] ||= create_sender(host, port)
+        sender = Thread.current[host_with_port]
+
+        facility = extract_placeholders(@facility, chunk.metadata)
+        severity = extract_placeholders(@severity, chunk.metadata)
+        program = extract_placeholders(@program, chunk.metadata)
+        hostname = extract_placeholders(@hostname, chunk.metadata)
+
+        packet_options = {facility: facility, severity: severity, program: program}
+        packet_options[:hostname] = hostname unless hostname.empty?
+
+        begin
+          chunk.open do |io|
+            io.each_line do |msg|
+              sender.transmit(msg.chomp!, packet_options)
+            end
           end
+        rescue
+          if Thread.current[host_with_port]
+            Thread.current[host_with_port].close
+            @senders.delete(Thread.current[host_with_port])
+            Thread.current[host_with_port] = nil
+          end
+          raise
         end
+      end
 
-        tag = rewrite_tag!(tag.dup)
-        @loggers[tag] ||= RemoteSyslogLogger::UdpSender.new(@host,
-          @port,
-          facility: record["facility"] || @facility,
-          severity: record["severity"] || @severity,
-          program: tag,
-          local_hostname: @hostname)
+      private
 
-        @loggers[tag].transmit format(tag, time, record)
+      def create_sender(host, port)
+        if @protocol == :tcp
+          options = {
+            tls: @tls,
+            whinyerrors: true,
+            packet_size: @packet_size,
+            timeout: @timeout,
+            timeout_exception: @timeout_exception,
+            keep_alive: @keep_alive,
+            keep_alive_idle: @keep_alive_idle,
+            keep_alive_cnt: @keep_alive_cnt,
+            keep_alive_intvl: @keep_alive_intvl,
+            program: @program,
+          }
+          options[:ca_file] = @ca_file if @ca_file
+          options[:verify_mode] = @verify_mode if @verify_mode
+          sender = RemoteSyslogSender::TcpSender.new(
+            host,
+            port,
+            options
+          )
+        else
+          sender = RemoteSyslogSender::UdpSender.new(
+            host,
+            port,
+            whinyerrors: true,
+            program: @program,
+          )
+        end
+        @senders << sender
+        sender
       end
     end
   end

data/test/plugin/out_remote_syslog.rb

@@ -1,62 +1,84 @@
 require "test_helper"
 require "fluent/plugin/out_remote_syslog"
 
-class RemoteSyslogOutputTest < MiniTest::Unit::TestCase
+class RemoteSyslogOutputTest < Test::Unit::TestCase
   def setup
     Fluent::Test.setup
   end
 
-  def create_driver(conf = CONFIG, tag = "test.remote_syslog")
-    Fluent::Test::OutputTestDriver.new(Fluent::RemoteSyslogOutput, tag) {}.configure(conf)
+  def create_driver(conf = CONFIG)
+    Fluent::Test::Driver::Output.new(Fluent::Plugin::RemoteSyslogOutput).configure(conf)
   end
 
   def test_configure
     d = create_driver %[
-      type remote_syslog
+      @type remote_syslog
       hostname foo.com
       host example.com
       port 5566
       severity debug
-      tag minitest
+      program minitest
     ]
 
-    d.run do
-      d.emit(message: "foo")
-    end
+    loggers = d.instance.instance_variable_get(:@senders)
+    assert_equal loggers, []
 
-    loggers = d.instance.instance_variable_get(:@loggers)
-    refute_empty loggers
+    assert_equal "example.com", d.instance.instance_variable_get(:@host)
+    assert_equal 5566, d.instance.instance_variable_get(:@port)
+    assert_equal "debug", d.instance.instance_variable_get(:@severity)
+  end
 
-    logger = loggers.values.first
+  def test_write
+    d = create_driver %[
+      @type remote_syslog
+      hostname foo.com
+      host example.com
+      port 5566
+      severity debug
+      program minitest
 
-    assert_equal "example.com", logger.instance_variable_get(:@remote_hostname)
-    assert_equal 5566, logger.instance_variable_get(:@remote_port)
+      <format>
+        @type single_value
+        message_key message
+      </format>
+    ]
 
-    p = logger.instance_variable_get(:@packet)
-    assert_equal "foo.com", p.hostname
-    assert_equal 1, p.facility
-    assert_equal "minitest", p.tag
-    assert_equal 7, p.severity
+    mock.proxy(RemoteSyslogSender::UdpSender).new("example.com", 5566, whinyerrors: true, program: "minitest") do |sender|
+      mock.proxy(sender).transmit("foo",  facility: "user", severity: "debug", program: "minitest", hostname: "foo.com")
+    end
+
+    d.run do
+      d.feed("tag", Fluent::EventTime.now, {"message" => "foo"})
+    end
   end
 
-  def test_rewrite_tag
+  def test_write_tcp
     d = create_driver %[
-      type remote_syslog
+      @type remote_syslog
       hostname foo.com
       host example.com
       port 5566
       severity debug
-      tag rewrited.${tag_parts[1]}
+      program minitest
+
+      protocol tcp
+
+      <format>
+        @type single_value
+        message_key message
+      </format>
     ]
 
-    d.run do
-      d.emit(message: "foo")
+    any_instance_of(RemoteSyslogSender::TcpSender) do |klass|
+      mock(klass).connect
     end
 
-    loggers = d.instance.instance_variable_get(:@loggers)
-    logger = loggers.values.first
+    mock.proxy(RemoteSyslogSender::TcpSender).new("example.com", 5566, whinyerrors: true, program: "minitest", tls: false, packet_size: 1024, timeout: nil, timeout_exception: false, keep_alive: false, keep_alive_cnt: nil, keep_alive_idle: nil, keep_alive_intvl: nil) do |sender|
+      mock(sender).transmit("foo",  facility: "user", severity: "debug", program: "minitest", hostname: "foo.com")
+    end
 
-    p = logger.instance_variable_get(:@packet)
-    assert_equal "rewrited.remote_syslog", p.tag
+    d.run do
+      d.feed("tag", Fluent::EventTime.now, {"message" => "foo"})
+    end
   end
 end

data/test/test_helper.rb

@@ -1,7 +1,10 @@
+require 'bundler/setup'
+require 'test/unit'
+
 $LOAD_PATH.unshift(File.join(File.dirname(__FILE__), "..", "lib"))
 $LOAD_PATH.unshift(File.dirname(__FILE__))
 
-require "minitest/autorun"
-require "minitest/unit"
-require "minitest/pride"
 require "fluent/test"
+require 'fluent/test/driver/output'
+
+require 'test/unit/rr'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-remote_syslog
 version: !ruby/object:Gem::Version
-  version: 0.3.3
+  version: 1.0.0
 platform: ruby
 authors:
 - Richard Lee
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-03-26 00:00:00.000000000 Z
+date: 2017-12-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -25,7 +25,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '10.0'
 - !ruby/object:Gem::Dependency
-  name: test-unit-minitest
+  name: test-unit
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -39,7 +39,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: minitest
+  name: test-unit-rr
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -67,61 +67,19 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: fluent-mixin-plaintextformatter
+  name: remote_syslog_sender
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: remote_syslog_logger
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 1.0.0
+        version: 1.1.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 1.0.0
-- !ruby/object:Gem::Dependency
-  name: fluent-mixin-config-placeholders
-  requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: fluent-mixin-rewrite-tag-name
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
+        version: 1.1.1
 description: ''
 email:
 - dlackty@gmail.com
@@ -161,7 +119,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.6.10
+rubygems_version: 2.6.13
 signing_key: 
 specification_version: 4
 summary: Fluentd output plugin for remote syslog