fluent-plugin-nats 0.1.0 → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 774b17902a6d2b133d83d093ae6ea1b2e1767112
-  data.tar.gz: 6d75ec076c1ab6e82916bd43da7101191a34d5b4
+  metadata.gz: 37e3edf9206fe66e157aa2c2d8a0ae58e2f5d934
+  data.tar.gz: 22a579e710b96489e1a29cb67f7982409a7f5b6e
 SHA512:
-  metadata.gz: 8dc09878eeee36105446dc2a07669f9cd57c55547d2c430fc1be0f944dcddaa45d94335411474a420639a07cb672078c7caaf8917f721d8136bbf16b24fd0fe7
-  data.tar.gz: 5f8a5cfbb0190f7600c8450d8e82afaeaaee9368da6e456d3bb11e0f61d289f430630c0d9c0bfe868d1382bee54b5212efb2ca8887ffa7a312aa039e448df746
+  metadata.gz: 20518c485938cf956976a732cfe8c3c64162834af602aac5dc0eb926c64942f9308933e8de083366aa22c25442122f8941abb14c48ec686fce587e0393e82215
+  data.tar.gz: 5a6b040380c3787da130b05d3a5c9bf93bda73e4d112289d56744714d5d4ee4f5e245a05a60d98c9f81bcd9d549fe951ad8336a17e000f88c9623cf40661d88a

data/README.md

@@ -6,6 +6,12 @@ fluent-plugin-nats
 
 [![Build Status](https://secure.travis-ci.org/achied/fluent-plugin-nats.png)](http://travis-ci.org/achied/fluent-plugin-nats)
 
+## Requirements
+
+| fluent-plugin-nats | Fluentd     | Ruby   |
+|--------------------|-------------|--------|
+| >= 1.0.0           | >= v0.14.20 | >= 2.1 |
+| < 1.0.0            | >= v0.12.0  | >= 1.9 |
 
 # Getting Started
 Setup the NATS input:
@@ -18,7 +24,7 @@ Setup the NATS input:
     port 4222
     user nats
     password nats
-    queue fluent.>,fluent2.>
+    queues fluent.>,fluent2.>
     ssl false
   </source>
 ~~~~~
@@ -31,6 +37,38 @@ Setup the match output:
   </match>
 ~~~~
 
+# Configuration
+
+## Plugin helpers
+
+* thread
+
+* See also: Fluent::Plugin::Input
+
+## Fluent::Plugin::NATSInput
+
+* **host** (string) (optional): NATS server hostname
+  * Default value: `localhost`
+* **user** (string) (optional): Username for authorized connection
+  * Default value: `nats`
+* **password** (string) (optional): Password for authorized connection
+  * Default value: `nats`
+* **port** (integer) (optional): NATS server port
+  * Default value: `4222`
+* **queues** (array) (optional): Subscribing queue names
+  * Default value: `["fluent.>"]`
+* **queue** (string) (optional):
+  * Default value: `fluent.>`
+  * Obsoleted: Use `queues` instead
+* **tag** (string) (optional): The tag prepend before queue name
+  * Default value: `nats`
+* **ssl** (bool) (optional): Enable secure SSL/TLS connection
+  * Default value: `false`
+* **max_reconnect_attempts** (integer) (optional): The max number of reconnect tries
+  * Default value: `150`
+* **reconnect_time_wait** (integer) (optional): The number of seconds to wait between reconnect tries
+  * Default value: `2`
+
 # Suitable Queues
 
 ## Components

data/Rakefile

@@ -6,8 +6,6 @@ Rake::TestTask.new(:test) do |test|
   test.libs << 'lib' << 'test'
   test.pattern = 'test/**/test_*.rb'
   test.test_files  = Dir['test/plugin/*.rb']
-  test.ruby_opts = ['-rubygems'] if defined? Gem
-  test.ruby_opts << '-I.'
   test.verbose = false
   test.warning = false
 end

data/fluent-plugin-nats.gemspec

@@ -1,14 +1,14 @@
-# -*- encoding: utf-8 -*-
 $:.push File.expand_path("../lib", __FILE__)
 
 Gem::Specification.new do |gem|
   gem.name        = "fluent-plugin-nats"
-  gem.version     = "0.1.0"
+  gem.version     = "1.0.0"
   gem.authors     = ["Eduardo Aceituno"]
   gem.email       = ["achied@gmail.com"]
   gem.homepage    = "https://github.com/achied/fluent-plugin-nats"
   gem.summary     = %q{nats plugin for fluentd, an event collector}
   gem.description = %q{nats plugin for fluentd, an event collector}
+  gem.license     = "Apache-2.0"
 
   gem.files         = `git ls-files`.split("\n")
   gem.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
@@ -16,9 +16,10 @@ Gem::Specification.new do |gem|
   gem.require_paths = ["lib"]
 
   gem.add_dependency "eventmachine", ">= 0.12.10"
-  gem.add_dependency "fluentd", ">= 0.10.7", "< 0.14"
+  gem.add_dependency "fluentd", ">= 0.14.20"
   gem.add_dependency "nats", ">= 0.4.22"
 
   gem.add_development_dependency "rake", ">= 0.9.2"
   gem.add_development_dependency "test-unit", "> 3.1"
+  gem.add_development_dependency "timecop"
 end

data/lib/fluent/plugin/in_nats.rb

@@ -1,83 +1,91 @@
-module Fluent
-  class NATSInput < Input
-    Fluent::Plugin.register_input("nats", self)
+require "fluent/plugin/input"
+require "nats/client"
 
-    config_param :host, :string, :default => "localhost"
-    config_param :user, :string, :default => "nats"
-    config_param :password, :string, :default => "nats", :secret => true
-    config_param :port, :integer, :default => 4222
-    config_param :queue, :string, :default => "fluent.>"
-    config_param :tag, :string, :default => "nats"
-    config_param :ssl, :bool, :default => false
-    config_param :max_reconnect_attempts, :integer, :default => 150
-    config_param :reconnect_time_wait, :integer, :default => 2
+module Fluent
+  module Plugin
+    class NATSInput < Fluent::Plugin::Input
+      Fluent::Plugin.register_input("nats", self)
 
-    def initialize
-      require "nats/client"
+      helpers :thread
 
-      NATS.on_error { |err| puts "Server Error: #{err}"; exit! }
-      super
-    end
+      desc "NATS server hostname"
+      config_param :host, :string, default: "localhost"
+      desc "Username for authorized connection"
+      config_param :user, :string, default: "nats"
+      desc "Password for authorized connection"
+      config_param :password, :string, default: "nats", secret: true
+      desc "NATS server port"
+      config_param :port, :integer, default: 4222
+      desc "Subscribing queue names"
+      config_param :queues, :array, default: ["fluent.>"]
+      config_param :queue, :string, default: "fluent.>", obsoleted: "Use `queues` instead"
+      desc "The tag prepend before queue name"
+      config_param :tag, :string, default: "nats"
+      desc "Enable secure SSL/TLS connection"
+      config_param :ssl, :bool, default: false
+      desc "The max number of reconnect tries"
+      config_param :max_reconnect_attempts, :integer, default: 150
+      desc "The number of seconds to wait between reconnect tries"
+      config_param :reconnect_time_wait, :integer, default: 2
 
-    def configure(conf)
-      super
+      def configure(conf)
+        super
 
-      unless @host && @queue
-        raise ConfigError, "'host' and 'queue' must be all specified."
+        @nats_config = {
+          uri: "nats://#{@host}:#{@port}",
+          ssl: @ssl,
+          user: @user,
+          pass: @password,
+          reconnect_time_wait: @reconnect_time_wait,
+          max_reconnect_attempts: @max_reconnect_attempts,
+        }
       end
 
-      @nats_config = {
-        :uri => "nats://#{@host}:#{@port}",
-        :ssl => @ssl,
-        :user => @user,
-        :pass => @password,
-        :reconnect_time_wait => @reconnect_time_wait,
-        :max_reconnect_attempts => @max_reconnect_attempts,
-      }
-    end
-
-    def start
-      super
-      run_reactor_thread
-      @thread = Thread.new(&method(:run))
-      log.info "listening nats on #{@uri}/#{@queue}"
-    end
+      def start
+        super
+        NATS.on_error do |error_message|
+          log.error "Server Error: #{error_message}"
+          exit!
+        end
+        run_reactor_thread
+        thread_create(:nats_input_main, &method(:run))
+        log.info "listening nats on #{@uri}/#{@queue}"
+      end
 
-    def shutdown
-      super
-      @nats_conn.close
-      @thread.join
-      EM.stop if EM.reactor_running?
-      @reactor_thread.join if @reactor_thread
-    end
+      def shutdown
+        @nats_conn.close
+        EM.stop if EM.reactor_running?
+        super
+      end
 
-    def run
-      queues = @queue.split(',')
-      EM.next_tick {
-        @nats_conn = NATS.connect(@nats_config) {
-          queues.each do |queue|
-            @nats_conn.subscribe(queue) do |msg, reply, sub|
-              tag = "#{@tag}.#{sub}"
-              begin
-                msg_json = JSON.parse(msg)
-              rescue JSON::ParserError => e
-                log.error "Failed parsing JSON #{e.inspect}.  Passing as a normal string"
-                msg_json = msg
+      def run
+        EM.next_tick do
+          @nats_conn = NATS.connect(@nats_config) do
+            @queues.each do |queue|
+              @nats_conn.subscribe(queue) do |msg, _reply, sub|
+                tag = "#{@tag}.#{sub}"
+                begin
+                  message = JSON.parse(msg)
+                rescue JSON::ParserError => e
+                  log.error "Failed parsing JSON #{e.inspect}.  Passing as a normal string"
+                  message = msg
+                end
+                time = Engine.now
+                router.emit(tag, time, message || {})
               end
-              time = Engine.now
-              router.emit(tag, time, msg_json || {})
             end
           end
-        }
-      }
-    end
+        end
+      end
+
+      private
 
-    private
-    def run_reactor_thread
-      unless EM.reactor_running?
-        @reactor_thread = Thread.new { EM.run }
+      def run_reactor_thread
+        return if EM.reactor_running?
+        thread_create(:nats_input_reactor_thread) do
+          EM.run
+        end
       end
     end
-
   end
 end

data/test/nats-publish-message.rb

@@ -17,7 +17,7 @@ def main(argv)
     queue = value
   end
   parser.on("--message=MESSAGE", "Message JSON") do |value|
-    message = value[1..-2]
+    message = value
   end
   begin
     parser.parse!

data/test/plugin/test_in_nats.rb

@@ -1,8 +1,6 @@
-require 'test/unit'
-require 'fluent/test'
-require 'lib/fluent/plugin/in_nats.rb'
-require 'nats/client'
-require 'test_helper'
+require "test_helper"
+require "fluent/test/driver/input"
+require "fluent/plugin/in_nats"
 
 class NATSInputTest < Test::Unit::TestCase
   include NATSTestHelper
@@ -16,13 +14,13 @@ class NATSInputTest < Test::Unit::TestCase
 
   def basic_queue_conf
     CONFIG + %[
-      queue fluent.>
+      queues fluent.>
     ]
   end
 
   def multiple_queue_conf
     CONFIG + %[
-      queue fluent.>, fluent2.>
+      queues fluent.>, fluent2.>
     ]
   end
 
@@ -33,35 +31,45 @@ class NATSInputTest < Test::Unit::TestCase
   end
 
   def create_driver(conf)
-    Fluent::Test::InputTestDriver.new(Fluent::NATSInput).configure(conf)
+    Fluent::Test::Driver::Input.new(Fluent::Plugin::NATSInput).configure(conf)
+  end
+
+  def setup
+    Fluent::Test.setup
+    @time = Time.parse("2011-01-02 13:14:15 UTC")
+    Timecop.freeze(@time)
+  end
+
+  def teardown
+    Timecop.return
   end
 
   sub_test_case "configure" do
     test "basic" do
       d = create_driver basic_queue_conf
       assert_equal 4222, d.instance.port
-      assert_equal 'localhost', d.instance.host
-      assert_equal 'nats', d.instance.user
-      assert_equal 'nats', d.instance.password
-      assert_equal 'fluent.>', d.instance.queue
+      assert_equal "localhost", d.instance.host
+      assert_equal "nats", d.instance.user
+      assert_equal "nats", d.instance.password
+      assert_equal ["fluent.>"], d.instance.queues
     end
 
     test "multiple queue" do
       d = create_driver multiple_queue_conf
       assert_equal 4222, d.instance.port
-      assert_equal 'localhost', d.instance.host
-      assert_equal 'nats', d.instance.user
-      assert_equal 'nats', d.instance.password
-      assert_equal 'fluent.>, fluent2.>', d.instance.queue
+      assert_equal "localhost", d.instance.host
+      assert_equal "nats", d.instance.user
+      assert_equal "nats", d.instance.password
+      assert_equal ["fluent.>", "fluent2.>"], d.instance.queues
     end
 
     test "basic with ssl" do
       d = create_driver ssl_conf
       assert_equal 4222, d.instance.port
-      assert_equal 'localhost', d.instance.host
-      assert_equal 'nats', d.instance.user
-      assert_equal 'nats', d.instance.password
-      assert_equal 'fluent.>', d.instance.queue
+      assert_equal "localhost", d.instance.host
+      assert_equal "nats", d.instance.user
+      assert_equal "nats", d.instance.password
+      assert_equal ["fluent.>"], d.instance.queues
       assert_equal true, d.instance.ssl
     end
   end
@@ -70,154 +78,149 @@ class NATSInputTest < Test::Unit::TestCase
     test "with credentials" do
       d = create_driver basic_queue_conf
 
-      time = Time.parse("2011-01-02 13:14:15 UTC").to_i
-      Fluent::Engine.now = time
+      time = @time.to_i
 
-      d.expect_emit "nats.fluent.test1", time, {"message"=>'nats', "fluent_timestamp"=>time}
-      d.expect_emit "nats.fluent.test2", time, {"message"=>'nats', "fluent_timestamp"=>time}
+      records = [
+        ["nats.fluent.test1", time, { "message" => "nats", "fluent_timestamp" => time }],
+        ["nats.fluent.test2", time, { "message" => "nats", "fluent_timestamp" => time }]
+      ]
 
       uri = generate_uri(d)
 
       run_server(uri) do
-        d.run do
-          d.expected_emits.each { |tag, _time, record|
+        d.run(expect_records: records.size, timeout: 5) do
+          records.each do |tag, _time, record|
             send(uri, tag[5..-1], record)
-            sleep 0.5
-          }
+          end
         end
+        assert_equal(records, d.events)
       end
     end
 
     test "without credentials" do
       d = create_driver basic_queue_conf
+      time = @time.to_i
 
-      time = Time.parse("2011-01-02 13:14:15 UTC").to_i
-      Fluent::Engine.now = time
-
-      d.expect_emit "nats.fluent.test1", time, {"message"=>'nats', "fluent_timestamp"=>time}
-      d.expect_emit "nats.fluent.test2", time, {"message"=>'nats', "fluent_timestamp"=>time}
+      records = [
+        ["nats.fluent.test1", time, { "message" => "nats", "fluent_timestamp" => time }],
+        ["nats.fluent.test2", time, { "message" => "nats", "fluent_timestamp" => time }]
+      ]
 
       uri = generate_uri(d)
 
       run_server(uri) do
-        d.run do
-          d.expected_emits.each { |tag, time, record|
+        d.run(expect_records: records.size, timeout: 5) do
+          records.each do |tag, _time, record|
             send(uri, tag[5..-1], record)
-            sleep 0.5
-          }
+          end
         end
+        assert_equal(records, d.events)
       end
     end
 
     test "multiple queues" do
       d = create_driver multiple_queue_conf
+      time = @time.to_i
 
-      time = Time.parse("2011-01-02 13:14:15 UTC").to_i
-      Fluent::Engine.now = time
-
-      d.expect_emit "nats.fluent.test1", time, {"message"=>'nats', "fluent_timestamp"=>time}
-      d.expect_emit "nats.fluent.test2", time, {"message"=>'nats', "fluent_timestamp"=>time}
-      d.expect_emit "nats.fluent2.test1", time, {"message"=>'nats', "fluent_timestamp"=>time}
-      d.expect_emit "nats.fluent2.test2", time, {"message"=>'nats', "fluent_timestamp"=>time}
+      records = [
+        ["nats.fluent.test1", time, { "message" => "nats", "fluent_timestamp" => time }],
+        ["nats.fluent.test2", time, { "message" => "nats", "fluent_timestamp" => time }],
+        ["nats.fluent2.test1", time, { "message" => "nats", "fluent_timestamp" => time }],
+        ["nats.fluent2.test2", time, { "message" => "nats", "fluent_timestamp" => time }]
+      ]
 
       uri = generate_uri(d)
 
       run_server(uri) do
-        d.run do
-          d.expected_emits.each { |tag, time, record|
+        d.run(expect_records: records.size, timeout: 5) do
+          records.each do |tag, _time, record|
             send(uri, tag[5..-1], record)
-            sleep 0.5
-          }
+          end
         end
+        assert_equal(records, d.events)
       end
     end
 
     test "without fluent timestamp" do
       d = create_driver basic_queue_conf
-
       time = Time.now.to_i
-      Fluent::Engine.now = time
 
-      d.expect_emit "nats.fluent.test1", time, {"message"=>'nats'}
+      records = [
+        ["nats.fluent.test1", time, { "message" => "nats" }]
+      ]
 
       uri = generate_uri(d)
       run_server(uri) do
-        d.run do
-          d.expected_emits.each do |tag, time, record|
+        d.run(expect_records: records.size, timeout: 5) do
+          records.each do |tag, _time, record|
             send(uri, tag[5..-1], record)
-            sleep 0.5
           end
         end
+        assert_equal(records, d.events)
       end
     end
 
     test "arrays" do
       d = create_driver basic_queue_conf
-
       time = Time.now.to_i
-      Fluent::Engine.now = time
-      
-      d.expect_emit "nats.fluent.empty_array", time, []
-      d.expect_emit "nats.fluent.string_array", time, %w(one two three)
-
-      user = d.instance.user
-      password = d.instance.password
-      uri = "nats://#{user}:#{password}@#{d.instance.host}:#{d.instance.port}"
+
+      records = [
+        ["nats.fluent.empty_array", time, []],
+        ["nats.fluent.string_array", time, %w(one two three)]
+      ]
+
+      uri = generate_uri(d)
       run_server(uri) do
-        d.run do
-          d.expected_emits.each do |tag, time, record|
+        d.run(expect_records: records.size, timeout: 5) do
+          records.each do |tag, _time, record|
             send(uri, tag[5..-1], record)
-            sleep 0.5
           end
         end
+        assert_equal(records, d.events)
       end
     end
 
     test "empty publish string" do
       d = create_driver basic_queue_conf
-
       time = Time.now.to_i
-      Fluent::Engine.now = time
 
-      d.expect_emit "nats.fluent.nil", time, {}
+      records = [
+        ["nats.fluent.nil", time, {}]
+      ]
 
       uri = generate_uri(d)
       run_server(uri) do
-        d.run do
-          d.expected_emits.each do |tag, time, record|
+        d.run(expect_records: records.size, timeout: 5) do
+          records.each do |tag, _time, _record|
             send(uri, tag[5..-1], nil)
-            sleep 0.5
           end
         end
+        assert_equal(records, d.events)
       end
     end
 
     test "regular publish string" do
       d = create_driver basic_queue_conf
-
       time = Time.now.to_i
-      Fluent::Engine.now = time
 
-      d.expect_emit "nats.fluent.string", time, "Lorem ipsum dolor sit amet"
+      records = [
+        ["nats.fluent.string", time, "Lorem ipsum dolor sit amet"]
+      ]
 
       uri = generate_uri(d)
       run_server(uri) do
-        d.run do
-          d.expected_emits.each do |tag, time, record|
+        d.run(expect_records: records.size, timeout: 5) do
+          records.each do |tag, _time, _record|
             send(uri, tag[5..-1], "Lorem ipsum dolor sit amet")
-            sleep 0.5
           end
         end
+        assert_equal(records, d.events)
       end
     end
   end
 
-  def setup
-    Fluent::Test.setup
-  end
-
   def send(uri, tag, msg)
-    system("test/nats-publish-message.rb", *%W[--uri=#{uri} --queue=#{tag} --message='#{msg.to_json}'])
+    system("test/nats-publish-message.rb", "--uri", uri, "--queue", tag, "--message", msg.to_json)
   end
 
   def generate_uri(driver)

data/test/test_helper.rb

@@ -1,31 +1,11 @@
-require 'rubygems'
-require 'bundler'
-begin
-  Bundler.setup(:default, :development)
-rescue Bundler::BundlerError => e
-  $stderr.puts e.message
-  $stderr.puts "Run `bundle install` to install missing gems"
-  exit e.status_code
-end
-require 'test/unit'
-
-$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
-$LOAD_PATH.unshift(File.dirname(__FILE__))
-require 'fluent/test'
-unless ENV.has_key?('VERBOSE')
-  nulllogger = Object.new
-  nulllogger.instance_eval {|obj|
-    def method_missing(method, *args)
-      # pass
-    end
-  }
-  $log = nulllogger
-end
-
-class Test::Unit::TestCase
-end
+require "bundler/setup"
+require "test/unit"
 
-require 'nats/client'
+$LOAD_PATH.unshift(File.join(__dir__, "..", "lib"))
+$LOAD_PATH.unshift(__dir__)
+require "fluent/test"
+require "nats/client"
+require "timecop"
 
 module NATSTestHelper
   def run_server(uri)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-nats
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 1.0.0
 platform: ruby
 authors:
 - Eduardo Aceituno
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-10-25 00:00:00.000000000 Z
+date: 2017-10-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: eventmachine
@@ -30,20 +30,14 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.10.7
-    - - "<"
-      - !ruby/object:Gem::Version
-        version: '0.14'
+        version: 0.14.20
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.10.7
-    - - "<"
-      - !ruby/object:Gem::Version
-        version: '0.14'
+        version: 0.14.20
 - !ruby/object:Gem::Dependency
   name: nats
   requirement: !ruby/object:Gem::Requirement
@@ -86,6 +80,20 @@ dependencies:
     - - ">"
       - !ruby/object:Gem::Version
         version: '3.1'
+- !ruby/object:Gem::Dependency
+  name: timecop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: nats plugin for fluentd, an event collector
 email:
 - achied@gmail.com
@@ -105,7 +113,8 @@ files:
 - test/plugin/test_in_nats.rb
 - test/test_helper.rb
 homepage: https://github.com/achied/fluent-plugin-nats
-licenses: []
+licenses:
+- Apache-2.0
 metadata: {}
 post_install_message: 
 rdoc_options: []