fluent-plugin-mqtt 0.0.7 → 0.0.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 9ebd832e07164e15a36a3617c61c78ac14cc1c10
-  data.tar.gz: 9ac4921e763bc8484beb993182513ff0e16937a5
+  metadata.gz: 19e8964bfff6ad942926e4e1607b91a099f8a1f9
+  data.tar.gz: dcf3560aca07e73b59eb1ff319620977dc63ac6b
 SHA512:
-  metadata.gz: bc796bd26c90e7ff7f1dbb93cd59b95083ff3ca8b26c6a97e1404efcee4ad936d976be1c41358708ace9ac2e889b9983725c8e54ad04f7f397f8c6f0de516b3c
-  data.tar.gz: 036f05679c777eab6d332c205b4fc804f17774c9276d63d56d3cedf79843eff4d7d27e7b7143e682a9c005eb568b6774244f0abbbbdfd7e4dfaf900e3b219161
+  metadata.gz: a86199765c2bdcd8309c2e79c0ea4a073a443acef5a55574ee8fc951d8974c45116f0fdaf5ae94d6c72c6477bb565746be676de52f6ca8ec44f9836cd382b378
+  data.tar.gz: b3d1b82362c57f3860338946b44f5af6419a133173caee3fc8b0fbf6f2978aa8f5b1a8189bc8f912797911d9bc5b1bdf9588a400a9dd50169fae9496b0ac3f4f

data/fluent-plugin-mqtt.gemspec

@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name          = "fluent-plugin-mqtt"
-  spec.version       = "0.0.7"
+  spec.version       = "0.0.8"
   spec.authors       = ["Yuuna Kurita"]
   spec.email         = ["yuuna.m@gmail.com"]
   spec.summary       = %q{fluentd input plugin for mqtt server}

data/lib/fluent/plugin/in_mqtt.rb

@@ -1,23 +1,23 @@
-module Fluent
+require 'mqtt'
+require 'fluent/plugin/input'
+require 'fluent/plugin/parser'
+
+module Fluent::Plugin
   class MqttInput < Input
-    Plugin.register_input('mqtt', self)
+    Fluent::Plugin.register_input('mqtt', self)
+
+    helpers :thread, :inject, :compat_parameters, :parser
+
+    DEFAULT_PARSER_TYPE = 'none'
 
-    include Fluent::SetTagKeyMixin
     config_set_default :include_tag_key, false
-    
-    include Fluent::SetTimeKeyMixin
     config_set_default :include_time_key, true
 
-    # Define `router` method of v0.12 to support v0.10 or earlier
-    unless method_defined?(:router)
-      define_method("router") { Fluent::Engine }
-    end
-    
-    
     config_param :port, :integer, :default => 1883
     config_param :bind, :string, :default => '127.0.0.1'
     config_param :topic, :string, :default => '#'
-    config_param :format, :string, :default => 'none'
+    config_param :format, :string, :default => DEFAULT_PARSER_TYPE
+    config_param :client_id, :string, :default => nil
     config_param :username, :string, :default => nil
     config_param :password, :string, :default => nil
     config_param :ssl, :bool, :default => nil
@@ -25,31 +25,33 @@ module Fluent
     config_param :key, :string, :default => nil
     config_param :cert, :string, :default => nil
 
-    require 'mqtt'
+    config_section :parse do
+      config_set_default :@type, DEFAULT_PARSER_TYPE
+    end
 
     def configure(conf)
+      compat_parameters_convert(conf, :inject, :parser)
       super
-      @bind ||= conf['bind']
-      @topic ||= conf['topic']
-      @port ||= conf['port']
-
       configure_parser(conf)
     end
 
     def configure_parser(conf)
-      @parser = Plugin.new_parser(@format)
-      @parser.configure(conf)
+      @parser = parser_create(usage: 'in_mqtt_parser', type: @format, conf: conf)
     end
 
     # Return [time (if not available return now), message]
     def parse(message)
-      return @parser.parse(message)[1], @parser.parse(message)[0] || Fluent::Engine.now
+      @parser.parse(message) {|time, record|
+        return (time || Fluent::Engine.now), record
+      }
     end
 
     def start
-      $log.debug "start mqtt #{@bind}"
+      super
+      log.debug "start mqtt #{@bind}"
       opts = {host: @bind,
               port: @port}
+      opts[:client_id] = @client_id if @client_id
       opts[:username] =  @username if @username
       opts[:password] = @password if @password
       opts[:ssl] = @ssl if @ssl
@@ -59,25 +61,26 @@ module Fluent
       @connect = MQTT::Client.connect(opts)
       @connect.subscribe(@topic)
 
-      @thread = Thread.new do
+      thread_create(:in_mqtt_worker) do
         @connect.get do |topic,message|
           topic.gsub!("/","\.")
-          $log.debug "#{topic}: #{message}"
+          log.debug "#{topic}: #{message}"
           begin
-            parsed_message = self.parse(message)
+            time, record = self.parse(message)
+            record = inject_values_to_record(topic, time, record)
           rescue Exception => e
-            $log.error e
+            log.error e
           end
-          emit topic, parsed_message[0], parsed_message[1]
+          emit topic, record, time
         end
       end
     end
 
-    
+
     def emit topic, message, time = Fluent::Engine.now
       if message.class == Array
         message.each do |data|
-          $log.debug "#{topic}: #{data}"
+          log.debug "#{topic}: #{data}"
           router.emit(topic , time , data)
         end
       else
@@ -86,9 +89,8 @@ module Fluent
     end
 
     def shutdown
-      @thread.kill
       @connect.disconnect
+      super
     end
   end
 end
-

data/lib/fluent/plugin/out_mqtt.rb

@@ -13,12 +13,14 @@ module Fluent
     config_param :bind, :string, :default => '127.0.0.1'
     config_param :topic, :string, :default => 'td-agent'
     config_param :format, :string, :default => 'none'
+    config_param :client_id, :string, :default => nil
     config_param :username, :string, :default => nil
     config_param :password, :string, :default => nil
     config_param :ssl, :bool, :default => nil
     config_param :ca, :string, :default => nil
     config_param :key, :string, :default => nil
     config_param :cert, :string, :default => nil
+    config_param :retain, :string, :default => true
 
     require 'mqtt'
 
@@ -38,8 +40,10 @@ module Fluent
     def configure(conf)
       super
       @bind ||= conf['bind']
-      @topic ||= conf['topic'] 
-      @port ||= conf['port']      
+      @topic ||= conf['topic']
+      @port ||= conf['port']
+      @formatter = Plugin.new_formatter(@format)
+      @formatter.configure(conf)
     end
 
     def start
@@ -49,6 +53,7 @@ module Fluent
       $log.debug "start mqtt #{@bind}"
       opts = {host: @bind,
               port: @port}
+      opts[:client_id] = @client_id if @client_id
       opts[:username] =  @username if @username
       opts[:password] = @password if @password
       opts[:ssl] = @ssl if @ssl
@@ -69,9 +74,9 @@ module Fluent
     end
 
     def write(chunk)
-      $log.debug "write"
       chunk.msgpack_each { |tag, time, record|
-        @connect.publish(tag, record , retain=true)
+        log.debug "write #{@topic} #{@formatter.format(tag,time,record)}"
+        @connect.publish(@topic, @formatter.format(tag,time,record), retain=@retain)
       }
     end
 

data/test/helper.rb

@@ -12,6 +12,7 @@ require 'test/unit'
 $LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
 $LOAD_PATH.unshift(File.dirname(__FILE__))
 require 'fluent/test'
+require 'fluent/test/helpers'
 unless ENV.has_key?('VERBOSE')
   nulllogger = Object.new
   nulllogger.instance_eval {|obj|

data/test/plugin/test_in_mqtt.rb

@@ -1,6 +1,7 @@
 require 'helper'
+require 'fluent/test/driver/input'
 
-class Fluent::MqttInput
+class Fluent::Plugin::MqttInput
   #def emit topic, message , time = Fluent::Engine.now
     #if message.class == Array
       #message.each do |data|
@@ -14,6 +15,8 @@ class Fluent::MqttInput
 
 end
 
+include Fluent::Test::Helpers
+
 class MqttInputTest < Test::Unit::TestCase
   def setup
     Fluent::Test.setup
@@ -24,7 +27,7 @@ class MqttInputTest < Test::Unit::TestCase
              format json ]
 
   def create_driver(conf = CONFIG)
-    Fluent::Test::InputTestDriver.new(Fluent::MqttInput).configure(conf)
+    Fluent::Test::Driver::Input.new(Fluent::Plugin::MqttInput).configure(conf)
   end
 
   def test_configure_1
@@ -45,7 +48,7 @@ class MqttInputTest < Test::Unit::TestCase
          time_key time]
     )
     assert_equal 'csv', d2.instance.format
-    assert_equal 'time', d2.instance.time_key
+    assert_equal 'time', d2.instance.inject_config.time_key
   end
 
   def test_configure_2
@@ -72,21 +75,20 @@ class MqttInputTest < Test::Unit::TestCase
          port 1883
          format json ]
     )
-    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+    time = event_time("2011-01-02 13:14:15 UTC")
     data = [
       {tag: "tag1", message: {"t" => time, "v" => {"a"=>1}}},
       {tag: "tag2", message: {"t" => time, "v" => {"a"=>1}}},
       {tag: "tag3", message: {"t" => time, "v" => {"a"=>32}}},
     ]
 
-    d.run do
+    d.run(expect_emits: 3, timeout: 5) do
       data.each do |record|
         send_data record[:tag], record[:message], d.instance.format
-        sleep 0.1
       end
     end
 
-    emits = d.emits
+    emits = d.events
     assert_equal('tag1', emits[0][0])
     assert_equal({"t" => time, "v" => {"a"=>1}}, emits[0][2])
 
@@ -104,7 +106,7 @@ class MqttInputTest < Test::Unit::TestCase
          format json
          time_key t ]
     )
-    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+    time = event_time("2011-01-02 13:14:15 UTC")
     data = [
       {tag: "tag1", message: {"t" => time, "v" => {"a"=>1}}},
       {tag: "tag2", message: {"t" => time, "v" => {"a"=>1}}},
@@ -112,16 +114,16 @@ class MqttInputTest < Test::Unit::TestCase
       {tag: "tag3", message: {"t" => time, "v" => {"a"=>32}}},
     ]
 
-    d.run do
+    d.run(expect_emits: 4, timeout: 5) do
       data.each do |record|
         send_data record[:tag], record[:message], d.instance.format
       end
     end
 
-    emits = d.emits
+    emits = d.events
     assert_equal('tag1', emits[0][0])
     assert_equal(time, emits[0][1])
-    assert_equal({"v" => {"a"=>1}}, emits[0][2])
+    assert_equal({"v" => {"a"=>1}, "t" => time}, emits[0][2])
   end
 
   def test_format_none
@@ -137,13 +139,13 @@ class MqttInputTest < Test::Unit::TestCase
       {tag: "tag3", message: ''},
     ]
 
-    d.run do
+    d.run(expect_emits: 3, timeout: 5) do
       data.each do |record|
         send_data record[:tag], record[:message], d.instance.format
       end
     end
 
-    emits = d.emits
+    emits = d.events
     time = Fluent::Engine.now
     assert_equal('tag1', emits[0][0])
     assert_equal({'message' => 'hello world'}, emits[0][2])
@@ -161,20 +163,20 @@ class MqttInputTest < Test::Unit::TestCase
          keys time,message]
     )
 
-    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+    time = event_time("2011-01-02 13:14:15 UTC")
     data = [
       {tag: "tag1", message: "#{time},hello world" },
       {tag: "tag2", message: "#{time},hello to you to" },
       {tag: "tag3", message: "#{time}," },
     ]
 
-    d.run do
+    d.run(expect_emits: 3, timeout: 5) do
       data.each do |record|
         send_data record[:tag], record[:message], d.instance.format
       end
     end
 
-    emits = d.emits
+    emits = d.events
     #puts 'emits length', emits.length.to_s
     assert_equal('tag1', emits[0][0])
     assert_equal({'time' => time.to_s, 'message' => 'hello world'}, emits[0][2])
@@ -192,10 +194,10 @@ class MqttInputTest < Test::Unit::TestCase
          format csv
          keys time2,message
          time_key time2
-         time_format %S]
+         time_format %s]
     )
 
-    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+    time = event_time("2011-01-02 13:14:15 UTC").to_i # to obtain unixtime stamp
     data = [
       {tag: "tag1", message: "#{time},abc" },
       {tag: "tag2", message: "#{time},def" },
@@ -203,24 +205,24 @@ class MqttInputTest < Test::Unit::TestCase
       {tag: "tag3", message: "#{time}," },
     ]
 
-    d.run do
+    d.run(expect_emits: 4, timeout: 5) do
       data.each do |record|
         send_data record[:tag], record[:message], d.instance.format
       end
     end
 
-    emits = d.emits
+    emits = d.events
     assert_equal('tag1', emits[0][0])
-    assert_equal({'message' => 'abc'}, emits[0][2])
+    assert_equal({'message' => 'abc', "time2" => time}, emits[0][2])
 
     assert_equal('tag2', emits[1][0])
-    assert_equal({'message' => 'def'}, emits[1][2])
+    assert_equal({'message' => 'def', "time2" => time}, emits[1][2])
 
     assert_equal('tag3', emits[2][0])
-    assert_equal({'message' => 'ghi'}, emits[2][2])
+    assert_equal({'message' => 'ghi', "time2" => time}, emits[2][2])
 
     assert_equal('tag3', emits[3][0])
-    assert_equal({'message' => nil}, emits[3][2])
+    assert_equal({'message' => nil, "time2"=> time}, emits[3][2])
   end
 
   def send_data tag, record, format
@@ -234,6 +236,5 @@ class MqttInputTest < Test::Unit::TestCase
       else
         sub_client.publish(tag, record)
     end
-    sleep 0.2
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-mqtt
 version: !ruby/object:Gem::Version
-  version: 0.0.7
+  version: 0.0.8
 platform: ruby
 authors:
 - Yuuna Kurita
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-04-29 00:00:00.000000000 Z
+date: 2017-04-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: mqtt
@@ -131,11 +131,10 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.5.1
+rubygems_version: 2.5.1
 signing_key: 
 specification_version: 4
 summary: fluentd input plugin for mqtt server
 test_files:
 - test/helper.rb
 - test/plugin/test_in_mqtt.rb
-has_rdoc: