fluent-plugin-mqtt-io 0.0.6 → 0.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 9a44345e5a1d555cccb9052cb5026a1d1c74ba99
-  data.tar.gz: 38f42494ed8fedaf6f183eb0ec424a19b1e5490f
+  metadata.gz: d296aa7e9f2bf32ab247b8f99ad7bdaf972cf763
+  data.tar.gz: 669477119ff620c36cb45bf2ed63ae2ed238b1d2
 SHA512:
-  metadata.gz: 69ae3c6e96f603776b577e139f4375acee20273bb885526def569e50e57977339ba0af0f145fd3dbdd2a9bbfea192dc673f80884468e71cc394041050fb32424
-  data.tar.gz: 2029b0f4d5cd40210d598ab5131367367c5478fe6a77537fb27357728ec0735b16f89353b7b077766211a615ef98c3608ce4b0ba7017795504d5a68cbfc2818d
+  metadata.gz: a105c66f6dd6e4e11a6e7f118c34295590da1c10ec0bd31e6393aafc21cceef2274dc3f2468c1d6a5c6105a4a44e6c53eb79e48d27fde883e866bbebd6e89d7e
+  data.tar.gz: 04fd6db671975fb44c8ba536fc10e31c43503d1d20eb0d76bda5a41bfc6a5657dedea25f9489d44495f22f76b5ec4b2158ca9b563c583c9f5b05391691f56fb7

data/README.md

@@ -26,6 +26,8 @@ Or install it yourself as:
 
 fluent-plugin-mqtt-io provides Input and Output Plugins for MQTT.
 
+### Input Plugin (Fluet::MqttInput)
+
 Input Plugin can be used via source directive in the configuration.
 
 ```
@@ -44,9 +46,11 @@ The default MQTT topic is "#". Configurable options are the following:
 - bind: IP address of MQTT broker
 - port: Port number of MQTT broker
 - format (mandatory): Input parser can be chosen, e.g. json, xml
-  - in order to use xml format, you need to install [fluent-plugin-xml-parser](https://github.com/toyokazu/fluent-plugin-xml-parser).
-  - default time_key field for json format is 'time'
+  - In order to use xml format, you need to install [fluent-plugin-xml-parser](https://github.com/toyokazu/fluent-plugin-xml-parser).
+  - Default time_key field for json format is 'time'
 - topic: Topic name to be subscribed
+- bulk_trans: Enable bulk transfer to support buffered output (mqtt_buf, Fluent::MqttBufferedOutput, defaut: true)
+- bulk_trans_sep: A message separator for bulk transfer. The default separator is "\t".
 - username: User name for authentication
 - password: Password for authentication
 - ssl: set true if you want to use SSL/TLS. If set to true, the following parameter must be provided
@@ -56,6 +60,8 @@ The default MQTT topic is "#". Configurable options are the following:
 
 Input Plugin supports @label directive.
 
+### Output Plugin (Fluent::MqttOutput, Fluent::MqttBufferedOutput)
+
 Output Plugin can be used via match directive.
 
 ```
@@ -68,7 +74,7 @@ Output Plugin can be used via match directive.
 
 ```
 
-The options are basically the same as Input Plugin. The difference is the following.
+The options are basically the same as Input Plugin except for "format" and "bulk_trans". Additional options for Output Plugin are the following.
 
 - time_key: An attribute name used for timestamp field genarated from fluentd time field. Default is nil (omitted).
   If this option is omitted, timestamp field is not appended to the output record.
@@ -93,7 +99,7 @@ The topic name or tag name, e.g. "topic", received from an event can not be publ
 
 ```
 
-You can also use mqtt_buf type which is implemented as BufferedOutput.
+You can also use mqtt_buf type which is implemented as Fluent::MqttBufferedOutput.
 
 ```
 
@@ -110,10 +116,13 @@ You can also use mqtt_buf type which is implemented as BufferedOutput.
 
 ```
 
+When a plugin implemented as Fluent::BufferedOutput, fluentd stores the received messages into buffers (Fluent::MemoryBuffer is used as default) separated by tag names. Writer Threads emit those stored messages periodically in the specified interval. In MqttBufferedOutput, the stored messages are concatenated with 'bulk_trans_sep' (default: "\t"). This function reduces the number of messages sent via MQTT if data producing interval of sensors are smaller than publish interval (flush_interval). The concatinated messages can be properly handled by Fluent::MqttInput plugin by specifying 'bulk_trans' option.
 
 ## Use case examples
 
-### Libelium sensor data collection
+### Sensor data collection (not for Libelium now)
+
+*additional description (2015-12-25)*: Thanks to the updates by Libelium, Meshlium farmware now supports flexible output format configuration including JSON. As a result, data conversion by XmlParser is not required for this use case. The following description is kept just for sharing know-how of data conversion in fluentd.
 
 There are many kinds of commercial sensor products on the market, e.g. [Libelium](http://www.libelium.com/). Major sensor products support MQTT to upload sensor data. The following example shows how to store uploaded Libelium sensor data into ElasticSearch.
 

data/fluent-plugin-mqtt-io.gemspec

@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name          = "fluent-plugin-mqtt-io"
-  spec.version       = "0.0.6"
+  spec.version       = "0.1.0"
   spec.authors       = ["Toyokazu Akiyama"]
   spec.email         = ["toyokazu@gmail.com"]
 

data/lib/fluent/plugin/in_mqtt.rb

@@ -6,6 +6,8 @@ module Fluent
     config_param :bind, :string, :default => '127.0.0.1'
     config_param :topic, :string, :default => '#'
     config_param :format, :string, :default => 'json'
+    config_param :bulk_trans, :bool, :default => true
+    config_param :bulk_trans_sep, :string, :default => "\t"
     config_param :username, :string, :default => nil
     config_param :password, :string, :default => nil
     config_param :ssl, :bool, :default => nil
@@ -24,6 +26,8 @@ module Fluent
       super
       @bind ||= conf['bind']
       @topic ||= conf['topic']
+      @bulk_trans ||= conf['bulk_trans']
+      @bulk_trans_sep ||= conf['bulk_trans_sep']
       @port ||= conf['port']
       @username ||= conf['username']
       @password ||= conf['password']
@@ -47,27 +51,56 @@ module Fluent
       opts[:ca_file] = @ca_file if @ca_file
       opts[:cert_file] = @cert_file if @cert_file
       opts[:key_file] = @key_file if @key_file
-      @connect = MQTT::Client.connect(opts)
-      @connect.subscribe(@topic)
 
-      @thread = Thread.new do
-        @connect.get do |topic,message|
-          emit topic, message
+      # In order to handle Exception raised from reading Thread
+      # in MQTT::Client caused by network disconnection (during read_byte),
+      # @connect_thread generates connection.
+      @client = MQTT::Client.new(opts)
+      @connect_thread = Thread.new do
+        while (true)
+          begin
+            @client.disconnect if @client.connected?
+            @client.connect
+            @client.subscribe(@topic)
+            @get_thread.kill if !@get_thread.nil? && @get_thread.alive?
+            @get_thread = Thread.new do
+              @client.get do |topic, message|
+                emit(topic, message)
+              end
+            end
+            sleep
+          rescue MQTT::ProtocolException => pe
+            $log.debug "Handling #{pe.class}: #{pe.message}"
+            next
+          rescue Timeout::Error => te
+            $log.debug "Handling #{te.class}: #{te.message}"
+            next
+          end
+        end
+      end
+    end
+
+    def parse(message)
+      @parser.parse(message) do |time, record|
+        if time.nil?
+          $log.debug "Since time_key field is nil, Fluent::Engine.now is used."
+          time = Fluent::Engine.now
         end
+        $log.debug "#{topic}, #{time}, #{record}"
+        return [time, record]
       end
     end
 
-    def emit topic, message
+    def emit(topic, message)
       begin
         topic.gsub!("/","\.")
-        @parser.parse(message) {|time, record|
-          if time.nil?
-            $log.debug "Since time_key field is nil, Fluent::Engine.now is used."
-            time = Fluent::Engine.now
+        if @bulk_trans
+          message.split(@bulk_trans_sep).each do |m|
+            router.emit(topic, *parse(m))
           end
-          $log.debug "#{topic}, #{time}, #{record}"
-          router.emit(topic, time, record)
-        }
+        else
+          router.emit(topic, *parse(message))
+        end
       rescue Exception => e
         $log.error :error => e.to_s
         $log.debug_backtrace(e.backtrace)
@@ -75,8 +108,9 @@ module Fluent
     end
 
     def shutdown
-      @thread.kill
-      @connect.disconnect
+      @get_thread.kill
+      @connect_thread.kill
+      @client.disconnect
     end
   end
 end

data/lib/fluent/plugin/mqtt_output_mixin.rb

@@ -12,10 +12,11 @@ module Fluent
       base.config_param :ca_file, :string, :default => nil
       base.config_param :key_file, :string, :default => nil
       base.config_param :cert_file, :string, :default => nil
-      base.config_param :time_key, :string, :default => nil
+      base.config_param :time_key, :string, :default => 'time'
       base.config_param :time_format, :string, :default => nil
       base.config_param :topic_rewrite_pattern, :string, :default => nil
       base.config_param :topic_rewrite_replacement, :string, :default => nil
+      base.config_param :bulk_trans_sep, :string, :default => "\t"
     end
 
     require 'mqtt'
@@ -35,6 +36,7 @@ module Fluent
       @time_format ||= conf['time_format']
       @topic_rewrite_pattern ||= conf['topic_rewrite_pattern']
       @topic_rewrite_replacement ||= conf['topic_rewrite_replacement']
+      @bulk_trans_sep ||= conf['bulk_trans_sep']
     end
 
     # This method is called when starting.
@@ -53,7 +55,25 @@ module Fluent
       opts[:ca_file] = @ca_file if @ca_file
       opts[:cert_file] = @cert_file if @cert_file
       opts[:key_file] = @key_file if @key_file
-      @connect = MQTT::Client.connect(opts)
+      # In order to handle Exception raised from reading Thread
+      # in MQTT::Client caused by network disconnection (during read_byte),
+      # @connect_thread generates connection.
+      @client = MQTT::Client.new(opts)
+      @connect_thread = Thread.new do
+        while (true)
+          begin
+            @client.disconnect if @client.connected?
+            @client.connect
+            sleep
+          rescue MQTT::ProtocolException => pe
+            $log.debug "Handling #{pe.class}: #{pe.message}"
+            next
+          rescue Timeout::Error => te
+            $log.debug "Handling #{te.class}: #{te.message}"
+            next
+          end
+        end
+      end
     end
 
     # This method is called when shutting down.
@@ -61,13 +81,19 @@ module Fluent
     def shutdown
       super
 
-      @connect.disconnect
+      @client.disconnect
     end
 
     def format_time(time)
-      if @time_format.nil?
+      case @time_format
+      when nil then
+        # default format is integer value
+        time
+      when "iso8601" then
+        # iso8601 format
         Time.at(time).iso8601
       else
+        # specified strftime format
         Time.at(time).strftime(@time_format)
       end
     end

data/lib/fluent/plugin/out_mqtt.rb

@@ -10,7 +10,7 @@ module Fluent
     def emit(tag, es, chain)
       es.each {|time,record|
         $log.debug "#{tag}, #{format_time(time)}, #{record}"
-        @connect.publish(rewrite_tag(tag), record.merge(timestamp_hash(time)).to_json)
+        @client.publish(rewrite_tag(tag), record.merge(timestamp_hash(time)).to_json)
       }
       $log.flush
 

data/lib/fluent/plugin/out_mqtt_buf.rb

@@ -1,5 +1,5 @@
 module Fluent
-  class MqttOutput < BufferedOutput
+  class MqttBufferedOutput < BufferedOutput
     require 'fluent/plugin/mqtt_output_mixin'
     include Fluent::MqttOutputMixin
 
@@ -10,7 +10,8 @@ module Fluent
     # This method is called when an event reaches to Fluentd.
     # Convert the event to a raw string.
     def format(tag, time, record)
-      [tag, time, record].to_json + "\n"
+      [tag, time, record].to_msgpack
+      #[tag, time, record].to_json + "\n"
     end
 
     # This method is called every flush interval. Write the buffer chunk
@@ -21,10 +22,21 @@ module Fluent
     #
     # NOTE! This method is called by internal thread, not Fluentd's main thread. So IO wait doesn't affect other plugins.
     def write(chunk)
-      json = json_parse(chunk.read)
-      $log.debug "#{json[0]}, #{format_time(json[1])}, #{json[2]}"
-      @connect.publish(rewrite_tag(json[0]), (json[2].merge(timestamp_hash(json[1]))).to_json)
+      messages = {}
+      chunk.msgpack_each do |tag, time, record|
+        #$log.debug "Thread ID: #{Thread.current.object_id}, tag: #{tag}, time: #{format_time(time)}, record: #{record}"
+        messages[tag] = [] if messages[tag].nil?
+        messages[tag] << record.merge(timestamp_hash(time))
+      end
+      messages.keys.each do |tag|
+        $log.debug "Thread ID: #{Thread.current.object_id}, topic: #{rewrite_tag(tag)}, message: #{messages[tag]}"
+        @client.publish(rewrite_tag(tag), messages[tag].map {|m| m.to_json}.join(@bulk_trans_sep))
+      end
       $log.flush
+      #json = json_parse(chunk.open {|io| io.readline})
+      #$log.debug "#{json[0]}, #{format_time(json[1])}, #{json[2]}"
+      #@client.publish(rewrite_tag(json[0]), (json[2].merge(timestamp_hash(json[1]))).to_json)
+      #$log.flush
     end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-mqtt-io
 version: !ruby/object:Gem::Version
-  version: 0.0.6
+  version: 0.1.0
 platform: ruby
 authors:
 - Toyokazu Akiyama
 autorequire: 
 bindir: []
 cert_chain: []
-date: 2015-12-24 00:00:00.000000000 Z
+date: 2015-12-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd