fluent-plugin-mqtt-io 0.3.1 → 0.3.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 97803d6c5941eb7ff068d9d8e7205c9a78cc629b
-  data.tar.gz: 8f492394371c32718ad1f97181597efe2192bf08
+  metadata.gz: de59247eae560befc4d28b446d68e3e9ef650e90
+  data.tar.gz: 6e9eb9e006557bdad477407ec7d648bacf73d9ce
 SHA512:
-  metadata.gz: a022d9dd82ed38ced4b9cfc89e7d01da5cca628dd6030bc36687df3594409dc25953a027ab9a5a2390f3aa7ef12081114b46c81311d8904a52e342f71f14cd3c
-  data.tar.gz: 00e999e1448d738401eca559d54670c2dff6586f534342af9d09c9caf3f9f745b4bcbbc3704e55210b55485f7bf210e8cce54c21358ce96c00b8be3a0728802b
+  metadata.gz: f96037a4572be86d1acf889569d41f814dfc13270eda73dbb7b704c2de05e8ad83723c3648d90e1f530a443ea52bd189f88e861982ed376e67d320964fef5afd
+  data.tar.gz: 18a41fef17638a172e500df27aa51b45da20e3ef022823893f358d26ed581bdae5d86af4e1bb0568ffb2054da1fc108051123b93d4010b46980f31905ff552a3

data/README.md

@@ -30,6 +30,8 @@ fluent-plugin-mqtt-io provides Input and Output Plugins for MQTT.
 
 Input Plugin can be used via source directive in the configuration.
 
+For fluent-plugin-mqtt-io <= 0.2.3
+
 ```
 
 <source>
@@ -41,6 +43,20 @@ Input Plugin can be used via source directive in the configuration.
 
 ```
 
+For fluent-plugin-mqtt-io ~> v0.3.0
+
+```
+
+<source>
+  @type mqtt
+  host 127.0.0.1
+  port 1883
+  format json
+</source>
+
+```
+
+
 The default MQTT topic is "#". Configurable options are the following:
 
 - **host**: IP address of MQTT broker
@@ -69,6 +85,8 @@ Input Plugin supports @label directive.
 
 Output Plugin can be used via match directive.
 
+For fluent-plugin-mqtt-io <= 0.2.3
+
 ```
 
 <match topic.**>
@@ -79,6 +97,18 @@ Output Plugin can be used via match directive.
 
 ```
 
+For fluent-plugin-mqtt-io ~> v0.3.0
+
+```
+
+<match topic.**>
+  @type mqtt
+  host 127.0.0.1
+  port 1883
+</match>
+
+```
+
 The options are basically the same as Input Plugin except for "format" and "bulk_trans" (only for Input). Additional options for Output Plugin are the following.
 
 - time_key: An attribute name used for timestamp field genarated from fluentd time field. Default is nil (omitted).
@@ -93,6 +123,7 @@ If you use different source, e.g. the other MQTT broker, log file and so on, the
 
 The topic name or tag name, e.g. "topic", received from an event can not be published without modification because if MQTT input plugin connecting to the identical MQTT broker is used as a source, the same message will become an input repeatedly. In order to support data conversion in single MQTT domain, simple topic rewriting should be supported. Since topic is rewritten using #gsub method, 'pattern' and 'replacement' are the same as #gsub arguments.
 
+For fluent-plugin-mqtt-io <= v0.2.3
 
 ```
 
@@ -106,6 +137,20 @@ The topic name or tag name, e.g. "topic", received from an event can not be publ
 
 ```
 
+For fluent-plugin-mqtt-io ~> v0.3.0
+
+```
+
+<match topic.**>
+  @type mqtt
+  host 127.0.0.1
+  port 1883
+  topic_rewrite_pattern '^([\w\/]+)$'
+  topic_rewrite_replacement '\1/rewritten'
+</match>
+
+```
+
 You can also use mqtt_buf type which is implemented as Fluent::MqttBufferedOutput.
 
 ```
@@ -123,6 +168,25 @@ You can also use mqtt_buf type which is implemented as Fluent::MqttBufferedOutpu
 
 ```
 
+For fluent-plugin-mqtt-io ~> v0.3.0
+
+```
+
+<match topic.**>
+  @type mqtt
+  host 127.0.0.1
+  port 1883
+  topic_rewrite_pattern '^([\w\/]+)$'
+  topic_rewrite_replacement '\1/rewritten'
+  # You can specify Buffer Plugin options
+  <buffer>
+    buffer_type memory
+    flush_interval 1s
+  </buffer>
+</match>
+
+```
+
 When a plugin implemented as Fluent::BufferedOutput, fluentd stores the received messages into buffers (Fluent::MemoryBuffer is used as default) separated by tag names. Writer Threads emit those stored messages periodically in the specified interval. In MqttBufferedOutput, the stored messages are concatenated with 'bulk_trans_sep' (default: "\t"). This function reduces the number of messages sent via MQTT if data producing interval of sensors are smaller than publish interval (flush_interval). The concatinated messages can be properly handled by Fluent::MqttInput plugin by specifying 'bulk_trans' option.
 
 ## Use case examples

data/fluent-plugin-mqtt-io.gemspec

@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name          = "fluent-plugin-mqtt-io"
-  spec.version       = "0.3.1"
+  spec.version       = "0.3.3"
   spec.authors       = ["Toyokazu Akiyama"]
   spec.email         = ["toyokazu@gmail.com"]
 
@@ -22,7 +22,7 @@ Gem::Specification.new do |spec|
   spec.required_ruby_version = '>= 2.2.0'
 
   spec.add_dependency 'fluentd', '~> 0.14'
-  spec.add_dependency "mqtt", "~> 0.4"
+  spec.add_dependency "mqtt", "~> 0.5"
 
   spec.add_development_dependency "bundler", "~> 1.14"
   spec.add_development_dependency "rake", "~> 12.0"

data/lib/fluent/plugin/in_mqtt.rb

@@ -1,54 +1,26 @@
 require 'fluent/plugin/input'
 require 'fluent/event'
 require 'fluent/time'
-require 'mqtt'
+require 'fluent/plugin/mqtt_proxy'
 
 module Fluent::Plugin
   class MqttInput < Input
+    include MqttProxy
+
     Fluent::Plugin.register_input('mqtt', self)
 
     helpers :thread, :compat_parameters, :parser
 
-    MQTT_PORT = 1883
-
-    desc 'The address to connect to.'
-    config_param :host, :string, default: '127.0.0.1'
-    desc 'The port to connect to.'
-    config_param :port, :integer, default: MQTT_PORT
     desc 'The topic to subscribe.'
     config_param :topic, :string, default: '#'
     desc 'The format to receive.'
     config_param :format, :string, default: 'json'
-    desc 'Specify keep alive interval.'
-    config_param :keep_alive, :integer, default: 15
-    desc 'Specify initial interval for reconnection.'
-    config_param :initial_interval, :integer, default: 1
-    desc 'Specify increasing ratio of reconnection interval.'
-    config_param :retry_inc_ratio, :integer, default: 2
 
     # bulk_trans is deprecated
     # multiple entries must be inputted as an Array
     #config_param :bulk_trans, :bool, default: true
     #config_param :bulk_trans_sep, :string, default: "\t"
 
-    config_section :security, required: false, multi: false do
-      ### User based authentication
-      desc 'The username for authentication'
-      config_param :username, :string, default: nil
-      desc 'The password for authentication'
-      config_param :password, :string, default: nil
-      desc 'Use TLS or not.'
-      config_param :use_tls, :bool, default: nil
-      config_section :tls, required: false, multi: true do
-        desc 'Specify TLS ca file.'
-        config_param :ca_file, :string, default: nil
-        desc 'Specify TLS key file.'
-        config_param :key_file, :string, default: nil
-        desc 'Specify TLS cert file.'
-        config_param :cert_file, :string, default: nil
-      end
-    end
-
     config_section :monitor, required: false, multi: false do
       desc 'Record received time into message or not.'
       config_param :recv_time, :bool, default: false
@@ -59,7 +31,6 @@ module Fluent::Plugin
     def configure(conf)
       super
       configure_parser(conf)
-      init_retry_interval
     end
 
     def configure_parser(conf)
@@ -68,71 +39,32 @@ module Fluent::Plugin
       @parser = parser_create(conf: parser_config)
     end
 
-    def init_retry_interval
-      @retry_interval = @initial_interval
+    def start
+      super
+      start_proxy
     end
 
-    def increment_retry_interval
-      @retry_interval = @retry_interval * @retry_inc_ratio
+    def shutdown
+      shutdown_proxy
+      super
     end
 
-    def sleep_retry_interval(e, message)
-      log.error "#{message},#{e.class},#{e.message}"
-      log.error "Retry in #{@retry_interval} sec"
-      sleep @retry_interval
-      increment_retry_interval
+    def current_plugin_name
+      :in_mqtt
     end
 
-    def start
-      log.debug "start mqtt #{@host}"
-      opts = {
-        host: @host,
-        port: @port,
-        keep_alive: @keep_alive
-      }
-      opts[:username] = @security.username if @security.respond_to?(:username)
-      opts[:password] = @security.password if @security.respond_to?(:password)
-      if @security.respond_to?(:use_tls) && @security.use_tls
-        opts[:ssl] = @security.use_tls
-        opts[:ca_file] = @security.tls.ca_file
-        opts[:cert_file] = @security.tls.cert_file
-        opts[:key_file] = @security.tls.key_file
-      end
-
-      # In order to handle Exception raised from reading Thread
-      # in MQTT::Client caused by network disconnection (during read_byte),
-      # @connect_thread generates connection.
-      @client = MQTT::Client.new(opts)
-      @get_thread = nil
-      @connect_thread = Thread.new(&method(:connect_loop))
+    def after_disconnection
     end
 
-    def connect_loop
-      while (true)
-        begin
-          @get_thread.kill if !@get_thread.nil? && @get_thread.alive?
-          @client.disconnect if @client.connected?
-          @client.connect
-          @client.subscribe(@topic)
-          @get_thread = Thread.new do
+    def after_connection
+      if @client.connected?
+        @client.subscribe(@topic)
+        thread_create(:in_mqtt_get) do
+          rescue_disconnection do
             @client.get do |topic, message|
               emit(topic, message)
             end
           end
-          init_retry_interval
-          sleep
-        rescue MQTT::ProtocolException => e
-          sleep_retry_interval(e, "Protocol error occurs.")
-          next
-        rescue Timeout::Error => e
-          sleep_retry_interval(e, "Timeout error occurs.")
-          next
-        rescue SystemCallError => e
-          sleep_retry_interval(e, "System call error occurs.")
-          next
-        rescue StandardError=> e
-          sleep_retry_interval(e, "The other error occurs.")
-          next
         end
       end
     end
@@ -175,11 +107,5 @@ module Fluent::Plugin
         log.debug_backtrace(e.backtrace)
       end
     end
-
-    def shutdown
-      @get_thread.kill
-      @connect_thread.kill
-      @client.disconnect
-    end
   end
 end

data/lib/fluent/plugin/mqtt_proxy.rb

@@ -0,0 +1,129 @@
+require 'mqtt'
+module Fluent::Plugin
+  module MqttProxy
+    MQTT_PORT = 1883
+
+    def self.included(base)
+      base.helpers :timer
+
+      base.desc 'The address to connect to.'
+      base.config_param :host, :string, default: '127.0.0.1'
+      base.desc 'The port to connect to.'
+      base.config_param :port, :integer, default: MQTT_PORT
+      base.desc 'Specify keep alive interval.'
+      base.config_param :keep_alive, :integer, default: 15
+      base.desc 'Specify initial connection retry interval.'
+      base.config_param :initial_interval, :integer, default: 1
+      base.desc 'Specify increasing ratio of connection retry interval.'
+      base.config_param :retry_inc_ratio, :integer, default: 2
+      base.desc 'Specify maximum connection retry interval.'
+      base.config_param :max_retry_interval, :integer, default: 300
+
+      base.config_section :security, required: false, multi: false do
+        ### User based authentication
+        base.desc 'The username for authentication'
+        base.config_param :username, :string, default: nil
+        base.desc 'The password for authentication'
+        base.config_param :password, :string, default: nil
+        base.desc 'Use TLS or not.'
+        base.config_param :use_tls, :bool, default: nil
+        base.config_section :tls, required: false, multi: true do
+          base.desc 'Specify TLS ca file.'
+          base.config_param :ca_file, :string, default: nil
+          base.desc 'Specify TLS key file.'
+          base.config_param :key_file, :string, default: nil
+          base.desc 'Specify TLS cert file.'
+          base.config_param :cert_file, :string, default: nil
+        end
+      end
+    end
+
+    class MqttProxyError
+    end
+
+    def current_plugin_name
+      # should be implemented
+    end
+
+    def start_proxy
+      log.debug "start mqtt proxy for #{current_plugin_name}"
+      log.debug "start to connect mqtt broker #{@host}:#{@port}"
+      opts = {
+        host: @host,
+        port: @port,
+        keep_alive: @keep_alive
+      }
+      opts[:username] = @security.username if @security.respond_to?(:username)
+      opts[:password] = @security.password if @security.respond_to?(:password)
+      if @security.respond_to?(:use_tls) && @security.use_tls
+        opts[:ssl] = @security.use_tls
+        opts[:ca_file] = @security.tls.ca_file
+        opts[:cert_file] = @security.tls.cert_file
+        opts[:key_file] = @security.tls.key_file
+      end
+
+      init_retry_interval
+      @client_mutex = Mutex.new
+      @client = MQTT::Client.new(opts)
+      connect
+    end
+
+    def shutdown_proxy
+      @client.disconnect
+    end
+
+    def init_retry_interval
+      @retry_interval = @initial_interval
+    end
+
+    def increment_retry_interval
+      return @retry_interval if @retry_interval >= @max_retry_interval
+      @retry_interval = @retry_interval * @retry_inc_ratio
+    end
+
+    def retry_connect(e, message)
+      log.error "#{message},#{e.class},#{e.message}"
+      log.error "Retry in #{@retry_interval} sec"
+      timer_execute("#{current_plugin_name}_connect".to_sym, @retry_interval, repeat: false, &method(:connect))
+      increment_retry_interval
+      after_disconnection
+      @client.disconnect if @client.connected?
+    end
+
+    def after_disconnection
+      # should be implemented
+    end
+
+    def rescue_disconnection(*block)
+      begin
+        yield *block
+      rescue MQTT::ProtocolException => e
+        # TODO:
+        # Currently MQTT::ProtocolException cannot be caught during @client.get
+        # and @client.publish. The reason must be investigated...
+        retry_connect(e, "Protocol error occurs.")
+      rescue Timeout::Error => e
+        retry_connect(e, "Timeout error occurs.")
+      rescue SystemCallError => e
+        retry_connect(e, "System call error occurs.")
+      rescue StandardError=> e
+        retry_connect(e, "The other error occurs.")
+      end
+    end
+
+    def after_connection
+      # should be implemented
+    end
+
+    def connect
+      @client_mutex.lock
+      rescue_disconnection do
+        @client.connect
+        log.debug "connected to mqtt broker #{@host}:#{@port} for #{current_plugin_name}"
+        init_retry_interval
+      end
+      @client_mutex.unlock
+      after_connection
+    end
+  end
+end

data/lib/fluent/plugin/out_mqtt.rb

@@ -1,48 +1,20 @@
 require 'fluent/plugin/output'
 require 'fluent/event'
 require 'fluent/time'
-require 'mqtt'
+require 'fluent/plugin/mqtt_proxy'
 
 module Fluent::Plugin
   class MqttOutput < Output
+    include MqttProxy
+
     Fluent::Plugin.register_output('mqtt', self)
 
     helpers :compat_parameters, :formatter, :inject
 
-    MQTT_PORT = 1883
-
-    desc 'The address to connect to.'
-    config_param :host, :string, default: '127.0.0.1'
-    desc 'The port to connect to.'
-    config_param :port, :integer, default: MQTT_PORT
-    desc 'MQTT keep alive interval.'
-    config_param :keep_alive, :integer, default: 15
     desc 'Topic rewrite matching pattern.'
     config_param :topic_rewrite_pattern, :string, default: nil
     desc 'Topic rewrite replacement string.'
     config_param :topic_rewrite_replacement, :string, default: nil
-    desc 'Initial retry interval.'
-    config_param :initial_interval, :integer, default: 1
-    desc 'Increasing ratio of retry interval.'
-    config_param :retry_inc_ratio, :integer, default: 2
-
-    config_section :security, required: false, multi: false do
-      ### User based authentication
-      desc 'The username for authentication'
-      config_param :username, :string, default: nil
-      desc 'The password for authentication'
-      config_param :password, :string, default: nil
-      desc 'Use TLS or not.'
-      config_param :use_tls, :bool, default: nil
-      config_section :tls, required: false, multi: true do
-        desc 'TLS ca file.'
-        config_param :ca_file, :string, default: nil
-        desc 'TLS key file.'
-        config_param :key_file, :string, default: nil
-        desc 'TLS cert file.'
-        config_param :cert_file, :string, default: nil
-      end
-    end
 
     config_section :monitor, required: false, multi: false do
       desc 'Recording send time for monitoring.'
@@ -60,22 +32,6 @@ module Fluent::Plugin
       formatter_config = conf.elements(name: 'format').first
       @formatter = formatter_create(conf: formatter_config)
       @has_buffer_section = conf.elements(name: 'buffer').size > 0
-      init_retry_interval
-    end
-
-    def init_retry_interval
-      @retry_interval = @initial_interval
-    end
-
-    def increment_retry_interval
-      @retry_interval = @retry_interval * @retry_inc_ratio
-    end
-
-    def sleep_retry_interval(e, message)
-      log.error "#{message},#{e.class},#{e.message}"
-      log.error "Retry in #{@retry_interval} sec"
-      sleep @retry_interval
-      increment_retry_interval
     end
 
     def rewrite_tag(tag)
@@ -94,89 +50,18 @@ module Fluent::Plugin
     # Open sockets or files here.
     def start
       super
-
-      log.debug "start to connect mqtt broker #{@host}:#{@port}"
-      opts = {
-        host: @host,
-        port: @port,
-        keep_alive: @keep_alive
-      }
-      opts[:username] = @security.username if @security.respond_to?(:username)
-      opts[:password] = @security.password if @security.respond_to?(:password)
-      if @security.respond_to?(:use_tls) && @security.use_tls
-        opts[:ssl] = @security.use_tls
-        opts[:ca_file] = @security.tls.ca_file
-        opts[:cert_file] = @security.tls.cert_file
-        opts[:key_file] = @security.tls.key_file
-      end
-      # In order to handle Exception raised from reading Thread
-      # in MQTT::Client caused by network disconnection (during read_byte),
-      # @connect_thread generates connection.
-      @client_mutex = Mutex.new
-      @client = MQTT::Client.new(opts)
-      connect_loop
+      start_proxy
     end
 
     # This method is called when shutting down.
     # Shutdown the thread and close sockets or files here.
     def shutdown
+      shutdown_proxy
       super
-
-      @client.disconnect
     end
 
-    def connect_loop
-      while (true)
-        @client_mutex.lock
-        begin
-          @client.disconnect if @client.connected?
-          @client.connect
-          init_retry_interval
-        rescue MQTT::ProtocolException => e
-          sleep_retry_interval(e, "Protocol error occurs.")
-          @client_mutex.unlock
-          next
-        rescue Timeout::Error => e
-          sleep_retry_interval(e, "Timeout error occurs.")
-          @client_mutex.unlock
-          next
-        rescue SystemCallError => e
-          sleep_retry_interval(e, "System call error occurs.")
-          @client_mutex.unlock
-          next
-        rescue StandardError=> e
-          sleep_retry_interval(e, "The other error occurs.")
-          @client_mutex.unlock
-          next
-        end
-        @client_mutex.unlock
-        break
-      end
-    end
-
-    def publish_error_handler
-      while(true)
-        begin
-          yield
-        rescue MQTT::ProtocolException => e
-          sleep_retry_interval(e, "Protocol error occurs.")
-          connect_loop
-          next
-        rescue Timeout::Error => e
-          sleep_retry_interval(e, "Timeout error occurs.")
-          connect_loop
-          next
-        rescue SystemCallError => e
-          sleep_retry_interval(e, "System call error occurs.")
-          connect_loop
-          next
-        rescue StandardError=> e
-          sleep_retry_interval(e, "The other error occurs.")
-          connect_loop
-          next
-        end
-        break
-      end
+    def current_plugin_name
+      :out_mqtt
     end
 
     def add_send_time(record)
@@ -192,8 +77,8 @@ module Fluent::Plugin
       if es.class == Fluent::OneEventStream
         es = inject_values_to_event_stream(tag, es)
         es.each do |time, record|
-          log.debug "#{rewrite_tag(tag)}, #{add_send_time(record)}"
-          publish_error_handler do
+          log.debug "#{rewrite_tag(tag)}, #{time}, #{add_send_time(record)}"
+          rescue_disconnection do
             @client.publish(rewrite_tag(tag), @formatter.format(tag, time, add_send_time(record)))
           end
         end
@@ -201,10 +86,10 @@ module Fluent::Plugin
         es = inject_values_to_event_stream(tag, es)
         array = []
         es.each do |time, record|
-          log.debug "#{rewrite_tag(tag)}, #{add_send_time(record)}"
+          log.debug "#{rewrite_tag(tag)}, #{time}, #{add_send_time(record)}"
           array << add_send_time(record)
         end
-        publish_error_handler do
+        rescue_disconnection do
           @client.publish(rewrite_tag(tag), @formatter.format(tag, Fluent::EventTime.now, array))
         end
       end
@@ -215,11 +100,22 @@ module Fluent::Plugin
       publish_event_stream(tag, es)
     end
 
+    def format(tag, time, record)
+      [tag, time, record].to_msgpack
+    end
+
+    def formatted_to_msgpack_binary
+      true
+    end
+
     def write(chunk)
       return if chunk.empty?
-      tag = chunk.metadata.tag
-
-      publish_event_stream(tag, es)
+      chunk.each do |tag, time, record|
+        rescue_disconnection do
+          log.debug "#{rewrite_tag(rewrite_tag(tag))}, #{time}, #{add_send_time(record)}"
+          @client.publish(rewrite_tag(tag), @formatter.format(tag, time, add_send_time(record)))
+        end
+      end
     end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-mqtt-io
 version: !ruby/object:Gem::Version
-  version: 0.3.1
+  version: 0.3.3
 platform: ruby
 authors:
 - Toyokazu Akiyama
 autorequire: 
 bindir: []
 cert_chain: []
-date: 2017-05-07 00:00:00.000000000 Z
+date: 2017-08-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -30,14 +30,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.4'
+        version: '0.5'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.4'
+        version: '0.5'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -96,6 +96,7 @@ files:
 - Rakefile
 - fluent-plugin-mqtt-io.gemspec
 - lib/fluent/plugin/in_mqtt.rb
+- lib/fluent/plugin/mqtt_proxy.rb
 - lib/fluent/plugin/out_mqtt.rb
 - test/helper.rb
 - test/plugin/test_in_mqtt.rb