fluent-plugin-mqtt-io 0.4.4 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e083f992d750102e55064612775e4d03b188e50fa1d6f3585837124bd681ea90
-  data.tar.gz: 5803786735463c1c29f47b608374926fc73106310ce5d8af8594fe909744f7ff
+  metadata.gz: ed837b9979b990f051238be99dac2c17ba2feb7b35c0f56910f706c3c426944a
+  data.tar.gz: 52628c53314f60215b6b97854103a3e99c0f313fe5a5031b1e6ed87e16cfc918
 SHA512:
-  metadata.gz: 6bdb809393dae2de2e85ad36a2f0e745ade9cc12834f136e4081aced2bd56aae14d9ed0e9dbfd307d91f34336b0ec98ae2921cbfee2a3f07599878d120c8f026
-  data.tar.gz: fa59faf530eb786320f0bf7a6f115b6b4db12e44334aba6bfcac56fa9b2e563d0e1efa499a7302e69a297ef98fc7e1ed3a73cd355a057c9a7b16dcf367b47dd5
+  metadata.gz: d2b38c5196fa566997d647fe63d3b2af44b323d218b5ec2c8e613cfa6662bd64401fbc99a68935950ddeb9b1b128f7571357e885276dfae5c6fbaaab62c088e9
+  data.tar.gz: 2d96a742d31d3b638f989999ea5c4919c95f61c64156d46d2dc01263842b036cf584bccc596c11634a3b95f358e84c112ce7b4c64b9eafc6fe48c367f0518381

data/README.md

@@ -5,7 +5,7 @@ Mqtt::IO Plugin is deeply inspired by Fluent::Plugin::Mqtt.
 
 https://github.com/yuuna/fluent-plugin-mqtt
 
-Mqtt::IO plugin focus on federating components, e.g. sensors, messaging platform and databases. Encryption/Decryption is not supported in this plugin but [fluent-plugin-jwt-filter](https://github.com/toyokazu/fluent-plugin-jwt-filter) can be used to encrypt/decrypt messages using JSON Web Token technology.
+Mqtt::IO plugin focus on federating components, e.g. sensors, messaging platform and databases. Connection encryption/decryption (TLS) is supported by ruby-mqtt but end-to-end encryption/decryption is not supported in this plugin. [fluent-plugin-jwt-filter](https://github.com/toyokazu/fluent-plugin-jwt-filter) can be used to encrypt/decrypt messages using JSON Web Token technology.
 
 ## Installation
 
@@ -83,6 +83,7 @@ The default MQTT topic is "#". Configurable options are the following:
 - **initial_interval**: An initial value of retry interval (s) (default 1)
 - **retry_inc_ratio**: An increase ratio of retry interval per connection failure (default 2 (double)). It may be better to set the value to 1 in a mobile environment for eager reconnection.
 - **max_retry_interval**: Maximum value of retry interval (default 300)
+- **max_retry_freq**: Threshold of retry frequency described by a number of retries per minutes. This option is provided for detecting failure via proxy services, e.g. ssh port forwarding. When the thresold is exceeded, MqttProxy::ExceedRetryFrequencyThresholdException is raised and the fluentd will be restarted. So, it is enough to be specified once for a MQTT server at a source/match directive in your configuration (default 10)
 
 Input Plugin supports @label directive.
 

data/fluent-plugin-mqtt-io.gemspec

@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name          = "fluent-plugin-mqtt-io"
-  spec.version       = "0.4.4"
+  spec.version       = "0.5.0"
   spec.authors       = ["Toyokazu Akiyama"]
   spec.email         = ["toyokazu@gmail.com"]
 
@@ -24,7 +24,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency 'fluentd', [">= 0.14.0", "< 2"]
   spec.add_dependency "mqtt", "~> 0.5"
 
-  spec.add_development_dependency "bundler", "~> 1.14"
-  spec.add_development_dependency "rake", "~> 12.0"
+  spec.add_development_dependency "bundler", [">= 1.14", "< 2.3"]
+  spec.add_development_dependency "rake", "~> 13.0"
   spec.add_development_dependency "test-unit"
 end

data/lib/fluent/plugin/in_mqtt.rb

@@ -20,11 +20,6 @@ module Fluent::Plugin
       config_param :@type, :string, default: 'none'
     end
 
-    # bulk_trans is deprecated
-    # multiple entries must be inputted as an Array
-    #config_param :bulk_trans, :bool, default: true
-    #config_param :bulk_trans_sep, :string, default: "\t"
-
     config_section :monitor, required: false, multi: false do
       desc 'Record received time into message or not.'
       config_param :recv_time, :bool, default: false
@@ -66,20 +61,28 @@ module Fluent::Plugin
       :in_mqtt
     end
 
-    def kill_thread
-      @get_thread.kill if !@get_thread.nil?
+    def exit_thread
+      @get_thread.exit if !@get_thread.nil?
+    end
+
+    def disconnect
+      begin
+        @client.disconnect if @client.connected?
+      rescue => e
+        log.error "Error in in_mqtt#disconnect,#{e.class},#{e.message}"
+      end
+      exit_thread
     end
 
-    def after_disconnection
-      kill_thread
+    def terminate
+      exit_thread
       super
     end
 
     def after_connection
       if @client.connected?
         @client.subscribe(@topic)
-        #@get_thread = thread_create(:in_mqtt_get) do
-        @get_thread = Thread.new do
+        @get_thread = thread_create(:in_mqtt_get) do
           @client.get do |topic, message|
             emit(topic, message)
           end

data/lib/fluent/plugin/mqtt_proxy.rb

@@ -20,8 +20,10 @@ module Fluent::Plugin
       base.config_param :initial_interval, :integer, default: 1
       base.desc 'Specify increasing ratio of connection retry interval.'
       base.config_param :retry_inc_ratio, :integer, default: 2
-      base.desc 'Specify maximum connection retry interval.'
+      base.desc 'Specify the maximum connection retry interval.'
       base.config_param :max_retry_interval, :integer, default: 300
+      base.desc 'Specify threshold of retry frequency as number of retries per minutes. Frequency is monitored per retry.'
+      base.config_param :max_retry_freq, :integer, default: 10
 
       base.config_section :security, required: false, multi: false do
         ### User based authentication
@@ -44,11 +46,20 @@ module Fluent::Plugin
 
     class MqttError < StandardError; end
 
+    class ExceedRetryFrequencyThresholdException < StandardError; end
+
     def current_plugin_name
       # should be implemented
     end
 
     def start_proxy
+      # Start a thread from main thread for handling a thread generated
+      # by MQTT::Client#get (in_mqtt). Dummy thread is used for out_mqtt
+      # to keep the same implementation style.
+      @proxy_thread = thread_create("#{current_plugin_name}_proxy".to_sym, &method(:proxy))
+    end
+
+    def proxy
       log.debug "start mqtt proxy for #{current_plugin_name}"
       log.debug "start to connect mqtt broker #{@host}:#{@port}"
       opts = {
@@ -68,12 +79,13 @@ module Fluent::Plugin
       end
 
       init_retry_interval
+      @retry_sequence = []
       @client = MQTT::Client.new(opts)
       connect
     end
 
     def shutdown_proxy
-      @client.disconnect
+      disconnect
     end
 
     def init_retry_interval
@@ -81,29 +93,43 @@ module Fluent::Plugin
     end
 
     def increment_retry_interval
-      return @retry_interval if @retry_interval >= @max_retry_interval
+      return @max_retry_interval if @retry_interval >= @max_retry_interval
       @retry_interval = @retry_interval * @retry_inc_ratio
     end
 
-    def retry_connect(e, message)
-      if !@_retrying
-        log.error "#{message},#{e.class},#{e.message}"
-        log.error "Retry in #{@retry_interval} sec"
-        timer_execute("#{current_plugin_name}_connect".to_sym, @retry_interval, repeat: false, &method(:connect))
-        @_retrying = true
-        increment_retry_interval
-        after_disconnection
-        @client.disconnect if @client.connected?
+    def update_retry_sequence(e) 
+      @retry_sequence << {time: Time.now, error: "#{e.class}: #{e.message}"}
+      # delete old retry records
+      while @retry_sequence[0][:time] < Time.now - 60
+        @retry_sequence.shift
+      end 
+    end
+
+    def check_retry_frequency
+      return if @retry_sequence.size <= 1
+      if @retry_sequence.size > @max_retry_freq
+        log.error "Retry frequency threshold is exceeded: #{@retry_sequence}"
+        raise ExceedRetryFrequencyThresholdException
       end
     end
 
-    def kill_connect_thread
-      @connect_thread.kill if !@connect_thread.nil?
+    def retry_connect(e, message)
+      log.error "#{message},#{e.class},#{e.message}"
+      log.error "Retry in #{@retry_interval} sec"
+      update_retry_sequence(e)
+      check_retry_frequency
+      disconnect
+      sleep @retry_interval
+      increment_retry_interval
+      connect
+      # never reach here
     end
 
-    def after_disconnection
+    def disconnect
       # should be implemented
-      kill_connect_thread
+    end
+
+    def terminate
     end
 
     def rescue_disconnection
@@ -114,22 +140,13 @@ module Fluent::Plugin
       begin
         yield
       rescue MQTT::ProtocolException => e
-        # TODO:
-        # Thread created via fluentd thread API, e.g. thread_create,
-        # cannot catch MQTT::ProtocolException raised from @read_thread
-        # in ruby-mqtt. So, the current version uses plugin local thread
-        # @connect_thread to catch it.
         retry_connect(e, "Protocol error occurs in #{current_plugin_name}.")
-        raise MqttError, "Protocol error occurs."
       rescue Timeout::Error => e
         retry_connect(e, "Timeout error occurs in #{current_plugin_name}.")
-        raise Timeout::Error, "Timeout error occurs."
       rescue SystemCallError => e
         retry_connect(e, "System call error occurs in #{current_plugin_name}.")
-        raise SystemCallError, "System call error occurs."
       rescue StandardError=> e
         retry_connect(e, "The other error occurs in #{current_plugin_name}.")
-        raise StandardError, "The other error occurs."
       rescue MQTT::NotConnectedException=> e
         # Since MQTT::NotConnectedException is raised only on publish,
         # connection error should be catched before this error.
@@ -137,7 +154,7 @@ module Fluent::Plugin
         # to prevent waistful increment of retry interval.
         #log.error "MQTT not connected exception occurs.,#{e.class},#{e.message}"
         #retry_connect(e, "MQTT not connected exception occurs.")
-        raise MqttError, "MQTT not connected exception occurs in #{current_plugin_name}."
+        #raise MqttError, "MQTT not connected exception occurs in #{current_plugin_name}."
       end
     end
 
@@ -148,16 +165,12 @@ module Fluent::Plugin
     end
 
     def connect
-      #@connect_thread = thread_create("#{current_plugin_name}_monitor".to_sym) do
-      @_retrying = false
-      @connect_thread = Thread.new do
-        rescue_disconnection do
-          @client.connect
-          log.debug "connected to mqtt broker #{@host}:#{@port} for #{current_plugin_name}"
-          init_retry_interval
-          thread = after_connection
-          thread.join
-        end
+      rescue_disconnection do
+        @client.connect
+        log.debug "connected to mqtt broker #{@host}:#{@port} for #{current_plugin_name}"
+        init_retry_interval
+        thread = after_connection
+        thread.join
       end
     end
   end

data/lib/fluent/plugin/out_mqtt.rb

@@ -89,27 +89,35 @@ module Fluent::Plugin
     # Shutdown the thread and close sockets or files here.
     def shutdown
       shutdown_proxy
-      kill_thread
+      exit_thread
       super
     end
 
-    def kill_thread
-      @dummy_thread.kill if !@dummy_thread.nil?
+    def exit_thread
+      @dummy_thread.exit if !@dummy_thread.nil?
     end
 
-    def after_connection
-      #@dummy_thread = thread_create(:out_mqtt_dummy) do
-      @dummy_thread = Thread.new do
-        Thread.stop
+    def disconnect
+      begin
+        @client.disconnect if @client.connected?
+      rescue => e
+        log.error "Error in out_mqtt#disconnect,#{e.class},#{e.message}"
       end
-      @dummy_thread
+      exit_thread
     end
 
-    def after_disconnection
-      kill_thread
+    def terminate
+      exit_thread
       super
     end
 
+    def after_connection
+      @dummy_thread = thread_create(:out_mqtt_dummy) do
+        Thread.stop
+      end
+      @dummy_thread
+    end
+
     def current_plugin_name
       :out_mqtt
     end
@@ -148,7 +156,7 @@ module Fluent::Plugin
     end
 
     def publish(tag, time, record)
-      log.debug "MqttOutput::#{caller_locations(1,1)[0].label}: #{rewrite_tag(rewrite_tag(tag))}, #{time}, #{add_send_time(record)}"
+      log.debug "MqttOutput::#{caller_locations(1,1)[0].label}: #{rewrite_tag(tag)}, #{time}, #{add_send_time(record)}"
       @client.publish(
         rewrite_tag(tag),
         @formatter.format(tag, time, add_send_time(record)),

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-mqtt-io
 version: !ruby/object:Gem::Version
-  version: 0.4.4
+  version: 0.5.0
 platform: ruby
 authors:
 - Toyokazu Akiyama
 autorequire: 
 bindir: []
 cert_chain: []
-date: 2019-12-19 00:00:00.000000000 Z
+date: 2020-12-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -48,30 +48,36 @@ dependencies:
   name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '1.14'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2.3'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '1.14'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2.3'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '12.0'
+        version: '13.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '12.0'
+        version: '13.0'
 - !ruby/object:Gem::Dependency
   name: test-unit
   requirement: !ruby/object:Gem::Requirement
@@ -127,7 +133,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
+rubygems_version: 3.1.4
 signing_key: 
 specification_version: 4
 summary: fluentd input/output plugin for mqtt broker