fluent-plugin-mqtt-io 0.2.1 → 0.2.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c7acf8fbbfbb33e609608751a49d4cf4b97ec0b5
-  data.tar.gz: a03ffed38eb93fa059f9b415bf58208fb7616b3a
+  metadata.gz: 5d00a8d71f16ee4a7f7c25d5c6432e1786159728
+  data.tar.gz: 54654b5f2cf2dca79f443a3e376a1bd323b4514f
 SHA512:
-  metadata.gz: 93ae1aec473651c34c0c9398b144b7d832e1691d73844f6dc4a840df5740b786ca209c3f4484b5e1327b94d541ae74fde1cd40f30bb545ba385a7c33bef355e5
-  data.tar.gz: 194bd3a8c5665788276f95ddb9dc1a99d89511b3e725f496af414e376385df2d44b07fcc1c169300f5fbac1c0364f1aca5938c52e75193beeaf77376296d57cb
+  metadata.gz: d061d3b17e90904f305b1310992e78aa4ebd26659fa9c10218e741b43b4d464157757d4c52c41a423b784dfb3409c23315e3bd0cc90f3cbb16f3d45af63b268b
+  data.tar.gz: 52628b2004bd6e0c492c12ce3fe4e24d073d4d02c9e21a7499ea0579c67313ee27108d71f6cd388156800f40d356d3bf57a386b71b3201afa6a020fef669083d

data/README.md

@@ -58,6 +58,8 @@ The default MQTT topic is "#". Configurable options are the following:
   - ca_file: CA certificate file path
   - key_file: private key file path
   - cert_file: certificate file path
+- recv_time: Add receive time to message in millisecond (ms) as integer for debug and performance/delay analysis
+- recv_time_key: An attribute of recv_time
 
 Input Plugin supports @label directive.
 
@@ -82,6 +84,8 @@ The options are basically the same as Input Plugin except for "format" and "bulk
 - time_format: Output format of timestamp field. Default is ISO8601. You can specify your own format by using TimeParser.
 - topic_rewrite_pattern: Regexp pattern to extract replacement words from received topic or tag name
 - topic_rewrite_replacement: Topic name used for the publish using extracted pattern
+- send_time: Add send time to message in millisecond (ms) as integer for debug and performance/delay analysis
+- send_time_key: An attribute of recv_time
 
 If you use different source, e.g. the other MQTT broker, log file and so on, there is no need to specifie topic rewriting. Skip the following descriptions.
 

data/fluent-plugin-mqtt-io.gemspec

@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name          = "fluent-plugin-mqtt-io"
-  spec.version       = "0.2.1"
+  spec.version       = "0.2.2"
   spec.authors       = ["Toyokazu Akiyama"]
   spec.email         = ["toyokazu@gmail.com"]
 

data/lib/fluent/plugin/in_mqtt.rb

@@ -15,6 +15,8 @@ module Fluent
     config_param :ca_file, :string, :default => nil
     config_param :key_file, :string, :default => nil
     config_param :cert_file, :string, :default => nil
+    config_param :recv_time, :bool, :default => false
+    config_param :recv_time_key, :string, :default => "recv_time"
 
     require 'mqtt'
 
@@ -25,14 +27,6 @@ module Fluent
 
     def configure(conf)
       super
-      @host ||= conf['host']
-      @topic ||= conf['topic']
-      @bulk_trans ||= conf['bulk_trans']
-      @bulk_trans_sep ||= conf['bulk_trans_sep']
-      @port ||= conf['port']
-      @username ||= conf['username']
-      @password ||= conf['password']
-      @keep_alive ||= conf['keep_alive']
       configure_parser(conf)
       init_retry_interval
     end
@@ -51,9 +45,8 @@ module Fluent
     end
 
     def sleep_retry_interval(e, message)
-      $log.debug "#{message}"
-      $log.debug "#{e.class}: #{e.message}"
-      $log.debug "Retry in #{@retry_interval} sec"
+      $log.error "#{message},#{e.class},#{e.message}"
+      $log.error "Retry in #{@retry_interval} sec"
       sleep @retry_interval
       increment_retry_interval
     end
@@ -107,14 +100,23 @@ module Fluent
       end
     end
 
+    def add_recv_time(record)
+      if @recv_time
+        # recv_time is recorded in ms
+        record.merge({@recv_time_key => Time.now.instance_eval { self.to_i * 1000 + (usec/1000) }})
+      else
+        record
+      end
+    end
+
     def parse(message)
       @parser.parse(message) do |time, record|
         if time.nil?
           $log.debug "Since time_key field is nil, Fluent::Engine.now is used."
           time = Fluent::Engine.now
         end
-        $log.debug "#{topic}, #{time}, #{record}"
-        return [time, record]
+        $log.debug "#{topic}, #{time}, #{add_recv_time(record)}"
+        return [time, add_recv_time(record)]
       end
     end
 

data/lib/fluent/plugin/mqtt_output_mixin.rb

@@ -18,6 +18,8 @@ module Fluent
       base.config_param :topic_rewrite_pattern, :string, :default => nil
       base.config_param :topic_rewrite_replacement, :string, :default => nil
       base.config_param :bulk_trans_sep, :string, :default => "\t"
+      base.config_param :send_time, :bool, :default => false
+      base.config_param :send_time_key, :string, :default => "send_time"
     end
 
     require 'mqtt'
@@ -27,18 +29,6 @@ module Fluent
     # If the configuration is invalid, raise Fluent::ConfigError.
     def configure(conf)
       super
-
-      # You can also refer raw parameter via conf[name].
-      @host ||= conf['host']
-      @port ||= conf['port']
-      @username ||= conf['username']
-      @password ||= conf['password']
-      @keep_alive ||= conf['keep_alive']
-      @time_key ||= conf['time_key']
-      @time_format ||= conf['time_format']
-      @topic_rewrite_pattern ||= conf['topic_rewrite_pattern']
-      @topic_rewrite_replacement ||= conf['topic_rewrite_replacement']
-      @bulk_trans_sep ||= conf['bulk_trans_sep']
       init_retry_interval
     end
 
@@ -51,9 +41,8 @@ module Fluent
     end
 
     def sleep_retry_interval(e, message)
-      $log.debug "#{message}"
-      $log.debug "#{e.class}: #{e.message}"
-      $log.debug "Retry in #{@retry_interval} sec"
+      $log.error "#{message},#{e.class},#{e.message}"
+      $log.error "Retry in #{@retry_interval} sec"
       sleep @retry_interval
       increment_retry_interval
     end
@@ -133,6 +122,16 @@ module Fluent
       end
     end
 
+    def add_send_time(record)
+      if @send_time
+        # send_time is recorded in ms
+        record.merge({@send_time_key => Time.now.instance_eval { self.to_i * 1000 + (usec/1000) }})
+      else
+        record
+      end
+    end
+                  
+
     def rewrite_tag(tag)
       if @topic_rewrite_pattern.nil?
         tag.gsub("\.", "/")

data/lib/fluent/plugin/out_mqtt.rb

@@ -9,8 +9,8 @@ module Fluent
 
     def emit(tag, es, chain)
       es.each {|time,record|
-        $log.debug "#{tag}, #{format_time(time)}, #{record}"
-        @client.publish(rewrite_tag(tag), record.merge(timestamp_hash(time)).to_json)
+        $log.debug "#{tag}, #{format_time(time)}, #{add_send_time(record)}"
+        @client.publish(rewrite_tag(tag), add_send_time(record).merge(timestamp_hash(time)).to_json)
       }
       $log.flush
 

data/lib/fluent/plugin/out_mqtt_buf.rb

@@ -11,7 +11,6 @@ module Fluent
     # Convert the event to a raw string.
     def format(tag, time, record)
       [tag, time, record].to_msgpack
-      #[tag, time, record].to_json + "\n"
     end
 
     # This method is called every flush interval. Write the buffer chunk
@@ -24,19 +23,14 @@ module Fluent
     def write(chunk)
       messages = {}
       chunk.msgpack_each do |tag, time, record|
-        #$log.debug "Thread ID: #{Thread.current.object_id}, tag: #{tag}, time: #{format_time(time)}, record: #{record}"
         messages[tag] = [] if messages[tag].nil?
-        messages[tag] << record.merge(timestamp_hash(time))
+        messages[tag] << add_send_time(record).merge(timestamp_hash(time))
       end
       messages.keys.each do |tag|
         $log.debug "Thread ID: #{Thread.current.object_id}, topic: #{rewrite_tag(tag)}, message: #{messages[tag]}"
         @client.publish(rewrite_tag(tag), messages[tag].map {|m| m.to_json}.join(@bulk_trans_sep))
       end
       $log.flush
-      #json = json_parse(chunk.open {|io| io.readline})
-      #$log.debug "#{json[0]}, #{format_time(json[1])}, #{json[2]}"
-      #@client.publish(rewrite_tag(json[0]), (json[2].merge(timestamp_hash(json[1]))).to_json)
-      #$log.flush
     end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-mqtt-io
 version: !ruby/object:Gem::Version
-  version: 0.2.1
+  version: 0.2.2
 platform: ruby
 authors:
 - Toyokazu Akiyama
 autorequire: 
 bindir: []
 cert_chain: []
-date: 2016-01-04 00:00:00.000000000 Z
+date: 2016-01-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -75,7 +75,6 @@ extra_rdoc_files: []
 files:
 - ".gitignore"
 - ".travis.yml"
-- CODE_OF_CONDUCT.md
 - Gemfile
 - LICENSE
 - README.md

data/CODE_OF_CONDUCT.md

@@ -1,13 +0,0 @@
-# Contributor Code of Conduct
-
-As contributors and maintainers of this project, we pledge to respect all people who contribute through reporting issues, posting feature requests, updating documentation, submitting pull requests or patches, and other activities.
-
-We are committed to making participation in this project a harassment-free experience for everyone, regardless of level of experience, gender, gender identity and expression, sexual orientation, disability, personal appearance, body size, race, ethnicity, age, or religion.
-
-Examples of unacceptable behavior by participants include the use of sexual language or imagery, derogatory comments or personal attacks, trolling, public or private harassment, insults, or other unprofessional conduct.
-
-Project maintainers have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct. Project maintainers who do not follow the Code of Conduct may be removed from the project team.
-
-Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by opening an issue or contacting one or more of the project maintainers.
-
-This Code of Conduct is adapted from the [Contributor Covenant](http://contributor-covenant.org), version 1.0.0, available at [http://contributor-covenant.org/version/1/0/0/](http://contributor-covenant.org/version/1/0/0/)