fluent-plugin-mqtt-io 0.3.5 → 0.3.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: d276b49bf5d719468bf9290820334ee0069c9cd1
-  data.tar.gz: 3256a5038f1673bd6abbc50fde3c422de8f0705d
+  metadata.gz: 6e38c839b0a95cb6cf2cebc4c54d2b6c7b40529c
+  data.tar.gz: 2070c495dd6162a84477057b41170b95880b82b7
 SHA512:
-  metadata.gz: ae2fd591fb25d976fea2f83d46c08ff97ec96469aa4af041cc71c87a7f0852ec460e6c87b23c1e047c50c8c81305c9159498914f14d5117979c7c7d309c4b971
-  data.tar.gz: d1def163b352d5a05f0e34bbed980093c32c370e9d51c3494d4578e7050c69cd36c763bc2e2c2bab41309f6309e165fbdaf13596b7cb8572b2698dcc7ce5ecdc
+  metadata.gz: 49a35a2dccbe28249c27731ac447a63d7f8462d0005486fa1736dc5f1316fa43ec46f09573ea758120b0d2c7dac8136b44f946e6cb7a3e16c39dac0b568eb5e1
+  data.tar.gz: d06614888b953bf57df4ab2110947f1ec77d2a489fe6bfcc27f0b1e3692397c140178d693aa6bb3981f807f9d32a5546cba9a8528a862c4ba8cf207a7d1d872b

data/README.md

@@ -76,8 +76,13 @@ The default MQTT topic is "#". Configurable options are the following:
   - **ca_file**: CA certificate file path
   - **key_file**: private key file path
   - **cert_file**: certificate file path
-- **recv_time**: Add receive time to message in millisecond (ms) as integer for debug and performance/delay analysis
-- **recv_time_key**: An attribute of recv_time
+- **recv_time**: Add receive time to message in millisecond (ms) as integer for debug and performance/delay analysis. only for fluent-plugin-mqtt-io <= 0.2.3
+- **recv_time_key**: An attribute of recv_time. only for fluent-plugin-mqtt-io <= 0.2.3
+- **monitor**: monitor section. only for fluent-plugin-mqtt-io ~> 0.3.0
+  - **recv_time**: Add receive time to message in millisecond (ms) as integer for debug and performance/delay analysis (default: false). only for fluent-plugin-mqtt-io ~> 0.3.0
+  - **recv_time_key**: An attribute of recv_time (default: "recv_time"). only for fluent-plugin-mqtt-io ~> 0.3.0
+  - **time_type**: Type of time format (string (default), unixtime, float) only for fluent-plugin-mqtt-io ~> 0.3.0
+  - **time_format**: Time format e.g. %FT%T.%N%:z (refer strftime) only for fluent-plugin-mqtt-io ~> 0.3.0
 - **initial_interval**: An initial value of retry interval (s) (default 1)
 - **retry_inc_ratio**: An increase ratio of retry interval per connection failure (default 2 (double)). It may be better to set the value to 1 in a mobile environment for eager reconnection.
 - **max_retry_interval**: Maximum value of retry interval (default 300) only for fluent-plugin-mqtt-io ~> 0.3.0
@@ -118,13 +123,18 @@ For fluent-plugin-mqtt-io ~> v0.3.0
 
 The options are basically the same as Input Plugin except for "format" and "bulk_trans" (only for Input). Additional options for Output Plugin are the following.
 
-- time_key: An attribute name used for timestamp field genarated from fluentd time field. Default is nil (omitted).
+- **time_key**: An attribute name used for timestamp field genarated from fluentd time field. Default is nil (omitted).
   If this option is omitted, timestamp field is not appended to the output record.
-- time_format: Output format of timestamp field. Default is ISO8601. You can specify your own format by using TimeParser.
-- topic_rewrite_pattern: Regexp pattern to extract replacement words from received topic or tag name
-- topic_rewrite_replacement: Topic name used for the publish using extracted pattern
-- send_time: Add send time to message in millisecond (ms) as integer for debug and performance/delay analysis
-- send_time_key: An attribute of recv_time
+- **time_format**: Output format of timestamp field. Default is ISO8601. You can specify your own format by using TimeParser.
+- **topic_rewrite_pattern**: Regexp pattern to extract replacement words from received topic or tag name
+- **topic_rewrite_replacement**: Topic name used for the publish using extracted pattern
+- **send_time**: Add send time to message in millisecond (ms) as integer for debug and performance/delay analysis. only for fluent-plugin-mqtt-io <= 0.2.3
+- **send_time_key**: An attribute of send_time. only for fluent-plugin-mqtt-io <= 0.2.3
+- **monitor**: monitor section. only for fluent-plugin-mqtt-io ~> 0.3.0
+  - **send_time**: Add send time to message in millisecond (ms) as integer for debug and performance/delay analysis. only for fluent-plugin-mqtt-io ~> 0.3.0
+  - **send_time_key**: An attribute of send_time. only for fluent-plugin-mqtt-io ~> 0.3.0
+  - **time_type**: Type of time format (string (default), unixtime, float) only for fluent-plugin-mqtt-io ~> 0.3.0
+  - **time_format**: Time format e.g. %FT%T.%N%:z (refer strftime) only for fluent-plugin-mqtt-io ~> 0.3.0
 
 If you use different source, e.g. the other MQTT broker, log file and so on, there is no need to specifie topic rewriting. Skip the following descriptions.
 

data/fluent-plugin-mqtt-io.gemspec

@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name          = "fluent-plugin-mqtt-io"
-  spec.version       = "0.3.5"
+  spec.version       = "0.3.6"
   spec.authors       = ["Toyokazu Akiyama"]
   spec.email         = ["toyokazu@gmail.com"]
 

data/lib/fluent/plugin/in_mqtt.rb

@@ -6,6 +6,7 @@ require 'fluent/plugin/mqtt_proxy'
 module Fluent::Plugin
   class MqttInput < Input
     include MqttProxy
+    include Fluent::TimeMixin::Formatter
 
     Fluent::Plugin.register_input('mqtt', self)
 
@@ -25,12 +26,21 @@ module Fluent::Plugin
       desc 'Record received time into message or not.'
       config_param :recv_time, :bool, default: false
       desc 'Specify the attribute name of received time.'
-      config_param :recv_time_key, :string, default: "recv_time"
+      config_param :recv_time_key, :string, default: 'recv_time'
+      desc 'Specify time type of recv_time (string, unixtime, float).'
+      config_param :time_type, :string, default: 'string'
+      desc 'Specify time format of recv_time (e.g. %FT%T.%N%:z).'
+      config_param :time_format, :string, default: nil
     end
 
     def configure(conf)
       super
       configure_parser(conf)
+      if !@monitor.nil?
+        @recv_time_formatter = time_formatter_create(
+          type: @monitor.time_type.to_sym, format: @monitor.time_format
+        )
+      end
     end
 
     def configure_parser(conf)
@@ -69,9 +79,9 @@ module Fluent::Plugin
     end
 
     def add_recv_time(record)
-      if @recv_time
+      if !@monitor.nil? && @monitor.recv_time
         # recv_time is recorded in ms
-        record.merge({"#{@recv_time_key}": Fluent::EventTime.now})
+        record.merge({"#{@monitor.recv_time_key}": @recv_time_formatter.format(Fluent::EventTime.now)})
       else
         record
       end
@@ -83,7 +93,6 @@ module Fluent::Plugin
           log.debug "Since time_key field is nil, Fluent::EventTime.now is used."
           time = Fluent::EventTime.now
         end
-        log.debug "#{topic}, #{time}, #{add_recv_time(record)}"
         return [time, add_recv_time(record)]
       end
     end
@@ -92,6 +101,7 @@ module Fluent::Plugin
       begin
         tag = topic.gsub("/","\.")
         time, record = parse(message)
+        log.debug "MqttInput#emit: #{tag}, #{time}, #{add_recv_time(record)}"
         if record.is_a?(Array)
           mes = Fluent::MultiEventStream.new
           record.each do |single_record|

data/lib/fluent/plugin/out_mqtt.rb

@@ -6,6 +6,7 @@ require 'fluent/plugin/mqtt_proxy'
 module Fluent::Plugin
   class MqttOutput < Output
     include MqttProxy
+    include Fluent::TimeMixin::Formatter
 
     Fluent::Plugin.register_output('mqtt', self)
 
@@ -21,6 +22,10 @@ module Fluent::Plugin
       config_param :send_time, :bool, default: false
       desc 'Recording key name of send time for monitoring.'
       config_param :send_time_key, :string, default: "send_time"
+      desc 'Specify time type of send_time (string, unixtime, float).'
+      config_param :time_type, :string, default: 'string'
+      desc 'Specify time format of send_time (e.g. %FT%T.%N%:z).'
+      config_param :time_format, :string, default: nil
     end
 
     # This method is called before starting.
@@ -32,6 +37,11 @@ module Fluent::Plugin
       formatter_config = conf.elements(name: 'format').first
       @formatter = formatter_create(conf: formatter_config)
       @has_buffer_section = conf.elements(name: 'buffer').size > 0
+      if !@monitor.nil?
+        @send_time_formatter = time_formatter_create(
+          type: @monitor.time_type.to_sym, format: @monitor.time_format
+        )
+      end
     end
 
     def rewrite_tag(tag)
@@ -72,9 +82,9 @@ module Fluent::Plugin
     end
 
     def add_send_time(record)
-      if @send_time
+      if !@monitor.nil? && @monitor.send_time
         # send_time is recorded in ms
-        record.merge({"#{@send_time_key}": Fluent::EventTime.now})
+        record.merge({"#{@monitor.send_time_key}": @send_time_formatter.format(Fluent::EventTime.now)})
       else
         record
       end
@@ -84,7 +94,7 @@ module Fluent::Plugin
       if es.class == Fluent::OneEventStream
         es = inject_values_to_event_stream(tag, es)
         es.each do |time, record|
-          log.debug "#{rewrite_tag(tag)}, #{time}, #{add_send_time(record)}"
+          log.debug "MqttOutput#publish_event_stream: #{rewrite_tag(tag)}, #{time}, #{add_send_time(record)}"
           rescue_disconnection do
             @client.publish(rewrite_tag(tag), @formatter.format(tag, time, add_send_time(record)))
           end
@@ -93,7 +103,7 @@ module Fluent::Plugin
         es = inject_values_to_event_stream(tag, es)
         array = []
         es.each do |time, record|
-          log.debug "#{rewrite_tag(tag)}, #{time}, #{add_send_time(record)}"
+          log.debug "MqttOutput#publish_event_stream: #{rewrite_tag(tag)}, #{time}, #{add_send_time(record)}"
           array << add_send_time(record)
         end
         rescue_disconnection do
@@ -120,7 +130,7 @@ module Fluent::Plugin
       return if chunk.empty?
       chunk.each do |tag, time, record|
         rescue_disconnection do
-          log.debug "#{rewrite_tag(rewrite_tag(tag))}, #{time}, #{add_send_time(record)}"
+          log.debug "MqttOutput#write: #{rewrite_tag(rewrite_tag(tag))}, #{time}, #{add_send_time(record)}"
           @client.publish(rewrite_tag(tag), @formatter.format(tag, time, add_send_time(record)))
         end
       end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-mqtt-io
 version: !ruby/object:Gem::Version
-  version: 0.3.5
+  version: 0.3.6
 platform: ruby
 authors:
 - Toyokazu Akiyama
 autorequire: 
 bindir: []
 cert_chain: []
-date: 2017-08-25 00:00:00.000000000 Z
+date: 2017-08-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd