fluent-plugin-mqtt-io 0.4.0 → 0.4.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 4fc0cec014b6bf816dd2d0b1772b88a1a070eacd
-  data.tar.gz: ea134724065a4c5bb99f7e3ee8fdaab0ca7a05b9
+  metadata.gz: 910f2e9e505f0b9ae26ac26f29357a849b2e11df
+  data.tar.gz: ede83bec8dac5beda12524e0ba2d939460c664da
 SHA512:
-  metadata.gz: e33d620814d9c7f5284163f8f22985fc583c2be740cc1f2dad6693540f72396b51c46fe7675232ce7a89418e47bdfaafb3ff0c5e39d6c8ab8394af1c52a19bda
-  data.tar.gz: e7c78a5f2b9b8da3c9a46345b0ae1ae40244e6911f1c89b9b0f1278f09eb833290be60c013d49f03b03820560b96da552b1fceebd57ffbe80f8b39710dc7f3a3
+  metadata.gz: 8f40b5f402574cf81877f0bfd70a9d9331f724e0b3b667b9404f9d71e4cf732c3321200b219f1922e9cae835543f2a8c2cec55c3e2b33018f376e5277a09773d
+  data.tar.gz: 81e7227607e0ed5d95fc33b4e4015a5c1e323813a6c8604259cbcf578a74b7c7f89879d872e0e39d86062031c8c67540edcef84be8a2d4808f4f99e2d0255f24

data/CHANGELOG.md

@@ -1,3 +1,8 @@
+## 0.4.0 (2018-01-05)
+Features:
+  - change fluentd dependency from ~> 0.14 to >= 0.14
+
+
 ## 0.3.0 (2017-03-20)
 
 Features:

data/README-old.md

@@ -0,0 +1,332 @@
+# Fluent::Plugin::Mqtt::IO
+
+Fluent plugin for MQTT Input/Output.
+Mqtt::IO Plugin is deeply inspired by Fluent::Plugin::Mqtt.
+
+https://github.com/yuuna/fluent-plugin-mqtt
+
+Mqtt::IO plugin focus on federating components, e.g. sensors, messaging platform and databases. Encryption/Decryption is not supported in this plugin but [fluent-plugin-jwt-filter](https://github.com/toyokazu/fluent-plugin-jwt-filter) can be used to encrypt/decrypt messages using JSON Web Token technology.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'fluent-plugin-mqtt-io'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install fluent-plugin-mqtt-io
+
+
+## Usage
+
+fluent-plugin-mqtt-io provides Input and Output Plugins for MQTT.
+
+### Input Plugin (Fluet::MqttInput)
+
+Input Plugin can be used via source directive in the configuration.
+
+For fluent-plugin-mqtt-io <= 0.2.3
+
+```
+
+<source>
+  type mqtt
+  host 127.0.0.1
+  port 1883
+  format json
+</source>
+
+```
+
+For fluent-plugin-mqtt-io >= v0.3.0
+
+```
+
+<source>
+  @type mqtt
+  host 127.0.0.1
+  port 1883
+  <parse>
+    @type json
+  </parse>
+</source>
+
+```
+
+
+The default MQTT topic is "#". Configurable options are the following:
+
+- **host**: IP address of MQTT broker
+- **port**: Port number of MQTT broker
+- **client_id**: Client ID that to connect to MQTT broker
+- **format** (mandatory): Input parser can be chosen, e.g. json, xml
+  - In order to use xml format, you need to install [fluent-plugin-xml-parser](https://github.com/toyokazu/fluent-plugin-xml-parser).
+  - Default time_key field for json format is 'time'
+- **topic**: Topic name to be subscribed
+- **bulk_trans**: Enable bulk transfer to support buffered output (mqtt_buf, Fluent::MqttBufferedOutput, defaut: true) only for fluent-plugin-mqtt-io <= 0.2.3
+- **bulk_trans_sep**: A message separator for bulk transfer. The default separator is "\t". only for fluent-plugin-mqtt-io <= 0.2.3
+- **username**: User name for authentication
+- **password**: Password for authentication
+- **keep_alive**: An interval of sending keep alive packet (default 15 sec)
+- **ssl**: set true if you want to use SSL/TLS. If set to true, the following parameter must be provided
+  - **ca_file**: CA certificate file path
+  - **key_file**: private key file path
+  - **cert_file**: certificate file path
+- **recv_time**: Add receive time to message in millisecond (ms) as integer for debug and performance/delay analysis. only for fluent-plugin-mqtt-io <= 0.2.3
+- **recv_time_key**: An attribute of recv_time. only for fluent-plugin-mqtt-io <= 0.2.3
+- **monitor**: monitor section. only for fluent-plugin-mqtt-io >= 0.3.0
+  - **recv_time**: Add receive time to message in millisecond (ms) as integer for debug and performance/delay analysis (default: false). only for fluent-plugin-mqtt-io >= 0.3.0
+  - **recv_time_key**: An attribute of recv_time (default: "recv_time"). only for fluent-plugin-mqtt-io >= 0.3.0
+  - **time_type**: Type of time format (string (default), unixtime, float) only for fluent-plugin-mqtt-io >= 0.3.0
+  - **time_format**: Time format e.g. %FT%T.%N%:z (refer strftime) only for fluent-plugin-mqtt-io >= 0.3.0
+- **initial_interval**: An initial value of retry interval (s) (default 1)
+- **retry_inc_ratio**: An increase ratio of retry interval per connection failure (default 2 (double)). It may be better to set the value to 1 in a mobile environment for eager reconnection.
+- **max_retry_interval**: Maximum value of retry interval (default 300) only for fluent-plugin-mqtt-io >= 0.3.0
+
+Input Plugin supports @label directive.
+
+### Output Plugin (Fluent::MqttOutput, Fluent::MqttBufferedOutput)
+
+Output Plugin can be used via match directive.
+
+For fluent-plugin-mqtt-io <= 0.2.3
+
+```
+
+<match topic.**>
+  type mqtt
+  host 127.0.0.1
+  port 1883
+</match>
+
+```
+
+For fluent-plugin-mqtt-io >= v0.3.0
+
+```
+
+<match topic.**>
+  @type mqtt
+  host 127.0.0.1
+  port 1883
+  <format>
+    @type json
+    add_newline false
+  </format>
+</match>
+
+```
+
+The options are basically the same as Input Plugin except for "format" and "bulk_trans" (only for Input). Additional options for Output Plugin are the following.
+
+- **time_key**: An attribute name used for timestamp field genarated from fluentd time field. Default is nil (omitted).
+  If this option is omitted, timestamp field is not appended to the output record.
+- **time_format**: Output format of timestamp field. Default is ISO8601. You can specify your own format by using TimeParser.
+- **topic_rewrite_pattern**: Regexp pattern to extract replacement words from received topic or tag name
+- **topic_rewrite_replacement**: Topic name used for the publish using extracted pattern
+- **send_time**: Add send time to message in millisecond (ms) as integer for debug and performance/delay analysis. only for fluent-plugin-mqtt-io <= 0.2.3
+- **send_time_key**: An attribute of send_time. only for fluent-plugin-mqtt-io <= 0.2.3
+- **monitor**: monitor section. only for fluent-plugin-mqtt-io >= 0.3.0
+  - **send_time**: Add send time to message in millisecond (ms) as integer for debug and performance/delay analysis. only for fluent-plugin-mqtt-io >= 0.3.0
+  - **send_time_key**: An attribute of send_time. only for fluent-plugin-mqtt-io >= 0.3.0
+  - **time_type**: Type of time format (string (default), unixtime, float) only for fluent-plugin-mqtt-io >= 0.3.0
+  - **time_format**: Time format e.g. %FT%T.%N%:z (refer strftime) only for fluent-plugin-mqtt-io >= 0.3.0
+
+If you use different source, e.g. the other MQTT broker, log file and so on, there is no need to specifie topic rewriting. Skip the following descriptions.
+
+The topic name or tag name, e.g. "topic", received from an event can not be published without modification because if MQTT input plugin connecting to the identical MQTT broker is used as a source, the same message will become an input repeatedly. In order to support data conversion in single MQTT domain, simple topic rewriting should be supported. Since topic is rewritten using #gsub method, 'pattern' and 'replacement' are the same as #gsub arguments.
+
+For fluent-plugin-mqtt-io <= v0.2.3
+
+```
+
+<match topic.**>
+  type mqtt
+  host 127.0.0.1
+  port 1883
+  topic_rewrite_pattern '^([\w\/]+)$'
+  topic_rewrite_replacement '\1/rewritten'
+</match>
+
+```
+
+For fluent-plugin-mqtt-io >= v0.3.0
+
+```
+
+<match topic.**>
+  @type mqtt
+  host 127.0.0.1
+  port 1883
+  <format>
+    @type json
+    add_newline false
+  </format>
+  topic_rewrite_pattern '^([\w\/]+)$'
+  topic_rewrite_replacement '\1/rewritten'
+</match>
+
+```
+
+You can also use mqtt_buf type which is implemented as Fluent::MqttBufferedOutput.
+
+```
+
+<match topic.**>
+  type mqtt_buf
+  host 127.0.0.1
+  port 1883
+  topic_rewrite_pattern '^([\w\/]+)$'
+  topic_rewrite_replacement '\1/rewritten'
+  # You can specify Buffer Plugin options
+  buffer_type memory
+  flush_interval 1s
+</match>
+
+```
+
+For fluent-plugin-mqtt-io >= v0.3.0
+
+```
+
+<match topic.**>
+  @type mqtt
+  host 127.0.0.1
+  port 1883
+  <format>
+    @type json
+  </format>
+  topic_rewrite_pattern '^([\w\/]+)$'
+  topic_rewrite_replacement '\1/rewritten'
+  # You can specify Buffer Plugin options
+  <buffer>
+    buffer_type memory
+    flush_interval 1s
+  </buffer>
+</match>
+
+```
+
+When a plugin implemented as Fluent::BufferedOutput, fluentd stores the received messages into buffers (Fluent::MemoryBuffer is used as default) separated by tag names. Writer Threads emit those stored messages periodically in the specified interval. In MqttBufferedOutput, the stored messages are concatenated with 'bulk_trans_sep' (default: "\t"). This function reduces the number of messages sent via MQTT if data producing interval of sensors are smaller than publish interval (flush_interval). The concatinated messages can be properly handled by Fluent::MqttInput plugin by specifying 'bulk_trans' option.
+
+## Use case examples
+
+### Sensor data collection (not for Libelium now)
+
+*additional description (2015-12-25)*: Thanks to the updates by Libelium, Meshlium farmware now supports flexible output format configuration including JSON. As a result, data conversion by XmlParser is not required for this use case. The following description is kept just for sharing know-how of data conversion in fluentd.
+
+There are many kinds of commercial sensor products on the market, e.g. [Libelium](http://www.libelium.com/). Major sensor products support MQTT to upload sensor data. The following example shows how to store uploaded Libelium sensor data into ElasticSearch.
+
+![Libelium sensor data collection](https://github.com/toyokazu/fluent-plugin-mqtt-io/blob/master/images/libelium_sensor_data_collection.png "Libelium sensor data")
+
+As described in the figure, fluent-plugin-mqtt-io, fluent-plugin-xml-parser and fluent-plugin-elasticsearch are used. The following is an example configuration.
+
+```
+<source>
+  type mqtt
+  host 192.168.1.100
+  port 1883
+  topic 'Libelium/+/+'
+  format xml
+  time_xpath '["cap:alert/cap:info/cap:onset", "text"]'
+  time_key '@timestamp'
+  attr_xpaths '[["cap:alert/cap:info/cap:parameter/cap:valueName", "text"]]'
+  value_xpaths '[["cap:alert/cap:info/cap:parameter/cap:value", "text"]]'
+  @label @MQTT_OUT
+</source>
+
+<label @MQTT_OUT>
+  <match **>
+    <store>
+      type elasticsearch
+      host localhost
+      port 9200
+      index_name libelium
+      type_name smartcity
+      include_tag_key true
+      tag_key sensor_id
+      logstash_format false
+    </store>
+  </match>
+</label>
+
+```
+
+The following mapping is assumed to be created at ElasticSearch.
+
+```
+
+curl -XPUT 'http://localhost:9200/libelium/_mapping/smartcity' -d '
+  {"smartcity":
+    {"properties":
+      {
+        "sensor_id":{"type": "string"},
+        "DUST":{"type": "float"},
+        "MCP":{"type": "float"},
+        "HUMA":{"type": "float"},
+        "TCA":{"type": "float"},
+        "BAT":{"type": "float"},
+        "@timestamp":{"type":"date","format":"dateOptionalTime"}
+      }
+    }
+  }'
+
+```
+
+### MQTT message conversion
+
+Sometimes, MQTT message conversion must be done in the network because the processing entities does not have the conversion function. In that case, the configuration similar to the above example can be used. The difference resides output configuration. In this example, since the same MQTT broker is used to upload converted data, topic rewriting function is used for separating messages before and after conversion.
+
+![MQTT message conversion](https://github.com/toyokazu/fluent-plugin-mqtt-io/blob/master/images/mqtt_message_conversion.png "MQTT message conversion")
+
+```
+<source>
+  type mqtt
+  host 192.168.1.100
+  port 1883
+  topic 'Libelium/+/+'
+  format xml
+  time_xpath '["cap:alert/cap:info/cap:onset", "text"]'
+  time_key '@timestamp'
+  attr_xpaths '[["cap:alert/cap:info/cap:parameter/cap:valueName", "text"]]'
+  value_xpaths '[["cap:alert/cap:info/cap:parameter/cap:value", "text"]]'
+  @label @MQTT_OUT
+</source>
+
+<label @MQTT_OUT>
+  <match **>
+    type mqtt
+    host 192.168.1.100
+    port 1883
+    topic_rewrite_pattern '^([\w\/]+)$'
+    topic_rewrite_replacement '\1/rewritten'
+  </match>
+</label>
+
+```
+
+
+### Sensor data uploads from tiny computers, e.g. Raspberry Pi, Edison, etc
+
+MQTT output plugin can be used as the following. If you have tiny computers like Raspberry Pi equipped with sensors and their data are outputted as files, you can use fluent-plugin-mqtt-io for uploading those data.
+
+![Sensor data uploads from tiny computers](https://github.com/toyokazu/fluent-plugin-mqtt-io/blob/master/images/sensor_data_uploads_from_tiny_computers.png "Sensor data uploads from tiny computers")
+
+
+## Contributing
+
+1. Fork it ( http://github.com/toyokazu/fluent-plugin-mqtt-io/fork )
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request
+
+
+## License
+
+The gem is available as open source under the terms of the [Apache License Version 2.0](https://www.apache.org/licenses/LICENSE-2.0).

data/README.md

@@ -30,23 +30,7 @@ fluent-plugin-mqtt-io provides Input and Output Plugins for MQTT.
 
 Input Plugin can be used via source directive in the configuration.
 
-For fluent-plugin-mqtt-io <= 0.2.3
-
-```
-
-<source>
-  type mqtt
-  host 127.0.0.1
-  port 1883
-  format json
-</source>
-
-```
-
-For fluent-plugin-mqtt-io ~> v0.3.0
-
 ```
-
 <source>
   @type mqtt
   host 127.0.0.1
@@ -58,35 +42,46 @@ For fluent-plugin-mqtt-io ~> v0.3.0
 
 ```
 
+When using security options, specify them in security section; for example:
+
+```
+<match>
+   @type mqtt
+   host 'your_host'
+   port 'your_port'
+   <security>
+     username 'your_username'
+     password 'your_password'
+   </security>
+   <format>
+     @type json
+   </format>
+</match>
+```
 
 The default MQTT topic is "#". Configurable options are the following:
 
 - **host**: IP address of MQTT broker
 - **port**: Port number of MQTT broker
 - **client_id**: Client ID that to connect to MQTT broker
-- **format** (mandatory): Input parser can be chosen, e.g. json, xml
-  - In order to use xml format, you need to install [fluent-plugin-xml-parser](https://github.com/toyokazu/fluent-plugin-xml-parser).
-  - Default time_key field for json format is 'time'
+- **parser**: Parser plugin can be specified ![Parser Plugin](https://docs.fluentd.org/v1.0/articles/parser-plugin-overview)
 - **topic**: Topic name to be subscribed
-- **bulk_trans**: Enable bulk transfer to support buffered output (mqtt_buf, Fluent::MqttBufferedOutput, defaut: true) only for fluent-plugin-mqtt-io <= 0.2.3
-- **bulk_trans_sep**: A message separator for bulk transfer. The default separator is "\t". only for fluent-plugin-mqtt-io <= 0.2.3
-- **username**: User name for authentication
-- **password**: Password for authentication
+- **security**
+  - **username**: User name for authentication
+  - **password**: Password for authentication
+  - **use_tls**: set true if you want to use SSL/TLS. If set to true, the following parameter must be provided
+    - **ca_file**: CA certificate file path
+    - **key_file**: private key file path
+    - **cert_file**: certificate file path
 - **keep_alive**: An interval of sending keep alive packet (default 15 sec)
-- **ssl**: set true if you want to use SSL/TLS. If set to true, the following parameter must be provided
-  - **ca_file**: CA certificate file path
-  - **key_file**: private key file path
-  - **cert_file**: certificate file path
-- **recv_time**: Add receive time to message in millisecond (ms) as integer for debug and performance/delay analysis. only for fluent-plugin-mqtt-io <= 0.2.3
-- **recv_time_key**: An attribute of recv_time. only for fluent-plugin-mqtt-io <= 0.2.3
-- **monitor**: monitor section. only for fluent-plugin-mqtt-io ~> 0.3.0
-  - **recv_time**: Add receive time to message in millisecond (ms) as integer for debug and performance/delay analysis (default: false). only for fluent-plugin-mqtt-io ~> 0.3.0
-  - **recv_time_key**: An attribute of recv_time (default: "recv_time"). only for fluent-plugin-mqtt-io ~> 0.3.0
-  - **time_type**: Type of time format (string (default), unixtime, float) only for fluent-plugin-mqtt-io ~> 0.3.0
-  - **time_format**: Time format e.g. %FT%T.%N%:z (refer strftime) only for fluent-plugin-mqtt-io ~> 0.3.0
+- **monitor**: monitor section. only for fluent-plugin-mqtt-io
+  - **recv_time**: Add receive time to message in millisecond (ms) as integer for debug and performance/delay analysis (default: false). only for fluent-plugin-mqtt-io
+  - **recv_time_key**: An attribute of recv_time (default: "recv_time"). only for fluent-plugin-mqtt-io
+  - **time_type**: Type of time format (string (default), unixtime, float) only for fluent-plugin-mqtt-io
+  - **time_format**: Time format e.g. %FT%T.%N%:z (refer strftime) only for fluent-plugin-mqtt-io
 - **initial_interval**: An initial value of retry interval (s) (default 1)
 - **retry_inc_ratio**: An increase ratio of retry interval per connection failure (default 2 (double)). It may be better to set the value to 1 in a mobile environment for eager reconnection.
-- **max_retry_interval**: Maximum value of retry interval (default 300) only for fluent-plugin-mqtt-io ~> 0.3.0
+- **max_retry_interval**: Maximum value of retry interval (default 300)
 
 Input Plugin supports @label directive.
 
@@ -94,22 +89,7 @@ Input Plugin supports @label directive.
 
 Output Plugin can be used via match directive.
 
-For fluent-plugin-mqtt-io <= 0.2.3
-
 ```
-
-<match topic.**>
-  type mqtt
-  host 127.0.0.1
-  port 1883
-</match>
-
-```
-
-For fluent-plugin-mqtt-io ~> v0.3.0
-
-```
-
 <match topic.**>
   @type mqtt
   host 127.0.0.1
@@ -119,46 +99,31 @@ For fluent-plugin-mqtt-io ~> v0.3.0
     add_newline false
   </format>
 </match>
-
 ```
 
-The options are basically the same as Input Plugin except for "format" and "bulk_trans" (only for Input). Additional options for Output Plugin are the following.
+The options are basically the same as Input Plugin except for "parser (for Input)/format (for Output)". Additional options for Output Plugin are the following.
 
+- **qos**: Quality of Service (QoS) for message publishing, 0 or 1 is valid. 2 is not supported by mqtt client. Default is 1.
+- **retain**: If set true the broker will keep the message even after sending it to all current subscribers. Default is false
 - **time_key**: An attribute name used for timestamp field genarated from fluentd time field. Default is nil (omitted).
   If this option is omitted, timestamp field is not appended to the output record.
-- **time_format**: Output format of timestamp field. Default is ISO8601. You can specify your own format by using TimeParser.
 - **topic_rewrite_pattern**: Regexp pattern to extract replacement words from received topic or tag name
 - **topic_rewrite_replacement**: Topic name used for the publish using extracted pattern
-- **send_time**: Add send time to message in millisecond (ms) as integer for debug and performance/delay analysis. only for fluent-plugin-mqtt-io <= 0.2.3
-- **send_time_key**: An attribute of send_time. only for fluent-plugin-mqtt-io <= 0.2.3
-- **monitor**: monitor section. only for fluent-plugin-mqtt-io ~> 0.3.0
-  - **send_time**: Add send time to message in millisecond (ms) as integer for debug and performance/delay analysis. only for fluent-plugin-mqtt-io ~> 0.3.0
-  - **send_time_key**: An attribute of send_time. only for fluent-plugin-mqtt-io ~> 0.3.0
-  - **time_type**: Type of time format (string (default), unixtime, float) only for fluent-plugin-mqtt-io ~> 0.3.0
-  - **time_format**: Time format e.g. %FT%T.%N%:z (refer strftime) only for fluent-plugin-mqtt-io ~> 0.3.0
+- **format**: Formatter plugin can be specified. ![Formatter Plugin](https://docs.fluentd.org/v1.0/articles/formatter-plugin-overview)
+- **monitor**: monitor section. only for fluent-plugin-mqtt-io
+  - **send_time**: Add send time to message in millisecond (ms) as integer for debug and performance/delay analysis. only for fluent-plugin-mqtt-io
+  - **send_time_key**: An attribute of send_time. only for fluent-plugin-mqtt-io
+  - **time_type**: Type of time format (string (default), unixtime, float) only for fluent-plugin-mqtt-io
+  - **time_format**: Time format e.g. %FT%T.%N%:z (refer strftime) only for fluent-plugin-mqtt-io
+- **buffer**: synchronous/asynchronous buffer is supported. Refer ![Buffer section configurations](https://docs.fluentd.org/v1.0/articles/buffer-section) for detailed configuration.
+  - **async**: Enable asynchronous buffer transfer. Default is false.
 
 If you use different source, e.g. the other MQTT broker, log file and so on, there is no need to specifie topic rewriting. Skip the following descriptions.
 
 The topic name or tag name, e.g. "topic", received from an event can not be published without modification because if MQTT input plugin connecting to the identical MQTT broker is used as a source, the same message will become an input repeatedly. In order to support data conversion in single MQTT domain, simple topic rewriting should be supported. Since topic is rewritten using #gsub method, 'pattern' and 'replacement' are the same as #gsub arguments.
 
-For fluent-plugin-mqtt-io <= v0.2.3
 
 ```
-
-<match topic.**>
-  type mqtt
-  host 127.0.0.1
-  port 1883
-  topic_rewrite_pattern '^([\w\/]+)$'
-  topic_rewrite_replacement '\1/rewritten'
-</match>
-
-```
-
-For fluent-plugin-mqtt-io ~> v0.3.0
-
-```
-
 <match topic.**>
   @type mqtt
   host 127.0.0.1
@@ -170,30 +135,10 @@ For fluent-plugin-mqtt-io ~> v0.3.0
   topic_rewrite_pattern '^([\w\/]+)$'
   topic_rewrite_replacement '\1/rewritten'
 </match>
-
-```
-
-You can also use mqtt_buf type which is implemented as Fluent::MqttBufferedOutput.
-
-```
-
-<match topic.**>
-  type mqtt_buf
-  host 127.0.0.1
-  port 1883
-  topic_rewrite_pattern '^([\w\/]+)$'
-  topic_rewrite_replacement '\1/rewritten'
-  # You can specify Buffer Plugin options
-  buffer_type memory
-  flush_interval 1s
-</match>
-
 ```
 
-For fluent-plugin-mqtt-io ~> v0.3.0
 
 ```
-
 <match topic.**>
   @type mqtt
   host 127.0.0.1
@@ -209,113 +154,7 @@ For fluent-plugin-mqtt-io ~> v0.3.0
     flush_interval 1s
   </buffer>
 </match>
-
-```
-
-When a plugin implemented as Fluent::BufferedOutput, fluentd stores the received messages into buffers (Fluent::MemoryBuffer is used as default) separated by tag names. Writer Threads emit those stored messages periodically in the specified interval. In MqttBufferedOutput, the stored messages are concatenated with 'bulk_trans_sep' (default: "\t"). This function reduces the number of messages sent via MQTT if data producing interval of sensors are smaller than publish interval (flush_interval). The concatinated messages can be properly handled by Fluent::MqttInput plugin by specifying 'bulk_trans' option.
-
-## Use case examples
-
-### Sensor data collection (not for Libelium now)
-
-*additional description (2015-12-25)*: Thanks to the updates by Libelium, Meshlium farmware now supports flexible output format configuration including JSON. As a result, data conversion by XmlParser is not required for this use case. The following description is kept just for sharing know-how of data conversion in fluentd.
-
-There are many kinds of commercial sensor products on the market, e.g. [Libelium](http://www.libelium.com/). Major sensor products support MQTT to upload sensor data. The following example shows how to store uploaded Libelium sensor data into ElasticSearch.
-
-![Libelium sensor data collection](https://github.com/toyokazu/fluent-plugin-mqtt-io/blob/master/images/libelium_sensor_data_collection.png "Libelium sensor data")
-
-As described in the figure, fluent-plugin-mqtt-io, fluent-plugin-xml-parser and fluent-plugin-elasticsearch are used. The following is an example configuration.
-
 ```
-<source>
-  type mqtt
-  host 192.168.1.100
-  port 1883
-  topic 'Libelium/+/+'
-  format xml
-  time_xpath '["cap:alert/cap:info/cap:onset", "text"]'
-  time_key '@timestamp'
-  attr_xpaths '[["cap:alert/cap:info/cap:parameter/cap:valueName", "text"]]'
-  value_xpaths '[["cap:alert/cap:info/cap:parameter/cap:value", "text"]]'
-  @label @MQTT_OUT
-</source>
-
-<label @MQTT_OUT>
-  <match **>
-    <store>
-      type elasticsearch
-      host localhost
-      port 9200
-      index_name libelium
-      type_name smartcity
-      include_tag_key true
-      tag_key sensor_id
-      logstash_format false
-    </store>
-  </match>
-</label>
-
-```
-
-The following mapping is assumed to be created at ElasticSearch.
-
-```
-
-curl -XPUT 'http://localhost:9200/libelium/_mapping/smartcity' -d '
-  {"smartcity":
-    {"properties":
-      {
-        "sensor_id":{"type": "string"},
-        "DUST":{"type": "float"},
-        "MCP":{"type": "float"},
-        "HUMA":{"type": "float"},
-        "TCA":{"type": "float"},
-        "BAT":{"type": "float"},
-        "@timestamp":{"type":"date","format":"dateOptionalTime"}
-      }
-    }
-  }'
-
-```
-
-### MQTT message conversion
-
-Sometimes, MQTT message conversion must be done in the network because the processing entities does not have the conversion function. In that case, the configuration similar to the above example can be used. The difference resides output configuration. In this example, since the same MQTT broker is used to upload converted data, topic rewriting function is used for separating messages before and after conversion.
-
-![MQTT message conversion](https://github.com/toyokazu/fluent-plugin-mqtt-io/blob/master/images/mqtt_message_conversion.png "MQTT message conversion")
-
-```
-<source>
-  type mqtt
-  host 192.168.1.100
-  port 1883
-  topic 'Libelium/+/+'
-  format xml
-  time_xpath '["cap:alert/cap:info/cap:onset", "text"]'
-  time_key '@timestamp'
-  attr_xpaths '[["cap:alert/cap:info/cap:parameter/cap:valueName", "text"]]'
-  value_xpaths '[["cap:alert/cap:info/cap:parameter/cap:value", "text"]]'
-  @label @MQTT_OUT
-</source>
-
-<label @MQTT_OUT>
-  <match **>
-    type mqtt
-    host 192.168.1.100
-    port 1883
-    topic_rewrite_pattern '^([\w\/]+)$'
-    topic_rewrite_replacement '\1/rewritten'
-  </match>
-</label>
-
-```
-
-
-### Sensor data uploads from tiny computers, e.g. Raspberry Pi, Edison, etc
-
-MQTT output plugin can be used as the following. If you have tiny computers like Raspberry Pi equipped with sensors and their data are outputted as files, you can use fluent-plugin-mqtt-io for uploading those data.
-
-![Sensor data uploads from tiny computers](https://github.com/toyokazu/fluent-plugin-mqtt-io/blob/master/images/sensor_data_uploads_from_tiny_computers.png "Sensor data uploads from tiny computers")
 
 
 ## Contributing

data/fluent-plugin-mqtt-io.gemspec

@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name          = "fluent-plugin-mqtt-io"
-  spec.version       = "0.4.0"
+  spec.version       = "0.4.1"
   spec.authors       = ["Toyokazu Akiyama"]
   spec.email         = ["toyokazu@gmail.com"]
 

data/lib/fluent/plugin/in_mqtt.rb

@@ -66,13 +66,20 @@ module Fluent::Plugin
       :in_mqtt
     end
 
+    def kill_thread
+      @get_thread.kill if !@get_thread.nil?
+    end
+
     def after_disconnection
+      kill_thread
+      super
     end
 
     def after_connection
       if @client.connected?
         @client.subscribe(@topic)
-        @get_thread = thread_create(:in_mqtt_get) do
+        #@get_thread = thread_create(:in_mqtt_get) do
+        @get_thread = Thread.new do
           @client.get do |topic, message|
             emit(topic, message)
           end

data/lib/fluent/plugin/mqtt_proxy.rb

@@ -92,13 +92,18 @@ module Fluent::Plugin
       @client.disconnect if @client.connected?
     end
 
+    def kill_connect_thread
+      @connect_thread.kill if !@connect_thread.nil?
+    end
+
     def after_disconnection
       # should be implemented
+      kill_connect_thread
     end
 
-    def rescue_disconnection(*block)
+    def rescue_disconnection
       begin
-        yield *block
+        yield
       rescue MQTT::ProtocolException => e
         # TODO:
         # Currently MQTT::ProtocolException cannot be caught during @client.get
@@ -110,6 +115,12 @@ module Fluent::Plugin
         retry_connect(e, "System call error occurs.")
       rescue StandardError=> e
         retry_connect(e, "The other error occurs.")
+      rescue MQTT::NotConnectedException=> e
+        # Since MQTT::NotConnectedException is raised only on publish,
+        # connection error should be catched before this error.
+        # So, reconnection process is omitted for this Exception
+        # to prevent waistful increment of retry interval.
+        log.error "MQTT not connected exception occurs.,#{e.class},#{e.message}"
       end
     end
 
@@ -120,7 +131,8 @@ module Fluent::Plugin
     end
 
     def connect
-      thread_create("#{current_plugin_name}_monitor".to_sym) do
+      #@connect_thread = thread_create("#{current_plugin_name}_monitor".to_sym) do
+      @connect_thread = Thread.new do
         rescue_disconnection do
           @client.connect
           log.debug "connected to mqtt broker #{@host}:#{@port} for #{current_plugin_name}"

data/lib/fluent/plugin/out_mqtt.rb

@@ -18,6 +18,11 @@ module Fluent::Plugin
     desc 'Topic rewrite replacement string.'
     config_param :topic_rewrite_replacement, :string, default: nil
 
+    desc 'Retain option which publishing'
+    config_param :retain, :bool, default: false
+    desc 'QoS option which publishing'
+    config_param :qos, :integer, default: 1
+
     config_section :format do
       desc 'The format to publish'
       config_param :@type, :string, default: 'single_value'
@@ -36,6 +41,11 @@ module Fluent::Plugin
       config_param :time_format, :string, default: nil
     end
 
+    config_section :buffer, required: false, multi: false do
+      desc 'Prefer asynchronous buffering'
+      config_param :async, :bool, default: false
+    end
+
     # This method is called before starting.
     # 'conf' is a Hash that includes configuration parameters.
     # If the configuration is invalid, raise Fluent::ConfigError.
@@ -64,6 +74,10 @@ module Fluent::Plugin
       @has_buffer_section
     end
 
+    def prefer_delayed_commit
+      @has_buffer_section && @buffer_config.async
+    end
+
     # This method is called when starting.
     # Open sockets or files here.
     def start
@@ -75,16 +89,27 @@ module Fluent::Plugin
     # Shutdown the thread and close sockets or files here.
     def shutdown
       shutdown_proxy
+      kill_thread
       super
     end
 
+    def kill_thread
+      @dummy_thread.kill if !@dummy_thread.nil?
+    end
+
     def after_connection
-      @dummy_thread = thread_create(:out_mqtt_dummy) do
+      #@dummy_thread = thread_create(:out_mqtt_dummy) do
+      @dummy_thread = Thread.new do
         Thread.stop
       end
       @dummy_thread
     end
 
+    def after_disconnection
+      kill_thread
+      super
+    end
+
     def current_plugin_name
       :out_mqtt
     end
@@ -99,23 +124,11 @@ module Fluent::Plugin
     end
 
     def publish_event_stream(tag, es)
-      if es.class == Fluent::OneEventStream
-        es = inject_values_to_event_stream(tag, es)
-        es.each do |time, record|
-          log.debug "MqttOutput#publish_event_stream: #{rewrite_tag(tag)}, #{time}, #{add_send_time(record)}"
-          rescue_disconnection do
-            @client.publish(rewrite_tag(tag), @formatter.format(tag, time, add_send_time(record)))
-          end
-        end
-      else
-        es = inject_values_to_event_stream(tag, es)
-        array = []
-        es.each do |time, record|
-          log.debug "MqttOutput#publish_event_stream: #{rewrite_tag(tag)}, #{time}, #{add_send_time(record)}"
-          array << add_send_time(record)
-        end
+      log.debug "publish_event_stream: #{es.class}"
+      es = inject_values_to_event_stream(tag, es)
+      es.each do |time, record|
         rescue_disconnection do
-          @client.publish(rewrite_tag(tag), @formatter.format(tag, Fluent::EventTime.now, array))
+          publish(tag, time, record)
         end
       end
       log.flush
@@ -134,13 +147,32 @@ module Fluent::Plugin
       true
     end
 
+    def publish(tag, time, record)
+      log.debug "MqttOutput::#{caller_locations(1,1)[0].label}: #{rewrite_tag(rewrite_tag(tag))}, #{time}, #{add_send_time(record)}"
+      @client.publish(
+        rewrite_tag(tag),
+        @formatter.format(tag, time, add_send_time(record)),
+        @retain,
+        @qos
+      )
+    end
+
     def write(chunk)
       return if chunk.empty?
       chunk.each do |tag, time, record|
         rescue_disconnection do
-          log.debug "MqttOutput#write: #{rewrite_tag(rewrite_tag(tag))}, #{time}, #{add_send_time(record)}"
-          @client.publish(rewrite_tag(tag), @formatter.format(tag, time, add_send_time(record)))
+          publish(tag, time, record)
+        end
+      end
+    end
+
+    def try_write(chunk)
+      return if chunk.empty?
+      rescue_disconnection do
+        chunk.each do |tag, time, record|
+          publish(tag, time, record)
         end
+        commit_write(chunk.unique_id)
       end
     end
   end

data/test/plugin/test_in_mqtt.rb

@@ -21,10 +21,9 @@ class MqttInputTest < Test::Unit::TestCase
           host 127.0.0.1
           port 1300
           client_id aa-bb-cc-dd
-          <parse>
-            @type json
-            time_format %FT%T%:z
-          </parse>
+          <monitor>
+            recv_time true
+          </monitor>
           <security>
             use_tls true
             <tls>
@@ -38,6 +37,10 @@ class MqttInputTest < Test::Unit::TestCase
       assert_equal '127.0.0.1', d.instance.host
       assert_equal 1300, d.instance.port
       assert_equal 'aa-bb-cc-dd', d.instance.client_id
+      
+      assert_equal 'none', d.instance.parser_configs.first[:@type]
+
+      assert_equal true, d.instance.monitor.recv_time
 
       assert_equal true, d.instance.security.use_tls
       assert_equal '/cert/cacert.pem', d.instance.security.tls.ca_file

data/test/plugin/test_out_mqtt.rb

@@ -21,9 +21,11 @@ class MqttOutputTest < Test::Unit::TestCase
         host 127.0.0.1
         port 1300
         client_id aa-bb-cc-dd
-        <format>
-          @type json
-        </format>
+        retain true
+        qos 2
+        <buffer>
+          async true
+        </buffer>
         <monitor>
           send_time true
         </monitor>
@@ -38,8 +40,15 @@ class MqttOutputTest < Test::Unit::TestCase
       ]
       assert_equal '127.0.0.1', d.instance.host
       assert_equal 1300, d.instance.port
-      assert_equal true, d.instance.monitor.send_time
       assert_equal 'aa-bb-cc-dd', d.instance.client_id
+      assert_equal true, d.instance.retain
+      assert_equal 2, d.instance.qos
+
+      assert_equal true, d.instance.buffer_config.async
+
+      assert_equal true, d.instance.monitor.send_time
+
+      # cannot test formatter configuration (default: single_value)
 
       assert_equal true, d.instance.security.use_tls
       assert_equal '/cert/cacert.pem', d.instance.security.tls.ca_file

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-mqtt-io
 version: !ruby/object:Gem::Version
-  version: 0.4.0
+  version: 0.4.1
 platform: ruby
 authors:
 - Toyokazu Akiyama
 autorequire: 
 bindir: []
 cert_chain: []
-date: 2018-01-05 00:00:00.000000000 Z
+date: 2018-06-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -98,6 +98,7 @@ files:
 - CHANGELOG.md
 - Gemfile
 - LICENSE
+- README-old.md
 - README.md
 - Rakefile
 - fluent-plugin-mqtt-io.gemspec