fluent-plugin-mqtt 0.0.8 → 0.0.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 19e8964bfff6ad942926e4e1607b91a099f8a1f9
-  data.tar.gz: dcf3560aca07e73b59eb1ff319620977dc63ac6b
+  metadata.gz: 87146ea70041d0d56d3a6c9db7f6cf8a1096bd9a
+  data.tar.gz: fe84c1296c4e329dbca1e794e0e3fd177a415666
 SHA512:
-  metadata.gz: a86199765c2bdcd8309c2e79c0ea4a073a443acef5a55574ee8fc951d8974c45116f0fdaf5ae94d6c72c6477bb565746be676de52f6ca8ec44f9836cd382b378
-  data.tar.gz: b3d1b82362c57f3860338946b44f5af6419a133173caee3fc8b0fbf6f2978aa8f5b1a8189bc8f912797911d9bc5b1bdf9588a400a9dd50169fae9496b0ac3f4f
+  metadata.gz: 4d4f98038168c1a48cd50911ee8e6f3ff617ffa66a5cef78ce0db37c13c13d378527025eee7d17715f5e1356ec3526bfef6737a66fe691e0ab6347ecc98122c0
+  data.tar.gz: 841606034f3c2b74a22c4e8e8c812d29571987b6ab57efe004b811959ed41f946809d03f8e0c609eb1fd8b5118a3bdab370584be37fd57dfaf833aac35602b09

data/fluent-plugin-mqtt.gemspec

@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name          = "fluent-plugin-mqtt"
-  spec.version       = "0.0.8"
+  spec.version       = "0.0.9"
   spec.authors       = ["Yuuna Kurita"]
   spec.email         = ["yuuna.m@gmail.com"]
   spec.summary       = %q{fluentd input plugin for mqtt server}

data/lib/fluent/plugin/out_mqtt.rb

@@ -1,14 +1,18 @@
-module Fluent
-  class OutMqtt < BufferedOutput
-    Plugin.register_output('mqtt', self)
+require 'mqtt'
+require 'msgpack'
+require 'fluent/plugin/output'
+
+module Fluent::Plugin
+  class OutMqtt < Output
+    Fluent::Plugin.register_output('mqtt', self)
+
+    helpers :compat_parameters, :inject, :formatter
+
+    DEFAULT_BUFFER_TYPE = "memory"
 
-    include Fluent::SetTagKeyMixin
     config_set_default :include_tag_key, false
-    
-    include Fluent::SetTimeKeyMixin
     config_set_default :include_time_key, true
 
-    
     config_param :port, :integer, :default => 1883
     config_param :bind, :string, :default => '127.0.0.1'
     config_param :topic, :string, :default => 'td-agent'
@@ -22,15 +26,19 @@ module Fluent
     config_param :cert, :string, :default => nil
     config_param :retain, :string, :default => true
 
-    require 'mqtt'
+    config_section :buffer do
+      config_set_default :@type, DEFAULT_BUFFER_TYPE
+      config_set_default :chunk_keys, ['tag']
+    end
 
-    unless method_defined?(:log)
-      define_method(:log) { $log }
+    config_section :inject do
+      config_set_default :time_key, "time"
+      config_set_default :time_type, "string"
+      config_set_default :time_format, "%Y-%m-%dT%H:%M:%S%z"
     end
 
     def initialize
       super
-      require 'msgpack'
 
       @clients = {}
       @connection_options = {}
@@ -38,19 +46,21 @@ module Fluent
     end
 
     def configure(conf)
+      compat_parameters_convert(conf, :buffer, :inject, :formatter)
       super
       @bind ||= conf['bind']
       @topic ||= conf['topic']
       @port ||= conf['port']
-      @formatter = Plugin.new_formatter(@format)
-      @formatter.configure(conf)
+      @formatter = formatter_create
+      if conf.has_key?('buffer_chunk_limit')
+        #check buffer_size
+        conf['buffer_chunk_limit'] = available_buffer_chunk_limit(conf)
+      end
     end
 
     def start
-      #check buffer_size
-      @buffer.buffer_chunk_limit = available_buffer_chunk_limit
 
-      $log.debug "start mqtt #{@bind}"
+      log.debug "start mqtt #{@bind}"
       opts = {host: @bind,
               port: @port}
       opts[:client_id] = @client_id if @client_id
@@ -70,11 +80,21 @@ module Fluent
     end
 
     def format(tag, time, record)
-      [tag, time, record].to_msgpack
+      [time, record].to_msgpack
+    end
+
+    def formatted_to_msgpack_binary
+      true
+    end
+
+    def multi_workers_ready?
+      true
     end
 
     def write(chunk)
-      chunk.msgpack_each { |tag, time, record|
+      tag = chunk.metadata.tag
+      chunk.msgpack_each { |time, record|
+        record = inject_values_to_record(tag, time, record)
         log.debug "write #{@topic} #{@formatter.format(tag,time,record)}"
         @connect.publish(@topic, @formatter.format(tag,time,record), retain=@retain)
       }
@@ -84,12 +104,12 @@ module Fluent
     # Following limits are heuristic. BSON is sometimes bigger than MessagePack and JSON.
     LIMIT_MQTT = 2 * 1024  # 2048kb
 
-    def available_buffer_chunk_limit
-      if @buffer.buffer_chunk_limit > LIMIT_MQTT
-        log.warn ":buffer_chunk_limit(#{@buffer.buffer_chunk_limit}) is large. Reset :buffer_chunk_limit with #{LIMIT_MQTT}"
+    def available_buffer_chunk_limit(conf)
+      if conf['buffer_chunk_limit'] > LIMIT_MQTT
+        log.warn ":buffer_chunk_limit(#{conf['buffer_chunk_limit']}) is large. Reset :buffer_chunk_limit with #{LIMIT_MQTT}"
         LIMIT_MQTT
       else
-        @buffer.buffer_chunk_limit
+        conf['buffer_chunk_limit']
       end
     end
   end

data/test/helper.rb

@@ -24,6 +24,9 @@ unless ENV.has_key?('VERBOSE')
 end
 
 require 'fluent/plugin/in_mqtt'
+require 'fluent/plugin/out_mqtt'
 
 class Test::Unit::TestCase
 end
+
+include Fluent::Test::Helpers

data/test/plugin/test_out_mqtt.rb

@@ -0,0 +1,118 @@
+require_relative '../helper'
+require 'fluent/test/driver/output'
+
+class MqttOutputTest < Test::Unit::TestCase
+  def setup
+    Fluent::Test.setup
+  end
+
+  CONFIG = %[ bind 127.0.0.1
+              port 1883
+              format json ]
+
+  def create_driver(conf = CONFIG)
+    Fluent::Test::Driver::Output.new(Fluent::Plugin::OutMqtt).configure(conf)
+  end
+
+  def sub_client(topic = "td-agent/#")
+    connect = MQTT::Client.connect("localhost")
+    connect.subscribe(topic)
+    return connect
+  end
+
+  def test_configure_1
+    d = create_driver(
+      %[ bind 127.0.0.1
+         port 1883
+         format json]
+    )
+    assert_equal '127.0.0.1', d.instance.bind
+    assert_equal 1883, d.instance.port
+    assert_equal 'json', d.instance.instance_variable_get(:@format)
+
+    d2 = create_driver(
+      %[ bind 127.0.0.1
+         port 1883
+         format csv
+         fields time,message
+         time_key time]
+    )
+    assert_equal 'csv', d2.instance.instance_variable_get(:@format)
+    assert_equal 'time', d2.instance.inject_config.time_key
+  end
+
+  class TestWithTimeZone < self
+    def setup
+      @timeZone = ENV['TZ']
+    end
+    def teardown
+      ENV['TZ'] = @timeZone
+    end
+
+    def test_format_csv
+      ENV['TZ'] = 'Asia/Tokyo'
+
+      d = create_driver(
+        %[ bind 127.0.0.1
+           port 1883
+           format csv
+           time_type string
+           time_format %Y-%m-%dT%H:%M:%S%z
+           fields time,message]
+      )
+
+      client = sub_client
+      time = event_time("2011-01-02 13:14:15 UTC")
+      data = [
+        {tag: "tag1", message: "#{time},hello world" },
+        {tag: "tag2", message: "#{time},hello to you to" },
+        {tag: "tag3", message: "#{time}," },
+      ]
+
+      d.run(default_tag: "test") do
+        data.each do |record|
+          d.feed(time, record)
+        end
+      end
+      3.times do |i|
+        record = client.get
+        assert_equal "td-agent", record[0]
+        assert_equal "\"2011-01-02T22:14:15+0900\",\"#{data[i][:message]}\"\n", record[1]
+      end
+    end
+
+    def test_format_json
+      ENV['TZ'] = 'Asia/Tokyo'
+
+      d = create_driver(
+        %[ bind 127.0.0.1
+           port 1883
+           format json
+           time_type string
+           time_format %Y-%m-%dT%H:%M:%S%z
+           fields time,message]
+      )
+
+      client = sub_client
+      time = event_time("2011-01-02 13:14:15 UTC")
+      data = [
+        {tag: "tag1", message: "#{time},hello world" },
+        {tag: "tag2", message: "#{time},hello to you to" },
+        {tag: "tag3", message: "#{time}," },
+      ]
+
+      d.run(default_tag: "test") do
+        data.each do |record|
+          d.feed(time, record)
+        end
+      end
+      3.times do |i|
+        record = client.get
+        assert_equal "td-agent", record[0]
+        assert_equal "{\"tag\":\"#{data[i][:tag]}\",\
+\"message\":\"#{data[i][:message]}\",\
+\"time\":\"2011-01-02T22:14:15+0900\"}\n", record[1]
+      end
+    end
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-mqtt
 version: !ruby/object:Gem::Version
-  version: 0.0.8
+  version: 0.0.9
 platform: ruby
 authors:
 - Yuuna Kurita
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-04-18 00:00:00.000000000 Z
+date: 2017-05-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: mqtt
@@ -111,6 +111,7 @@ files:
 - lib/fluent/plugin/out_mqtt.rb
 - test/helper.rb
 - test/plugin/test_in_mqtt.rb
+- test/plugin/test_out_mqtt.rb
 homepage: http://github.com/yuuna/fluent-plugin-mqtt
 licenses:
 - MIT
@@ -131,10 +132,11 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.5.1
+rubygems_version: 2.6.11
 signing_key: 
 specification_version: 4
 summary: fluentd input plugin for mqtt server
 test_files:
 - test/helper.rb
 - test/plugin/test_in_mqtt.rb
+- test/plugin/test_out_mqtt.rb