fluent-plugin-mqtt 0.0.3 → 0.0.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: d4e453c77b000aeef76dbe664b9fb67d63a6e532
-  data.tar.gz: f22a3f8159fdfc1bc4571619406534292a082652
+  metadata.gz: 58d93e842a532198c75c7520cedb5417db36252a
+  data.tar.gz: b12b3618be94456a9307fc4d816e0e44b0255858
 SHA512:
-  metadata.gz: d38f63aaabf190906296242b259ad742c9f43d659abda789dda94007d34271caa68852a2976626207c1285ca8d060a7682effe8251977bdb0ca301d958e9ec56
-  data.tar.gz: 972613c5cd9484810ba8757577748781a9659404dba58459e572183a22986dc682e39482b45b85ed6266737b2bcfef1cb66ce726881765cbf4d54076cdbb3cf1
+  metadata.gz: e65f421d83d3b588430f378b5b87d18af278162ec42b21fb0d53a4ccbc8aa2d34b1f7f0c88f3acbaf0d3a8b2d62f983e54ae6e5a8ebd234fc96a44d2c388f4b5
+  data.tar.gz: f3e3113d8cadafe1d4347f0f8bc41d982a18fd1375df7aea0f7059beaf41e9d89f8baa571074f2511359c087b8dcbc5a8d1215ae91a631c9f8d9f8d0cedf7dc2

data/README.md

@@ -27,6 +27,8 @@ MQTT topic is set "#".
   type mqtt
   bind 127.0.0.1
   port 1883
+  username username
+  password password
 </source>
 
 ```

data/fluent-plugin-mqtt.gemspec

@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name          = "fluent-plugin-mqtt"
-  spec.version       = "0.0.3"
+  spec.version       = "0.0.4"
   spec.authors       = ["Yuuna Kurita"]
   spec.email         = ["yuuna.m@gmail.com"]
   spec.summary       = %q{fluentd input plugin for mqtt server}
@@ -17,12 +17,10 @@ Gem::Specification.new do |spec|
   spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
   spec.require_paths = ["lib"]
 
-  spec.add_development_dependency "bundler", "~> 1.5"
-  spec.add_development_dependency "rake"
-  spec.add_development_dependency "mqtt"
-  spec.add_development_dependency "fluentd"
-  spec.add_runtime_dependency "mqtt"
+  spec.add_runtime_dependency "mqtt", "~> 0.3.1"
   spec.add_runtime_dependency "fluentd"
   spec.add_runtime_dependency "yajl-ruby"
-
+  spec.add_runtime_dependency "test-unit"
+  spec.add_development_dependency "bundler", "~> 1.5"
+  spec.add_development_dependency "rake"
 end

data/lib/fluent/plugin/in_mqtt.rb

@@ -11,6 +11,13 @@ module Fluent
     config_param :port, :integer, :default => 1883
     config_param :bind, :string, :default => '127.0.0.1'
     config_param :topic, :string, :default => '#'
+    config_param :format, :string, :default => 'none'
+    config_param :username, :string, :default => nil
+    config_param :password, :string, :default => nil
+    config_param :ssl, :bool, :default => nil
+    config_param :ca, :string, :default => nil
+    config_param :key, :string, :default => nil
+    config_param :cert, :string, :default => nil
 
     require 'mqtt'
 
@@ -19,23 +26,48 @@ module Fluent
       @bind ||= conf['bind']
       @topic ||= conf['topic']
       @port ||= conf['port']
+
+      configure_parser(conf)
+    end
+
+    def configure_parser(conf)
+      @parser = Plugin.new_parser(@format)
+      @parser.configure(conf)
+    end
+
+    # Return [time (if not available return now), message]
+    def parse(message)
+      return @parser.parse(message)[1], @parser.parse(message)[0] || Fluent::Engine.now
     end
 
     def start
       $log.debug "start mqtt #{@bind}"
-      @connect = MQTT::Client.connect({remote_host: @bind, remote_port: @port})
+      opts = {host: @bind,
+              port: @port,
+              username: @username,
+              password: @password}
+      opts[:ssl] = @ssl if @ssl
+      opts[:ca_file] = @ca if @ca
+      opts[:crt_file] = @crt if @crt
+      opts[:key_file] = @key if @key
+      @connect = MQTT::Client.connect(opts)
       @connect.subscribe(@topic)
 
       @thread = Thread.new do
         @connect.get do |topic,message|
           topic.gsub!("/","\.")
           $log.debug "#{topic}: #{message}"
-          emit topic, json_parse(message)
+          begin
+            parsed_message = self.parse(message)
+          rescue Exception => e
+            $log.error e
+          end
+          emit topic, parsed_message[0], parsed_message[1]
         end
       end
     end
 
-    def emit topic, message , time = Fluent::Engine.now
+    def emit topic, message, time = Fluent::Engine.now
       if message.class == Array
         message.each do |data|
           $log.debug "#{topic}: #{data}"
@@ -46,15 +78,6 @@ module Fluent
       end
     end
 
-    def json_parse message
-      begin
-        y = Yajl::Parser.new
-        y.parse(message)
-      rescue
-        $log.error "JSON parse error", :error => $!.to_s, :error_class => $!.class.to_s
-        $log.warn_backtrace $!.backtrace         
-      end
-    end
     def shutdown
       @thread.kill
       @connect.disconnect

data/test/plugin/test_in_mqtt.rb

@@ -1,16 +1,16 @@
 require 'helper'
 
-class Fluent::MqttInput 
-  def emit topic, message , time = Fluent::Engine.now
-    if message.class == Array
-      message.each do |data|
-        $log.debug "#{topic}: #{data}"
-        Fluent::Engine.emit(topic, message["t"], data)
-      end
-    else
-      Fluent::Engine.emit(topic, message["t"], message)
-    end
-  end
+class Fluent::MqttInput
+  #def emit topic, message , time = Fluent::Engine.now
+    #if message.class == Array
+      #message.each do |data|
+        #$log.debug "#{topic}: #{data}"
+        #Fluent::Engine.emit(topic, message["t"], data)
+      #end
+    #else
+      #Fluent::Engine.emit(topic, message["t"], message)
+    #end
+  #end
 
 end
 
@@ -19,50 +19,221 @@ class MqttInputTest < Test::Unit::TestCase
     Fluent::Test.setup
   end
 
+ CONFIG = %[ bind 127.0.0.1
+             port 1883
+             format json ]
 
- CONFIG = %[
-  ]
-
-  def create_driver(conf = CONFIG) 
+  def create_driver(conf = CONFIG)
     Fluent::Test::InputTestDriver.new(Fluent::MqttInput).configure(conf)
   end
-  
-  def test_configure
+
+  def test_configure_1
     d = create_driver(
       %[ bind 127.0.0.1
-         port 1300 ] 
+         port 1883
+         format none]
     )
     assert_equal '127.0.0.1', d.instance.bind
-    assert_equal 1300, d.instance.port
+    assert_equal 1883, d.instance.port
+    assert_equal 'none', d.instance.format
+
+    d2 = create_driver(
+      %[ bind 127.0.0.1
+         port 1883
+         format csv
+         keys time,message
+         time_key time]
+    )
+    assert_equal 'csv', d2.instance.format
+    assert_equal 'time', d2.instance.time_key
   end
 
+  def test_configure_2
+    d = create_driver(
+      %[ bind 127.0.0.1
+         port 1883
+         format csv
+         keys time,message ]
+    )
+    assert_equal '127.0.0.1', d.instance.bind
+    assert_equal 1883, d.instance.port
+    assert_equal 'csv', d.instance.format
+  end
 
   def sub_client
-    connect = MQTT::Client.connect
+    connect = MQTT::Client.connect("localhost")
     connect.subscribe('#')
     return connect
   end
 
+  def test_format_json_without_time_key
+    d = create_driver(
+      %[ bind 127.0.0.1
+         port 1883
+         format json ]
+    )
+    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+    data = [
+      {tag: "tag1", message: {"t" => time, "v" => {"a"=>1}}},
+      {tag: "tag2", message: {"t" => time, "v" => {"a"=>1}}},
+      {tag: "tag3", message: {"t" => time, "v" => {"a"=>32}}},
+    ]
+
+    d.run do
+      data.each do |record|
+        send_data record[:tag], record[:message], d.instance.format
+        sleep 0.1
+      end
+    end
+
+    emits = d.emits
+    assert_equal('tag1', emits[0][0])
+    assert_equal({"t" => time, "v" => {"a"=>1}}, emits[0][2])
+
+    assert_equal('tag2', emits[1][0])
+    assert_equal({"t" => time, "v" => {"a"=>1}}, emits[1][2])
+
+    assert_equal('tag3', emits[2][0])
+    assert_equal({"t" => time, "v" => {"a"=>32}}, emits[2][2])
+  end
+
+  def test_format_json_with_time_key
+    d = create_driver(
+      %[ bind 127.0.0.1
+         port 1883
+         format json
+         time_key t ]
+    )
+    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+    data = [
+      {tag: "tag1", message: {"t" => time, "v" => {"a"=>1}}},
+      {tag: "tag2", message: {"t" => time, "v" => {"a"=>1}}},
+      {tag: "tag3", message: {"t" => time, "v" => {"a"=>31}}},
+      {tag: "tag3", message: {"t" => time, "v" => {"a"=>32}}},
+    ]
+
+    d.run do
+      data.each do |record|
+        send_data record[:tag], record[:message], d.instance.format
+      end
+    end
+
+    emits = d.emits
+    assert_equal('tag1', emits[0][0])
+    assert_equal(time, emits[0][1])
+    assert_equal({"v" => {"a"=>1}}, emits[0][2])
+  end
+
+  def test_format_none
+    d = create_driver(
+      %[ bind 127.0.0.1
+         port 1883
+         format none]
+    )
 
-  def test_client
-    d = create_driver
-    time = Time.parse("2011-01-02 13:14:15 UTC").to_i    
-    d.expect_emit "tag1", time, {"t" => time, "v" => {"a"=>1}}
-    d.expect_emit "tag2", time, {"t" => time, "v" => {"a"=>2}}
-    d.expect_emit "tag3", time, {"t" => time, "v" => {"a"=>31}}
-    d.expect_emit "tag3", time, {"t" => time, "v" => {"a"=>32}}
+    data = [
+      {tag: "tag1", message: 'hello world'},
+      {tag: "tag2", message: 'another world'},
+      {tag: "tag3", message: ''},
+    ]
 
     d.run do
-      d.expected_emits.each {|tag,time,record|
-        send_data tag, time, record
-      }
-      send_data "tag3", time , [{"t" => time, "v" => {"a"=>31}} , {"t" => time, "v" => {"a"=>32}}] 
-      sleep 0.5
+      data.each do |record|
+        send_data record[:tag], record[:message], d.instance.format
+      end
     end
 
+    emits = d.emits
+    time = Fluent::Engine.now
+    assert_equal('tag1', emits[0][0])
+    assert_equal({'message' => 'hello world'}, emits[0][2])
+    assert_equal('tag2', emits[1][0])
+    assert_equal({'message' => 'another world'}, emits[1][2])
+    assert_equal('tag3', emits[2][0])
+    assert_equal({'message' => ''}, emits[2][2])
   end
 
-  def send_data tag, time, record
-    sub_client.publish(tag, record.to_json)
+  def test_format_csv
+    d = create_driver(
+      %[ bind 127.0.0.1
+         port 1883
+         format csv
+         keys time,message]
+    )
+
+    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+    data = [
+      {tag: "tag1", message: "#{time},hello world" },
+      {tag: "tag2", message: "#{time},hello to you to" },
+      {tag: "tag3", message: "#{time}," },
+    ]
+
+    d.run do
+      data.each do |record|
+        send_data record[:tag], record[:message], d.instance.format
+      end
+    end
+
+    emits = d.emits
+    #puts 'emits length', emits.length.to_s
+    assert_equal('tag1', emits[0][0])
+    assert_equal({'time' => time.to_s, 'message' => 'hello world'}, emits[0][2])
+
+    assert_equal('tag2', emits[1][0])
+    assert_equal({'time' => time.to_s, 'message' => 'hello to you to'}, emits[1][2])
+    assert_equal('tag3', emits[2][0])
+    assert_equal({'time' => time.to_s, 'message' => nil}, emits[2][2])
+  end
+
+  def test_format_csv_with_time_key
+    d = create_driver(
+      %[ bind 127.0.0.1
+         port 1883
+         format csv
+         keys time2,message
+         time_key time2
+         time_format %S]
+    )
+
+    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+    data = [
+      {tag: "tag1", message: "#{time},abc" },
+      {tag: "tag2", message: "#{time},def" },
+      {tag: "tag3", message: "#{time},ghi" },
+      {tag: "tag3", message: "#{time}," },
+    ]
+
+    d.run do
+      data.each do |record|
+        send_data record[:tag], record[:message], d.instance.format
+      end
+    end
+
+    emits = d.emits
+    assert_equal('tag1', emits[0][0])
+    assert_equal({'message' => 'abc'}, emits[0][2])
+
+    assert_equal('tag2', emits[1][0])
+    assert_equal({'message' => 'def'}, emits[1][2])
+
+    assert_equal('tag3', emits[2][0])
+    assert_equal({'message' => 'ghi'}, emits[2][2])
+
+    assert_equal('tag3', emits[3][0])
+    assert_equal({'message' => nil}, emits[3][2])
+  end
+
+  def send_data tag, record, format
+    case format
+      when 'none'
+        sub_client.publish(tag, record)
+      when 'json'
+        sub_client.publish(tag, record.to_json)
+      when 'csv'
+        sub_client.publish(tag, record)
+      else
+        sub_client.publish(tag, record)
+    end
+    sleep 0.2
   end
 end

metadata

@@ -1,111 +1,97 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-mqtt
 version: !ruby/object:Gem::Version
-  version: 0.0.3
+  version: 0.0.4
 platform: ruby
 authors:
 - Yuuna Kurita
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-04-17 00:00:00.000000000 Z
+date: 2015-10-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: bundler
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ~>
-      - !ruby/object:Gem::Version
-        version: '1.5'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ~>
-      - !ruby/object:Gem::Version
-        version: '1.5'
-- !ruby/object:Gem::Dependency
-  name: rake
+  name: mqtt
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
+        version: 0.3.1
+  type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.3.1
 - !ruby/object:Gem::Dependency
-  name: mqtt
+  name: fluentd
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  type: :development
+  type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: fluentd
+  name: yajl-ruby
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  type: :development
+  type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: mqtt
+  name: test-unit
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: fluentd
+  name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
+        version: '1.5'
+  type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '1.5'
 - !ruby/object:Gem::Dependency
-  name: yajl-ruby
+  name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  type: :runtime
+  type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 description: fluentd input plugin for mqtt server
@@ -115,7 +101,7 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- .gitignore
+- ".gitignore"
 - Gemfile
 - LICENSE.txt
 - README.md
@@ -134,17 +120,17 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.0.6
+rubygems_version: 2.2.2
 signing_key: 
 specification_version: 4
 summary: fluentd input plugin for mqtt server