fluent-plugin-zmq-pub 0.0.2 → 0.0.4

data/README.rdoc

@@ -8,6 +8,18 @@ Fluentd plugin to publish records to ZeroMQ.
 
 Sometimes I wanted to 'sniff' fluentd stream -- running my own programs to the stream without changing fluentd configuration and restarting fluentd. With this plugin, fluentd records are always published to ZeroMQ regardless of the existance of subscriber. After that I can start and stop my subcriber programs at any time. 
 
+== Dependence
+
+This plugin use ffi-rzmq to support ZMQ, and need v3.2 or greater version of ZMQ library installed in your system.
+
+== Installation
+
+You need to install ZeroMQ libraries before installing this plugin
+
+  ## (RedHat/CentOS)
+  # yum install zeromq3 zeromq3-devel
+  # fluent-gem install fluent-plugin-zmq-pub
+
 == Configuration
 
   <match zmq.**>
@@ -15,6 +27,7 @@ Sometimes I wanted to 'sniff' fluentd stream -- running my own programs to the s
     pubkey ${tag}:${key1}
     bindaddr tcp://*:5556
     flush_interval 1s
+    bulk_send true
   </match>
 
 * 'pubkey' specifies the publish key to ZeroMQ. 
@@ -22,6 +35,7 @@ Sometimes I wanted to 'sniff' fluentd stream -- running my own programs to the s
   * Actual record to be published is '<pubkey> <reocord.to_msgpack>'.
   * Subscriber can subscribe by '<pubkey>'.
 * 'bindaddr' is the address to which ZeroMQ publisher socket to be bound.
+* If 'bulk_send' is set to true, send multiple records with the same publish key in one 'send_string' method. This improves the performance.
 
 == Example usage
 
@@ -29,7 +43,7 @@ Put the configuration above to fluentd.conf, and save this sample code as 'sampl
 
   #!/usr/bin/env ruby
   
-  require 'zmq'
+  require 'ffi-rzmq'
   require 'msgpack'
   
   context = ZMQ::Context.new(1)
@@ -45,11 +59,13 @@ Put the configuration above to fluentd.conf, and save this sample code as 'sampl
   end
   
   while true
-    if msg = subscriber.recv(ZMQ::NOBLOCK)
-      (tag, time, record) =  MessagePack.unpack(msg.split(" ",2)[1])
-      puts "tag: #{tag}"
-      puts "time: #{time}"
-      puts "record: #{record.to_s}"
+    msg = ''
+    while subscriber.recv_string(msg,ZMQ::DONTWAIT) && msg.size > 0
+      record =  MessagePack.unpack(msg.split(" ",2)[1])
+      puts "tag: #{record[0]}"
+      puts "time: #{record[1]}"
+      puts "record: #{record[2]}"
+      msg = ''
     end
     sleep(0.1)
   end
@@ -73,6 +89,21 @@ Then you will get the following output from sample_sub.rb
 
 The nice thing is that once you put this plugin to your fluentd.conf and start fluentd, you can start and stop any subscriber programs without changing fluentd configuration.
 
+== zmq_sub input plugin
+
+Input plugin to subscribe the output of zmq_pub is also included. Here is the example configuration.
+
+    <source>
+       type zmq_sub
+       publisher tcp://127.0.0.1:5556
+       bulk_send true
+       subkey zmq.,zmq2
+    </source>
+
+* If zmq_pub set `bulk_send` to true, zmq_sub also set it to true.
+* `subkey` is a comma separated list of keys to subscribe. In this example, keys starting "zmq." or "zmq2." will be subscribed.
+
+
 == Copyright
 
 * Copyright (c) 2013- OGIBAYASHI Hironori (@angostura11)

data/fluent-plugin-zmq-pub.gemspec

@@ -4,8 +4,8 @@ $:.push File.expand_path('../lib', __FILE__)
 Gem::Specification.new do |gem|
   gem.authors       = ["OGIBAYASHI Hironori"]
   gem.email         = ["ogibayashi@gmail.com"]
-  gem.description   = %q{0MQ publisher plugin for fluentd}
-  gem.summary       = %q{0MQ publisher plugin for fluentd}
+  gem.description   = %q{0MQ publisher/subscriber plugin for fluentd}
+  gem.summary       = %q{0MQ publisher/subscriber plugin for fluentd, use 0MQ v3.2 or greater version}
   gem.homepage      = ""
   gem.licenses	    = ["Apache License, Version 2.0"]
 
@@ -14,9 +14,9 @@ Gem::Specification.new do |gem|
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.name          = "fluent-plugin-zmq-pub"
   gem.require_paths = ["lib"]
-  gem.version       = "0.0.2"
+  gem.version       = "0.0.4"
   gem.add_development_dependency "fluentd"
   gem.add_runtime_dependency "fluentd"
-  gem.add_development_dependency "zmq"
-  gem.add_runtime_dependency "zmq"
+  gem.add_development_dependency "ffi-rzmq"
+  gem.add_runtime_dependency "ffi-rzmq"
 end

data/lib/fluent/plugin/in_zmq_sub.rb

@@ -0,0 +1,87 @@
+module Fluent
+
+  class ZmqSubInput < Fluent::Input
+    Fluent::Plugin.register_input('zmq_sub', self)
+
+    config_param :subkey, :string, :default => ""
+    config_param :publisher, :string, :default => "tcp://127.0.0.1:5556"
+    config_param :bulk_send, :bool, :default => false
+
+    attr_reader :subkeys
+
+    def initialize
+      super
+      require 'ffi-rzmq'
+    end
+
+    def configure(conf)
+      super
+      @subkeys = @subkey.split(",")
+    end
+
+    def start
+      super
+      @context =ZMQ::Context.new()
+      @thread = Thread.new(&method(:run))
+    end
+
+    def shutdown
+      super
+      Thread.kill(@thread)
+      @thread.join
+      @context.terminate
+    end
+
+    def run
+      begin
+        @subscriber = @context.socket(ZMQ::SUB)
+        @subscriber.connect(@publisher)
+        if @subkeys.size > 0
+          @subkeys.each do |k|
+            @subscriber.setsockopt(ZMQ::SUBSCRIBE,k)
+          end
+        else
+          @subscriber.setsockopt(ZMQ::SUBSCRIBE,'')
+        end
+        loop do
+          msg = ''
+          while @subscriber.recv_string(msg,ZMQ::DONTWAIT) && msg.size > 0
+            begin
+              (key, records) = msg.split(" ",2)
+              records = MessagePack.unpack(records)
+              if @bulk_send && records[0].class == Array
+                es = MultiEventStream.new
+                prev_tag = nil
+                records.each do |tag, time, record|
+                  if prev_tag && prev_tag != tag
+                    Engine.emit_stream(prev_tag, es)
+                    es = MultiEventStream.new
+                  end
+                  es.add(time, record)
+                  prev_tag = tag
+                end
+                Engine.emit_stream(prev_tag, es) if es.to_a.size > 0
+              else
+                Engine.emit(*records)
+              end
+            rescue => e
+              log.warn "Error in processing message.",:error_class => e.class, :error => e
+              log.warn_backtrace
+            end
+            msg = ''
+          end
+          sleep(0.1)
+        end
+      rescue => e
+        log.error "error occurred while executing plugin.", :error_class => e.class, :error => e
+        log.warn_backtrace
+      ensure
+        if @subscriber
+          @subscriber.close
+        end
+      end
+    end
+
+  end
+
+end

data/lib/fluent/plugin/out_zmq_pub.rb

@@ -4,10 +4,13 @@ module Fluent
 
     config_param :pubkey, :string
     config_param :bindaddr, :string, :default => 'tcp://*:5556'
+    config_param :highwatermark, :integer, :default => 1000
+    # Send multiple record with the same publish key at once
+    config_param :bulk_send, :bool, :default => false
 
     def initialize
       super
-      require 'zmq'
+      require 'ffi-rzmq'
       @mutex = Mutex.new
     end
 
@@ -17,8 +20,9 @@ module Fluent
     
     def start
       super
-      @context = ZMQ::Context.new(1)
+      @context = ZMQ::Context.new()
       @publisher = @context.socket(ZMQ::PUB)
+      @publisher.setsockopt(ZMQ::SNDHWM, @highwatermark)
       @publisher.bind(@bindaddr)
     end
 
@@ -27,6 +31,8 @@ module Fluent
     end
 
     def write(chunk)
+      records = { }
+      #  to_msgpack in format, unpack in write, then to_msgpack again... better way?
       chunk.msgpack_each{ |record|
         pubkey_replaced = @pubkey.gsub(/\${(.*?)}/){ |s|
           case $1
@@ -36,18 +42,29 @@ module Fluent
             record[2][$1]
           end
         }
-
-        #  to_msgpack in format, unpack in write, then to_msgpack again... better way?
+        if @bulk_send
+          records[pubkey_replaced] ||= []
+          records[pubkey_replaced] << record
+        else
+          @mutex.synchronize { 
+            @publisher.send_string(pubkey_replaced + " " + record.to_msgpack,ZMQ::DONTWAIT)
+          }
+        end
+      }
+      if @bulk_send
         @mutex.synchronize { 
-          @publisher.send(pubkey_replaced + " " + record.to_msgpack,ZMQ::NOBLOCK)
+          records.each{  |k,v|
+            @publisher.send_string(k + " " + v.to_msgpack,ZMQ::DONTWAIT)
+          }
         }
-      }
+      end
+        
     end
  
     def shutdown
       super
       @publisher.close
-      @context.close
+      @context.terminate
     end
 
   end

data/test/helper.rb

@@ -23,6 +23,7 @@ unless ENV.has_key?('VERBOSE')
 end
 
 require 'fluent/plugin/out_zmq_pub'
+require 'fluent/plugin/in_zmq_sub'
 
 class Test::Unit::TestCase
 end

data/test/plugin/test_in_zmq_sub.rb

@@ -0,0 +1,97 @@
+require 'helper'
+require 'ffi-rzmq'
+
+class ZmqSubIutputTest < Test::Unit::TestCase
+  def setup
+    Fluent::Test.setup
+    @context = ZMQ::Context.new()
+    @publisher = @context.socket(ZMQ::PUB)
+    @publisher.bind("tcp://*:5556")
+  end
+  
+  def teardown
+    @publisher.close
+    @context.terminate
+  end
+
+  PUBLISHER = "tcp://127.0.0.1:5556"
+  CONFIG = %[
+     publisher #{PUBLISHER}
+     subkey test1.,test2.
+  ]
+  
+  
+  def create_driver(conf=CONFIG)
+    Fluent::Test::InputTestDriver.new(Fluent::ZmqSubInput).configure(conf)
+  end
+  
+  def test_configure
+    d = create_driver(CONFIG + "bulk_send true")
+    assert_equal PUBLISHER, d.instance.publisher
+    assert_equal ["test1.","test2."], d.instance.subkeys
+    assert_equal true, d.instance.bulk_send
+  end
+  
+  def test_receive
+    d = create_driver
+    
+    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+    Fluent::Engine.now = time
+    
+    d.expect_emit "test1.aa", time, {"a"=>1}
+    d.expect_emit "test2.bb", time, {"a"=>2}
+    
+    d.run do
+      d.expected_emits.each {|tag,time,record|
+        send_record("dummy",time,record)  # This record should not be received.
+        send_record(tag,time,record)
+      }
+      sleep 1
+    end
+  end
+
+  def test_no_subkey
+    d = create_driver("")
+    
+    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+    Fluent::Engine.now = time
+    
+    d.expect_emit "test1.aa", time, {"a"=>1}
+    d.expect_emit "test2.bb", time, {"a"=>2}
+    
+    d.run do
+      d.expected_emits.each {|tag,time,record|
+        send_record(tag,time,record)
+      }
+      sleep 1
+    end
+  end
+
+
+  def test_receive_bulk
+    d = create_driver(CONFIG + "bulk_send true")
+    
+    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+    Fluent::Engine.now = time
+    
+    d.expect_emit "test3.aa", time, {"a"=>1}
+    d.expect_emit "test4.bb", time, {"a"=>2}
+    
+    d.run do
+      record_to_send = []
+      d.expected_emits.each {|tag,time,record|
+        record_to_send << [tag,time,record]
+      }
+      send_record_bulk("test1.aa",record_to_send)
+      sleep 1
+    end
+  end
+
+  def send_record(tag,time,record)
+    @publisher.send_string(tag + " " + [tag,time,record].to_msgpack)
+  end
+
+  def send_record_bulk(tag,records)
+    @publisher.send_string(tag + " " + records.to_msgpack)
+  end
+end

data/test/plugin/test_out_zeromq_pub.rb

@@ -1,5 +1,5 @@
 require 'helper'
-require 'zmq'
+require 'ffi-rzmq'
 
 class ZmqPubOutputTest < Test::Unit::TestCase
   def setup
@@ -11,13 +11,24 @@ class ZmqPubOutputTest < Test::Unit::TestCase
 
   def teardown
     @subscriber.close
-    @context.close
+    @context.terminate
   end
   
   CONFIG = %[
       pubkey ${tag}:${key1}
       bindaddr tcp://*:5556
   ]
+
+  CONFIG_BULK = %[
+      pubkey ${tag}:${key1}
+      bindaddr tcp://*:5556
+      bulk_send true
+  ]
+
+  CONFIG_BY_TAG = %[
+      pubkey ${tag}
+      bindaddr tcp://*:5556
+  ]
   
   def create_driver(conf = CONFIG, tag='test')
     Fluent::Test::BufferedOutputTestDriver.new(Fluent::ZmqPubOutput, tag).configure(conf)
@@ -50,16 +61,44 @@ class ZmqPubOutputTest < Test::Unit::TestCase
     time = Time.parse("2011-01-02 13:14:15 UTC").to_i
     d.emit({"key1"=>"aaa"}, time)
     d.emit({"key1"=>"bbb", "key2"=>3}, time)
+    d.emit({"key1"=>"aaa", "key2"=>4}, time)
     
     d.run
     sleep 1
 
-    msg = @subscriber.recv(ZMQ::NOBLOCK)
+    msg = ''
+    @subscriber.recv_string(msg,ZMQ::DONTWAIT)
     (key, record) = msg.split(" ",2)
     assert_equal ["test",time.to_i,{ "key1" => "aaa"}].to_msgpack, record
 
+    msg = ''
+    @subscriber.recv_string(msg,ZMQ::DONTWAIT)
+    (key, record) = msg.split(" ",2)
+    assert_equal ["test",time.to_i,{ "key1" => "aaa","key2" => 4 }].to_msgpack, record
+
   end
 
+  def test_write_bulk
+    d = create_driver(CONFIG_BULK)
+    @subscriber.setsockopt(ZMQ::SUBSCRIBE,"test:aaa")
+    
+    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+    d.emit({"key1"=>"aaa"}, time)
+    d.emit({"key1"=>"bbb", "key2"=>3}, time)
+    d.emit({"key1"=>"aaa", "key2"=>4}, time)
+    
+    d.run
+    sleep 1
+
+    msg = ''
+    @subscriber.recv_string(msg,ZMQ::DONTWAIT)
+    (key, record) = msg.split(" ",2)
+    assert_equal [["test",time.to_i,{ "key1" => "aaa"}],
+                  ["test",time.to_i,{ "key1" => "aaa","key2" => 4 }]].to_msgpack, record
+
+  end
+
+
 end
 
 

metadata

@@ -2,7 +2,7 @@
 name: fluent-plugin-zmq-pub
 version: !ruby/object:Gem::Version 
   prerelease: 
-  version: 0.0.2
+  version: 0.0.4
 platform: ruby
 authors: 
 - OGIBAYASHI Hironori
@@ -10,7 +10,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2014-03-27 00:00:00 Z
+date: 2014-10-09 00:00:00 Z
 dependencies: 
 - !ruby/object:Gem::Dependency 
   name: fluentd
@@ -35,7 +35,7 @@ dependencies:
   type: :runtime
   version_requirements: *id002
 - !ruby/object:Gem::Dependency 
-  name: zmq
+  name: ffi-rzmq
   prerelease: false
   requirement: &id003 !ruby/object:Gem::Requirement 
     none: false
@@ -46,7 +46,7 @@ dependencies:
   type: :development
   version_requirements: *id003
 - !ruby/object:Gem::Dependency 
-  name: zmq
+  name: ffi-rzmq
   prerelease: false
   requirement: &id004 !ruby/object:Gem::Requirement 
     none: false
@@ -56,7 +56,7 @@ dependencies:
         version: "0"
   type: :runtime
   version_requirements: *id004
-description: 0MQ publisher plugin for fluentd
+description: 0MQ publisher/subscriber plugin for fluentd
 email: 
 - ogibayashi@gmail.com
 executables: []
@@ -72,8 +72,10 @@ files:
 - README.rdoc
 - Rakefile
 - fluent-plugin-zmq-pub.gemspec
+- lib/fluent/plugin/in_zmq_sub.rb
 - lib/fluent/plugin/out_zmq_pub.rb
 - test/helper.rb
+- test/plugin/test_in_zmq_sub.rb
 - test/plugin/test_out_zeromq_pub.rb
 homepage: ""
 licenses: 
@@ -101,7 +103,8 @@ rubyforge_project:
 rubygems_version: 1.8.25
 signing_key: 
 specification_version: 3
-summary: 0MQ publisher plugin for fluentd
+summary: 0MQ publisher/subscriber plugin for fluentd, use 0MQ v3.2 or greater version
 test_files: 
 - test/helper.rb
+- test/plugin/test_in_zmq_sub.rb
 - test/plugin/test_out_zeromq_pub.rb