fluent-plugin-zmq-pub 0.0.1

data/.gitignore

@@ -0,0 +1,17 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in fluent-plugin-zmq-pub.gemspec
+gemspec

data/LICENSE

@@ -0,0 +1,22 @@
+Copyright (c) 2012 Hironori Ogibayashi
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.rdoc

@@ -0,0 +1,80 @@
+= fluent-plugin-zmq-pub
+
+== Overview
+
+Fluentd plugin to publish records to ZeroMQ.
+
+== Why this plugin was created?
+
+Sometimes I wanted to 'sniff' fluentd stream -- running my own programs to the stream without changing fluentd configuration and restarting fluentd. With this plugin, fluentd records are always published to ZeroMQ regardless of the existance of subscriber. After that I can start and stop my subcriber programs at any time. 
+
+== Configuration
+
+  <match zmq.**>
+    type zmq_pub
+    pubkey ${tag}:${key1}
+    bindaddr tcp://*:5556
+    flush_interval 1s
+  </match>
+
+* 'pubkey' specifies the publish key to ZeroMQ. 
+  * '${tag}' is replace by fluentd tag. '${name}' is replaced by fluentd record['name']. 
+  * Actual record to be published is '<pubkey> <reocord.to_msgpack>'.
+  * Subscriber can subscribe by '<pubkey>'.
+* 'bindaddr' is the address to which ZeroMQ publisher socket to be bound.
+
+== Example usage
+
+Put the configuration above to fluentd.conf, and save this sample code as 'sample_sub.rb'.
+
+  #!/usr/bin/env ruby
+  
+  require 'zmq'
+  require 'msgpack'
+  
+  context = ZMQ::Context.new(1)
+  subscriber = context.socket(ZMQ::SUB)
+  subscriber.connect("tcp://localhost:5556")
+  
+  if ARGV.length > 0
+    ARGV.each{|s|
+      subscriber.setsockopt(ZMQ::SUBSCRIBE,s)
+    }
+  else
+    subscriber.setsockopt(ZMQ::SUBSCRIBE,"")
+  end
+  
+  while true
+    if msg = subscriber.recv(ZMQ::NOBLOCK)
+      (tag, time, record) =  MessagePack.unpack(msg.split(" ",2)[1])
+      puts "tag: #{tag}"
+      puts "time: #{time}"
+      puts "record: #{record.to_s}"
+    end
+    sleep(0.1)
+  end
+
+Run sample_sub.rb. Argument is the key to subscribe. (Correspond to  'pubkey' in zmq_pub configuration). If you give no arguments, all key will be subscribed.
+
+  % ./sample_sub.rb zmq.test.tag:aaa
+
+Submit records to fluentd.
+
+  % echo '{"key1": "aaa", "key2":"foo"}' | fluent-cat zmq.test.tag
+  % echo '{"key1": "bbb", "key2":"foo"}' | fluent-cat zmq.test.tag
+
+Then you will get the following output from sample_sub.rb
+
+  tag: zmq.test.tag
+  time: 1376033265
+  record: {"key1"=>"aaa", "key2"=>"foo"}
+
+(You should not get the second record("key1":"bbb") because the publish key to zmq was "zmq.test.tag:bbb" and specified subscibe key was "zmq.test.tag:aaa")
+
+The nice thing is that once you put this plugin to your fluentd.conf and start fluentd, you can start and stop any subscriber programs without changing fluentd configuration.
+
+== Copyright
+
+* Copyright (c) 2013- OGIBAYASHI Hironori (@angostura11)
+* License
+  * Apache License, Version 2.0

data/Rakefile

@@ -0,0 +1,11 @@
+#!/usr/bin/env rake
+require "bundler/gem_tasks"
+
+require 'rake/testtask'
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.pattern = 'test/**/test_*.rb'
+  test.verbose = true
+end
+
+task :default => :test

data/fluent-plugin-zmq-pub.gemspec

@@ -0,0 +1,21 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path('../lib', __FILE__)
+
+Gem::Specification.new do |gem|
+  gem.authors       = ["OGIBAYASHI Hironori"]
+  gem.email         = ["ogibayashi@gmail.com"]
+  gem.description   = %q{0MQ publisher plugin for fluentd}
+  gem.summary       = %q{0MQ publisher plugin for fluentd}
+  gem.homepage      = ""
+
+  gem.files         = `git ls-files`.split($\)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.name          = "fluent-plugin-zmq-pub"
+  gem.require_paths = ["lib"]
+  gem.version       = "0.0.1"
+  gem.add_development_dependency "fluentd"
+  gem.add_runtime_dependency "fluentd"
+  gem.add_development_dependency "zmq"
+  gem.add_runtime_dependency "zmq"
+end

data/lib/fluent/plugin/out_zmq_pub.rb

@@ -0,0 +1,53 @@
+module Fluent
+  class Fluent::ZmqPubOutput < Fluent::BufferedOutput
+    Fluent::Plugin.register_output('zmq_pub', self)
+
+    config_param :pubkey, :string
+    config_param :bindaddr, :string, :default => 'tcp://*:5556'
+
+    def initialize
+      super
+      require 'zmq'
+    end
+
+    def configure(conf)
+      super
+    end
+    
+    def start
+      super
+      @context = ZMQ::Context.new(1)
+      @publisher = @context.socket(ZMQ::PUB)
+      @publisher.bind(@bindaddr)
+    end
+
+    def format(tag, time, record)
+      [tag,time,record].to_msgpack
+    end
+
+    def write(chunk)
+      chunk.msgpack_each{ |record|
+        pubkey_replaced = @pubkey.gsub(/\${(.*?)}/){ |s|
+          case $1
+          when 'tag'
+            record[0]
+          else
+            record[2][$1]
+          end
+        }
+
+        #  to_msgpack in format, unpack in write, then to_msgpack again... better way?
+        @publisher.send(pubkey_replaced + " " + record.to_msgpack)
+      }
+      
+    end
+ 
+    def shutdown
+      super
+      @publisher.close
+      @context.close
+    end
+
+  end
+
+end

data/test/helper.rb

@@ -0,0 +1,28 @@
+require 'rubygems'
+require 'bundler'
+begin
+  Bundler.setup(:default, :development)
+rescue Bundler::BundlerError => e
+  $stderr.puts e.message
+  $stderr.puts "Run `bundle install` to install missing gems"
+  exit e.status_code
+end
+require 'test/unit'
+
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+require 'fluent/test'
+unless ENV.has_key?('VERBOSE')
+  nulllogger = Object.new
+  nulllogger.instance_eval {|obj|
+    def method_missing(method, *args)
+      # pass
+    end
+  }
+  $log = nulllogger
+end
+
+require 'fluent/plugin/out_zmq_pub'
+
+class Test::Unit::TestCase
+end

data/test/plugin/test_out_zeromq_pub.rb

@@ -0,0 +1,65 @@
+require 'helper'
+require 'zmq'
+
+class ZmqPubOutputTest < Test::Unit::TestCase
+  def setup
+    Fluent::Test.setup
+    @context = ZMQ::Context.new(1)
+    @subscriber = @context.socket(ZMQ::SUB)
+    @subscriber.connect('tcp://localhost:5556')
+  end
+
+  def teardown
+    @subscriber.close
+    @context.close
+  end
+  
+  CONFIG = %[
+      pubkey ${tag}:${key1}
+      bindaddr tcp://*:5556
+  ]
+  
+  def create_driver(conf = CONFIG, tag='test')
+    Fluent::Test::BufferedOutputTestDriver.new(Fluent::ZmqPubOutput, tag).configure(conf)
+  end
+  
+  def test_configure
+    d = create_driver
+
+    assert_equal '${tag}:${key1}', d.instance.pubkey
+    assert_equal 'tcp://*:5556', d.instance.bindaddr
+  end
+  
+  def test_format
+    d = create_driver
+    
+    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+    d.emit({"key1"=>"aaa"}, time)
+    d.emit({"key1"=>"bbb", "key2"=>3}, time)
+    
+    d.expect_format ["test",time.to_i,{ "key1" => "aaa"}].to_msgpack
+    d.expect_format ["test",time.to_i,{ "key1" => "bbb", "key2" => 3}].to_msgpack
+    
+    d.run
+  end
+  
+  def test_write
+    d = create_driver
+    @subscriber.setsockopt(ZMQ::SUBSCRIBE,"test:aaa")
+    
+    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+    d.emit({"key1"=>"aaa"}, time)
+    d.emit({"key1"=>"bbb", "key2"=>3}, time)
+    
+    d.run
+    sleep 1
+
+    msg = @subscriber.recv(ZMQ::NOBLOCK)
+    (key, record) = msg.split(" ",2)
+    assert_equal ["test",time.to_i,{ "key1" => "aaa"}].to_msgpack, record
+
+  end
+
+end
+
+

metadata

@@ -0,0 +1,107 @@
+--- !ruby/object:Gem::Specification 
+name: fluent-plugin-zmq-pub
+version: !ruby/object:Gem::Version 
+  prerelease: 
+  version: 0.0.1
+platform: ruby
+authors: 
+- OGIBAYASHI Hironori
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2013-08-09 00:00:00 Z
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: fluentd
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+  type: :development
+  version_requirements: *id001
+- !ruby/object:Gem::Dependency 
+  name: fluentd
+  prerelease: false
+  requirement: &id002 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+  type: :runtime
+  version_requirements: *id002
+- !ruby/object:Gem::Dependency 
+  name: zmq
+  prerelease: false
+  requirement: &id003 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+  type: :development
+  version_requirements: *id003
+- !ruby/object:Gem::Dependency 
+  name: zmq
+  prerelease: false
+  requirement: &id004 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+  type: :runtime
+  version_requirements: *id004
+description: 0MQ publisher plugin for fluentd
+email: 
+- ogibayashi@gmail.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- .gitignore
+- Gemfile
+- LICENSE
+- README.rdoc
+- Rakefile
+- fluent-plugin-zmq-pub.gemspec
+- lib/fluent/plugin/out_zmq_pub.rb
+- test/helper.rb
+- test/plugin/test_out_zeromq_pub.rb
+homepage: ""
+licenses: []
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.8.25
+signing_key: 
+specification_version: 3
+summary: 0MQ publisher plugin for fluentd
+test_files: 
+- test/helper.rb
+- test/plugin/test_out_zeromq_pub.rb