fluent-plugin-fedmsg 0.0.1 → 0.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 07e0e35e50a48c784740e30c5e72724ab777fcde
-  data.tar.gz: 9f68c00a6bf1c3a7543aec0f3966c7d55c04284a
+  metadata.gz: 8b1f278f7df87e6c3ce1080d07f25a18f08affe1
+  data.tar.gz: 6343aadf248afa6a0c9573f31d6b8b6fd4ed52b0
 SHA512:
-  metadata.gz: 8fdf99207957031f281e9b840279b5a958254ccbaefd5f358872d14c1ee447e09d45ea8965d20c9e912ffdabc3de951330fc3a65a2c9fc26cf9b35c412945178
-  data.tar.gz: 3096204a1601d5328f683161e2813a386c4fcd868484408e55fad4ad3b4daa094e934b3a4a0152874115b2cde3630e12a05d5cd77b228e4367b0985456395e16
+  metadata.gz: 81059067eb76f509a78c630b0df9c8fdbe4952ef6da751213958c2d4e695fd0cba2d14f512297c1b2a5d55d4ea8c2a0644ad1eed5615afb52247a20173c6eea3
+  data.tar.gz: fc2808158c63f12f0180394a661280ee56d6c83a64ced70795fee5df7410d21a6b67deffd9313b1c1b1be515b145121002a0b91582020c4faa0d761527098387

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in fluent-plugin-zmq-pub.gemspec
+gemspec

data/Rakefile

@@ -0,0 +1,11 @@
+#!/usr/bin/env rake
+require "bundler/gem_tasks"
+
+require 'rake/testtask'
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.pattern = 'test/**/test_*.rb'
+  test.verbose = true
+end
+
+task :default => :test

data/fluent-plugin-fedmsg.gemspec

@@ -0,0 +1,22 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path('../lib', __FILE__)
+
+Gem::Specification.new do |gem|
+  gem.authors       = ["Anton Sherkhonov"]
+  gem.email         = ["sherkhonov@gmail.com"]
+  gem.description   = %q{FedMsg subscriber plugin for fluentd}
+  gem.summary       = %q{FedMsg subscriber plugin for fluentd, use 0MQ v3.2 or greater version}
+  gem.homepage      = ""
+  gem.licenses	    = ["Apache License, Version 2.0"]
+
+  gem.files         = `git ls-files`.split($\)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.name          = "fluent-plugin-fedmsg"
+  gem.require_paths = ["lib"]
+  gem.version       = "0.0.2"
+  gem.add_development_dependency "rake"
+  gem.add_development_dependency "test-unit", "~> 3.2.0"
+  gem.add_runtime_dependency "fluentd"
+  gem.add_runtime_dependency "ffi-rzmq"
+end

data/lib/fluent/plugin/in_fedmsg.rb

@@ -0,0 +1,89 @@
+require 'fluent/input'
+
+module Fluent
+
+  class ZmqSubInput < Fluent::Input
+    Fluent::Plugin.register_input('fedmsg', self)
+
+    config_param :subkey, :string, :default => ""
+    config_param :publisher, :string, :default => "tcp://127.0.0.1:5556"
+    config_param :tag_prefix, :string, :default => ""
+    config_param :drop_fields, :string, :default => "username,certificate,i,crypto,signature"
+
+    attr_reader :subkeys
+
+    def initialize
+      super
+      require 'ffi-rzmq'
+    end
+
+    def configure(conf)
+      super
+      @subkeys = @subkey.split(",")
+      @drop_fields_arr = @drop_fields.split(",")
+    end
+
+    def start
+      super
+      @context =ZMQ::Context.new()
+      @thread = Thread.new(&method(:run))
+    end
+
+    def shutdown
+      Thread.kill(@thread)
+      @thread.join
+      @context.terminate
+      super
+    end
+
+    def run
+      begin
+        @subscriber = @context.socket(ZMQ::SUB)
+        @subscriber.connect(@publisher)
+        @subscriber.setsockopt(ZMQ::SNDHWM, 100)
+        @subscriber.setsockopt(ZMQ::RCVHWM, 100)
+        if @subkeys.size > 0
+          @subkeys.each do |k|
+            @subscriber.setsockopt(ZMQ::SUBSCRIBE,k)
+          end
+        else
+          @subscriber.setsockopt(ZMQ::SUBSCRIBE,'')
+        end
+        loop do
+          msg = ''
+          while @subscriber.recv_string(msg,ZMQ::DONTWAIT) && msg.size > 0
+            begin
+              record = JSON.parse(msg)
+            rescue JSON::ParserError => e
+              log.trace "Ignoring non-JSON message '#{msg}'"
+              next
+            rescue => e
+              log.warn "Unknown error parsing JSON message.",:error_class => e.class, :error => e
+              log.warn_backtrace
+            end
+            begin
+              tag = @tag_prefix + "." + record['topic']
+              time = record['timestamp']
+              record.delete_if { |key, value| @drop_fields_arr.include? key}
+              Engine.emit(tag, time, record)
+            rescue => e
+              log.warn "Error in processing message.",:error_class => e.class, :error => e
+              log.warn_backtrace
+            end
+            msg = ''
+          end
+          sleep(0.1)
+        end
+      rescue => e
+        log.error "error occurred while executing plugin.", :error_class => e.class, :error => e
+        log.warn_backtrace
+      ensure
+        if @subscriber
+          @subscriber.close
+        end
+      end
+    end
+
+  end
+
+end

data/test/helper.rb

@@ -0,0 +1,28 @@
+require 'rubygems'
+require 'bundler'
+begin
+  Bundler.setup(:default, :development)
+rescue Bundler::BundlerError => e
+  $stderr.puts e.message
+  $stderr.puts "Run `bundle install` to install missing gems"
+  exit e.status_code
+end
+require 'test/unit'
+
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+require 'fluent/test'
+unless ENV.has_key?('VERBOSE')
+  nulllogger = Object.new
+  nulllogger.instance_eval {|obj|
+    def method_missing(method, *args)
+      # pass
+    end
+  }
+  $log = nulllogger
+end
+
+require 'fluent/plugin/in_fedmsg'
+
+class Test::Unit::TestCase
+end

data/test/plugin/test_in_fedmsg.rb

@@ -0,0 +1,97 @@
+require 'helper'
+require 'ffi-rzmq'
+
+class FedMsgIntputTest < Test::Unit::TestCase
+  def setup
+    Fluent::Test.setup
+    @context = ZMQ::Context.new()
+    @publisher = @context.socket(ZMQ::PUB)
+    @publisher.bind("tcp://*:5556")
+  end
+  
+  def teardown
+    @publisher.close
+    @context.terminate
+  end
+
+  PUBLISHER = "tcp://127.0.0.1:5556"
+  CONFIG = %[
+     publisher #{PUBLISHER}
+     subkey test1.,test2.
+  ]
+  
+  
+  def create_driver(conf=CONFIG)
+    Fluent::Test::InputTestDriver.new(Fluent::ZmqSubInput).configure(conf)
+  end
+  
+  def test_configure
+    d = create_driver(CONFIG + "tag_prefix fed_msg_stuff")
+    assert_equal PUBLISHER, d.instance.publisher
+    assert_equal ["test1.","test2."], d.instance.subkeys
+    assert_equal "fed_msg_stuff", d.instance.tag_prefix
+  end
+  
+  def test_receive
+    d = create_driver
+    
+    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+    Fluent::Engine.now = time
+    
+    d.expect_emit "test1.aa", time, {"a"=>1}
+    d.expect_emit "test2.bb", time, {"a"=>2}
+    
+    d.run do
+      d.expected_emits.each {|tag,time,record|
+        send_record("dummy",time,record)  # This record should not be received.
+        send_record(tag,time,record)
+      }
+      sleep 1
+    end
+  end
+
+  def test_no_subkey
+    d = create_driver("")
+    
+    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+    Fluent::Engine.now = time
+    
+    d.expect_emit "test1.aa", time, {"a"=>1}
+    d.expect_emit "test2.bb", time, {"a"=>2}
+    
+    d.run do
+      d.expected_emits.each {|tag,time,record|
+        send_record(tag,time,record)
+      }
+      sleep 1
+    end
+  end
+
+
+  def test_receive_bulk
+    d = create_driver(CONFIG + "bulk_send true")
+    
+    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+    Fluent::Engine.now = time
+    
+    d.expect_emit "test3.aa", time, {"a"=>1}
+    d.expect_emit "test4.bb", time, {"a"=>2}
+    
+    d.run do
+      record_to_send = []
+      d.expected_emits.each {|tag,time,record|
+        record_to_send << [tag,time,record]
+      }
+      send_record_bulk("test1.aa",record_to_send)
+      sleep 1
+    end
+  end
+
+  def send_record(tag,time,record)
+    @publisher.send_string(tag + " " + [tag,time,record].to_msgpack)
+  end
+
+  def send_record_bulk(tag,records)
+    @publisher.send_string(tag + " " + records.to_msgpack)
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-fedmsg
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.0.2
 platform: ruby
 authors:
 - Anton Sherkhonov
@@ -74,8 +74,14 @@ extensions: []
 extra_rdoc_files: []
 files:
 - .gitignore
+- Gemfile
 - LICENSE
 - README.md
+- Rakefile
+- fluent-plugin-fedmsg.gemspec
+- lib/fluent/plugin/in_fedmsg.rb
+- test/helper.rb
+- test/plugin/test_in_fedmsg.rb
 homepage: ''
 licenses:
 - Apache License, Version 2.0
@@ -100,4 +106,6 @@ rubygems_version: 2.0.14.1
 signing_key: 
 specification_version: 4
 summary: FedMsg subscriber plugin for fluentd, use 0MQ v3.2 or greater version
-test_files: []
+test_files:
+- test/helper.rb
+- test/plugin/test_in_fedmsg.rb