fluent-plugin-buffered-resque 1.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 3a22892fefa524a310d6c88abebba487cd95617a
+  data.tar.gz: 14a1cb1c2d1140a28ebcb95049ed23948f50327f
+SHA512:
+  metadata.gz: 393fc7b84dc4b6cb880a54153246e8a11274814a8d575e7d20974c72777daf148701beaba07e03eef4dc4f664209f582fb691818f9b6158ff47bc81c1c2332a9
+  data.tar.gz: 8bff4e477eebf3c08312cddde26b0669085c6a3d7b7271e6f47954798c416e41266be0f69e5132a39229bf5e5d6dc4a2373cdefc74caf61bd882a08d5c0e7285

data/.gitignore

@@ -0,0 +1,5 @@
+/fluent/
+/pkg/
+/coverage/
+/vendor/
+Gemfile.lock

data/.travis.yml

@@ -0,0 +1,5 @@
+rvm:
+ - 1.9.2
+ - 1.9.3
+
+script: bundle exec rake test

data/AUTHORS

@@ -0,0 +1 @@
+Yuichi Tateno <hotchpotch _at_ gmail.com>

data/Gemfile

@@ -0,0 +1,5 @@
+source :rubygems
+
+gemspec
+
+gem "simplecov", :require => false

data/README.rdoc

@@ -0,0 +1,26 @@
+= Fluent event to Resque queue plugin
+
+= Installation
+
+  $ fluent-gem install fluent-plugin-buffered-resque
+
+= Usage
+
+    <match resque>
+      type resque
+      redis hostname:port/namespace
+      queue file_serve
+      worker_class MyClass
+      bulk_queuing true
+    </match>
+
+    fluent_logger.post('resque', {
+      :user_id => xxx,
+      :body => yyy
+    })
+    #=> enqueue ('file_serve', 'MyClass', {:user_id => xxx, :body => yyy})
+
+= Copyright
+
+Copyright:: Copyright (c) 2012- Yuichi Tateno
+License::   Apache License, Version 2.0

data/Rakefile

@@ -0,0 +1,17 @@
+require 'bundler'
+Bundler::GemHelper.install_tasks
+
+require 'rake/testtask'
+
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.test_files = FileList['test/plugin/*.rb']
+  test.verbose = true
+end
+
+task :coverage do |t|
+  ENV['SIMPLE_COV'] = '1'
+  Rake::Task["test"].invoke
+end
+
+task :default => [:build]

data/VERSION

@@ -0,0 +1 @@
+1.0.0

data/fluent-plugin-buffered-resque.gemspec

@@ -0,0 +1,25 @@
+# encoding: utf-8
+$:.push File.expand_path('../lib', __FILE__)
+
+Gem::Specification.new do |gem|
+  gem.name        = "fluent-plugin-buffered-resque"
+  gem.description = "Resque output plugin for fluent event collector. Has extra features like buffering and setting a worker class in the config."
+  gem.homepage    = "https://github.com/hfwang/fluent-plugin-buffered-resque"
+  gem.summary     = gem.description
+  gem.version     = File.read("VERSION").strip
+  gem.authors     = ["Hsiu-Fan Wang", "Yuichi Tateno"]
+  gem.email       = "hfwang@porkbuns.net"
+  gem.has_rdoc    = false
+  gem.files       = `git ls-files`.split("\n")
+  gem.test_files  = `git ls-files -- {test,spec,features}/*`.split("\n")
+  gem.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  gem.require_paths = ['lib']
+
+  gem.add_dependency "fluentd", ">= 0.10.8"
+  gem.add_dependency "multi_json", "~> 1.0"
+  gem.add_dependency "redis-namespace",  ">= 1.0.2"
+
+  gem.add_development_dependency "rake", ">= 0.9.2"
+  gem.add_development_dependency "simplecov", ">= 0.5.4"
+  gem.add_development_dependency "rr", ">= 1.0.0"
+end

data/lib/fluent/plugin/out_buffered_resque.rb

@@ -0,0 +1,107 @@
+
+module Fluent
+  class BufferedResqueOutput < BufferedOutput
+    Fluent::Plugin.register_output('buffered_resque', self)
+
+    include SetTagKeyMixin
+    config_set_default :include_tag_key, false
+
+    include SetTimeKeyMixin
+    config_set_default :include_time_key, true
+
+    config_param :queue, :string
+    config_param :redis, :string, :default => nil
+    config_param :worker_class_name_tag, :string, :default => 'class'
+    config_param :worker_class, :string, :default => nil
+    config_param :bulk_queueing, :bool, :default => false
+
+    # Define `log` method for v0.10.42 or earlier
+    unless method_defined?(:log)
+      define_method("log") { $log }
+    end
+
+    def initialize
+      super
+      require 'multi_json'
+      require 'redis'
+      require 'redis-namespace'
+    end
+
+    def configure(conf)
+      super
+
+      @worker_class_name_tag = conf['worker_class_name_tag'] || 'class'
+      @worker_class = conf['worker_class']
+      self.redis = conf['redis'] if conf['redis']
+    end
+
+    # code from resque.rb
+    def redis=(server)
+      case server
+      when String
+        if server =~ /redis\:\/\//
+          redis = Redis.connect(:url => server, :thread_safe => true)
+        else
+          server, namespace = server.split('/', 2)
+          host, port, db = server.split(':')
+          redis = Redis.new(:host => host, :port => port,
+                            :thread_safe => true, :db => db)
+        end
+        namespace ||= :resque
+
+        @redis = Redis::Namespace.new(namespace, :redis => redis)
+      when Redis::Namespace
+        @redis = server
+      else
+        @redis = Redis::Namespace.new(:resque, :redis => server)
+      end
+    end
+
+    def redis
+      return @redis if @redis && !@redis.kind_of?(String)
+      self.redis = Redis.respond_to?(:connect) ? Redis.connect : "localhost:6379"
+      self.redis
+    end
+
+    def enqueue(queue, klass, args)
+      redis.sadd(:queues, queue.to_s)
+      redis.rpush("queue:#{queue}", ::MultiJson.encode(:class => klass, :args => [args]))
+    end
+
+    def start
+      super
+    end
+
+    def shutdown
+      super
+    end
+
+    def format(tag, time, record)
+      [tag, time, record].to_msgpack
+    end
+
+    def write(chunk)
+      queue_name = @queue_mapped ? chunk.key : @queue
+
+      if klass = @worker_class and @bulk_queueing
+        records = []
+        chunk.msgpack_each {|tag, time, record|
+          record.delete(@worker_class_name_tag)
+          records << record
+        }
+        log.debug("Flushing #{records.size} records to #{queue_name}:#{klass}")
+        enqueue(queue_name, klass, records)
+      else
+        chunk.msgpack_each {|tag, time, record|
+          klass = @worker_class || record.delete(@worker_class_name_tag)
+          if klass && !klass.empty?
+            log.debug("Enqueuing one record to #{queue_name}:#{klass}")
+            enqueue(queue_name, klass, record)
+          else
+            log.error("Neither worker_class param nor #{@worker_class_name_tag} record key was supplied.")
+          end
+        }
+      end
+    end
+  end
+end

data/test/plugin/out_resque.rb

@@ -0,0 +1,73 @@
+require 'test_helper'
+require 'fluent/plugin/out_resque'
+require 'multi_json'
+
+class ResqueOutputTest < Test::Unit::TestCase
+  def setup
+    super
+    Fluent::Test.setup
+    @subject = Object.new
+    any_instance_of(Fluent::ResqueOutput, :redis= => lambda {}, :redis => @subject)
+  end
+
+  CONFIG = %[
+    type resque
+    queue test_queue
+    time_format %y-%m-%d %H:%M:%S
+  ]
+
+  def create_driver(conf = CONFIG)
+    Fluent::Test::BufferedOutputTestDriver.new(Fluent::ResqueOutput) {
+    }.configure(conf)
+  end
+
+  def check_enqueue(queue, klass, args)
+    mock(@subject).sadd(:queues, "test_queue").any_times
+    mock(@subject).rpush("queue:#{queue}", ::MultiJson.encode(:class => klass, :args => [args]))
+  end
+
+  def test_write
+    d = create_driver
+    time = Time.at(Time.now.to_i).utc
+    d.emit({'a' => 1, "class" => "WorkerTest"}, time)
+    d.emit({'b' => 2, "class" => "WorkerTest"}, time)
+    check_enqueue("test_queue", "WorkerTest", {"a" => 1, "time" => time.strftime("%y-%m-%d %H:%M:%S")})
+    check_enqueue("test_queue", "WorkerTest", {"b" => 2, "time" => time.strftime("%y-%m-%d %H:%M:%S")})
+    d.run
+    assert_equal true, true
+  end
+
+  def test_write_with_bulk_queueing
+    d = create_driver(CONFIG + "\nworker_class WorkerTest\nbulk_queueing true")
+    time = Time.at(Time.now.to_i).utc
+    d.emit({'a' => 1, "class" => "WorkerTest"}, time)
+    d.emit({'b' => 2, "class" => "WorkerTest"}, time)
+    check_enqueue("test_queue", "WorkerTest", [{"a" => 1, "time" => time.strftime("%y-%m-%d %H:%M:%S")}, {"b" => 2, "time" => time.strftime("%y-%m-%d %H:%M:%S")}])
+    d.run
+    assert_equal true, true
+  end
+
+  def test_write_except_time_key
+    d = create_driver(CONFIG + "\ninclude_time_key false")
+    time = Time.at(Time.now.to_i).utc
+    d.emit({'a' => 1, 'class' => 'WorkerTest'}, time)
+    check_enqueue("test_queue", "WorkerTest", {"a" => 1})
+    d.run
+  end
+
+  def test_write_include_tag_key
+    d = create_driver(CONFIG + "\ninclude_tag_key true")
+    time = Time.at(Time.now.to_i).utc
+    d.emit({'a' => 1, 'class' => 'WorkerTest'}, time)
+    check_enqueue("test_queue", "WorkerTest", {"a" => 1, "tag" => 'test', "time" => time.strftime("%y-%m-%d %H:%M:%S")})
+    d.run
+  end
+
+  def test_write_change_worker_class_name_tag
+    d = create_driver(CONFIG + "\nworker_class_name_tag klass")
+    time = Time.at(Time.now.to_i).utc
+    d.emit({'a' => 1, 'klass' => 'WorkerTest::Test'}, time)
+    check_enqueue("test_queue", "WorkerTest::Test", {"a" => 1, "time" => time.strftime("%y-%m-%d %H:%M:%S")})
+    d.run
+  end
+end

data/test/test_helper.rb

@@ -0,0 +1,20 @@
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+
+require 'rr'
+require 'test/unit'
+class Test::Unit::TestCase
+  include RR::Adapters::TestUnit
+end
+
+if ENV['SIMPLE_COV']
+  require 'simplecov'
+  SimpleCov.start do 
+    add_filter 'test/'
+    add_filter 'pkg/'
+    add_filter 'vendor/'
+  end
+end
+
+require 'test/unit'
+require 'fluent/test'

metadata

@@ -0,0 +1,142 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-buffered-resque
+version: !ruby/object:Gem::Version
+  version: 1.0.0
+platform: ruby
+authors:
+- Hsiu-Fan Wang
+- Yuichi Tateno
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-10-14 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.10.8
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.10.8
+- !ruby/object:Gem::Dependency
+  name: multi_json
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: redis-namespace
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.0.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.0.2
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.9.2
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.9.2
+- !ruby/object:Gem::Dependency
+  name: simplecov
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.5.4
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.5.4
+- !ruby/object:Gem::Dependency
+  name: rr
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.0.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.0.0
+description: Resque output plugin for fluent event collector. Has extra features like
+  buffering and setting a worker class in the config.
+email: hfwang@porkbuns.net
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".travis.yml"
+- AUTHORS
+- Gemfile
+- README.rdoc
+- Rakefile
+- VERSION
+- fluent-plugin-buffered-resque.gemspec
+- lib/fluent/plugin/out_buffered_resque.rb
+- test/plugin/out_resque.rb
+- test/test_helper.rb
+homepage: https://github.com/hfwang/fluent-plugin-buffered-resque
+licenses: []
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.2
+signing_key: 
+specification_version: 4
+summary: Resque output plugin for fluent event collector. Has extra features like
+  buffering and setting a worker class in the config.
+test_files:
+- test/plugin/out_resque.rb
+- test/test_helper.rb