fluent-plugin-resque 0.1.0 → 0.2.0

data/Gemfile

@@ -2,4 +2,7 @@ source :rubygems
 
 gemspec
 
+# ToDo remove use handle_tag_name_mixin branch
+gem "fluentd", :git => "git://github.com/fluent/fluentd.git", :branch => "handle_tag_name_mixin"
+
 gem "simplecov", :require => false

data/README.rdoc

@@ -6,19 +6,19 @@
 
 = Usage
 
-    <match queue.name>
+    <match resque>
       type resque
       redis hostname:port/namespace
       queue file_serve
-      # Resque.enqueue_to('file_serve', 'Queue::Name', record)
-    </match>
-
-    <match queue.name.foo>
-      type resque
-      queue file_serve
-      remove_tag_prefix queue.
-      # Resque.enqueue_to('file_serve', 'Name::Foo', record)
+      worker_class_name_tag class # default class
     </match>
+    
+    fluent_logger.post('resque', {
+      :class => 'Worker::MyClass',
+      :user_id => xxx,
+      :body => yyy
+    })
+    #=> enqueue ('file_serve', 'Worker::MyClass', {:user_id => xxx, :body => yyy})
 
 = Copyright
 

data/VERSION

@@ -1 +1 @@
-0.1.0
+0.2.0

data/fluent-plugin-resque.gemspec

@@ -15,8 +15,10 @@ Gem::Specification.new do |gem|
   gem.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
   gem.require_paths = ['lib']
 
-  gem.add_dependency "fluentd", ">= 0.10.8"
-  gem.add_dependency "resque", ">= 1.0.0"
+  # gem.add_dependency "fluentd", ">= 0.10.8"
+  gem.add_dependency "multi_json", "~> 1.0"
+  gem.add_dependency "redis-namespace",  "~> 1.0.2"
+
   gem.add_development_dependency "rake", ">= 0.9.2"
   gem.add_development_dependency "simplecov", ">= 0.5.4"
   gem.add_development_dependency "rr", ">= 1.0.0"

data/lib/fluent/plugin/out_resque.rb

@@ -11,23 +11,55 @@ module Fluent
 
     config_param :queue, :string
     config_param :redis, :string, :default => nil
-    config_param :remove_tag_prefix, :string, :default => nil
+    config_param :worker_class_name_tag, :string, :default => 'class'
 
     def initialize
       super
-      require 'resque'
+      require 'multi_json'
+      require 'redis'
+      require 'redis-namespace'
     end
 
     def configure(conf)
       super
 
-      Resque.redis = conf['redis'] if conf['redis']
+      @worker_class_name_tag = conf['worker_class_name_tag'] || 'class'
+      self.redis = conf['redis'] if conf['redis']
+    end
+
+    # code from resque.rb
+    def redis=(server)
+      case server
+      when String
+        if server =~ /redis\:\/\//
+          redis = Redis.connect(:url => server, :thread_safe => true)
+        else
+          server, namespace = server.split('/', 2)
+          host, port, db = server.split(':')
+          redis = Redis.new(:host => host, :port => port,
+                            :thread_safe => true, :db => db)
+        end
+        namespace ||= :resque
 
-      if remove_tag_prefix = conf['remove_tag_prefix']
-        @remove_tag_prefix = Regexp.new('^' + Regexp.escape(remove_tag_prefix))
+        @redis = Redis::Namespace.new(namespace, :redis => redis)
+      when Redis::Namespace
+        @redis = server
+      else
+        @redis = Redis::Namespace.new(:resque, :redis => server)
       end
     end
 
+    def redis
+      return @redis if @redis && !@redis.kind_of?(String)
+      self.redis = Redis.respond_to?(:connect) ? Redis.connect : "localhost:6379"
+      self.redis
+    end
+
+    def enqueue(queue, klass, args)
+      redis.sadd(:queues, queue.to_s)
+      redis.rpush("queue:#{queue}", ::MultiJson.encode(:class => klass, :args => [args]))
+    end
+
     def start
       super
     end
@@ -44,20 +76,13 @@ module Fluent
       queue_name = @queue_mapped ? chunk.key : @queue
 
       chunk.msgpack_each {|tag, time, record|
-        record[@time_key] = Time.at(time || record[@time_key]) if @include_time_key
-        tag = remove_prefix(tag) if @remove_tag_prefix
-        record[@tag_key] = tag if @include_tag_key
-        Resque.enqueue_to(queue_name, camelize(tag), record) 
+        klass = record.delete(@worker_class_name_tag)
+        if klass && !klass.empty?
+          enqueue(queue_name, klass, record)
+        else
+          $log.error("record have not #{@worker_class_name_tag} key.")
+        end
       }
     end
-
-    private
-    def remove_prefix(tag)
-      tag.to_s.sub(@remove_tag_prefix, '')
-    end
-
-    def camelize(name)
-      name.to_s.gsub(/\.(.?)/) { "::" + $1.upcase }.gsub(/(^|_)(.)/) { $2.upcase }
-    end
   end
 end

data/test/plugin/out_resque.rb

@@ -1,15 +1,19 @@
 require 'test_helper'
-require 'resque'
+require 'fluent/plugin/out_resque'
+require 'multi_json'
 
 class ResqueOutputTest < Test::Unit::TestCase
   def setup
+    super
     Fluent::Test.setup
-    require 'fluent/plugin/out_resque'
+    @subject = Object.new
+    any_instance_of(Fluent::ResqueOutput, :redis= => lambda {}, :redis => @subject)
   end
 
   CONFIG = %[
     type resque
     queue test_queue
+    time_format %y-%m-%d %H:%M:%S
   ]
 
   def create_driver(conf = CONFIG)
@@ -17,42 +21,43 @@ class ResqueOutputTest < Test::Unit::TestCase
     }.configure(conf)
   end
 
+  def check_enqueue(queue, klass, args)
+    mock(@subject).sadd(:queues, "test_queue").any_times
+    mock(@subject).rpush("queue:#{queue}", ::MultiJson.encode(:class => klass, :args => [args]))
+  end
+
   def test_write
     d = create_driver
-    time = Time.at Time.now.to_i
-    d.emit({'a' => 1}, time)
-    d.emit({'b' => 2}, time)
-    mock(Resque).enqueue_to("test_queue", "Test", {"a" => 1 , "time" => time})
-    mock(Resque).enqueue_to("test_queue", "Test", {"b" => 2 , "time" => time})
+    time = Time.at(Time.now.to_i).utc
+    d.emit({'a' => 1, "class" => "WorkerTest"}, time)
+    d.emit({'b' => 2, "class" => "WorkerTest"}, time)
+    check_enqueue("test_queue", "WorkerTest", {"a" => 1, "time" => time.strftime("%y-%m-%d %H:%M:%S")})
+    check_enqueue("test_queue", "WorkerTest", {"b" => 2, "time" => time.strftime("%y-%m-%d %H:%M:%S")})
     d.run
+    assert_equal true, true
   end
 
   def test_write_except_time_key
     d = create_driver(CONFIG + "\ninclude_time_key false")
-    time = Time.at Time.now.to_i
-    d.emit({'a' => 1}, time)
-    mock(Resque).enqueue_to("test_queue", "Test", {"a" => 1})
+    time = Time.at(Time.now.to_i).utc
+    d.emit({'a' => 1, 'class' => 'WorkerTest'}, time)
+    check_enqueue("test_queue", "WorkerTest", {"a" => 1})
     d.run
   end
 
   def test_write_include_tag_key
     d = create_driver(CONFIG + "\ninclude_tag_key true")
-    time = Time.at Time.now.to_i
-    d.emit({'a' => 1}, time)
-    mock(Resque).enqueue_to("test_queue", "Test", {"a" => 1, "time" => time, "tag" => 'test'})
+    time = Time.at(Time.now.to_i).utc
+    d.emit({'a' => 1, 'class' => 'WorkerTest'}, time)
+    check_enqueue("test_queue", "WorkerTest", {"a" => 1, "tag" => 'test', "time" => time.strftime("%y-%m-%d %H:%M:%S")})
     d.run
   end
 
-  def test_write_with_remove_tag_prefix
-    d = create_driver(CONFIG + "\nremove_tag_prefix te")
-    time = Time.at Time.now.to_i
-    d.emit({'a' => 1}, time)
-    mock(Resque).enqueue_to("test_queue", "St", {"a" => 1, "time" => time})
+  def test_write_change_worker_class_name_tag
+    d = create_driver(CONFIG + "\nworker_class_name_tag klass")
+    time = Time.at(Time.now.to_i).utc
+    d.emit({'a' => 1, 'klass' => 'WorkerTest::Test'}, time)
+    check_enqueue("test_queue", "WorkerTest::Test", {"a" => 1, "time" => time.strftime("%y-%m-%d %H:%M:%S")})
     d.run
   end
-
-  def test_change_redis_host
-    mock(Resque).redis = "localhost:11111/namespace"
-    d = create_driver(CONFIG + "\nredis localhost:11111/namespace")
-  end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-resque
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,33 +9,33 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-02-29 00:00:00.000000000 Z
+date: 2012-03-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: fluentd
-  requirement: &2158383160 !ruby/object:Gem::Requirement
+  name: multi_json
+  requirement: &2191051140 !ruby/object:Gem::Requirement
     none: false
     requirements:
-    - - ! '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: 0.10.8
+        version: '1.0'
   type: :runtime
   prerelease: false
-  version_requirements: *2158383160
+  version_requirements: *2191051140
 - !ruby/object:Gem::Dependency
-  name: resque
-  requirement: &2158382640 !ruby/object:Gem::Requirement
+  name: redis-namespace
+  requirement: &2191050660 !ruby/object:Gem::Requirement
     none: false
     requirements:
-    - - ! '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: 1.0.0
+        version: 1.0.2
   type: :runtime
   prerelease: false
-  version_requirements: *2158382640
+  version_requirements: *2191050660
 - !ruby/object:Gem::Dependency
   name: rake
-  requirement: &2158382160 !ruby/object:Gem::Requirement
+  requirement: &2191050200 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -43,10 +43,10 @@ dependencies:
         version: 0.9.2
   type: :development
   prerelease: false
-  version_requirements: *2158382160
+  version_requirements: *2191050200
 - !ruby/object:Gem::Dependency
   name: simplecov
-  requirement: &2158381700 !ruby/object:Gem::Requirement
+  requirement: &2191049740 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -54,10 +54,10 @@ dependencies:
         version: 0.5.4
   type: :development
   prerelease: false
-  version_requirements: *2158381700
+  version_requirements: *2191049740
 - !ruby/object:Gem::Dependency
   name: rr
-  requirement: &2158381240 !ruby/object:Gem::Requirement
+  requirement: &2191049280 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -65,7 +65,7 @@ dependencies:
         version: 1.0.0
   type: :development
   prerelease: false
-  version_requirements: *2158381240
+  version_requirements: *2191049280
 description: Resque enqueue plugin
 email: hotchpotch@gmail.com
 executables: []
@@ -95,15 +95,21 @@ required_ruby_version: !ruby/object:Gem::Requirement
   - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
+      segments:
+      - 0
+      hash: 1609622363767887906
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
   - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
+      segments:
+      - 0
+      hash: 1609622363767887906
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.16
+rubygems_version: 1.8.17
 signing_key: 
 specification_version: 3
 summary: Resque enqueue plugin