fluent-plugin-resque-ex 0.1.0

data/.gitignore

@@ -0,0 +1,5 @@
+/fluent/
+/pkg/
+/coverage/
+/vendor/
+Gemfile.lock

data/.travis.yml

@@ -0,0 +1,5 @@
+rvm:
+ - 1.9.2
+ - 1.9.3
+
+script: bundle exec rake test

data/AUTHORS

@@ -0,0 +1 @@
+Yuichi Tateno <hotchpotch _at_ gmail.com>

data/Gemfile

@@ -0,0 +1,5 @@
+source :rubygems
+
+gemspec
+
+gem "simplecov", :require => false

data/README.rdoc

@@ -0,0 +1,26 @@
+= Fluent event to Resque queue plugin
+
+= Installation
+
+  $ fluent-gem install fluent-plugin-resque-ex
+
+= Usage
+
+    <match resque>
+      type resque
+      redis hostname:port/namespace
+      queue file_serve
+      worker_class_name_tag class # default class
+    </match>
+    
+    fluent_logger.post('resque', {
+      :class => 'Worker::MyClass',
+      :user_id => xxx,
+      :body => yyy
+    })
+    #=> enqueue ('file_serve', 'Worker::MyClass', {:user_id => xxx, :body => yyy})
+
+= Copyright
+
+Copyright:: Copyright (c) 2012- Yuichi Tateno
+License::   Apache License, Version 2.0

data/Rakefile

@@ -0,0 +1,17 @@
+require 'bundler'
+Bundler::GemHelper.install_tasks
+
+require 'rake/testtask'
+
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.test_files = FileList['test/plugin/*.rb']
+  test.verbose = true
+end
+
+task :coverage do |t|
+  ENV['SIMPLE_COV'] = '1'
+  Rake::Task["test"].invoke
+end
+
+task :default => [:build]

data/VERSION

@@ -0,0 +1 @@
+0.1.0

data/fluent-plugin-resque-ex.gemspec

@@ -0,0 +1,25 @@
+# encoding: utf-8
+$:.push File.expand_path('../lib', __FILE__)
+
+Gem::Specification.new do |gem|
+  gem.name        = "fluent-plugin-resque-ex"
+  gem.description = "Resque-ex enqueue plugin"
+  gem.homepage    = "https://github.com/iyagi15/fluent-plugin-resque-ex"
+  gem.summary     = gem.description
+  gem.version     = File.read("VERSION").strip
+  gem.authors     = ["Yuichi Tateno"]
+  gem.email       = "hotchpotch@gmail.com"
+  gem.has_rdoc    = false
+  gem.files       = `git ls-files`.split("\n")
+  gem.test_files  = `git ls-files -- {test,spec,features}/*`.split("\n")
+  gem.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  gem.require_paths = ['lib']
+
+  gem.add_dependency "fluentd", ">= 0.10.8"
+  gem.add_dependency "multi_json", "~> 1.0"
+  gem.add_dependency "redis-namespace",  "~> 1.0.2"
+
+  gem.add_development_dependency "rake", ">= 0.9.2"
+  gem.add_development_dependency "simplecov", ">= 0.5.4"
+  gem.add_development_dependency "rr", ">= 1.0.0"
+end

data/lib/fluent/plugin/out_resque_ex.rb

@@ -0,0 +1,90 @@
+module Fluent
+  class ResqueExOutput < BufferedOutput
+    Fluent::Plugin.register_output('resque_ex', self)
+
+    include SetTagKeyMixin
+    config_set_default :include_tag_key, false
+
+    include SetTimeKeyMixin
+    config_set_default :include_time_key, true
+
+    config_param :queue, :string
+    config_param :redis, :string, :default => nil
+    config_param :worker_class_name_tag, :string, :default => 'class'
+
+    def initialize
+      super
+      require 'multi_json'
+      require 'redis'
+      require 'redis-namespace'
+      require 'uuidtools'
+      require 'digest/md5'
+    end
+
+    def configure(conf)
+      super
+
+      @worker_class_name_tag = conf['worker_class_name_tag'] || 'class'
+      self.redis = conf['redis'] if conf['redis']
+    end
+
+    # code from resque.rb
+    def redis=(server)
+      case server
+      when String
+        if server =~ /redis\:\/\//
+          redis = Redis.connect(:url => server, :thread_safe => true)
+        else
+          server, namespace = server.split('/', 2)
+          host, port, db = server.split(':')
+          redis = Redis.new(:host => host, :port => port,
+                            :thread_safe => true, :db => db)
+        end
+        namespace ||= :resque
+
+        @redis = Redis::Namespace.new(namespace, :redis => redis)
+      when Redis::Namespace
+        @redis = server
+      else
+        @redis = Redis::Namespace.new(:resque, :redis => server)
+      end
+    end
+
+    def redis
+      return @redis if @redis && !@redis.kind_of?(String)
+      self.redis = Redis.respond_to?(:connect) ? Redis.connect : "localhost:6379"
+      self.redis
+    end
+
+    def enqueue(queue, klass, args)
+      id = Digest::MD5.hexdigest(UUIDTools::UUID.random_create.to_s)
+      redis.sadd(:queues, queue.to_s)
+      redis.rpush("queue:#{queue}", ::MultiJson.encode(:class => klass, :id => id, :args => [args]))
+    end
+
+    def start
+      super
+    end
+
+    def shutdown
+      super
+    end
+
+    def format(tag, time, record)
+      [tag, time, record].to_msgpack
+    end
+
+    def write(chunk)
+      queue_name = @queue_mapped ? chunk.key : @queue
+
+      chunk.msgpack_each {|tag, time, record|
+        klass = record.delete(@worker_class_name_tag)
+        if klass && !klass.empty?
+          enqueue(queue_name, klass, record)
+        else
+          $log.error("record have not #{@worker_class_name_tag} key.")
+        end
+      }
+    end
+  end
+end

data/test/plugin/out_resque.rb

@@ -0,0 +1,63 @@
+require 'test_helper'
+require 'fluent/plugin/out_resque'
+require 'multi_json'
+
+class ResqueOutputTest < Test::Unit::TestCase
+  def setup
+    super
+    Fluent::Test.setup
+    @subject = Object.new
+    any_instance_of(Fluent::ResqueOutput, :redis= => lambda {}, :redis => @subject)
+  end
+
+  CONFIG = %[
+    type resque
+    queue test_queue
+    time_format %y-%m-%d %H:%M:%S
+  ]
+
+  def create_driver(conf = CONFIG)
+    Fluent::Test::BufferedOutputTestDriver.new(Fluent::ResqueOutput) {
+    }.configure(conf)
+  end
+
+  def check_enqueue(queue, klass, args)
+    mock(@subject).sadd(:queues, "test_queue").any_times
+    mock(@subject).rpush("queue:#{queue}", ::MultiJson.encode(:class => klass, :args => [args]))
+  end
+
+  def test_write
+    d = create_driver
+    time = Time.at(Time.now.to_i).utc
+    d.emit({'a' => 1, "class" => "WorkerTest"}, time)
+    d.emit({'b' => 2, "class" => "WorkerTest"}, time)
+    check_enqueue("test_queue", "WorkerTest", {"a" => 1, "time" => time.strftime("%y-%m-%d %H:%M:%S")})
+    check_enqueue("test_queue", "WorkerTest", {"b" => 2, "time" => time.strftime("%y-%m-%d %H:%M:%S")})
+    d.run
+    assert_equal true, true
+  end
+
+  def test_write_except_time_key
+    d = create_driver(CONFIG + "\ninclude_time_key false")
+    time = Time.at(Time.now.to_i).utc
+    d.emit({'a' => 1, 'class' => 'WorkerTest'}, time)
+    check_enqueue("test_queue", "WorkerTest", {"a" => 1})
+    d.run
+  end
+
+  def test_write_include_tag_key
+    d = create_driver(CONFIG + "\ninclude_tag_key true")
+    time = Time.at(Time.now.to_i).utc
+    d.emit({'a' => 1, 'class' => 'WorkerTest'}, time)
+    check_enqueue("test_queue", "WorkerTest", {"a" => 1, "tag" => 'test', "time" => time.strftime("%y-%m-%d %H:%M:%S")})
+    d.run
+  end
+
+  def test_write_change_worker_class_name_tag
+    d = create_driver(CONFIG + "\nworker_class_name_tag klass")
+    time = Time.at(Time.now.to_i).utc
+    d.emit({'a' => 1, 'klass' => 'WorkerTest::Test'}, time)
+    check_enqueue("test_queue", "WorkerTest::Test", {"a" => 1, "time" => time.strftime("%y-%m-%d %H:%M:%S")})
+    d.run
+  end
+end

data/test/test_helper.rb

@@ -0,0 +1,20 @@
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+
+require 'rr'
+require 'test/unit'
+class Test::Unit::TestCase
+  include RR::Adapters::TestUnit
+end
+
+if ENV['SIMPLE_COV']
+  require 'simplecov'
+  SimpleCov.start do 
+    add_filter 'test/'
+    add_filter 'pkg/'
+    add_filter 'vendor/'
+  end
+end
+
+require 'test/unit'
+require 'fluent/test'

metadata

@@ -0,0 +1,153 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-resque-ex
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+  prerelease: 
+platform: ruby
+authors:
+- Yuichi Tateno
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-04-21 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 0.10.8
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 0.10.8
+- !ruby/object:Gem::Dependency
+  name: multi_json
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: redis-namespace
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.0.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.0.2
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 0.9.2
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 0.9.2
+- !ruby/object:Gem::Dependency
+  name: simplecov
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 0.5.4
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 0.5.4
+- !ruby/object:Gem::Dependency
+  name: rr
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 1.0.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 1.0.0
+description: Resque-ex enqueue plugin
+email: hotchpotch@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- .travis.yml
+- AUTHORS
+- Gemfile
+- README.rdoc
+- Rakefile
+- VERSION
+- fluent-plugin-resque-ex.gemspec
+- lib/fluent/plugin/out_resque_ex.rb
+- test/plugin/out_resque.rb
+- test/test_helper.rb
+homepage: https://github.com/iyagi15/fluent-plugin-resque-ex
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.23
+signing_key: 
+specification_version: 3
+summary: Resque-ex enqueue plugin
+test_files:
+- test/plugin/out_resque.rb
+- test/test_helper.rb