fluent-plugin-resque 0.1.0

data/.gitignore

@@ -0,0 +1,5 @@
+/fluent/
+/pkg/
+/coverage/
+/vendor/
+Gemfile.lock

data/.travis.yml

@@ -0,0 +1,5 @@
+rvm:
+ - 1.9.2
+ - 1.9.3
+
+script: bundle exec rake test

data/AUTHORS

@@ -0,0 +1 @@
+Yuichi Tateno <hotchpotch _at_ gmail.com>

data/Gemfile

@@ -0,0 +1,5 @@
+source :rubygems
+
+gemspec
+
+gem "simplecov", :require => false

data/README.rdoc

@@ -0,0 +1,26 @@
+= Fluent event to Resque queue plugin
+
+= Installation
+
+  $ fluent-gem install fluent-plugin-resque
+
+= Usage
+
+    <match queue.name>
+      type resque
+      redis hostname:port/namespace
+      queue file_serve
+      # Resque.enqueue_to('file_serve', 'Queue::Name', record)
+    </match>
+
+    <match queue.name.foo>
+      type resque
+      queue file_serve
+      remove_tag_prefix queue.
+      # Resque.enqueue_to('file_serve', 'Name::Foo', record)
+    </match>
+
+= Copyright
+
+Copyright:: Copyright (c) 2012- Yuichi Tateno
+License::   Apache License, Version 2.0

data/Rakefile

@@ -0,0 +1,17 @@
+require 'bundler'
+Bundler::GemHelper.install_tasks
+
+require 'rake/testtask'
+
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.test_files = FileList['test/plugin/*.rb']
+  test.verbose = true
+end
+
+task :coverage do |t|
+  ENV['SIMPLE_COV'] = '1'
+  Rake::Task["test"].invoke
+end
+
+task :default => [:build]

data/VERSION

@@ -0,0 +1 @@
+0.1.0

data/fluent-plugin-resque.gemspec

@@ -0,0 +1,23 @@
+# encoding: utf-8
+$:.push File.expand_path('../lib', __FILE__)
+
+Gem::Specification.new do |gem|
+  gem.name        = "fluent-plugin-resque"
+  gem.description = "Resque enqueue plugin"
+  gem.homepage    = "https://github.com/hotchpotch/fluent-plugin-resque"
+  gem.summary     = gem.description
+  gem.version     = File.read("VERSION").strip
+  gem.authors     = ["Yuichi Tateno"]
+  gem.email       = "hotchpotch@gmail.com"
+  gem.has_rdoc    = false
+  gem.files       = `git ls-files`.split("\n")
+  gem.test_files  = `git ls-files -- {test,spec,features}/*`.split("\n")
+  gem.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  gem.require_paths = ['lib']
+
+  gem.add_dependency "fluentd", ">= 0.10.8"
+  gem.add_dependency "resque", ">= 1.0.0"
+  gem.add_development_dependency "rake", ">= 0.9.2"
+  gem.add_development_dependency "simplecov", ">= 0.5.4"
+  gem.add_development_dependency "rr", ">= 1.0.0"
+end

data/lib/fluent/plugin/out_resque.rb

@@ -0,0 +1,63 @@
+
+module Fluent
+  class ResqueOutput < BufferedOutput
+    Fluent::Plugin.register_output('resque', self)
+
+    include SetTagKeyMixin
+    config_set_default :include_tag_key, false
+
+    include SetTimeKeyMixin
+    config_set_default :include_time_key, true
+
+    config_param :queue, :string
+    config_param :redis, :string, :default => nil
+    config_param :remove_tag_prefix, :string, :default => nil
+
+    def initialize
+      super
+      require 'resque'
+    end
+
+    def configure(conf)
+      super
+
+      Resque.redis = conf['redis'] if conf['redis']
+
+      if remove_tag_prefix = conf['remove_tag_prefix']
+        @remove_tag_prefix = Regexp.new('^' + Regexp.escape(remove_tag_prefix))
+      end
+    end
+
+    def start
+      super
+    end
+
+    def shutdown
+      super
+    end
+
+    def format(tag, time, record)
+      [tag, time, record].to_msgpack
+    end
+
+    def write(chunk)
+      queue_name = @queue_mapped ? chunk.key : @queue
+
+      chunk.msgpack_each {|tag, time, record|
+        record[@time_key] = Time.at(time || record[@time_key]) if @include_time_key
+        tag = remove_prefix(tag) if @remove_tag_prefix
+        record[@tag_key] = tag if @include_tag_key
+        Resque.enqueue_to(queue_name, camelize(tag), record) 
+      }
+    end
+
+    private
+    def remove_prefix(tag)
+      tag.to_s.sub(@remove_tag_prefix, '')
+    end
+
+    def camelize(name)
+      name.to_s.gsub(/\.(.?)/) { "::" + $1.upcase }.gsub(/(^|_)(.)/) { $2.upcase }
+    end
+  end
+end

data/test/plugin/out_resque.rb

@@ -0,0 +1,58 @@
+require 'test_helper'
+require 'resque'
+
+class ResqueOutputTest < Test::Unit::TestCase
+  def setup
+    Fluent::Test.setup
+    require 'fluent/plugin/out_resque'
+  end
+
+  CONFIG = %[
+    type resque
+    queue test_queue
+  ]
+
+  def create_driver(conf = CONFIG)
+    Fluent::Test::BufferedOutputTestDriver.new(Fluent::ResqueOutput) {
+    }.configure(conf)
+  end
+
+  def test_write
+    d = create_driver
+    time = Time.at Time.now.to_i
+    d.emit({'a' => 1}, time)
+    d.emit({'b' => 2}, time)
+    mock(Resque).enqueue_to("test_queue", "Test", {"a" => 1 , "time" => time})
+    mock(Resque).enqueue_to("test_queue", "Test", {"b" => 2 , "time" => time})
+    d.run
+  end
+
+  def test_write_except_time_key
+    d = create_driver(CONFIG + "\ninclude_time_key false")
+    time = Time.at Time.now.to_i
+    d.emit({'a' => 1}, time)
+    mock(Resque).enqueue_to("test_queue", "Test", {"a" => 1})
+    d.run
+  end
+
+  def test_write_include_tag_key
+    d = create_driver(CONFIG + "\ninclude_tag_key true")
+    time = Time.at Time.now.to_i
+    d.emit({'a' => 1}, time)
+    mock(Resque).enqueue_to("test_queue", "Test", {"a" => 1, "time" => time, "tag" => 'test'})
+    d.run
+  end
+
+  def test_write_with_remove_tag_prefix
+    d = create_driver(CONFIG + "\nremove_tag_prefix te")
+    time = Time.at Time.now.to_i
+    d.emit({'a' => 1}, time)
+    mock(Resque).enqueue_to("test_queue", "St", {"a" => 1, "time" => time})
+    d.run
+  end
+
+  def test_change_redis_host
+    mock(Resque).redis = "localhost:11111/namespace"
+    d = create_driver(CONFIG + "\nredis localhost:11111/namespace")
+  end
+end

data/test/test_helper.rb

@@ -0,0 +1,20 @@
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+
+require 'rr'
+require 'test/unit'
+class Test::Unit::TestCase
+  include RR::Adapters::TestUnit
+end
+
+if ENV['SIMPLE_COV']
+  require 'simplecov'
+  SimpleCov.start do 
+    add_filter 'test/'
+    add_filter 'pkg/'
+    add_filter 'vendor/'
+  end
+end
+
+require 'test/unit'
+require 'fluent/test'

metadata

@@ -0,0 +1,112 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-resque
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+  prerelease: 
+platform: ruby
+authors:
+- Yuichi Tateno
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-02-29 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: &2158383160 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 0.10.8
+  type: :runtime
+  prerelease: false
+  version_requirements: *2158383160
+- !ruby/object:Gem::Dependency
+  name: resque
+  requirement: &2158382640 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 1.0.0
+  type: :runtime
+  prerelease: false
+  version_requirements: *2158382640
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: &2158382160 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 0.9.2
+  type: :development
+  prerelease: false
+  version_requirements: *2158382160
+- !ruby/object:Gem::Dependency
+  name: simplecov
+  requirement: &2158381700 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 0.5.4
+  type: :development
+  prerelease: false
+  version_requirements: *2158381700
+- !ruby/object:Gem::Dependency
+  name: rr
+  requirement: &2158381240 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 1.0.0
+  type: :development
+  prerelease: false
+  version_requirements: *2158381240
+description: Resque enqueue plugin
+email: hotchpotch@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- .travis.yml
+- AUTHORS
+- Gemfile
+- README.rdoc
+- Rakefile
+- VERSION
+- fluent-plugin-resque.gemspec
+- lib/fluent/plugin/out_resque.rb
+- test/plugin/out_resque.rb
+- test/test_helper.rb
+homepage: https://github.com/hotchpotch/fluent-plugin-resque
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.16
+signing_key: 
+specification_version: 3
+summary: Resque enqueue plugin
+test_files:
+- test/plugin/out_resque.rb
+- test/test_helper.rb