fluent-plugin-resque 0.1.0 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
data/Gemfile CHANGED
@@ -2,4 +2,7 @@ source :rubygems
2
2
 
3
3
  gemspec
4
4
 
5
+ # ToDo remove use handle_tag_name_mixin branch
6
+ gem "fluentd", :git => "git://github.com/fluent/fluentd.git", :branch => "handle_tag_name_mixin"
7
+
5
8
  gem "simplecov", :require => false
@@ -6,19 +6,19 @@
6
6
 
7
7
  = Usage
8
8
 
9
- <match queue.name>
9
+ <match resque>
10
10
  type resque
11
11
  redis hostname:port/namespace
12
12
  queue file_serve
13
- # Resque.enqueue_to('file_serve', 'Queue::Name', record)
14
- </match>
15
-
16
- <match queue.name.foo>
17
- type resque
18
- queue file_serve
19
- remove_tag_prefix queue.
20
- # Resque.enqueue_to('file_serve', 'Name::Foo', record)
13
+ worker_class_name_tag class # default class
21
14
  </match>
15
+
16
+ fluent_logger.post('resque', {
17
+ :class => 'Worker::MyClass',
18
+ :user_id => xxx,
19
+ :body => yyy
20
+ })
21
+ #=> enqueue ('file_serve', 'Worker::MyClass', {:user_id => xxx, :body => yyy})
22
22
 
23
23
  = Copyright
24
24
 
data/VERSION CHANGED
@@ -1 +1 @@
1
- 0.1.0
1
+ 0.2.0
@@ -15,8 +15,10 @@ Gem::Specification.new do |gem|
15
15
  gem.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
16
16
  gem.require_paths = ['lib']
17
17
 
18
- gem.add_dependency "fluentd", ">= 0.10.8"
19
- gem.add_dependency "resque", ">= 1.0.0"
18
+ # gem.add_dependency "fluentd", ">= 0.10.8"
19
+ gem.add_dependency "multi_json", "~> 1.0"
20
+ gem.add_dependency "redis-namespace", "~> 1.0.2"
21
+
20
22
  gem.add_development_dependency "rake", ">= 0.9.2"
21
23
  gem.add_development_dependency "simplecov", ">= 0.5.4"
22
24
  gem.add_development_dependency "rr", ">= 1.0.0"
@@ -11,23 +11,55 @@ module Fluent
11
11
 
12
12
  config_param :queue, :string
13
13
  config_param :redis, :string, :default => nil
14
- config_param :remove_tag_prefix, :string, :default => nil
14
+ config_param :worker_class_name_tag, :string, :default => 'class'
15
15
 
16
16
  def initialize
17
17
  super
18
- require 'resque'
18
+ require 'multi_json'
19
+ require 'redis'
20
+ require 'redis-namespace'
19
21
  end
20
22
 
21
23
  def configure(conf)
22
24
  super
23
25
 
24
- Resque.redis = conf['redis'] if conf['redis']
26
+ @worker_class_name_tag = conf['worker_class_name_tag'] || 'class'
27
+ self.redis = conf['redis'] if conf['redis']
28
+ end
29
+
30
+ # code from resque.rb
31
+ def redis=(server)
32
+ case server
33
+ when String
34
+ if server =~ /redis\:\/\//
35
+ redis = Redis.connect(:url => server, :thread_safe => true)
36
+ else
37
+ server, namespace = server.split('/', 2)
38
+ host, port, db = server.split(':')
39
+ redis = Redis.new(:host => host, :port => port,
40
+ :thread_safe => true, :db => db)
41
+ end
42
+ namespace ||= :resque
25
43
 
26
- if remove_tag_prefix = conf['remove_tag_prefix']
27
- @remove_tag_prefix = Regexp.new('^' + Regexp.escape(remove_tag_prefix))
44
+ @redis = Redis::Namespace.new(namespace, :redis => redis)
45
+ when Redis::Namespace
46
+ @redis = server
47
+ else
48
+ @redis = Redis::Namespace.new(:resque, :redis => server)
28
49
  end
29
50
  end
30
51
 
52
+ def redis
53
+ return @redis if @redis && !@redis.kind_of?(String)
54
+ self.redis = Redis.respond_to?(:connect) ? Redis.connect : "localhost:6379"
55
+ self.redis
56
+ end
57
+
58
+ def enqueue(queue, klass, args)
59
+ redis.sadd(:queues, queue.to_s)
60
+ redis.rpush("queue:#{queue}", ::MultiJson.encode(:class => klass, :args => [args]))
61
+ end
62
+
31
63
  def start
32
64
  super
33
65
  end
@@ -44,20 +76,13 @@ module Fluent
44
76
  queue_name = @queue_mapped ? chunk.key : @queue
45
77
 
46
78
  chunk.msgpack_each {|tag, time, record|
47
- record[@time_key] = Time.at(time || record[@time_key]) if @include_time_key
48
- tag = remove_prefix(tag) if @remove_tag_prefix
49
- record[@tag_key] = tag if @include_tag_key
50
- Resque.enqueue_to(queue_name, camelize(tag), record)
79
+ klass = record.delete(@worker_class_name_tag)
80
+ if klass && !klass.empty?
81
+ enqueue(queue_name, klass, record)
82
+ else
83
+ $log.error("record have not #{@worker_class_name_tag} key.")
84
+ end
51
85
  }
52
86
  end
53
-
54
- private
55
- def remove_prefix(tag)
56
- tag.to_s.sub(@remove_tag_prefix, '')
57
- end
58
-
59
- def camelize(name)
60
- name.to_s.gsub(/\.(.?)/) { "::" + $1.upcase }.gsub(/(^|_)(.)/) { $2.upcase }
61
- end
62
87
  end
63
88
  end
@@ -1,15 +1,19 @@
1
1
  require 'test_helper'
2
- require 'resque'
2
+ require 'fluent/plugin/out_resque'
3
+ require 'multi_json'
3
4
 
4
5
  class ResqueOutputTest < Test::Unit::TestCase
5
6
  def setup
7
+ super
6
8
  Fluent::Test.setup
7
- require 'fluent/plugin/out_resque'
9
+ @subject = Object.new
10
+ any_instance_of(Fluent::ResqueOutput, :redis= => lambda {}, :redis => @subject)
8
11
  end
9
12
 
10
13
  CONFIG = %[
11
14
  type resque
12
15
  queue test_queue
16
+ time_format %y-%m-%d %H:%M:%S
13
17
  ]
14
18
 
15
19
  def create_driver(conf = CONFIG)
@@ -17,42 +21,43 @@ class ResqueOutputTest < Test::Unit::TestCase
17
21
  }.configure(conf)
18
22
  end
19
23
 
24
+ def check_enqueue(queue, klass, args)
25
+ mock(@subject).sadd(:queues, "test_queue").any_times
26
+ mock(@subject).rpush("queue:#{queue}", ::MultiJson.encode(:class => klass, :args => [args]))
27
+ end
28
+
20
29
  def test_write
21
30
  d = create_driver
22
- time = Time.at Time.now.to_i
23
- d.emit({'a' => 1}, time)
24
- d.emit({'b' => 2}, time)
25
- mock(Resque).enqueue_to("test_queue", "Test", {"a" => 1 , "time" => time})
26
- mock(Resque).enqueue_to("test_queue", "Test", {"b" => 2 , "time" => time})
31
+ time = Time.at(Time.now.to_i).utc
32
+ d.emit({'a' => 1, "class" => "WorkerTest"}, time)
33
+ d.emit({'b' => 2, "class" => "WorkerTest"}, time)
34
+ check_enqueue("test_queue", "WorkerTest", {"a" => 1, "time" => time.strftime("%y-%m-%d %H:%M:%S")})
35
+ check_enqueue("test_queue", "WorkerTest", {"b" => 2, "time" => time.strftime("%y-%m-%d %H:%M:%S")})
27
36
  d.run
37
+ assert_equal true, true
28
38
  end
29
39
 
30
40
  def test_write_except_time_key
31
41
  d = create_driver(CONFIG + "\ninclude_time_key false")
32
- time = Time.at Time.now.to_i
33
- d.emit({'a' => 1}, time)
34
- mock(Resque).enqueue_to("test_queue", "Test", {"a" => 1})
42
+ time = Time.at(Time.now.to_i).utc
43
+ d.emit({'a' => 1, 'class' => 'WorkerTest'}, time)
44
+ check_enqueue("test_queue", "WorkerTest", {"a" => 1})
35
45
  d.run
36
46
  end
37
47
 
38
48
  def test_write_include_tag_key
39
49
  d = create_driver(CONFIG + "\ninclude_tag_key true")
40
- time = Time.at Time.now.to_i
41
- d.emit({'a' => 1}, time)
42
- mock(Resque).enqueue_to("test_queue", "Test", {"a" => 1, "time" => time, "tag" => 'test'})
50
+ time = Time.at(Time.now.to_i).utc
51
+ d.emit({'a' => 1, 'class' => 'WorkerTest'}, time)
52
+ check_enqueue("test_queue", "WorkerTest", {"a" => 1, "tag" => 'test', "time" => time.strftime("%y-%m-%d %H:%M:%S")})
43
53
  d.run
44
54
  end
45
55
 
46
- def test_write_with_remove_tag_prefix
47
- d = create_driver(CONFIG + "\nremove_tag_prefix te")
48
- time = Time.at Time.now.to_i
49
- d.emit({'a' => 1}, time)
50
- mock(Resque).enqueue_to("test_queue", "St", {"a" => 1, "time" => time})
56
+ def test_write_change_worker_class_name_tag
57
+ d = create_driver(CONFIG + "\nworker_class_name_tag klass")
58
+ time = Time.at(Time.now.to_i).utc
59
+ d.emit({'a' => 1, 'klass' => 'WorkerTest::Test'}, time)
60
+ check_enqueue("test_queue", "WorkerTest::Test", {"a" => 1, "time" => time.strftime("%y-%m-%d %H:%M:%S")})
51
61
  d.run
52
62
  end
53
-
54
- def test_change_redis_host
55
- mock(Resque).redis = "localhost:11111/namespace"
56
- d = create_driver(CONFIG + "\nredis localhost:11111/namespace")
57
- end
58
63
  end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-resque
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.0
4
+ version: 0.2.0
5
5
  prerelease:
6
6
  platform: ruby
7
7
  authors:
@@ -9,33 +9,33 @@ authors:
9
9
  autorequire:
10
10
  bindir: bin
11
11
  cert_chain: []
12
- date: 2012-02-29 00:00:00.000000000 Z
12
+ date: 2012-03-16 00:00:00.000000000 Z
13
13
  dependencies:
14
14
  - !ruby/object:Gem::Dependency
15
- name: fluentd
16
- requirement: &2158383160 !ruby/object:Gem::Requirement
15
+ name: multi_json
16
+ requirement: &2191051140 !ruby/object:Gem::Requirement
17
17
  none: false
18
18
  requirements:
19
- - - ! '>='
19
+ - - ~>
20
20
  - !ruby/object:Gem::Version
21
- version: 0.10.8
21
+ version: '1.0'
22
22
  type: :runtime
23
23
  prerelease: false
24
- version_requirements: *2158383160
24
+ version_requirements: *2191051140
25
25
  - !ruby/object:Gem::Dependency
26
- name: resque
27
- requirement: &2158382640 !ruby/object:Gem::Requirement
26
+ name: redis-namespace
27
+ requirement: &2191050660 !ruby/object:Gem::Requirement
28
28
  none: false
29
29
  requirements:
30
- - - ! '>='
30
+ - - ~>
31
31
  - !ruby/object:Gem::Version
32
- version: 1.0.0
32
+ version: 1.0.2
33
33
  type: :runtime
34
34
  prerelease: false
35
- version_requirements: *2158382640
35
+ version_requirements: *2191050660
36
36
  - !ruby/object:Gem::Dependency
37
37
  name: rake
38
- requirement: &2158382160 !ruby/object:Gem::Requirement
38
+ requirement: &2191050200 !ruby/object:Gem::Requirement
39
39
  none: false
40
40
  requirements:
41
41
  - - ! '>='
@@ -43,10 +43,10 @@ dependencies:
43
43
  version: 0.9.2
44
44
  type: :development
45
45
  prerelease: false
46
- version_requirements: *2158382160
46
+ version_requirements: *2191050200
47
47
  - !ruby/object:Gem::Dependency
48
48
  name: simplecov
49
- requirement: &2158381700 !ruby/object:Gem::Requirement
49
+ requirement: &2191049740 !ruby/object:Gem::Requirement
50
50
  none: false
51
51
  requirements:
52
52
  - - ! '>='
@@ -54,10 +54,10 @@ dependencies:
54
54
  version: 0.5.4
55
55
  type: :development
56
56
  prerelease: false
57
- version_requirements: *2158381700
57
+ version_requirements: *2191049740
58
58
  - !ruby/object:Gem::Dependency
59
59
  name: rr
60
- requirement: &2158381240 !ruby/object:Gem::Requirement
60
+ requirement: &2191049280 !ruby/object:Gem::Requirement
61
61
  none: false
62
62
  requirements:
63
63
  - - ! '>='
@@ -65,7 +65,7 @@ dependencies:
65
65
  version: 1.0.0
66
66
  type: :development
67
67
  prerelease: false
68
- version_requirements: *2158381240
68
+ version_requirements: *2191049280
69
69
  description: Resque enqueue plugin
70
70
  email: hotchpotch@gmail.com
71
71
  executables: []
@@ -95,15 +95,21 @@ required_ruby_version: !ruby/object:Gem::Requirement
95
95
  - - ! '>='
96
96
  - !ruby/object:Gem::Version
97
97
  version: '0'
98
+ segments:
99
+ - 0
100
+ hash: 1609622363767887906
98
101
  required_rubygems_version: !ruby/object:Gem::Requirement
99
102
  none: false
100
103
  requirements:
101
104
  - - ! '>='
102
105
  - !ruby/object:Gem::Version
103
106
  version: '0'
107
+ segments:
108
+ - 0
109
+ hash: 1609622363767887906
104
110
  requirements: []
105
111
  rubyforge_project:
106
- rubygems_version: 1.8.16
112
+ rubygems_version: 1.8.17
107
113
  signing_key:
108
114
  specification_version: 3
109
115
  summary: Resque enqueue plugin