fluent-plugin-resque 0.1.0 → 0.2.0
Sign up to get free protection for your applications and to get access to all the features.
- data/Gemfile +3 -0
- data/README.rdoc +9 -9
- data/VERSION +1 -1
- data/fluent-plugin-resque.gemspec +4 -2
- data/lib/fluent/plugin/out_resque.rb +43 -18
- data/test/plugin/out_resque.rb +28 -23
- metadata +25 -19
data/Gemfile
CHANGED
data/README.rdoc
CHANGED
@@ -6,19 +6,19 @@
|
|
6
6
|
|
7
7
|
= Usage
|
8
8
|
|
9
|
-
<match
|
9
|
+
<match resque>
|
10
10
|
type resque
|
11
11
|
redis hostname:port/namespace
|
12
12
|
queue file_serve
|
13
|
-
#
|
14
|
-
</match>
|
15
|
-
|
16
|
-
<match queue.name.foo>
|
17
|
-
type resque
|
18
|
-
queue file_serve
|
19
|
-
remove_tag_prefix queue.
|
20
|
-
# Resque.enqueue_to('file_serve', 'Name::Foo', record)
|
13
|
+
worker_class_name_tag class # default class
|
21
14
|
</match>
|
15
|
+
|
16
|
+
fluent_logger.post('resque', {
|
17
|
+
:class => 'Worker::MyClass',
|
18
|
+
:user_id => xxx,
|
19
|
+
:body => yyy
|
20
|
+
})
|
21
|
+
#=> enqueue ('file_serve', 'Worker::MyClass', {:user_id => xxx, :body => yyy})
|
22
22
|
|
23
23
|
= Copyright
|
24
24
|
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.
|
1
|
+
0.2.0
|
@@ -15,8 +15,10 @@ Gem::Specification.new do |gem|
|
|
15
15
|
gem.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
|
16
16
|
gem.require_paths = ['lib']
|
17
17
|
|
18
|
-
gem.add_dependency "fluentd", ">= 0.10.8"
|
19
|
-
gem.add_dependency "
|
18
|
+
# gem.add_dependency "fluentd", ">= 0.10.8"
|
19
|
+
gem.add_dependency "multi_json", "~> 1.0"
|
20
|
+
gem.add_dependency "redis-namespace", "~> 1.0.2"
|
21
|
+
|
20
22
|
gem.add_development_dependency "rake", ">= 0.9.2"
|
21
23
|
gem.add_development_dependency "simplecov", ">= 0.5.4"
|
22
24
|
gem.add_development_dependency "rr", ">= 1.0.0"
|
@@ -11,23 +11,55 @@ module Fluent
|
|
11
11
|
|
12
12
|
config_param :queue, :string
|
13
13
|
config_param :redis, :string, :default => nil
|
14
|
-
config_param :
|
14
|
+
config_param :worker_class_name_tag, :string, :default => 'class'
|
15
15
|
|
16
16
|
def initialize
|
17
17
|
super
|
18
|
-
require '
|
18
|
+
require 'multi_json'
|
19
|
+
require 'redis'
|
20
|
+
require 'redis-namespace'
|
19
21
|
end
|
20
22
|
|
21
23
|
def configure(conf)
|
22
24
|
super
|
23
25
|
|
24
|
-
|
26
|
+
@worker_class_name_tag = conf['worker_class_name_tag'] || 'class'
|
27
|
+
self.redis = conf['redis'] if conf['redis']
|
28
|
+
end
|
29
|
+
|
30
|
+
# code from resque.rb
|
31
|
+
def redis=(server)
|
32
|
+
case server
|
33
|
+
when String
|
34
|
+
if server =~ /redis\:\/\//
|
35
|
+
redis = Redis.connect(:url => server, :thread_safe => true)
|
36
|
+
else
|
37
|
+
server, namespace = server.split('/', 2)
|
38
|
+
host, port, db = server.split(':')
|
39
|
+
redis = Redis.new(:host => host, :port => port,
|
40
|
+
:thread_safe => true, :db => db)
|
41
|
+
end
|
42
|
+
namespace ||= :resque
|
25
43
|
|
26
|
-
|
27
|
-
|
44
|
+
@redis = Redis::Namespace.new(namespace, :redis => redis)
|
45
|
+
when Redis::Namespace
|
46
|
+
@redis = server
|
47
|
+
else
|
48
|
+
@redis = Redis::Namespace.new(:resque, :redis => server)
|
28
49
|
end
|
29
50
|
end
|
30
51
|
|
52
|
+
def redis
|
53
|
+
return @redis if @redis && !@redis.kind_of?(String)
|
54
|
+
self.redis = Redis.respond_to?(:connect) ? Redis.connect : "localhost:6379"
|
55
|
+
self.redis
|
56
|
+
end
|
57
|
+
|
58
|
+
def enqueue(queue, klass, args)
|
59
|
+
redis.sadd(:queues, queue.to_s)
|
60
|
+
redis.rpush("queue:#{queue}", ::MultiJson.encode(:class => klass, :args => [args]))
|
61
|
+
end
|
62
|
+
|
31
63
|
def start
|
32
64
|
super
|
33
65
|
end
|
@@ -44,20 +76,13 @@ module Fluent
|
|
44
76
|
queue_name = @queue_mapped ? chunk.key : @queue
|
45
77
|
|
46
78
|
chunk.msgpack_each {|tag, time, record|
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
79
|
+
klass = record.delete(@worker_class_name_tag)
|
80
|
+
if klass && !klass.empty?
|
81
|
+
enqueue(queue_name, klass, record)
|
82
|
+
else
|
83
|
+
$log.error("record have not #{@worker_class_name_tag} key.")
|
84
|
+
end
|
51
85
|
}
|
52
86
|
end
|
53
|
-
|
54
|
-
private
|
55
|
-
def remove_prefix(tag)
|
56
|
-
tag.to_s.sub(@remove_tag_prefix, '')
|
57
|
-
end
|
58
|
-
|
59
|
-
def camelize(name)
|
60
|
-
name.to_s.gsub(/\.(.?)/) { "::" + $1.upcase }.gsub(/(^|_)(.)/) { $2.upcase }
|
61
|
-
end
|
62
87
|
end
|
63
88
|
end
|
data/test/plugin/out_resque.rb
CHANGED
@@ -1,15 +1,19 @@
|
|
1
1
|
require 'test_helper'
|
2
|
-
require '
|
2
|
+
require 'fluent/plugin/out_resque'
|
3
|
+
require 'multi_json'
|
3
4
|
|
4
5
|
class ResqueOutputTest < Test::Unit::TestCase
|
5
6
|
def setup
|
7
|
+
super
|
6
8
|
Fluent::Test.setup
|
7
|
-
|
9
|
+
@subject = Object.new
|
10
|
+
any_instance_of(Fluent::ResqueOutput, :redis= => lambda {}, :redis => @subject)
|
8
11
|
end
|
9
12
|
|
10
13
|
CONFIG = %[
|
11
14
|
type resque
|
12
15
|
queue test_queue
|
16
|
+
time_format %y-%m-%d %H:%M:%S
|
13
17
|
]
|
14
18
|
|
15
19
|
def create_driver(conf = CONFIG)
|
@@ -17,42 +21,43 @@ class ResqueOutputTest < Test::Unit::TestCase
|
|
17
21
|
}.configure(conf)
|
18
22
|
end
|
19
23
|
|
24
|
+
def check_enqueue(queue, klass, args)
|
25
|
+
mock(@subject).sadd(:queues, "test_queue").any_times
|
26
|
+
mock(@subject).rpush("queue:#{queue}", ::MultiJson.encode(:class => klass, :args => [args]))
|
27
|
+
end
|
28
|
+
|
20
29
|
def test_write
|
21
30
|
d = create_driver
|
22
|
-
time = Time.at
|
23
|
-
d.emit({'a' => 1}, time)
|
24
|
-
d.emit({'b' => 2}, time)
|
25
|
-
|
26
|
-
|
31
|
+
time = Time.at(Time.now.to_i).utc
|
32
|
+
d.emit({'a' => 1, "class" => "WorkerTest"}, time)
|
33
|
+
d.emit({'b' => 2, "class" => "WorkerTest"}, time)
|
34
|
+
check_enqueue("test_queue", "WorkerTest", {"a" => 1, "time" => time.strftime("%y-%m-%d %H:%M:%S")})
|
35
|
+
check_enqueue("test_queue", "WorkerTest", {"b" => 2, "time" => time.strftime("%y-%m-%d %H:%M:%S")})
|
27
36
|
d.run
|
37
|
+
assert_equal true, true
|
28
38
|
end
|
29
39
|
|
30
40
|
def test_write_except_time_key
|
31
41
|
d = create_driver(CONFIG + "\ninclude_time_key false")
|
32
|
-
time = Time.at
|
33
|
-
d.emit({'a' => 1}, time)
|
34
|
-
|
42
|
+
time = Time.at(Time.now.to_i).utc
|
43
|
+
d.emit({'a' => 1, 'class' => 'WorkerTest'}, time)
|
44
|
+
check_enqueue("test_queue", "WorkerTest", {"a" => 1})
|
35
45
|
d.run
|
36
46
|
end
|
37
47
|
|
38
48
|
def test_write_include_tag_key
|
39
49
|
d = create_driver(CONFIG + "\ninclude_tag_key true")
|
40
|
-
time = Time.at
|
41
|
-
d.emit({'a' => 1}, time)
|
42
|
-
|
50
|
+
time = Time.at(Time.now.to_i).utc
|
51
|
+
d.emit({'a' => 1, 'class' => 'WorkerTest'}, time)
|
52
|
+
check_enqueue("test_queue", "WorkerTest", {"a" => 1, "tag" => 'test', "time" => time.strftime("%y-%m-%d %H:%M:%S")})
|
43
53
|
d.run
|
44
54
|
end
|
45
55
|
|
46
|
-
def
|
47
|
-
d = create_driver(CONFIG + "\
|
48
|
-
time = Time.at
|
49
|
-
d.emit({'a' => 1}, time)
|
50
|
-
|
56
|
+
def test_write_change_worker_class_name_tag
|
57
|
+
d = create_driver(CONFIG + "\nworker_class_name_tag klass")
|
58
|
+
time = Time.at(Time.now.to_i).utc
|
59
|
+
d.emit({'a' => 1, 'klass' => 'WorkerTest::Test'}, time)
|
60
|
+
check_enqueue("test_queue", "WorkerTest::Test", {"a" => 1, "time" => time.strftime("%y-%m-%d %H:%M:%S")})
|
51
61
|
d.run
|
52
62
|
end
|
53
|
-
|
54
|
-
def test_change_redis_host
|
55
|
-
mock(Resque).redis = "localhost:11111/namespace"
|
56
|
-
d = create_driver(CONFIG + "\nredis localhost:11111/namespace")
|
57
|
-
end
|
58
63
|
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-resque
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.2.0
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -9,33 +9,33 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2012-
|
12
|
+
date: 2012-03-16 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
|
-
name:
|
16
|
-
requirement: &
|
15
|
+
name: multi_json
|
16
|
+
requirement: &2191051140 !ruby/object:Gem::Requirement
|
17
17
|
none: false
|
18
18
|
requirements:
|
19
|
-
- -
|
19
|
+
- - ~>
|
20
20
|
- !ruby/object:Gem::Version
|
21
|
-
version: 0
|
21
|
+
version: '1.0'
|
22
22
|
type: :runtime
|
23
23
|
prerelease: false
|
24
|
-
version_requirements: *
|
24
|
+
version_requirements: *2191051140
|
25
25
|
- !ruby/object:Gem::Dependency
|
26
|
-
name:
|
27
|
-
requirement: &
|
26
|
+
name: redis-namespace
|
27
|
+
requirement: &2191050660 !ruby/object:Gem::Requirement
|
28
28
|
none: false
|
29
29
|
requirements:
|
30
|
-
- -
|
30
|
+
- - ~>
|
31
31
|
- !ruby/object:Gem::Version
|
32
|
-
version: 1.0.
|
32
|
+
version: 1.0.2
|
33
33
|
type: :runtime
|
34
34
|
prerelease: false
|
35
|
-
version_requirements: *
|
35
|
+
version_requirements: *2191050660
|
36
36
|
- !ruby/object:Gem::Dependency
|
37
37
|
name: rake
|
38
|
-
requirement: &
|
38
|
+
requirement: &2191050200 !ruby/object:Gem::Requirement
|
39
39
|
none: false
|
40
40
|
requirements:
|
41
41
|
- - ! '>='
|
@@ -43,10 +43,10 @@ dependencies:
|
|
43
43
|
version: 0.9.2
|
44
44
|
type: :development
|
45
45
|
prerelease: false
|
46
|
-
version_requirements: *
|
46
|
+
version_requirements: *2191050200
|
47
47
|
- !ruby/object:Gem::Dependency
|
48
48
|
name: simplecov
|
49
|
-
requirement: &
|
49
|
+
requirement: &2191049740 !ruby/object:Gem::Requirement
|
50
50
|
none: false
|
51
51
|
requirements:
|
52
52
|
- - ! '>='
|
@@ -54,10 +54,10 @@ dependencies:
|
|
54
54
|
version: 0.5.4
|
55
55
|
type: :development
|
56
56
|
prerelease: false
|
57
|
-
version_requirements: *
|
57
|
+
version_requirements: *2191049740
|
58
58
|
- !ruby/object:Gem::Dependency
|
59
59
|
name: rr
|
60
|
-
requirement: &
|
60
|
+
requirement: &2191049280 !ruby/object:Gem::Requirement
|
61
61
|
none: false
|
62
62
|
requirements:
|
63
63
|
- - ! '>='
|
@@ -65,7 +65,7 @@ dependencies:
|
|
65
65
|
version: 1.0.0
|
66
66
|
type: :development
|
67
67
|
prerelease: false
|
68
|
-
version_requirements: *
|
68
|
+
version_requirements: *2191049280
|
69
69
|
description: Resque enqueue plugin
|
70
70
|
email: hotchpotch@gmail.com
|
71
71
|
executables: []
|
@@ -95,15 +95,21 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
95
95
|
- - ! '>='
|
96
96
|
- !ruby/object:Gem::Version
|
97
97
|
version: '0'
|
98
|
+
segments:
|
99
|
+
- 0
|
100
|
+
hash: 1609622363767887906
|
98
101
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
99
102
|
none: false
|
100
103
|
requirements:
|
101
104
|
- - ! '>='
|
102
105
|
- !ruby/object:Gem::Version
|
103
106
|
version: '0'
|
107
|
+
segments:
|
108
|
+
- 0
|
109
|
+
hash: 1609622363767887906
|
104
110
|
requirements: []
|
105
111
|
rubyforge_project:
|
106
|
-
rubygems_version: 1.8.
|
112
|
+
rubygems_version: 1.8.17
|
107
113
|
signing_key:
|
108
114
|
specification_version: 3
|
109
115
|
summary: Resque enqueue plugin
|