resque-mongo 1.3.1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (61) hide show
  1. data/.gitignore +3 -0
  2. data/.kick +26 -0
  3. data/CONTRIBUTORS +14 -0
  4. data/HISTORY.md +53 -0
  5. data/LICENSE +20 -0
  6. data/README.markdown +755 -0
  7. data/Rakefile +65 -0
  8. data/bin/resque +57 -0
  9. data/bin/resque-web +18 -0
  10. data/config.ru +14 -0
  11. data/deps.rip +8 -0
  12. data/examples/async_helper.rb +31 -0
  13. data/examples/demo/README.markdown +71 -0
  14. data/examples/demo/Rakefile +3 -0
  15. data/examples/demo/app.rb +27 -0
  16. data/examples/demo/config.ru +19 -0
  17. data/examples/demo/job.rb +12 -0
  18. data/examples/god/resque.god +52 -0
  19. data/examples/god/stale.god +26 -0
  20. data/examples/instance.rb +11 -0
  21. data/examples/simple.rb +30 -0
  22. data/init.rb +1 -0
  23. data/lib/resque.rb +224 -0
  24. data/lib/resque/errors.rb +7 -0
  25. data/lib/resque/failure.rb +63 -0
  26. data/lib/resque/failure/base.rb +58 -0
  27. data/lib/resque/failure/hoptoad.rb +88 -0
  28. data/lib/resque/failure/mongo.rb +32 -0
  29. data/lib/resque/helpers.rb +65 -0
  30. data/lib/resque/job.rb +103 -0
  31. data/lib/resque/server.rb +182 -0
  32. data/lib/resque/server/public/idle.png +0 -0
  33. data/lib/resque/server/public/jquery-1.3.2.min.js +19 -0
  34. data/lib/resque/server/public/jquery.relatize_date.js +95 -0
  35. data/lib/resque/server/public/poll.png +0 -0
  36. data/lib/resque/server/public/ranger.js +24 -0
  37. data/lib/resque/server/public/reset.css +48 -0
  38. data/lib/resque/server/public/style.css +75 -0
  39. data/lib/resque/server/public/working.png +0 -0
  40. data/lib/resque/server/views/error.erb +1 -0
  41. data/lib/resque/server/views/failed.erb +35 -0
  42. data/lib/resque/server/views/key.erb +17 -0
  43. data/lib/resque/server/views/layout.erb +38 -0
  44. data/lib/resque/server/views/next_more.erb +10 -0
  45. data/lib/resque/server/views/overview.erb +4 -0
  46. data/lib/resque/server/views/queues.erb +46 -0
  47. data/lib/resque/server/views/stats.erb +62 -0
  48. data/lib/resque/server/views/workers.erb +78 -0
  49. data/lib/resque/server/views/working.erb +67 -0
  50. data/lib/resque/stat.rb +55 -0
  51. data/lib/resque/tasks.rb +39 -0
  52. data/lib/resque/version.rb +3 -0
  53. data/lib/resque/worker.rb +415 -0
  54. data/tasks/redis.rake +133 -0
  55. data/tasks/resque.rake +2 -0
  56. data/test/dump.rdb +0 -0
  57. data/test/redis-test.conf +132 -0
  58. data/test/resque_test.rb +191 -0
  59. data/test/test_helper.rb +87 -0
  60. data/test/worker_test.rb +229 -0
  61. metadata +162 -0
@@ -0,0 +1,133 @@
1
+ # Inspired by rabbitmq.rake the Redbox project at http://github.com/rick/redbox/tree/master
2
+ require 'fileutils'
3
+ require 'open-uri'
4
+
5
+ class RedisRunner
6
+
7
+ def self.redisdir
8
+ "/tmp/redis/"
9
+ end
10
+
11
+ def self.redisconfdir
12
+ server_dir = File.dirname(`which redis-server`)
13
+ conf_file = "#{server_dir}/../etc/redis.conf"
14
+ unless File.exists? conf_file
15
+ conf_file = "#{server_dir}/../../etc/redis.conf"
16
+ end
17
+ conf_file
18
+ end
19
+
20
+ def self.dtach_socket
21
+ '/tmp/redis.dtach'
22
+ end
23
+
24
+ # Just check for existance of dtach socket
25
+ def self.running?
26
+ File.exists? dtach_socket
27
+ end
28
+
29
+ def self.start
30
+ puts 'Detach with Ctrl+\ Re-attach with rake redis:attach'
31
+ sleep 1
32
+ exec "dtach -A #{dtach_socket} redis-server #{redisconfdir}"
33
+ end
34
+
35
+ def self.attach
36
+ exec "dtach -a #{dtach_socket}"
37
+ end
38
+
39
+ def self.stop
40
+ sh 'echo "SHUTDOWN" | nc localhost 6379'
41
+ end
42
+
43
+ end
44
+
45
+ namespace :redis do
46
+
47
+ desc 'About redis'
48
+ task :about do
49
+ puts "\nSee http://code.google.com/p/redis/ for information about redis.\n\n"
50
+ end
51
+
52
+ desc 'Start redis'
53
+ task :start do
54
+ RedisRunner.start
55
+ end
56
+
57
+ desc 'Stop redis'
58
+ task :stop do
59
+ RedisRunner.stop
60
+ end
61
+
62
+ desc 'Restart redis'
63
+ task :restart do
64
+ RedisRunner.stop
65
+ RedisRunner.start
66
+ end
67
+
68
+ desc 'Attach to redis dtach socket'
69
+ task :attach do
70
+ RedisRunner.attach
71
+ end
72
+
73
+ desc 'Install the latest verison of Redis from Github (requires git, duh)'
74
+ task :install => [:about, :download, :make] do
75
+ ENV['PREFIX'] and bin_dir = "#{ENV['PREFIX']}/bin" or bin_dir = '/usr/bin'
76
+ %w(redis-benchmark redis-cli redis-server).each do |bin|
77
+ sh "cp /tmp/redis/#{bin} #{bin_dir}"
78
+ end
79
+
80
+ puts "Installed redis-benchmark, redis-cli and redis-server to #{bin_dir}"
81
+
82
+ ENV['PREFIX'] and conf_dir = "#{ENV['PREFIX']}/etc" or conf_dir = '/etc'
83
+ unless File.exists?("#{conf_dir}")
84
+ sh "cp /tmp/redis/redis.conf #{conf_dir}"
85
+ puts "Installed redis.conf to #{conf_dir} \n You should look at this file!"
86
+ end
87
+ end
88
+
89
+ task :make do
90
+ sh "cd #{RedisRunner.redisdir} && make clean"
91
+ sh "cd #{RedisRunner.redisdir} && make"
92
+ end
93
+
94
+ desc "Download package"
95
+ task :download do
96
+ sh 'rm -rf /tmp/redis/' if File.exists?("#{RedisRunner.redisdir}/.svn")
97
+ sh 'git clone git://github.com/antirez/redis.git /tmp/redis' unless File.exists?(RedisRunner.redisdir)
98
+ sh "cd #{RedisRunner.redisdir} && git pull" if File.exists?("#{RedisRunner.redisdir}/.git")
99
+ end
100
+
101
+ end
102
+
103
+ namespace :dtach do
104
+
105
+ desc 'About dtach'
106
+ task :about do
107
+ puts "\nSee http://dtach.sourceforge.net/ for information about dtach.\n\n"
108
+ end
109
+
110
+ desc 'Install dtach 0.8 from source'
111
+ task :install => [:about] do
112
+
113
+ Dir.chdir('/tmp/')
114
+ unless File.exists?('/tmp/dtach-0.8.tar.gz')
115
+ require 'net/http'
116
+
117
+ url = 'http://downloads.sourceforge.net/project/dtach/dtach/0.8/dtach-0.8.tar.gz'
118
+ open('/tmp/dtach-0.8.tar.gz', 'wb') do |file| file.write(open(url).read) end
119
+ end
120
+
121
+ unless File.directory?('/tmp/dtach-0.8')
122
+ system('tar xzf dtach-0.8.tar.gz')
123
+ end
124
+
125
+ ENV['PREFIX'] and bin_dir = "#{ENV['PREFIX']}/bin" or bin_dir = "/usr/bin"
126
+ Dir.chdir('/tmp/dtach-0.8/')
127
+ sh 'cd /tmp/dtach-0.8/ && ./configure && make'
128
+ sh "cp /tmp/dtach-0.8/dtach #{bin_dir}"
129
+
130
+ puts "Dtach successfully installed to #{bin_dir}"
131
+ end
132
+ end
133
+
@@ -0,0 +1,2 @@
1
+ $LOAD_PATH.unshift File.dirname(__FILE__) + '/../lib'
2
+ require 'resque/tasks'
Binary file
@@ -0,0 +1,132 @@
1
+ # Redis configuration file example
2
+
3
+ # By default Redis does not run as a daemon. Use 'yes' if you need it.
4
+ # Note that Redis will write a pid file in /var/run/redis.pid when daemonized.
5
+ daemonize yes
6
+
7
+ # When run as a daemon, Redis write a pid file in /var/run/redis.pid by default.
8
+ # You can specify a custom pid file location here.
9
+ pidfile ./test/redis-test.pid
10
+
11
+ # Accept connections on the specified port, default is 6379
12
+ port 9736
13
+
14
+ # If you want you can bind a single interface, if the bind option is not
15
+ # specified all the interfaces will listen for connections.
16
+ #
17
+ # bind 127.0.0.1
18
+
19
+ # Close the connection after a client is idle for N seconds (0 to disable)
20
+ timeout 300
21
+
22
+ # Save the DB on disk:
23
+ #
24
+ # save <seconds> <changes>
25
+ #
26
+ # Will save the DB if both the given number of seconds and the given
27
+ # number of write operations against the DB occurred.
28
+ #
29
+ # In the example below the behaviour will be to save:
30
+ # after 900 sec (15 min) if at least 1 key changed
31
+ # after 300 sec (5 min) if at least 10 keys changed
32
+ # after 60 sec if at least 10000 keys changed
33
+ save 900 1
34
+ save 300 10
35
+ save 60 10000
36
+
37
+ # The filename where to dump the DB
38
+ dbfilename dump.rdb
39
+
40
+ # For default save/load DB in/from the working directory
41
+ # Note that you must specify a directory not a file name.
42
+ dir ./test/
43
+
44
+ # Set server verbosity to 'debug'
45
+ # it can be one of:
46
+ # debug (a lot of information, useful for development/testing)
47
+ # notice (moderately verbose, what you want in production probably)
48
+ # warning (only very important / critical messages are logged)
49
+ loglevel debug
50
+
51
+ # Specify the log file name. Also 'stdout' can be used to force
52
+ # the demon to log on the standard output. Note that if you use standard
53
+ # output for logging but daemonize, logs will be sent to /dev/null
54
+ logfile stdout
55
+
56
+ # Set the number of databases. The default database is DB 0, you can select
57
+ # a different one on a per-connection basis using SELECT <dbid> where
58
+ # dbid is a number between 0 and 'databases'-1
59
+ databases 16
60
+
61
+ ################################# REPLICATION #################################
62
+
63
+ # Master-Slave replication. Use slaveof to make a Redis instance a copy of
64
+ # another Redis server. Note that the configuration is local to the slave
65
+ # so for example it is possible to configure the slave to save the DB with a
66
+ # different interval, or to listen to another port, and so on.
67
+
68
+ # slaveof <masterip> <masterport>
69
+
70
+ ################################## SECURITY ###################################
71
+
72
+ # Require clients to issue AUTH <PASSWORD> before processing any other
73
+ # commands. This might be useful in environments in which you do not trust
74
+ # others with access to the host running redis-server.
75
+ #
76
+ # This should stay commented out for backward compatibility and because most
77
+ # people do not need auth (e.g. they run their own servers).
78
+
79
+ # requirepass foobared
80
+
81
+ ################################### LIMITS ####################################
82
+
83
+ # Set the max number of connected clients at the same time. By default there
84
+ # is no limit, and it's up to the number of file descriptors the Redis process
85
+ # is able to open. The special value '0' means no limts.
86
+ # Once the limit is reached Redis will close all the new connections sending
87
+ # an error 'max number of clients reached'.
88
+
89
+ # maxclients 128
90
+
91
+ # Don't use more memory than the specified amount of bytes.
92
+ # When the memory limit is reached Redis will try to remove keys with an
93
+ # EXPIRE set. It will try to start freeing keys that are going to expire
94
+ # in little time and preserve keys with a longer time to live.
95
+ # Redis will also try to remove objects from free lists if possible.
96
+ #
97
+ # If all this fails, Redis will start to reply with errors to commands
98
+ # that will use more memory, like SET, LPUSH, and so on, and will continue
99
+ # to reply to most read-only commands like GET.
100
+ #
101
+ # WARNING: maxmemory can be a good idea mainly if you want to use Redis as a
102
+ # 'state' server or cache, not as a real DB. When Redis is used as a real
103
+ # database the memory usage will grow over the weeks, it will be obvious if
104
+ # it is going to use too much memory in the long run, and you'll have the time
105
+ # to upgrade. With maxmemory after the limit is reached you'll start to get
106
+ # errors for write operations, and this may even lead to DB inconsistency.
107
+
108
+ # maxmemory <bytes>
109
+
110
+ ############################### ADVANCED CONFIG ###############################
111
+
112
+ # Glue small output buffers together in order to send small replies in a
113
+ # single TCP packet. Uses a bit more CPU but most of the times it is a win
114
+ # in terms of number of queries per second. Use 'yes' if unsure.
115
+ glueoutputbuf yes
116
+
117
+ # Use object sharing. Can save a lot of memory if you have many common
118
+ # string in your dataset, but performs lookups against the shared objects
119
+ # pool so it uses more CPU and can be a bit slower. Usually it's a good
120
+ # idea.
121
+ #
122
+ # When object sharing is enabled (shareobjects yes) you can use
123
+ # shareobjectspoolsize to control the size of the pool used in order to try
124
+ # object sharing. A bigger pool size will lead to better sharing capabilities.
125
+ # In general you want this value to be at least the double of the number of
126
+ # very common strings you have in your dataset.
127
+ #
128
+ # WARNING: object sharing is experimental, don't enable this feature
129
+ # in production before of Redis 1.0-stable. Still please try this feature in
130
+ # your development environment so that we can test it better.
131
+ shareobjects no
132
+ shareobjectspoolsize 1024
@@ -0,0 +1,191 @@
1
+ require File.dirname(__FILE__) + '/test_helper'
2
+
3
+ context "Resque" do
4
+ setup do
5
+ Resque.drop
6
+
7
+ Resque.push(:people, { 'name' => 'chris' })
8
+ Resque.push(:people, { 'name' => 'bob' })
9
+ Resque.push(:people, { 'name' => 'mark' })
10
+ end
11
+
12
+ test "can put jobs on a queue" do
13
+ assert Resque::Job.create(:jobs, 'SomeJob', 20, '/tmp')
14
+ assert Resque::Job.create(:jobs, 'SomeJob', 20, '/tmp')
15
+ end
16
+
17
+ test "can grab jobs off a queue" do
18
+ Resque::Job.create(:jobs, 'some-job', 20, '/tmp')
19
+
20
+ job = Resque.reserve(:jobs)
21
+
22
+ assert_kind_of Resque::Job, job
23
+ assert_equal SomeJob, job.payload_class
24
+ assert_equal 20, job.args[0]
25
+ assert_equal '/tmp', job.args[1]
26
+ end
27
+
28
+ test "can re-queue jobs" do
29
+ Resque::Job.create(:jobs, 'some-job', 20, '/tmp')
30
+
31
+ job = Resque.reserve(:jobs)
32
+ job.recreate
33
+
34
+ assert_equal job, Resque.reserve(:jobs)
35
+ end
36
+
37
+ test "can put jobs on a queue by way of an ivar" do
38
+ assert_equal 0, Resque.size(:ivar)
39
+ assert Resque.enqueue(SomeIvarJob, 20, '/tmp')
40
+ assert Resque.enqueue(SomeIvarJob, 20, '/tmp')
41
+
42
+ job = Resque.reserve(:ivar)
43
+
44
+ assert_kind_of Resque::Job, job
45
+ assert_equal SomeIvarJob, job.payload_class
46
+ assert_equal 20, job.args[0]
47
+ assert_equal '/tmp', job.args[1]
48
+
49
+ assert Resque.reserve(:ivar)
50
+ assert_equal nil, Resque.reserve(:ivar)
51
+ end
52
+
53
+ test "jobs have a nice #inspect" do
54
+ assert Resque::Job.create(:jobs, 'SomeJob', 20, '/tmp')
55
+ job = Resque.reserve(:jobs)
56
+ assert_equal '(Job{jobs} | SomeJob | [20, "/tmp"])', job.inspect
57
+ end
58
+
59
+ test "jobs can test for equality" do
60
+ assert Resque::Job.create(:jobs, 'SomeJob', 20, '/tmp')
61
+ assert Resque::Job.create(:jobs, 'some-job', 20, '/tmp')
62
+ assert_equal Resque.reserve(:jobs), Resque.reserve(:jobs)
63
+
64
+ assert Resque::Job.create(:jobs, 'SomeMethodJob', 20, '/tmp')
65
+ assert Resque::Job.create(:jobs, 'SomeJob', 20, '/tmp')
66
+ assert_not_equal Resque.reserve(:jobs), Resque.reserve(:jobs)
67
+
68
+ assert Resque::Job.create(:jobs, 'SomeJob', 20, '/tmp')
69
+ assert Resque::Job.create(:jobs, 'SomeJob', 30, '/tmp')
70
+ assert_not_equal Resque.reserve(:jobs), Resque.reserve(:jobs)
71
+ end
72
+
73
+ test "can put jobs on a queue by way of a method" do
74
+ assert_equal 0, Resque.size(:method)
75
+ assert Resque.enqueue(SomeMethodJob, 20, '/tmp')
76
+ assert Resque.enqueue(SomeMethodJob, 20, '/tmp')
77
+
78
+ job = Resque.reserve(:method)
79
+
80
+ assert_kind_of Resque::Job, job
81
+ assert_equal SomeMethodJob, job.payload_class
82
+ assert_equal 20, job.args[0]
83
+ assert_equal '/tmp', job.args[1]
84
+
85
+ assert Resque.reserve(:method)
86
+ assert_equal nil, Resque.reserve(:method)
87
+ end
88
+
89
+ test "needs to infer a queue with enqueue" do
90
+ assert_raises Resque::NoQueueError do
91
+ Resque.enqueue(SomeJob, 20, '/tmp')
92
+ end
93
+ end
94
+
95
+ test "can put items on a queue" do
96
+ assert Resque.push(:people, { 'name' => 'jon' })
97
+ end
98
+
99
+ test "can pull items off a queue" do
100
+ assert_equal({ 'name' => 'chris' }, Resque.pop(:people))
101
+ assert_equal({ 'name' => 'bob' }, Resque.pop(:people))
102
+ assert_equal({ 'name' => 'mark' }, Resque.pop(:people))
103
+ assert_equal nil, Resque.pop(:people)
104
+ end
105
+
106
+ test "knows how big a queue is" do
107
+ assert_equal 3, Resque.size(:people)
108
+
109
+ assert_equal({ 'name' => 'chris' }, Resque.pop(:people))
110
+ assert_equal 2, Resque.size(:people)
111
+
112
+ assert_equal({ 'name' => 'bob' }, Resque.pop(:people))
113
+ assert_equal({ 'name' => 'mark' }, Resque.pop(:people))
114
+ assert_equal 0, Resque.size(:people)
115
+ end
116
+
117
+ test "can peek at a queue" do
118
+ assert_equal({ 'name' => 'chris' }, Resque.peek(:people))
119
+ assert_equal 3, Resque.size(:people)
120
+ end
121
+
122
+ test "can peek multiple items on a queue" do
123
+ assert_equal({ 'name' => 'bob' }, Resque.peek(:people, 1, 1))
124
+
125
+ assert_equal([{ 'name' => 'bob' }, { 'name' => 'mark' }], Resque.peek(:people, 1, 2))
126
+ assert_equal([{ 'name' => 'chris' }, { 'name' => 'bob' }], Resque.peek(:people, 0, 2))
127
+ assert_equal([{ 'name' => 'chris' }, { 'name' => 'bob' }, { 'name' => 'mark' }], Resque.peek(:people, 0, 3))
128
+ assert_equal({ 'name' => 'mark' }, Resque.peek(:people, 2, 1))
129
+ assert_equal nil, Resque.peek(:people, 3)
130
+ assert_equal [], Resque.peek(:people, 3, 2)
131
+ end
132
+
133
+ test "knows what queues it is managing" do
134
+ assert_equal %w( people ), Resque.queues
135
+ Resque.push(:cars, { 'make' => 'bmw' })
136
+ assert_equal %w( cars people ), Resque.queues
137
+ end
138
+
139
+ test "queues are always a list" do
140
+ Resque.drop
141
+ assert_equal [], Resque.queues
142
+ end
143
+
144
+ test "can delete a queue" do
145
+ Resque.push(:cars, { 'make' => 'bmw' })
146
+ assert_equal %w( cars people ), Resque.queues
147
+ Resque.remove_queue(:people)
148
+ assert_equal %w( cars ), Resque.queues
149
+ assert_equal nil, Resque.pop(:people)
150
+ end
151
+
152
+ test "keeps track of resque keys" do
153
+ assert_equal ["people"], Resque.keys
154
+ end
155
+
156
+ test "badly wants a class name, too" do
157
+ assert_raises Resque::NoClassError do
158
+ Resque::Job.create(:jobs, nil)
159
+ end
160
+ end
161
+
162
+ test "keeps stats" do
163
+ Resque::Job.create(:jobs, SomeJob, 20, '/tmp')
164
+ Resque::Job.create(:jobs, BadJob)
165
+ Resque::Job.create(:jobs, GoodJob)
166
+
167
+ Resque::Job.create(:others, GoodJob)
168
+ Resque::Job.create(:others, GoodJob)
169
+
170
+ stats = Resque.info
171
+ assert_equal 8, stats[:pending]
172
+
173
+ @worker = Resque::Worker.new(:jobs)
174
+ @worker.register_worker
175
+ 2.times { @worker.process }
176
+ job = @worker.reserve
177
+ @worker.working_on job
178
+
179
+ stats = Resque.info
180
+ assert_equal 1, stats[:working]
181
+ assert_equal 1, stats[:workers]
182
+
183
+ @worker.done_working
184
+
185
+ stats = Resque.info
186
+ assert_equal 2, stats[:queues]
187
+ assert_equal 3, stats[:processed]
188
+ assert_equal 1, stats[:failed]
189
+ assert_equal ['localhost:27017'], stats[:servers]
190
+ end
191
+ end