resque-mongo 1.3.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (61) hide show
  1. data/.gitignore +3 -0
  2. data/.kick +26 -0
  3. data/CONTRIBUTORS +14 -0
  4. data/HISTORY.md +53 -0
  5. data/LICENSE +20 -0
  6. data/README.markdown +755 -0
  7. data/Rakefile +65 -0
  8. data/bin/resque +57 -0
  9. data/bin/resque-web +18 -0
  10. data/config.ru +14 -0
  11. data/deps.rip +8 -0
  12. data/examples/async_helper.rb +31 -0
  13. data/examples/demo/README.markdown +71 -0
  14. data/examples/demo/Rakefile +3 -0
  15. data/examples/demo/app.rb +27 -0
  16. data/examples/demo/config.ru +19 -0
  17. data/examples/demo/job.rb +12 -0
  18. data/examples/god/resque.god +52 -0
  19. data/examples/god/stale.god +26 -0
  20. data/examples/instance.rb +11 -0
  21. data/examples/simple.rb +30 -0
  22. data/init.rb +1 -0
  23. data/lib/resque.rb +224 -0
  24. data/lib/resque/errors.rb +7 -0
  25. data/lib/resque/failure.rb +63 -0
  26. data/lib/resque/failure/base.rb +58 -0
  27. data/lib/resque/failure/hoptoad.rb +88 -0
  28. data/lib/resque/failure/mongo.rb +32 -0
  29. data/lib/resque/helpers.rb +65 -0
  30. data/lib/resque/job.rb +103 -0
  31. data/lib/resque/server.rb +182 -0
  32. data/lib/resque/server/public/idle.png +0 -0
  33. data/lib/resque/server/public/jquery-1.3.2.min.js +19 -0
  34. data/lib/resque/server/public/jquery.relatize_date.js +95 -0
  35. data/lib/resque/server/public/poll.png +0 -0
  36. data/lib/resque/server/public/ranger.js +24 -0
  37. data/lib/resque/server/public/reset.css +48 -0
  38. data/lib/resque/server/public/style.css +75 -0
  39. data/lib/resque/server/public/working.png +0 -0
  40. data/lib/resque/server/views/error.erb +1 -0
  41. data/lib/resque/server/views/failed.erb +35 -0
  42. data/lib/resque/server/views/key.erb +17 -0
  43. data/lib/resque/server/views/layout.erb +38 -0
  44. data/lib/resque/server/views/next_more.erb +10 -0
  45. data/lib/resque/server/views/overview.erb +4 -0
  46. data/lib/resque/server/views/queues.erb +46 -0
  47. data/lib/resque/server/views/stats.erb +62 -0
  48. data/lib/resque/server/views/workers.erb +78 -0
  49. data/lib/resque/server/views/working.erb +67 -0
  50. data/lib/resque/stat.rb +55 -0
  51. data/lib/resque/tasks.rb +39 -0
  52. data/lib/resque/version.rb +3 -0
  53. data/lib/resque/worker.rb +415 -0
  54. data/tasks/redis.rake +133 -0
  55. data/tasks/resque.rake +2 -0
  56. data/test/dump.rdb +0 -0
  57. data/test/redis-test.conf +132 -0
  58. data/test/resque_test.rb +191 -0
  59. data/test/test_helper.rb +87 -0
  60. data/test/worker_test.rb +229 -0
  61. metadata +162 -0
@@ -0,0 +1,133 @@
1
+ # Inspired by rabbitmq.rake the Redbox project at http://github.com/rick/redbox/tree/master
2
+ require 'fileutils'
3
+ require 'open-uri'
4
+
5
+ class RedisRunner
6
+
7
+ def self.redisdir
8
+ "/tmp/redis/"
9
+ end
10
+
11
+ def self.redisconfdir
12
+ server_dir = File.dirname(`which redis-server`)
13
+ conf_file = "#{server_dir}/../etc/redis.conf"
14
+ unless File.exists? conf_file
15
+ conf_file = "#{server_dir}/../../etc/redis.conf"
16
+ end
17
+ conf_file
18
+ end
19
+
20
+ def self.dtach_socket
21
+ '/tmp/redis.dtach'
22
+ end
23
+
24
+ # Just check for existance of dtach socket
25
+ def self.running?
26
+ File.exists? dtach_socket
27
+ end
28
+
29
+ def self.start
30
+ puts 'Detach with Ctrl+\ Re-attach with rake redis:attach'
31
+ sleep 1
32
+ exec "dtach -A #{dtach_socket} redis-server #{redisconfdir}"
33
+ end
34
+
35
+ def self.attach
36
+ exec "dtach -a #{dtach_socket}"
37
+ end
38
+
39
+ def self.stop
40
+ sh 'echo "SHUTDOWN" | nc localhost 6379'
41
+ end
42
+
43
+ end
44
+
45
+ namespace :redis do
46
+
47
+ desc 'About redis'
48
+ task :about do
49
+ puts "\nSee http://code.google.com/p/redis/ for information about redis.\n\n"
50
+ end
51
+
52
+ desc 'Start redis'
53
+ task :start do
54
+ RedisRunner.start
55
+ end
56
+
57
+ desc 'Stop redis'
58
+ task :stop do
59
+ RedisRunner.stop
60
+ end
61
+
62
+ desc 'Restart redis'
63
+ task :restart do
64
+ RedisRunner.stop
65
+ RedisRunner.start
66
+ end
67
+
68
+ desc 'Attach to redis dtach socket'
69
+ task :attach do
70
+ RedisRunner.attach
71
+ end
72
+
73
+ desc 'Install the latest verison of Redis from Github (requires git, duh)'
74
+ task :install => [:about, :download, :make] do
75
+ ENV['PREFIX'] and bin_dir = "#{ENV['PREFIX']}/bin" or bin_dir = '/usr/bin'
76
+ %w(redis-benchmark redis-cli redis-server).each do |bin|
77
+ sh "cp /tmp/redis/#{bin} #{bin_dir}"
78
+ end
79
+
80
+ puts "Installed redis-benchmark, redis-cli and redis-server to #{bin_dir}"
81
+
82
+ ENV['PREFIX'] and conf_dir = "#{ENV['PREFIX']}/etc" or conf_dir = '/etc'
83
+ unless File.exists?("#{conf_dir}")
84
+ sh "cp /tmp/redis/redis.conf #{conf_dir}"
85
+ puts "Installed redis.conf to #{conf_dir} \n You should look at this file!"
86
+ end
87
+ end
88
+
89
+ task :make do
90
+ sh "cd #{RedisRunner.redisdir} && make clean"
91
+ sh "cd #{RedisRunner.redisdir} && make"
92
+ end
93
+
94
+ desc "Download package"
95
+ task :download do
96
+ sh 'rm -rf /tmp/redis/' if File.exists?("#{RedisRunner.redisdir}/.svn")
97
+ sh 'git clone git://github.com/antirez/redis.git /tmp/redis' unless File.exists?(RedisRunner.redisdir)
98
+ sh "cd #{RedisRunner.redisdir} && git pull" if File.exists?("#{RedisRunner.redisdir}/.git")
99
+ end
100
+
101
+ end
102
+
103
+ namespace :dtach do
104
+
105
+ desc 'About dtach'
106
+ task :about do
107
+ puts "\nSee http://dtach.sourceforge.net/ for information about dtach.\n\n"
108
+ end
109
+
110
+ desc 'Install dtach 0.8 from source'
111
+ task :install => [:about] do
112
+
113
+ Dir.chdir('/tmp/')
114
+ unless File.exists?('/tmp/dtach-0.8.tar.gz')
115
+ require 'net/http'
116
+
117
+ url = 'http://downloads.sourceforge.net/project/dtach/dtach/0.8/dtach-0.8.tar.gz'
118
+ open('/tmp/dtach-0.8.tar.gz', 'wb') do |file| file.write(open(url).read) end
119
+ end
120
+
121
+ unless File.directory?('/tmp/dtach-0.8')
122
+ system('tar xzf dtach-0.8.tar.gz')
123
+ end
124
+
125
+ ENV['PREFIX'] and bin_dir = "#{ENV['PREFIX']}/bin" or bin_dir = "/usr/bin"
126
+ Dir.chdir('/tmp/dtach-0.8/')
127
+ sh 'cd /tmp/dtach-0.8/ && ./configure && make'
128
+ sh "cp /tmp/dtach-0.8/dtach #{bin_dir}"
129
+
130
+ puts "Dtach successfully installed to #{bin_dir}"
131
+ end
132
+ end
133
+
@@ -0,0 +1,2 @@
1
+ $LOAD_PATH.unshift File.dirname(__FILE__) + '/../lib'
2
+ require 'resque/tasks'
Binary file
@@ -0,0 +1,132 @@
1
+ # Redis configuration file example
2
+
3
+ # By default Redis does not run as a daemon. Use 'yes' if you need it.
4
+ # Note that Redis will write a pid file in /var/run/redis.pid when daemonized.
5
+ daemonize yes
6
+
7
+ # When run as a daemon, Redis write a pid file in /var/run/redis.pid by default.
8
+ # You can specify a custom pid file location here.
9
+ pidfile ./test/redis-test.pid
10
+
11
+ # Accept connections on the specified port, default is 6379
12
+ port 9736
13
+
14
+ # If you want you can bind a single interface, if the bind option is not
15
+ # specified all the interfaces will listen for connections.
16
+ #
17
+ # bind 127.0.0.1
18
+
19
+ # Close the connection after a client is idle for N seconds (0 to disable)
20
+ timeout 300
21
+
22
+ # Save the DB on disk:
23
+ #
24
+ # save <seconds> <changes>
25
+ #
26
+ # Will save the DB if both the given number of seconds and the given
27
+ # number of write operations against the DB occurred.
28
+ #
29
+ # In the example below the behaviour will be to save:
30
+ # after 900 sec (15 min) if at least 1 key changed
31
+ # after 300 sec (5 min) if at least 10 keys changed
32
+ # after 60 sec if at least 10000 keys changed
33
+ save 900 1
34
+ save 300 10
35
+ save 60 10000
36
+
37
+ # The filename where to dump the DB
38
+ dbfilename dump.rdb
39
+
40
+ # For default save/load DB in/from the working directory
41
+ # Note that you must specify a directory not a file name.
42
+ dir ./test/
43
+
44
+ # Set server verbosity to 'debug'
45
+ # it can be one of:
46
+ # debug (a lot of information, useful for development/testing)
47
+ # notice (moderately verbose, what you want in production probably)
48
+ # warning (only very important / critical messages are logged)
49
+ loglevel debug
50
+
51
+ # Specify the log file name. Also 'stdout' can be used to force
52
+ # the demon to log on the standard output. Note that if you use standard
53
+ # output for logging but daemonize, logs will be sent to /dev/null
54
+ logfile stdout
55
+
56
+ # Set the number of databases. The default database is DB 0, you can select
57
+ # a different one on a per-connection basis using SELECT <dbid> where
58
+ # dbid is a number between 0 and 'databases'-1
59
+ databases 16
60
+
61
+ ################################# REPLICATION #################################
62
+
63
+ # Master-Slave replication. Use slaveof to make a Redis instance a copy of
64
+ # another Redis server. Note that the configuration is local to the slave
65
+ # so for example it is possible to configure the slave to save the DB with a
66
+ # different interval, or to listen to another port, and so on.
67
+
68
+ # slaveof <masterip> <masterport>
69
+
70
+ ################################## SECURITY ###################################
71
+
72
+ # Require clients to issue AUTH <PASSWORD> before processing any other
73
+ # commands. This might be useful in environments in which you do not trust
74
+ # others with access to the host running redis-server.
75
+ #
76
+ # This should stay commented out for backward compatibility and because most
77
+ # people do not need auth (e.g. they run their own servers).
78
+
79
+ # requirepass foobared
80
+
81
+ ################################### LIMITS ####################################
82
+
83
+ # Set the max number of connected clients at the same time. By default there
84
+ # is no limit, and it's up to the number of file descriptors the Redis process
85
+ # is able to open. The special value '0' means no limts.
86
+ # Once the limit is reached Redis will close all the new connections sending
87
+ # an error 'max number of clients reached'.
88
+
89
+ # maxclients 128
90
+
91
+ # Don't use more memory than the specified amount of bytes.
92
+ # When the memory limit is reached Redis will try to remove keys with an
93
+ # EXPIRE set. It will try to start freeing keys that are going to expire
94
+ # in little time and preserve keys with a longer time to live.
95
+ # Redis will also try to remove objects from free lists if possible.
96
+ #
97
+ # If all this fails, Redis will start to reply with errors to commands
98
+ # that will use more memory, like SET, LPUSH, and so on, and will continue
99
+ # to reply to most read-only commands like GET.
100
+ #
101
+ # WARNING: maxmemory can be a good idea mainly if you want to use Redis as a
102
+ # 'state' server or cache, not as a real DB. When Redis is used as a real
103
+ # database the memory usage will grow over the weeks, it will be obvious if
104
+ # it is going to use too much memory in the long run, and you'll have the time
105
+ # to upgrade. With maxmemory after the limit is reached you'll start to get
106
+ # errors for write operations, and this may even lead to DB inconsistency.
107
+
108
+ # maxmemory <bytes>
109
+
110
+ ############################### ADVANCED CONFIG ###############################
111
+
112
+ # Glue small output buffers together in order to send small replies in a
113
+ # single TCP packet. Uses a bit more CPU but most of the times it is a win
114
+ # in terms of number of queries per second. Use 'yes' if unsure.
115
+ glueoutputbuf yes
116
+
117
+ # Use object sharing. Can save a lot of memory if you have many common
118
+ # string in your dataset, but performs lookups against the shared objects
119
+ # pool so it uses more CPU and can be a bit slower. Usually it's a good
120
+ # idea.
121
+ #
122
+ # When object sharing is enabled (shareobjects yes) you can use
123
+ # shareobjectspoolsize to control the size of the pool used in order to try
124
+ # object sharing. A bigger pool size will lead to better sharing capabilities.
125
+ # In general you want this value to be at least the double of the number of
126
+ # very common strings you have in your dataset.
127
+ #
128
+ # WARNING: object sharing is experimental, don't enable this feature
129
+ # in production before of Redis 1.0-stable. Still please try this feature in
130
+ # your development environment so that we can test it better.
131
+ shareobjects no
132
+ shareobjectspoolsize 1024
@@ -0,0 +1,191 @@
1
+ require File.dirname(__FILE__) + '/test_helper'
2
+
3
+ context "Resque" do
4
+ setup do
5
+ Resque.drop
6
+
7
+ Resque.push(:people, { 'name' => 'chris' })
8
+ Resque.push(:people, { 'name' => 'bob' })
9
+ Resque.push(:people, { 'name' => 'mark' })
10
+ end
11
+
12
+ test "can put jobs on a queue" do
13
+ assert Resque::Job.create(:jobs, 'SomeJob', 20, '/tmp')
14
+ assert Resque::Job.create(:jobs, 'SomeJob', 20, '/tmp')
15
+ end
16
+
17
+ test "can grab jobs off a queue" do
18
+ Resque::Job.create(:jobs, 'some-job', 20, '/tmp')
19
+
20
+ job = Resque.reserve(:jobs)
21
+
22
+ assert_kind_of Resque::Job, job
23
+ assert_equal SomeJob, job.payload_class
24
+ assert_equal 20, job.args[0]
25
+ assert_equal '/tmp', job.args[1]
26
+ end
27
+
28
+ test "can re-queue jobs" do
29
+ Resque::Job.create(:jobs, 'some-job', 20, '/tmp')
30
+
31
+ job = Resque.reserve(:jobs)
32
+ job.recreate
33
+
34
+ assert_equal job, Resque.reserve(:jobs)
35
+ end
36
+
37
+ test "can put jobs on a queue by way of an ivar" do
38
+ assert_equal 0, Resque.size(:ivar)
39
+ assert Resque.enqueue(SomeIvarJob, 20, '/tmp')
40
+ assert Resque.enqueue(SomeIvarJob, 20, '/tmp')
41
+
42
+ job = Resque.reserve(:ivar)
43
+
44
+ assert_kind_of Resque::Job, job
45
+ assert_equal SomeIvarJob, job.payload_class
46
+ assert_equal 20, job.args[0]
47
+ assert_equal '/tmp', job.args[1]
48
+
49
+ assert Resque.reserve(:ivar)
50
+ assert_equal nil, Resque.reserve(:ivar)
51
+ end
52
+
53
+ test "jobs have a nice #inspect" do
54
+ assert Resque::Job.create(:jobs, 'SomeJob', 20, '/tmp')
55
+ job = Resque.reserve(:jobs)
56
+ assert_equal '(Job{jobs} | SomeJob | [20, "/tmp"])', job.inspect
57
+ end
58
+
59
+ test "jobs can test for equality" do
60
+ assert Resque::Job.create(:jobs, 'SomeJob', 20, '/tmp')
61
+ assert Resque::Job.create(:jobs, 'some-job', 20, '/tmp')
62
+ assert_equal Resque.reserve(:jobs), Resque.reserve(:jobs)
63
+
64
+ assert Resque::Job.create(:jobs, 'SomeMethodJob', 20, '/tmp')
65
+ assert Resque::Job.create(:jobs, 'SomeJob', 20, '/tmp')
66
+ assert_not_equal Resque.reserve(:jobs), Resque.reserve(:jobs)
67
+
68
+ assert Resque::Job.create(:jobs, 'SomeJob', 20, '/tmp')
69
+ assert Resque::Job.create(:jobs, 'SomeJob', 30, '/tmp')
70
+ assert_not_equal Resque.reserve(:jobs), Resque.reserve(:jobs)
71
+ end
72
+
73
+ test "can put jobs on a queue by way of a method" do
74
+ assert_equal 0, Resque.size(:method)
75
+ assert Resque.enqueue(SomeMethodJob, 20, '/tmp')
76
+ assert Resque.enqueue(SomeMethodJob, 20, '/tmp')
77
+
78
+ job = Resque.reserve(:method)
79
+
80
+ assert_kind_of Resque::Job, job
81
+ assert_equal SomeMethodJob, job.payload_class
82
+ assert_equal 20, job.args[0]
83
+ assert_equal '/tmp', job.args[1]
84
+
85
+ assert Resque.reserve(:method)
86
+ assert_equal nil, Resque.reserve(:method)
87
+ end
88
+
89
+ test "needs to infer a queue with enqueue" do
90
+ assert_raises Resque::NoQueueError do
91
+ Resque.enqueue(SomeJob, 20, '/tmp')
92
+ end
93
+ end
94
+
95
+ test "can put items on a queue" do
96
+ assert Resque.push(:people, { 'name' => 'jon' })
97
+ end
98
+
99
+ test "can pull items off a queue" do
100
+ assert_equal({ 'name' => 'chris' }, Resque.pop(:people))
101
+ assert_equal({ 'name' => 'bob' }, Resque.pop(:people))
102
+ assert_equal({ 'name' => 'mark' }, Resque.pop(:people))
103
+ assert_equal nil, Resque.pop(:people)
104
+ end
105
+
106
+ test "knows how big a queue is" do
107
+ assert_equal 3, Resque.size(:people)
108
+
109
+ assert_equal({ 'name' => 'chris' }, Resque.pop(:people))
110
+ assert_equal 2, Resque.size(:people)
111
+
112
+ assert_equal({ 'name' => 'bob' }, Resque.pop(:people))
113
+ assert_equal({ 'name' => 'mark' }, Resque.pop(:people))
114
+ assert_equal 0, Resque.size(:people)
115
+ end
116
+
117
+ test "can peek at a queue" do
118
+ assert_equal({ 'name' => 'chris' }, Resque.peek(:people))
119
+ assert_equal 3, Resque.size(:people)
120
+ end
121
+
122
+ test "can peek multiple items on a queue" do
123
+ assert_equal({ 'name' => 'bob' }, Resque.peek(:people, 1, 1))
124
+
125
+ assert_equal([{ 'name' => 'bob' }, { 'name' => 'mark' }], Resque.peek(:people, 1, 2))
126
+ assert_equal([{ 'name' => 'chris' }, { 'name' => 'bob' }], Resque.peek(:people, 0, 2))
127
+ assert_equal([{ 'name' => 'chris' }, { 'name' => 'bob' }, { 'name' => 'mark' }], Resque.peek(:people, 0, 3))
128
+ assert_equal({ 'name' => 'mark' }, Resque.peek(:people, 2, 1))
129
+ assert_equal nil, Resque.peek(:people, 3)
130
+ assert_equal [], Resque.peek(:people, 3, 2)
131
+ end
132
+
133
+ test "knows what queues it is managing" do
134
+ assert_equal %w( people ), Resque.queues
135
+ Resque.push(:cars, { 'make' => 'bmw' })
136
+ assert_equal %w( cars people ), Resque.queues
137
+ end
138
+
139
+ test "queues are always a list" do
140
+ Resque.drop
141
+ assert_equal [], Resque.queues
142
+ end
143
+
144
+ test "can delete a queue" do
145
+ Resque.push(:cars, { 'make' => 'bmw' })
146
+ assert_equal %w( cars people ), Resque.queues
147
+ Resque.remove_queue(:people)
148
+ assert_equal %w( cars ), Resque.queues
149
+ assert_equal nil, Resque.pop(:people)
150
+ end
151
+
152
+ test "keeps track of resque keys" do
153
+ assert_equal ["people"], Resque.keys
154
+ end
155
+
156
+ test "badly wants a class name, too" do
157
+ assert_raises Resque::NoClassError do
158
+ Resque::Job.create(:jobs, nil)
159
+ end
160
+ end
161
+
162
+ test "keeps stats" do
163
+ Resque::Job.create(:jobs, SomeJob, 20, '/tmp')
164
+ Resque::Job.create(:jobs, BadJob)
165
+ Resque::Job.create(:jobs, GoodJob)
166
+
167
+ Resque::Job.create(:others, GoodJob)
168
+ Resque::Job.create(:others, GoodJob)
169
+
170
+ stats = Resque.info
171
+ assert_equal 8, stats[:pending]
172
+
173
+ @worker = Resque::Worker.new(:jobs)
174
+ @worker.register_worker
175
+ 2.times { @worker.process }
176
+ job = @worker.reserve
177
+ @worker.working_on job
178
+
179
+ stats = Resque.info
180
+ assert_equal 1, stats[:working]
181
+ assert_equal 1, stats[:workers]
182
+
183
+ @worker.done_working
184
+
185
+ stats = Resque.info
186
+ assert_equal 2, stats[:queues]
187
+ assert_equal 3, stats[:processed]
188
+ assert_equal 1, stats[:failed]
189
+ assert_equal ['localhost:27017'], stats[:servers]
190
+ end
191
+ end