resque-filter 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,9 @@
1
+ *.gem
2
+ *.DS_Store
3
+ .bundle
4
+ Gemfile.lock
5
+ pkg/*
6
+ .idea
7
+ spec/dump.rdb
8
+ spec/redis-server.log
9
+ spec/redis.pid
@@ -0,0 +1,10 @@
1
+ language: ruby
2
+ rvm:
3
+ - 1.8.7
4
+ - 1.9.2
5
+ - 1.9.3
6
+ - jruby-18mode # JRuby in 1.8 mode
7
+ - jruby-19mode # JRuby in 1.9 mode
8
+ - rbx-18mode
9
+ # - rbx-19mode
10
+ script: bundle exec rspec spec
@@ -0,0 +1,24 @@
1
+ 0.6.0
2
+ -----
3
+
4
+ Allow multiple patterns per priority bucket, along with negation patterns
5
+
6
+ 0.5.3
7
+ -----
8
+
9
+ these are not the cutnpaste errors you are looking for <7f20764> [Matt Conway]
10
+
11
+ 0.5.2
12
+ -----
13
+
14
+ fix helper method name clash <afbd5f4> [Matt Conway]
15
+
16
+ 0.5.1
17
+ -----
18
+
19
+ Fixes to UI <d2742ae> [Matt Conway]
20
+
21
+ 0.5.0
22
+ -----
23
+
24
+ Initial version
data/Gemfile ADDED
@@ -0,0 +1,4 @@
1
+ source "http://rubygems.org"
2
+
3
+ # Specify your gem's dependencies in resque-filter.gemspec
4
+ gemspec
data/LICENSE ADDED
@@ -0,0 +1,20 @@
1
+ Copyright (c) 2011 Matt Conway (matt@conwaysplace.com)
2
+
3
+ Permission is hereby granted, free of charge, to any person obtaining
4
+ a copy of this software and associated documentation files (the
5
+ "Software"), to deal in the Software without restriction, including
6
+ without limitation the rights to use, copy, modify, merge, publish,
7
+ distribute, sublicense, and/or sell copies of the Software, and to
8
+ permit persons to whom the Software is furnished to do so, subject to
9
+ the following conditions:
10
+
11
+ The above copyright notice and this permission notice shall be
12
+ included in all copies or substantial portions of the Software.
13
+
14
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
15
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
16
+ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
17
+ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
18
+ LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
19
+ OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
20
+ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
@@ -0,0 +1,35 @@
1
+ A resque plugin that allows jobs to execute only if a filter is true, re-queueing otherwise. For example, one can author jobs that are filter-aware so that they can be scheduled with resque-scheduler to run on specific hosts.
2
+
3
+ Authored against Resque 1.20.0, so it at least works with that - try running the tests if you use a different version of resque
4
+
5
+ [![Build Status](https://secure.travis-ci.org/wr0ngway/resque-filter.png)](http://travis-ci.org/wr0ngway/resque-filter)
6
+
7
+ Usage:
8
+
9
+ Install the gem, then define a worker with a filter method. If that method returns true, the job will run, otherwise it will get re-enqueued.
10
+
11
+ class MyFilteredWorker
12
+
13
+ def self.perform(*args)
14
+ puts "I ran"
15
+ end
16
+
17
+ def self.filter(*args)
18
+ return `hostname`.chomp == args.first
19
+ end
20
+
21
+ end
22
+
23
+
24
+ You can also configure an alternate queue management strategy like so:
25
+
26
+ Resque::Plugins::Filter::JobFilter.configure do |config|
27
+ # The queue strategy to use when filtering job:
28
+ # :simple - pops, checks filter, pushes if not runnable
29
+ # :optimistic - peeks, checks filter, pops if runnable (not distributed client safe)
30
+ config.strategy = :simple
31
+ end
32
+
33
+ Contributors:
34
+
35
+ Matt Conway ( https://github.com/wr0ngway )
@@ -0,0 +1,67 @@
1
+ require 'bundler'
2
+ Bundler::GemHelper.install_tasks
3
+
4
+ task :my_release => ['changelog', 'release'] do
5
+ end
6
+
7
+ task :changelog do
8
+
9
+ helper = Bundler::GemHelper.new(Dir.pwd)
10
+ version = "v#{helper.gemspec.version}"
11
+
12
+ changelog_file = 'CHANGELOG'
13
+ entries = ""
14
+
15
+ # Get a list of current tags
16
+ tags = `git tag -l`.split
17
+ tags = tags.sort_by {|t| t[1..-1].split(".").collect {|s| s.to_i } }
18
+ newest_tag = tags[-1]
19
+
20
+ if version == newest_tag
21
+ puts "You need to update version, same as most recent tag: #{version}"
22
+ exit
23
+ end
24
+
25
+ # If we already have a changelog, make the last tag be the
26
+ # last one in the changelog, and the next one be the one
27
+ # following that in the tag list
28
+ newest_changelog_version = nil
29
+ if File.exist?(changelog_file)
30
+ entries = File.read(changelog_file)
31
+ head = entries.split.first
32
+ if head =~ /\d\.\d\.\d/
33
+ newest_changelog_version = "v#{head}"
34
+
35
+ if version == newest_changelog_version
36
+ puts "You need to update version, same as most recent changelog: #{version}"
37
+ exit
38
+ end
39
+
40
+ end
41
+ end
42
+
43
+ # Generate changelog from repo
44
+ log=`git log --pretty='format:%s <%h> [%cn]' #{newest_tag}..HEAD`
45
+
46
+ # Strip out maintenance entries
47
+ log = log.lines.to_a.delete_if do |l|
48
+ l =~ /^Regenerated? gemspec/ ||
49
+ l =~ /^version bump/i ||
50
+ l =~ /^Updated changelog/ ||
51
+ l =~ /^Merged? branch/
52
+ end
53
+
54
+ # Write out changelog file
55
+ File.open(changelog_file, 'w') do |out|
56
+ out.puts version.gsub(/^v/, '')
57
+ out.puts "-----"
58
+ out.puts "\n"
59
+ out.puts log
60
+ out.puts "\n"
61
+ out.puts entries
62
+ end
63
+
64
+ # Commit and push
65
+ sh "git ci -m'Updated changelog' #{changelog_file}"
66
+ sh "git push"
67
+ end
@@ -0,0 +1,6 @@
1
+ require 'resque'
2
+ require 'resque/job'
3
+ require 'resque/plugins/filter/version'
4
+ require 'resque/plugins/filter/job_filter'
5
+
6
+ Resque::Job.send(:extend, Resque::Plugins::Filter::JobFilter)
@@ -0,0 +1,107 @@
1
+ require 'resque/worker'
2
+
3
+ # To configure resque filter, add something like the
4
+ # following to an initializer (defaults shown):
5
+ #
6
+ # Resque::Plugins::Filter::JobFilter.configure do |config|
7
+ # # The queue strategy to use when filtering job:
8
+ # # :simple - pops, checks filter, pushes if not runnable
9
+ # # :optimistic - peeks, checks filter, pops if runnable (not distributed client safe)
10
+ # config.strategy = :simple
11
+ # end
12
+ module Resque
13
+ module Plugins
14
+ module Filter
15
+
16
+ module JobFilter
17
+
18
+ # Allows configuring via class accessors
19
+ class << self
20
+ # optional
21
+ attr_accessor :strategy
22
+
23
+ def strategy=(s)
24
+ raise "Invalid strategy: #{s}" unless [:simple, :optimistic].include?(s)
25
+ @strategy = s
26
+ end
27
+ end
28
+
29
+ # default values
30
+ self.strategy = :simple
31
+
32
+ # Allows configuring via class accessors
33
+ def self.configure
34
+ yield self
35
+ end
36
+
37
+ def self.extended(receiver)
38
+ class << receiver
39
+ alias reserve_without_filter reserve
40
+ alias reserve reserve_with_filter
41
+ end
42
+ end
43
+
44
+ def reserve_with_filter(queue)
45
+ send("#{JobFilter.strategy}_reserve_with_filter", queue)
46
+ end
47
+
48
+ def simple_reserve_with_filter(queue)
49
+ return unless job = reserve_without_filter(queue)
50
+
51
+ # if the class participates in filter, and doesn't want to be run,
52
+ # then push it back onto queue
53
+ if filter(job)
54
+ return job
55
+ else
56
+ Resque.push(queue, job.payload)
57
+ return nil
58
+ end
59
+ end
60
+
61
+ # if filtering on hostname, in a cluster of many workers, it could
62
+ # take a while for the job to get to the machine in question as unrelated
63
+ # workers would thrash on this job, thereby preventing it from reaching
64
+ # the right worker. By peeking at the data and only popping it if filtered,
65
+ # then the right worker would get the job quicker. However, if there are
66
+ # other jobs on the queue, it may still take a while. This is also not
67
+ # DistributedRedis client safe due to the use of watch/multi/exec
68
+ #
69
+ def optimistic_reserve_with_filter(queue)
70
+ # http://redis.io/topics/transactions
71
+ # WATCH mykey
72
+ # val = GET mykey
73
+ # val = val + 1
74
+ # MULTI
75
+ # SET mykey $val
76
+ # EXEC
77
+
78
+ key = "queue:#{queue}"
79
+ redis.watch(key)
80
+
81
+ return unless payload = decode(redis.lindex(key, 0))
82
+
83
+ job = new(queue, payload)
84
+
85
+ if filter(job)
86
+ success = redis.multi do
87
+ redis.lpop(key)
88
+ end
89
+ return job if success
90
+ end
91
+
92
+ return nil
93
+ end
94
+
95
+ def filter(job)
96
+ if job.payload_class.respond_to?(:filter)
97
+ return job.payload_class.filter(*job.args)
98
+ else
99
+ return true
100
+ end
101
+ end
102
+
103
+ end
104
+
105
+ end
106
+ end
107
+ end
@@ -0,0 +1,7 @@
1
+ module Resque
2
+ module Plugins
3
+ module Filter
4
+ VERSION = "0.5.0"
5
+ end
6
+ end
7
+ end
@@ -0,0 +1,31 @@
1
+ # -*- encoding: utf-8 -*-
2
+ $:.push File.expand_path("../lib", __FILE__)
3
+ require 'resque/plugins/filter/version'
4
+
5
+ Gem::Specification.new do |s|
6
+ s.name = "resque-filter"
7
+ s.version = Resque::Plugins::Filter::VERSION
8
+ s.authors = ["Matt Conway"]
9
+ s.email = ["matt@conwaysplace.com"]
10
+ s.homepage = ""
11
+ s.summary = %q{A resque plugin that allows jobs to execute only if a filter is true, re-queueing otherwise.}
12
+ s.description = %q{A resque plugin that allows jobs to execute only if a filter is true, re-queueing otherwise. For example, one can author jobs that are filter-aware so that they can be scheduled with resque-scheduler to run on specific hosts.}
13
+
14
+ s.rubyforge_project = "resque-filter"
15
+
16
+ s.files = `git ls-files`.split("\n")
17
+ s.test_files = `git ls-files -- {test,spec,features}/*`.split("\n")
18
+ s.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
19
+ s.require_paths = ["lib"]
20
+
21
+ s.add_dependency("resque", '~> 1.10')
22
+
23
+ s.add_development_dependency('rake')
24
+ s.add_development_dependency('rspec', '~> 2.5')
25
+ s.add_development_dependency('rack-test', '~> 0.5.4')
26
+
27
+ # Needed for testing newer resque on ruby 1.8.7
28
+ s.add_development_dependency('json')
29
+ # Needed for correct ordering when passing hash params to rack-test
30
+ s.add_development_dependency('orderedhash')
31
+ end
@@ -0,0 +1,80 @@
1
+ require "spec_helper"
2
+
3
+ describe "Resque Job Filter" do
4
+
5
+ before(:each) do
6
+ Resque.redis.flushall
7
+ end
8
+
9
+ after(:each) do
10
+ Resque.redis.lrange("failed", 0, -1).size.should == 0
11
+ Resque.redis.get("stat:failed").to_i.should == 0
12
+ end
13
+
14
+ context "basic resque behavior still works" do
15
+
16
+ it "can work on multiple queues" do
17
+ Resque::Job.create(:high, SomeJob)
18
+ Resque::Job.create(:critical, SomeJob)
19
+
20
+ worker = Resque::Worker.new(:critical, :high)
21
+
22
+ worker.process
23
+ Resque.size(:high).should == 1
24
+ Resque.size(:critical).should == 0
25
+
26
+ worker.process
27
+ Resque.size(:high).should == 0
28
+ end
29
+
30
+ it "should pass lint" do
31
+ Resque::Plugin.lint(Resque::Plugins::Filter)
32
+ end
33
+
34
+ end
35
+
36
+ it "should fail for unknown strategy" do
37
+ lambda {
38
+ Resque::Plugins::Filter::JobFilter.strategy = :badstrategy
39
+ }.should raise_error(RuntimeError, "Invalid strategy: badstrategy")
40
+ Resque::Plugins::Filter::JobFilter.strategy.should == :simple
41
+ end
42
+
43
+ [:simple, :optimistic].each do |strategy|
44
+
45
+ context "basic filtering with #{strategy} strategy" do
46
+
47
+ before(:each) do
48
+ Resque::Plugins::Filter::JobFilter.strategy = strategy
49
+ end
50
+
51
+ after(:each) do
52
+ Resque::Plugins::Filter::JobFilter.strategy = :simple
53
+ end
54
+
55
+ it "runs job when filter is true" do
56
+ Resque::Job.should_receive("#{strategy}_reserve_with_filter").with(:myqueue)
57
+ Resque::Job.reserve(:myqueue)
58
+ end
59
+
60
+ it "calls correct strategy method" do
61
+ Resque::Job.create(:myqueue, FilterJob, true)
62
+ worker = Resque::Worker.new("*")
63
+
64
+ worker.work(0)
65
+ Resque.size(:myqueue).should == 0
66
+ end
67
+
68
+ it "doesn't run job when filter is false" do
69
+ Resque::Job.create(:myqueue, FilterJob, false)
70
+ worker = Resque::Worker.new("*")
71
+
72
+ worker.work(0)
73
+ Resque.size(:myqueue).should == 1
74
+ end
75
+
76
+ end
77
+
78
+ end
79
+
80
+ end
@@ -0,0 +1,312 @@
1
+ # Redis configuration file example
2
+
3
+ # Note on units: when memory size is needed, it is possible to specifiy
4
+ # it in the usual form of 1k 5GB 4M and so forth:
5
+ #
6
+ # 1k => 1000 bytes
7
+ # 1kb => 1024 bytes
8
+ # 1m => 1000000 bytes
9
+ # 1mb => 1024*1024 bytes
10
+ # 1g => 1000000000 bytes
11
+ # 1gb => 1024*1024*1024 bytes
12
+ #
13
+ # units are case insensitive so 1GB 1Gb 1gB are all the same.
14
+
15
+ # By default Redis does not run as a daemon. Use 'yes' if you need it.
16
+ # Note that Redis will write a pid file in ./tmp/run/redis.pid when daemonized.
17
+ daemonize yes
18
+
19
+ # When running daemonized, Redis writes a pid file in ./tmp/run/redis.pid by
20
+ # default. You can specify a custom pid file location here.
21
+ pidfile ./redis.pid
22
+
23
+ # Accept connections on the specified port, default is 6379
24
+ port 6379
25
+
26
+ # If you want you can bind a single interface, if the bind option is not
27
+ # specified all the interfaces will listen for incoming connections.
28
+ #
29
+ bind 127.0.0.1
30
+
31
+ # Close the connection after a client is idle for N seconds (0 to disable)
32
+ timeout 300
33
+
34
+ # Set server verbosity to 'debug'
35
+ # it can be one of:
36
+ # debug (a lot of information, useful for development/testing)
37
+ # verbose (many rarely useful info, but not a mess like the debug level)
38
+ # notice (moderately verbose, what you want in production probably)
39
+ # warning (only very important / critical messages are logged)
40
+ loglevel verbose
41
+
42
+ # Specify the log file name. Also 'stdout' can be used to force
43
+ # Redis to log on the standard output. Note that if you use standard
44
+ # output for logging but daemonize, logs will be sent to /dev/null
45
+ logfile ./redis-server.log
46
+
47
+ # Set the number of databases. The default database is DB 0, you can select
48
+ # a different one on a per-connection basis using SELECT <dbid> where
49
+ # dbid is a number between 0 and 'databases'-1
50
+ databases 16
51
+
52
+ ################################ SNAPSHOTTING #################################
53
+ #
54
+ # Save the DB on disk:
55
+ #
56
+ # save <seconds> <changes>
57
+ #
58
+ # Will save the DB if both the given number of seconds and the given
59
+ # number of write operations against the DB occurred.
60
+ #
61
+ # In the example below the behaviour will be to save:
62
+ # after 900 sec (15 min) if at least 1 key changed
63
+ # after 300 sec (5 min) if at least 10 keys changed
64
+ # after 60 sec if at least 10000 keys changed
65
+ #
66
+ # Note: you can disable saving at all commenting all the "save" lines.
67
+
68
+ save 900 1
69
+ save 300 10
70
+ save 60 10000
71
+
72
+ # Compress string objects using LZF when dump .rdb databases?
73
+ # For default that's set to 'yes' as it's almost always a win.
74
+ # If you want to save some CPU in the saving child set it to 'no' but
75
+ # the dataset will likely be bigger if you have compressible values or keys.
76
+ rdbcompression yes
77
+
78
+ # The filename where to dump the DB
79
+ dbfilename dump.rdb
80
+
81
+ # The working directory.
82
+ #
83
+ # The DB will be written inside this directory, with the filename specified
84
+ # above using the 'dbfilename' configuration directive.
85
+ #
86
+ # Also the Append Only File will be created inside this directory.
87
+ #
88
+ # Note that you must specify a directory here, not a file name.
89
+ dir .
90
+
91
+ ################################# REPLICATION #################################
92
+
93
+ # Master-Slave replication. Use slaveof to make a Redis instance a copy of
94
+ # another Redis server. Note that the configuration is local to the slave
95
+ # so for example it is possible to configure the slave to save the DB with a
96
+ # different interval, or to listen to another port, and so on.
97
+ #
98
+ # slaveof <masterip> <masterport>
99
+
100
+ # If the master is password protected (using the "requirepass" configuration
101
+ # directive below) it is possible to tell the slave to authenticate before
102
+ # starting the replication synchronization process, otherwise the master will
103
+ # refuse the slave request.
104
+ #
105
+ # masterauth <master-password>
106
+
107
+ ################################## SECURITY ###################################
108
+
109
+ # Require clients to issue AUTH <PASSWORD> before processing any other
110
+ # commands. This might be useful in environments in which you do not trust
111
+ # others with access to the host running redis-server.
112
+ #
113
+ # This should stay commented out for backward compatibility and because most
114
+ # people do not need auth (e.g. they run their own servers).
115
+ #
116
+ # Warning: since Redis is pretty fast an outside user can try up to
117
+ # 150k passwords per second against a good box. This means that you should
118
+ # use a very strong password otherwise it will be very easy to break.
119
+ #
120
+ # requirepass foobared
121
+
122
+ ################################### LIMITS ####################################
123
+
124
+ # Set the max number of connected clients at the same time. By default there
125
+ # is no limit, and it's up to the number of file descriptors the Redis process
126
+ # is able to open. The special value '0' means no limits.
127
+ # Once the limit is reached Redis will close all the new connections sending
128
+ # an error 'max number of clients reached'.
129
+ #
130
+ # maxclients 128
131
+
132
+ # Don't use more memory than the specified amount of bytes.
133
+ # When the memory limit is reached Redis will try to remove keys with an
134
+ # EXPIRE set. It will try to start freeing keys that are going to expire
135
+ # in little time and preserve keys with a longer time to live.
136
+ # Redis will also try to remove objects from free lists if possible.
137
+ #
138
+ # If all this fails, Redis will start to reply with errors to commands
139
+ # that will use more memory, like SET, LPUSH, and so on, and will continue
140
+ # to reply to most read-only commands like GET.
141
+ #
142
+ # WARNING: maxmemory can be a good idea mainly if you want to use Redis as a
143
+ # 'state' server or cache, not as a real DB. When Redis is used as a real
144
+ # database the memory usage will grow over the weeks, it will be obvious if
145
+ # it is going to use too much memory in the long run, and you'll have the time
146
+ # to upgrade. With maxmemory after the limit is reached you'll start to get
147
+ # errors for write operations, and this may even lead to DB inconsistency.
148
+ #
149
+ # maxmemory <bytes>
150
+
151
+ ############################## APPEND ONLY MODE ###############################
152
+
153
+ # By default Redis asynchronously dumps the dataset on disk. If you can live
154
+ # with the idea that the latest records will be lost if something like a crash
155
+ # happens this is the preferred way to run Redis. If instead you care a lot
156
+ # about your data and don't want to that a single record can get lost you should
157
+ # enable the append only mode: when this mode is enabled Redis will append
158
+ # every write operation received in the file appendonly.aof. This file will
159
+ # be read on startup in order to rebuild the full dataset in memory.
160
+ #
161
+ # Note that you can have both the async dumps and the append only file if you
162
+ # like (you have to comment the "save" statements above to disable the dumps).
163
+ # Still if append only mode is enabled Redis will load the data from the
164
+ # log file at startup ignoring the dump.rdb file.
165
+ #
166
+ # IMPORTANT: Check the BGREWRITEAOF to check how to rewrite the append
167
+ # log file in background when it gets too big.
168
+
169
+ appendonly no
170
+
171
+ # The name of the append only file (default: "appendonly.aof")
172
+ # appendfilename appendonly.aof
173
+
174
+ # The fsync() call tells the Operating System to actually write data on disk
175
+ # instead to wait for more data in the output buffer. Some OS will really flush
176
+ # data on disk, some other OS will just try to do it ASAP.
177
+ #
178
+ # Redis supports three different modes:
179
+ #
180
+ # no: don't fsync, just let the OS flush the data when it wants. Faster.
181
+ # always: fsync after every write to the append only log . Slow, Safest.
182
+ # everysec: fsync only if one second passed since the last fsync. Compromise.
183
+ #
184
+ # The default is "everysec" that's usually the right compromise between
185
+ # speed and data safety. It's up to you to understand if you can relax this to
186
+ # "no" that will will let the operating system flush the output buffer when
187
+ # it wants, for better performances (but if you can live with the idea of
188
+ # some data loss consider the default persistence mode that's snapshotting),
189
+ # or on the contrary, use "always" that's very slow but a bit safer than
190
+ # everysec.
191
+ #
192
+ # If unsure, use "everysec".
193
+
194
+ # appendfsync always
195
+ appendfsync everysec
196
+ # appendfsync no
197
+
198
+ ################################ VIRTUAL MEMORY ###############################
199
+
200
+ # Virtual Memory allows Redis to work with datasets bigger than the actual
201
+ # amount of RAM needed to hold the whole dataset in memory.
202
+ # In order to do so very used keys are taken in memory while the other keys
203
+ # are swapped into a swap file, similarly to what operating systems do
204
+ # with memory pages.
205
+ #
206
+ # To enable VM just set 'vm-enabled' to yes, and set the following three
207
+ # VM parameters accordingly to your needs.
208
+
209
+ vm-enabled no
210
+ # vm-enabled yes
211
+
212
+ # This is the path of the Redis swap file. As you can guess, swap files
213
+ # can't be shared by different Redis instances, so make sure to use a swap
214
+ # file for every redis process you are running. Redis will complain if the
215
+ # swap file is already in use.
216
+ #
217
+ # The best kind of storage for the Redis swap file (that's accessed at random)
218
+ # is a Solid State Disk (SSD).
219
+ #
220
+ # *** WARNING *** if you are using a shared hosting the default of putting
221
+ # the swap file under /tmp is not secure. Create a dir with access granted
222
+ # only to Redis user and configure Redis to create the swap file there.
223
+ vm-swap-file ./tmp/redis.swap
224
+
225
+ # vm-max-memory configures the VM to use at max the specified amount of
226
+ # RAM. Everything that deos not fit will be swapped on disk *if* possible, that
227
+ # is, if there is still enough contiguous space in the swap file.
228
+ #
229
+ # With vm-max-memory 0 the system will swap everything it can. Not a good
230
+ # default, just specify the max amount of RAM you can in bytes, but it's
231
+ # better to leave some margin. For instance specify an amount of RAM
232
+ # that's more or less between 60 and 80% of your free RAM.
233
+ vm-max-memory 0
234
+
235
+ # Redis swap files is split into pages. An object can be saved using multiple
236
+ # contiguous pages, but pages can't be shared between different objects.
237
+ # So if your page is too big, small objects swapped out on disk will waste
238
+ # a lot of space. If you page is too small, there is less space in the swap
239
+ # file (assuming you configured the same number of total swap file pages).
240
+ #
241
+ # If you use a lot of small objects, use a page size of 64 or 32 bytes.
242
+ # If you use a lot of big objects, use a bigger page size.
243
+ # If unsure, use the default :)
244
+ vm-page-size 32
245
+
246
+ # Number of total memory pages in the swap file.
247
+ # Given that the page table (a bitmap of free/used pages) is taken in memory,
248
+ # every 8 pages on disk will consume 1 byte of RAM.
249
+ #
250
+ # The total swap size is vm-page-size * vm-pages
251
+ #
252
+ # With the default of 32-bytes memory pages and 134217728 pages Redis will
253
+ # use a 4 GB swap file, that will use 16 MB of RAM for the page table.
254
+ #
255
+ # It's better to use the smallest acceptable value for your application,
256
+ # but the default is large in order to work in most conditions.
257
+ vm-pages 134217728
258
+
259
+ # Max number of VM I/O threads running at the same time.
260
+ # This threads are used to read/write data from/to swap file, since they
261
+ # also encode and decode objects from disk to memory or the reverse, a bigger
262
+ # number of threads can help with big objects even if they can't help with
263
+ # I/O itself as the physical device may not be able to couple with many
264
+ # reads/writes operations at the same time.
265
+ #
266
+ # The special value of 0 turn off threaded I/O and enables the blocking
267
+ # Virtual Memory implementation.
268
+ vm-max-threads 4
269
+
270
+ ############################### ADVANCED CONFIG ###############################
271
+
272
+ # Glue small output buffers together in order to send small replies in a
273
+ # single TCP packet. Uses a bit more CPU but most of the times it is a win
274
+ # in terms of number of queries per second. Use 'yes' if unsure.
275
+ glueoutputbuf yes
276
+
277
+ # Hashes are encoded in a special way (much more memory efficient) when they
278
+ # have at max a given numer of elements, and the biggest element does not
279
+ # exceed a given threshold. You can configure this limits with the following
280
+ # configuration directives.
281
+ hash-max-zipmap-entries 64
282
+ hash-max-zipmap-value 512
283
+
284
+ # Active rehashing uses 1 millisecond every 100 milliseconds of CPU time in
285
+ # order to help rehashing the main Redis hash table (the one mapping top-level
286
+ # keys to values). The hash table implementation redis uses (see dict.c)
287
+ # performs a lazy rehashing: the more operation you run into an hash table
288
+ # that is rhashing, the more rehashing "steps" are performed, so if the
289
+ # server is idle the rehashing is never complete and some more memory is used
290
+ # by the hash table.
291
+ #
292
+ # The default is to use this millisecond 10 times every second in order to
293
+ # active rehashing the main dictionaries, freeing memory when possible.
294
+ #
295
+ # If unsure:
296
+ # use "activerehashing no" if you have hard latency requirements and it is
297
+ # not a good thing in your environment that Redis can reply form time to time
298
+ # to queries with 2 milliseconds delay.
299
+ #
300
+ # use "activerehashing yes" if you don't have such hard requirements but
301
+ # want to free memory asap when possible.
302
+ activerehashing yes
303
+
304
+ ################################## INCLUDES ###################################
305
+
306
+ # Include one or more other config files here. This is useful if you
307
+ # have a standard template that goes to all redis server but also need
308
+ # to customize a few per-server settings. Include files can include
309
+ # other files, so use this wisely.
310
+ #
311
+ # include /path/to/local.conf
312
+ # include /path/to/other.conf
@@ -0,0 +1,41 @@
1
+ require 'rspec'
2
+ require 'resque-filter'
3
+
4
+ # No need to start redis when running in Travis
5
+ unless ENV['CI']
6
+
7
+ begin
8
+ Resque.queues
9
+ rescue Errno::ECONNREFUSED
10
+ spec_dir = File.dirname(File.expand_path(__FILE__))
11
+ REDIS_CMD = "redis-server #{spec_dir}/redis-test.conf"
12
+
13
+ puts "Starting redis for testing at localhost..."
14
+ puts `cd #{spec_dir}; #{REDIS_CMD}`
15
+
16
+ # Schedule the redis server for shutdown when tests are all finished.
17
+ at_exit do
18
+ puts 'Stopping redis'
19
+ pid = File.read("#{spec_dir}/redis.pid").to_i rescue nil
20
+ system ("kill -9 #{pid}") if pid.to_i != 0
21
+ File.delete("#{spec_dir}/redis.pid") rescue nil
22
+ File.delete("#{spec_dir}/redis-server.log") rescue nil
23
+ File.delete("#{spec_dir}/dump.rdb") rescue nil
24
+ end
25
+ end
26
+
27
+ end
28
+
29
+ class SomeJob
30
+ def self.perform(*args)
31
+ end
32
+ end
33
+
34
+ class FilterJob
35
+ def self.perform(*args)
36
+ end
37
+
38
+ def self.filter(*args)
39
+ return args && args.first
40
+ end
41
+ end
metadata ADDED
@@ -0,0 +1,168 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: resque-filter
3
+ version: !ruby/object:Gem::Version
4
+ hash: 11
5
+ prerelease:
6
+ segments:
7
+ - 0
8
+ - 5
9
+ - 0
10
+ version: 0.5.0
11
+ platform: ruby
12
+ authors:
13
+ - Matt Conway
14
+ autorequire:
15
+ bindir: bin
16
+ cert_chain: []
17
+
18
+ date: 2012-04-27 00:00:00 Z
19
+ dependencies:
20
+ - !ruby/object:Gem::Dependency
21
+ name: resque
22
+ prerelease: false
23
+ requirement: &id001 !ruby/object:Gem::Requirement
24
+ none: false
25
+ requirements:
26
+ - - ~>
27
+ - !ruby/object:Gem::Version
28
+ hash: 27
29
+ segments:
30
+ - 1
31
+ - 10
32
+ version: "1.10"
33
+ type: :runtime
34
+ version_requirements: *id001
35
+ - !ruby/object:Gem::Dependency
36
+ name: rake
37
+ prerelease: false
38
+ requirement: &id002 !ruby/object:Gem::Requirement
39
+ none: false
40
+ requirements:
41
+ - - ">="
42
+ - !ruby/object:Gem::Version
43
+ hash: 3
44
+ segments:
45
+ - 0
46
+ version: "0"
47
+ type: :development
48
+ version_requirements: *id002
49
+ - !ruby/object:Gem::Dependency
50
+ name: rspec
51
+ prerelease: false
52
+ requirement: &id003 !ruby/object:Gem::Requirement
53
+ none: false
54
+ requirements:
55
+ - - ~>
56
+ - !ruby/object:Gem::Version
57
+ hash: 9
58
+ segments:
59
+ - 2
60
+ - 5
61
+ version: "2.5"
62
+ type: :development
63
+ version_requirements: *id003
64
+ - !ruby/object:Gem::Dependency
65
+ name: rack-test
66
+ prerelease: false
67
+ requirement: &id004 !ruby/object:Gem::Requirement
68
+ none: false
69
+ requirements:
70
+ - - ~>
71
+ - !ruby/object:Gem::Version
72
+ hash: 3
73
+ segments:
74
+ - 0
75
+ - 5
76
+ - 4
77
+ version: 0.5.4
78
+ type: :development
79
+ version_requirements: *id004
80
+ - !ruby/object:Gem::Dependency
81
+ name: json
82
+ prerelease: false
83
+ requirement: &id005 !ruby/object:Gem::Requirement
84
+ none: false
85
+ requirements:
86
+ - - ">="
87
+ - !ruby/object:Gem::Version
88
+ hash: 3
89
+ segments:
90
+ - 0
91
+ version: "0"
92
+ type: :development
93
+ version_requirements: *id005
94
+ - !ruby/object:Gem::Dependency
95
+ name: orderedhash
96
+ prerelease: false
97
+ requirement: &id006 !ruby/object:Gem::Requirement
98
+ none: false
99
+ requirements:
100
+ - - ">="
101
+ - !ruby/object:Gem::Version
102
+ hash: 3
103
+ segments:
104
+ - 0
105
+ version: "0"
106
+ type: :development
107
+ version_requirements: *id006
108
+ description: A resque plugin that allows jobs to execute only if a filter is true, re-queueing otherwise. For example, one can author jobs that are filter-aware so that they can be scheduled with resque-scheduler to run on specific hosts.
109
+ email:
110
+ - matt@conwaysplace.com
111
+ executables: []
112
+
113
+ extensions: []
114
+
115
+ extra_rdoc_files: []
116
+
117
+ files:
118
+ - .gitignore
119
+ - .travis.yml
120
+ - CHANGELOG
121
+ - Gemfile
122
+ - LICENSE
123
+ - README.md
124
+ - Rakefile
125
+ - lib/resque-filter.rb
126
+ - lib/resque/plugins/filter/job_filter.rb
127
+ - lib/resque/plugins/filter/version.rb
128
+ - resque-filter.gemspec
129
+ - spec/job_filter_spec.rb
130
+ - spec/redis-test.conf
131
+ - spec/spec_helper.rb
132
+ homepage: ""
133
+ licenses: []
134
+
135
+ post_install_message:
136
+ rdoc_options: []
137
+
138
+ require_paths:
139
+ - lib
140
+ required_ruby_version: !ruby/object:Gem::Requirement
141
+ none: false
142
+ requirements:
143
+ - - ">="
144
+ - !ruby/object:Gem::Version
145
+ hash: 3
146
+ segments:
147
+ - 0
148
+ version: "0"
149
+ required_rubygems_version: !ruby/object:Gem::Requirement
150
+ none: false
151
+ requirements:
152
+ - - ">="
153
+ - !ruby/object:Gem::Version
154
+ hash: 3
155
+ segments:
156
+ - 0
157
+ version: "0"
158
+ requirements: []
159
+
160
+ rubyforge_project: resque-filter
161
+ rubygems_version: 1.8.15
162
+ signing_key:
163
+ specification_version: 3
164
+ summary: A resque plugin that allows jobs to execute only if a filter is true, re-queueing otherwise.
165
+ test_files:
166
+ - spec/job_filter_spec.rb
167
+ - spec/redis-test.conf
168
+ - spec/spec_helper.rb