sidekiq-dynamic-queues 0.5.0

Sign up to get free protection for your applications and to get access to all the features.
@@ -0,0 +1,5 @@
1
+ module Sidekiq
2
+ module DynamicQueues
3
+ VERSION = "0.5.0"
4
+ end
5
+ end
@@ -0,0 +1,31 @@
1
+ # -*- encoding: utf-8 -*-
2
+ $:.push File.expand_path("../lib", __FILE__)
3
+ require 'sidekiq/dynamic_queues/version'
4
+
5
+ Gem::Specification.new do |s|
6
+ s.name = "sidekiq-dynamic-queues"
7
+ s.version = Sidekiq::DynamicQueues::VERSION
8
+ s.platform = Gem::Platform::RUBY
9
+ s.authors = ["Matt Conway"]
10
+ s.email = ["matt@conwaysplace.com"]
11
+ s.homepage = ""
12
+ s.summary = %q{A sidekiq plugin for specifying the queues a worker pulls from with wildcards, negations, or dynamic look up from redis}
13
+ s.description = %q{A sidekiq plugin for specifying the queues a worker pulls from with wildcards, negations, or dynamic look up from redis}
14
+
15
+ s.rubyforge_project = "sidekiq-dynamic-queues"
16
+
17
+ s.files = `git ls-files`.split("\n")
18
+ s.test_files = `git ls-files -- {test,spec,features}/*`.split("\n")
19
+ s.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
20
+ s.require_paths = ["lib"]
21
+
22
+ s.add_dependency("sidekiq", '~> 2.9')
23
+
24
+ s.add_development_dependency('rake')
25
+ s.add_development_dependency('rspec', '~> 2.5')
26
+ s.add_development_dependency('sinatra')
27
+ s.add_development_dependency('slim')
28
+ s.add_development_dependency('rack-test', '~> 0.5.4')
29
+
30
+ end
31
+
@@ -0,0 +1,198 @@
1
+ require "spec_helper"
2
+
3
+ describe "Dynamic Queues" do
4
+
5
+ include Sidekiq::DynamicQueues::Attributes
6
+
7
+ def watch_queues(*queues)
8
+ Sidekiq.redis do |r|
9
+ queues.each {|q| r.sadd('queues', q) }
10
+ end
11
+ end
12
+
13
+ before(:each) do
14
+ SomeJob.sidekiq_options('retry' => false, 'queue' => 'default')
15
+ SomeJob.result = nil
16
+ Sidekiq.redis {|r| r.flushall }
17
+ end
18
+
19
+ context "basic behavior still works" do
20
+
21
+ it "can work on different queues" do
22
+ SomeJob.perform_async(1)
23
+ run_queues("default")
24
+ SomeJob.result.should eq [1]
25
+ enqueue_on("other", SomeJob, 2)
26
+ run_queues("other")
27
+ SomeJob.result.should eq [2]
28
+ end
29
+
30
+ it "can work on multiple queues" do
31
+ SomeJob.perform_async(1)
32
+ run_queues("other", "default")
33
+ SomeJob.result.should eq [1]
34
+ enqueue_on("other", SomeJob, 2)
35
+ run_queues("default", "other")
36
+ SomeJob.result.should eq [2]
37
+ end
38
+
39
+ end
40
+
41
+ context "attributes" do
42
+
43
+ it "should always have a fallback pattern" do
44
+ get_dynamic_queues.should == {'default' => ['*']}
45
+ end
46
+
47
+ it "should allow setting single patterns" do
48
+ get_dynamic_queue('foo').should == ['*']
49
+ set_dynamic_queue('foo', ['bar'])
50
+ get_dynamic_queue('foo').should == ['bar']
51
+ end
52
+
53
+ it "should allow setting multiple patterns" do
54
+ set_dynamic_queues({'foo' => ['bar'], 'baz' => ['boo']})
55
+ get_dynamic_queues.should == {'foo' => ['bar'], 'baz' => ['boo'], 'default' => ['*']}
56
+ end
57
+
58
+ it "should remove mapping when setting empty value" do
59
+ get_dynamic_queues
60
+ set_dynamic_queues({'foo' => ['bar'], 'baz' => ['boo']})
61
+ get_dynamic_queues.should == {'foo' => ['bar'], 'baz' => ['boo'], 'default' => ['*']}
62
+
63
+ set_dynamic_queues({'foo' => [], 'baz' => ['boo']})
64
+ get_dynamic_queues.should == {'baz' => ['boo'], 'default' => ['*']}
65
+ set_dynamic_queues({'baz' => nil})
66
+ get_dynamic_queues.should == {'default' => ['*']}
67
+
68
+ set_dynamic_queues({'foo' => ['bar'], 'baz' => ['boo']})
69
+ set_dynamic_queue('foo', [])
70
+ get_dynamic_queues.should == {'baz' => ['boo'], 'default' => ['*']}
71
+ set_dynamic_queue('baz', nil)
72
+ get_dynamic_queues.should == {'default' => ['*']}
73
+ end
74
+
75
+
76
+ end
77
+
78
+ context "basic queue patterns" do
79
+
80
+ Fetch = Sidekiq::DynamicQueues::Fetch
81
+ SFTO = Sidekiq::Fetcher::TIMEOUT
82
+
83
+ before(:each) do
84
+ watch_queues(*%w[high_x foo high_y superhigh_z])
85
+ end
86
+
87
+ it "can specify simple queues" do
88
+ fetch = Fetch.new(:queues => %w[foo], :strict => true)
89
+ fetch.queues_cmd.should eq ["queue:foo", SFTO]
90
+
91
+ fetch = Fetch.new(:queues => %w[foo bar], :strict => true)
92
+ fetch.queues_cmd.should eq ["queue:foo", "queue:bar", SFTO]
93
+ end
94
+
95
+ it "can specify simple wildcard" do
96
+ fetch = Fetch.new(:queues => %w[*], :strict => true)
97
+ fetch.queues_cmd.should eq ["queue:foo", "queue:high_x",
98
+ "queue:high_y", "queue:superhigh_z", SFTO]
99
+ end
100
+
101
+ it "can include queues with pattern"do
102
+ fetch = Fetch.new(:queues => %w[high*], :strict => true)
103
+ fetch.queues_cmd.should eq ["queue:high_x", "queue:high_y", SFTO]
104
+
105
+ fetch = Fetch.new(:queues => %w[*high_z], :strict => true)
106
+ fetch.queues_cmd.should eq ["queue:superhigh_z", SFTO]
107
+
108
+ fetch = Fetch.new(:queues => %w[*high*], :strict => true)
109
+ fetch.queues_cmd.should eq ["queue:high_x", "queue:high_y",
110
+ "queue:superhigh_z", SFTO]
111
+ end
112
+
113
+ it "can blacklist queues" do
114
+ fetch = Fetch.new(:queues => %w[* !foo], :strict => true)
115
+ fetch.queues_cmd.should eq ["queue:high_x", "queue:high_y",
116
+ "queue:superhigh_z", SFTO]
117
+ end
118
+
119
+ it "can blacklist queues with pattern" do
120
+ fetch = Fetch.new(:queues => %w[* !*high*], :strict => true)
121
+ fetch.queues_cmd.should eq ["queue:foo", SFTO]
122
+ end
123
+
124
+ it "randomizes when not strict ordering" do
125
+ fetch = Fetch.new(:queues => %w[*], :strict => false)
126
+
127
+ 3.times.any? { fetch.queues_cmd != fetch.queues_cmd }.should be_true
128
+
129
+ %w[high_x foo high_y superhigh_z].all? do |q|
130
+ fetch.queues_cmd.include?("queue:{q}")
131
+ end
132
+ end
133
+
134
+ end
135
+
136
+ context "redis backed queues" do
137
+
138
+ it "can dynamically lookup queues" do
139
+ set_dynamic_queue("mykey", ["foo", "bar"])
140
+ fetch = Fetch.new(:queues => %w[@mykey], :strict => true)
141
+ fetch.queues_cmd.should eq ["queue:bar", "queue:foo", SFTO]
142
+ end
143
+
144
+ it "can blacklist dynamic queues" do
145
+ watch_queues(*%w[high_x foo high_y superhigh_z])
146
+
147
+ set_dynamic_queue("mykey", ["foo"])
148
+ fetch = Fetch.new(:queues => %w[* !@mykey], :strict => true)
149
+ fetch.queues_cmd.should eq ["queue:high_x", "queue:high_y",
150
+ "queue:superhigh_z", SFTO]
151
+ end
152
+
153
+ it "can blacklist dynamic queues with negation" do
154
+ watch_queues(*%w[high_x foo high_y superhigh_z])
155
+
156
+ set_dynamic_queue("mykey", ["!foo", "high_x"])
157
+ fetch = Fetch.new(:queues => %w[!@mykey], :strict => true)
158
+ fetch.queues_cmd.should eq ["queue:foo", SFTO]
159
+ end
160
+
161
+ it "uses hostname as default key in dynamic queues" do
162
+ host = `hostname`.chomp
163
+ set_dynamic_queue(host, ["foo", "bar"])
164
+ fetch = Fetch.new(:queues => %w[@], :strict => true)
165
+ fetch.queues_cmd.should eq ["queue:bar", "queue:foo", SFTO]
166
+ end
167
+
168
+ it "can use wildcards in dynamic queues" do
169
+ watch_queues(*%w[high_x foo high_y superhigh_z])
170
+
171
+ set_dynamic_queue("mykey", ["*high*", "!high_y"])
172
+ fetch = Fetch.new(:queues => %w[@mykey], :strict => true)
173
+ fetch.queues_cmd.should eq ["queue:high_x", "queue:superhigh_z", SFTO]
174
+ end
175
+
176
+ it "falls back to default queues when missing" do
177
+ set_dynamic_queue("default", ["foo", "bar"])
178
+ fetch = Fetch.new(:queues => %w[@mykey], :strict => true)
179
+ fetch.queues_cmd.should eq ["queue:bar", "queue:foo", SFTO]
180
+ end
181
+
182
+ it "falls back to all queues when missing and no default" do
183
+ watch_queues(*%w[high_x foo])
184
+ fetch = Fetch.new(:queues => %w[@mykey], :strict => true)
185
+ fetch.queues_cmd.should eq ["queue:foo", "queue:high_x", SFTO]
186
+ end
187
+
188
+ it "falls back to all queues when missing and no default and keep up to date" do
189
+ watch_queues(*%w[high_x foo])
190
+ fetch = Fetch.new(:queues => %w[@mykey], :strict => true)
191
+ fetch.queues_cmd.should eq ["queue:foo", "queue:high_x", SFTO]
192
+ watch_queues(*%w[bar])
193
+ fetch.queues_cmd.should eq ["queue:bar", "queue:foo", "queue:high_x", SFTO]
194
+ end
195
+
196
+ end
197
+
198
+ end
@@ -0,0 +1,312 @@
1
+ # Redis configuration file example
2
+
3
+ # Note on units: when memory size is needed, it is possible to specifiy
4
+ # it in the usual form of 1k 5GB 4M and so forth:
5
+ #
6
+ # 1k => 1000 bytes
7
+ # 1kb => 1024 bytes
8
+ # 1m => 1000000 bytes
9
+ # 1mb => 1024*1024 bytes
10
+ # 1g => 1000000000 bytes
11
+ # 1gb => 1024*1024*1024 bytes
12
+ #
13
+ # units are case insensitive so 1GB 1Gb 1gB are all the same.
14
+
15
+ # By default Redis does not run as a daemon. Use 'yes' if you need it.
16
+ # Note that Redis will write a pid file in ./tmp/run/redis.pid when daemonized.
17
+ daemonize yes
18
+
19
+ # When running daemonized, Redis writes a pid file in ./tmp/run/redis.pid by
20
+ # default. You can specify a custom pid file location here.
21
+ pidfile ./redis.pid
22
+
23
+ # Accept connections on the specified port, default is 6379
24
+ port 6379
25
+
26
+ # If you want you can bind a single interface, if the bind option is not
27
+ # specified all the interfaces will listen for incoming connections.
28
+ #
29
+ bind 127.0.0.1
30
+
31
+ # Close the connection after a client is idle for N seconds (0 to disable)
32
+ timeout 300
33
+
34
+ # Set server verbosity to 'debug'
35
+ # it can be one of:
36
+ # debug (a lot of information, useful for development/testing)
37
+ # verbose (many rarely useful info, but not a mess like the debug level)
38
+ # notice (moderately verbose, what you want in production probably)
39
+ # warning (only very important / critical messages are logged)
40
+ loglevel verbose
41
+
42
+ # Specify the log file name. Also 'stdout' can be used to force
43
+ # Redis to log on the standard output. Note that if you use standard
44
+ # output for logging but daemonize, logs will be sent to /dev/null
45
+ logfile ./redis-server.log
46
+
47
+ # Set the number of databases. The default database is DB 0, you can select
48
+ # a different one on a per-connection basis using SELECT <dbid> where
49
+ # dbid is a number between 0 and 'databases'-1
50
+ databases 16
51
+
52
+ ################################ SNAPSHOTTING #################################
53
+ #
54
+ # Save the DB on disk:
55
+ #
56
+ # save <seconds> <changes>
57
+ #
58
+ # Will save the DB if both the given number of seconds and the given
59
+ # number of write operations against the DB occurred.
60
+ #
61
+ # In the example below the behaviour will be to save:
62
+ # after 900 sec (15 min) if at least 1 key changed
63
+ # after 300 sec (5 min) if at least 10 keys changed
64
+ # after 60 sec if at least 10000 keys changed
65
+ #
66
+ # Note: you can disable saving at all commenting all the "save" lines.
67
+
68
+ save 900 1
69
+ save 300 10
70
+ save 60 10000
71
+
72
+ # Compress string objects using LZF when dump .rdb databases?
73
+ # For default that's set to 'yes' as it's almost always a win.
74
+ # If you want to save some CPU in the saving child set it to 'no' but
75
+ # the dataset will likely be bigger if you have compressible values or keys.
76
+ rdbcompression yes
77
+
78
+ # The filename where to dump the DB
79
+ dbfilename dump.rdb
80
+
81
+ # The working directory.
82
+ #
83
+ # The DB will be written inside this directory, with the filename specified
84
+ # above using the 'dbfilename' configuration directive.
85
+ #
86
+ # Also the Append Only File will be created inside this directory.
87
+ #
88
+ # Note that you must specify a directory here, not a file name.
89
+ dir .
90
+
91
+ ################################# REPLICATION #################################
92
+
93
+ # Master-Slave replication. Use slaveof to make a Redis instance a copy of
94
+ # another Redis server. Note that the configuration is local to the slave
95
+ # so for example it is possible to configure the slave to save the DB with a
96
+ # different interval, or to listen to another port, and so on.
97
+ #
98
+ # slaveof <masterip> <masterport>
99
+
100
+ # If the master is password protected (using the "requirepass" configuration
101
+ # directive below) it is possible to tell the slave to authenticate before
102
+ # starting the replication synchronization process, otherwise the master will
103
+ # refuse the slave request.
104
+ #
105
+ # masterauth <master-password>
106
+
107
+ ################################## SECURITY ###################################
108
+
109
+ # Require clients to issue AUTH <PASSWORD> before processing any other
110
+ # commands. This might be useful in environments in which you do not trust
111
+ # others with access to the host running redis-server.
112
+ #
113
+ # This should stay commented out for backward compatibility and because most
114
+ # people do not need auth (e.g. they run their own servers).
115
+ #
116
+ # Warning: since Redis is pretty fast an outside user can try up to
117
+ # 150k passwords per second against a good box. This means that you should
118
+ # use a very strong password otherwise it will be very easy to break.
119
+ #
120
+ # requirepass foobared
121
+
122
+ ################################### LIMITS ####################################
123
+
124
+ # Set the max number of connected clients at the same time. By default there
125
+ # is no limit, and it's up to the number of file descriptors the Redis process
126
+ # is able to open. The special value '0' means no limits.
127
+ # Once the limit is reached Redis will close all the new connections sending
128
+ # an error 'max number of clients reached'.
129
+ #
130
+ # maxclients 128
131
+
132
+ # Don't use more memory than the specified amount of bytes.
133
+ # When the memory limit is reached Redis will try to remove keys with an
134
+ # EXPIRE set. It will try to start freeing keys that are going to expire
135
+ # in little time and preserve keys with a longer time to live.
136
+ # Redis will also try to remove objects from free lists if possible.
137
+ #
138
+ # If all this fails, Redis will start to reply with errors to commands
139
+ # that will use more memory, like SET, LPUSH, and so on, and will continue
140
+ # to reply to most read-only commands like GET.
141
+ #
142
+ # WARNING: maxmemory can be a good idea mainly if you want to use Redis as a
143
+ # 'state' server or cache, not as a real DB. When Redis is used as a real
144
+ # database the memory usage will grow over the weeks, it will be obvious if
145
+ # it is going to use too much memory in the long run, and you'll have the time
146
+ # to upgrade. With maxmemory after the limit is reached you'll start to get
147
+ # errors for write operations, and this may even lead to DB inconsistency.
148
+ #
149
+ # maxmemory <bytes>
150
+
151
+ ############################## APPEND ONLY MODE ###############################
152
+
153
+ # By default Redis asynchronously dumps the dataset on disk. If you can live
154
+ # with the idea that the latest records will be lost if something like a crash
155
+ # happens this is the preferred way to run Redis. If instead you care a lot
156
+ # about your data and don't want to that a single record can get lost you should
157
+ # enable the append only mode: when this mode is enabled Redis will append
158
+ # every write operation received in the file appendonly.aof. This file will
159
+ # be read on startup in order to rebuild the full dataset in memory.
160
+ #
161
+ # Note that you can have both the async dumps and the append only file if you
162
+ # like (you have to comment the "save" statements above to disable the dumps).
163
+ # Still if append only mode is enabled Redis will load the data from the
164
+ # log file at startup ignoring the dump.rdb file.
165
+ #
166
+ # IMPORTANT: Check the BGREWRITEAOF to check how to rewrite the append
167
+ # log file in background when it gets too big.
168
+
169
+ appendonly no
170
+
171
+ # The name of the append only file (default: "appendonly.aof")
172
+ # appendfilename appendonly.aof
173
+
174
+ # The fsync() call tells the Operating System to actually write data on disk
175
+ # instead to wait for more data in the output buffer. Some OS will really flush
176
+ # data on disk, some other OS will just try to do it ASAP.
177
+ #
178
+ # Redis supports three different modes:
179
+ #
180
+ # no: don't fsync, just let the OS flush the data when it wants. Faster.
181
+ # always: fsync after every write to the append only log . Slow, Safest.
182
+ # everysec: fsync only if one second passed since the last fsync. Compromise.
183
+ #
184
+ # The default is "everysec" that's usually the right compromise between
185
+ # speed and data safety. It's up to you to understand if you can relax this to
186
+ # "no" that will will let the operating system flush the output buffer when
187
+ # it wants, for better performances (but if you can live with the idea of
188
+ # some data loss consider the default persistence mode that's snapshotting),
189
+ # or on the contrary, use "always" that's very slow but a bit safer than
190
+ # everysec.
191
+ #
192
+ # If unsure, use "everysec".
193
+
194
+ # appendfsync always
195
+ appendfsync everysec
196
+ # appendfsync no
197
+
198
+ ################################ VIRTUAL MEMORY ###############################
199
+
200
+ # Virtual Memory allows Redis to work with datasets bigger than the actual
201
+ # amount of RAM needed to hold the whole dataset in memory.
202
+ # In order to do so very used keys are taken in memory while the other keys
203
+ # are swapped into a swap file, similarly to what operating systems do
204
+ # with memory pages.
205
+ #
206
+ # To enable VM just set 'vm-enabled' to yes, and set the following three
207
+ # VM parameters accordingly to your needs.
208
+
209
+ vm-enabled no
210
+ # vm-enabled yes
211
+
212
+ # This is the path of the Redis swap file. As you can guess, swap files
213
+ # can't be shared by different Redis instances, so make sure to use a swap
214
+ # file for every redis process you are running. Redis will complain if the
215
+ # swap file is already in use.
216
+ #
217
+ # The best kind of storage for the Redis swap file (that's accessed at random)
218
+ # is a Solid State Disk (SSD).
219
+ #
220
+ # *** WARNING *** if you are using a shared hosting the default of putting
221
+ # the swap file under /tmp is not secure. Create a dir with access granted
222
+ # only to Redis user and configure Redis to create the swap file there.
223
+ vm-swap-file ./tmp/redis.swap
224
+
225
+ # vm-max-memory configures the VM to use at max the specified amount of
226
+ # RAM. Everything that deos not fit will be swapped on disk *if* possible, that
227
+ # is, if there is still enough contiguous space in the swap file.
228
+ #
229
+ # With vm-max-memory 0 the system will swap everything it can. Not a good
230
+ # default, just specify the max amount of RAM you can in bytes, but it's
231
+ # better to leave some margin. For instance specify an amount of RAM
232
+ # that's more or less between 60 and 80% of your free RAM.
233
+ vm-max-memory 0
234
+
235
+ # Redis swap files is split into pages. An object can be saved using multiple
236
+ # contiguous pages, but pages can't be shared between different objects.
237
+ # So if your page is too big, small objects swapped out on disk will waste
238
+ # a lot of space. If you page is too small, there is less space in the swap
239
+ # file (assuming you configured the same number of total swap file pages).
240
+ #
241
+ # If you use a lot of small objects, use a page size of 64 or 32 bytes.
242
+ # If you use a lot of big objects, use a bigger page size.
243
+ # If unsure, use the default :)
244
+ vm-page-size 32
245
+
246
+ # Number of total memory pages in the swap file.
247
+ # Given that the page table (a bitmap of free/used pages) is taken in memory,
248
+ # every 8 pages on disk will consume 1 byte of RAM.
249
+ #
250
+ # The total swap size is vm-page-size * vm-pages
251
+ #
252
+ # With the default of 32-bytes memory pages and 134217728 pages Redis will
253
+ # use a 4 GB swap file, that will use 16 MB of RAM for the page table.
254
+ #
255
+ # It's better to use the smallest acceptable value for your application,
256
+ # but the default is large in order to work in most conditions.
257
+ vm-pages 134217728
258
+
259
+ # Max number of VM I/O threads running at the same time.
260
+ # This threads are used to read/write data from/to swap file, since they
261
+ # also encode and decode objects from disk to memory or the reverse, a bigger
262
+ # number of threads can help with big objects even if they can't help with
263
+ # I/O itself as the physical device may not be able to couple with many
264
+ # reads/writes operations at the same time.
265
+ #
266
+ # The special value of 0 turn off threaded I/O and enables the blocking
267
+ # Virtual Memory implementation.
268
+ vm-max-threads 4
269
+
270
+ ############################### ADVANCED CONFIG ###############################
271
+
272
+ # Glue small output buffers together in order to send small replies in a
273
+ # single TCP packet. Uses a bit more CPU but most of the times it is a win
274
+ # in terms of number of queries per second. Use 'yes' if unsure.
275
+ glueoutputbuf yes
276
+
277
+ # Hashes are encoded in a special way (much more memory efficient) when they
278
+ # have at max a given numer of elements, and the biggest element does not
279
+ # exceed a given threshold. You can configure this limits with the following
280
+ # configuration directives.
281
+ hash-max-zipmap-entries 64
282
+ hash-max-zipmap-value 512
283
+
284
+ # Active rehashing uses 1 millisecond every 100 milliseconds of CPU time in
285
+ # order to help rehashing the main Redis hash table (the one mapping top-level
286
+ # keys to values). The hash table implementation redis uses (see dict.c)
287
+ # performs a lazy rehashing: the more operation you run into an hash table
288
+ # that is rhashing, the more rehashing "steps" are performed, so if the
289
+ # server is idle the rehashing is never complete and some more memory is used
290
+ # by the hash table.
291
+ #
292
+ # The default is to use this millisecond 10 times every second in order to
293
+ # active rehashing the main dictionaries, freeing memory when possible.
294
+ #
295
+ # If unsure:
296
+ # use "activerehashing no" if you have hard latency requirements and it is
297
+ # not a good thing in your environment that Redis can reply form time to time
298
+ # to queries with 2 milliseconds delay.
299
+ #
300
+ # use "activerehashing yes" if you don't have such hard requirements but
301
+ # want to free memory asap when possible.
302
+ activerehashing yes
303
+
304
+ ################################## INCLUDES ###################################
305
+
306
+ # Include one or more other config files here. This is useful if you
307
+ # have a standard template that goes to all redis server but also need
308
+ # to customize a few per-server settings. Include files can include
309
+ # other files, so use this wisely.
310
+ #
311
+ # include /path/to/local.conf
312
+ # include /path/to/other.conf