rq-ruby1.8 3.4.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (67) hide show
  1. data/Gemfile +22 -0
  2. data/Gemfile.lock +22 -0
  3. data/INSTALL +166 -0
  4. data/LICENSE +10 -0
  5. data/Makefile +6 -0
  6. data/README +1183 -0
  7. data/Rakefile +37 -0
  8. data/TODO +24 -0
  9. data/TUTORIAL +230 -0
  10. data/VERSION +1 -0
  11. data/bin/rq +902 -0
  12. data/bin/rqmailer +865 -0
  13. data/example/a.rb +7 -0
  14. data/extconf.rb +198 -0
  15. data/gemspec.rb +40 -0
  16. data/install.rb +210 -0
  17. data/lib/rq.rb +155 -0
  18. data/lib/rq/arrayfields.rb +371 -0
  19. data/lib/rq/backer.rb +31 -0
  20. data/lib/rq/configfile.rb +82 -0
  21. data/lib/rq/configurator.rb +40 -0
  22. data/lib/rq/creator.rb +54 -0
  23. data/lib/rq/cron.rb +144 -0
  24. data/lib/rq/defaultconfig.txt +5 -0
  25. data/lib/rq/deleter.rb +51 -0
  26. data/lib/rq/executor.rb +40 -0
  27. data/lib/rq/feeder.rb +527 -0
  28. data/lib/rq/ioviewer.rb +48 -0
  29. data/lib/rq/job.rb +51 -0
  30. data/lib/rq/jobqueue.rb +947 -0
  31. data/lib/rq/jobrunner.rb +110 -0
  32. data/lib/rq/jobrunnerdaemon.rb +193 -0
  33. data/lib/rq/lister.rb +47 -0
  34. data/lib/rq/locker.rb +43 -0
  35. data/lib/rq/lockfile.rb +564 -0
  36. data/lib/rq/logging.rb +124 -0
  37. data/lib/rq/mainhelper.rb +189 -0
  38. data/lib/rq/orderedautohash.rb +39 -0
  39. data/lib/rq/orderedhash.rb +240 -0
  40. data/lib/rq/qdb.rb +733 -0
  41. data/lib/rq/querier.rb +98 -0
  42. data/lib/rq/rails.rb +80 -0
  43. data/lib/rq/recoverer.rb +28 -0
  44. data/lib/rq/refresher.rb +80 -0
  45. data/lib/rq/relayer.rb +283 -0
  46. data/lib/rq/resource.rb +22 -0
  47. data/lib/rq/resourcemanager.rb +40 -0
  48. data/lib/rq/resubmitter.rb +100 -0
  49. data/lib/rq/rotater.rb +98 -0
  50. data/lib/rq/sleepcycle.rb +46 -0
  51. data/lib/rq/snapshotter.rb +40 -0
  52. data/lib/rq/sqlite.rb +286 -0
  53. data/lib/rq/statuslister.rb +48 -0
  54. data/lib/rq/submitter.rb +113 -0
  55. data/lib/rq/toucher.rb +182 -0
  56. data/lib/rq/updater.rb +94 -0
  57. data/lib/rq/usage.rb +1222 -0
  58. data/lib/rq/util.rb +304 -0
  59. data/rdoc.sh +17 -0
  60. data/rq-ruby1.8.gemspec +120 -0
  61. data/test/.gitignore +1 -0
  62. data/test/test_rq.rb +145 -0
  63. data/white_box/crontab +2 -0
  64. data/white_box/joblist +8 -0
  65. data/white_box/killrq +18 -0
  66. data/white_box/rq_killer +27 -0
  67. metadata +208 -0
@@ -0,0 +1,155 @@
1
+ unless defined? $__rq__
2
+ module RQ
3
+ #--{{{
4
+ require 'rbconfig'
5
+
6
+ C = Config::CONFIG
7
+
8
+ AUTHOR = 'Ara Howard <ara.t.howard@gmail.com>'
9
+ AUTHOR2 = 'Pjotr Prins <pjotr.public01@thebird.nl>'
10
+ WEBSITE = 'https://github.com/pjotrp/rq'
11
+ LIBNAME = 'rq'
12
+ src = File.dirname(__FILE__)
13
+ VERSION = File.read(File.join(src,'..','VERSION')).strip
14
+ LIBVER = "#{ LIBNAME }-#{ VERSION }"
15
+ DIRNAME = File::dirname(File::expand_path(__FILE__)) + File::SEPARATOR
16
+ ROOTDIR = File::dirname(DIRNAME)
17
+ #LIBDIR = File::join(DIRNAME, LIBVER) + File::SEPARATOR
18
+ LIBDIR = File::join(DIRNAME, LIBNAME) + File::SEPARATOR
19
+ LOCALDIR = File::join(LIBDIR, 'local') + File::SEPARATOR
20
+ LOCALBINDIR = File::join(LOCALDIR, 'bin') + File::SEPARATOR
21
+ LOCALLIBDIR = File::join(LOCALDIR, 'lib') + File::SEPARATOR
22
+ ARCH = C['sitearch'] || C['arch']
23
+ ARCHLIBDIR = File::join(LIBDIR, ARCH) + File::SEPARATOR
24
+ EXIT_SUCCESS = 0
25
+ EXIT_FAILURE = 1
26
+ #
27
+ # builtin
28
+ #
29
+ require 'optparse'
30
+ require 'logger'
31
+ require 'socket'
32
+ require 'optparse'
33
+ require 'logger'
34
+ require 'yaml'
35
+ require 'yaml/store'
36
+ require 'pp'
37
+ require 'socket'
38
+ require 'pathname'
39
+ require 'tempfile'
40
+ require 'fileutils'
41
+ require 'tmpdir'
42
+ require 'drb/drb'
43
+ require 'digest/md5'
44
+ #
45
+ # try to load rubygems
46
+ #
47
+ begin
48
+ require 'rubygems'
49
+ rescue LoadError
50
+ nil
51
+ end
52
+
53
+ #
54
+ # depends - http://raa.ruby-lang.org
55
+ #
56
+ $:.push LIBDIR
57
+ $:.push ARCHLIBDIR
58
+ begin
59
+ require 'arrayfields'
60
+ rescue LoadError
61
+ begin
62
+ require LIBDIR + 'arrayfields'
63
+ rescue LoadError
64
+ abort "require arrayfields - http://raa.ruby-lang.org/project/arrayfields/"
65
+ end
66
+ end
67
+
68
+ begin
69
+ require 'lockfile'
70
+ rescue LoadError
71
+ begin
72
+ require LIBDIR + 'lockfile'
73
+ rescue LoadError
74
+ abort "require lockfile - http://raa.ruby-lang.org/project/lockfile/"
75
+ end
76
+ end
77
+
78
+ begin
79
+ require 'posixlock'
80
+ rescue LoadError
81
+ begin
82
+ require ARCHLIBDIR + 'posixlock'
83
+ rescue LoadError
84
+ abort "require posixlock - http://raa.ruby-lang.org/project/posixlock/"
85
+ end
86
+ end
87
+
88
+ #
89
+ # setup local require/lib/path/environment
90
+ #
91
+ $VERBOSE = nil
92
+ ENV['PATH'] = [LOCALBINDIR, ENV['PATH']].join(':')
93
+ ENV['LD_LIBRARY_PATH'] = [LOCALLIBDIR, ENV['LD_LIBRARY_PATH']].join(':')
94
+ begin
95
+ $:.unshift ARCHLIBDIR
96
+ $:.unshift LIBDIR
97
+ $:.unshift '/var/lib/gems/1.8/gems/sqlite-1.3.1/lib'
98
+
99
+ require 'sqlite'
100
+ # p $:
101
+ # require 'sqlite/version'
102
+ # print("SQLite.version=",SQLite::Version::STRING,"\n")
103
+
104
+ rescue LoadError
105
+ p $:
106
+ abort "require sqlite in load path - http://raa.ruby-lang.org/project/sqlite-ruby/"
107
+ ensure
108
+ $:.shift
109
+ $:.shift
110
+ end
111
+ #system "ldd #{ ARCHLIBDIR }/_sqlite.so"
112
+ #system "which sqlite"
113
+
114
+ #
115
+ # rq support libs
116
+ #
117
+ require LIBDIR + 'util'
118
+ require LIBDIR + 'logging'
119
+ require LIBDIR + 'orderedhash'
120
+ require LIBDIR + 'orderedautohash'
121
+ require LIBDIR + 'sleepcycle'
122
+ require LIBDIR + 'qdb'
123
+ require LIBDIR + 'jobqueue'
124
+ require LIBDIR + 'job'
125
+ require LIBDIR + 'jobrunner'
126
+ require LIBDIR + 'jobrunnerdaemon'
127
+ require LIBDIR + 'usage'
128
+ require LIBDIR + 'mainhelper'
129
+ require LIBDIR + 'creator'
130
+ require LIBDIR + 'submitter'
131
+ require LIBDIR + 'resubmitter'
132
+ require LIBDIR + 'lister'
133
+ require LIBDIR + 'statuslister'
134
+ require LIBDIR + 'deleter'
135
+ require LIBDIR + 'updater'
136
+ require LIBDIR + 'querier'
137
+ require LIBDIR + 'executor'
138
+ require LIBDIR + 'configurator'
139
+ require LIBDIR + 'snapshotter'
140
+ require LIBDIR + 'locker'
141
+ require LIBDIR + 'backer'
142
+ require LIBDIR + 'rotater'
143
+ require LIBDIR + 'feeder'
144
+ require LIBDIR + 'recoverer'
145
+ require LIBDIR + 'ioviewer'
146
+ require LIBDIR + 'toucher'
147
+ #require LIBDIR + 'resourcemanager'
148
+ #require LIBDIR + 'resource'
149
+ require LIBDIR + 'cron'
150
+ require LIBDIR + 'rails'
151
+
152
+ #--}}}
153
+ end # module rq
154
+ $__rq__ = __FILE__
155
+ end
@@ -0,0 +1,371 @@
1
+ #
2
+ # The ArrayFields module implements methods which allow an Array to be indexed
3
+ # by String or Symbol. It is not required to manually use this module to
4
+ # extend Arrays - they are auto-extended on a per-object basis when
5
+ # Array#fields= is called
6
+ #
7
+ module ArrayFields
8
+ self::VERSION = '3.8.0' unless defined? self::VERSION
9
+ def self.version() VERSION end
10
+ #
11
+ # multiton cache of fields - wraps fields and fieldpos map to save memory
12
+ #
13
+ class FieldSet
14
+ class << self
15
+ def new fields
16
+ @sets[fields] ||= super
17
+ end
18
+ def init_sets
19
+ @sets = {}
20
+ end
21
+ end
22
+
23
+ init_sets
24
+
25
+ attr :fields
26
+ attr :fieldpos
27
+ def initialize fields
28
+ raise ArgumentError, "<#{ fields.inspect }> not inject-able" unless
29
+ fields.respond_to? :inject
30
+
31
+ @fieldpos =
32
+ fields.inject({}) do |h, f|
33
+ unless String === f or Symbol === f
34
+ raise ArgumentError, "<#{ f.inspect }> neither String nor Symbol"
35
+ end
36
+ h[f] = h.size
37
+ h
38
+ end
39
+
40
+ @fields = fields
41
+ end
42
+ def pos f
43
+ return @fieldpos[f] if @fieldpos.has_key? f
44
+ f = f.to_s
45
+ return @fieldpos[f] if @fieldpos.has_key? f
46
+ f = f.intern
47
+ return @fieldpos[f] if @fieldpos.has_key? f
48
+ nil
49
+ end
50
+ end
51
+ #
52
+ # methods redefined to work with fields as well as numeric indexes
53
+ #
54
+ def [] idx, *args
55
+ if @fieldset and (String === idx or Symbol === idx)
56
+ pos = @fieldset.pos idx
57
+ return nil unless pos
58
+ super(pos, *args)
59
+ else
60
+ super
61
+ end
62
+ end
63
+ def slice idx, *args
64
+ if @fieldset and (String === idx or Symbol === idx)
65
+ pos = @fieldset.pos idx
66
+ return nil unless pos
67
+ super(pos, *args)
68
+ else
69
+ super
70
+ end
71
+ end
72
+
73
+ def []=(idx, *args)
74
+ if @fieldset and (String === idx or Symbol === idx)
75
+ pos = @fieldset.pos idx
76
+ unless pos
77
+ @fieldset.fields << idx
78
+ @fieldset.fieldpos[idx] = pos = size
79
+ end
80
+ super(pos, *args)
81
+ else
82
+ super
83
+ end
84
+ end
85
+ def at idx
86
+ if @fieldset and (String === idx or Symbol === idx)
87
+ pos = @fieldset.pos idx
88
+ return nil unless pos
89
+ super pos
90
+ else
91
+ super
92
+ end
93
+ end
94
+ def delete_at idx
95
+ if @fieldset and (String === idx or Symbol === idx)
96
+ pos = @fieldset.pos idx
97
+ return nil unless pos
98
+ super pos
99
+ else
100
+ super
101
+ end
102
+ end
103
+ def fill(obj, *args)
104
+ idx = args.first
105
+ if idx and @fieldset and (String === idx or Symbol === idx)
106
+ idx = args.shift
107
+ pos = @fieldset.pos idx
108
+ super(obj, pos, *args)
109
+ else
110
+ super
111
+ end
112
+ end
113
+
114
+ def values_at(*idxs)
115
+ idxs.flatten!
116
+ if @fieldset
117
+ idxs.map!{|i| (String === i or Symbol === i) ? @fieldset.pos(i) : i}
118
+ end
119
+ super(*idxs)
120
+ end
121
+ def indices(*idxs)
122
+ idxs.flatten!
123
+ if @fieldset
124
+ idxs.map!{|i| (String === i or Symbol === i) ? @fieldset.pos(i) : i}
125
+ end
126
+ super(*idxs)
127
+ end
128
+ def indexes(*idxs)
129
+ idxs.flatten!
130
+ if @fieldset
131
+ idxs.map!{|i| (String === i or Symbol === i) ? @fieldset.pos(i) : i}
132
+ end
133
+ super(*idxs)
134
+ end
135
+
136
+ def slice!(*args)
137
+ ret = self[*args]
138
+ self[*args] = nil
139
+ ret
140
+ end
141
+ def each_with_field
142
+ each_with_index do |elem, i|
143
+ yield elem, @fieldset.fields[i]
144
+ end
145
+ end
146
+ #
147
+ # methods which give a hash-like interface
148
+ #
149
+ def each_pair
150
+ each_with_index do |elem, i|
151
+ yield @fieldset.fields[i], elem
152
+ end
153
+ end
154
+ def each_key
155
+ @fieldset.each{|field| yield field}
156
+ end
157
+ def each_value(*args, &block)
158
+ each(*args, &block)
159
+ end
160
+ def fetch key
161
+ self[key] or raise IndexError, 'key not found'
162
+ end
163
+
164
+ def has_key? key
165
+ @fieldset.fields.include? key
166
+ end
167
+ def member? key
168
+ @fieldset.fields.include? key
169
+ end
170
+ def key? key
171
+ @fieldset.fields.include? key
172
+ end
173
+
174
+ def has_value? value
175
+ if respond_to? 'include?'
176
+ self.include? value
177
+ else
178
+ a = []
179
+ each{|val| a << val}
180
+ a.include? value
181
+ end
182
+ end
183
+ def value? value
184
+ if respond_to? 'include?'
185
+ self.include? value
186
+ else
187
+ a = []
188
+ each{|val| a << val}
189
+ a.include? value
190
+ end
191
+ end
192
+
193
+ def keys
194
+ fields
195
+ end
196
+ def store key, value
197
+ self[key] = value
198
+ end
199
+ def values
200
+ if respond_to? 'to_ary'
201
+ self.to_ary
202
+ else
203
+ a = []
204
+ each{|val| a << val}
205
+ a
206
+ end
207
+ end
208
+
209
+ def to_hash
210
+ if respond_to? 'to_ary'
211
+ h = {}
212
+ @fieldset.fields.zip(to_ary){|f,e| h[f] = e}
213
+ h
214
+ else
215
+ a = []
216
+ each{|val| a << val}
217
+ h = {}
218
+ @fieldset.fields.zip(a){|f,e| h[f] = e}
219
+ h
220
+ end
221
+ end
222
+ def to_h
223
+ if respond_to? 'to_ary'
224
+ h = {}
225
+ @fieldset.fields.zip(to_ary){|f,e| h[f] = e}
226
+ h
227
+ else
228
+ a = []
229
+ each{|val| a << val}
230
+ h = {}
231
+ @fieldset.fields.zip(a){|f,e| h[f] = e}
232
+ h
233
+ end
234
+ end
235
+
236
+ def update other
237
+ other.each{|k,v| self[k] = v}
238
+ to_hash
239
+ end
240
+ def replace other
241
+ Hash === other ? update(other) : super
242
+ end
243
+ def invert
244
+ to_hash.invert
245
+ end
246
+ end
247
+ Arrayfields = ArrayFields
248
+ #
249
+ # Fieldable encapsulates methods in common for classes which may have their
250
+ # fields set and subsequently be auto-extended by ArrayFields
251
+ #
252
+ module Fieldable
253
+ #
254
+ # sets fields an dynamically extends this Array instance with methods for
255
+ # keyword access
256
+ #
257
+ def fields= fields
258
+ extend ArrayFields unless defined? @fieldset
259
+
260
+ @fieldset =
261
+ if ArrayFields::FieldSet === fields
262
+ fields
263
+ else
264
+ ArrayFields::FieldSet.new fields
265
+ end
266
+ end
267
+ #
268
+ # access to fieldset
269
+ #
270
+ attr_reader :fieldset
271
+ #
272
+ # access to field list
273
+ #
274
+ def fields
275
+ @fieldset and @fieldset.fields
276
+ end
277
+ end
278
+ #
279
+ # Array instances are extened with two methods only: Fieldable#fields= and
280
+ # Fieldable#fields. only when Fieldable#fields= is called will the full set
281
+ # of ArrayFields methods auto-extend the Array instance. the Array class also
282
+ # has added a class generator when the fields are known apriori.
283
+ #
284
+ class Array
285
+ include Fieldable
286
+
287
+ class << self
288
+ def fields *fields
289
+ Class.new(self) do
290
+ const_set :FIELDS, ArrayFields::FieldSet.new(fields.flatten)
291
+ include ArrayFields
292
+ def initialize *a, &b
293
+ super
294
+ ensure
295
+ @fieldset = self.class.const_get :FIELDS
296
+ end
297
+ end
298
+ end
299
+ alias_method 'struct', 'fields'
300
+ end
301
+ end
302
+ #
303
+ # proxy class that allows an array to be wrapped in a way that still allows #
304
+ # keyword access. also facilitate usage of ArrayFields with arraylike objects.
305
+ # thnx to Sean O'Dell for the suggestion.
306
+ #
307
+ # sample usage
308
+ #
309
+ # fa = FieldedArray.new %w(zero one two), [0,1,2]
310
+ # p fa['zero'] #=> 0
311
+ #
312
+ #
313
+ class FieldedArray
314
+ include Fieldable
315
+ class << self
316
+ def [](*pairs)
317
+ pairs.flatten!
318
+ raise ArgumentError, "argument must be key/val pairs" unless
319
+ (pairs.size % 2 == 0)
320
+ fields, elements = [], []
321
+ while((f = pairs.shift) and (e = pairs.shift))
322
+ fields << f and elements << e
323
+ end
324
+ new fields, elements
325
+ end
326
+ end
327
+ def initialize fields = [], array = []
328
+ @a = array
329
+ self.fields = fields
330
+ end
331
+ def method_missing(meth, *args, &block)
332
+ @a.send(meth, *args, &block)
333
+ end
334
+ delegates =
335
+ %w(
336
+ to_s
337
+ to_str
338
+ inspect
339
+ )
340
+ delegates.each do |meth|
341
+ class_eval "def #{ meth }(*a,&b); @a.#{ meth }(*a,&b);end"
342
+ end
343
+ end
344
+ Fieldedarray = FieldedArray
345
+
346
+ class PseudoHash < ::Array
347
+ class << self
348
+ def [](*pairs)
349
+ pairs.flatten!
350
+ raise ArgumentError, "argument must be key/val pairs" unless
351
+ (pairs.size % 2 == 0 and pairs.size >= 2)
352
+ keys, values = [], []
353
+ while((k = pairs.shift) and (v = pairs.shift))
354
+ keys << k and values << v
355
+ end
356
+ new keys, values
357
+ end
358
+ end
359
+ def initialize keys = [], values = []
360
+ self.fields = keys
361
+ self.replace values
362
+ end
363
+ def to_yaml opts = {}
364
+ YAML::quick_emit object_id, opts do |out|
365
+ out.map taguri, to_yaml_style do |map|
366
+ each_pair{|f,v| map.add f,v}
367
+ end
368
+ end
369
+ end
370
+ end
371
+ Pseudohash = PseudoHash