rq-ruby1.8 3.4.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (67) hide show
  1. data/Gemfile +22 -0
  2. data/Gemfile.lock +22 -0
  3. data/INSTALL +166 -0
  4. data/LICENSE +10 -0
  5. data/Makefile +6 -0
  6. data/README +1183 -0
  7. data/Rakefile +37 -0
  8. data/TODO +24 -0
  9. data/TUTORIAL +230 -0
  10. data/VERSION +1 -0
  11. data/bin/rq +902 -0
  12. data/bin/rqmailer +865 -0
  13. data/example/a.rb +7 -0
  14. data/extconf.rb +198 -0
  15. data/gemspec.rb +40 -0
  16. data/install.rb +210 -0
  17. data/lib/rq.rb +155 -0
  18. data/lib/rq/arrayfields.rb +371 -0
  19. data/lib/rq/backer.rb +31 -0
  20. data/lib/rq/configfile.rb +82 -0
  21. data/lib/rq/configurator.rb +40 -0
  22. data/lib/rq/creator.rb +54 -0
  23. data/lib/rq/cron.rb +144 -0
  24. data/lib/rq/defaultconfig.txt +5 -0
  25. data/lib/rq/deleter.rb +51 -0
  26. data/lib/rq/executor.rb +40 -0
  27. data/lib/rq/feeder.rb +527 -0
  28. data/lib/rq/ioviewer.rb +48 -0
  29. data/lib/rq/job.rb +51 -0
  30. data/lib/rq/jobqueue.rb +947 -0
  31. data/lib/rq/jobrunner.rb +110 -0
  32. data/lib/rq/jobrunnerdaemon.rb +193 -0
  33. data/lib/rq/lister.rb +47 -0
  34. data/lib/rq/locker.rb +43 -0
  35. data/lib/rq/lockfile.rb +564 -0
  36. data/lib/rq/logging.rb +124 -0
  37. data/lib/rq/mainhelper.rb +189 -0
  38. data/lib/rq/orderedautohash.rb +39 -0
  39. data/lib/rq/orderedhash.rb +240 -0
  40. data/lib/rq/qdb.rb +733 -0
  41. data/lib/rq/querier.rb +98 -0
  42. data/lib/rq/rails.rb +80 -0
  43. data/lib/rq/recoverer.rb +28 -0
  44. data/lib/rq/refresher.rb +80 -0
  45. data/lib/rq/relayer.rb +283 -0
  46. data/lib/rq/resource.rb +22 -0
  47. data/lib/rq/resourcemanager.rb +40 -0
  48. data/lib/rq/resubmitter.rb +100 -0
  49. data/lib/rq/rotater.rb +98 -0
  50. data/lib/rq/sleepcycle.rb +46 -0
  51. data/lib/rq/snapshotter.rb +40 -0
  52. data/lib/rq/sqlite.rb +286 -0
  53. data/lib/rq/statuslister.rb +48 -0
  54. data/lib/rq/submitter.rb +113 -0
  55. data/lib/rq/toucher.rb +182 -0
  56. data/lib/rq/updater.rb +94 -0
  57. data/lib/rq/usage.rb +1222 -0
  58. data/lib/rq/util.rb +304 -0
  59. data/rdoc.sh +17 -0
  60. data/rq-ruby1.8.gemspec +120 -0
  61. data/test/.gitignore +1 -0
  62. data/test/test_rq.rb +145 -0
  63. data/white_box/crontab +2 -0
  64. data/white_box/joblist +8 -0
  65. data/white_box/killrq +18 -0
  66. data/white_box/rq_killer +27 -0
  67. metadata +208 -0
@@ -0,0 +1,124 @@
1
+ unless defined? $__rq_logging__
2
+ module RQ
3
+ #--{{{
4
+ LIBDIR = File::dirname(File::expand_path(__FILE__)) + File::SEPARATOR unless
5
+ defined? LIBDIR
6
+
7
+ require "logger"
8
+
9
+ require LIBDIR + 'util'
10
+ #
11
+ # module which adds logging methods to all RQ classes
12
+ #
13
+ module Logging
14
+ #--{{{
15
+ #
16
+ # a module that adds an accessor to Logging objects in ored to fix a bug where
17
+ # not all logging devices are put into sync mode, resulting in improper log
18
+ # rolling. this is a hack.
19
+ #
20
+ module LoggerExt
21
+ #--{{{
22
+ attr :logdev
23
+ #--}}}
24
+ end # module LoggerExt
25
+ #
26
+ # implementations of the methods shared by both classes and objects of classes
27
+ # which include Logging
28
+ #
29
+ module LogMethods
30
+ #--{{{
31
+ def logger
32
+ #--{{{
33
+ if defined?(@logger) and @logger
34
+ @logger
35
+ else
36
+ if Class === self
37
+ @logger = self.default_logger
38
+ else
39
+ @logger = self::class::logger
40
+ end
41
+ raise "@logger is undefined!" unless defined?(@logger) and @logger
42
+ @logger
43
+ end
44
+ #--}}}
45
+ end
46
+ def logger= log
47
+ #--{{{
48
+ @logger = log
49
+ @logger.extend LoggerExt
50
+ @logger.logdev.dev.sync = true
51
+ @logger
52
+ #--}}}
53
+ end
54
+ def debug(*args, &block); logger.debug(*args, &block); end
55
+ def info(*args, &block); logger.info(*args, &block) ; end
56
+ def warn(*args, &block); logger.warn(*args, &block) ; end
57
+ def error(*args, &block); logger.error(*args, &block); end
58
+ def fatal(*args, &block); logger.fatal(*args, &block); end
59
+ def logerr e
60
+ #--{{{
61
+ if logger.debug?
62
+ error{ Util::errmsg e }
63
+ else
64
+ error{ Util::emsg e }
65
+ end
66
+ #--}}}
67
+ end
68
+ #--}}}
69
+ end # module LogMethods
70
+
71
+ module LogClassMethods
72
+ #--{{{
73
+ def default_logger
74
+ #--{{{
75
+ if defined?(@default_logger) and @default_logger
76
+ @default_logger
77
+ else
78
+ self.default_logger = Logger::new STDERR
79
+ @default_logger = Logger::INFO
80
+ @default_logger.warn{ "<#{ self }> using default logger"}
81
+ @default_logger
82
+ end
83
+ #--}}}
84
+ end
85
+ def default_logger= log
86
+ #--{{{
87
+ @default_logger = (Logger === log ? log : Logger::new(log))
88
+ @default_logger.extend LoggerExt
89
+ @default_logger.logdev.dev.sync = true
90
+ @default_logger
91
+ #--}}}
92
+ end
93
+ #--}}}
94
+ end
95
+
96
+ EOL = "\n"
97
+ DIV0 = ("." * 79) << EOL
98
+ DIV1 = ("-" * 79) << EOL
99
+ DIV2 = ("=" * 79) << EOL
100
+ DIV3 = ("#" * 79) << EOL
101
+ SEC0 = ("." * 16) << EOL
102
+ SEC1 = ("-" * 16) << EOL
103
+ SEC2 = ("=" * 16) << EOL
104
+ SEC3 = ("#" * 16) << EOL
105
+
106
+ class << self
107
+ #--{{{
108
+ def append_features c
109
+ #--{{{
110
+ ret = super
111
+ c.extend LogMethods
112
+ c.extend LogClassMethods
113
+ ret
114
+ #--}}}
115
+ end
116
+ #--}}}
117
+ end
118
+ include LogMethods
119
+ #--}}}
120
+ end # module Logging
121
+ #--}}}
122
+ end # module rq
123
+ $__rq_logging__ = __FILE__
124
+ end
@@ -0,0 +1,189 @@
1
+ unless defined? $__rq_mainhelper__
2
+ module RQ
3
+ #--{{{
4
+ LIBDIR = File::dirname(File::expand_path(__FILE__)) + File::SEPARATOR unless
5
+ defined? LIBDIR
6
+
7
+ require 'tempfile'
8
+
9
+ require LIBDIR + 'util'
10
+ require LIBDIR + 'logging'
11
+
12
+ #
13
+ # the MainHelper class abstracts some of the common functions the various
14
+ # Main delegates require
15
+ #
16
+ class MainHelper
17
+ #--{{{
18
+ include Util
19
+ include Logging
20
+ attr :main
21
+ attr :argv
22
+ attr :env
23
+ attr :program
24
+ attr :stdin
25
+ attr :job_stdin
26
+ attr :cmd
27
+ attr :options
28
+ attr :qpath
29
+ attr :mode
30
+ attr :quiet
31
+ attr :q
32
+ attr :fields
33
+ attr :dot_rq_dir
34
+ attr :loops
35
+
36
+ alias_method 'stdin?', 'stdin'
37
+ alias_method 'job_stdin?', 'job_stdin'
38
+ alias_method 'quiet?', 'quiet'
39
+ def initialize main
40
+ #--{{{
41
+ @main = main
42
+ @logger = main.logger
43
+ @argv = main.argv
44
+ @env = main.env
45
+ @program = main.program
46
+ @stdin = main.stdin
47
+ @data = main.data
48
+ @job_stdin = main.job_stdin
49
+ @cmd = main.cmd
50
+ @options = main.options
51
+ @qpath = main.qpath
52
+ @mode = main.mode
53
+ @quiet = main.quiet
54
+ @fields = main.fields
55
+ @dot_rq_dir = main.dot_rq_dir
56
+ @loops = main.loops
57
+ @q = nil
58
+ #--}}}
59
+ end
60
+ def set_q
61
+ #--{{{
62
+ raise "q <#{ @qpath }> does not exist" unless test ?d, @qpath
63
+ @q = JobQueue::new @qpath, 'logger' => @logger
64
+ if @options['snapshot']
65
+ ss = "#{ $0 }_#{ Process::pid }_#{ Thread::current.object_id.abs }_#{ rand Time::now.to_i }".gsub(%r|/|o,'_')
66
+ qtmp = File::join Dir::tmpdir, ss
67
+ @q = @q.snapshot qtmp, @options['retries']
68
+ at_exit{ FileUtils::rm_rf qtmp }
69
+ end
70
+ #--}}}
71
+ end
72
+ def loadio io, path, jobs
73
+ #--{{{
74
+ while((line = io.gets))
75
+ if line =~ %r/^---\s*$/o
76
+ loaded = YAML::load io
77
+ raise "no jobs in <#{ path }>" unless
78
+ Array === loaded and
79
+ Hash === loaded.first and Hash === loaded.last
80
+ loaded.each{|job| jobs << job}
81
+ loaded = nil
82
+ else
83
+ # line.gsub!(%r/(?:^\s+)|(?:\s+$)|(?:#.*$)/o, '')
84
+ line.strip!
85
+ next if line.empty?
86
+ job = Job::new
87
+ if((m = %r/^\s*(?:jid\s*=\s*)?(\d+)\s*$/io.match(line)))
88
+ job['jid'] = Integer(m[1])
89
+ else
90
+ job['command'] = line
91
+ end
92
+ jobs << job
93
+ end
94
+ end
95
+ #--}}}
96
+ end
97
+ def loadio io, path, jobs
98
+ #--{{{
99
+ while((line = io.gets))
100
+ if line =~ %r/^---\s*$/o
101
+ loadyaml io, path, jobs
102
+ else
103
+ # line.gsub!(%r/(?:^\s+)|(?:\s+$)|(?:#.*$)/o, '')
104
+ line.strip!
105
+ next if line.empty?
106
+ job = Job::new
107
+ if((m = %r/^\s*(?:jid\s*=\s*)?(\d+)\s*$/io.match(line)))
108
+ job['jid'] = Integer(m[1])
109
+ else
110
+ job['command'] = line
111
+ end
112
+ jobs << job
113
+ end
114
+ end
115
+ #--}}}
116
+ end
117
+ def loadyaml io, path, jobs
118
+ #--{{{
119
+ h = nil
120
+ while((line = io.gets))
121
+ line.strip!
122
+ next if line.empty?
123
+ case line
124
+ when %r/^\s*-\s*$/
125
+ jobs << h if h
126
+ h = {}
127
+ else
128
+ k, v = line.split %r/:/, 2
129
+ k.strip!
130
+ v.strip!
131
+ h[k] = v
132
+ end
133
+ end
134
+ jobs << h if h
135
+ #--}}}
136
+ end
137
+ def dumping_yaml_tuples
138
+ #--{{{
139
+ fields = nil
140
+ dump = lambda do |tuple|
141
+ puts '---'
142
+ if fields.nil?
143
+ if @fields
144
+ fields = field_match @fields, tuple.fields
145
+ else
146
+ fields = tuple.fields
147
+ end
148
+ end
149
+ dump = lambda do |tuple|
150
+ puts '-'
151
+ fields.each{|f| puts " #{ f }: #{ tuple[ f ] }"}
152
+ end
153
+ dump[tuple]
154
+ end
155
+ lambda{|tuple| dump[tuple]}
156
+ #--}}}
157
+ end
158
+ def field_match srclist, dstlist
159
+ #--{{{
160
+ fields = dstlist.select do |dst|
161
+ srclist.map do |src|
162
+ re =
163
+ if src =~ %r/^[a-zA-Z0-9_-]+$/
164
+ %r/^#{ src }/i
165
+ else
166
+ %r/#{ src }/i
167
+ end
168
+ src == dst or dst =~ re
169
+ end.any?
170
+ end.uniq
171
+ #--}}}
172
+ end
173
+ def init_job_stdin!
174
+ #--{{{
175
+ if @job_stdin == '-'
176
+ tmp = Tempfile::new "#{ Process::pid }_#{ rand 42 }"
177
+ while((buf = STDIN.read(8192))); tmp.write buf; end
178
+ tmp.close
179
+ @job_stdin = tmp.path
180
+ end
181
+ @job_stdin
182
+ #--}}}
183
+ end
184
+ #--}}}
185
+ end # class MainHelper
186
+ #--}}}
187
+ end # module RQ
188
+ $__rq_mainhelper__ = __FILE__
189
+ end
@@ -0,0 +1,39 @@
1
+ unless defined? $__rq_orderedautohash__
2
+ module RQ
3
+ #--{{{
4
+ LIBDIR = File::dirname(File::expand_path(__FILE__)) + File::SEPARATOR unless
5
+ defined? LIBDIR
6
+
7
+ require LIBDIR + 'orderedhash'
8
+
9
+ #if false
10
+ class ::OrderedHash
11
+ #--{{{
12
+ def initialize(*a, &b)
13
+ #--{{{
14
+ super
15
+ @order = []
16
+ #--}}}
17
+ end
18
+ #--}}}
19
+ end
20
+ #end
21
+
22
+ class OrderedAutoHash < ::OrderedHash
23
+ #--{{{
24
+ def initialize(*args)
25
+ #--{{{
26
+ super(*args){|a,k| a[k] = OrderedAutoHash::new(*args)}
27
+ #--}}}
28
+ end
29
+ def class
30
+ #--{{{
31
+ ::Hash
32
+ #--}}}
33
+ end
34
+ #--}}}
35
+ end # class OrderedAutoHash
36
+ #--}}}
37
+ end # module RQ
38
+ $__rq_orderedautohash__ = __FILE__
39
+ end
@@ -0,0 +1,240 @@
1
+ # AUTHOR
2
+ # jan molic /mig/at/1984/dot/cz/
3
+ #
4
+ # DESCRIPTION
5
+ # Hash with preserved order and some array-like extensions
6
+ # Public domain.
7
+ #
8
+ # THANKS
9
+ # Andrew Johnson for his suggestions and fixes of Hash[],
10
+ # merge, to_a, inspect and shift
11
+ #
12
+ # USAGE
13
+ # just require this file and use OrderedHash instead of Hash (examples are at the end)
14
+ # you can try to run this file (ruby orderedhash.rb)
15
+ #
16
+
17
+ class OrderedHash < Hash
18
+
19
+ attr_accessor :order
20
+
21
+ class << self
22
+ def [] *args
23
+ hsh = OrderedHash.new
24
+ if Hash === args[0]
25
+ hsh.replace args[0]
26
+ elsif (args.size % 2) != 0
27
+ raise ArgumentError, "odd number of elements for Hash"
28
+ else
29
+ hsh[args.shift] = args.shift while args.size > 0
30
+ end
31
+ hsh
32
+ end
33
+ end
34
+
35
+ def initialize
36
+ @order = []
37
+ end
38
+
39
+ def store_only a,b
40
+ store a,b
41
+ end
42
+
43
+ alias orig_store store
44
+ def store a,b
45
+ @order.push a unless has_key? a
46
+ super a,b
47
+ end
48
+ alias []= store
49
+
50
+ def == hsh2
51
+ return false if @order != hsh2.order
52
+ super hsh2
53
+ end
54
+
55
+ def clear
56
+ @order = []
57
+ super
58
+ end
59
+
60
+ def delete key
61
+ @order.delete key
62
+ super
63
+ end
64
+
65
+ def each_key
66
+ @order.each { |k| yield k }
67
+ self
68
+ end
69
+
70
+ def each_value
71
+ @order.each { |k| yield self[k] }
72
+ self
73
+ end
74
+
75
+ def each
76
+ @order.each { |k| yield k,self[k] }
77
+ self
78
+ end
79
+ alias each_pair each
80
+
81
+ def delete_if
82
+ @order.clone.each { |k|
83
+ delete k if yield
84
+ }
85
+ self
86
+ end
87
+
88
+ def values
89
+ ary = []
90
+ @order.each { |k| ary.push self[k] }
91
+ ary
92
+ end
93
+
94
+ def keys
95
+ @order
96
+ end
97
+
98
+ def invert
99
+ hsh2 = Hash.new
100
+ @order.each { |k| hsh2[self[k]] = k }
101
+ hsh2
102
+ end
103
+
104
+ def reject &block
105
+ self.dup.delete_if &block
106
+ end
107
+
108
+ def reject! &block
109
+ hsh2 = reject &block
110
+ self == hsh2 ? nil : hsh2
111
+ end
112
+
113
+ def replace hsh2
114
+ @order = hsh2.keys
115
+ super hsh2
116
+ end
117
+
118
+ def shift
119
+ key = @order.first
120
+ key ? [key,delete(key)] : super
121
+ end
122
+
123
+ def unshift k,v
124
+ unless self.include? k
125
+ @order.unshift k
126
+ orig_store(k,v)
127
+ true
128
+ else
129
+ false
130
+ end
131
+ end
132
+
133
+ def push k,v
134
+ unless self.include? k
135
+ @order.push k
136
+ orig_store(k,v)
137
+ true
138
+ else
139
+ false
140
+ end
141
+ end
142
+
143
+ def pop
144
+ key = @order.last
145
+ key ? [key,delete(key)] : nil
146
+ end
147
+
148
+ def to_a
149
+ ary = []
150
+ each { |k,v| ary << [k,v] }
151
+ ary
152
+ end
153
+
154
+ def to_s
155
+ self.to_a.to_s
156
+ end
157
+
158
+ def inspect
159
+ ary = []
160
+ each {|k,v| ary << k.inspect + "=>" + v.inspect}
161
+ '{' + ary.join(", ") + '}'
162
+ end
163
+
164
+ def update hsh2
165
+ hsh2.each { |k,v| self[k] = v }
166
+ self
167
+ end
168
+ alias :merge! update
169
+
170
+ def merge hsh2
171
+ self.dup update(hsh2)
172
+ end
173
+
174
+ def select
175
+ ary = []
176
+ each { |k,v| ary << [k,v] if yield k,v }
177
+ ary
178
+ end
179
+
180
+ attr_accessor "to_yaml_style"
181
+ def yaml_inline= bool
182
+ if respond_to?("to_yaml_style")
183
+ self.to_yaml_style = :inline
184
+ else
185
+ unless defined? @__yaml_inline_meth
186
+ @__yaml_inline_meth =
187
+ lambda {|opts|
188
+ YAML::quick_emit(object_id, opts) {|emitter|
189
+ emitter << '{ ' << map{|kv| kv.join ': '}.join(', ') << ' }'
190
+ }
191
+ }
192
+ class << self
193
+ def to_yaml opts = {}
194
+ begin
195
+ @__yaml_inline ? @__yaml_inline_meth[ opts ] : super
196
+ rescue
197
+ @to_yaml_style = :inline
198
+ super
199
+ end
200
+ end
201
+ end
202
+ end
203
+ end
204
+ @__yaml_inline = bool
205
+ end
206
+ def yaml_inline!() self.yaml_inline = true end
207
+
208
+ end
209
+
210
+ if __FILE__ == $0
211
+
212
+ # You can do simply
213
+ hsh = OrderedHash.new
214
+ hsh['z'] = 1
215
+ hsh['a'] = 2
216
+ hsh['c'] = 3
217
+ p hsh.keys # ['z','a','c']
218
+
219
+ # or using OrderedHash[] method
220
+ hsh = OrderedHash['z', 1, 'a', 2, 'c', 3]
221
+ p hsh.keys # ['z','a','c']
222
+
223
+ # but this don't preserve order
224
+ hsh = OrderedHash['z'=>1, 'a'=>2, 'c'=>3]
225
+ p hsh.keys # ['a','c','z']
226
+
227
+ # OrderedHash has useful extensions: push, pop and unshift
228
+ p hsh.push('to_end', 15) # true, key added
229
+ p hsh.push('to_end', 30) # false, already - nothing happen
230
+ p hsh.unshift('to_begin', 50) # true, key added
231
+ p hsh.unshift('to_begin', 60) # false, already - nothing happen
232
+ p hsh.keys # ["to_begin", "a", "c", "z", "to_end"]
233
+ p hsh.pop # ["to_end", 15], if nothing remains, return nil
234
+ p hsh.keys # ["to_begin", "a", "c", "z"]
235
+ p hsh.shift # ["to_begin", 30], if nothing remains, return nil
236
+
237
+ end
238
+
239
+
240
+ # END