bluth 0.5.3 → 0.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
data/lib/bluth/worker.rb CHANGED
@@ -1,7 +1,21 @@
1
+ require 'benchmark'
1
2
  require 'eventmachine'
2
3
  require 'rufus/scheduler'
3
4
  require 'daemonizing'
4
- require 'timeout'
5
+
6
+ class Rufus::Scheduler::SchedulerCore
7
+ # See lib/rufus/sc/scheduler.rb
8
+ def handle_exception(task, exception)
9
+ case exception
10
+ when SystemExit
11
+ exit
12
+ else
13
+ Familia.info exception.message
14
+ Familia.info exception.backtrace
15
+ task.unschedule
16
+ end
17
+ end
18
+ end
5
19
 
6
20
  module Bluth
7
21
  @salt = rand.gibbler.shorten(10).freeze
@@ -10,30 +24,29 @@ module Bluth
10
24
  end
11
25
 
12
26
  module WorkerBase
13
-
14
- def id
15
- @id ||= [host, user, rand, Time.now].gibbler.short
27
+
28
+ def init(h=nil, u=nil, w=nil)
29
+ @host, @user, @wid, = h || Bluth.sysinfo.hostname, u || Bluth.sysinfo.user, w
30
+ @pid_file ||= "/tmp/#{self.class.prefix}-#{wid}.pid"
31
+ @log_file ||= "/tmp/#{self.class.prefix}-#{wid}.log"
32
+ @success ||= 0
33
+ @failure ||= 0
34
+ @problem ||= 0
16
35
  end
17
36
 
18
- def longid
19
- [host, user, id].join('-')
37
+ def wid
38
+ @wid ||= [host, user, rand, Time.now.to_f].gibbler.short
39
+ @wid
20
40
  end
21
41
 
22
42
  # Used by daemonize as the process name (linux only)
23
43
  def name
24
- "bs-#{self.class.prefix}-#{id}"
44
+ "bluth-#{self.class.prefix}-#{wid}"
25
45
  end
26
46
 
27
- def key(suffix=nil)
28
- self.class.key longid, suffix
29
- end
30
-
31
- def initialize
32
- @host, @user = Bluth.sysinfo.hostname, Bluth.sysinfo.user
33
- @pid_file ||= "/tmp/#{self.class.prefix}-#{id}.pid"
34
- @log_file ||= "/tmp/#{self.class.prefix}-#{id}.log"
35
- @success, @failure, @problem = 0, 0, 0
36
- end
47
+ #def rediskey(suffix=nil)
48
+ # self.class.rediskey index, suffix
49
+ #end
37
50
 
38
51
  def current_job
39
52
  Gibbler::Digest.new(@current_job || '')
@@ -41,12 +54,12 @@ module Bluth
41
54
 
42
55
  def kill(force=false)
43
56
  if force || host == Bluth.sysinfo.hostname
44
- STDERR.puts "Destroying #{self.index} (this machine is: #{Bluth.sysinfo.hostname}; worker is: #{host})"
57
+ Familia.info "Destroying #{self.index} (this machine is: #{Bluth.sysinfo.hostname}; worker is: #{host})"
45
58
  Worker.kill self.pid_file if File.exists?(self.pid_file) rescue Errno::ESRCH
46
59
  File.delete self.log_file if File.exists?(self.log_file)
47
60
  destroy!
48
61
  else
49
- STDERR.puts "Worker #{self.index} not running on #{Bluth.sysinfo.hostname}"
62
+ Familia.info "Worker #{self.index} not running on #{Bluth.sysinfo.hostname}"
50
63
  end
51
64
  end
52
65
 
@@ -62,49 +75,57 @@ module Bluth
62
75
 
63
76
  module ClassMethods
64
77
  def from_redis(wid)
65
- me = new
66
- me.id = wid
67
- super(me.longid)
78
+ me = new nil, nil, wid
79
+ super(me.index)
68
80
  end
69
-
70
81
  def run!(*args)
71
82
  me = new
72
- Familia.info "Created: #{me.key}"
83
+ Familia.info "Created: #{me.rediskey}"
73
84
  me.run!
74
85
  me
75
86
  end
76
-
77
87
  def run(*args)
78
88
  me = new
79
- Familia.info "Created: #{me.key}"
89
+ Familia.info "Created: #{me.rediskey}"
80
90
  me.run
81
91
  me
82
92
  end
83
-
84
93
  def kill(pid_file)
94
+ self.class.runblock :onexit
85
95
  pid = read_pid_file pid_file
86
96
  super(pid_file, 10)
87
97
  end
88
-
89
-
98
+ def onstart &blk
99
+ @onstart = blk unless blk.nil?
100
+ @onstart
101
+ end
102
+ def onexit &blk
103
+ @onexit = blk unless blk.nil?
104
+ @onexit
105
+ end
106
+ # A convenience method for calling onstart/onexit blocks
107
+ def runblock meth
108
+ blk = self.send(meth)
109
+ return if blk.nil?
110
+ instance_eval &blk
111
+ end
90
112
  end
91
-
92
113
  end
93
114
 
94
115
  class Worker < Storable
95
- include WorkerBase
96
- @interval = 2.seconds
116
+ @interval = 2 #.seconds
97
117
  class << self
98
118
  attr_accessor :interval
99
119
  end
120
+ include WorkerBase
100
121
  include Familia
101
122
  include Logging
102
123
  include Daemonizable
103
124
  prefix :worker
104
- index :id
125
+ index [:host, :user, :wid]
105
126
  field :host
106
127
  field :user
107
- field :id
128
+ field :wid
108
129
  field :process_id => Integer
109
130
  field :pid_file
110
131
  field :log_file
@@ -113,35 +134,28 @@ module Bluth
113
134
  field :failure => Integer
114
135
  field :problem => Integer
115
136
  include Familia::Stamps
116
- def success!
117
- @success += 1
118
- @current_job = ""
119
- update_time
120
- save
121
- end
122
- def failure!
123
- @failure += 1
124
- @current_job = ""
125
- update_time
126
- save
127
- end
128
- def problem!
129
- @problem += 1
130
- @current_job = ""
131
- update_time
132
- save
137
+ [:success, :failure, :problem].each do |name|
138
+ define_method "#{name}!" do
139
+ v = self.send(name) + 1
140
+ self.send :"#{name}=", v
141
+ self.instance_variable_set '@current_job', ''
142
+ update_time! # calls save
143
+ end
133
144
  end
134
145
 
135
146
  def run!
136
147
  begin
148
+ self.class.runblock :onstart
137
149
  find_gob
138
150
  rescue => ex
139
151
  msg = "#{ex.class}: #{ex.message}"
140
- STDERR.puts msg
141
- Familia.ld :EXCEPTION, msg, caller[1] if Familia.debug?
152
+ Familia.info msg
153
+ Familia.trace :EXCEPTION, msg, caller[1] if Familia.debug?
154
+ self.class.runblock :onexit
142
155
  destroy!
143
156
  rescue Interrupt => ex
144
157
  puts $/, "Exiting..."
158
+ self.class.runblock :onexit
145
159
  destroy!
146
160
  end
147
161
  end
@@ -150,19 +164,10 @@ module Bluth
150
164
  begin
151
165
  @process_id = $$
152
166
  save
153
-
167
+ self.class.runblock :onstart
154
168
  scheduler = Rufus::Scheduler.start_new
155
169
  Familia.info "Setting interval: #{Worker.interval} sec (poptimeout: #{Bluth.poptimeout})"
156
170
  Familia.reconnect_all! # Need to reconnect after daemonize
157
- ## TODO: Works but needs to restart scheduler
158
- ##Signal.trap("USR1") do
159
- ## Worker.interval += 1
160
- ## Familia.info "Setting interval: #{Worker.interval} sec"
161
- ##end
162
- ##Signal.trap("USR2") do
163
- ## Worker.interval -= 1
164
- ## Familia.info "Setting interval: #{Worker.interval}"
165
- ##end
166
171
  scheduler.every Worker.interval, :blocking => true do |task|
167
172
  Familia.ld "#{$$} TICK @ #{Time.now.utc}"
168
173
  sleep rand
@@ -172,8 +177,9 @@ module Bluth
172
177
 
173
178
  rescue => ex
174
179
  msg = "#{ex.class}: #{ex.message}"
175
- STDERR.puts msg
176
- Familia.ld :EXCEPTION, msg, caller[1] if Familia.debug?
180
+ Familia.info msg
181
+ Familia.trace :EXCEPTION, msg, caller[1] if Familia.debug?
182
+ self.class.runblock :onexit
177
183
  destroy!
178
184
  rescue Interrupt => ex
179
185
  puts <<-EOS.gsub(/(?:^|\n)\s*/, "\n")
@@ -183,6 +189,7 @@ module Bluth
183
189
  EOS
184
190
  # We reconnect to the queue in case we're currently
185
191
  # waiting on a brpop (blocking pop) timeout.
192
+ self.class.runblock :onexit
186
193
  destroy!
187
194
  end
188
195
 
@@ -190,13 +197,13 @@ module Bluth
190
197
 
191
198
 
192
199
  private
193
- require 'benchmark'
200
+
194
201
  # DO NOT return from this method
195
202
  def find_gob(task=nil)
196
203
  begin
197
204
  job = Bluth.pop
198
205
  unless job.nil?
199
- job.wid = self.id
206
+ job.wid = self.wid
200
207
  if job.delayed?
201
208
  job.attempts = 0
202
209
  job.retry!
@@ -204,7 +211,7 @@ module Bluth
204
211
  job.failure! "Too many attempts"
205
212
  else
206
213
  job.stime = Time.now.utc.to_i
207
- self.working! job.id
214
+ self.working! job.jobid
208
215
  tms = Benchmark.measure do
209
216
  job.perform
210
217
  end
@@ -243,114 +250,76 @@ module Bluth
243
250
  #end
244
251
  end
245
252
  end
246
-
247
253
  end
248
254
 
255
+ # TODO: Refactor somehow. When this is subclassed
256
+ # (eg BS::SchduleWorker) the self.object is not created.
249
257
  class ScheduleWorker < Storable
250
258
  include WorkerBase
259
+ include Familia
260
+ include Logging
261
+ include Daemonizable
251
262
  @interval = 20
252
263
  @timeout = 60 #not working
264
+ @every = []
253
265
  class << self
254
- attr_accessor :interval, :timeout
266
+ attr_accessor :interval, :timeout, :schedule
255
267
  def interval(v=nil)
256
268
  @interval = v unless v.nil?
257
269
  @interval
258
270
  end
271
+ def every interval=nil, opts={}, &blk
272
+ unless interval.nil?
273
+ @every << [interval, opts, blk]
274
+ end
275
+ @every
276
+ end
259
277
  end
260
- include Familia
261
- include Logging
262
- include Daemonizable
263
278
  prefix :scheduler
264
- index :id
279
+ index [:host, :user, :wid]
265
280
  field :host
266
281
  field :user
267
- field :id
282
+ field :wid
268
283
  field :process_id => Integer
269
284
  field :pid_file
270
285
  field :log_file
271
- field :scheduled => Integer
272
- field :monitored => Integer
273
- field :timeouts => Integer
274
286
  include Familia::Stamps
275
- attr_reader :schedule
276
- attr_reader :monitors
277
-
278
- def scheduled!(count=1)
279
- @scheduled ||= 0
280
- @scheduled += count
281
- update_time
282
- save
283
- end
284
- def monitored!(count=1)
285
- @monitored ||= 0
286
- @monitored += count
287
- update_time
288
- save
289
- end
290
- def timeout!(count=1)
291
- @timeouts ||= 0
292
- @timeouts += count
293
- update_time
294
- save
295
- end
287
+
296
288
  def run!
297
289
  run
298
290
  end
291
+
299
292
  def run
300
293
  begin
301
- raise Familia::Problem, "Only 1 scheduler at a time" if ScheduleWorker.any?
302
-
294
+ raise Familia::Problem, "Only 1 scheduler at a time" if !ScheduleWorker.instances.empty?
303
295
  EM.run {
304
296
  @process_id = $$
305
297
  srand(Bluth.salt.to_i(16) ** @process_id)
306
- @schedule = Rufus::Scheduler::EmScheduler.start_new
298
+ ScheduleWorker.schedule = Rufus::Scheduler::EmScheduler.start_new
307
299
  save # persist and make note the scheduler is running
308
- prepare
309
- @schedule.every self.class.interval, :tags => :keeper do |keeper_task|
310
- begin
311
- scheduled_work(keeper_task)
312
- rescue => ex
313
- msg = "#{ex.class}: #{ex.message}"
314
- STDERR.puts msg
315
- STDERR.puts ex.backtrace
316
- Familia.ld :EXCEPTION, msg, caller[1] if Familia.debug?
317
- end
318
- sleep rand # prevent thrashing
300
+ self.class.runblock :onstart
301
+ self.class.every.each do |args|
302
+ interval, opts, blk = *args
303
+ Familia.ld " scheduling every #{interval}: #{opts}"
304
+ ScheduleWorker.schedule.every interval, opts, &blk
319
305
  end
320
306
  }
321
307
  rescue => ex
322
308
  msg = "#{ex.class}: #{ex.message}"
323
309
  puts msg
324
- STDERR.puts ex.backtrace
325
- Familia.ld :EXCEPTION, msg, caller[1] if Familia.debug?
310
+ Familia.info ex.backtrace
311
+ Familia.trace :EXCEPTION, msg, caller[1] if Familia.debug?
312
+ self.class.runblock :onexit
326
313
  destroy!
327
314
  rescue Interrupt => ex
328
315
  puts $/, "Exiting..."
316
+ self.class.runblock :onexit
329
317
  destroy!
330
318
  end
331
319
  end
332
320
 
333
- protected
334
-
335
- def prepare
336
- end
337
-
338
- def scheduled_work(keeper)
339
- STDOUT.puts "Come on!"
340
- end
341
-
342
321
  end
343
322
 
323
+ Bluth.scheduler = Bluth::ScheduleWorker
344
324
  end
345
325
 
346
- class Rufus::Scheduler::SchedulerCore
347
- # See lib/rufus/sc/scheduler.rb
348
- def handle_exception(job, exception)
349
- case exception
350
- when SystemExit
351
- exit
352
- else
353
- super
354
- end
355
- end
356
- end