bjj 1.0.3

Sign up to get free protection for your applications and to get access to all the features.
@@ -0,0 +1,120 @@
1
+ module Attributes
2
+ Attributes::VERSION = '5.0.0' unless defined? Attributes::VERSION
3
+ def self.version() Attributes::VERSION end
4
+
5
+ class List < ::Array
6
+ def << element
7
+ super
8
+ self
9
+ ensure
10
+ uniq!
11
+ index!
12
+ end
13
+ def index!
14
+ @index ||= Hash.new
15
+ each{|element| @index[element] = true}
16
+ end
17
+ def include? element
18
+ @index ||= Hash.new
19
+ @index[element] ? true : false
20
+ end
21
+ def initializers
22
+ @initializers ||= Hash.new
23
+ end
24
+ end
25
+
26
+ def attributes *a, &b
27
+ unless a.empty?
28
+ returned = Hash.new
29
+
30
+ hashes, names = a.partition{|x| Hash === x}
31
+ names_and_defaults = {}
32
+ hashes.each{|h| names_and_defaults.update h}
33
+ names.flatten.compact.each{|name| names_and_defaults.update name => nil}
34
+
35
+ initializers = __attributes__.initializers
36
+
37
+ names_and_defaults.each do |name, default|
38
+ raise NameError, "bad instance variable name '@#{ name }'" if "@#{ name }" =~ %r/[!?=]$/o
39
+ name = name.to_s
40
+
41
+ initialize = b || lambda { default }
42
+ initializer = lambda do |this|
43
+ Object.instance_method('instance_eval').bind(this).call &initialize
44
+ end
45
+ initializer_id = initializer.object_id
46
+ __attributes__.initializers[name] = initializer
47
+
48
+ module_eval <<-code
49
+ def #{ name }=(*value, &block)
50
+ value.unshift block if block
51
+ @#{ name } = value.first
52
+ end
53
+ code
54
+
55
+ module_eval <<-code
56
+ def #{ name }(*value, &block)
57
+ value.unshift block if block
58
+ return self.send('#{ name }=', value.first) unless value.empty?
59
+ #{ name }! unless defined? @#{ name }
60
+ @#{ name }
61
+ end
62
+ code
63
+
64
+ module_eval <<-code
65
+ def #{ name }!
66
+ initializer = ObjectSpace._id2ref #{ initializer_id }
67
+ self.#{ name } = initializer.call(self)
68
+ @#{ name }
69
+ end
70
+ code
71
+
72
+ module_eval <<-code
73
+ def #{ name }?
74
+ #{ name }
75
+ end
76
+ code
77
+
78
+ attributes << name
79
+ returned[name] = initializer
80
+ end
81
+
82
+ returned
83
+ else
84
+ begin
85
+ __attribute_list__
86
+ rescue NameError
87
+ singleton_class =
88
+ class << self
89
+ self
90
+ end
91
+ klass = self
92
+ singleton_class.module_eval do
93
+ attribute_list = List.new
94
+ define_method('attribute_list'){ klass == self ? attribute_list : raise(NameError) }
95
+ alias_method '__attribute_list__', 'attribute_list'
96
+ end
97
+ __attribute_list__
98
+ end
99
+ end
100
+ end
101
+
102
+ %w( __attributes__ __attribute__ attribute ).each{|dst| alias_method dst, 'attributes'}
103
+ end
104
+
105
+ =begin
106
+ class Object
107
+ def attributes *a, &b
108
+ sc =
109
+ class << self
110
+ self
111
+ end
112
+ sc.attributes *a, &b
113
+ end
114
+ %w( __attributes__ __attribute__ attribute ).each{|dst| alias_method dst, 'attributes'}
115
+ end
116
+ =end
117
+
118
+ class Module
119
+ include Attributes
120
+ end
@@ -0,0 +1,72 @@
1
+ class Bj
2
+ module ClassMethods
3
+ attribute("rails_root"){ Util.const_or_env("RAILS_ROOT"){ "." } }
4
+ attribute("rails_env"){ Util.const_or_env("RAILS_ENV"){ "development" } }
5
+ attribute("database_yml"){ File.join rails_root, "config", "database.yml" }
6
+ attribute("configurations"){ YAML::load(ERB.new(IO.read(database_yml)).result) }
7
+ attribute("tables"){ Table.list }
8
+ attribute("hostname"){ Socket.gethostname }
9
+ attribute("logger"){ Bj::Logger.off STDERR }
10
+ attribute("ruby"){ Util.which_ruby }
11
+ attribute("rake"){ Util.which_rake }
12
+ attribute("script"){ Util.find_script "bj" }
13
+ attribute("ttl"){ Integer(Bj::Table::Config["ttl"] || (twenty_four_hours = 24 * 60 * 60)) }
14
+ attribute("table"){ Table }
15
+ attribute("config"){ table.config }
16
+ attribute("util"){ Util }
17
+ attribute("runner"){ Runner }
18
+ attribute("joblist"){ Joblist }
19
+ attribute("default_path"){ %w'/bin /usr/bin /usr/local/bin /opt/local/bin'.join(File::PATH_SEPARATOR) }
20
+
21
+ def transaction options = {}, &block
22
+ options.to_options!
23
+
24
+ cur_rails_env = Bj.rails_env.to_s
25
+ new_rails_env = options[:rails_env].to_s
26
+
27
+ cur_spec = configurations[cur_rails_env]
28
+ table.establish_connection(cur_spec) unless table.connected?
29
+
30
+ if(new_rails_env.empty? or cur_rails_env == new_rails_env)
31
+ table.transaction{ block.call(table.connection) }
32
+ else
33
+ new_spec = configurations[new_rails_env]
34
+ table.establish_connection(new_spec)
35
+ Bj.rails_env = new_rails_env
36
+ begin
37
+ table.transaction{ block.call(table.connection) }
38
+ ensure
39
+ table.establish_connection(cur_spec)
40
+ Bj.rails_env = cur_rails_env
41
+ end
42
+ end
43
+ end
44
+
45
+ def chroot options = {}, &block
46
+ if defined? @chrooted and @chrooted
47
+ return(block ? block.call(@chrooted) : @chrooted)
48
+ end
49
+ if block
50
+ begin
51
+ chrooted = @chrooted
52
+ Dir.chdir(@chrooted = rails_root) do
53
+ raise RailsRoot, "<#{ Dir.pwd }> is not a rails root" unless Util.valid_rails_root?(Dir.pwd)
54
+ block.call(@chrooted)
55
+ end
56
+ ensure
57
+ @chrooted = chrooted
58
+ end
59
+ else
60
+ Dir.chdir(@chrooted = rails_root)
61
+ raise RailsRoot, "<#{ Dir.pwd }> is not a rails root" unless Util.valid_rails_root?(Dir.pwd)
62
+ @chrooted
63
+ end
64
+ end
65
+
66
+ def boot
67
+ load File.join(rails_root, "config", "boot.rb")
68
+ load File.join(rails_root, "config", "environment.rb")
69
+ end
70
+ end
71
+ send :extend, ClassMethods
72
+ end
@@ -0,0 +1,4 @@
1
+ class Bj
2
+ class Error < ::StandardError; end
3
+ class RailsRoot < Error; end
4
+ end
@@ -0,0 +1,112 @@
1
+ class Bj
2
+ class JobList < ::Array
3
+ module ClassMethods
4
+ def for jobs, options = {}
5
+ if Joblist === jobs
6
+ jobs.update options
7
+ return jobs
8
+ end
9
+ options.to_options!
10
+ jobs = [jobs].flatten.compact
11
+ list = []
12
+ jobs.each do |arg|
13
+ list.push(
14
+ case arg
15
+ when String
16
+ case arg
17
+ when %r/^\d+$/
18
+ job_from_id arg
19
+ else
20
+ job_from_string arg
21
+ end
22
+ when Hash
23
+ job_from_hash arg
24
+ when Io
25
+ jobs_from_io arg
26
+ when Fixnum, Bignum
27
+ job_from_number arg
28
+ else
29
+ job_from_string arg
30
+ end
31
+ )
32
+ end
33
+ list.flatten!
34
+ list.compact!
35
+ list.map!{|job| job.reverse_merge! options}
36
+ list
37
+ end
38
+
39
+ def job_from_hash arg
40
+ arg.to_hash.to_options!
41
+ end
42
+
43
+ def job_from_string arg
44
+ unless arg.strip.empty?
45
+ { :command => arg.to_s }
46
+ else
47
+ nil
48
+ end
49
+ end
50
+
51
+ def job_from_number arg
52
+ id = arg.to_i
53
+ Table::Job.find(id).to_hash
54
+ end
55
+
56
+ def jobs_from_io arg
57
+ if arg == "-"
58
+ load_from_io STDIN
59
+ else
60
+ if arg.respond_to? :read
61
+ load_from_io arg
62
+ else
63
+ open(arg, "r"){|fd| load_from_io fd}
64
+ end
65
+ end
66
+ end
67
+
68
+ def load_from_io io
69
+ list = []
70
+ io.each do |line|
71
+ line.strip!
72
+ next if line.empty?
73
+ list << job_from_string(line)
74
+ end
75
+ list
76
+ end
77
+
78
+ def jobs_from_yaml arg
79
+ object =
80
+ if arg == "-"
81
+ YAML.load STDIN
82
+ else
83
+ if arg.respond_to? :read
84
+ YAML.load arg
85
+ else
86
+ open(arg, "r"){|fd| YAML.load fd}
87
+ end
88
+ end
89
+ Joblist.for object
90
+ end
91
+ end
92
+ send :extend, ClassMethods
93
+
94
+ module InstanceMethods
95
+ def update options = {}
96
+ options.to_options!
97
+ each{|job| job.update options}
98
+ end
99
+
100
+ def push other
101
+ Joblist.for(other).each do |job|
102
+ super job
103
+ end
104
+ self
105
+ end
106
+ alias_method "<<", "push"
107
+ end
108
+ send :include, InstanceMethods
109
+ end
110
+
111
+ Joblist = JobList
112
+ end
@@ -0,0 +1,50 @@
1
+ class Bj
2
+ class Logger < ::Logger
3
+ def self.new *a, &b
4
+ super(*a, &b).instance_eval{ @default_formatter = @formatter = Formatter.new; self }
5
+ end
6
+ def format_message(severity, datetime, progname, msg)
7
+ (@formatter || @default_formatter).call(severity, datetime, progname, msg)
8
+ end
9
+
10
+ def device
11
+ @logdev.instance_eval{ @dev }
12
+ end
13
+
14
+ def tty?
15
+ device.respond_to?('tty?') and device.tty?
16
+ end
17
+
18
+ def turn which
19
+ @logdev.extend OnOff unless OnOff === @logdev
20
+ @logdev.turn which
21
+ end
22
+
23
+ module OnOff
24
+ def turn which
25
+ @turned = which.to_s =~ %r/on/i ? :on : :off
26
+ end
27
+
28
+ def write message
29
+ return message.to_s.size if @turned == :off
30
+ super
31
+ end
32
+ end
33
+
34
+ def on
35
+ turn :on
36
+ end
37
+ alias_method "on!", "on"
38
+ def self.on *a, &b
39
+ new(*a, &b).instance_eval{ turn :on; self }
40
+ end
41
+
42
+ def off
43
+ turn :off
44
+ end
45
+ alias_method "off!", "off"
46
+ def self.off *a, &b
47
+ new(*a, &b).instance_eval{ turn :off; self }
48
+ end
49
+ end
50
+ end
@@ -0,0 +1,359 @@
1
+ class Bj
2
+ class Runner
3
+ class Background
4
+ def self.for(*a, &b) new(*a, &b) end
5
+
6
+ attribute "command"
7
+ attribute "thread"
8
+ attribute "pid"
9
+
10
+ def initialize command
11
+ @command = command
12
+ @thread = new_thread
13
+ end
14
+
15
+ def inspect
16
+ {
17
+ "command" => command,
18
+ "pid" => pid,
19
+ }.inspect
20
+ end
21
+
22
+
23
+ # TODO - auto start runner?
24
+
25
+ def new_thread
26
+ this = self
27
+ Thread.new do
28
+ Thread.current.abort_on_exception = true
29
+ loop do
30
+ cleanup = lambda{}
31
+
32
+ IO.popen command, "r+" do |pipe|
33
+ this.pid = pid = pipe.pid
34
+ cleanup = lambda do
35
+ cleanup = lambda{}
36
+ begin; Process.kill(Runner.kill_signal, pid); rescue Exception; 42; end
37
+ end
38
+ at_exit &cleanup
39
+ begin ; Process.wait; rescue Exception; 42; end
40
+ end
41
+
42
+ Bj.logger.error{ "#{ command } failed with #{ $?.inspect }" } unless
43
+ [0, 42].include?($?.exitstatus)
44
+
45
+ cleanup.call
46
+
47
+ sleep 42
48
+ end
49
+ end
50
+ end
51
+ end
52
+
53
+ module ClassMethods
54
+ attribute("thread"){ Thread.current }
55
+ attribute("hup_signal"){ Signal.list.keys.index("HUP") ? "HUP" : "ABRT" }
56
+ attribute("hup_signaled"){ false }
57
+ attribute("kill_signal"){ "TERM" }
58
+ attribute("kill_signaled"){ false }
59
+
60
+ def tickle
61
+ return nil if Bj.config[Runner.no_tickle_key]
62
+ ping or start
63
+ end
64
+
65
+ def ping
66
+ begin
67
+ pid = nil
68
+ uri = nil
69
+ process = nil
70
+ Bj.transaction do
71
+ pid = Bj.config[Runner.key(Process.pid)] || Bj.config[Runner.key]
72
+ uri = Bj.config["#{ pid }.uri"]
73
+ process =
74
+ if uri
75
+ require "drb"
76
+ # DRb.start_service "druby://localhost:0"
77
+ DRbObject.new(nil, uri)
78
+ else
79
+ Process
80
+ end
81
+ end
82
+ return nil unless pid
83
+ pid = Integer pid
84
+ begin
85
+ process.kill Runner.hup_signal, pid
86
+ pid
87
+ rescue Exception => e
88
+ false
89
+ end
90
+ rescue Exception => e
91
+ false
92
+ end
93
+ end
94
+
95
+ def key ppid = 0
96
+ ppid ||= 0
97
+ #"#{ Bj.rails_env }.#{ ppid }.pid"
98
+ "runner.#{ ppid }.pid"
99
+ end
100
+
101
+ def no_tickle_key
102
+ # "#{ Bj.rails_env }.no_tickle"
103
+ "no_tickle"
104
+ end
105
+
106
+ def start options = {}
107
+ options.to_options!
108
+ background.delete Bj.rails_env if options[:force]
109
+ background[Bj.rails_env] ||= Background.for(command)
110
+ end
111
+
112
+ def background
113
+ @background ||= Hash.new
114
+ end
115
+
116
+ def background= value
117
+ @background ||= value
118
+ end
119
+
120
+ def command
121
+ "#{ Bj.ruby } " + %W[
122
+ #{ Bj.script }
123
+ run
124
+ --forever
125
+ --redirect=#{ log }
126
+ --ppid=#{ Process.pid }
127
+ --rails_env=#{ Bj.rails_env }
128
+ --rails_root=#{ Bj.rails_root }
129
+ ].map{|word| word.inspect}.join(" ")
130
+ end
131
+
132
+ def log
133
+ File.join logdir, "bj.#{ Bj.hostname }.#{ Bj.rails_env }.log"
134
+ end
135
+
136
+ def logdir
137
+ File.join File.expand_path(Bj.rails_root), 'log'
138
+ end
139
+
140
+ def run options = {}, &block
141
+ new(options, &block).run
142
+ end
143
+ end
144
+ send :extend, ClassMethods
145
+
146
+ module Instance_Methods
147
+ attribute "options"
148
+ attribute "block"
149
+
150
+ def initialize options = {}, &block
151
+ options.to_options!
152
+ @options, @block = options, block
153
+ end
154
+
155
+ def run
156
+ wait = options[:wait] || 42
157
+ limit = options[:limit]
158
+ forever = options[:forever]
159
+
160
+ limit = false if forever
161
+ wait = Integer wait
162
+ loopno = 0
163
+
164
+ Runner.thread = Thread.current
165
+ Bj.chroot
166
+
167
+ register or exit!(EXIT::WARNING)
168
+
169
+ Bj.logger.info{ "STARTED" }
170
+ at_exit{ Bj.logger.info{ "STOPPED" } }
171
+
172
+ fill_morgue
173
+ install_signal_handlers
174
+
175
+ loop do
176
+ ping_parent
177
+
178
+ loopno += 1
179
+ break if(limit and loopno > limit)
180
+
181
+ archive_jobs
182
+
183
+ catch :no_jobs do
184
+ loop do
185
+ job = thread = stdout = stderr = nil
186
+
187
+ Bj.transaction(options) do
188
+ now = Time.now
189
+
190
+ job = Bj::Table::Job.find :first,
191
+ :conditions => ["state = ? and submitted_at <= ?", "pending", now],
192
+ :order => "priority DESC, submitted_at ASC",
193
+ :limit => 1,
194
+ :lock => true
195
+ throw :no_jobs unless job
196
+
197
+
198
+ Bj.logger.info{ "#{ job.title } - started" }
199
+
200
+ command = job.command
201
+ env = job.env ? YAML.load(job.env) : {}
202
+ stdin = job.stdin || ''
203
+ stdout = job.stdout || ''
204
+ stderr = job.stderr || ''
205
+ started_at = Time.now
206
+
207
+ thread = Util.start command, :cwd=>Bj.rails_root, :env=>env, :stdin=>stdin, :stdout=>stdout, :stderr=>stderr
208
+
209
+ job.state = "running"
210
+ job.runner = Bj.hostname
211
+ job.pid = thread.pid
212
+ job.started_at = started_at
213
+ job.save!
214
+ job.reload
215
+ end
216
+
217
+ exit_status = thread.value
218
+ finished_at = Time.now
219
+
220
+ Bj.transaction(options) do
221
+ job = Bj::Table::Job.find job.id
222
+ break unless job
223
+ job.state = "finished"
224
+ job.finished_at = finished_at
225
+ job.stdout = stdout
226
+ job.stderr = stderr
227
+ job.exit_status = exit_status
228
+ job.save!
229
+ job.reload
230
+ Bj.logger.info{ "#{ job.title } - exit_status=#{ job.exit_status }" }
231
+ end
232
+ end
233
+ end
234
+
235
+ Runner.hup_signaled false
236
+ wait.times do
237
+ break if Runner.hup_signaled?
238
+ break if Runner.kill_signaled?
239
+ sleep 1
240
+ end
241
+
242
+ break unless(limit or limit == false)
243
+ break if Runner.kill_signaled?
244
+ end
245
+ end
246
+
247
+ def ping_parent
248
+ ppid = options[:ppid]
249
+ return unless ppid
250
+ begin
251
+ Process.kill 0, Integer(ppid)
252
+ rescue Errno::ESRCH
253
+ Kernel.exit 42
254
+ rescue Exception
255
+ 42
256
+ end
257
+ end
258
+
259
+ def install_signal_handlers
260
+ Runner.hup_signaled false
261
+ hup_handler = nil
262
+ hup_handler =
263
+ trap Runner.hup_signal do |*a|
264
+ begin
265
+ Runner.hup_signaled true
266
+ rescue Exception => e
267
+ Bj.logger.error{ e } rescue nil
268
+ end
269
+ hup_handler.call *a rescue nil
270
+ end
271
+
272
+ Runner.kill_signaled false
273
+ kill_handler = nil
274
+ kill_handler =
275
+ trap Runner.kill_signal do |*a|
276
+ begin
277
+ Runner.kill_signaled true
278
+ rescue Exception => e
279
+ Bj.logger.error{ e } rescue nil
280
+ end
281
+ kill_handler.call *a rescue nil
282
+ end
283
+
284
+ begin
285
+ trap("INT"){ exit }
286
+ rescue Exception
287
+ end
288
+ end
289
+
290
+ def fill_morgue
291
+ Bj.transaction do
292
+ now = Time.now
293
+ jobs = Bj::Table::Job.find :all,
294
+ :conditions => ["state = 'running' and runner = ?", Bj.hostname]
295
+ jobs.each do |job|
296
+ if job.is_restartable?
297
+ Bj.logger.info{ "#{ job.title } - found dead and bloated but resubmitted" }
298
+ %w[ runner pid started_at finished_at stdout stderr exit_status ].each do |column|
299
+ job[column] = nil
300
+ end
301
+ job.state = 'pending'
302
+ else
303
+ Bj.logger.info{ "#{ job.title } - found dead and bloated" }
304
+ job.state = 'dead'
305
+ job.finished_at = now
306
+ end
307
+ job.save!
308
+ end
309
+ end
310
+ end
311
+
312
+ def archive_jobs
313
+ Bj.transaction do
314
+ now = Time.now
315
+ too_old = now - Bj.ttl
316
+ jobs = Bj::Table::Job.find :all,
317
+ :conditions => ["(state = 'finished' or state = 'dead') and submitted_at < ?", too_old]
318
+ jobs.each do |job|
319
+ Bj.logger.info{ "#{ job.title } - archived" }
320
+ hash = job.to_hash.update(:archived_at => now)
321
+ Bj::Table::JobArchive.create! hash
322
+ job.destroy
323
+ end
324
+ end
325
+ end
326
+
327
+ def register
328
+ Bj.transaction do
329
+ pid = Bj.config[key]
330
+ return false if Util.alive?(pid)
331
+ Bj.config[key] = Process.pid
332
+ unless Bj.util.ipc_signals_supported? # not winblows
333
+ require "drb"
334
+ DRb.start_service "druby://localhost:0", Process
335
+ Bj.config["#{ Process.pid }.uri"] = DRb.uri
336
+ end
337
+ end
338
+ at_exit{ unregister }
339
+ true
340
+ rescue Exception
341
+ false
342
+ end
343
+
344
+ def unregister
345
+ Bj.transaction do
346
+ Bj.config.delete key
347
+ end
348
+ true
349
+ rescue Exception
350
+ false
351
+ end
352
+
353
+ def key
354
+ @key ||= ( options[:ppid] ? Runner.key(options[:ppid]) : Runner.key )
355
+ end
356
+ end
357
+ send :include, Instance_Methods
358
+ end
359
+ end