webapp_worker 0.0.1 → 0.0.2
Sign up to get free protection for your applications and to get access to all the features.
- data/README.markdown +64 -2
- data/bin/waw +1 -1
- data/lib/webapp_worker/application.rb +231 -15
- data/lib/webapp_worker/version.rb +1 -1
- data/webapp_worker.gemspec +3 -2
- metadata +18 -7
data/README.markdown
CHANGED
@@ -12,11 +12,73 @@ or use it in your Gemfile
|
|
12
12
|
|
13
13
|
## Quick Test
|
14
14
|
|
15
|
-
|
15
|
+
waw -e development -f jobs.yml (parses data)
|
16
|
+
waw -e development -f jobs.yml -j (parses and shows jobs)
|
17
|
+
waw -e development -f jobs.yml -n 1 (parses and shows next X number of jobs that will run)
|
18
|
+
waw -e development -f jobs.yml -r (parses and starts to run)
|
16
19
|
|
17
20
|
## Using in your webapp
|
18
21
|
|
19
|
-
|
22
|
+
Create a jobs yaml file.
|
23
|
+
|
24
|
+
---
|
25
|
+
development:
|
26
|
+
mailto:
|
27
|
+
server_host_name:
|
28
|
+
- :command: "rake job:run"
|
29
|
+
:minute: 0-59/5
|
30
|
+
|
31
|
+
Load this file somewhere in your web app, like an initializer file.
|
32
|
+
|
33
|
+
jobs = "config/jobs.yml"
|
34
|
+
|
35
|
+
a = WebappWorker::Application.new(environment:"development",mailto:"")
|
36
|
+
a.parse_yaml(jobs)
|
37
|
+
a.run
|
38
|
+
|
39
|
+
You don't have to use a jobs file, you can specify the yaml or hash in the code above like this:
|
40
|
+
|
41
|
+
job = {:command=>"rake job:run", :minute=>"0-59/5", :hour=>"0-4/2", :day=>1, :month=>"0-12/1"}
|
42
|
+
|
43
|
+
a = WebappWorker::Application.new(environment:"development",mailto:"",jobs:[job])
|
44
|
+
a.run
|
45
|
+
|
46
|
+
## Example Output of waw
|
47
|
+
|
48
|
+
$ waw -e local -f config/jobs.yml -j
|
49
|
+
Job File: config/jobs.yml
|
50
|
+
|
51
|
+
Host: localhost
|
52
|
+
Mailto:
|
53
|
+
Environment: development
|
54
|
+
Amount of Jobs: 9
|
55
|
+
|
56
|
+
Command to Run: rake job:run
|
57
|
+
Next Run: [2012-01-03 22:00:00 -0700]
|
58
|
+
Command to Run: rake job:run
|
59
|
+
Next Run: [2012-01-03 22:02:00 -0700]
|
60
|
+
Command to Run: rake job:run
|
61
|
+
Next Run: [2012-01-03 21:12:00 -0700]
|
62
|
+
Command to Run: rake job:run
|
63
|
+
Next Run: [2012-01-03 21:14:00 -0700]
|
64
|
+
Command to Run: rake job:run
|
65
|
+
Next Run: [2012-01-03 21:16:00 -0700]
|
66
|
+
Command to Run: rake job:run
|
67
|
+
Next Run: [2012-01-03 21:18:00 -0700]
|
68
|
+
Command to Run: rake job:run
|
69
|
+
Next Run: [2012-01-03 21:22:00 -0700]
|
70
|
+
Command to Run: rake job:run
|
71
|
+
Next Run: [2012-01-03 21:24:00 -0700]
|
72
|
+
Command to Run: rake job:run
|
73
|
+
Next Run: [2012-01-03 21:30:00 -0700]
|
74
|
+
|
75
|
+
## Roadmap
|
76
|
+
|
77
|
+
- Process also needs to understand when to die and to start back up. (new version being used in the web app server)
|
78
|
+
- Start having the webapp worker registering to a central point or do UDP mutlicasting to find each other.
|
79
|
+
- Once self registering is enabled, webapp_workers need to communicate effectively.
|
80
|
+
- Once communication is esatablished webapp_workers need to do the scheduling for themselves.
|
81
|
+
- Spit out reports of the different jobs and how fast they run.
|
20
82
|
|
21
83
|
## Contributing
|
22
84
|
|
data/bin/waw
CHANGED
@@ -1,11 +1,27 @@
|
|
1
1
|
require 'socket'
|
2
|
+
require 'timeout'
|
3
|
+
require 'open4'
|
4
|
+
require 'logger'
|
5
|
+
|
6
|
+
module Process
|
7
|
+
class << self
|
8
|
+
def alive?(pid)
|
9
|
+
begin
|
10
|
+
Process.kill(0, pid.to_i)
|
11
|
+
true
|
12
|
+
rescue Errno::ESRCH
|
13
|
+
false
|
14
|
+
end
|
15
|
+
end
|
16
|
+
end
|
17
|
+
end
|
2
18
|
|
3
19
|
module WebappWorker
|
4
20
|
class Application
|
5
|
-
attr_accessor :hostname, :mailto, :environment, :jobs
|
21
|
+
attr_accessor :hostname, :mailto, :environment, :jobs, :file, :file_mtime
|
6
22
|
|
7
23
|
def initialize(user_supplied_hash={})
|
8
|
-
standard_hash = { hostname:"#{self.hostname}", mailto:"", environment:"local", jobs:"" }
|
24
|
+
standard_hash = { hostname:"#{self.hostname}", mailto:"", environment:"local", jobs:"", file:"" }
|
9
25
|
|
10
26
|
user_supplied_hash = {} unless user_supplied_hash
|
11
27
|
user_supplied_hash = standard_hash.merge(user_supplied_hash)
|
@@ -18,15 +34,34 @@ module WebappWorker
|
|
18
34
|
end
|
19
35
|
|
20
36
|
def parse_yaml(yaml)
|
21
|
-
@
|
22
|
-
@
|
37
|
+
@file = yaml
|
38
|
+
@file_mtime = File.mtime(@file)
|
39
|
+
@mailto = (YAML.load_file(@file))[@environment]["mailto"] unless @mailto
|
40
|
+
@jobs = (YAML.load_file(@file))[@environment][@hostname] unless @hostname.nil?
|
23
41
|
end
|
24
42
|
|
25
43
|
def hostname
|
26
44
|
return Socket.gethostname.downcase
|
27
45
|
end
|
28
46
|
|
29
|
-
def
|
47
|
+
def next_command_run_time?
|
48
|
+
commands = {}
|
49
|
+
c = {}
|
50
|
+
|
51
|
+
@jobs.each do |job|
|
52
|
+
j = WebappWorker::Job.new(job)
|
53
|
+
commands.store(j.command,j.next_run?)
|
54
|
+
end
|
55
|
+
(commands.sort_by { |key,value| value }).collect { |key,value| c.store(key,value) }
|
56
|
+
|
57
|
+
c.each do |key,value|
|
58
|
+
return value[0]
|
59
|
+
end
|
60
|
+
end
|
61
|
+
|
62
|
+
def commands_to_run
|
63
|
+
self.check_file_modification_time
|
64
|
+
|
30
65
|
commands = {}
|
31
66
|
c = {}
|
32
67
|
next_commands = {}
|
@@ -37,31 +72,180 @@ module WebappWorker
|
|
37
72
|
end
|
38
73
|
(commands.sort_by { |key,value| value }).collect { |key,value| c.store(key,value) }
|
39
74
|
|
40
|
-
counter = 0
|
41
75
|
c.each do |key,value|
|
42
76
|
next_commands.store(key,value)
|
43
|
-
counter = counter + 1
|
44
|
-
break if counter >= til
|
45
77
|
end
|
46
78
|
|
47
79
|
return next_commands
|
48
80
|
end
|
49
81
|
|
50
|
-
def
|
51
|
-
|
82
|
+
def check_file_modification_time
|
83
|
+
mtime = File.mtime(@file)
|
84
|
+
|
85
|
+
if mtime != @file_mtime
|
86
|
+
@file_mtime = mtime
|
87
|
+
self.parse_yaml(@file)
|
88
|
+
end
|
89
|
+
end
|
90
|
+
|
91
|
+
def check_for_directory
|
92
|
+
dir = "/tmp/webapp_worker"
|
93
|
+
|
94
|
+
if Dir.exists?(dir)
|
95
|
+
else
|
96
|
+
Dir.mkdir(dir, 0700)
|
97
|
+
end
|
98
|
+
end
|
99
|
+
|
100
|
+
def create_pid(logger)
|
101
|
+
logger.info "Creating Pid File at /tmp/webapp_worker/waw.pid"
|
102
|
+
|
103
|
+
File.open("/tmp/webapp_worker/waw.pid", 'w') { |f| f.write(Process.pid) }
|
104
|
+
$0="Web App Worker - Job File: #{@file}"
|
105
|
+
|
106
|
+
logger.info "Pid File created: #{Process.pid} at /tmp/webapp_worker/waw.pid"
|
107
|
+
end
|
108
|
+
|
109
|
+
def check_for_process(logger)
|
110
|
+
file = "/tmp/webapp_worker/waw.pid"
|
111
|
+
|
112
|
+
if File.exists?(file)
|
113
|
+
possible_pid = ""
|
114
|
+
pid_file = File.open(file, 'r').each { |f| possible_pid+= f }
|
115
|
+
pid_file.close
|
116
|
+
|
117
|
+
if Process.alive?(possible_pid)
|
118
|
+
puts "Already found webapp_worker running, pid is: #{possible_pid}, exiting..."
|
119
|
+
logger.fatal "Found webapp_worker already running with pid: #{possible_pid}, Pid File: #{file} exiting..."
|
120
|
+
exit 1
|
121
|
+
else
|
122
|
+
logger.warn "Found pid file, but no process running, recreating pid file with my pid: #{Process.pid}"
|
123
|
+
File.delete(file)
|
124
|
+
self.create_pid(logger)
|
125
|
+
end
|
126
|
+
else
|
127
|
+
self.create_pid(logger)
|
128
|
+
end
|
129
|
+
|
130
|
+
logger.info "Starting Webapp Worker"
|
131
|
+
end
|
132
|
+
|
133
|
+
def graceful_termination(logger)
|
134
|
+
stop_loop = true
|
135
|
+
|
136
|
+
begin
|
137
|
+
puts
|
138
|
+
puts "Graceful Termination started, waiting 60 seconds before KILL signal send"
|
139
|
+
logger.info "Graceful Termination started, waiting 60 seconds before KILL signal send"
|
140
|
+
|
141
|
+
Timeout::timeout(60) do
|
142
|
+
@command_processes.each do |pid,command|
|
143
|
+
logger.debug "Sending INT Signal to #{command} Process with PID: #{pid}"
|
144
|
+
begin
|
145
|
+
Process.kill("INT",pid.to_i)
|
146
|
+
rescue => error
|
147
|
+
end
|
148
|
+
end
|
149
|
+
|
150
|
+
@threads.each do |thread,command|
|
151
|
+
thread.join
|
152
|
+
end
|
153
|
+
end
|
154
|
+
rescue Timeout::Error
|
155
|
+
puts "Graceful Termination bypassed, killing processes and threads"
|
156
|
+
logger.info "Graceful Termination bypassed, killing processes and threads"
|
157
|
+
|
158
|
+
@command_processes.each do |pid,command|
|
159
|
+
logger.debug "Killing #{command} Process with PID: #{pid}"
|
160
|
+
begin
|
161
|
+
Process.kill("KILL",pid.to_i)
|
162
|
+
rescue => error
|
163
|
+
end
|
164
|
+
end
|
165
|
+
|
166
|
+
@threads.each do |thread,command|
|
167
|
+
logger.debug "Killing Command Thread: #{command}"
|
168
|
+
Thread.kill(thread)
|
169
|
+
end
|
170
|
+
end
|
171
|
+
|
172
|
+
puts "Stopping Webapp Worker"
|
173
|
+
logger.info "Stopping Webapp Worker"
|
174
|
+
file = "/tmp/webapp_worker/waw.pid"
|
175
|
+
File.delete(file)
|
176
|
+
exit 0
|
177
|
+
end
|
178
|
+
|
179
|
+
def run(debug=nil,verbose=nil)
|
180
|
+
self.check_for_directory
|
181
|
+
|
182
|
+
logger = Logger.new("/tmp/webapp_worker/#{@environment}.log", 5, 5242880)
|
183
|
+
|
184
|
+
if debug
|
185
|
+
logger.level = Logger::DEBUG
|
186
|
+
elsif verbose
|
187
|
+
logger.level = Logger::INFO
|
188
|
+
else
|
189
|
+
logger.level = Logger::WARN
|
190
|
+
end
|
191
|
+
|
52
192
|
p = Process.fork do
|
53
|
-
|
193
|
+
begin
|
194
|
+
self.check_for_process(logger)
|
195
|
+
rescue => error
|
196
|
+
puts error.inspect
|
197
|
+
logger.fatal error.inspect
|
198
|
+
end
|
54
199
|
|
200
|
+
@command_processes = {}
|
55
201
|
@threads = {}
|
202
|
+
stop_loop = false
|
203
|
+
|
204
|
+
Signal.trap('HUP', 'IGNORE')
|
205
|
+
|
206
|
+
%w(INT QUIT TERM TSTP).each do |sig|
|
207
|
+
Signal.trap(sig) do
|
208
|
+
logger.warn "Received a #{sig} signal, stopping current commands."
|
209
|
+
self.graceful_termination(logger)
|
210
|
+
end
|
211
|
+
end
|
212
|
+
|
213
|
+
Signal.trap('USR1') do
|
214
|
+
version = WebappWorker::VERSION
|
215
|
+
puts
|
216
|
+
puts "Webapp Worker Version: #{version}"
|
217
|
+
logger.info "Received USR1 signal, sent version: #{version}"
|
218
|
+
end
|
219
|
+
|
220
|
+
Signal.trap('USR2') do
|
221
|
+
logger.level = Logger::DEBUG
|
222
|
+
puts
|
223
|
+
puts "Changed logger level to Debug"
|
224
|
+
logger.info "Changed logger level to Debug"
|
225
|
+
end
|
56
226
|
|
57
|
-
|
227
|
+
#Signal.trap('STOP') do |s|
|
228
|
+
# #Stop Looping until
|
229
|
+
# stop_loop = true
|
230
|
+
# logger.warn "Received signal #{s}, pausing current loop."
|
231
|
+
#end
|
232
|
+
#
|
233
|
+
#Signal.trap('CONT') do
|
234
|
+
# #Start Looping again (catch throw?)
|
235
|
+
# stop_loop = false
|
236
|
+
# logger.warn "Received signal #{s}, starting current loop."
|
237
|
+
#end
|
238
|
+
|
239
|
+
logger.debug "Going into Loop"
|
240
|
+
until stop_loop
|
58
241
|
@threads.each do |thread,command|
|
59
242
|
if thread.status == false
|
243
|
+
logger.debug "Deleting Old Thread from Array of Jobs"
|
60
244
|
@threads.delete(thread)
|
61
245
|
end
|
62
246
|
end
|
63
247
|
|
64
|
-
data = self.
|
248
|
+
data = self.commands_to_run
|
65
249
|
|
66
250
|
data.each do |command,time|
|
67
251
|
time = time[0]
|
@@ -69,15 +253,47 @@ module WebappWorker
|
|
69
253
|
range = (time - now).to_i
|
70
254
|
|
71
255
|
if @threads.detect { |thr,com| com == command }
|
72
|
-
|
256
|
+
data.delete(command)
|
73
257
|
else
|
74
258
|
t = Thread.new do
|
259
|
+
logger.debug "Creating New Thread for command: #{command} - may need to sleep for: #{range} seconds"
|
75
260
|
sleep(range) unless range <= 0
|
76
|
-
|
261
|
+
logger.debug "Running Command: #{command}"
|
262
|
+
|
263
|
+
pid, stdin, stdout, stderr = Open4::popen4 command
|
264
|
+
@command_processes.store(pid,command)
|
265
|
+
|
266
|
+
ignored, status = Process::waitpid2 pid
|
267
|
+
|
268
|
+
if status.to_i == 0
|
269
|
+
logger.debug "Completed Command: #{command}"
|
270
|
+
else
|
271
|
+
logger.fatal "Command: #{command} Failure! Exited with Status: #{status.to_i}, Standard Out and Error Below"
|
272
|
+
logger.fatal "STDOUT BELOW:"
|
273
|
+
stdout.each_line do |line|
|
274
|
+
logger.fatal line
|
275
|
+
end
|
276
|
+
logger.fatal "STDERR BELOW:"
|
277
|
+
stderr.each_line do |line|
|
278
|
+
logger.fatal line
|
279
|
+
end
|
280
|
+
logger.fatal "Command: #{command} Unable to Complete! Standard Out and Error Above"
|
281
|
+
end
|
77
282
|
end
|
78
283
|
@threads.store(t,command)
|
79
284
|
end
|
80
285
|
end
|
286
|
+
|
287
|
+
logger.debug Thread.list
|
288
|
+
logger.debug @threads.inspect
|
289
|
+
logger.debug @command_processes.inspect
|
290
|
+
|
291
|
+
time = self.next_command_run_time?
|
292
|
+
now = Time.now.utc
|
293
|
+
range = (time - now).to_i
|
294
|
+
range = range - 1
|
295
|
+
logger.debug "Sleeping for #{range} seconds after looping through all jobs found"
|
296
|
+
sleep(range) unless range <= 0
|
81
297
|
end
|
82
298
|
end
|
83
299
|
|
data/webapp_worker.gemspec
CHANGED
@@ -7,7 +7,7 @@ Gem::Specification.new do |s|
|
|
7
7
|
s.version = WebappWorker::VERSION
|
8
8
|
s.authors = ["Nick Willever"]
|
9
9
|
s.email = ["nickwillever@gmail.com"]
|
10
|
-
s.homepage = ""
|
10
|
+
s.homepage = "https://nictrix.github.com/webapp_worker"
|
11
11
|
s.summary = %q{Provides a worker for your webapp}
|
12
12
|
s.description = %q{Allow the webapp to handle your workers, no need to use a job scheduler}
|
13
13
|
|
@@ -16,5 +16,6 @@ Gem::Specification.new do |s|
|
|
16
16
|
s.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
|
17
17
|
s.require_paths = ["lib"]
|
18
18
|
|
19
|
-
s.add_runtime_dependency 'trollop'
|
19
|
+
s.add_runtime_dependency 'trollop', '=1.16.2'
|
20
|
+
s.add_runtime_dependency 'open4', '=1.3.0'
|
20
21
|
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: webapp_worker
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.2
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -9,19 +9,30 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date:
|
12
|
+
date: 2012-01-16 00:00:00.000000000Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: trollop
|
16
|
-
requirement: &
|
16
|
+
requirement: &83276430 !ruby/object:Gem::Requirement
|
17
17
|
none: false
|
18
18
|
requirements:
|
19
|
-
- -
|
19
|
+
- - =
|
20
20
|
- !ruby/object:Gem::Version
|
21
|
-
version:
|
21
|
+
version: 1.16.2
|
22
22
|
type: :runtime
|
23
23
|
prerelease: false
|
24
|
-
version_requirements: *
|
24
|
+
version_requirements: *83276430
|
25
|
+
- !ruby/object:Gem::Dependency
|
26
|
+
name: open4
|
27
|
+
requirement: &83272260 !ruby/object:Gem::Requirement
|
28
|
+
none: false
|
29
|
+
requirements:
|
30
|
+
- - =
|
31
|
+
- !ruby/object:Gem::Version
|
32
|
+
version: 1.3.0
|
33
|
+
type: :runtime
|
34
|
+
prerelease: false
|
35
|
+
version_requirements: *83272260
|
25
36
|
description: Allow the webapp to handle your workers, no need to use a job scheduler
|
26
37
|
email:
|
27
38
|
- nickwillever@gmail.com
|
@@ -40,7 +51,7 @@ files:
|
|
40
51
|
- lib/webapp_worker/job.rb
|
41
52
|
- lib/webapp_worker/version.rb
|
42
53
|
- webapp_worker.gemspec
|
43
|
-
homepage:
|
54
|
+
homepage: https://nictrix.github.com/webapp_worker
|
44
55
|
licenses: []
|
45
56
|
post_install_message:
|
46
57
|
rdoc_options: []
|