webapp_worker 0.0.1 → 0.0.2

data/README.markdown

@@ -12,11 +12,73 @@ or use it in your Gemfile
 
 ## Quick Test
 
-``
+	waw -e development -f jobs.yml (parses data)
+	waw -e development -f jobs.yml  -j (parses and shows jobs)
+	waw -e development -f jobs.yml  -n 1 (parses and shows next X number of jobs that will run)
+	waw -e development -f jobs.yml -r (parses and starts to run)
 
 ## Using in your webapp
 
-
+Create a jobs yaml file.
+
+	---
+	development:
+		mailto:
+		server_host_name:
+			- :command: "rake job:run"
+				:minute: 0-59/5
+
+Load this file somewhere in your web app, like an initializer file.
+
+	jobs = "config/jobs.yml"
+
+	a = WebappWorker::Application.new(environment:"development",mailto:"")
+	a.parse_yaml(jobs)
+	a.run
+
+You don't have to use a jobs file, you can specify the yaml or hash in the code above like this:
+
+	job = {:command=>"rake job:run", :minute=>"0-59/5", :hour=>"0-4/2", :day=>1, :month=>"0-12/1"}
+
+	a = WebappWorker::Application.new(environment:"development",mailto:"",jobs:[job])
+	a.run
+
+## Example Output of waw
+
+	$ waw -e local -f config/jobs.yml -j
+	Job File: config/jobs.yml
+
+	Host: localhost
+	Mailto:
+	Environment: development
+	Amount of Jobs: 9
+
+	Command to Run: rake job:run
+	   Next Run: [2012-01-03 22:00:00 -0700]
+	Command to Run: rake job:run
+	   Next Run: [2012-01-03 22:02:00 -0700]
+	Command to Run: rake job:run
+	   Next Run: [2012-01-03 21:12:00 -0700]
+	Command to Run: rake job:run
+	   Next Run: [2012-01-03 21:14:00 -0700]
+	Command to Run: rake job:run
+	   Next Run: [2012-01-03 21:16:00 -0700]
+	Command to Run: rake job:run
+	   Next Run: [2012-01-03 21:18:00 -0700]
+	Command to Run: rake job:run
+	   Next Run: [2012-01-03 21:22:00 -0700]
+	Command to Run: rake job:run
+	   Next Run: [2012-01-03 21:24:00 -0700]
+	Command to Run: rake job:run
+	   Next Run: [2012-01-03 21:30:00 -0700]
+
+## Roadmap
+
+- Process also needs to understand when to die and to start back up. (new version being used in the web app server)
+- Start having the webapp worker registering to a central point or do UDP mutlicasting to find each other.
+- Once self registering is enabled, webapp_workers need to communicate effectively.
+- Once communication is esatablished webapp_workers need to do the scheduling for themselves.
+- Spit out reports of the different jobs and how fast they run.
 
 ## Contributing
 

data/bin/waw

@@ -54,5 +54,5 @@ elsif opts[:run] == false && opts[:jobs] == true
 	end
 else
 	puts "Running Jobs"
-	a.run
+	a.run(opts[:debug],opts[:verbose])
 end

data/lib/webapp_worker/application.rb

@@ -1,11 +1,27 @@
 require 'socket'
+require 'timeout'
+require 'open4'
+require 'logger'
+
+module Process
+  class << self
+    def alive?(pid)
+      begin
+        Process.kill(0, pid.to_i)
+        true
+      rescue Errno::ESRCH
+        false
+      end
+    end
+  end
+end
 
 module WebappWorker
 	class Application
-		attr_accessor :hostname, :mailto, :environment, :jobs
+		attr_accessor :hostname, :mailto, :environment, :jobs, :file, :file_mtime
 
 		def initialize(user_supplied_hash={})
-			standard_hash = { hostname:"#{self.hostname}", mailto:"", environment:"local", jobs:"" }
+			standard_hash = { hostname:"#{self.hostname}", mailto:"", environment:"local", jobs:"", file:"" }
 
 			user_supplied_hash = {} unless user_supplied_hash
 			user_supplied_hash = standard_hash.merge(user_supplied_hash)
@@ -18,15 +34,34 @@ module WebappWorker
 		end
 
 		def parse_yaml(yaml)
-			@mailto = (YAML.load_file(yaml))[@environment]["mailto"] unless @mailto
-			@jobs = (YAML.load_file(yaml))[@environment][@hostname] unless @hostname.nil?
+			@file = yaml
+			@file_mtime = File.mtime(@file)
+			@mailto = (YAML.load_file(@file))[@environment]["mailto"] unless @mailto
+			@jobs = (YAML.load_file(@file))[@environment][@hostname] unless @hostname.nil?
 		end
 
 		def hostname
 			return Socket.gethostname.downcase
 		end
 
-		def next_command_run?(til)
+		def next_command_run_time?
+			commands = {}
+			c = {}
+
+			@jobs.each do |job|
+				j = WebappWorker::Job.new(job)
+				commands.store(j.command,j.next_run?)
+			end
+			(commands.sort_by { |key,value| value }).collect { |key,value| c.store(key,value) }
+
+			c.each do |key,value|
+				return value[0]
+			end
+		end
+
+		def commands_to_run
+			self.check_file_modification_time
+
 			commands = {}
 			c = {}
 			next_commands = {}
@@ -37,31 +72,180 @@ module WebappWorker
 			end
 			(commands.sort_by { |key,value| value }).collect { |key,value| c.store(key,value) }
 
-			counter = 0
 			c.each do |key,value|
 				next_commands.store(key,value)
-				counter = counter + 1
-				break if counter >= til
 			end
 
 			return next_commands
 		end
 
-		def run
-			#Going to need to do memory/process management, or fork processes not threads...
+		def check_file_modification_time
+			mtime = File.mtime(@file)
+
+			if mtime != @file_mtime
+				@file_mtime = mtime
+				self.parse_yaml(@file)
+			end
+		end
+
+		def check_for_directory
+			dir = "/tmp/webapp_worker"
+
+			if Dir.exists?(dir)
+			else
+				Dir.mkdir(dir, 0700)
+			end
+		end
+
+		def create_pid(logger)
+			logger.info "Creating Pid File at /tmp/webapp_worker/waw.pid"
+
+			File.open("/tmp/webapp_worker/waw.pid", 'w') { |f| f.write(Process.pid) }
+			$0="Web App Worker - Job File: #{@file}"
+
+			logger.info "Pid File created: #{Process.pid} at /tmp/webapp_worker/waw.pid"
+		end
+
+		def check_for_process(logger)
+			file = "/tmp/webapp_worker/waw.pid"
+
+			if File.exists?(file)
+				possible_pid = ""
+				pid_file = File.open(file, 'r').each { |f| possible_pid+= f }
+				pid_file.close
+
+				if Process.alive?(possible_pid)
+					puts "Already found webapp_worker running, pid is: #{possible_pid}, exiting..."
+					logger.fatal "Found webapp_worker already running with pid: #{possible_pid}, Pid File: #{file} exiting..."
+					exit 1
+				else
+					logger.warn "Found pid file, but no process running, recreating pid file with my pid: #{Process.pid}"
+					File.delete(file)
+					self.create_pid(logger)
+				end
+			else
+				self.create_pid(logger)
+			end
+
+			logger.info "Starting Webapp Worker"
+		end
+
+		def graceful_termination(logger)
+			stop_loop = true
+
+			begin
+				puts
+				puts "Graceful Termination started, waiting 60 seconds before KILL signal send"
+				logger.info "Graceful Termination started, waiting 60 seconds before KILL signal send"
+
+				Timeout::timeout(60) do
+					@command_processes.each do |pid,command|
+						logger.debug "Sending INT Signal to #{command} Process with PID: #{pid}"
+						begin
+							Process.kill("INT",pid.to_i)
+						rescue => error
+						end
+					end
+
+					@threads.each do |thread,command|
+						thread.join
+					end
+				end
+			rescue Timeout::Error
+				puts "Graceful Termination bypassed, killing processes and threads"
+				logger.info "Graceful Termination bypassed, killing processes and threads"
+
+				@command_processes.each do |pid,command|
+					logger.debug "Killing #{command} Process with PID: #{pid}"
+					begin
+						Process.kill("KILL",pid.to_i)
+					rescue => error
+					end
+				end
+
+				@threads.each do |thread,command|
+					logger.debug "Killing Command Thread: #{command}"
+					Thread.kill(thread)
+				end
+			end
+
+			puts "Stopping Webapp Worker"
+			logger.info "Stopping Webapp Worker"
+			file = "/tmp/webapp_worker/waw.pid"
+			File.delete(file)
+			exit 0
+		end
+
+		def run(debug=nil,verbose=nil)
+			self.check_for_directory
+
+			logger = Logger.new("/tmp/webapp_worker/#{@environment}.log", 5, 5242880)
+
+			if debug
+				logger.level = Logger::DEBUG
+			elsif verbose
+				logger.level = Logger::INFO
+			else
+				logger.level = Logger::WARN
+			end
+
 			p = Process.fork do
-				Signal.trap('HUP', 'IGNORE')
+				begin
+					self.check_for_process(logger)
+				rescue => error
+					puts error.inspect
+					logger.fatal error.inspect
+				end
 
+				@command_processes = {}
 				@threads = {}
+				stop_loop = false
+
+				Signal.trap('HUP', 'IGNORE')
+
+				%w(INT QUIT TERM TSTP).each do |sig|
+					Signal.trap(sig) do
+						logger.warn "Received a #{sig} signal, stopping current commands."
+						self.graceful_termination(logger)
+					end
+				end
+
+				Signal.trap('USR1') do
+					version = WebappWorker::VERSION
+					puts
+					puts "Webapp Worker Version: #{version}"
+					logger.info "Received USR1 signal, sent version: #{version}"
+				end
+
+				Signal.trap('USR2') do
+					logger.level = Logger::DEBUG
+					puts
+					puts "Changed logger level to Debug"
+					logger.info "Changed logger level to Debug"
+				end
 
-				loop do
+				#Signal.trap('STOP') do |s|
+				#	#Stop Looping until
+				#	stop_loop = true
+				#	logger.warn "Received signal #{s}, pausing current loop."
+				#end
+				#
+				#Signal.trap('CONT') do
+				#	#Start Looping again (catch throw?)
+				#	stop_loop = false
+				#	logger.warn "Received signal #{s}, starting current loop."
+				#end
+
+				logger.debug "Going into Loop"
+				until stop_loop
 					@threads.each do |thread,command|
 						if thread.status == false
+							logger.debug "Deleting Old Thread from Array of Jobs"
 							@threads.delete(thread)
 						end
 					end
 
-					data = self.next_command_run?(1)
+					data = self.commands_to_run
 
 					data.each do |command,time|
 						time = time[0]
@@ -69,15 +253,47 @@ module WebappWorker
 						range = (time - now).to_i
 
 						if @threads.detect { |thr,com| com == command }
-							sleep(range) unless range <= 0
+							data.delete(command)
 						else
 							t = Thread.new do
+								logger.debug "Creating New Thread for command: #{command} - may need to sleep for: #{range} seconds"
 								sleep(range) unless range <= 0
-								`#{command}`
+								logger.debug "Running Command: #{command}"
+
+								pid, stdin, stdout, stderr = Open4::popen4 command
+								@command_processes.store(pid,command)
+
+								ignored, status = Process::waitpid2 pid
+
+								if status.to_i == 0
+									logger.debug "Completed Command: #{command}"
+								else
+									logger.fatal "Command: #{command} Failure! Exited with Status: #{status.to_i}, Standard Out and Error Below"
+									logger.fatal "STDOUT BELOW:"
+									stdout.each_line do |line|
+										logger.fatal line
+									end
+									logger.fatal "STDERR BELOW:"
+									stderr.each_line do |line|
+										logger.fatal line
+									end
+									logger.fatal "Command: #{command} Unable to Complete! Standard Out and Error Above"
+								end
 							end
 							@threads.store(t,command)
 						end
 					end
+
+					logger.debug Thread.list
+					logger.debug @threads.inspect
+					logger.debug @command_processes.inspect
+
+					time = self.next_command_run_time?
+					now = Time.now.utc
+					range = (time - now).to_i
+					range = range - 1
+					logger.debug "Sleeping for #{range} seconds after looping through all jobs found"
+					sleep(range) unless range <= 0
 				end
 			end
 

data/lib/webapp_worker/version.rb

@@ -1,3 +1,3 @@
 module WebappWorker
-  VERSION = "0.0.1"
+  VERSION = "0.0.2"
 end

data/webapp_worker.gemspec

@@ -7,7 +7,7 @@ Gem::Specification.new do |s|
   s.version     = WebappWorker::VERSION
   s.authors     = ["Nick Willever"]
   s.email       = ["nickwillever@gmail.com"]
-  s.homepage    = ""
+  s.homepage    = "https://nictrix.github.com/webapp_worker"
   s.summary     = %q{Provides a worker for your webapp}
   s.description = %q{Allow the webapp to handle your workers, no need to use a job scheduler}
 
@@ -16,5 +16,6 @@ Gem::Specification.new do |s|
   s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
   s.require_paths = ["lib"]
 
-  s.add_runtime_dependency 'trollop'
+  s.add_runtime_dependency 'trollop', '=1.16.2'
+  s.add_runtime_dependency 'open4', '=1.3.0'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: webapp_worker
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.0.2
   prerelease: 
 platform: ruby
 authors:
@@ -9,19 +9,30 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2011-12-15 00:00:00.000000000Z
+date: 2012-01-16 00:00:00.000000000Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: trollop
-  requirement: &81315550 !ruby/object:Gem::Requirement
+  requirement: &83276430 !ruby/object:Gem::Requirement
     none: false
     requirements:
-    - - ! '>='
+    - - =
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 1.16.2
   type: :runtime
   prerelease: false
-  version_requirements: *81315550
+  version_requirements: *83276430
+- !ruby/object:Gem::Dependency
+  name: open4
+  requirement: &83272260 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - =
+      - !ruby/object:Gem::Version
+        version: 1.3.0
+  type: :runtime
+  prerelease: false
+  version_requirements: *83272260
 description: Allow the webapp to handle your workers, no need to use a job scheduler
 email:
 - nickwillever@gmail.com
@@ -40,7 +51,7 @@ files:
 - lib/webapp_worker/job.rb
 - lib/webapp_worker/version.rb
 - webapp_worker.gemspec
-homepage: ''
+homepage: https://nictrix.github.com/webapp_worker
 licenses: []
 post_install_message: 
 rdoc_options: []