rack-app-worker 0.1.0

data/lib/rack/app/worker/cli.rb

@@ -0,0 +1,37 @@
+require 'rack/app/worker'
+module Rack::App::Worker::CLI
+
+  extend(self)
+
+  def start(options)
+    observer = Rack::App::Worker::Observer.new
+    daemonizer.daemonize if options[:daemonize]
+    daemonizer.subscribe_to_signals
+    # daemonizer.on_shutdown{ observer.stop }
+    # daemonizer.on_halt{ observer.stop }
+    observer.start
+  end
+
+  def stop(options)
+    daemonizer.send_signal('HUP')
+  end
+
+  def halt(options)
+    daemonizer.send_signal('TERM')
+  end
+
+  def reload(options)
+    daemonizer.send_signal('USR1')
+  end
+
+  protected
+
+  def method_missing(command)
+    $stderr.puts("Unknown worker command: #{command}")
+  end
+
+  def daemonizer
+    @daemonizer ||= Rack::App::Worker::Daemonizer.new('master')
+  end
+
+end

data/lib/rack/app/worker/client_proxy.rb

@@ -0,0 +1,27 @@
+class Rack::App::Worker::ClientProxy
+
+  require 'rack/app/worker/client_proxy/wrapper'
+
+  def initialize(name)
+    @name = name
+  end
+
+  def send
+    Rack::App::Worker::ClientProxy::Wrapper.new(rabbitmq.send_exchange(@name))
+  end
+
+  alias to_one send
+
+  def broadcast
+    Rack::App::Worker::ClientProxy::Wrapper.new(rabbitmq.broadcast_exchange(@name))
+  end
+
+  alias to_all broadcast
+
+  protected
+
+  def rabbitmq
+    @rabbitmq ||= Rack::App::Worker::RabbitMQ.new
+  end
+
+end

data/lib/rack/app/worker/client_proxy/wrapper.rb

@@ -0,0 +1,16 @@
+require 'yaml'
+class Rack::App::Worker::ClientProxy::Wrapper < BasicObject
+
+  def initialize(exchange)
+    @exchange = exchange
+  end
+
+  protected
+
+  def method_missing(method_name, *args)
+    headers = {'method_name' => method_name.to_s}
+    @exchange.publish(::YAML.dump(args), :headers => headers)
+    nil
+  end
+
+end

data/lib/rack/app/worker/consumer.rb

@@ -0,0 +1,76 @@
+require 'yaml'
+# Bunny::Consumer
+class Rack::App::Worker::Consumer
+
+  def initialize(definition)
+    @definition = definition
+    @instance = @definition[:class].new
+    @subscriptions = []
+    @shutdown_requested = false
+  end
+
+  def start
+    daemonizer.spawn do |d|
+      d.process_title("rack-app-worker/#{@definition[:name]}/#{d.id}")
+      start_working
+    end
+  end
+
+  def stop
+    daemonizer.send_signal('HUP', 1)
+  end
+
+  def stop_all
+    daemonizer.send_signal('HUP')
+  end
+
+  protected
+
+  def start_working
+    logger.info "consumer start working for #{@definition[:name]}"
+    rabbit = Rack::App::Worker::RabbitMQ.new
+    subscribe(rabbit.send_queue(@definition[:name]))
+    subscribe(rabbit.create_broadcast_queue(@definition[:name]))
+    wait_for_shutdown
+  end
+
+  def wait_for_shutdown
+    sleep(1) until @shutdown_requested
+  end
+
+  def handle_message(queue, delivery_info, properties, payload)
+    method_name = properties[:headers]['method_name']
+    args = YAML.load(payload)
+    @instance.public_send(method_name, *args)
+    queue.channel.ack(delivery_info.delivery_tag, false)
+  rescue Exception
+    queue.channel.nack(delivery_info.delivery_tag, false, true)
+  end
+
+  def at_shutdown
+    logger.info 'cancel subscriptions'
+    @subscriptions.each { |c| c.cancel }
+    @shutdown_requested = true
+  end
+
+  def daemonizer
+    @daemonizer ||= proc {
+      daemonizer_instance = Rack::App::Worker::Daemonizer.new(@definition[:name])
+      daemonizer_instance.on_shutdown { at_shutdown }
+      daemonizer_instance.on_halt { at_shutdown }
+      daemonizer_instance
+    }.call
+  end
+
+  def subscribe(queue)
+    logger.info "creating subscription for #{queue.name}"
+    @subscriptions << queue.subscribe(:manual_ack => true) do |delivery_info, properties, payload|
+      handle_message(queue, delivery_info, properties, payload)
+    end
+  end
+
+  def logger
+    @logger ||= Rack::App::Worker::Logger.new
+  end
+
+end

data/lib/rack/app/worker/daemonizer.rb

@@ -0,0 +1,184 @@
+require 'timeout'
+require 'securerandom'
+class Rack::App::Worker::Daemonizer
+  DEFAULT_KILL_SIGNAL = 'HUP'.freeze
+
+  def initialize(daemon_name)
+    @daemon_name = daemon_name.to_s
+    @on_shutdown, @on_halt, @on_reload = proc {}, proc {}, proc {}
+  end
+
+  def id
+    @id ||= SecureRandom.uuid
+  end
+
+  def spawn(&block)
+
+    parent_pid = current_pid
+    spawn_block = proc do
+      subscribe_to_signals
+      bind(parent_pid)
+      save_current_process_pid
+      redirect
+      block.call(self)
+    end
+
+    try_fork(&spawn_block)
+
+  end
+
+  def daemonize
+    case try_fork
+
+      when NilClass #child
+        subscribe_to_signals
+        save_current_process_pid
+        redirect
+
+      else #parent
+        Kernel.exit
+
+    end
+  end
+
+  def has_running_process?
+    pids.any? { |pid| Rack::App::Worker::Utils.process_alive?(pid) }
+  end
+
+  def process_title(new_title)
+    if Process.respond_to?(:setproctitle)
+      Process.setproctitle(new_title)
+    else
+
+      $0 = new_title
+    end
+  end
+
+  def send_signal(signal, to_amount_of_worker=pids.length)
+    pids.take(to_amount_of_worker).each do |pid|
+      kill(signal, pid)
+    end
+  end
+
+  def bind(to_pid)
+    Thread.new do
+      sleep(1) while Rack::App::Worker::Utils.process_alive?(to_pid)
+
+      at_shutdown
+    end
+  end
+
+  def on_shutdown(&block)
+    raise('block not given!') unless block.is_a?(Proc)
+    @on_shutdown = block
+  end
+
+  def on_halt(&block)
+    raise('block not given!') unless block.is_a?(Proc)
+    @on_halt = block
+  end
+
+  def on_reload(&block)
+    raise('block not given!') unless block.is_a?(Proc)
+    @on_reload = block
+  end
+
+  def subscribe_to_signals
+    ::Signal.trap('INT'){ at_shutdown }
+    ::Signal.trap('HUP'){ at_shutdown }
+    ::Signal.trap('TERM'){ at_halt }
+    ::Signal.trap('USR1'){ at_reload }
+  end
+
+  protected
+
+  # Try and read the existing pid from the pid file and signal the
+  # process. Returns true for a non blocking status.
+  def kill(signal, pid)
+    ::Process.kill(signal, pid)
+    true
+  rescue Errno::ESRCH
+    $stdout.puts "The process #{pid} did not exist: Errno::ESRCH"
+    true
+  rescue Errno::EPERM
+    $stderr.puts "Lack of privileges to manage the process #{pid}: Errno::EPERM"
+    false
+  rescue ::Exception => e
+    $stderr.puts "While signaling the PID, unexpected #{e.class}: #{e}"
+    false
+  end
+
+  def at_shutdown
+    @on_shutdown.call
+  ensure
+    at_stop
+  end
+
+  def at_halt
+    Timeout.timeout(10) { @on_halt.call } rescue nil
+  ensure
+    at_stop
+  end
+
+  def at_reload
+    @on_reload.call
+  end
+
+  def at_stop
+    File.write('/Users/aluzsi/Works/rack-app/worker/sandbox/out', pid_file_path)
+    File.delete(pid_file_path) if File.exist?(pid_file_path)
+    ::Kernel.exit
+  end
+
+  def try_fork(&block)
+    pid = nil
+    Timeout.timeout(15) { (Kernel.sleep(1) while (pid = ::Kernel.fork(&block)) == -1) }
+    return pid
+  rescue Timeout::Error
+    raise('Fork failed!')
+  end
+
+  # Attempts to write the pid of the forked process to the pid file.
+  def save_current_process_pid
+    File.write(pid_file_path, current_pid)
+  rescue ::Exception => e
+    $stderr.puts "While writing the PID to file, unexpected #{e.class}: #{e}"
+    Kernel.exit
+  end
+
+  def redirect
+    Timeout.timeout(5) { try_redirect }
+  rescue Timeout::Error
+    raise('Cannot redirect standard io channels!')
+  end
+
+  # Send stdout and stderr to log files for the child process
+  def try_redirect
+    $stdin.reopen(Rack::App::Utils.devnull_path)
+    $stdout.reopen(Rack::App::Worker::Environment.stdout)
+    $stderr.reopen(Rack::App::Worker::Environment.stderr)
+    $stdout.sync = $stderr.sync = true
+  rescue Errno::ENOENT
+    retry
+  end
+
+  def pids
+    sorted_pid_files = Dir.glob(File.join(pids_folder_path, '*')).sort_by { |fp| File.mtime(fp) }
+    sorted_pid_files.map { |file_path| File.read(file_path).to_i }
+  end
+
+  def pid_file_path
+    File.join(pids_folder_path, id)
+  end
+
+  def pids_folder_path
+    path = Rack::App::Utils.pwd('pids', 'workers', @daemon_name)
+    FileUtils.mkdir_p(path)
+    path
+  end
+
+  def current_pid
+    Process.pid rescue $$
+  end
+
+end

data/lib/rack/app/worker/dsl.rb

@@ -0,0 +1,4 @@
+module Rack::App::Worker::DSL
+  require 'rack/app/worker/dsl/for_class'
+  require 'rack/app/worker/dsl/for_endpoints'
+end

data/lib/rack/app/worker/dsl/for_class.rb

@@ -0,0 +1,12 @@
+module Rack::App::Worker::DSL::ForClass
+
+  def worker(name, &block)
+    Rack::App::Worker::Register.add(name,block) unless block.nil?
+  end
+  alias define_worker worker
+
+  def workers
+    Rack::App::Worker::Register
+  end
+
+end

data/lib/rack/app/worker/dsl/for_endpoints.rb

@@ -0,0 +1,7 @@
+module Rack::App::Worker::DSL::ForEndpoints
+
+  def workers
+    Rack::App::Worker::Register::Clients
+  end
+
+end

data/lib/rack/app/worker/environment.rb

@@ -0,0 +1,71 @@
+require 'rack/app/worker'
+module Rack::App::Worker::Environment
+  extend(self)
+
+  DEFAULT_QOS = 50
+  DEFAULT_WORKER_CLUSTER = 'main'.freeze
+  DEFAULT_WORKER_NAMESPACE = 'rack-app-worker'.freeze
+  DEFAULT_HEARTBEAT_INTERVAL = 10
+  DEFAULT_MESSAGE_COUNT_LIMIT = 50
+  DEFAULT_MAX_CONSUMER_NUMBER = Rack::App::Worker::Utils.maximum_allowed_process_number
+
+  def worker_cluster
+    (ENV['WORKER_CLUSTER'] || DEFAULT_WORKER_CLUSTER).to_s
+  end
+
+  def queue_qos
+    (ENV['WORKER_QOS'] || DEFAULT_QOS).to_i
+  end
+
+  def namespace
+    (ENV['WORKER_NAMESPACE'] || DEFAULT_WORKER_NAMESPACE).to_s
+  end
+
+  def heartbeat_interval
+    (ENV['WORKER_HEARTBEAT_INTERVAL'] || DEFAULT_HEARTBEAT_INTERVAL).to_i
+  end
+
+  def message_count_limit
+    (ENV['WORKER_MESSAGE_COUNT_LIMIT'] || DEFAULT_MESSAGE_COUNT_LIMIT).to_i
+  end
+
+  def max_consumer_number
+    (ENV['WORKER_MAX_CONSUMER_NUMBER'] || DEFAULT_MAX_CONSUMER_NUMBER).to_i
+  end
+
+  def log_level
+    case ENV['WORKER_LOG_LEVEL'].to_s.upcase
+
+      when 'DEBUG', '0'
+        0
+
+      when 'INFO', '1'
+        1
+
+      when 'WARN', '2'
+        2
+
+      when 'ERROR', '3'
+        3
+
+      when 'FATAL', '4'
+        4
+
+      when 'UNKNOWN', '5'
+        5
+
+      else
+        3
+
+    end
+  end
+
+  def stdout
+    (ENV['WORKER_STDOUT'] || Rack::App::Utils.devnull_path).to_s
+  end
+
+  def stderr
+    (ENV['WORKER_STDOUT'] || Rack::App::Utils.devnull_path).to_s
+  end
+
+end