loops 2.0.0

data/lib/loops/process_manager.rb

@@ -0,0 +1,114 @@
+module Loops
+  class ProcessManager
+    attr_reader :logger
+
+    def initialize(config, logger)
+      @config = {
+        'poll_period' => 1,
+        'workers_engine' => 'fork'
+      }.merge(config)
+
+      @logger = logger
+      @worker_pools = {}
+      @shutdown = false
+    end
+
+    def start_workers(name, number, &blk)
+      raise ArgumentError, "Need a worker block!" unless block_given?
+
+      logger.debug("Creating a workers pool of #{number} workers for #{name} loop...")
+      @worker_pools[name] = Loops::WorkerPool.new(name, self, @config['workers_engine'], &blk)
+      @worker_pools[name].start_workers(number)
+    end
+
+    def monitor_workers
+      setup_signals
+
+      logger.info('Starting workers monitoring code...')
+      loop do
+        logger.debug("Checking workers' health...")
+        @worker_pools.each do |name, pool|
+          break if shutdown?
+          pool.check_workers
+        end
+
+        break if shutdown?
+        logger.debug("Sleeping for #{@config['poll_period']} seconds...")
+        sleep(@config['poll_period'])
+      end
+    ensure
+      unless wait_for_workers(10)
+        logger.info("Some workers are still alive after 10 seconds of waiting. Killing them...")
+        stop_workers(true)
+        wait_for_workers(5)
+      end
+    end
+
+    def setup_signals
+      # Zombie reapers
+      trap('CHLD') {}
+      trap('EXIT') {}
+    end
+
+    def wait_for_workers(seconds)
+      seconds.times do
+        logger.info("Shutting down... waiting for workers to die (we have #{seconds} seconds)...")
+        running_total = 0
+
+        @worker_pools.each do |name, pool|
+          running_total += pool.wait_workers
+        end
+
+        if running_total.zero?
+          logger.info("All workers are dead. Exiting...")
+          return true
+        end
+
+        logger.info("#{running_total} workers are still running! Sleeping for a second...")
+        sleep(1)
+      end
+
+      return false
+    end
+
+    def stop_workers(force = false)
+      # Return if already shuting down (and not forced to stop)
+      return if shutdown? && !force
+
+      # Set shutdown flag
+      logger.info("Stopping workers#{force ? ' (forced)' : ''}...")
+      start_shutdown!
+
+      # Termination loop
+      @worker_pools.each do |name, pool|
+        pool.stop_workers(force)
+      end
+    end
+
+    def stop_workers!
+      # return if already shutting down
+      return if shutdown?
+
+      # Set shutdown flag
+      start_shutdown!
+
+      # Ask gently to stop
+      stop_workers(false)
+
+      # Give it a second
+      sleep(1)
+
+      # Forcefully stop the workers
+      stop_workers(true)
+    end
+
+    def shutdown?
+      @shutdown
+    end
+
+    def start_shutdown!
+      logger.info("Starting shutdown (shutdown flag set)...")
+      @shutdown = true
+    end
+  end
+end

data/lib/loops/queue.rb

@@ -0,0 +1,78 @@
+begin
+  require 'stomp'
+rescue LoadError
+  puts "Can't load stomp gem - all queue loops will be disabled!"
+end
+
+require 'timeout'
+
+module Loops
+  class Queue < Base
+    def self.check_dependencies
+      raise "No stomp gem installed!" unless defined?(Stomp::Client)
+    end
+
+    def run
+      create_client
+
+      config['queue_name'] ||= "/queue/loops/#{name}"
+      config['prefetch_size'] ||= 1
+      debug "Subscribing for the queue #{config['queue_name']}..."
+
+      headers = { :ack => :client }
+      headers["activemq.prefetchSize"] = config['prefetch_size'] if config['prefetch_size']
+
+      @total_served = 0
+      @client.subscribe(config['queue_name'], headers) do |msg|
+        begin
+          if config['action_timeout']
+            timeout(config['action_timeout']) { process_message(msg) }
+          else
+            process_message(msg)
+          end
+
+          @client.acknowledge(msg)
+          @total_served += 1
+          if config['max_requests'] && @total_served >= config['max_requests'].to_i
+            disconnect_client_and_exit
+          end
+        rescue Exception => e
+          error "Exception from process message! We won't be ACKing the message."
+          error "Details: #{e} at #{e.backtrace.first}"
+          disconnect_client_and_exit
+        end
+      end
+
+      @client.join
+    rescue Exception => e
+      error "Closing queue connection because of exception: #{e} at #{e.backtrace.first}"
+      disconnect_client_and_exit
+    end
+
+    def process_message(msg)
+      raise "This method process_message(msg) should be overriden in the loop class!"
+    end
+
+  private
+
+    def create_client
+      config['port'] ||= config['port'].to_i == 0 ? 61613 : config['port'].to_i
+      config['host'] ||= 'localhost'
+
+      @client = Stomp::Client.open(config['user'], config['password'], config['host'], config['port'], true)
+      setup_signals
+    end
+
+    def disconnect_client_and_exit
+      debug "Unsubscribing..."
+      @client.unsubscribe(name) rescue nil
+      @client.close() rescue nil
+      exit(0)
+    end
+
+    def setup_signals
+      Signal.trap('INT') { disconnect_client_and_exit }
+      Signal.trap('TERM') { disconnect_client_and_exit }
+    end
+  end
+end

data/lib/loops/version.rb

@@ -0,0 +1,31 @@
+# Contains information about currently used Loops version.
+#
+# @example
+#   puts "Loops #{Loops::Version}"
+#
+class Loops::Version
+  # @return [Hash<Symbol, Integer>]
+  #   a +Hash+ containing major, minor, and patch version parts.
+  CURRENT = YAML.load_file(File.join(Loops::LIB_ROOT, '../VERSION.yml'))
+
+  # @return [Integer]
+  #   a major part of the Loops version.
+  MAJOR = CURRENT[:major]
+  # @return [Integer]
+  #   a minor part of the Loops version.
+  MINOR = CURRENT[:minor]
+  # @return [Integer]
+  #   a patch part of the Loops version.
+  PATCH = CURRENT[:patch]
+
+  # @return [String]
+  #   a string representation of the Loops version.
+  STRING = "%d.%d.%d" % [MAJOR, MINOR, PATCH]
+
+  # @return [String]
+  #   a string representation of the Loops version.
+  #
+  def self.to_s
+    STRING
+  end
+end

data/lib/loops/worker.rb

@@ -0,0 +1,101 @@
+module Loops
+  class Worker
+    attr_reader :name
+    attr_reader :pid
+
+    def initialize(name, pm, engine, &blk)
+      raise ArgumentError, "Need a worker block!" unless block_given?
+
+      @name = name
+      @pm = pm
+      @engine = engine
+      @worker_block = blk
+    end
+
+    def logger
+      @pm.logger
+    end
+
+    def shutdown?
+      @pm.shutdown?
+    end
+
+    def run
+      return if shutdown?
+      if @engine == 'fork'
+        # Enable COW-friendly garbage collector in Ruby Enterprise Edition
+        # See http://www.rubyenterpriseedition.com/faq.html#adapt_apps_for_cow for more details
+        if GC.respond_to?(:copy_on_write_friendly=)
+          GC.copy_on_write_friendly = true
+        end
+
+        @pid = Kernel.fork do
+          @pid = Process.pid
+          normal_exit = false
+          begin
+            $0 = "loop worker: #{@name}\0"
+            @worker_block.call
+            normal_exit = true
+            exit(0)
+          rescue Exception => e
+            message = SystemExit === e ? "exit(#{e.status})" : e.to_s
+            if SystemExit === e and e.success?
+              if normal_exit
+                logger.info("Worker finished: normal return")
+              else
+                logger.info("Worker exited: #{message} at #{e.backtrace.first}")
+              end
+            else
+              logger.fatal("Worker exited with error: #{message}\n  #{e.backtrace.join("\n  ")}")
+            end
+            logger.fatal("Terminating #{@name} worker: #{@pid}")
+            raise # so that the error gets written to stderr
+          end
+        end
+      elsif @engine == 'thread'
+        @thread = Thread.start do
+          @worker_block.call
+        end
+      else
+        raise ArgumentError, "Invalid engine name: #{@engine}"
+      end
+    rescue Exception => e
+      logger.error("Exception from worker: #{e} at #{e.backtrace.first}")
+    end
+
+    def running?(verbose = false)
+      if @engine == 'fork'
+        return false unless @pid
+        begin
+          Process.waitpid(@pid, Process::WNOHANG)
+          res = Process.kill(0, @pid)
+          logger.debug("KILL(#{@pid}) = #{res}") if verbose
+          return true
+        rescue Errno::ESRCH, Errno::ECHILD, Errno::EPERM => e
+          logger.error("Exception from kill: #{e} at #{e.backtrace.first}") if verbose
+          return false
+        end
+      elsif @engine == 'thread'
+        @thread && @thread.alive?
+      else
+        raise ArgumentError, "Invalid engine name: #{@engine}"
+      end
+    end
+
+    def stop(force = false)
+      if @engine == 'fork'
+        begin
+          sig = force ? 'SIGKILL' : 'SIGTERM'
+          logger.debug("Sending #{sig} to ##{@pid}")
+          Process.kill(sig, @pid)
+        rescue Errno::ESRCH, Errno::ECHILD, Errno::EPERM=> e
+          logger.error("Exception from kill: #{e} at #{e.backtrace.first}")
+        end
+      elsif @engine == 'thread'
+        force && !defined?(::JRuby) ? @thread.kill! : @thread.kill
+      else
+        raise ArgumentError, "Invalid engine name: #{@engine}"
+      end
+    end
+  end
+end

data/lib/loops/worker_pool.rb

@@ -0,0 +1,55 @@
+module Loops
+  class WorkerPool
+    attr_reader :name
+
+    def initialize(name, pm, engine, &blk)
+      @name = name
+      @pm = pm
+      @worker_block = blk
+      @engine = engine
+      @workers = []
+    end
+
+    def logger
+      @pm.logger
+    end
+
+    def shutdown?
+      @pm.shutdown?
+    end
+
+    def start_workers(number)
+      logger.info("Creating #{number} workers for #{name} loop...")
+      number.times do
+        @workers << Worker.new(name, @pm, @engine, &@worker_block)
+      end
+    end
+
+    def check_workers
+      logger.debug("Checking loop #{name} workers...")
+      @workers.each do |worker|
+        next if worker.running? || worker.shutdown?
+        logger.info("Worker #{worker.name} is not running. Restart!")
+        worker.run
+      end
+    end
+
+    def wait_workers
+      running = 0
+      @workers.each do |worker|
+        next unless worker.running?
+        running += 1
+        logger.info("Worker #{name} is still running (#{worker.pid})")
+      end
+      return running
+    end
+
+    def stop_workers(force)
+      logger.info("Stopping loop #{name} workers...")
+      @workers.each do |worker|
+        next unless worker.running?
+        worker.stop(force)
+      end
+    end
+  end
+end

data/loops.gemspec

@@ -0,0 +1,98 @@
+# Generated by jeweler
+# DO NOT EDIT THIS FILE DIRECTLY
+# Instead, edit Jeweler::Tasks in Rakefile, and run the gemspec command
+# -*- encoding: utf-8 -*-
+
+Gem::Specification.new do |s|
+  s.name = %q{loops}
+  s.version = "2.0.0"
+
+  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+  s.authors = ["Alexey Kovyrin", "Dmytro Shteflyuk"]
+  s.date = %q{2010-03-19}
+  s.description = %q{Loops is a small and lightweight framework for Ruby on Rails, Merb and other ruby frameworks created to support simple background loops in your application which are usually used to do some background data processing on your servers (queue workers, batch tasks processors, etc).}
+  s.email = %q{alexey@kovyrin.net}
+  s.executables = ["loops", "loops-memory-stats"]
+  s.extra_rdoc_files = [
+    "LICENSE",
+     "README.rdoc"
+  ]
+  s.files = [
+    ".gitignore",
+     "LICENSE",
+     "README.rdoc",
+     "Rakefile",
+     "VERSION.yml",
+     "bin/loops",
+     "bin/loops-memory-stats",
+     "generators/loops/loops_generator.rb",
+     "generators/loops/templates/app/loops/APP_README",
+     "generators/loops/templates/app/loops/queue_loop.rb",
+     "generators/loops/templates/app/loops/simple_loop.rb",
+     "generators/loops/templates/config/loops.yml",
+     "generators/loops/templates/script/loops",
+     "init.rb",
+     "lib/loops.rb",
+     "lib/loops/autoload.rb",
+     "lib/loops/base.rb",
+     "lib/loops/cli.rb",
+     "lib/loops/cli/commands.rb",
+     "lib/loops/cli/options.rb",
+     "lib/loops/command.rb",
+     "lib/loops/commands/debug_command.rb",
+     "lib/loops/commands/list_command.rb",
+     "lib/loops/commands/start_command.rb",
+     "lib/loops/commands/stats_command.rb",
+     "lib/loops/commands/stop_command.rb",
+     "lib/loops/daemonize.rb",
+     "lib/loops/engine.rb",
+     "lib/loops/errors.rb",
+     "lib/loops/logger.rb",
+     "lib/loops/process_manager.rb",
+     "lib/loops/queue.rb",
+     "lib/loops/version.rb",
+     "lib/loops/worker.rb",
+     "lib/loops/worker_pool.rb",
+     "loops.gemspec",
+     "spec/loop_lock_spec.rb",
+     "spec/loops/base_spec.rb",
+     "spec/loops/cli_spec.rb",
+     "spec/loops_spec.rb",
+     "spec/rails/another_loop.rb",
+     "spec/rails/app/loops/complex_loop.rb",
+     "spec/rails/app/loops/simple_loop.rb",
+     "spec/rails/config.yml",
+     "spec/rails/config/boot.rb",
+     "spec/rails/config/environment.rb",
+     "spec/rails/config/loops.yml",
+     "spec/spec_helper.rb"
+  ]
+  s.homepage = %q{http://github.com/kovyrin/loops}
+  s.rdoc_options = ["--charset=UTF-8"]
+  s.require_paths = ["lib"]
+  s.rubygems_version = %q{1.3.6}
+  s.summary = %q{Simple background loops framework for ruby}
+  s.test_files = [
+    "spec/loop_lock_spec.rb",
+     "spec/loops/base_spec.rb",
+     "spec/loops/cli_spec.rb",
+     "spec/loops_spec.rb",
+     "spec/rails/another_loop.rb",
+     "spec/rails/app/loops/complex_loop.rb",
+     "spec/rails/app/loops/simple_loop.rb",
+     "spec/rails/config/boot.rb",
+     "spec/rails/config/environment.rb",
+     "spec/spec_helper.rb"
+  ]
+
+  if s.respond_to? :specification_version then
+    current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
+    s.specification_version = 3
+
+    if Gem::Version.new(Gem::RubyGemsVersion) >= Gem::Version.new('1.2.0') then
+    else
+    end
+  else
+  end
+end
+