loops 2.0.0

data/generators/loops/loops_generator.rb

@@ -0,0 +1,28 @@
+# This generator bootstraps a Rails project for use with loops
+class LoopsGenerator < Rails::Generator::Base
+  def manifest
+    record do |m|
+      # Generate app/loops directory and an example loop files
+      m.directory 'app'
+      m.directory 'app/loops'
+      m.file      'app/loops/APP_README',     'app/loops/README'
+      m.file      'app/loops/simple_loop.rb', 'app/loops/simple_loop.rb'
+      m.file      'app/loops/queue_loop.rb',  'app/loops/queue_loop.rb'
+      
+      # Generate script/loops file
+      m.directory 'script'
+      m.file      'script/loops',             'script/loops', :chmod => 0755
+      
+      # Generate config/loops.yml file
+      m.directory 'config'
+      m.file      'config/loops.yml',         'config/loops.yml'
+    end
+  end
+
+protected
+
+  def banner
+    "Usage: #{$0} loops"
+  end
+
+end

data/generators/loops/templates/app/loops/APP_README

@@ -0,0 +1 @@
+This directory should be used to hold all application loops.

data/generators/loops/templates/app/loops/queue_loop.rb

@@ -0,0 +1,8 @@
+class QueueLoop < Loops::Queue
+  def process_message(message)
+    debug "Received a message: #{message.body}"
+    debug "sleeping..."
+    sleep(0.5 + rand(10) / 10.0)
+    debug "done..."
+  end
+end

data/generators/loops/templates/app/loops/simple_loop.rb

@@ -0,0 +1,12 @@
+# Simple loop with its own custom run method
+#
+# Does nothing aside from printing loop's name, pid and current time every second
+#
+class SimpleLoop < Loops::Base
+  def run
+    with_period_of(1) do
+      debug(Time.now)
+      sleep(5)
+    end
+  end
+end

data/generators/loops/templates/config/loops.yml

@@ -0,0 +1,34 @@
+# This file is a configuration file for loops rails plugin
+#
+
+# This section is used to control loops manager
+global:
+  logger: stdout
+  poll_period: 5
+  workers_engine: fork
+
+# Each record in this section represents one loop which could be ran using loops plugin.
+# Each loop should have a file in app/loops directory with the same name as its config record.
+loops:
+
+  # Simple time printing loop
+  simple:
+    type: simple
+    workers_number: 1
+    logger: default
+
+  # An example of a STOMP queue-based loop
+  queue:
+    type: queue
+    workers_number: 2
+    logger: log/loops/queue.log
+    # logger_rotate: daily       # TODO
+    # logger_max_size: 10000000  # TODO
+  
+    host: 127.0.0.1
+    port: 61613
+    disabled: true
+
+  # This loop has no loop class file so it should not be started
+  non_existent:
+    type: simple

data/generators/loops/templates/script/loops

@@ -0,0 +1,20 @@
+#!/usr/bin/env ruby
+
+vendored_loops_binary = Dir[File.join(File.dirname(__FILE__),
+                                         '..',
+                                         'vendor',
+                                         '{gems,plugins}',
+                                         'loops*',
+                                         'bin',
+                                         'loops')].first
+
+# Initialize loops root variable
+ENV['LOOPS_ROOT'] = File.expand_path(File.join(File.dirname(__FILE__), '..'))
+
+if vendored_loops_binary
+  load File.expand_path(vendored_loops_binary)
+else
+  require 'rubygems' unless ENV['NO_RUBYGEMS']
+  require 'loops'
+  load Loops::BINARY
+end

data/init.rb

@@ -0,0 +1 @@
+require File.join(File.dirname(__FILE__), 'lib/loops')

data/lib/loops.rb

@@ -0,0 +1,167 @@
+require 'yaml'
+require 'erb'
+require 'pathname'
+
+module Loops
+  # @return [String]
+  #   a full path to the loops "lib" directory.
+  LIB_ROOT = File.expand_path(File.dirname(__FILE__)) unless const_defined?('LIB_ROOT')
+  # @return [String]
+  #   a full path to the loops binary file.
+  BINARY   = File.expand_path(File.join(LIB_ROOT, '../bin/loops')) unless const_defined?('BINARY')
+
+  # Loops root directory.
+  #
+  # Usually it is initialized with framework's root dir (RAILS_ROOT or MERB_ROOT),
+  # but you can specify another directory using command line arguments.
+  #
+  # Loops current directory will is set to this value (chdir).
+  #
+  # @return [Pathname, nil]
+  #   the loops root directory.
+  #
+  def self.root
+    @@root
+  end
+
+  # Set loops root directory.
+  #
+  # This is internal method used to set the loops root directory.
+  #
+  # @param [String] path
+  #   the absolute path of the loops root directory.
+  # @return [Pathname]
+  #   the loops root directory.
+  #
+  # @private
+  #
+  def self.root=(path)
+    @@root = Pathname.new(path).realpath
+  end
+
+  # Get loops config file full path.
+  #
+  # @return [Pathname]
+  #   the loops config file path.
+  #
+  def self.config_file
+    @@config_file ||= root.join('config/loops.yml')
+  end
+
+  # Set loops config file path.
+  #
+  # This is internal method used to set the loops config file path.
+  #
+  # @param [String] path
+  #   the absolute or relative to the loops root path of the loops
+  #   config file.
+  # @return [Pathname]
+  #   the loops config file path.
+  #
+  # @private
+  #
+  def self.config_file=(config_file)
+    @@config_file = root.join(config_file) if config_file
+  end
+
+  # Get directory containing loops classes.
+  #
+  # @return [Pathname]
+  #   the loops directory path.
+  #
+  def self.loops_root
+    @@loops_root ||= root.join('app/loops')
+  end
+
+  # Set loops classes directory path.
+  #
+  # This is internal method used to set directory where loops classes
+  # will be searched.
+  #
+  # @param [String] path
+  #   the absolute or relative to the loops classes directory.
+  # @return [Pathname]
+  #   the loops classes directory path.
+  #
+  # @private
+  #
+  def self.loops_root=(loops_root)
+    @@loops_root = root.join(loops_root) if loops_root
+  end
+
+  # Get the loops monitor process pid file path.
+  #
+  # @return [Pathname]
+  #   the loops monitor process pid file path.
+  #
+  def self.pid_file
+    @@pid_file ||= root.join('loops.pid')
+  end
+
+  # Set the loops monitor pid file path.
+  #
+  # This is internal method used to set the loops monitor pid file path.
+  #
+  # @param [String] path
+  #   the absolute or relative to the loops monitor pid file.
+  # @return [Pathname]
+  #   the loops monitor pid file path.
+  #
+  # @private
+  #
+  def self.pid_file=(pid_file)
+    @@pid_file = root.join(pid_file) if pid_file
+  end
+
+  # Get the current loops logger.
+  #
+  # There are two contexts where different devices (usually) will be
+  # configured for this logger instance:
+  #
+  # 1. In context of loops monitor logger device will be retrieved from
+  #    the global section of the loops config file, or STDOUT when it
+  #    was not configured.
+  # 2. In context of loop proccess logger device will be configured
+  #    based on logger value of the particular loop section in the config
+  #    file.
+  #
+  # @example Put all Rails logging into the loop log file (add this to the environment.rb)
+  #   Rails.logger = Loops.logger
+  #
+  # @return [Loops::Logger]
+  #   the current loops logger instance.
+  #
+  def self.logger
+    @@logger ||= ::Loops::Logger.new($stdout)
+  end
+
+  # Set the current loops logger.
+  #
+  def self.logger=(logger)
+    @@logger = logger
+  end
+
+  # Get the current framework's default logger.
+  #
+  # @return [Logger]
+  #   the default logger for currently used framework.
+  #
+  def self.default_logger
+    @@default_logger
+  end
+
+  # Set the current framework's default logger.
+  #
+  # @param [Logger] logger
+  #   the default logger for currently used framework.
+  # @return [Logger]
+  #   the default logger for currently used framework.
+  #
+  # @private
+  #
+  def self.default_logger=(logger)
+    @@default_logger = logger
+  end
+end
+
+require File.join(Loops::LIB_ROOT, 'loops/autoload')

data/lib/loops/autoload.rb

@@ -0,0 +1,20 @@
+module Loops
+  # @private
+  def self.__p(*path) File.join(Loops::LIB_ROOT, 'loops', *path) end
+
+  autoload :Base,           __p('base')
+  autoload :CLI,            __p('cli')
+  autoload :Command,        __p('command')
+  autoload :Commands,       __p('command')
+  autoload :Daemonize,      __p('daemonize')
+  autoload :Engine,         __p('engine')
+  autoload :Errors,         __p('errors')
+  autoload :Logger,         __p('logger')
+  autoload :ProcessManager, __p('process_manager')
+  autoload :Queue,          __p('queue')
+  autoload :Worker,         __p('worker')
+  autoload :WorkerPool,     __p('worker_pool')
+  autoload :Version,        __p('version')
+
+  include Errors
+end

data/lib/loops/base.rb

@@ -0,0 +1,148 @@
+# Base class for all loop processes.
+#
+# To create a new loop just inherit this class and override the
+# {#run} method (see example below).
+#
+# In most cases it's a good idea to re-run your loop periodically.
+# In this case your process will free all unused memory to the system,
+# and all leaked resources (yes, it's real life).
+#
+# @example
+#   class MySuperLoop < Loops::Base
+#     def self.check_dependencies
+#       gem 'tinder', '=1.3.1'
+#     end
+#
+#     def run
+#       1000.times do
+#         if shutdown?
+#           info("Shutting down!")
+#           exit(0)
+#         end
+#
+#         unless item = UploadItems.get_next
+#           sleep(config['sleep_time'])
+#           next
+#         end
+#
+#         item.perform_upload
+#       end
+#     end
+#   end
+#
+class Loops::Base
+  # @return [String]
+  #   loop name.
+  attr_reader :name
+
+  # @return [Hash<String, Object>]
+  #   The hash of loop options from config.
+  attr_reader :config
+
+  # Initializes a new instance of loop.
+  #
+  # @param [ProcessManager] pm
+  #   the instance of process manager.
+  # @param [String] name
+  #   the loop name.
+  # @param [Hash<String, Object>]
+  #   the loop configuration options from the config file.
+  #
+  def initialize(pm, name, config)
+    @pm     = pm
+    @name   = name
+    @config = config
+  end
+
+  # Get the logger instance.
+  #
+  # @return [Logger]
+  #   the logger instance.
+  #
+  def logger
+    @pm.logger
+  end
+
+  # Get a value indicating whether shutdown is in the progress.
+  #
+  # Check this flag periodically and if loop is in the shutdown,
+  # close all open handlers, update your data, and exit loop as
+  # soon as possible to not to loose any sensitive data.
+  #
+  # @return [Boolean]
+  #   a value indicating whether shutdown is in the progress.
+  #
+  # @example
+  #   if shutdown?
+  #     info('Shutting down!')
+  #     exit(0)
+  #   end
+  #
+  def shutdown?
+    @pm.shutdown?
+  end
+
+  # Verifies loop dependencies.
+  #
+  # Override this method if your loop depends on any external
+  # libraries, resources, etc. Verify your dependencies here,
+  # and raise an exception in case of any trouble to disallow
+  # this loop to start.
+  #
+  # @example
+  #   def self.check_dependencies
+  #     gem 'tinder', '=1.3.1'
+  #   end
+  #
+  def self.check_dependencies
+  end
+
+  # A loop entry point. Should be overridden in descendants.
+  #
+  def run
+    raise 'Generic loop has nothing to do'
+  end
+
+  # Proxy logger calls to our logger
+  [ :debug, :info, :warn, :error, :fatal ].each do |meth_name|
+    class_eval <<-EVAL, __FILE__, __LINE__
+      def #{meth_name}(message)
+        logger.#{meth_name}("loop[\#{name}/\#{Process.pid}]: \#{message}")
+      end
+    EVAL
+  end
+
+  def with_lock(entity_ids, loop_id, timeout, entity_name = '', &block)
+    entity_name = 'item' if entity_name.to_s.empty?
+    entity_ids = [entity_ids] unless Array === entity_ids
+
+    entity_ids.each do |entity_id|
+      debug("Locking #{entity_name} #{entity_id}")
+      lock = LoopLock.lock(:entity_id => entity_id, :loop => loop_id.to_s, :timeout => timeout)
+      unless lock
+        warn("Race condition detected for the #{entity_name}: #{entity_id}. Skipping the item.")
+        next
+      end
+
+      begin
+        result = if block.arity == 1
+          yield entity_id
+        else
+          yield
+        end
+        return result
+      ensure
+        debug("Unlocking #{entity_name} #{entity_id}")
+        LoopLock.unlock(:entity_id => entity_id, :loop => loop_id.to_s)
+      end
+    end
+  end
+
+  def with_period_of(seconds)
+    raise ArgumentError, "No block given!" unless block_given?
+    loop do
+      yield
+      sleep(seconds)
+    end
+  end
+end