loops 2.0.0

data/lib/loops/commands/debug_command.rb

@@ -0,0 +1,8 @@
+class Loops::Commands::DebugCommand < Loops::Command
+  def execute
+    Loops.logger.write_to_console = true
+    puts "Starting one loop in debug mode: #{options[:args].first}"
+    engine.debug_loop!(options[:args].first)
+    exit(0)
+  end
+end

data/lib/loops/commands/list_command.rb

@@ -0,0 +1,11 @@
+class Loops::Commands::ListCommand < Loops::Command
+  def execute
+    puts 'Available loops:'
+    engine.loops_config.each do |name, config|
+      puts "  Loop: #{name}" + (config['disabled'] ? ' (disabled)' : '')
+      config.each do |k, v|
+        puts "     - #{k}: #{v}"
+      end
+    end
+  end
+end

data/lib/loops/commands/start_command.rb

@@ -0,0 +1,24 @@
+class Loops::Commands::StartCommand < Loops::Command
+  def execute
+    # Pid file check
+    if Loops::Daemonize.check_pid(Loops.pid_file)
+      puts "Can't start, another process exists!"
+      exit(1)
+    end
+
+    # Daemonization
+    if options[:daemonize]
+      app_name = "loops monitor: #{options[:args].join(' ') rescue 'all'}\0"
+      Loops::Daemonize.daemonize(app_name)
+    end
+
+    # Pid file creation
+    Loops::Daemonize.create_pid(Loops.pid_file)
+
+    # Workers processing
+    engine.start_loops!(options[:args])
+
+    # Workers exited, cleaning up
+    File.delete(Loops.pid_file) rescue nil
+  end
+end

data/lib/loops/commands/stats_command.rb

@@ -0,0 +1,5 @@
+class Loops::Commands::StatsCommand < Loops::Command
+  def execute
+    system File.join(Loops::LIB_ROOT, '../bin/loops-memory-stats')
+  end
+end

data/lib/loops/commands/stop_command.rb

@@ -0,0 +1,18 @@
+class Loops::Commands::StopCommand < Loops::Command
+  def execute
+    STDOUT.sync = true
+    raise "No pid file or a stale pid file!" unless Loops::Daemonize.check_pid(Loops.pid_file)
+    pid = Loops::Daemonize.read_pid(Loops.pid_file)
+    print "Killing the process: #{pid}: "
+
+    loop do
+      Process.kill('SIGTERM', pid)
+      sleep(1)
+      break unless Loops::Daemonize.check_pid(Loops.pid_file)
+      print(".")
+    end
+
+    puts " Done!"
+    exit(0)
+  end
+end

data/lib/loops/daemonize.rb

@@ -0,0 +1,68 @@
+module Loops
+  module Daemonize
+    def self.read_pid(pid_file)
+      File.open(pid_file) do |f|
+        f.gets.to_i
+      end
+    rescue Errno::ENOENT
+      0
+    end
+
+    def self.check_pid(pid_file)
+      pid = read_pid(pid_file)
+      return false if pid.zero?
+      if defined?(::JRuby)
+        system "kill -0 #{pid} &> /dev/null"
+        return $? == 0
+      else
+        Process.kill(0, pid)
+      end
+      true
+    rescue Errno::ESRCH, Errno::ECHILD, Errno::EPERM
+      false
+    end
+
+    def self.create_pid(pid_file)
+      if File.exist?(pid_file)
+        puts "Pid file #{pid_file} exists! Checking the process..."
+        if check_pid(pid_file)
+          puts "Can't create new pid file because another process is runnig!"
+          return false
+        end
+        puts "Stale pid file! Removing..."
+        File.delete(pid_file)
+      end
+
+      File.open(pid_file, 'w') do |f|
+        f.puts(Process.pid)
+      end
+
+      return true
+    end
+
+    def self.daemonize(app_name)
+      if defined?(::JRuby)
+        puts "WARNING: daemonize method is not implemented for JRuby (yet), please consider using nohup."
+        return
+      end
+
+      fork && exit # Fork and exit from the parent
+
+      # Detach from the controlling terminal
+      unless sess_id = Process.setsid
+        raise Daemons.RuntimeException.new('cannot detach from controlling terminal')
+      end
+
+      # Prevent the possibility of acquiring a controlling terminal
+      trap 'SIGHUP', 'IGNORE'
+      exit if pid = fork
+
+      $0 = app_name if app_name
+
+      Dir.chdir(Loops.root) # Make sure we're in the working directory
+      File.umask(0000) # Insure sensible umask
+
+      return sess_id
+    end
+  end
+end

data/lib/loops/engine.rb

@@ -0,0 +1,207 @@
+class Loops::Engine
+  attr_reader :config
+
+  attr_reader :loops_config
+
+  attr_reader :global_config
+
+  def initialize
+    load_config
+  end
+
+  def load_config
+    # load and parse with erb
+    raw_config = File.read(Loops.config_file)
+    erb_config = ERB.new(raw_config).result
+
+    @config = YAML.load(erb_config)
+    @loops_config  = @config['loops']
+    @global_config = {
+      'poll_period'    => 1,
+      'workers_engine' => 'fork'
+    }.merge(@config['global'])
+
+    Loops.logger.default_logfile = @global_config['logger'] || $stdout
+    Loops.logger.colorful_logs = @global_config['colorful_logs'] || @global_config['colourful_logs']
+  end
+
+  def start_loops!(loops_to_start = [])
+    @running_loops = []
+    @pm = Loops::ProcessManager.new(global_config, Loops.logger)
+
+    # Start all loops
+    loops_config.each do |name, config|
+      next if config['disabled']
+      next unless loops_to_start.empty? || loops_to_start.member?(name)
+
+      klass = load_loop_class(name, config)
+      next unless klass
+
+      start_loop(name, klass, config)
+      @running_loops << name
+    end
+
+    # Do not continue if there is nothing to run
+    if @running_loops.empty?
+      puts 'WARNING: No loops to run! Exiting...'
+      return
+    end
+
+    # Start monitoring loop
+    setup_signals
+    @pm.monitor_workers
+
+    info 'Loops are stopped now!'
+  end
+
+  def debug_loop!(loop_name)
+    @pm = Loops::ProcessManager.new(global_config, Loops.logger)
+    loop_config = loops_config[loop_name] || {}
+
+    # Adjust loop config values before starting it in debug mode
+    loop_config['workers_number'] = 1
+    loop_config['debug_loop'] = true
+
+    # Load loop class
+    unless klass = load_loop_class(loop_name, loop_config)
+      puts "Can't load loop class!"
+      return false
+    end
+
+    # Start the loop
+    start_loop(loop_name, klass, loop_config)
+  end
+
+  private
+
+    # Proxy logger calls to the default loops logger
+    [ :debug, :error, :fatal, :info, :warn ].each do |meth_name|
+      class_eval <<-EVAL, __FILE__, __LINE__
+        def #{meth_name}(message)
+          Loops.logger.#{meth_name} "loops[RUNNER/\#{Process.pid}]: \#{message}"
+        end
+      EVAL
+    end
+
+    def load_loop_class(name, config)
+      loop_name = config['loop_name'] || name
+
+      klass_files = [Loops.loops_root + "#{loop_name}_loop.rb", "#{loop_name}_loop"]
+      begin
+        klass_file = klass_files.shift
+        debug "Loading class file: #{klass_file}"
+        require(klass_file)
+      rescue LoadError
+        retry unless klass_files.empty?
+        error "Can't load the class file: #{klass_file}. Worker #{name} won't be started!"
+        return false
+      end
+
+      klass_name = "#{loop_name}_loop".capitalize.gsub(/_(.)/) { $1.upcase }
+      klass = Object.const_get(klass_name) rescue nil
+
+      unless klass
+        error "Can't find class: #{klass_name}. Worker #{name} won't be started!"
+        return false
+      end
+
+      begin
+        klass.check_dependencies
+      rescue Exception => e
+        error "Loop #{name} dependencies check failed: #{e} at #{e.backtrace.first}"
+        return false
+      end
+
+      return klass
+    end
+
+    def start_loop(name, klass, config)
+      info "Starting loop: #{name}"
+      info " - config: #{config.inspect}"
+
+      loop_proc = Proc.new do
+        the_logger = begin
+            if Loops.logger.is_a?(Loops::Logger) && @global_config['workers_engine'] == 'fork'
+              # this is happening right after the fork, therefore no need for teardown at the end of the proc
+              Loops.logger.logfile = config['logger']
+              Loops.logger
+            else
+              # for backwards compatibility and handling threading engine
+              create_logger(name, config)
+            end
+        end
+
+        # Set logger level
+        if String === config['log_level']
+          level = Logger::Severity.const_get(config['log_level'].upcase) rescue nil
+          the_logger.level = level if level
+        elsif Integer === config['log_level']
+          the_logger.level = config['log_level']
+        end
+
+        # Colorize logging?
+        if the_logger.respond_to?(:colorful_logs=) && (config.has_key?('colorful_logs') || config.has_key?('colourful_logs'))
+          the_logger.colorful_logs = config['colorful_logs'] || config['colourful_logs']
+        end
+
+        debug "Instantiating class: #{klass}"
+        the_loop = klass.new(@pm, name, config)
+
+        debug "Starting the loop #{name}!"
+        fix_ar_after_fork
+        # reseed the random number generator in case Loops calls
+        # srand or rand prior to forking
+        srand
+        the_loop.run
+      end
+
+      # If the loop is in debug mode, no need to use all kinds of
+      # process managers here
+      if config['debug_loop']
+        loop_proc.call
+      else
+        @pm.start_workers(name, config['workers_number'] || 1) { loop_proc.call }
+      end
+    end
+
+    def create_logger(loop_name, config)
+      config['logger'] ||= 'default'
+
+      return Loops.default_logger if config['logger'] == 'default'
+      Loops::Logger.new(config['logger'])
+
+    rescue Exception => e
+      message = "Can't create a logger for the #{loop_name} loop! Will log to the default logger!"
+      puts "ERROR: #{message}"
+
+      message << "\nException: #{e} at #{e.backtrace.first}"
+      error(message)
+
+      return Loops.default_logger
+    end
+
+    def setup_signals
+      trap('TERM') {
+        warn "Received a TERM signal... stopping..."
+        @pm.stop_workers!
+      }
+
+      trap('INT') {
+        warn "Received an INT signal... stopping..."
+        @pm.stop_workers!
+      }
+
+      trap('EXIT') {
+        warn "Received a EXIT 'signal'... stopping..."
+        @pm.stop_workers!
+      }
+    end
+
+    def fix_ar_after_fork
+      if Object.const_defined?('ActiveRecord')
+        ActiveRecord::Base.allow_concurrency = true
+        ActiveRecord::Base.clear_active_connections!
+        ActiveRecord::Base.verify_active_connections!
+      end
+    end
+end

data/lib/loops/errors.rb

@@ -0,0 +1,6 @@
+module Loops::Errors
+  Error = Class.new(RuntimeError)
+
+  InvalidFrameworkError = Class.new(Error)
+  InvalidCommandError   = Class.new(Error)
+end

data/lib/loops/logger.rb

@@ -0,0 +1,212 @@
+require 'logger'
+require 'delegate'
+require 'fileutils'
+
+class Loops::Logger < ::Delegator
+  # @return [Boolean]
+  #   A value indicating whether all logging output should be
+  #   also duplicated to the console.
+  attr_reader :write_to_console
+
+  # @return [Boolean]
+  #   A value inidicating whether critical errors should be highlighted
+  #   with ANSI colors in the log.
+  attr_reader :colorful_logs
+
+  # Initializes a new instance of the {Logger} class.
+  #
+  # @param [String, IO] logfile
+  #   The log device.  This is a filename (String), <tt>'stdout'</tt> or
+  #   <tt>'stderr'</tt> (String), <tt>'default'</tt> for default framework's
+  #   log file, or +IO+ object (typically +STDOUT+, +STDERR+,
+  #   or an open file).
+  # @param [Integer] level
+  #   Logging level. Constants are defined in +Logger+ namespace: +DEBUG+, +INFO+,
+  #   +WARN+, +ERROR+, +FATAL+, or +UNKNOWN+.
+  # @param [Integer] number_of_files
+  #   A number of files to keep.
+  # @param [Integer] max_file_size
+  #   A max file size. When file become larger, next one will be created.
+  # @param [Boolean] write_to_console
+  #   When +true+, all logging output will be dumped to the +STDOUT+ also.
+  #
+  def initialize(logfile = $stdout, level = ::Logger::INFO, number_of_files = 10, max_file_size = 100 * 1024 * 1024,
+                 write_to_console = false)
+    @number_of_files, @level, @max_file_size, @write_to_console =
+        number_of_files, level, max_file_size, write_to_console
+    self.logfile = logfile
+    super(@implementation)
+  end
+
+  # Sets the default log file (see {#logfile=}).
+  #
+  # @param [String, IO] logfile
+  #   the log file path or IO.
+  # @return [String, IO]
+  #   the log file path or IO.
+  #
+  def default_logfile=(logfile)
+    @default_logfile = logfile
+    self.logfile = logfile
+  end
+
+  # Sets the log file.
+  #
+  # @param [String, IO] logfile
+  #   The log device.  This is a filename (String), <tt>'stdout'</tt> or
+  #   <tt>'stderr'</tt> (String), <tt>'default'</tt> for default framework's
+  #   log file, or +IO+ object (typically +STDOUT+, +STDERR+,
+  #   or an open file).
+  # @return [String, IO]
+  #   the log device.
+  #
+  def logfile=(logfile)
+    logfile = @default_logfile || $stdout if logfile == 'default'
+    coerced_logfile =
+        case logfile
+        when 'stdout' then $stdout
+        when 'stderr' then $stderr
+        when IO, StringIO then logfile
+        else
+          if Loops.root
+            logfile =~ /^\// ? logfile : Loops.root.join(logfile).to_s
+          else
+            logfile
+          end
+        end
+    # Ensure logging directory does exist
+    FileUtils.mkdir_p(File.dirname(coerced_logfile)) if String === coerced_logfile
+
+    # Create a logger implementation.
+    @implementation = LoggerImplementation.new(coerced_logfile, @number_of_files, @max_file_size, @write_to_console, @colorful_logs)
+    @implementation.level = @level
+    logfile
+  end
+
+  # Remember the level at the proxy level.
+  #
+  # @param [Integer] level
+  #   Logging severity.
+  # @return [Integer]
+  #   Logging severity.
+  #
+  def level=(level)
+    @level = level
+    @implementation.level = @level if @implementation
+    level
+  end
+
+  # Sets a value indicating whether to dump all logs to the console.
+  #
+  # @param [Boolean] value
+  #   a value indicating whether to dump all logs to the console.
+  # @return [Boolean]
+  #   a value indicating whether to dump all logs to the console.
+  #
+  def write_to_console=(value)
+    @write_to_console = value
+    @implementation.write_to_console = value if @implementation
+    value
+  end
+
+  # Sets a value indicating whether to highlight with red ANSI color
+  # all critical messages.
+  #
+  # @param [Boolean] value
+  #   a value indicating whether to highlight critical errors in log.
+  # @return [Boolean]
+  #   a value indicating whether to highlight critical errors in log.
+  #
+  def colorful_logs=(value)
+    @colorful_logs = value
+    @implementation.colorful_logs = value if @implementation
+    value
+  end
+
+  # @private
+  # Send everything else to @implementation.
+  def __getobj__
+    @implementation or raise "Logger implementation not initialized"
+  end
+
+  # @private
+  # Delegator's method_missing ignores the &block argument (!!!?)
+  def method_missing(m, *args, &block)
+    target = self.__getobj__
+    unless target.respond_to?(m)
+      super(m, *args, &block)
+    else
+      target.__send__(m, *args, &block)
+    end
+  end
+
+  # @private
+  class LoggerImplementation < ::Logger
+
+    attr_reader :prefix
+
+    attr_accessor :write_to_console, :colorful_logs
+
+    class Formatter
+
+      def initialize(logger)
+        @logger = logger
+      end
+
+      def call(severity, time, progname, message)
+        if (@logger.prefix || '').empty?
+          "#{severity[0..0]} : #{time.strftime('%Y-%m-%d %H:%M:%S')} : #{message || progname}\n"
+        else
+          "#{severity[0..0]} : #{time.strftime('%Y-%m-%d %H:%M:%S')} : #{@logger.prefix} : #{message || progname}\n"
+        end
+      end
+    end
+
+    def initialize(log_device, number_of_files = 10, max_file_size = 10 * 1024 * 1024, write_to_console = true, colorful_logs = false)
+      super(log_device, number_of_files, max_file_size)
+      self.formatter    = Formatter.new(self)
+      @write_to_console = write_to_console
+      @colorful_logs    = colorful_logs
+      @prefix           = nil
+    end
+
+    def add(severity, message = nil, progname = nil, &block)
+      begin
+        if @colorful_logs
+          message = color_errors(severity, message)
+          progname = color_errors(severity, progname)
+        end
+        super(severity, message, progname, &block)
+        if @write_to_console && (message || progname)
+          puts self.formatter.call(%w(D I W E F A)[severity] || 'A', Time.now, progname, message)
+        end
+      rescue
+        # ignore errors in logging
+      end
+    end
+
+    def with_prefix(prefix)
+      raise "No block given" unless block_given?
+      old_prefix = @prefix
+      @prefix = prefix
+      begin
+        yield
+      ensure
+        @prefix = old_prefix
+      end
+    end
+
+    def color_errors(severity, line)
+      if severity < ::Logger::ERROR
+        line
+      else
+        if line && line !~ /\e/
+          "\e[31m#{line}\e[0m"
+        else
+          line
+        end
+      end
+    end
+
+  end
+end