revenant 0.0.2

data/lib/locks/mysql.rb

@@ -0,0 +1,56 @@
+# Each lock module must implement a singleton method called +lock_function+
+# This method should return a proc that will be called with a lock_name argument.
+# The proc should return true if the lock has been acquired, false otherwise
+#
+# Lock modules may expose other methods to users as needed, but only
+# +lock_function+ is required.
+# Modules register new lock types by calling Revenant.register(name, klass)
+module Revenant
+  module MySQL
+    extend self
+
+    def lock_function
+      Proc.new do |lock_name|
+        ::Revenant::MySQL.acquire_lock(lock_name)
+      end
+    end
+
+    # Expects the connection to behave like an instance of +Mysql+
+    # If you need something else, replace +acquire_lock+ with your own code.
+    # Or define your own lock_function while configuring a new Revenant task.
+    def acquire_lock(lock_name)
+      begin
+        acquired = false
+        sql = lock_query(lock_name)
+        connection.query(sql) do |result|
+          acquired = result.fetch_row.first == "1"
+        end
+        acquired
+      rescue ::Exception
+        false
+      end
+    end
+
+    def lock_query(lock_name)
+      "select get_lock('#{lock_name}',0);"
+    end
+
+    # Currently defaults to the ActiveRecord connection if AR is loaded.
+    # Set this in your task setup block if that is not what you want.
+    def connection
+      @connection ||= if defined?(ActiveRecord)
+                        ActiveRecord::Base.connection.raw_connection
+                      else
+                        raise "No connection established or discovered. Use Revenant::MySQL::connection="
+                      end
+    end
+
+    def connection=(conn)
+      @connection = conn
+    end
+  end
+
+  # This is how you register a new lock_type
+  register :mysql, MySQL
+end
+

data/lib/plugins/daemon.rb

@@ -0,0 +1,115 @@
+require 'revenant/task'
+require 'revenant/pid'
+require 'revenant/manager'
+
+# "startup" and "shutdown" are the methods Task expects modules like
+# this one to replace.
+module ::Revenant::Daemon
+  # Installs this plugin in the given +task+.
+  # Out of the box, this is always to provide daemon support.
+  # +install+ is expected to know when to do nothing.
+  def self.install(task)
+    if task.daemon?
+      class << task
+        include ::Revenant::Daemon
+      end
+    end
+  end
+
+  def startup
+    @original_dir = ::Revenant.working_directory
+    daemonize
+    log "#{name} is starting"
+  end
+
+  def shutdown
+    @pid.remove
+
+    if restart_pending?
+      log "#{name} is restarting"
+      if @original_dir
+        Dir.chdir @original_dir
+      end
+      system script
+    else
+      log "#{name} is shutting down"
+    end
+
+    exit 0
+  end
+
+  ##
+  ## Everything else is a daemon implementation detail.
+  ##
+
+  def pid_file
+    @options[:pid_file] ||= File.join("/tmp", "#{@name}.pid")
+  end
+
+  def log_file
+    @options[:log_file]
+  end
+
+  def script
+    @options[:script] ||= File.expand_path($0)
+  end
+
+  protected
+
+  def daemonize
+    verify_permissions
+    ::Revenant::Manager.daemonize(name, log_file)
+    @pid.create
+    daemon_signals
+  end
+
+  def verify_permissions
+    unless File.executable?(script)
+      error "script file is not executable: #{script.inspect}"
+      exit 1
+    end
+
+    dir = File.dirname(pid_file)
+    unless File.directory?(dir) && File.writable?(dir)
+      error "pid file is not writeable: #{pid_file.inspect}"
+      exit 1
+    end
+
+    @pid = ::Revenant::PID.new(pid_file)
+    if @pid.exists?
+      error "pid file exists: #{pid_file.inspect}. unclean shutdown?"
+      exit 1
+    end
+
+    if log_file && dir = File.dirname(log_file)
+      unless File.directory?(dir) && File.writable?(dir)
+        error "log file is not writeable: #{log_file.inspect}"
+        exit 1
+      end
+    end
+  end
+
+  def daemon_signals
+    trap("TERM") do
+      log "Received TERM signal"
+      shutdown_soon
+    end
+
+    trap("QUIT") do
+      log "QUIT: #{caller.inspect}"
+      shutdown_soon
+    end
+
+    trap("USR1") do
+      log "TRACE: #{caller.inspect}"
+    end
+
+    trap("USR2") do
+      log "Received USR2 signal"
+      restart_soon
+    end
+  end
+end
+
+# Register this plugin
+::Revenant.plugins[:daemon] = ::Revenant::Daemon

data/lib/revenant.rb

@@ -0,0 +1,72 @@
+module Revenant
+  VERSION = "0.0.1"
+
+  # Register a new type of lock.
+  # User code specifies which by setting lock_type = :something
+  # while configuring a Revenant::Task
+  def self.register(lock_type, klass)
+    @lock_types ||= {}
+    if klass.respond_to?(:lock_function)
+      @lock_types[lock_type.to_sym] = klass
+    else
+      raise ArgumentError, "#{klass} must have a `lock_function` that returns a callable object"
+    end
+  end
+
+  def self.find_module(lock_type)
+    @lock_types ||= {}
+    @lock_types.fetch(lock_type.to_sym) do
+      raise ArgumentError, "unknown lock type: #{lock_type.inspect}"
+    end
+  end
+
+  def self.plugins
+    @plugins ||= {}
+  end
+
+  def self.init
+    require 'revenant/task'
+    require_dir "locks"
+    require_dir "plugins"
+  end
+
+  # Given 'locks/foo', will require ./locks/foo/*.rb' with normalized
+  # (relative) paths. (e.g. require 'locks/foo/example' for example.rb)
+  def self.require_dir(relative_path)
+    current = File.expand_path('..', __FILE__)
+    make_relative = /#{current}\//
+    $LOAD_PATH << current unless $LOAD_PATH.include?(current)
+    pattern = File.join(current, relative_path, '*.rb')
+    Dir[pattern].each do |full_path|
+      relative_name = full_path.gsub(make_relative,'').gsub(/\.rb$/,'')
+      require relative_name
+    end
+  end
+
+  def self.working_directory
+    # If the 'PWD' environment variable points to our
+    # current working directory, use it instead of Dir.pwd.
+    # It may have a better name for the same destination,
+    # in the presence of symlinks.
+    e = File.stat(env_pwd = ENV['PWD'])
+    p = File.stat(Dir.pwd)
+    e.ino == p.ino && e.dev == p.dev ? env_pwd : Dir.pwd
+  rescue
+    Dir.pwd
+  end
+end
+
+module Kernel
+  def revenant(name = nil)
+    unless String === name || Symbol === name
+      raise ArgumentError, "Usage: task = revenant('example') {|r| configure_as_needed }"
+    end
+    instance = ::Revenant::Task.new(name)
+    instance.daemon = true # daemonized by default if available
+    yield instance if block_given?
+    instance
+  end
+end
+
+::Revenant.init
+

data/lib/revenant/manager.rb

@@ -0,0 +1,62 @@
+module Revenant
+  module Manager
+    extend self
+
+    def daemonize(name, log_file = nil)
+      # Firstly, get rid of the filthy dirty original process.
+      exit!(0) if fork
+
+      # Now that we aren't attached to a terminal, we can become
+      # a session leader.
+      begin
+        Process.setsid
+      rescue Errno::EPERM
+        raise SystemCallError, "setsid failed. terminal failed to detach?"
+      end
+      trap 'SIGHUP', 'IGNORE' # don't do anything crazy when this process exits
+
+      # Finally, time to create a daemonized process
+      exit!(0) if fork
+
+      $0 = name.to_s # set the process name
+      close_open_files
+      redirect_io_to(log_file)
+      srand # re-seed the PRNG with our 'final' pid
+    end
+
+    # Close anything that is not one of the three standard IO streams.
+    def close_open_files
+      ObjectSpace.each_object(IO) do |io|
+        next if [STDIN, STDOUT, STDERR].include?(io)
+        begin
+          io.close unless io.closed?
+        rescue ::Exception
+        end
+      end
+    end
+
+    # Redirects STDIN, STDOUT, and STDERR to the specified +log_file+
+    # or to /dev/null if none is given.
+    def redirect_io_to(log_file)
+      log_file ||= "/dev/null"
+      reopen_io STDIN, "/dev/null"
+      reopen_io STDOUT, log_file, "a"
+      reopen_io STDERR, STDOUT
+      STDERR.sync = STDOUT.sync = true
+    end
+
+    # Attempts to reopen an IO object.
+    def reopen_io(io, path, mode = nil)
+      begin
+        if mode
+          io.reopen(path, mode)
+        else
+          io.reopen(path)
+        end
+        io.binmode
+      rescue ::Exception
+      end
+    end
+  end
+end
+

data/lib/revenant/pid.rb

@@ -0,0 +1,24 @@
+module Revenant
+  class PID
+    def initialize(file)
+      @file = file
+    end
+
+    def exists?
+      File.exists?(@file)
+    end
+
+    def create
+      return false if exists?
+
+      File.open(@file, 'w') do |f|
+        f.write(Process.pid)
+      end
+      true
+    end
+
+    def remove
+      File.unlink(@file) if exists?
+    end
+  end
+end

data/lib/revenant/task.rb

@@ -0,0 +1,244 @@
+require 'revenant'
+require 'time'
+
+module Revenant
+  class Task
+    attr_reader :name
+    attr_accessor :options
+    attr_writer :logger
+
+    def initialize(name = nil)
+      unless String === name || Symbol === name
+        raise ArgumentError, "Usage: new(task_name)"
+      end
+      @name = name.to_sym
+      @options = {}
+    end
+
+    # Generally overridden when Revenant::Daemon is included
+    def startup
+      log "#{name} is starting"
+      trap("INT") { shutdown_soon }
+    end
+
+    ## Generally overridden when Revenant::Daemon is included
+    # The stack gets deeper here on every restart; this is here
+    # largely to ease testing.
+    # Implement your own plugin providing +shutdown+ if you want to
+    # make something serious that calls this code after a
+    # restart signal.
+    def shutdown
+      if restart_pending? && @work
+        log "#{name} is restarting"
+        run(&@work)
+      else
+        log "#{name} is shutting down"
+      end
+    end
+
+    # Takes actual block of code that is to be guarded by
+    # the lock. The +run_loop+ method does the actual work.
+    #
+    # If 'daemon?' is true, your code (including +on_load+)
+    # will execute after a fork.
+    #
+    # Make sure you don't open files and sockets in the exiting
+    # parent process by mistake. Open them in code that is called
+    # via +on_load+.
+    def run(&block)
+      unless @work = block
+        raise ArgumentError, "Usage: run { while_we_have_the_lock }"
+      end
+      @shutdown = false
+      @restart = false
+      install_plugins
+      startup # typically daemonizes the process, can have various implementations
+      on_load.call(self) if on_load
+      run_loop(&@work)
+      on_exit.call(self) if on_exit
+      shutdown
+    end
+
+    # Code to run just before the task looks for a lock
+    # This code runs after any necessary forks, and is
+    # therefore the proper place to open databases, logfiles,
+    # and any other resources you require.
+    def on_load(&block)
+      @on_load ||= block
+    end
+
+    # Code to run when the task is exiting.
+    def on_exit(&block)
+      @on_exit ||= block
+    end
+
+    # Used to pick the Task's +lock_module+
+    # Particular lock types may offer various helpful features
+    # via this lock module.
+    # Defaults to :mysql
+    def lock_type
+      @lock_type ||= :mysql
+    end
+
+    # Set a new lock type for this Task.
+    def lock_type=(val)
+      @lock_type = val.to_sym
+    end
+
+    # Set your own lock function. Will be called with a lock name as the arg.
+    # Should return true if a lock has been acquired, false otherwise.
+    # task.lock_function {|name| # .. }
+    def lock_function(&block)
+      if block_given?
+        @lock_function = block
+      else
+        @lock_function ||= lock_module.lock_function
+      end
+    end
+
+    # Returns a module that knows how to do some distributed locking.
+    # May not be the code that actually performs the lock, if this
+    # Task has had a +lock_function+ assigned to it explicitly.
+    def lock_module
+      ::Revenant.find_module(lock_type)
+    end
+
+    # How many work loops to perform before re-acquiring the lock.
+    # Defaults to 5.
+    # Setting it to 0 or nil will assume the lock is forever valid after
+    # acquisition.
+    def relock_every
+      @relock_every ||= 5
+    end
+
+    # Set the frequency with which locks are re-acquired.
+    # Setting it to 0 or nil will assume the lock is forever valid after
+    # acquisition.
+    def relock_every=(loops)
+      loops ||= 0
+      if Integer === loops && loops >= 0
+        @relock_every = loops
+      else
+        raise ArgumentError, "argument must be nil or an integer >= 0"
+      end
+    end
+
+    # How many seconds to sleep after each work loop.
+    # When we don't have the lock, how long to sleep before checking again.
+    # Default is 5 seconds.
+    def sleep_for
+      @sleep_for ||= 5
+    end
+
+    # Set the number of seconds to sleep for after a work loop.
+    def sleep_for=(seconds)
+      seconds ||= 0
+      if Integer === seconds && seconds >= 0
+        @sleep_for = seconds
+      else
+        raise ArgumentError, "argument must be nil or an integer >= 0"
+      end
+    end
+
+    # This could be the moment.
+    def shutdown_pending?
+      @shutdown ||= false
+    end
+
+    # At last, back to war.
+    def restart_pending?
+      @restart ||= false
+    end
+
+    # Task will restart at the earliest safe opportunity after
+    # +restart_soon+ is called.
+    def restart_soon
+      @restart = true
+      @shutdown = true
+    end
+
+    # Task will shut down at the earliest safe opportunity after
+    # +shutdown_soon+ is called.
+    def shutdown_soon
+      @restart = false
+      @shutdown = true
+    end
+
+    ## Used to lazily store/retrieve options that may be needed by plugins.
+    # We may want to capture, say, +log_file+ before actually loading the
+    # code that might care about such a concept.
+    def method_missing(name, *args)
+      name = name.to_s
+      last_char = name[-1,1]
+      super(name, *args) unless last_char == "=" || last_char == "?"
+      attr_name = name[0..-2].to_sym # :foo for 'foo=' or 'foo?'
+      if last_char == "="
+        @options[attr_name] = args.at(0)
+      else
+        @options[attr_name]
+      end
+    end
+
+    def log(message)
+      logger.puts "[#{$$}] #{Time.now.iso8601(2)} - #{message}"
+    end
+
+    def error(message)
+      logger.puts "[#{$$}] #{Time.now.iso8601(2)} - ERROR: #{message}"
+    end
+
+    def logger
+      @logger ||= STDERR
+    end
+
+    # Install any plugins that have registered themselves, or a custom
+    # list if the user has set it themselves.
+    def install_plugins
+      ::Revenant.plugins.each do |name, plugin|
+        plugin.install(self)
+      end
+    end
+
+    # Run until we receive a shutdown/reload signal,
+    # or when the worker raises an Interrupt.
+    # Runs after a fork when Revenant::Daemon is enabled.
+    def run_loop(&block)
+      acquired = false
+      begin
+        until shutdown_pending?
+          # The usual situation
+          if relock_every != 0
+            i ||= 0
+            # 0 % anything is 0, so we always try to get the lock on the first loop.
+            if (i %= relock_every) == 0
+              acquired = lock_function.call(@name)
+            end
+          else
+            # With relock_every set to 0, only acquire the lock once.
+            # Hope you're sure that lock beongs to you.
+            acquired ||= lock_function.call(@name)
+            i = 0 # no point in incrementing something we don't check.
+          end
+
+          yield if acquired
+
+          # Sleep one second at a time so we can quickly respond to
+          # shutdown requests.
+          sleep_for.times do
+            sleep(1) unless shutdown_pending?
+          end
+          i += 1
+        end # loop
+      rescue ::Interrupt => ex
+        log "shutting down after interrupt: #{ex.class} - #{ex.message}"
+        shutdown_soon # Always shut down from an Interrupt, even mid-restart.
+        return
+      rescue ::Exception => ex
+        error "restarting after error: #{ex.class} - #{ex.message}"
+        error "backtrace: #{ex.backtrace.join("\n")}"
+        restart_soon # Restart if we run into an exception.
+      end # begin block
+    end # run_loop
+  end # Task
+end # Revenant
+

metadata

@@ -0,0 +1,50 @@
+--- !ruby/object:Gem::Specification
+name: revenant
+version: !ruby/object:Gem::Version
+  version: 0.0.2
+  prerelease: 
+platform: ruby
+authors:
+- Wilson Bilkovich
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2010-06-06 00:00:00.000000000 Z
+dependencies: []
+description: A framework for building reliable distributed workers.
+email: wilson@supremetyrant.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/locks/mysql.rb
+- lib/plugins/daemon.rb
+- lib/revenant/manager.rb
+- lib/revenant/pid.rb
+- lib/revenant/task.rb
+- lib/revenant.rb
+homepage: http://github.com/wilson/revenant
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: 1.3.7
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.24
+signing_key: 
+specification_version: 2
+summary: Distributed daemons that just will not die.
+test_files: []