updater 0.9.2 → 0.9.3

data/VERSION

@@ -1 +1 @@
-0.9.2
+0.9.3

data/bin/updater

@@ -1,18 +1,111 @@
 #!/usr/bin/env ruby
 
-puts "starting update deamon..."
+require 'optparse'
 
-require 'rubygems'
-require 'yaml'
-require 'updater'
-equire 'updater/worker'
+OPERATIONS = %w{start stop noop} # restart}
+WORKERS = %w{fork thread}
 
+options = {}
 
-dbconfig = YAML.load_file('config/database.yml')
+opts = OptionParser.new do |opts|
+  opts.banner = <<EOF
+Updater: Ruby Job Processor. Copyright John F. Miller 2009-2010  
 
-DataMapper.setup(dbconfig['development'])
+Usage: updater (#{OPERATIONS.join('|')}) [options]"
+  Start of stop the Updater job queue processor.
+EOF
+  opts.on("-l","--local", "Use local 'lib' directory instead of Gems.") do
+    $LOAD_PATH << File.join(File.dirname(__FILE__), '../lib')
+  end
+  
+  opts.on("-c","--config-file [FILE]", "File containing configuration data.", "  default: ENV['UPDATE_CONFIG'] || updater.config") do |f|
+    options[:config_file] = f
+  end
+  
+  opts.on("--log-file [FILE]", "Where to send the log output.", "  default: STDOUT") do |f|
+    options[:log_file] = f
+  end
+  
+  opts.on("-d","--debug", "Output lots of extra data to the log file") do
+    options[:log_level] = "DEBUG"
+  end
+  
+  opts.on("-w","--worker WORKERTYPE", "(#{WORKERS.join('|')}) type of worker to use", "  default: fork (must be set to 'thread' for Windows)") do |w|
+    unless WORKERS.include? w
+      puts "** Invalid worker type **\n\n",opts 
+      exit -1 
+    end
+    options[:worker] = w
+  end
+  
+  opts.on("-m","--monitor", "Start/Stop the HTTP queue monitor", "  Not yet implimented") do  #TODO
+    puts "HTTP Monitor Not Yet Implimented."
+    exit  -1  
+  end
+  
+  opts.separator "\n  -- IPC Options: These options will need to be matched by the client --\n"
+  opts.on('-p','--pid-file FILE', 'The name of the PID file.') do |f|
+    options[:pid_file] = f
+  end
+  
+  opts.on('-s','--unix-socket FILE', 'Socket to be used for UNIX Socket IPC.') do |f|
+    options[:socket] = f
+  end
+  
+  opts.on('--host [HOST]', 'For UDC and TCP the host name. (See Security section in README)', '  default: localhost') do |h|
+    options[:host] = 'localhost'
+  end
+  
+  opts.on('-u','--udp PORT', 'Port to send/recieve UDP messages. (See Security section in README)') do |p|
+    options[:udp] = p
+  end
+  
+  opts.on('-t','--tcp PORT', 'Port to send/recieve tcp messages. (See Security section in README)') do |p|
+    options[:tcp] = p
+  end
+  
+  opts.on('-r','--remote-http PORT', 'Port to send/recieve HTTP service requests.', ' Not Yet Implimented') do |p|
+    options[:tcp] = p
+  end
+
+  
+  opts.separator "\n  -- Additional Information --\n"
+  opts.on_tail("-v", "--version", "Show Version information.") do
+    options[:version] = true
+  end
+  
+  opts.on_tail("-h", "--help", "show this message.") do
+    puts opts
+    exit
+  end
+end
+
+opts.parse!
+
+require 'updater' #must wait for -l option
+
+if options[:version]
+  puts(<<EOF)
+Updater: Ruby Job Processor. Version #{Updater::VERSION}
+  Copyright John F. Miller 2009-2010    
+EOF
+  exit
+end
+
+operation = ARGV.shift
+
+operation = operation.downcase if operation
+
+unless OPERATIONS.include? operation
+  puts "#{operation}: operation not supported" if operation
+  puts opts
+  exit
+end
+
+require 'updater/setup'
+
+Updater::Setup.send(operation,options)
 
-Worker.new.start
 
 
 

data/lib/updater/orm/datamapper.rb

@@ -76,7 +76,7 @@ module Updater
         end
 
         define_method mode do
-          chains.all(:occasion=>mode)
+          chains.all(:occasion=>mode).map {|job| Updater.new(i.target).tap {|u| u.params = job.params}}
         end
       end
 

data/lib/updater/orm/mongo.rb

@@ -0,0 +1,276 @@
+require 'mongo'
+require 'active_support/inflector' #to get classes from strings
+require 'active_support/core_ext/object/try'
+
+module Updater
+  module ORM
+    class Mongo
+            
+      FINDER= :get
+      ID=:_id
+      
+      def initialize(hash = {})
+        @hash = {}
+        hash.each do |key, val|
+          if respond_to? "#{key}="
+            send("#{key}=", val)
+          else
+            @hash[key] = val
+          end
+        end
+      end
+      
+      %w{time finder finder_args method method_args name persistant lock_name}.each do |field|
+        eval(<<-EOF)  #,__LINE__+1,__FILE__)
+          def #{field};@hash['#{field}'];end
+          def #{field}=(value);@hash['#{field}'] = value;end
+        EOF
+      end
+      
+      def _id
+        @hash['_id'] || @hash[:_id]
+      end
+      
+      def _id=(val)
+        val = BSON::ObjectID.from_string(val.to_s) unless val.kind_of? BSON::ObjectID
+        @hash[:_id] = val
+      end
+      
+      alias :id :_id
+      alias :'id=' :'_id='
+      
+      def target
+        @hash['target'].try :constantize
+      end
+      
+      def target=(value)
+        @hash['target'] = value.to_s
+      end
+      
+      def save
+        #todo validation
+        [:failure,:success,:ensure].each do |mode|
+          next unless @hash[mode]
+          @hash[mode] = @hash[mode].map do |job|
+            if job.kind_of? Updater::Update
+              job.save unless job.id
+              job = job.id
+            end
+            job
+          end
+        end
+        _id = self.class.collection.save @hash
+      end
+      
+      def destroy
+        self.class.collection.remove({:_id=>id})
+      end
+      
+      def [](arg)  #this allows easy mapping for time when a value coud either be U::ORM::Mongo or an ordered hash
+        @hash[arg]
+      end
+      
+      def lock(worker)
+        raise NotImplimentedError, "Use lock_next"
+      end
+      
+      # Non API Standard.  This method returns the collection used by this instance.
+      # This is used to create Job Chains
+      def collection
+        self.class.instance_variable_get(:@collection)
+      end
+
+      #key :time, Integer, :numeric=>true
+      #key :target, String, :required => true
+      #key :finder, String
+     # key :finder_args, Array
+      #key :method, String :required => true
+      #key :method_args, String :required => true
+      #key :name
+     # key :persistant
+      #key :lock_name
+      
+      %w{failure success ensure}.each do |mode|
+        eval(<<-EOF, binding ,__FILE__, __LINE__+1)
+          def #{mode}
+            @#{mode} ||= init_chain(:#{mode})
+          end
+          
+          def #{mode}=(chain)
+            chain = [chain] unless chain.kind_of? Array
+            @#{mode} , @hash[:#{mode}]  = build_chain_arrays(chain)
+            attach_intellegent_insertion(@#{mode},:#{mode},self) if @#{mode}
+          end
+        EOF
+      end
+      
+    private
+      # this method is calld from he chain asignment methods eg. failure=(chain) 
+      # chain is an array which may contain BSON::ObjectID's or Updater::Update's or both
+      # For BSON::ObjectID's we cannot initialize them as this could leed to infinate loops.
+      # (an object pool would solve this problem, but feels like overkill)
+      # The final result must be a @hash containing all the BSON::ObjectID' (forign keys)
+      # and possibly @failure containting all instanciated UpdaterUpdates read to be called
+      # or @failure set to nil with the chain instanciated on first use.
+      def build_chain_arrays(arr, build = false)
+        build ||= arr.any? {|j| k,_ = j; Updater::Update === k || Hash === k}
+        output = arr.inject({:ids=>[],:instances=>[]}) do |accl,j|
+          inst, id = rationalize_instance(j)
+          if inst.nil? && build
+            real_id,params = id #id could be an array
+            inst = Updater::Update.new(self.class.new(collection.find_one(real_id)))
+            inst.params = params #set params to the second element of the id array.  If id is scale then  set to nil.
+          end
+          accl[:ids] << id || inst #id will be nil only if inst has not ben saved.
+          accl[:instances] << inst if inst
+          accl
+        end
+        if build
+          return [output[:instances],output[:ids]]
+        end
+        [nil,output[:ids]]
+      end
+      
+      # This method takes something that may be a reference to an instance(BSON::ObjectID/String),
+      # an instance its self (Updater::Update), or a Hash 
+      # and returns a touple of the  Updater::Update,BSON::ObjectID.
+      # This method will bot instanciate object from BSON::ObjectID's
+      # nor will it save Hashes inorder to obtain an ID (it will creat a new Updater::Update from the hash).
+      # Instead it will return nil in the appropriate place.
+      def rationalize_instance(val)
+        val = BSON::ObjectID.fron_string(val) if val.kind_of? String
+        case val  #aval is the actual runable object, hval is a BSON::ObjectID that we can put into the Database
+          when Updater::Update
+            val.params ? [val,[val.id,val.params]] : [val,val.id]
+          when Hash
+            [Updater::Update.new(val),val['_id']]
+          when BSON::ObjectID
+            [nil,val]
+          when Array
+            rationalize_instance(val[0]).tap do |ret| 
+              ret[0].params = val[1] if ret[0]
+              ret[1] = [ret[1],val[1]]
+            end
+        end  
+      end
+      
+      def attach_intellegent_insertion(arr,mode,parent)
+        arr.define_singleton_method '<<' do |val|
+          inst, id = rationalize_instance(val)
+          inst = Updater::Update.new(self.class.new(parent.collection.find_one(id))) unless inst
+          parent.instance_variable_get(:@hash)[mode] ||= []
+          parent.instance_variable_get(:@hash)[mode] << id || inst
+          super inst
+        end
+        arr
+      end
+      
+      def init_chain(mode)
+        ret, @hash[mode] = build_chain_arrays(@hash[mode] || [],true)
+        attach_intellegent_insertion(ret,mode,self)
+      end
+      
+      class << self
+        attr_accessor :db, :collection, :logger
+        
+        # Availible options:
+        # * :database - *required* either an established Mongo::DB database OR the name of the database 
+        # * :collection - which collection to store jobs in. Default: "updater"
+        #
+        # If a connection to the database must be established (ie :database is not a Mongo::DB)
+        # these options may be used to establish that connection.
+        # * :host - the host to connect to.  Default: "localhost"
+        # * :port - the port to connect to.  Default: 27017
+        # * :username/:password - if these are present, they will be used to authenticate against the database
+        def setup(options)
+          logger ||= options[:logger]
+          raise ArgumentError, "Must spesify the name of a databas when setting up Mongo driver" unless options[:database]
+          if options[:database].kind_of? ::Mongo::DB
+            @db = options[:database]
+          else
+            logger.info "Attempting to connect to mongodb at #{[options[:host] || "localhost", options[:port] || 27017].join(':')} database: \"#{options[:database]}\""
+            @db = ::Mongo::Connection.new(options[:host] || "localhost", options[:port] || 27017).db(options[:database].to_s)
+            if options[:username] && options[:password]
+              success = db.authenticate(options[:username] , options[:password])
+              raise RunTimeError, "Could not Authenticate with MongoDb \"#{options[:database]}\" Please check the username and password."
+            end
+          end
+          collection_name = options[:collection] || 'updater'
+          unless db.collection_names.include? collection_name
+            logger.warn "Updater MongoDB Driver is creating a new collection, \"#{collection_name}\" in \"#{options[:database]}\""
+          end
+          @collection = db.collection(collection_name)
+        end
+        
+        def before_fork
+          @db.connection.close
+        end
+        
+        def after_fork
+          
+        end
+        
+        def lock_next(worker)
+          hash = Hash.new
+          hash['findandmodify'] =@collection.name
+          hash['query'] = {:time=>{'$lte'=>tnow},:lock_name=>nil}
+          hash['sort'] =[[:time,'ascending']] #oldest first
+          hash['update'] = {'$set'=>{:lock_name=>worker.name}}
+          hash['new'] = true
+          
+          
+          ret = @db.command hash, :check_response=>false
+          return nil unless ret['ok'] == 1
+          return new(ret['value'])
+        end 
+        
+        def get(id)
+          id = BSON::ObjectID.from_string(id) if id.kind_of? String
+          new(@collection.find_one(id))
+        end
+        
+        def current
+          raise NotImplementedError, "Mongo does not support lazy evaluation"
+        end
+        
+        def current_load
+          @collection.find(:time=>{'$lte'=>tnow}, :lock_name=>nil).count
+        end
+        
+        def delayed
+          @collection.find(:time=>{'$gt'=>tnow}).count
+        end
+        
+        def future(start, finish)
+          @collection.find(:time=>{'$gt'=>start+tnow,'$lt'=>finish+tnow}).count
+        end
+        
+        def queue_time
+          nxt = @collection.find_one({:lock_name=>nil, :time=>{'$ne'=>nil}}, :sort=>[[:time, :asc]], :fields=>[:time])
+          return nil unless nxt
+          return 0 if nxt['time'] <= tnow
+          return nxt['time'] - tnow
+        end
+        
+        def create(hash)
+          ret = new(hash)
+          ret.save and ret
+        end
+        
+        def clear_all
+          @collection.remove
+        end
+        
+        def clear_locks(worker)
+          @collection.update({:lock_name=>worker.name},{'$unset'=>{:lock_name=>1}},:multi=>true)
+        end
+        
+        private
+        def tnow
+          Updater::Update.time.now.to_i
+        end
+      end
+      
+    end
+  end
+end

data/lib/updater/orm/mongo.rb~

@@ -0,0 +1,188 @@
+require 'mongo'
+require 'active_support/inflector' #to get classes from strings
+require 'active_support/core_ext/object/try'
+
+module Updater
+  module ORM
+    class Mongo
+      
+      FINDER= :get
+      ID=:_id
+      
+      def initialize(hash = {})
+        @hash = {}
+        hash.each do |key, val|
+          if respond_to? "#{key}="
+            send("#{key}=", val)
+          else
+            @hash[key] = val
+          end
+        end
+      end
+      
+      %w{time finder finder_args method method_args name persistance lock_name}.each do |field|
+        eval(<<-EOF)  #,__LINE__+1,__FILE__)
+          def #{field};@hash['#{field}'];end
+          def #{field}=(value);@hash['#{field}'] = value;end
+        EOF
+      end
+      
+      def _id
+        @hash['_id'] || @hash[:_id]
+      end
+      
+      def _id=(val)
+        val = BSON::ObjectID.from_string(val.to_s) unless val.kind_of? BSON::ObjectID
+        @hash[:_id] = val
+      end
+      
+      alias :id :_id
+      alias :'id=' :'_id='
+      
+      def target
+        @hash['target'].try :constantize
+      end
+      
+      def target=(value)
+        @hash['target'] = value.to_s
+      end
+      
+      def save
+        #todo validation
+        self.class.collection.save @hash
+      end
+      
+      def destroy
+        @collection.remove({:_id=>id})
+      end
+      
+      def [](arg)  #this allows easy mapping for time when a value coud either be U::ORM::Mongo or an ordered hash
+        @hash[arg]
+      end
+
+      #key :time, Integer, :numeric=>true
+      #key :target, String, :required => true
+      #key :finder, String
+     # key :finder_args, Array
+      #key :method, String :required => true
+      #key :method_args, String :required => true
+      #key :name
+     # key :persistant
+      #key :lock_name
+      
+      %w{failure success ensure}.each do |mode|
+        eval(<<-EOF) #,__FILE__,__LINE__+1)
+          def #{mode}
+            @#{mode} ||= init_chain(:#{mode})
+          end
+        EOF
+      end
+      
+      def init_chain(mode)
+        ret = [@hash[mode.to_s] || []].flatten
+        unless ret.empty?
+          ret = @collection.find(:_id=>{'$in'=>ret}).map {|i| self.class.new(i)}
+        end
+        ret.define_singleton_method '<<' do |val|
+          val = BSON::ObjectID.fron_string(val) if val.kind_of? String
+          aval,hval = case val
+            when self.class
+              [val,val.id]
+            when Hash
+              [self.class.new(val),val['_id']]
+            when BSON::ObjectID
+              [@collection.find_one(val),val]
+          end
+          @hash[mode] ||= []
+          @hash[mode] << hval
+          super aval
+        end
+      end
+      
+      def lock(worker)
+        raise NotImplimentedError, "Use lock_next"
+      end
+      
+      class << self
+        attr_accessor :db, :collection, :logger
+        
+        # Availible options:
+        # * :database - the name of the database *required*
+        # * :host - the host to connect to.  Default: "localhost"
+        # * :port - the port to connect to.  Default: 27017
+        # * :collection - which collection to store jobs in. Default: "Updater"
+        # * :username/:password - if these are present, they will be used to authenticate against the database
+        def setup(options)
+          logger = options[:logger]
+          raise ArgumentError, "Must spesify the name of a databas when setting up Mongo driver" unless options[:database]
+          logger.info "Attempting to connect to mongodb at #{[options[:host] || "localhost", options[:port] || 27017}
+          @db = ::Mongo::Connection.new(options[:host] || "localhost", options[:port] || 27017).db(options[:database].to_s)
+          if options[:username] && options[:password]
+            success = db.authenticate(options[:username] , options[:password])
+            raise RunTimeError, "Could not Authenticate with MongoDb \"#{options[:database]}\" Please check the username and password."
+          end
+          collection_name = options[:collection] || 'updater'
+          unless db.collection_names.include collection_name
+            logger.warn "Updater MongoDB Driver is creating a new collection, \"#{collection_name}\" in \"#{options[:database]}\""
+          end
+          @collection = db.collection(collection_name)
+        end
+        
+        def lock_next(worker)
+          hash = OrderedHash.new
+          hash['findandmodify'] =@collection.name
+          hash['query'] = {:time=>{'$lte'=>tnow},:lock_name=>nil}
+          hash['sort'] =[[:time,'ascending']] #oldest first
+          hash['update'] = {'$set'=>{:lock_name=>worker.name}}
+          hash['new'] = true
+          
+          ret = @db.command hash
+          return nil unless ret['ok'] == 1
+          return new(ret['value'])
+        end 
+        
+        def get(id)
+          id = BSON::ObjectID.from_string(id) if id.kind_of? String
+          new(@collection.find_one(id))
+        end
+        
+        def current
+          raise NotImplementedError, "Mongo does not support lazy evaluation"
+        end
+        
+        def current_load
+          @collection.find(:time=>{'$lte'=>tnow}).count
+        end
+        
+        def delayed
+          @collection.find(:time=>{'$gt'=>tnow}).count
+        end
+        
+        def future(start, finish)
+          @collection.find(:time=>{'$gt'=>start+tnow,'$lt'=>finish+tnow}).count
+        end
+        
+        def queue_time
+          nxt = @collection.find_one({:time=>{'$gt'=>3,'$lt'=>4}, :lock_name=>'foobar'}, :sort=>[[:time, :asc]], :fields=>[:time])
+          return nil unless nxt
+          return 0 if nxt['time'] <= tnow
+          return nxt['time'] - tnow
+        end
+        
+        def create(hash)
+          new(hash).save
+        end
+        
+        def clear_all
+          @collection.remove
+        end
+        
+        private
+        def tnow
+          Updater::Update.time.now.to_i
+        end
+      end
+      
+    end
+  end
+end

data/lib/updater/orm/orm.rb

@@ -106,7 +106,7 @@ module Updater
     class Base
       
       # Every ORM should set this constant to a symbol that matches the most
-      # obvious method used to retrive a known instance.  :get or:find are likely
+      # obvious method used to retrive a known instance.  :get or :find are likely
       # candidates. 
       FINDER= nil
       

data/lib/updater/setup.rb

@@ -6,24 +6,30 @@ require 'erb'
 module Updater
   class Setup
     class << self
-      def start
-        new(config_file).start
+      def start(options={})
+        new(config_file(options), options).start
       end
       
-      def stop
-        new(config_file).stop
+      def stop(options={})
+        new(config_file(options), options).stop
+      end
+      
+      def noop(options={})
+        new(config_file(options), options).noop
       end
       
       def client_setup(options = {})
-        new(config_file, options).client_setup
+        new(config_file(options), options).client_setup
       end
       
       def monitor
         
       end
           
-      def config_file
-        if ENV['UPDATE_CONFIG'] && File.exists(ENV['UPDATE_CONFIG'])
+      def config_file(options = {})
+        if options[:config_file] && File.exists?(options[:config_file])
+          options[:config_file]
+        elsif ENV['UPDATE_CONFIG'] && File.exists(ENV['UPDATE_CONFIG'])
           ENV['UPDATE_CONFIG']
         else
           (Dir.glob('{config,.}/updater.config') + Dir.glob('.updater')).first
@@ -36,12 +42,14 @@ module Updater
     #extended used for clients who wnat to override parameters
     def initialize(file_or_hash, extended = {})
       @options = file_or_hash.kind_of?(Hash) ? file_or_hash : load_file(file_or_hash)
-      @options.merge(extended)
+      @options.merge!(extended)
       @options[:pid_file] ||= File.join(ROOT,'updater.pid')
       @options[:host] ||= "localhost"
       @logger = @options[:logger] || Logger.new(@options[:log_file] || STDOUT)
       level = Logger::SEV_LABEL.index(@options[:log_level].upcase) if @options[:log_level]
-      @logger.level = level || Logger::WARN 
+      @logger.level = level || Logger::WARN unless @options[:logger] #only set this if we were not handed a logger
+      @logger.debug "Debugging output enabled"
+      Update.logger = @logger
     end
     
     def start
@@ -54,13 +62,26 @@ module Updater
     
     def stop
       Process.kill("TERM",File.read(@options[:pid_file]).to_i)
+      sleep 1.0
+    end
+    
+    def noop
+      @logger.warn "NOOP: will not start service"
+      set_orm
+      init_orm
+      load_models
+      client_setup
+      @logger.debug @options.inspect
+      exit
     end
     
-    # The client is responcible for loading classes and making connections.  We will simply setup the Updater spesifics
+    # The client is responcible for loading classes and making connections.  We will simply setup the Updater spesifics.
     def client_setup
+      @logger.info "Updater Client is being initialized..."
       set_orm
       
       if @options[:socket] && File.exists?(@options[:socket])
+        @logger.debug "Using UNIX Socket \"#{@options[:socket]}\""
         Updater::Update.socket = UNIXSocket.new(@options[:socket])
       elsif @options[:udp]
         socket = UDPSocket.new()
@@ -86,13 +107,13 @@ module Updater
       #don't setup twice.  Client setup might call this as part of server setup in which case it is already done
       return false if Updater::Update.orm
       orm = @options[:orm] || "datamapper"
-      case orm.downcase
+      case orm.to_s.downcase
         when "datamapper"
           require 'updater/orm/datamapper'
           Updater::Update.orm = ORM::DataMapper
         when "mongodb"
-          require 'updater/orm/mongodb'
-          Updater::Update.orm = ORM::MongoDB
+          require 'updater/orm/mongo'
+          Updater::Update.orm = ORM::Mongo
         when "activerecord"
           require 'updater/orm/activerecord'
           Updater::Update.orm = ORM::ActiveRecord
@@ -103,18 +124,31 @@ module Updater
       @logger.info "Data store '#{orm}' selected"
     end
     
-    def _start
-      #set ORM
-      set_orm
-      #init DataStore
+    def init_orm
       default_options = {:adapter=>'sqlite3', :database=>'./default.db'}
       Updater::Update.orm.setup((@options[:database] || @options[:orm_setup] || default_options).merge(:logger=>@logger))
-      #load Models
-      
+    end
+    
+    def load_models
+      @logger.info "Loading Models..."
       models = @options[:models] || Dir.glob('./app/models/**/*.rb')
       models.each do |file|
-        require file
+        @logger.debug "  - loading file: #{file}"
+        begin
+          require file
+        rescue LoadError
+          require File.expand_path(File.join(Dir.pwd,file))
+        end
       end
+    end
+        
+    def _start
+      #set ORM
+      set_orm
+      #init DataStore
+      init_orm
+      #load Models
+      load_models
       
       #establish Connections
       @options[:host] ||= 'localhost'
@@ -164,7 +198,7 @@ module Updater
       @config_file = File.expand_path(file.path)
       YAML.load(ERB.new(file.read).result(binding)) || {}
     ensure
-      file.close
+      file.close if file.kind_of?(IO) && !file.closed?
     end
   end
-end
+end

data/lib/updater/setup.rb~

@@ -0,0 +1,175 @@
+require 'logger'
+require 'yaml'
+require 'socket'
+require 'erb'
+
+module Updater
+  class Setup
+    class << self
+      def start
+        new(config_file).start
+      end
+      
+      def stop
+        new(config_file).stop
+      end
+      
+      def client_setup(options = {})
+        new(config_file, options).client_setup
+      end
+      
+      def monitor
+        
+      end
+          
+      def config_file(options = {})
+        if options[:config_file] && File.exists(options[:config_file])
+          option[:config_file]
+        elsif ENV['UPDATE_CONFIG'] && File.exists(ENV['UPDATE_CONFIG'])
+          ENV['UPDATE_CONFIG']
+        else
+          (Dir.glob('{config,.}/updater.config') + Dir.glob('.updater')).first
+        end
+      end
+    end
+    
+    ROOT = File.dirname(self.config_file || Dir.pwd)
+    
+    #extended used for clients who wnat to override parameters
+    def initialize(file_or_hash, extended = {})
+      @options = file_or_hash.kind_of?(Hash) ? file_or_hash : load_file(file_or_hash)
+      @options.merge!(extended)
+      @options[:pid_file] ||= File.join(ROOT,'updater.pid')
+      @options[:host] ||= "localhost"
+      @logger = @options[:logger] || Logger.new(@options[:log_file] || STDOUT)
+      level = Logger::SEV_LABEL.index(@options[:log_level].upcase) if @options[:log_level]
+      @logger.level = level || Logger::WARN unless @options[:logger] #only set this if we were not handed a logger
+    end
+    
+    def start
+      pid = Process.fork do
+        _start
+      end
+      @logger.warn "Successfully started Master Loop at pid #{pid}"
+      puts "Job Queue Processor Started at PID: #{pid}"
+    end
+    
+    def stop
+      Process.kill("TERM",File.read(@options[:pid_file]).to_i)
+    end
+    
+    # The client is responcible for loading classes and making connections.  We will simply setup the Updater spesifics
+    def client_setup
+      @logger.info "Updater Client is being initialized..."
+      set_orm
+      
+      if @options[:socket] && File.exists?(@options[:socket])
+        @logger.debug "Using UNIX Socket \"#{@options[:socket]}\""
+        Updater::Update.socket = UNIXSocket.new(@options[:socket])
+      elsif @options[:udp]
+        socket = UDPSocket.new()
+        socket.connect(@options[:host],@options[:udp])
+        Updater::Update.socket = socket
+      elsif @options[:tcp]
+        Updater::Update.socket = TCPSocket.new(@options[:host],@options[:tcp])
+      elsif @options[:remote]
+        raise NotImplimentedError #For future Authenticated Http Rest Server
+      end
+      
+      #set PID
+      if File.exists? @options[:pid_file]
+        Updater::Update.pid = File.read(@options[:pid_file]).strip
+      end
+      
+      
+    end
+    
+    private
+    
+    def set_orm
+      #don't setup twice.  Client setup might call this as part of server setup in which case it is already done
+      return false if Updater::Update.orm
+      orm = @options[:orm] || "datamapper"
+      @logger.info "Updater setting ORM to \"#{orm}\""
+      case orm.to_s.downcase
+        when "datamapper"
+          require 'updater/orm/datamapper'
+          Updater::Update.orm = ORM::DataMapper
+        when "mongodb"
+          require 'updater/orm/mongo'
+          Updater::Update.orm = ORM::Mongo
+        when "activerecord"
+          require 'updater/orm/activerecord'
+          Updater::Update.orm = ORM::ActiveRecord
+        else
+          require "update/orm/#{orm}"
+          Updater::Update.orm = Object.const_get("ORM").const_get(orm.capitalize)
+      end
+      @logger.info "Data store '#{orm}' selected"
+    end
+    
+    def _start
+      #set ORM
+      set_orm
+      #init DataStore
+      default_options = {:adapter=>'sqlite3', :database=>'./default.db'}
+      Updater::Update.orm.setup((@options[:database] || @options[:orm_setup] || default_options).merge(:logger=>@logger))
+      #load Models
+      
+      models = @options[:models] || Dir.glob('./app/models/**/*.rb')
+      models.each do |file|
+        require file
+      end
+      
+      #establish Connections
+      @options[:host] ||= 'localhost'
+      #Unix Socket -- name at @options[:socket]
+      if @options[:socket]
+        File.unlink @options[:socket] if File.exists? @options[:socket]
+        @options[:sockets] ||= []
+        @options[:sockets] << UNIXServer.new(@options[:socket])
+        @logger.info "Now listening on UNIX Socket: #{@options[:socket]}"
+      end
+      
+      #UDP potentially unsafe user monitor server for Authenticated Connections (TODO)
+      if @options[:udp]
+        @options[:sockets] ||= []
+        udp = UDPSocket.new
+        udp.bind(@options[:host],@options[:udp])
+        @options[:sockets] << udp
+        @logger.info "Now listening for UDP: #{@options[:host]}:#{@options[:udp]}"
+      end
+      
+      #TCP Unsafe user monitor server for Authenticated Connections (TODO)
+      if @options[:tcp]
+        @options[:sockets] ||= []
+        @options[:sockets] << TCPServer.new(@options[:host],@options[:tcp])
+        @logger.info "Now listening for TCP: #{@options[:host]}:#{@options[:tcp]}"
+      end
+      
+      #Log PID
+      File.open(@options[:pid_file],'w') { |f| f.write(Process.pid.to_s)}
+      
+      client_setup
+      
+      #start Worker
+      worker = @options[:worker] || 'fork'  #todo make this line windows safe
+      require "updater/#{worker}_worker"
+      worker_class = Updater.const_get("#{worker.capitalize}Worker")
+      worker_class.logger = @logger
+      @logger.info "Using #{worker_class.to_s} to run jobs:"
+      worker_class.start(@options)
+      File.unlink(@options[:pid_file])
+      File.unlink @options[:socket] if @options[:socket] && File.exists?(@options[:socket])
+    end
+    
+    def load_file(file)
+      return {} if file.nil?
+      file = File.open(file) if file.kind_of?(String)
+      @config_file = File.expand_path(file.path)
+      YAML.load(ERB.new(file.read).result(binding)) || {}
+    ensure
+      file.close if file.kind_of?(IO) && !file.closed?
+    end
+  end
+end

data/lib/updater/thread_worker.rb

@@ -7,7 +7,6 @@ module Updater
 
   #This class repeatedly searches the database for active jobs and runs them
   class ThreadWorker
-    cattr_accessor :logger
     attr_accessor :pid
     attr_accessor :name
     
@@ -89,4 +88,4 @@ module Updater
     end
   end
   
-end
+end

data/lib/updater/thread_worker.rb~

@@ -0,0 +1,92 @@
+# This file based the file of the same name in the delayed_job gem by
+# Tobias Luetke (Coypright (c) 2005) under the MIT License.
+
+require 'benchmark'
+
+module Updater
+
+  #This class repeatedly searches the database for active jobs and runs them
+  class ThreadWorker
+    cattr_accessor :logger
+    attr_accessor :pid
+    attr_accessor :name
+    
+    def initialize(options={})
+      @quiet = options[:quiet]
+      @name = options[:name] || "host:#{Socket.gethostname} pid:#{Process.pid}" rescue "pid:#{Process.pid}"
+      @pid = Process.pid
+    end
+    
+    def start
+      say "*** Starting job worker #{@name}"
+      @t = run_job_loop
+      
+      trap('TERM') { terminate_with @t }
+      trap('INT')  { terminate_with @t }
+      
+      trap('USR1') do
+        old_proc = trap('USR1','IGNORE')
+        run_loop
+        trap('USR1',old_proc)
+      end
+      
+      Thread.pass
+     
+      sleep unless $exit
+    end
+    
+    def say(text)
+      puts text unless @quiet
+      logger.info text if logger      
+    end
+
+    def stop
+      raise RuntimeError unless @t
+      terminate_with @t
+    end
+    
+    def run_loop
+      if @t.alive?
+        @t.wakeup #calling run here is a Bad Idea
+      else
+        say " ~~ Restarting Job Loop"
+        @t = run_job_loop
+      end
+    end
+    
+  private
+  
+    def run_job_loop
+      Thread.new do
+        loop do
+          begin
+            delay = Update.work_off(self)
+            break if $exit
+            if delay 
+              sleep delay 
+            else
+              sleep
+            end
+            break if $exit
+          rescue
+            say "Caught exception in Job Loop"
+            sleep 0.1
+            retry
+          end
+        end
+        say "Worker thread exiting!"
+        Update.clear_locks(self)
+      end
+    end
+  
+    def terminate_with(t)
+      say "Exiting..."
+      $exit = true
+      t.run if t.alive?
+      say "Forcing Shutdown" unless status = t.join(15) #Nasty inline assignment
+      Update.clear_locks(self)
+      exit status ? 0 : 1
+    end
+  end
+  
+end

data/lib/updater/update.rb

@@ -7,13 +7,14 @@ module Updater
     # Contains the Error class after an error is caught in +run+. Not stored to the database
     attr_reader :error
     attr_reader :orm
+    attr_accessor :params
     
     #Run the action on this traget compleating any chained actions
-    def run(job=nil,params=nil)
+    def run(job=nil)
       ret = true #put return in scope
       begin
         t = target 
-        final_args = sub_args(job,params,@orm.method_args)
+        final_args = sub_args(job,@orm.method_args)
         t.send(@orm.method.to_sym,*final_args)
       rescue => e
         @error = e
@@ -24,7 +25,8 @@ module Updater
         run_chain :ensure
         begin
           @orm.destroy unless @orm.persistant
-        rescue DataObjects::ConnectionError
+        rescue StandardError => e
+          raise e unless e.class.to_s =~ /Connection/
           sleep 0.1
           retry
         end
@@ -43,6 +45,7 @@ module Updater
     end
     
     def initialize(orm_inst)
+      raise ArgumentError if orm_inst.nil?
       @orm = orm_inst
     end
     
@@ -75,14 +78,14 @@ module Updater
     
   private
       
-    def sub_args(job,params,a)
+    def sub_args(job,a)
       a.map do |e| 
         begin
           case e.to_s
             when '__job__'
               job
             when '__params__'
-              params
+              @params
             when '__self__'
               self
             else
@@ -102,7 +105,7 @@ module Updater
       chains = @orm.send(name)
       return unless chains
       chains.each do |job|
-        Update.new(job.target).run(self,job.params)
+        job.run(self)
       end
     rescue NameError
       puts @orm.inspect
@@ -123,13 +126,21 @@ module Updater
       # This is an open IO socket that will be writen to when a job is scheduled. If it is unset
       # then @pid is signaled instead.
       attr_accessor :socket
+      attr_writer :logger
+      
+      def logger
+        @logger ||= Logger.new(STDOUT)
+      end
       
       #Gets a single job form the queue, locks and runs it.  it returns the number of second
       #Until the next job is scheduled, or 0 is there are more current jobs, or nil if there 
       #are no jobs scheduled.
       def work_off(worker)
         inst = @orm.lock_next(worker)
-        new(inst).run if inst
+        if inst
+          worker.logger.debug "  running job #{inst.id}" 
+          new(inst).run
+        end
         @orm.queue_time
       ensure
         clear_locks(worker)
@@ -346,7 +357,7 @@ module Updater
       
       # Given some instance return the information needed to recreate that target 
       def target_for(inst)
-        return [inst, nil, nil] if inst.kind_of? Class
+        return [inst, nil, nil] if (inst.kind_of?(Class) || inst.kind_of?(Module))
         [inst.class,@orm::FINDER,inst.send(orm::ID)]
       end
       

data/spec/spec_helper.rb

@@ -3,11 +3,12 @@ require "rubygems"
 ROOT = File.join(File.dirname(__FILE__), '..')
 $LOAD_PATH << File.join(File.dirname(__FILE__), '../lib')
 
-require "spec" # Satisfies Autotest and anyone else not using the Rake tasks
+require "rspec" # Satisfies Autotest and anyone else not using the Rake tasks
 require "dm-core"
+require 'dm-migrations'
 
 require 'updater'
-require 'updater/thread_worker'
+#require 'updater/thread_worker'
 require 'updater/fork_worker'
 require 'updater/orm/datamapper'
 

data/spec/spec_helper.rb~

@@ -0,0 +1,23 @@
+require "rubygems"
+
+ROOT = File.join(File.dirname(__FILE__), '..')
+$LOAD_PATH << File.join(File.dirname(__FILE__), '../lib')
+
+require "rspec" # Satisfies Autotest and anyone else not using the Rake tasks
+require "dm-core"
+require  'dm-migrations'
+
+require 'updater'
+#require 'updater/thread_worker'
+require 'updater/fork_worker'
+require 'updater/orm/datamapper'
+
+Updater::Update.orm = Updater::ORM::DataMapper
+
+DataMapper.setup(:default, 'sqlite3::memory:')
+DataMapper.auto_migrate!
+
+require 'timecop'
+require 'chronic'
+
+

metadata

@@ -5,8 +5,8 @@ version: !ruby/object:Gem::Version
   segments: 
   - 0
   - 9
-  - 2
-  version: 0.9.2
+  - 3
+  version: 0.9.3
 platform: ruby
 authors: 
 - John F. Miller
@@ -14,13 +14,14 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-05-07 00:00:00 -07:00
+date: 2010-08-25 00:00:00 -07:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
   name: datamapper
   prerelease: false
   requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
@@ -35,6 +36,7 @@ dependencies:
   name: rspec
   prerelease: false
   requirement: &id002 !ruby/object:Gem::Requirement 
+    none: false
     requirements: 
     - - "="
       - !ruby/object:Gem::Version 
@@ -49,6 +51,7 @@ dependencies:
   name: timecop
   prerelease: false
   requirement: &id003 !ruby/object:Gem::Requirement 
+    none: false
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
@@ -63,6 +66,7 @@ dependencies:
   name: chronic
   prerelease: false
   requirement: &id004 !ruby/object:Gem::Requirement 
+    none: false
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
@@ -89,7 +93,9 @@ files:
 - Rakefile
 - VERSION
 - lib/updater.rb
+- lib/updater/thread_worker.rb~
 - lib/updater/setup.rb
+- lib/updater/setup.rb~
 - lib/updater/util.rb
 - lib/updater/update.rb
 - lib/updater/fork_worker.rb
@@ -97,12 +103,15 @@ files:
 - lib/updater/simulated.db
 - lib/updater/tasks.rb
 - lib/updater/thread_worker.rb
+- lib/updater/orm/mongo.rb~
 - lib/updater/orm/orm.rb
 - lib/updater/orm/datamapper.rb
+- lib/updater/orm/mongo.rb
 - spec/fork_worker_instance_spec.rb
 - spec/thread_worker_spec.rb
 - spec/schedule_spec.rb
 - spec/lock_spec.rb
+- spec/spec_helper.rb~
 - spec/params_sub_spec.rb
 - spec/chained_spec.rb
 - spec/fooclass.rb
@@ -124,6 +133,7 @@ rdoc_options: []
 require_paths: 
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
@@ -131,6 +141,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       - 0
       version: "0"
 required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
@@ -140,7 +151,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
 requirements: []
 
 rubyforge_project: 
-rubygems_version: 1.3.6
+rubygems_version: 1.3.7
 signing_key: 
 specification_version: 3
 summary: A Gem for queuing methods for later calling which is ORM Agnostic, and has advanced Error Handling