updater 0.9.4 → 0.10.0

data/Rakefile

@@ -1,7 +1,6 @@
 require 'rubygems'
 require 'rake/gempackagetask'
-gem 'rspec', '=1.3.0'
-require 'spec/rake/spectask'
+require 'rspec/core/rake_task'
 
 
 VERSION_FILE = File.join(File.dirname(__FILE__), 'VERSION')
@@ -26,7 +25,7 @@ spec = Gem::Specification.new do |s|
   s.email = EMAIL
   s.homepage = HOMEPAGE
   s.add_development_dependency('datamapper', '>= 0.10.2')
-  s.add_development_dependency('rspec', '=1.3.0')
+  s.add_development_dependency('rspec', '>= 2.0.0')
   s.add_development_dependency('timecop', '>= 0.2.1')
   s.add_development_dependency('chronic', '>= 0.2.3')
   s.require_path = 'lib'
@@ -40,21 +39,21 @@ Rake::GemPackageTask.new(spec) do |pkg|
   pkg.gem_spec = spec
 end
 
-Spec::Rake::SpecTask.new do |t|
-  t.warning = false
+RSpec::Core::RakeTask.new do |t|
+  ruby_opts="-w"
   t.rcov = false
 end
 
-Spec::Rake::SpecTask.new do |t|
+RSpec::Core::RakeTask.new do |t|
   t.name="failing"
   #todo Make this run only failing specs
-  t.warning = false
+  ruby_opts="-w"
   t.rcov = false
 end
 
-Spec::Rake::SpecTask.new do |t|
+RSpec::Core::RakeTask.new do |t|
   t.name="rcov"
-  t.warning = false
+  ruby_opts="-w"
   t.rcov = true
 end
 

data/VERSION

@@ -1 +1 @@
-0.9.4
+0.10.0

data/lib/updater/fork_worker.rb

@@ -231,7 +231,7 @@ module Updater
       
       def add_worker(worker_number)
         worker = WorkerMonitor.new(worker_number,Updater::Util.tempio)
-        Update.orm.before_fork
+        Update.orm.before_fork if Update.orm.respond_to? :before_fork
         pid = Process.fork do
           fork_cleanup
           self.new(@pipe,worker).run

data/lib/updater/orm/activerocord.rb

@@ -0,0 +1,23 @@
+module Updater
+  module ORm
+
+    class ActiveRecord
+      raise NoImplimentationError
+      def lock(worker)
+        return true if locked? && locked_by == worker.name
+        #all this to make sure the check and the lock are simultanious:
+        ccnt = self.class.where(id: self.id, lock: nil).update_all(:lock=>worker.name)
+        if 0 != cnt
+          @lock_name = worker.name
+          true
+        else
+          worker.say( "Worker #{worker.name} Failed to aquire lock on job #{id}" )
+          false
+        end
+      end
+      
+    end
+    
+  end
+  
+end

data/lib/updater/orm/datamapper.rb

@@ -1,5 +1,6 @@
 require "dm-core"
 require "dm-types"
+require "dm-migrations"
 
 module Updater
   module ORM
@@ -31,6 +32,10 @@ module Updater
       
       has n, :chains, :model=>'Updater::ORM::DMChained', :child_key=>[:caller_id]
       
+      def method
+        self[:method]
+      end
+      
       #attempt to lock this record for the worker
       def lock(worker)
         return true if locked? && locked_by == worker.name
@@ -55,28 +60,33 @@ module Updater
         define_method "#{mode}=" do |chain|
           case chain
             when self.class
-              chains.new(:target=>chain,:occasion=>mode)
+              self.chains.new(:target=>chain,:occasion=>mode)
             when Updater::Update
-              chains.new(:target=>chain.orm,:occasion=>mode)
+              self.chains.new(:target=>chain.orm,:occasion=>mode)
             when Hash
               chain.each do |target, params|
                 target = target.orm if target.kind_of? Updater::Update
-                chains.new(:target=>target,:params=>params, :occasion=>mode)
+                self.chains.new(:target=>target,:params=>params, :occasion=>mode)
               end
             when Array
               chain.each do |target|
+                target, params = target
                 target = target.orm if target.kind_of? Updater::Update
-                chains.new(:target=>target,:occasion=>mode)
+                self.chains.new(:target=>target,:occasion=>mode, :params=>params)
               end
+            when Integer
+              self.chains.new(:target_id=>chain,:occasion=>mode)
             when nil
-              chains=[]
+              self.chains=[]
             else
-              raise ArgumentError
+              raise ArgumentError, "Cannot add #{chain.inspect} to a chain (%s:%s)" % [__FILE__,__LINE__]
           end
+          save
+          chain
         end
 
         define_method mode do
-          chains.all(:occasion=>mode).map {|job| Updater.new(i.target).tap {|u| u.params = job.params}}
+          chains.all(:occasion=>mode).map {|job| Update.new(job.target).tap {|u| u.params = job.params}}
         end
       end
 
@@ -145,7 +155,7 @@ module Updater
           search = all(
               :target=>mytarget,
               :finder=>myfinder,
-              :finder_args=>myfinder_args, 
+              :finder_args=>myfinder_args.to_yaml, 
               :lock_name=>nil
             )
           myname ? search.all(:name=>myname ) : search
@@ -154,7 +164,17 @@ module Updater
         #For the server only, setup the connection to the database
         def setup(options)
           ::DataMapper.logger = options.delete(:logger)
+          auto_migrate = options.delete(:auto_migrate)
           ::DataMapper.setup(:default,options)
+          ::DataMapper.auto_migrate! if auto_migrate
+        end
+        
+        def logger
+          ::DataMapper.logger
+        end
+        
+        def logger=(input)
+          ::DataMapper.logger = input
         end
         
         # For pooled connections it is necessary to empty the pool of the parents connections so that they

data/lib/updater/orm/mock.rb

@@ -0,0 +1,155 @@
+module Updater
+  module ORM
+    class Mock
+      
+      FINDER = :find
+      
+      ID = :id
+      
+      class << self
+        attr_accessor :logger
+        
+        def index
+          @index ||= 0
+          @index += 1
+        end
+        
+        def get(id)
+          storage[id]
+        end
+        
+       def create(hash)
+          new(hash).tap {|n| storage[n.id] = n}
+        end
+        
+        def current
+          storage.values.find_all{|x|
+            x.time && x.time <= tnow && !x.lock_name 
+          }.sort{|a,b| a.time <=> b.time}
+        end
+        
+        def current_load
+          current.length
+        end
+        
+        def _delayed
+          storage.values.find_all{|x|
+            x.time && x.time > tnow
+          }.sort{|a,b| a.time <=> b.time}
+        end
+        
+        def delayed
+          _delayed.length
+        end
+        
+        def future(start, finish)
+          _delayed.find_all{|x| x.time >= start+tnow && x.time < finish+tnow}
+        end
+        
+        def queue_time
+          return 0 unless current.empty?
+          return nil if (d = _delayed).empty? #tricky assignment in conditional
+          d.first.time - tnow
+        end
+        
+        def lock_next(worker)
+          job = current.first
+          job.lock(worker) if job
+          job
+        end
+        
+        def clear_locks(worker)
+          storage.values.each{|x| x.lock_name = nil if x.lock_name == worker.name}
+        end
+        
+
+        def clear_all
+          @storage = {}
+        end
+
+        def setup(options)
+          @storage = {} 
+        end
+        
+        def for(target, finder, finder_args, name=nil)
+          @storage.values.find_all do |x|
+            naming = name ? x.name == name : true
+            naming && x.finder == finder && x.finder_args == finder_args && x.target == target && !x.lock_name
+          end
+        end
+        
+        def storage
+          @storage ||= {}
+        end
+        
+        private
+        def tnow
+          Updater::Update.time.now.to_i
+        end
+      end #class << self
+      
+      attr_reader :id
+      
+      attr_accessor :time, :target, :finder, :finder_args, :method, :method_args, :name, :lock_name, :persistant
+      
+      def initialize(hash = {})
+        @id = self.class.index
+        hash.each do |k,v| 
+          self.send("#{k}=",v)
+        end
+      end
+      
+      def lock(worker)
+        return false if @lock_name && @lock_name != worker.name
+        @lock_name = worker.name
+      end
+      
+      def save
+        self.class.storage[id] = self
+      end
+      
+      def destroy
+        self.class.storage.delete(id)
+      end
+      
+      %w{failure success ensure}.each do |mode|
+        eval(<<-EOF, binding ,__FILE__, __LINE__+1)
+          def #{mode}
+            @#{mode} ||= []
+          end
+          
+          def #{mode}=(chain)
+            return @#{mode} = nil if chain.nil?
+            @#{mode} ||= []
+            mchain = chain.kind_of?(Array) ? chain : [chain]
+            @#{mode} += mchain.map { |x| rationalize_instance(x) }.flatten
+            # attach_intellegent_insertion(@#{mode},:#{mode},self) if @#{mode}
+            chain
+          end
+        EOF
+      end
+      
+      private
+      
+      def rationalize_instance(val)
+        case val
+          when Updater::Update
+            val
+          when self.class
+            Updater::Update.new(val)
+          when Integer
+            Updater::Update.new(self.class.storage[val])
+          when Hash
+            val.map do |target, params|
+              rationalize_instance(target).tap{|u| u.params = params}
+            end
+          when Array #an array within an array
+            rationalize_instance(val[0]).tap {|u| u.params = val[1]}
+          else
+            raise ArgumentError, "Cannot add #{val.inspect} to a chain (%s:%s)" % [__FILE__,__LINE__]
+        end
+      end
+      
+    end
+  end
+end

data/lib/updater/orm/mongo.rb

@@ -71,7 +71,16 @@ module Updater
       end
       
       def lock(worker)
-        raise NotImplimentedError, "Use lock_next"
+          return true if @lock_name && @lock_name == worker.name
+          hash = Hash.new
+          hash['findandmodify'] = self.class.collection.name
+          hash['query'] = {:_id=>id,:lock_name=>nil}
+          hash['update'] = {'$set'=>{:lock_name=>worker.name}}
+          hash['new'] = true
+
+          ret = self.class.db.command hash, :check_response=>false
+          @lock_name = worker.name if ret['ok'] == 1
+          return ret['ok'] == 1
       end
       
       # Non API Standard.  This method returns the collection used by this instance.
@@ -97,23 +106,32 @@ module Updater
           end
           
           def #{mode}=(chain)
-            chain = [chain] unless chain.kind_of? Array
-            @#{mode} , @hash[:#{mode}]  = build_chain_arrays(chain)
+            return @#{mode} = @hash[:#{mode}] = nil if chain.nil?
+            mchain = chain.kind_of?(Array) ? chain : [chain]
+            inst, hsh = build_chain_arrays(mchain)
+            @#{mode} = (@#{mode} || [] ) + inst if inst
+            @hash[:#{mode}] = (@hash[:#{mode}] || []) + hsh
             attach_intellegent_insertion(@#{mode},:#{mode},self) if @#{mode}
+            chain
           end
         EOF
       end
       
+      def ==(other)
+        other.class == self.class && id == other.id
+      end
+      
     private
-      # this method is calld from he chain asignment methods eg. failure=(chain) 
+      # This method is calleppd from he chain asignment methods eg. failure=(chain) 
       # chain is an array which may contain BSON::ObjectId's or Updater::Update's or both
-      # For BSON::ObjectId's we cannot initialize them as this could leed to infinate loops.
+      # For BSON::ObjectId's we cannot initialize them as this could lead to infinate loops.
       # (an object pool would solve this problem, but feels like overkill)
       # The final result must be a @hash containing all the BSON::ObjectId' (forign keys)
       # and possibly @failure containting all instanciated UpdaterUpdates read to be called
       # or @failure set to nil with the chain instanciated on first use.
       def build_chain_arrays(arr, build = false)
-        build ||= arr.any? {|j| k,_ = j; Updater::Update === k || Hash === k}
+        arr = arr[0].to_a if arr[0].is_a? Hash
+        build ||= arr.any? {|j| k,_ = j; Updater::Update === k || self.class === k}
         output = arr.inject({:ids=>[],:instances=>[]}) do |accl,j|
           inst, id = rationalize_instance(j)
           if inst.nil? && build
@@ -134,7 +152,7 @@ module Updater
       # This method takes something that may be a reference to an instance(BSON::ObjectId/String),
       # an instance its self (Updater::Update), or a Hash 
       # and returns a touple of the  Updater::Update,BSON::ObjectId.
-      # This method will bot instanciate object from BSON::ObjectId's
+      # This method will not instanciate object from BSON::ObjectId's
       # nor will it save Hashes inorder to obtain an ID (it will creat a new Updater::Update from the hash).
       # Instead it will return nil in the appropriate place.
       def rationalize_instance(val)
@@ -142,8 +160,10 @@ module Updater
         case val  #aval is the actual runable object, hval is a BSON::ObjectId that we can put into the Database
           when Updater::Update
             val.params ? [val,[val.id,val.params]] : [val,val.id]
-          when Hash
-            [Updater::Update.new(val),val['_id']]
+          #~ when Hash # I think this ought to be deleted outright, but I whant to pass the functional test before I delete it.
+            #~ [Updater::Update.new(val),val['_id']]
+          when self.class
+            [Updater::Update.new(val), val.id]
           when BSON::ObjectId
             [nil,val]
           when Array
@@ -228,11 +248,15 @@ module Updater
         
         def get(id)
           id = BSON::ObjectId.from_string(id) if id.kind_of? String
-          new(@collection.find_one(id))
+          inst = @collection.find_one(id)
+          inst && new(inst)
         end
         
         def current
-          raise NotImplementedError, "Mongo does not support lazy evaluation"
+          # raise NotImplementedError, "Mongo does not support lazy evaluation"
+          @collection.find(:time=>{'$lte'=>tnow}).map do |x|
+            new(x)
+          end
         end
         
         def current_load
@@ -268,12 +292,23 @@ module Updater
           @collection.update({:lock_name=>worker.name},{'$unset'=>{:lock_name=>1}},:multi=>true)
         end
         
-        def for(target,finder,args,name)
+        def for(target,finder,args,name=nil)
           if name
-            @collection.find(:target=>target.to_s, :finder_args=>args,:name=>name)
+            @collection.find(
+              :target=>target.to_s, 
+              :finder_args=>args, 
+              :finder=>finder, 
+              :lock_name=>nil,
+              :name=>name
+            )
           else
-            @collection.find(:target=>target.to_s, :finder_args=>args)
-          end
+            @collection.find(
+              :target=>target.to_s, 
+              :finder_args=>args, 
+              :finder=>finder, 
+              :lock_name=>nil,
+            )
+          end.map {|x| new(x) }
         end
         
         private

data/lib/updater/orm/orm.rb

@@ -30,7 +30,7 @@ module Updater
     # always in reference to Updater::Update.time (by default Time).  This value will be
     # nil for chained methods.
     #
-    # target [Class]: The class of the target for this job.  the API spesifies that it must be
+    # target [Class]: The class of the target for this job.  The API spesifies that it must be
     # a Ruby class currently in scope in both the workers' and clients' frames of reference.
     # (see configuration documentation for how to achieve this.)  The writer must accept an
     # actual class, which it may store in the datastore as a string (by calling to_s on it).  The
@@ -232,6 +232,8 @@ module Updater
       end
       
       def after_fork
+        
+      end
       
       # Optional, but strongly recomended.
       #

data/lib/updater/setup.rb

@@ -6,6 +6,7 @@ require 'erb'
 module Updater
   class Setup
     class << self
+      attr_accessor :init
       #start a new server
       def start(options={})
         new(config_file(options), options).start
@@ -27,6 +28,10 @@ module Updater
         new(config_file(options), options).client_setup
       end
       
+      def test_setup(options = {})
+        new(options).test_setup
+      end
+      
       # pendeing
       def monitor
         
@@ -83,14 +88,14 @@ module Updater
     end
     
     # The client is responcible for loading classes and making connections.  We will simply setup the Updater spesifics.
-    def client_setup
-      @logger.info "Updater Client is being initialized..."
+    def client_setup(init = true)
+      @logger.info "Updater Client is being initialized... (#{__FILE__}:#{__LINE__})"
+      @logger.debug "  Call Stack:\n - #{caller[0..3].join("\n - ")}"
       set_orm
       
       Updater::Update.socket = socket_for_client
       
-      
-      init_orm
+      init_orm if init
       
       #set PID
       if File.exists? @options[:pid_file]
@@ -101,6 +106,16 @@ module Updater
       self
     end
     
+    def test_setup
+      @options[:orm] ||= 'mock' 
+      set_orm
+      init_orm
+      
+      Updater::Update.socket = @options[:socket] ||= File.open('/dev/null','w')
+      
+      self
+    end
+    
     def socket_for_client
       if @options[:socket] && File.exists?(@options[:socket])
         @logger.debug "Using UNIX Socket \"#{@options[:socket]}\""
@@ -145,13 +160,15 @@ module Updater
           require 'updater/orm/activerecord'
           Updater::Update.orm = ORM::ActiveRecord
         else
-          require "update/orm/#{orm}"
-          Updater::Update.orm = Object.const_get("ORM").const_get(orm.capitalize)
+          require "updater/orm/#{orm}"
+          Updater::Update.orm = Updater::ORM.const_get(orm.capitalize)
       end
       @logger.info "Data store '#{orm}' selected"
     end
     
     def init_orm
+      return false if self.class.init
+      self.class.init = true
       default_options = {:adapter=>'sqlite3', :database=>'./default.db'}
       Updater::Update.orm.setup((@options[:database] || @options[:orm_setup] || default_options).merge(:logger=>@logger))
     end
@@ -206,7 +223,7 @@ module Updater
       #Log PID
       File.open(@options[:pid_file],'w') { |f| f.write(Process.pid.to_s)}
       
-      client_setup
+      client_setup(false)
       
       #start Worker
       worker = @options[:worker] || 'fork'  #todo make this line windows safe

data/lib/updater/thread_worker.rb

@@ -9,6 +9,7 @@ module Updater
   class ThreadWorker
     attr_accessor :pid
     attr_accessor :name
+    attr_accessor :logger
     
     def initialize(options={})
       @quiet = options[:quiet]
@@ -53,6 +54,10 @@ module Updater
       end
     end
     
+    def logger
+      @logger ||= Logger.new(nil)
+    end
+    
   private
   
     def run_job_loop

data/lib/updater/update.rb

@@ -53,14 +53,17 @@ module Updater
     # (or the recreation of an object) on EACH invocation.  Methods that need to refer to the target more then once should
     # take care to store this value locally after initial retreavel.
     def target
-      target = @orm.finder.nil? ? @orm.target : @orm.target.send(@orm.finder,@orm.finder_args)
+      target = @orm.finder.nil? ? @orm.target : @orm.target.send(@orm.finder,*@orm.finder_args)
       raise TargetMissingError, "Target missing --Class:'#{@orm.target}' Finder:'#{@orm.finder}', Args:'#{@orm.finder_args.inspect}'" unless target
       target
     end
     
     # orm_inst must be set to an instacne of the class Update.orm
     def initialize(orm_inst)
-      raise ArgumentError if orm_inst.nil? || !orm_inst.kind_of?(self.class.orm)
+      if orm_inst.nil? || !orm_inst.kind_of?(self.class.orm)
+        raise ArgumentError, 
+          "Update has been set to use %s but recieved a %s  (%s:%s)\n  recieved %s." % [self.class.orm.inspect, orm_inst.class.inspect, __FILE__,__LINE__-1,orm_inst.inspect]
+      end
       @orm = orm_inst
     end
     
@@ -80,7 +83,7 @@ module Updater
     end
     
     def ==(other)
-      id = other.id
+      other.kind_of?(self.class) && id == other.id 
     end
     
     # If this is true, the job will NOT be removed after it is run.  This is usually true for chained Jobs.
@@ -172,7 +175,7 @@ module Updater
       #
       # MongoDB is one significant exception to this rule.  The Updater Mongo ORM layer uses the
       # 10gen MongoDB dirver directly without an ORM such as Mongoid or Mongo_Mapper.  If the
-      # application uses ond of thes ORMs #finder_method and #finder_id should be explicitly set.
+      # application uses one of thes ORMs #finder_method and #finder_id should be explicitly set.
       attr_accessor :finder_method
       
       # This is the application level default method to call on an instance type target.  It  should 
@@ -480,7 +483,7 @@ module Updater
       
       # Given some instance return the information needed to recreate that target 
       def target_for(inst,options = {})
-        return [inst, nil, nil] if (inst.kind_of?(Class) || inst.kind_of?(Module))
+        return [inst, options[:finder], options[:finder_args]] if (inst.kind_of?(Class) || inst.kind_of?(Module))
         [ inst.class, #target's class
           options[:finder] || @finder_method || orm::FINDER, #method to call on targets class to find/create target
           options[:finder_args] || inst.send(@finder_id || orm::ID) #value to pass to above method 

data/spec/chained_spec.rb

@@ -8,15 +8,16 @@ describe "Chained Methods:" do
   
   before :each do
     Update.clear_all
-    Foo.all.destroy!
+    Foo.reset
     @u = Update.chain(Foo,:chained,[:__job__,:__params__])
     @v = Update.chain(Foo,:chained2,[:__job__,:__params__])
+    #pending "Chained Worker not implimented in datamapper,  Waiting form ORM code refactor"
   end
   
   [:failure, :success, :ensure].each do |mode|
     specify "adding '#{mode.to_s}' chain" do
       v = Update.immidiate(Foo,:method1,[],mode=>@u)
-      v.orm.send(mode).should_not be_empty
+      v.send(mode).should_not be_empty
     end
   end
   

data/spec/datamapper_orm_spec.rb

@@ -0,0 +1,7 @@
+require File.join( File.dirname(__FILE__),  "spec_helper" )
+require 'updater/orm/datamapper'
+
+describe Updater::ORM::DataMapper do
+  it_behaves_like "an orm", :adapter=>'sqlite3',:database=>':memory:',:auto_migrate=>true
+  
+end

data/spec/errors_spec.rb

@@ -6,7 +6,7 @@ require  File.join( File.dirname(__FILE__),  "fooclass" )
 
 describe "Job Error Handeling" do
   before(:each) do
-    Foo.all.destroy!
+    Foo.reset
   end
   
   it "should return false when run" do