updater 0.2.2 → 0.3.0

data/lib/updater/update_dm.rb

@@ -0,0 +1,298 @@
+module Updater
+
+  #the basic class that drives updater
+  class Update
+    # Contains the Error class after an error is caught in +run+. Not stored to the database.
+    attr_reader :error
+    
+    include DataMapper::Resource
+    
+    property :id, Serial
+    property :target, Class
+    property :ident, Yaml
+    property :method, String
+    property :finder, String
+    property :args, Object, :lazy=>false
+    property :time, Integer
+    property :name, String
+    property :lock_name, String
+    
+    #will be called if an error occurs
+    belongs_to :failure, :model=>'Update', :child_key=>[:failure_id], :nullable=>true
+    
+    # Returns the Class or instance that will recieve the method call.  See +Updater.at+ for 
+    # information about how a target is derived.
+    def target
+      return @target if @ident.nil?
+      @target.send(@finder||:get, @ident)
+    end
+    
+    # Send the method with args to the target.
+    def run(job=nil)
+      t = target #do not trap errors here
+      final_args = job ? sub_args(job,args.dup) : args
+      begin
+        t.send(@method.to_sym,*final_args)
+      rescue => e
+        @error = e
+        failure.run(self) if failure
+        destroy unless nil == time
+        return false
+      end
+      destroy unless nil == time
+      true
+    end
+    
+    def sub_args(job,a)
+      a.map {|e| :__job__ == e ? job : e}
+    end
+    
+    #atempt to lock this record for the worker
+    def lock(worker)
+      return true if locked? && locked_by == worker.name
+      #all this to make sure the check and the lock are simultanious:
+      cnt = repository.update({properties[:lock_name]=>worker.name},self.class.all(:id=>self.id,:lock_name=>nil))
+      if 0 != cnt
+        @lock_name = worker.name
+        true
+      else
+        worker.say( "Worker #{worker.name} Failed to aquire lock on job #{id}" )
+        false
+      end
+    end
+    
+    def locked?
+      not @lock_name.nil?
+    end
+    
+    def locked_by
+      @lock_name
+    end
+    
+    #Like run but first aquires a lock for the worker.  Will return the result of run or nil
+    #if the record could not be locked
+    def run_with_lock(worker)
+      run if lock(worker)
+    end
+    
+    class << self
+      
+      # Request that the target be sent the method with args at the given time.
+      #
+      # == Parameters
+      # time <Integer | Object responding to to_i>,  by default the number of seconds sence the epoch.  
+      #What 'time'  references  can be set by sending the a substitute class to the time= method.
+      #
+      # target  <Class | instance> .  If target is a class then 'method' will be sent to that class (unless the 
+      # finder option is used.  Otherwise, the target will be assumed to be the result of 
+      # (target.class).get(target.id).  The finder method (:get by default) and the finder_args 
+      # (target.id by default) can be set in the options.  A DataMapper instance passed as the target
+      # will "just work."  Any object can be found in this mannor is known as a 'conforming instance'.
+      # 
+      # method <Symbol>.  The method that will be sent to the calculated target.
+      #
+      # args <Array> a list of arguments to be sent to with the method call.  Note: 'args' must be seirialiable
+      # with Marshal.dump.  Defaults to []
+      #
+      # options <Hash>  Addational options that will be used to configure the request.  see Options 
+      # section below.
+      #
+      # == Options
+      #
+      # :finder <Symbol> This method will be sent to the stored target class (either target or target.class) 
+      # inorder to extract the instance on which to preform the request.  By default :get is used.  For
+      # example to use on an ActiveRecord class 
+      #    :finder=>:find
+      #
+      # :finder_args <Array> | <Object>.  This is passed to the finder function.  By default it is 
+      # target.id.  Note that by setting :finder_args you will force Updater to calculate in instance
+      # as the computed target even if you pass a Class as the target.
+      #
+      # :name <String> A string sent by the requesting class to identify the request.  'name' must be 
+      # unique for a given computed target.  Names cannot be used effectivally when a Class has non-
+      # conforming instances as there is no way predict the results of a finder call.  'name' can be used
+      # in conjunction with the +for+ method to manipulate requests effecting an object or class after
+      # they are set.  See +for+ for examples
+      #
+      # :failure <Updater> an other request to be run if this request raises an error.  Usually the 
+      # failure request will be created with the +chane+ method.
+      #
+      # == Examples
+      #
+      #    Updater.at(Chronic.parse('tomorrow'),Foo,:bar,[]) # will run Foo.bar() tomorrow at midnight
+      #    
+      #    f = Foo.create
+      #    u = Updater.at(Chronic.parse('2 hours form now'),f,:bar,[]) # will run Foo.get(f.id).bar in 2 hours
+      def at(time,target,method,args=[],options={})
+        finder, finder_args = [:finder,:finder_args].map {|key| options.delete(key)}
+        hash = {:method=>method.to_s,:args=>args}
+        hash[:target] = target_for(target)
+        hash[:ident] = ident_for(target,finder,finder_args)
+        hash[:finder] = finder || :get
+        hash[:time] = time
+        ret = create(hash.merge(options))
+        Process.kill('USR2',pid) if pid
+        ret
+      rescue Errno::ESRCH
+        @pid = nil
+        puts "PID invalid"
+        #log this as well
+      end
+      
+      # like +at+ but with time as time.now.  Generally this will be used to run a long running operation in
+      # asyncronously in a differen process.  See +at+ for details
+      def immidiate(*args)
+        at(time.now,*args)
+      end
+      
+      # like +at+ but without a time to run.  This is used to create requests that run in responce to the 
+      # failure of other requests.  See +at+ for details
+      def chain(*args)
+        at(nil,*args)
+      end
+      
+      # Retrieves all updates for a conforming target possibly limiting the results to the named
+      # request.
+      #
+      # == Parameters
+      #
+      # target <Class | Object> a class or conforming object that postentially is the calculated target
+      # of a request.
+      #
+      # name(optional) <String>  If a name is sent, the first request with fot this target with this name
+      # will be returned.
+      #
+      # ==Returns
+      #
+      # <Array[Updater]> unless name is given then only a single [Updater] instance. 
+      def for(target,name=nil)
+        ident = ident_for(target)
+        target = target_for(target)
+        if name
+          first(:target=>target,:ident=>ident,:name=>name)
+        else
+          all(:target=>target,:ident=>ident)
+        end
+      end
+      
+      #The time class used by Updater.  See time= 
+      def time
+        @@time ||= Time
+      end
+      
+      # By default Updater will use the system time (Time class) to get the current time.  The application
+      # that Updater was developed for used a game clock that could be paused or restarted.  This method
+      # allows us to substitute a custom class for Time.  This class must respond with in interger or Time to
+      # the #now method.
+      def time=(klass)
+        @@time = klass
+      end
+      
+      #A filter for all requests that are ready to run, that is they requested to be run before or at time.now
+      def current
+        all(:time.lte=>time.now.to_i, :lock_name=>nil)
+      end
+      
+      #A filter for all requests that are not yet ready to run, that is time is after time.now
+      def delayed
+        all(:time.gt=>time.now.to_i)
+      end
+      
+      #how many jobs will happen in the next n seconds
+      def future(n)
+        ct = time.now.to_i
+        all(:time.gt=>ct,:time.lt=>ct+n)
+      end
+      
+      #Sets the process id of the worker process if known.  If this 
+      #is set then an attempt will be made to signal the worker any
+      #time a new update is made.
+      #
+      #If pid is not set, or is set to nil then the scheduleing program 
+      #is responcible for waking-up a potentially sleeping worker process
+      #in another way.
+      def pid=(p)
+        return @pid = nil unless p #tricky assignment in return
+        @pid = Integer("#{p}")
+        Process::kill 0, @pid
+        @pid
+      rescue Errno::ESRCH, ArgumentError
+        raise ArgumentError "PID was invalid"
+      end
+      
+      def pid
+        @pid
+      end
+      
+      ####################
+      # Worker Functions #
+      ####################
+      
+      #This returns a set of update requests.
+      #The first parameter is the maximum number to return (get a few other workers may be in compitition)
+      #The second optional parameter is a list of options to be past to DataMapper.
+      def worker_set(limit = 5, options={})
+        #TODO: add priority to this.
+        options = {:lock_name=>nil,:limit=>limit, :order=>[:time.asc]}.merge(options)
+        current.all(options)
+      end
+
+      #Gets a single job form the queue, locks and runs it.
+      def work_off(worker)
+        updates = worker_set
+        unless updates.empty?
+          #concept copied form delayed_job.  If there are a number of 
+          #different processes working on the queue, the niave approch
+          #would result in every instance trying to lock the same record.
+          #by shuffleing our results we greatly reduce the chances that
+          #multilpe workers try to lock the same process
+          updates = updates.to_a.sort_by{rand()}
+          updates.each do |u|
+            t = u.run_with_lock(worker)
+            break unless nil == t
+          end
+        end
+      rescue DataObjects::ConnectionError
+        sleep 0.1
+        retry
+      ensure
+        clear_locks(worker)
+        return queue_time
+      end
+      
+      def queue_time
+        nxt = self.first(:time.not=>nil,:lock_name=>nil, :order=>[:time.asc])
+        return nil unless nxt
+        return 0 if nxt.time <= time.now.to_i
+        return nxt.time - time.now.to_i
+      end
+      
+      def clear_locks(worker)
+        all(:lock_name=>worker.name).update(:lock_name=>nil)
+      end
+      
+    private
+      
+      # Computes the stored class an instance or class
+      def target_for(inst)
+        return inst if inst.kind_of? Class
+        inst.class
+      end
+      
+      # Compute the agrument sent to the finder method
+      def ident_for(target,finder=nil,args=nil)
+        if !(target.kind_of?(Class)) || finder
+          args || target.id
+        end
+        #Otherwize the target is the class and ident should be nil
+      end
+    
+    end
+    
+    #:nodoc:
+    def inspect
+      "#<Updater id=#{id} target=#{target.inspect} time=#{time}>"
+    end
+  end
+  
+end

data/lib/updater/util.rb

@@ -0,0 +1,22 @@
+require 'tmpdir'
+
+module Updater
+  class Util
+    class << self 
+      def tempio
+        fp = begin
+          File.open("#{Dir::tmpdir}/#{rand}",
+                    File::RDWR|File::CREAT|File::EXCL, 0600)
+        rescue Errno::EEXIST
+          retry
+        end
+        File.unlink(fp.path)
+        fp.binmode
+        fp.sync = true
+        fp
+      end
+    
+    
+    end
+  end
+end

data/lib/updater.rb

@@ -1,8 +1,5 @@
 require "rubygems"
 
-require 'dm-core'
-require 'dm-types'
-
 module Updater
   VERSION = File.read(File.join(File.dirname(__FILE__),'..','VERSION')).strip
 end

data/spec/chained_spec.rb

@@ -0,0 +1,81 @@
+require File.join( File.dirname(__FILE__),  "spec_helper" )
+
+include Updater
+
+require  File.join( File.dirname(__FILE__),  "fooclass" )
+
+describe "Chained Methods:" do
+  
+  before :each do
+    Update.clear_all
+    Foo.all.destroy!
+    @u = Update.chain(Foo,:chained,[:__job__,:__params__])
+    @v = Update.chain(Foo,:chained2,[:__job__,:__params__])
+  end
+  
+  [:failure, :success, :ensure].each do |mode|
+    specify "adding '#{mode.to_s}' chain" do
+      v = Update.immidiate(Foo,:method1,[],mode=>@u)
+      v.orm.send(mode).should_not be_empty
+    end
+  end
+  
+  specify "'failure' should run after an error" do
+    v = Update.immidiate(Foo,:method1,[],:failure=>@u)
+    Foo.should_receive(:method1).and_raise(RuntimeError)
+    Foo.should_receive(:chained).with(v,anything())
+    v.run
+  end
+  
+  specify "'failure' should NOT run if their is no error" do
+    v = Update.immidiate(Foo,:method1,[],:failure=>@u)
+    Foo.should_receive(:method1).and_return(:anything)
+    Foo.should_not_receive(:chained)
+    v.run
+  end
+  
+  specify "'success' should NOT run after an error" do
+    v = Update.immidiate(Foo,:method1,[],:success=>@u)
+    Foo.should_receive(:method1).and_raise(RuntimeError)
+    Foo.should_not_receive(:chained)
+    v.run
+  end
+  
+  specify "'success' should run if their is no error" do
+    v = Update.immidiate(Foo,:method1,[],:success=>@u)
+    Foo.should_receive(:method1).and_return(:anything)
+    Foo.should_receive(:chained).with(v,anything())
+    v.run
+  end
+  
+  specify "'ensure' should run after an error" do
+    v = Update.immidiate(Foo,:method1,[],:ensure=>@u)
+    Foo.should_receive(:method1).and_raise(RuntimeError)
+    Foo.should_receive(:chained).with(v,anything())
+    v.run
+  end
+  
+  specify "'ensure' should run if their is no error" do
+    v = Update.immidiate(Foo,:method1,[],:ensure=>@u)
+    Foo.should_receive(:method1).and_return(:anything)
+    Foo.should_receive(:chained).with(v,anything())
+    v.run
+  end
+  
+  specify "params should be availible" do
+    v = Update.immidiate(Foo,:method1,[],:ensure=>{@u=>'hi', @v=>'bye'})
+    Foo.should_receive(:method1).and_return(:anything)
+    Foo.should_receive(:chained).with(anything(), 'hi')
+    Foo.should_receive(:chained2).with(anything(), 'bye')
+    v.run
+  end
+  
+  specify "add an Array" do
+    v = Update.immidiate(Foo,:method1,[],:ensure=>[@u,@v])
+    Foo.should_receive(:method1).and_return(:anything)
+    Foo.should_receive(:chained).with(anything(), anything())
+    Foo.should_receive(:chained2).with(anything(), anything())
+    v.run
+  end
+  
+end

data/spec/errors_spec.rb

@@ -0,0 +1,31 @@
+require File.join( File.dirname(__FILE__),  "spec_helper" )
+
+include Updater
+
+require  File.join( File.dirname(__FILE__),  "fooclass" )
+
+describe "Job Error Handeling" do
+  before(:each) do
+    Foo.all.destroy!
+  end
+  
+  it "should return false when run" do
+    u = Update.immidiate(Foo,:bar,[:arg1,:arg2])
+    Foo.should_receive(:bar).with(:arg1,:arg2).and_raise(RuntimeError)
+    u.run.should be_false
+  end
+  
+  it "should trap errors" do
+    u = Update.immidiate(Foo,:bar,[:arg1,:arg2])
+    Foo.should_receive(:bar).with(:arg1,:arg2).and_raise(RuntimeError)
+    lambda {u.run}.should_not raise_error
+  end
+  
+  it "should run the failure task" do
+    err = Update.chain(Foo,:bar,[:error])
+    u = Update.immidiate(Foo,:bar,[:arg1,:arg2],:failure=>err)
+    Foo.should_receive(:bar).with(:arg1,:arg2).and_raise(RuntimeError)
+    Foo.should_receive(:bar).with(:error)
+    u.run
+  end
+end

data/spec/fooclass.rb

@@ -0,0 +1,14 @@
+
+  class Foo
+    include DataMapper::Resource
+    
+    property :id, Serial
+    property :name, String
+    
+    def bar(*args)
+      Foo.bar(:instance,*args)
+    end
+    
+  end
+
+Foo.auto_migrate!

data/spec/fork_worker_instance_spec.rb

@@ -0,0 +1,56 @@
+require File.join( File.dirname(__FILE__),  "spec_helper" )
+
+include Updater
+
+describe "Fork Worker Instance" do
+  
+  before :each do
+    @worker = ForkWorker::WorkerMonitor.new(1,Updater::Util.tempio)
+    @w = ForkWorker.new(IO.pipe,@worker)
+  end
+  
+  it "should have a heartbeat" do
+    @w.instance_variable_set(:@continue, true) #otherwise heartbeat is skipped 
+    mode = @worker.heartbeat.stat.mode
+    @w.heartbeat
+    @worker.heartbeat.stat.mode.should_not == mode
+  end
+  
+  describe "#smoke_pipe" do
+    
+    before :each do
+      @pipe = IO.pipe
+    end
+  
+    it "should remove exactly 1 char from a pipe" do
+      @pipe.last.write '..'
+      @w.smoke_pipe(@pipe.first).should be_true
+      @pipe.first.read_nonblock(2).should == '.'
+    end
+    
+    it "should not raise errors or block on empty pipes" do
+      lambda { @w.smoke_pipe(@pipe.first) }.should_not raise_error
+    end
+    
+  end
+  
+  describe "#wait_for" do
+    
+    it "should have specs"
+    
+    describe "when there are pending jobs" do
+      
+      it "should NOT wait for a signal"
+      
+      it "should smoke the pipe"
+      
+    end
+    
+    it "should wake as soon as a new job signal is placed on the pipe"
+    
+    
+    it "should run the heartbeat every 'timeout' seconds"
+    
+  end
+  
+end