updater 0.2.2 → 0.3.0

data/spec/fork_worker_spec.rb

@@ -0,0 +1,290 @@
+require File.join( File.dirname(__FILE__),  "spec_helper" )
+require 'logger'
+include Updater
+
+def fake_process_status(estat=0)
+  stub("Process Status",:pid=>1234,:exit_status=>estat)
+end
+
+def fake_iostream
+  stub("IOStream").as_null_object
+end
+
+ForkWorker.logger = Logger.new(nil)
+
+describe ForkWorker do
+  
+  describe "#reap_all_workers" do
+    it "should remove workers" do
+      Process.should_receive(:waitpid2).with(-1,Process::WNOHANG).twice\
+        .and_return([1234,fake_process_status],nil)
+      ForkWorker.should_receive(:remove_worker).with(1234).once
+      ForkWorker.reap_all_workers
+    end
+    
+    it "should not fail if there are no child processes" do
+      Process.should_receive(:waitpid2).with(-1,Process::WNOHANG).once\
+        .and_raise(Errno::ECHILD)
+      lambda{ForkWorker.reap_all_workers}.should_not raise_error
+    end
+    
+  end
+  
+  describe "#remove_worker" do
+    
+    it "should silently ignore missing workers" do
+      lambda{ForkWorker.remove_worker(1234)}.should_not raise_error
+    end
+    
+    it "should remove the worker from the set" do
+      worker = ForkWorker::WorkerMonitor.new(1,stub("IOStream").as_null_object)
+      ForkWorker.instance_variable_set :@workers, {1234=>worker}
+      ForkWorker.remove_worker(1234)
+      ForkWorker.instance_variable_get(:@workers).should == {}
+    end
+    
+    it "should close the removed workers heartbeat file" do
+      ios = mock("IOStream")
+      ios.should_receive(:close).and_return(nil)
+      worker = ForkWorker::WorkerMonitor.new(1,ios)
+      ForkWorker.instance_variable_set :@workers, {1234=>worker}
+      ForkWorker.remove_worker(1234)
+    end
+    
+    it "should not fail if the heartbeat file is already closed" do
+      ios = mock("IOStream")
+      ios.should_receive(:close).and_raise(IOError)
+      worker = ForkWorker::WorkerMonitor.new(1,ios)
+      ForkWorker.instance_variable_set :@workers, {1234=>worker}
+      lambda{ForkWorker.remove_worker(1234)}.should_not raise_error
+    end
+    
+  end
+  
+  describe "#add_worker" do
+    
+    before :each do
+      ForkWorker.initial_setup({})
+    end
+    
+    it "should add the new worker to the set" do
+      Process.stub!(:fork).and_return(1234)
+      ForkWorker.add_worker(1)
+      ForkWorker.instance_variable_get(:@workers).values.should include(1)
+      ForkWorker.instance_variable_get(:@workers).keys.should include(1234)
+    end
+    
+    it "should run a new worker instance in a fork" do
+      ForkWorker.instance_variable_set :@workers, {}
+      Process.should_receive(:fork).with(no_args()).and_yield.and_return(1234)
+      ForkWorker.should_receive(:fork_cleanup).and_return(nil) #
+      ForkWorker.should_receive(:new).with(anything(),duck_type(:number,:heartbeat)).and_return(stub("Worker",:run=>nil))
+      ForkWorker.add_worker(1)
+    end
+    
+  end
+  
+  describe "#spawn_missing_workers" do
+    
+    it "should add a worker with an empty set" do
+      ForkWorker.should_receive(:add_worker).with(0)
+      ForkWorker.instance_variable_set :@current_workers, 1
+      ForkWorker.instance_variable_set :@workers, {}
+      ForkWorker.spawn_missing_workers
+    end
+    
+    it "should add a worker when there are fewer then needed" do
+      ForkWorker.should_receive(:add_worker).with(1)
+      ForkWorker.instance_variable_set :@current_workers, 2
+      ForkWorker.instance_variable_set :@workers, {1233=>ForkWorker::WorkerMonitor.new(0,nil)}
+      ForkWorker.spawn_missing_workers
+    end
+    
+    it "should add a worker when one has gon missing" do
+      ForkWorker.should_receive(:add_worker).with(0)
+      ForkWorker.should_receive(:add_worker).with(2)
+      ForkWorker.instance_variable_set :@current_workers, 3
+      ForkWorker.instance_variable_set :@workers, {1233=>ForkWorker::WorkerMonitor.new(1,nil)}
+      ForkWorker.spawn_missing_workers
+    end
+    
+    it "should not add workers if thier are already enough" do
+      ForkWorker.should_not_receive(:add_worker)
+      ForkWorker.instance_variable_set :@current_workers, 1
+      ForkWorker.instance_variable_set :@workers, {1233=>ForkWorker::WorkerMonitor.new(0,nil)}
+      ForkWorker.spawn_missing_workers
+    end
+    
+  end
+  
+  describe "#initial_setup" do
+    
+    it "should set up a logger when one does not exist" do
+      ForkWorker.initial_setup({})
+      ForkWorker.logger.should_not be_nil
+      %w{debug info warn error fatal}.each do |n|
+        ForkWorker.logger.should respond_to(n.to_sym)
+      end
+    end
+    
+    it "should set workers set to empty" do
+      ForkWorker.initial_setup({})
+      ForkWorker.instance_variable_get(:@workers).should be_empty
+    end
+    
+    it "should create a pipe for children" do
+      pipe = ForkWorker.instance_variable_get(:@pipe)
+      pipe.length.should ==2
+      pipe.each {|io| io.should be_an IO}
+    end
+    
+  end
+  
+  describe "#handle_signal_queue" do
+    
+    before :each do
+      ForkWorker.initial_setup({})
+    end
+    
+    [:QUIT, :INT].each do |sig|
+      it "it should exicute a graceful shutdown on #{sig.to_s}" do
+        ForkWorker.should_receive(:stop).with(true)
+        ForkWorker.stub!(:awaken_master, true)
+        
+        ForkWorker.queue_signal(sig)
+        ForkWorker.handle_signal_queue.should be_false
+      end
+    end
+    
+    it "it should exicute a rapid shutdown on TERM" do
+      ForkWorker.should_receive(:stop).with(false)
+      ForkWorker.stub!(:awaken_master, true)
+      
+      ForkWorker.queue_signal(:TERM)
+      ForkWorker.handle_signal_queue.should be_false
+    end
+    
+    [:USR2, :DATA].each do |sig|
+      it "should write to the pipe on #{sig.to_s}" do
+        ForkWorker.queue_signal(sig)
+        ForkWorker.handle_signal_queue.should be_true
+        pipe = ForkWorker.instance_variable_get(:@pipe)
+        lambda{pipe.first.read(1)}.should_not raise_error
+      end
+    end
+    
+    it "should do maintance when the queue is empty" do
+      ForkWorker.should_receive(:murder_lazy_workers)
+      ForkWorker.should_receive(:maintain_worker_count)
+      ForkWorker.should_receive(:master_sleep)
+      
+      ForkWorker.handle_signal_queue.should be_true      
+    end
+    
+    it "should increase max_workers on TTIN and decrease on TTOU" do
+      max = ForkWorker.instance_variable_get(:@max_workers)
+      ForkWorker.queue_signal(:TTIN)
+      ForkWorker.handle_signal_queue.should be_true
+      ForkWorker.instance_variable_get(:@max_workers).should == max+1
+      
+      ForkWorker.queue_signal(:TTOU)
+      ForkWorker.handle_signal_queue.should be_true
+      ForkWorker.instance_variable_get(:@max_workers).should == max
+    end
+    
+    it "should never allow max_workers to be less then 1" do
+      ForkWorker.instance_variable_set(:@max_workers,1)
+      ForkWorker.queue_signal(:TTOU)
+      ForkWorker.handle_signal_queue.should be_true
+      ForkWorker.instance_variable_get(:@max_workers).should == 1
+    end
+  
+  end
+
+  describe "Master loop control" do
+    
+    before(:each) do
+      ForkWorker.initial_setup({})
+    end
+  
+    describe "#master_sleep" do
+      
+      it "should return to run maintance if there is no signal" do
+        IO.should_receive(:select).and_return(nil)
+        ForkWorker.master_sleep.should be_nil
+        ForkWorker.instance_variable_get(:@signal_queue).should be_empty
+      end
+      
+      it "should return if there is data on self_pipe" do
+        self_pipe = ForkWorker.instance_variable_get(:@self_pipe)
+        IO.should_receive(:select).and_return([[self_pipe.first],[],[]])
+        ForkWorker.master_sleep.should be_nil
+        ForkWorker.instance_variable_get(:@signal_queue).should be_empty
+      end
+      
+      it "should also add ':DATA' to queue when a stream other then self_pipe is ready." do
+        # pending "Testing Error  stream not raising error?"
+        stream = stub('Extern IO')
+        stream.should_receive(:read_nonblock).and_raise(Errno::EAGAIN)
+        IO.should_receive(:select).and_return([[stream],[],[]])
+        ForkWorker.master_sleep.should be_nil
+        ForkWorker.instance_variable_get(:@signal_queue).should include(:DATA)
+      end
+      
+    end
+
+  end
+  
+  
+  [true,false].each do |graceful|
+    
+    describe "#stop(#{graceful})" do
+      before :each do
+        ForkWorker.initial_setup({})
+      end
+      
+      describe "with no workers" do
+        it "should not fail" do
+          [true,false].each {|graceful| lambda{ForkWorker.stop(graceful)}.should_not raise_error}
+        end
+        
+        it "should not signal any workers" do
+          ForkWorker.should_not_receive(:signal_worker)
+          [true,false].each {|graceful| ForkWorker.stop(graceful)}
+        end
+      end
+      
+      describe "with workers" do
+        
+        before :each do
+          ForkWorker.instance_variable_set :@workers, 
+            {
+              1233=>ForkWorker::WorkerMonitor.new(0,fake_iostream),
+              1234=>ForkWorker::WorkerMonitor.new(0,fake_iostream)
+            }
+          Process.stub!(:waitpid2) do |_1,_2|
+            [ ForkWorker.instance_variable_get(:@workers).keys.first,
+              fake_process_status
+            ]
+          end
+        end
+        
+        it "should signals each worker to end" do
+          ForkWorker.instance_variable_get(:@workers).keys.each do |pid|
+            ForkWorker.should_receive(:signal_worker).with(graceful ? :QUIT : :TERM,pid)
+          end
+          ForkWorker.stop(graceful)
+        end
+        
+        it "should not kill workers that successfully quit" do
+          ForkWorker.should_not_receive(:signal_worker).with(:KILL,anything())
+          ForkWorker.stop(graceful)
+        end
+        
+      end
+      
+    end
+    
+  end
+  
+end

data/spec/lock_spec.rb

@@ -2,23 +2,29 @@ require File.join( File.dirname(__FILE__),  "spec_helper" )
 
 include Updater
 
+require  File.join( File.dirname(__FILE__),  "fooclass" )
+
 describe "Update Locking:" do
   
-  class Foo
-    include DataMapper::Resource
-    
-    property :id, Serial
-    property :name, String
+  
+  class Worker
+    attr_accessor :pid
+    attr_accessor :name
     
-    def bar(*args)
-      Foo.bar(:instance,*args)
+    def initialize(options={})
+      @quiet = options[:quiet]
+      @name = options[:name] || "host:#{Socket.gethostname} pid:#{Process.pid}" rescue "pid:#{Process.pid}"
+      @pid = Process.pid
     end
     
-  end
-  
-  Foo.auto_migrate!
+    def say(text)
+      puts text
+      nil
+    end
+  end    
   
   before :each do
+    Foo.all.destroy!
     @u = Update.immidiate(Foo,:bar,[])
     @w = Worker.new(:name=>"first", :quiet=>true)
   end
@@ -44,36 +50,13 @@ describe "Update Locking:" do
     @u.lock(@w).should be_true
     @u.lock(@w).should be_true
   end
-  
-  describe "#run_with_lock" do
-    
-    it "should run an unlocked record" do
-      u = Update.immidiate(Foo,:bar,[:arg1,:arg2])
-      Foo.should_receive(:bar).with(:arg1,:arg2)
-      u.run_with_lock(@w).should be_true
-    end
-    
-    it "should NOT run an already locked record" do
-      u = Update.immidiate(Foo,:bar,[:arg1,:arg2])
-      u.lock(Worker.new)
-      Foo.should_not_receive(:bar)
-      u.run_with_lock(@w).should be_nil
-    end
-    
-    it "should return false if the update ran but there was an error" do
-      u = Update.immidiate(Foo,:bar,[:arg1,:arg2])
-      Foo.should_receive(:bar).with(:arg1,:arg2).and_raise(RuntimeError)
-      u.run_with_lock(@w).should be_false
-    end
-    
-  end
-  
+   
   it "#clear_locks should clear all locks from a worker" do
     @v = Update.immidiate(Foo,:bar,[:arg1,:arg2])
     @u.lock(@w)
     @v.lock(@w)
     @u.locked?.should be_true
-    @w.clear_locks
+    Update.clear_locks(@w)
     @u.reload.locked?.should be_false
     @v.reload.locked?.should be_false
   end

data/spec/named_request_spec.rb

@@ -0,0 +1,36 @@
+require File.join( File.dirname(__FILE__),  "spec_helper" )
+
+include Updater
+
+require  File.join( File.dirname(__FILE__),  "fooclass" )
+
+describe "named request" do
+  
+  before(:each) do
+    Foo.all.destroy!
+  end
+  
+  it "should be found by name when target is an instance" do
+    f = Foo.create(:name=>'Honey')
+    u = Update.immidiate(f,:bar,[:named],:name=>'Now')
+    u.name.should ==("Now")
+    pending "'for' not implemented"
+    Update.for(f, "Now").should ==(u)
+  end
+  
+  it "should be found by name when target is a class" do
+    u = Update.immidiate(Foo,:bar,[:named],:name=>'Now')
+    u.name.should ==("Now")
+    pending "'for' not implemented"
+    Update.for(Foo, "Now").should ==(u)
+  end
+  
+  it "should return all updates for a given target" do
+    u1 = Update.immidiate(Foo,:bar,[:arg1,:arg2])
+    u2 = Update.immidiate(Foo,:bar,[:arg3,:arg4])
+    pending "'for' not implemented"
+    Update.for(Foo).should include(u1,u2)
+  end
+
+  
+end

data/spec/params_sub_spec.rb

@@ -0,0 +1,27 @@
+require File.join( File.dirname(__FILE__),  "spec_helper" )
+
+include Updater
+
+require  File.join( File.dirname(__FILE__),  "fooclass" )
+
+describe "Special Parameter Substitution" do
+  before :each do
+    Update.clear_all
+    @u = Update.chain(Foo,:chained, [:__job__,:__params__,:__self__, 'job params'])
+  end
+  
+  it "should substitute __job__ with job that chained in" do
+    Foo.should_receive(:chained).with(:arg1,anything(),anything(),'job params')
+    @u.run(:arg1)
+  end
+  
+  it "should substitute __params__ with params" do
+    Foo.should_receive(:chained).with(anything(),:arg2,anything(), 'job params')
+    @u.run(:arg1,:arg2)
+  end
+  
+  it "should substitute __self__ with the current job" do
+    Foo.should_receive(:chained).with(anything(),anything(),@u, 'job params')
+    @u.run
+  end
+end

data/spec/schedule_spec.rb

@@ -0,0 +1,89 @@
+require File.join( File.dirname(__FILE__),  "spec_helper" )
+
+include Updater
+
+require  File.join( File.dirname(__FILE__),  "fooclass" )
+
+describe "adding an immidiate update request" do
+  before(:each) do
+    Foo.all.destroy!
+  end
+  it "with a class target" do
+    u = Update.immidiate(Foo,:bar,[])
+    u.target.should == Foo
+    Update.current.get(u.id).should_not be_nil
+    Update.delayed.should == 0
+  end
+  
+  it "with an conforming instance target" do
+    f = Foo.create
+    u = Update.immidiate(f,:bar,[])
+    u.target.should == f
+    Update.current.get(u.id).should_not be_nil
+    Update.delayed.should == 0
+  end
+  
+  it "with an custome finder" do
+    f = Foo.create(:name=>'baz')
+    u = Update.immidiate(Foo,:bar,[],:finder=>:first, :finder_args=>[{:name=>'baz'}])
+    u.target.should == f
+    Update.current.get(u.id).should_not be_nil
+    Update.delayed.should == 0
+  end
+  
+end
+
+describe "chained request" do
+  before :each do
+    Update.clear_all
+  end
+  
+  it "should not be in current or delayed queue" do
+    u = Update.chain(Foo,:bar,[:error])
+    u.time.should be_nil
+    Update.current.should_not include(u)
+    Update.delayed.should == 0
+  end
+  
+  it "should be persistant" do
+    u = Update.chain(Foo,:bar,[:error])
+    u.should be_persistant
+  end
+  
+end
+ 
+describe "adding an delayed update request" do
+  before :each do
+    Update.clear_all
+    Foo.all.destroy
+  end
+
+  
+  it "with a class target" do
+    u = Update.at(Chronic.parse('tomorrow'),Foo,:bar,[])
+    u.target.should == Foo
+    Update.current.should_not include(u)
+    Update.delayed.should  == 1
+  end
+  
+  it "with an conforming instance target" do
+    f = Foo.create
+    u = Update.at(Chronic.parse('tomorrow'),f,:bar,[])
+    u.target.should == f
+    Update.current.should_not include(u)
+    Update.delayed.should == 1
+  end
+  
+  it "with an custome finder" do
+    f = Foo.create(:name=>'baz')
+    u = Update.at(Chronic.parse('tomorrow'),Foo,:bar,[],:finder=>:first, :finder_args=>[{:name=>'baz'}])
+    u.target.should == f
+    Update.current.should_not include(u)
+    Update.delayed.should == 1
+  end
+  
+end
+
+
+
+

data/spec/spec_helper.rb

@@ -7,7 +7,11 @@ require "spec" # Satisfies Autotest and anyone else not using the Rake tasks
 require "dm-core"
 
 require 'updater'
-require 'updater/worker'
+require 'updater/thread_worker'
+require 'updater/fork_worker'
+require 'updater/orm/datamapper'
+
+Updater::Update.orm = Updater::ORM::DataMapper
 
 DataMapper.setup(:default, 'sqlite3::memory:')
 DataMapper.auto_migrate!
@@ -15,3 +19,4 @@ DataMapper.auto_migrate!
 require 'timecop'
 require 'chronic'
 
+

data/spec/{worker_spec.rb → thread_worker_spec.rb}

@@ -2,10 +2,10 @@ require File.join( File.dirname(__FILE__),  "spec_helper" )
 
 include Updater
 
-describe Worker do
+describe ThreadWorker do
   
   it "should not print anything when quiet" do
-    w = Worker.new :quiet=>true
+    w = ThreadWorker.new :quiet=>true
     out = StringIO.new
     $stdout = out
     w.say "hello world"
@@ -14,7 +14,7 @@ describe Worker do
   end
   
   it "should have a name" do
-    Worker.new.name.should be_a String
+    ThreadWorker.new.name.should be_a String
   end
   
   describe "loop thread control:" do
@@ -35,7 +35,7 @@ describe Worker do
     
     specify "The loop should run when the worker is started" do
       pending
-      worker = Worker.new(:quiet=>true, :name=>"testing")
+      worker = ThreadWorker.new(:quiet=>true, :name=>"testing")
       Update.should_receive(:work_off).with(worker).once.and_return(nil)
       t = Thread.new do
         worker.start
@@ -73,13 +73,13 @@ describe "working off jobs:" do
   describe "Update#work_off" do
     
     before :each do
-      Update.all.destroy!
+      Update.clear_all
     end 
   
     it "should run and immidiate job"do
       u = Update.immidiate(Foo,:bar,[:arg1,:arg2])
       Foo.should_receive(:bar).with(:arg1,:arg2)
-      Update.work_off(Worker.new)
+      Update.work_off(ThreadWorker.new)
     end
     
     it "should aviod conflicts among mutiple workers" do
@@ -87,14 +87,14 @@ describe "working off jobs:" do
       u2 = Update.immidiate(Foo,:baz,[:arg2])
       Foo.should_receive(:bar).with(:arg1)
       Foo.should_receive(:baz).with(:arg2)
-      Update.work_off(Worker.new(:name=>"first", :quiet=>true))
-      Update.work_off(Worker.new(:name=>"second", :quiet=>true))
+      Update.work_off(ThreadWorker.new(:name=>"first", :quiet=>true))
+      Update.work_off(ThreadWorker.new(:name=>"second", :quiet=>true))
     end
     
     it "should return 0 if there are more jobs waiting" do
       u1 = Update.immidiate(Foo,:bar,[:arg1])
       u2 = Update.immidiate(Foo,:baz,[:arg2])
-      Update.work_off(Worker.new(:name=>"first", :quiet=>true)).should == 0
+      Update.work_off(ThreadWorker.new(:name=>"first", :quiet=>true)).should == 0
     end
     
     it "should return the number of seconds till the next job if there are no jobs to be run" do
@@ -102,12 +102,12 @@ describe "working off jobs:" do
       u1 = Update.at(Time.now + 30, Foo,:bar,[:arg1])
       Update.at(Time.now + 35, Foo,:bar,[:arg1])
       Update.at(Time.now + 1000, Foo,:bar,[:arg1])
-      Update.work_off(Worker.new(:name=>"first", :quiet=>true)).should == 30
+      Update.work_off(ThreadWorker.new(:name=>"first", :quiet=>true)).should == 30
     end
     
     it "should return nil if the job queue is empty" do
       u1 = Update.immidiate(Foo,:bar,[:arg1])
-      Update.work_off(Worker.new(:name=>"first", :quiet=>true)).should be_nil
+      Update.work_off(ThreadWorker.new(:name=>"first", :quiet=>true)).should be_nil
     end
     
   end

data/spec/update_runner_spec.rb

@@ -0,0 +1,48 @@
+require File.join( File.dirname(__FILE__),  "spec_helper" )
+
+include Updater
+
+require  File.join( File.dirname(__FILE__),  "fooclass" )
+
+describe "running an update" do
+  
+  before :each do
+    Update.clear_all
+    Foo.all.destroy!
+  end
+  
+  it "should call the named method with a class target" do
+    u = Update.immidiate(Foo,:bar,[:arg1,:arg2])
+    Foo.should_receive(:bar).with(:arg1,:arg2)
+    u.run
+  end
+  
+  it "should call the named method with an conforming instance target" do
+    f = Foo.create
+    u = Update.immidiate(f,:bar,[:arg1,:arg2])
+    Foo.should_receive(:bar).with(:instance,:arg1,:arg2)
+    u.run
+  end
+  
+  it "should delete the record once it is run" do
+    u = Update.immidiate(Foo,:bar,[:arg1,:arg2])
+    Foo.should_receive(:bar).with(:arg1,:arg2)
+    u.run
+    u.should_not be_saved #NOTE: not a theological statment
+  end
+  
+  it "should delete the record if there is a failure" do
+    u = Update.immidiate(Foo,:bar,[:arg1,:arg2])
+    Foo.should_receive(:bar).with(:arg1,:arg2).and_raise(RuntimeError)
+    u.run
+    u.should_not be_saved #NOTE: not a theological statment
+  end
+  
+  it "should NOT delete the record if it is a chain record" do
+    u = Update.chain(Foo,:bar,[:arg1,:arg2])
+    Foo.should_receive(:bar).with(:arg1,:arg2).and_raise(RuntimeError)
+    u.run
+    u.should be_saved
+  end
+  
+end