updater 0.3.2 → 0.9.0

data/VERSION

@@ -1 +1 @@
-0.3.2
+0.9.0

data/lib/updater/fork_worker.rb

@@ -35,7 +35,7 @@ module Updater
         logger.info "Max Workers set to #{@max_workers}"
         @timeout = options[:timeout] || 60
         logger.info "Timeout set to #{@timeout} sec."
-        @current_workers = 1
+        @current_workers = 1 #we will actually add this worker the first time through the master loop
         @workers = {} #key is pid value is worker class
         @uptime = Time.now
         @downtime = Time.now
@@ -92,10 +92,9 @@ module Updater
       # * :timeout : how long can a worker be inactive before being killed
       # * :sockets: 0 or more IO objects that should wake up master to alert it that new data is availible
       
-      def start(stream,options = {})
+      def start(options = {})
         initial_setup(options) #need this for logger
         logger.info "*** Starting Master Process***"
-        @stream = stream
         logger.info "* Adding the first round of workers *"
         maintain_worker_count
         QUEUE_SIGS.each { |sig| trap_deferred(sig) }
@@ -121,13 +120,13 @@ module Updater
           logger.fatal "10 consecutive errors! Abandoning Master process"
         end
         stop # gracefully shutdown all workers on our way out
-        logger.info "master process Exiting"
+        logger.warn "-=-=-=- master process Exiting -=-=-=-\n\n"
       end
       
       def stop(graceful = true)
         trap(:USR2,"IGNORE")
         [:INT,:TERM].each {|signal| trap(signal,"DEFAULT") }
-        puts "Quitting. I need 30 seconds to stop my workers..."
+        puts "Quitting. I need 30 seconds to stop my workers..." unless @workers.empty?
         limit = Time.now + 30
         signal_each_worker(graceful ? :QUIT : :TERM)
         until @workers.empty? || Time.now > limit
@@ -140,15 +139,26 @@ module Updater
       def master_sleep
         begin
           timeout = calc_timeout
-          logger.debug { "Sleeping for #{timeout}" } #TODO return to debug 
+          logger.debug { "Sleeping for #{timeout}" }
           ready, _1, _2 = IO.select(@wakeup_set, nil, nil, timeout)
-          return unless ready && ready.first #just wakeup and run maintance
-          @signal_queue << :DATA unless ready.first == @self_pipe.first #somebody wants our attention
+          return unless ready && ready.first #timeout hit,  just wakeup and run maintance
+          add_connection(ready.first) and return if ready.first.respond_to?(:accept) #open a new incomming connection 
+          @signal_queue << :DATA unless ready.first == @self_pipe.first
           loop {ready.first.read_nonblock(16 * 1024)}
+        rescue EOFError #somebody closed thier connection
+          logger.info "closed socket connection"
+          @wakeup_set.delete ready.first
+          ready.first.close
         rescue Errno::EAGAIN, Errno::EINTR
         end
       end
       
+      def add_connection(server)
+        @wakeup_set << server.accept_nonblock
+        logger.info "opened socket connection: [#{@wakeup_set.last.addr.join(', ')}]"
+      rescue Errno::EAGAIN, Errno::EINTR
+      end
+      
       def calc_timeout
         Time.now - [@uptime, @downtime].max < @timeout ? @timeout / 8 : 2*@timeout
       end
@@ -219,6 +229,7 @@ module Updater
       
       def add_worker(worker_number)
         worker = WorkerMonitor.new(worker_number,Updater::Util.tempio)
+        Update.orm.before_fork
         pid = Process.fork do
           fork_cleanup
           self.new(@pipe,worker).run
@@ -229,6 +240,7 @@ module Updater
       
       def fork_cleanup
         QUEUE_SIGS.each { |signal| trap(signal,"IGNORE") }
+        Update.orm.after_fork
         if @self_pipe !=nil
           @self_pipe.each {|io| io.close}
         end
@@ -327,8 +339,8 @@ module Updater
           wait_for(delay) if @continue
         rescue Exception=> e
           say "Caught exception in Job Loop"
-          say e.message
-          say "||=========\n|| Backtrace\n|| " + e.backtrace.join("\n|| ") + "\n||========="
+          say e.inspect
+          say "\n||=========\n|| Backtrace\n|| " + e.backtrace.join("\n|| ") + "\n||========="
           Update.clear_locks(self)
           exit; #die and be replaced by the master process
         end
@@ -392,8 +404,8 @@ module Updater
       #need to wait for another job
       t = Time.now + delay
       while Time.now < t && @continue
-        delay = [@timeout,t-Time.now].min
-        debug "No Jobs; #{name} sleeping for #{delay}:  [#{@timeout},#{t - Time.now}].min"
+        delay = [@timeout/2,t-Time.now].min
+        debug "No Jobs; #{name} sleeping for #{delay}:  [#{@timeout/2},#{t - Time.now}].min"
         wakeup,_1,_2 = select([@stream],nil,nil,delay)
         heartbeat
         if wakeup

data/lib/updater/orm/datamapper.rb

@@ -19,13 +19,13 @@ module Updater
       storage_names[:default] = "updates"
     
       property :id, Serial
-      property :time, Integer
-      property :target, Class
-      property :finder, String
-      property :finder_args, Yaml
+      property :time, Integer, :index=>true
+      property :target, Class, :index=>:for_target
+      property :finder, String, :index=>:for_target
+      property :finder_args, Yaml, :index=>:for_target
       property :method, String
       property :method_args, Object, :lazy=>false
-      property :name, String, :length=>255
+      property :name, String, :length=>255, :index=>true
       property :lock_name, String
       property :persistant, Boolean
       
@@ -112,6 +112,7 @@ module Updater
           return nxt.time - tnow
         end
         
+        #Returns the Locked Job or nil if no jobs were availible.
         def lock_next(worker)
           updates = worker_set
           unless updates.empty?
@@ -124,6 +125,7 @@ module Updater
             updates.each do |u|
               return u if u.lock(worker)
             end
+            return nil
           end
         rescue DataObjects::ConnectionError
           sleep 0.1
@@ -140,7 +142,32 @@ module Updater
         end
         
         def for(mytarget, myfinder, myfinder_args, myname=nil)
-          #TODO
+          search = all(
+              :target=>mytarget,
+              :finder=>myfinder,
+              :finder_args=>myfinder_args, 
+              :lock_name=>nil
+            )
+          myname ? search.all(:name=>myname ) : search
+        end
+        
+        #For the server only, setup the connection to the database
+        def setup(options)
+          ::DataMapper.logger = options.delete(:logger)
+          ::DataMapper.setup(:default,options)
+        end
+        
+        # For pooled connections it is necessary to empty the pool of the parents connections so that they
+        # do not comtiminate the child pool. Note that while Datamapper is thread safe, it is not safe accross a process fork.
+        def before_fork
+          return unless (defined? ::DataObjects::Pooling)
+          return if ::DataMapper.repository.adapter.kind_of?(::DataMapper::Adapters::Sqlite3Adapter)
+          ::DataMapper.logger.debug "+-+-+-+-+ Cleaning up connection pool (#{::DataObjects::Pooling.pools.length}) +-+-+-+-+"
+          ::DataObjects::Pooling.pools.each {|p| p.dispose} 
+        end
+        
+        def after_fork
+          
         end
         
       private
@@ -152,7 +179,7 @@ module Updater
           options = {:lock_name=>nil,:limit=>limit, :order=>[:time.asc]}.merge(options)
           current.all(options)
         end
-        
+
         def lock
           
         end
@@ -168,8 +195,8 @@ module Updater
       belongs_to :caller, :model=>Updater::ORM::DataMapper, :child_key=>[:caller_id]
       belongs_to :target, :model=>Updater::ORM::DataMapper, :child_key=>[:target_id]
 
-      property :params, Object, :nullable=>true #:required=>false
-      property :occasion, String,  :nullable=>false #:required=>true
+      property :params, Object, :required=>false 
+      property :occasion, String,  :required=>true
     end
 
   end#ORM

data/lib/updater/orm/orm.rb

@@ -217,6 +217,22 @@ module Updater
         NotImplementedError
       end
       
+      # This method is the generic way to setup the datastore.  Options is a hash one of whose fields
+      # will be :logger, the logger instance to pass on to the ORM.  The rest of the options are ORM 
+      # spesific.  The function should prepair a connection to the datastore using the given options.
+      # If the connection cannot be prepaired then an appropriate error should be raised.
+      def setup(options)
+        NotImplementedError 
+      end
+      
+      # This method is called by the child before a fork call.  It allows the ORM to clean up any connections
+      # Made by the parent and establish new connections if necessary.
+      def before_fork
+        
+      end
+      
+      def after_fork
+      
       # Optional, but strongly recomended.
       #
       # For any datastore that permits, return and Array of all delayed, chained, and current but not locked jobs that reference 

data/lib/updater/setup.rb

@@ -14,6 +14,10 @@ module Updater
         new(config_file).stop
       end
       
+      def client_setup(options = {})
+        new(config_file, options).client_setup
+      end
+      
       def monitor
         
       end
@@ -27,44 +31,84 @@ module Updater
       end
     end
     
-    ROOT = File.dirname(self.config_file)
+    ROOT = File.dirname(self.config_file || Dir.pwd)
     
-    def initialize(file_or_hash)
+    #extended used for clients who wnat to override parameters
+    def initialize(file_or_hash, extended = {})
       @options = file_or_hash.kind_of?(Hash) ? file_or_hash : load_file(file_or_hash)
+      @options.merge(extended)
       @options[:pid_file] ||= File.join(ROOT,'updater.pid')
       @options[:host] ||= "localhost"
-      @logger = Logger.new(@options[:log_file] || STDOUT)
+      @logger = @options[:logger] || Logger.new(@options[:log_file] || STDOUT)
       level = Logger::SEV_LABEL.index(@options[:log_level].upcase) if @options[:log_level]
       @logger.level = level || Logger::WARN 
     end
     
     def start
-      @logger.warn "Starting Loop"
       pid = Process.fork do
         _start
       end
-      @logger.warn "Rake Successfully started Master Loop at pid #{pid}"
+      @logger.warn "Successfully started Master Loop at pid #{pid}"
+      puts "Job Queue Processor Started at PID: #{pid}"
     end
     
     def stop
       Process.kill("TERM",File.read(@options[:pid_file]).to_i)
     end
     
-    def client
+    # The client is responcible for loading classes and making connections.  We will simply setup the Updater spesifics
+    def client_setup
+      set_orm
+      
+      if @options[:socket] && File.exists?(@options[:socket])
+        Updater::Update.socket = UNIXSocket.new(@options[:socket])
+      elsif @options[:udp]
+        socket = UDPSocket.new()
+        socket.connect(@options[:host],@options[:udp])
+        Updater::Update.socket = socket
+      elsif @options[:tcp]
+        Updater::Update.socket = TCPSocket.new(@options[:host],@options[:tcp])
+      elsif @options[:remote]
+        raise NotImplimentedError #For future Authenticated Http Rest Server
+      end
+      
+      #set PID
+      if File.exists? @options[:pid_file]
+        Updater::Update.pid = File.read(@options[:pid_file]).strip
+      end
+      
       
     end
     
     private
     
+    def set_orm
+      #don't setup twice.  Client setup might call this as part of server setup in which case it is already done
+      return false if Updater::Update.orm
+      orm = @options[:orm] || "datamapper"
+      case orm.downcase
+        when "datamapper"
+          require 'updater/orm/datamapper'
+          Updater::Update.orm = ORM::DataMapper
+        when "mongodb"
+          require 'updater/orm/mongodb'
+          Updater::Update.orm = ORM::MongoDB
+        when "activerecord"
+          require 'updater/orm/activerecord'
+          Updater::Update.orm = ORM::ActiveRecord
+        else
+          require "update/orm/#{orm}"
+          Updater::Update.orm = Object.const_get("ORM").const_get(orm.capitalize)
+      end
+      @logger.info "Data store '#{orm}' selected"
+    end
+    
     def _start
       #set ORM
-      require 'updater/orm/datamapper'
-      Updater::Update.orm = ORM::DataMapper
-      
+      set_orm
       #init DataStore
-      DataMapper.logger = @logger
-      DataMapper.setup(:default, :adapter=>'sqlite3', :database=>'./simulated.db')
-      
+      default_options = {:adapter=>'sqlite3', :database=>'./default.db'}
+      Updater::Update.orm.setup((@options[:database] || @options[:orm_setup] || default_options).merge(:logger=>@logger))
       #load Models
       
       models = @options[:models] || Dir.glob('./app/models/**/*.rb')
@@ -73,11 +117,13 @@ module Updater
       end
       
       #establish Connections
+      @options[:host] ||= 'localhost'
       #Unix Socket -- name at @options[:socket]
       if @options[:socket]
         File.unlink @options[:socket] if File.exists? @options[:socket]
         @options[:sockets] ||= []
         @options[:sockets] << UNIXServer.new(@options[:socket])
+        @logger.info "Now listening on UNIX Socket: #{@options[:socket]}"
       end
       
       #UDP potentially unsafe user monitor server for Authenticated Connections (TODO)
@@ -86,28 +132,39 @@ module Updater
         udp = UDPSocket.new
         udp.bind(@options[:host],@options[:udp])
         @options[:sockets] << udp
+        @logger.info "Now listening for UDP: #{@options[:host]}:#{@options[:udp]}"
       end
       
       #TCP Unsafe user monitor server for Authenticated Connections (TODO)
       if @options[:tcp]
         @options[:sockets] ||= []
         @options[:sockets] << TCPServer.new(@options[:host],@options[:tcp])
+        @logger.info "Now listening for TCP: #{@options[:host]}:#{@options[:tcp]}"
       end
       
       #Log PID
       File.open(@options[:pid_file],'w') { |f| f.write(Process.pid.to_s)}
+      
+      client_setup
+      
       #start Worker
-      require 'updater/fork_worker'
-      worker_class = ForkWorker
+      worker = @options[:worker] || 'fork'  #todo make this line windows safe
+      require "updater/#{worker}_worker"
+      worker_class = Updater.const_get("#{worker.capitalize}Worker")
       worker_class.logger = @logger
+      @logger.info "Using #{worker_class.to_s} to run jobs:"
       worker_class.start(@options)
+      File.unlink(@options[:pid_file])
+      File.unlink @options[:socket] if @options[:socket] && File.exists?(@options[:socket])
     end
     
     def load_file(file)
       return {} if file.nil?
       file = File.open(file) if file.kind_of?(String)
       @config_file = File.expand_path(file.path)
-      YAML.load(ERB.new(File.read(file)).result(binding)) || {}
+      YAML.load(ERB.new(file.read).result(binding)) || {}
+    ensure
+      file.close
     end
   end
 end

data/lib/updater/update.rb

@@ -22,7 +22,12 @@ module Updater
       ensure
         run_chain :success if ret
         run_chain :ensure
-        @orm.destroy unless @orm.persistant
+        begin
+          @orm.destroy unless @orm.persistant
+        rescue DataObjects::ConnectionError
+          sleep 0.1
+          retry
+        end
       end
       ret
     end
@@ -33,7 +38,7 @@ module Updater
     
     def target
       target = @orm.finder.nil? ? @orm.target : @orm.target.send(@orm.finder,@orm.finder_args)
-      raise TargetMissingError, "Class:'#{@orm.target}' Finder:'#{@orm.finder}', Args:'#{@orm.finder_args.inspect}'" unless target
+      raise TargetMissingError, "Target missing --Class:'#{@orm.target}' Finder:'#{@orm.finder}', Args:'#{@orm.finder_args.inspect}'" unless target
       target
     end
     
@@ -49,17 +54,23 @@ module Updater
       @orm.name
     end
     
-    #This is the appropriate valut ot use for a chanable field value
+    #This is the appropriate value to use for a chanable field value
     def id
       @orm.id
     end
     
+    def ==(other)
+      id = other.id
+    end
+    
     def persistant?
       @orm.persistant
     end
     
     def inspect
       "#<Updater::Update target=#{target.inspect} time=#{orm.time}>"
+    rescue TargetMissingError
+      "#<Updater::Update target=<missing> time=#{orm.time}>"
     end
     
   private
@@ -93,6 +104,9 @@ module Updater
       chains.each do |job|
         Update.new(job.target).run(self,job.params)
       end
+    rescue NameError
+      puts @orm.inspect
+      raise
     end
     
     class << self
@@ -100,6 +114,16 @@ module Updater
       #This attribute must be set to some ORM that will persist the data
       attr_accessor :orm
       
+      #remove once Bug is discovered
+      def orm=(input)
+        raise ArgumentError, "Must set ORM to and appropriate class" unless input.kind_of? Class
+        @orm = input
+      end
+      
+      # This is an open IO socket that will be writen to when a job is scheduled. If it is unset
+      # then @pid is signaled instead.
+      attr_accessor :socket
+      
       #Gets a single job form the queue, locks and runs it.  it returns the number of second
       #Until the next job is scheduled, or 0 is there are more current jobs, or nil if there 
       #are no jobs scheduled.
@@ -198,8 +222,12 @@ module Updater
       # Advanced: This method allows values to be passed directly to the ORM layer's create method.
       # use +at+ and friends for everyday use cases.
       def schedule(hash)
-        new(@orm.create(hash))
+        r = new(@orm.create(hash))
         signal_worker
+        r
+      rescue NoMethodError
+        raise ArgumentError, "ORM not initialized!" if @orm.nil?
+        raise
       end
 
       # Create a new job having the same charistics as the old, except that 'hash' will override the original.
@@ -237,7 +265,9 @@ module Updater
       #
       # <Array[Updater]> unless name is given then only a single [Updater] instance. 
       def for(target,name=nil)
-        #TODO
+        target,finder,args = target_for(target)
+        ret = @orm.for(target,finder,args,name).map {|i| new(i)}
+        name ? ret.first : ret
       end
       
             #The time class used by Updater.  See time= 
@@ -286,6 +316,8 @@ module Updater
       #is set then an attempt will be made to signal the worker any
       #time a new update is made.
       #
+      #The PID will not be signaled if @socket is availible, but should be set as a back-up
+      #
       #If pid is not set, or is set to nil then the scheduleing program 
       #is responcible for waking-up a potentially sleeping worker process
       #in another way.
@@ -295,6 +327,7 @@ module Updater
         Process::kill 0, @pid
         @pid
       rescue Errno::ESRCH, ArgumentError
+        @pid = nil
         raise ArgumentError, "PID was invalid"
       end
       
@@ -304,7 +337,9 @@ module Updater
       
     private
       def signal_worker
-        if @pid
+        if @socket
+          @socket.write '.'
+        elsif @pid
           Process::kill "USR2", @pid
         end
       end

data/spec/named_request_spec.rb

@@ -8,29 +8,49 @@ describe "named request" do
   
   before(:each) do
     Foo.all.destroy!
+    Update.clear_all
   end
   
   it "should be found by name when target is an instance" do
     f = Foo.create(:name=>'Honey')
     u = Update.immidiate(f,:bar,[:named],:name=>'Now')
     u.name.should ==("Now")
-    pending "'for' not implemented"
     Update.for(f, "Now").should ==(u)
   end
   
   it "should be found by name when target is a class" do
     u = Update.immidiate(Foo,:bar,[:named],:name=>'Now')
     u.name.should ==("Now")
-    pending "'for' not implemented"
     Update.for(Foo, "Now").should ==(u)
   end
   
   it "should return all updates for a given target" do
-    u1 = Update.immidiate(Foo,:bar,[:arg1,:arg2])
+    u1 = Update.immidiate(Foo,:bar,[:arg1,:arg2], :name=>'First')
     u2 = Update.immidiate(Foo,:bar,[:arg3,:arg4])
-    pending "'for' not implemented"
     Update.for(Foo).should include(u1,u2)
   end
-
+  
+  #locked updates are already running and can therefore not be modified
+  it "should not include locked updates" do
+    u = Update.immidiate(Foo,:bar,[:named],:name=>'Now')
+    u.orm.lock(Struct.new(:name).new('test_worker'))
+    u.orm.should be_locked
+    Update.for(Foo).should_not include(u)
+    Update.for(Foo).should be_empty
+  end
+  
+  it "should not return rusults with the wrong name" do
+    u = Update.immidiate(Foo,:bar,[:named],:name=>'Now')
+    u.name.should ==("Now")
+    Update.for(Foo, "Then").should be_nil
+  end
+  
+  it "should not return results for the wring target" do
+    f = Foo.create(:name=>'Honey')
+    g = Foo.create(:name=>'Sweetie Pie')
+    u = Update.immidiate(f,:bar,[:named],:name=>'Now')
+    Update.for(f).should include(u)
+    Update.for(g).should be_empty
+  end
   
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: updater
 version: !ruby/object:Gem::Version 
-  version: 0.3.2
+  version: 0.9.0
 platform: ruby
 authors: 
 - John F. Miller
@@ -9,7 +9,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-02-18 00:00:00 -08:00
+date: 2010-03-26 00:00:00 -07:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency