backnob 0.0.1

data/History.txt

@@ -0,0 +1,4 @@
+== 0.0.1 2007-09-18
+
+* 1 major enhancement:
+  * Initial release

data/License.txt

@@ -0,0 +1,20 @@
+Copyright (c) 2007 Jeremy Wells, Boost New Media Ltd.
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/Manifest.txt

@@ -0,0 +1,23 @@
+History.txt
+License.txt
+Manifest.txt
+README.txt
+Rakefile
+bin/backnob
+examples/backnob.yml
+lib/backnob.rb
+lib/backnob/client.rb
+lib/backnob/hash.rb
+lib/backnob/options.rb
+lib/backnob/server.rb
+lib/backnob/version.rb
+lib/backnob/worker.rb
+lib/backnob_control.rb
+scripts/txt2html
+setup.rb
+test/backnob.yml
+test/test_client.rb
+test/test_controller.rb
+test/test_helper.rb
+test/test_server.rb
+test/worker.rb

data/README.txt

@@ -0,0 +1,3 @@
+README for backnob
+==================
+

data/Rakefile

@@ -0,0 +1,125 @@
+require 'rubygems'
+require 'rake'
+require 'rake/clean'
+require 'rake/testtask'
+require 'rake/packagetask'
+require 'rake/gempackagetask'
+require 'rake/rdoctask'
+require 'rake/contrib/rubyforgepublisher'
+require 'fileutils'
+require 'hoe'
+
+include FileUtils
+require File.join(File.dirname(__FILE__), 'lib', 'backnob', 'version')
+
+AUTHOR = 'Jeremy Wells'  # can also be an array of Authors
+EMAIL = "jeremy@boost.co.nz"
+DESCRIPTION = "Background workers"
+GEM_NAME = 'backnob' # what ppl will type to install your gem
+
+@config_file = "~/.rubyforge/user-config.yml"
+@config = nil
+def rubyforge_username
+  unless @config
+    begin
+      @config = YAML.load(File.read(File.expand_path(@config_file)))
+    rescue
+      puts <<-EOS
+ERROR: No rubyforge config file found: #{@config_file}"
+Run 'rubyforge setup' to prepare your env for access to Rubyforge
+ - See http://newgem.rubyforge.org/rubyforge.html for more details
+      EOS
+      exit
+    end
+  end
+  @rubyforge_username ||= @config["username"]
+end
+
+RUBYFORGE_PROJECT = 'backnob' # The unix name for your project
+HOMEPATH = "http://#{RUBYFORGE_PROJECT}.rubyforge.org"
+DOWNLOAD_PATH = "http://rubyforge.org/projects/#{RUBYFORGE_PROJECT}"
+
+NAME = "backnob"
+REV = nil 
+# UNCOMMENT IF REQUIRED: 
+# REV = `svn info`.each {|line| if line =~ /^Revision:/ then k,v = line.split(': '); break v.chomp; else next; end} rescue nil
+VERS = Backnob::VERSION::STRING + (REV ? ".#{REV}" : "")
+CLEAN.include ['**/.*.sw?', '*.gem', '.config', '**/.DS_Store']
+RDOC_OPTS = ['--quiet', '--title', 'backnob documentation',
+    "--opname", "index.html",
+    "--line-numbers", 
+    "--main", "README",
+    "--inline-source"]
+
+class Hoe
+  def extra_deps 
+    @extra_deps.reject { |x| Array(x).first == 'hoe' } 
+  end 
+end
+
+# Generate all the Rake tasks
+# Run 'rake -T' to see list of generated tasks (from gem root directory)
+hoe = Hoe.new(GEM_NAME, VERS) do |p|
+  p.author = AUTHOR 
+  p.description = DESCRIPTION
+  p.email = EMAIL
+  p.summary = DESCRIPTION
+  p.url = HOMEPATH
+  p.rubyforge_name = RUBYFORGE_PROJECT if RUBYFORGE_PROJECT
+  p.test_globs = ["test/**/test_*.rb"]
+  p.clean_globs |= CLEAN  #An array of file patterns to delete on clean.
+  
+  # == Optional
+  p.changes = p.paragraphs_of("History.txt", 0..1).join("\n\n")
+  #p.extra_deps = []     # An array of rubygem dependencies [name, version], e.g. [ ['active_support', '>= 1.3.1'] ]
+  #p.spec_extras = {}    # A hash of extra values to set in the gemspec.
+end
+
+CHANGES = hoe.paragraphs_of('History.txt', 0..1).join("\n\n")
+PATH    = (RUBYFORGE_PROJECT == GEM_NAME) ? RUBYFORGE_PROJECT : "#{RUBYFORGE_PROJECT}/#{GEM_NAME}"
+hoe.remote_rdoc_dir = File.join(PATH.gsub(/^#{RUBYFORGE_PROJECT}\/?/,''), 'rdoc')
+
+desc 'Generate website files'
+task :website_generate do
+  Dir['website/**/*.txt'].each do |txt|
+    sh %{ ruby scripts/txt2html #{txt} > #{txt.gsub(/txt$/,'html')} }
+  end
+end
+
+desc 'Upload website files to rubyforge'
+task :website_upload do
+  host = "#{rubyforge_username}@rubyforge.org"
+  remote_dir = "/var/www/gforge-projects/#{PATH}/"
+  local_dir = 'website'
+  sh %{rsync -aCv #{local_dir}/ #{host}:#{remote_dir}}
+end
+
+desc 'Generate and upload website files'
+task :website => [:website_generate, :website_upload, :publish_docs]
+
+desc 'Release the website and new gem version'
+task :deploy => [:check_version, :website, :release] do
+  puts "Remember to create SVN tag:"
+  puts "svn copy svn+ssh://#{rubyforge_username}@rubyforge.org/var/svn/#{PATH}/trunk " +
+    "svn+ssh://#{rubyforge_username}@rubyforge.org/var/svn/#{PATH}/tags/REL-#{VERS} "
+  puts "Suggested comment:"
+  puts "Tagging release #{CHANGES}"
+end
+
+desc 'Runs tasks website_generate and install_gem as a local deployment of the gem'
+task :local_deploy => [:website_generate, :install_gem]
+
+task :check_version do
+  unless ENV['VERSION']
+    puts 'Must pass a VERSION=x.y.z release version'
+    exit
+  end
+  unless ENV['VERSION'] == VERS
+    puts "Please update your version.rb to match the release version, currently #{VERS}"
+    exit
+  end
+end
+
+task :test do
+  system('ruby lib/backnob_control.rb stop')
+end

data/bin/backnob

@@ -0,0 +1,12 @@
+#!/usr/bin/env ruby
+#
+#  Created by Jeremy Wells on 2007-9-18.
+#  Copyright (c) 2007 Boost New Media Ltd. All rights reserved.
+
+begin
+  require 'rubygems'
+rescue LoadError
+  # no rubygems to load, so we fail silently
+end
+
+require 'backnob_control'

data/examples/backnob.yml

@@ -0,0 +1,3 @@
+---
+  workers: lib/workers
+  listen: 127.0.0.1:6444

data/lib/backnob/client.rb

@@ -0,0 +1,58 @@
+require 'drb/drb'
+require 'pathname'
+
+module Backnob
+  SERVER_URI = 'druby://127.0.0.1:6444'
+  
+  class Client        
+    def initialize(uri = nil)
+      @uri = uri || SERVER_URI
+    end
+    
+    def create_worker(name, options = {})    
+      server do |s|
+        s.create_worker(name, options)
+      end
+    end
+    
+    def add_worker(file)
+      server do |s|
+        s.add_file(Pathname.new(file).realpath.to_s)
+      end
+    end
+    
+    def available?
+      begin
+        server do |s|
+          s.ping
+        end
+      rescue
+        return false
+      end
+    end
+    
+    def results(key, hk = nil)
+      server do |s|
+        s.results(key, hk)
+      end
+    end
+    
+    def server
+      unless block_given?
+        DRb.start_service
+        return DRbObject.new_with_uri(@uri)
+      else
+        begin
+          DRb.start_service
+          s = DRbObject.new_with_uri(@uri)        
+
+          ret = yield(s)
+        ensure
+          DRb.stop_service
+        end
+
+        ret
+      end
+    end
+  end
+end

data/lib/backnob/hash.rb

@@ -0,0 +1,12 @@
+class Hash
+  # Destructively convert all keys to symbols.
+  def symbolize_keys!
+    keys.each do |key|
+      unless key.is_a?(Symbol)
+        self[key.to_sym] = self[key]
+        delete(key)
+      end
+    end
+    self
+  end
+end

data/lib/backnob/options.rb

@@ -0,0 +1,17 @@
+require 'pathname'
+require File.dirname(__FILE__) + '/hash'
+
+module Backnob
+  module Options
+    def sanitize!
+      self.symbolize_keys!
+      self[:workers] = [self[:workers]].compact unless self[:workers].is_a?(Array)
+      self[:workers].collect!{|f| Pathname.new(f).realpath.to_s } if self[:workers]
+      
+      if self[:listen]
+        self[:listen] = 'druby://' + self[:listen] unless self[:listen] =~ /\:\/\//
+        self[:listen] << ':6444' unless self[:listen] =~ /\:\d+/      
+      end
+    end
+  end
+end

data/lib/backnob/server.rb

@@ -0,0 +1,208 @@
+require 'rubygems'
+require 'logger'
+require 'drb/drb'
+require 'observer'
+require 'drb/observer'
+require 'digest/sha1'
+require 'singleton'
+require 'slave'
+require 'thread'
+require 'yaml'
+require File.dirname(__FILE__) + '/options'
+require File.dirname(__FILE__) + '/worker'
+
+module Backnob  
+  class Server
+    include Singleton
+    
+    # Don't let this object be marshalled
+    include DRb::DRbUndumped
+    
+    # Default options
+    OPTIONS = {:listen => '127.0.0.1:6444'}
+    # Array of worker classes
+    WORKER_KLASSES = []
+  
+    # Pass any calls to class to instance
+    def self.method_missing(method, *args, &block)
+      self.instance.__send__(method, *args)
+    end
+    
+    # Stop the server and all workers
+    # If receieved twice then stops even if
+    # workers are still running
+    def stop
+      unless true #@stopping
+        @stopping = true
+        logger.info "Waiting for workers to exit"
+        @workers.values.each{|w| w.wait}
+      else
+        logger.info "Forcing workers to exit"
+        @workers.values.each{|w| w.shutdown}
+      end
+      
+      logger.info "Stopping DRb service"
+      DRb.stop_service
+    end
+    
+    # Start the server
+    def start(options = {})    
+      options.symbolize_keys!        
+      
+      # Sanitize the options
+      @options = OPTIONS.merge(options)
+      @options.extend(Backnob::Options)
+      @options.sanitize!
+      
+      if @options[:path] && File.exists?(@options[:path])
+        Dir.chdir(@options[:path])
+      end
+      
+      # Add the workers directory
+      add_file File.dirname(__FILE__) + '/workers'
+      
+      # Set default variables
+      @workers = {}
+      @results = {}
+      @rqueue = Queue.new
+
+      m = self
+      
+      # Trap TERM and INT signals to run the
+      # stop method
+      Signal.trap("TERM") do
+        m.stop
+      end
+
+      Signal.trap("INT") do
+        m.stop
+      end      
+            
+      DRb.start_service(@options[:listen], m)
+      logger.info "Waiting on #{@options[:listen]}"
+      
+      # This thread now waits for exit, and also
+      # caches results off the results queue.
+      # It also cleans up workers that have quit.
+      while DRb.thread && DRb.thread.alive?
+        sleep 0.25
+        
+        unless @rqueue.empty?
+          # Unmarshal results
+          data = ::Marshal.load(@rqueue.pop)
+          key = data[0]
+          
+          @results[key] = {} unless @results.has_key?(key)
+          @results[key][data[1]] = data[2]        
+          
+          if data[1] == :error
+            logger.debug "Error occurred in worker #{key}"
+            logger.debug data[2].to_s
+            logger.debug data[2].backtrace if data[2].backtrace
+          end
+          
+          if data[1] == :running && data[2] == false
+            @workers[key].shutdown(:quiet => true)
+            @workers.delete(key)
+          end
+        end
+      end
+    end
+    
+    # Get results from a worker. Results are cached
+    # if possible
+    def results(key, hk = nil)
+      results = {}
+      if @results[key]
+        results = @results[key]
+      else
+        results = @workers[key].object.results() rescue {}      
+      end
+      (hk ? results[hk] : results)
+    end
+    
+    # Just return true. Used by client to test server is responding
+    def ping
+      return true
+    end
+    
+    # Update method for observing workers. This recieved marshalled
+    # result data and puts it on the receive queue for caching
+    def update(data)
+      @rqueue << data
+    end
+    
+    # Return a default logger for this process 
+    def logger
+      @logger ||= Logger.new($stdout)
+    end
+
+    # Create a new worker. Given a name it will find the first
+    # class that matches that name. Options hash is passed to
+    # the worker. A key is returned which can be used to retrieve
+    # the worker results.
+    def create_worker(name, options = {})
+      load_workers
+      
+      klass = WORKER_KLASSES.detect do |w|
+        w.name =~ /#{name}/i
+      end
+      
+      if klass
+        key = Digest::SHA1.hexdigest(Time.now.to_i.to_s + klass.name)
+
+        logger.debug "Creating worker #{klass.name} with key #{key}"
+        
+        worker = Slave.new :object => klass.new(key, options)
+        worker.object.add_observer(self)        
+        worker.object.start
+        @workers[key] = worker
+
+        key
+      end
+    end
+    
+    # Register a class as a worker
+    def register(klass)
+      WORKER_KLASSES.delete(klass) rescue nil
+      WORKER_KLASSES << klass
+    end
+    
+    # Add a worker file
+    def add_file(file)
+      @options[:workers] ||= []
+      
+      if File.exists?(file)
+        @options[:workers].delete(file)
+        @options[:workers] << file
+
+        logger.debug "Added worker file #{file}"
+      end
+      
+      @options[:workers]
+    end
+    
+    private
+    
+    # Load each worker file. Worker files should
+    # have calls to register their classes.
+    def load_workers
+      @options[:workers].each do |file|
+        if File.exists?(file)
+          # If the file is a directory then load all
+          # the rb files inside it.
+          if File.directory?(file)
+            Dir.glob(file + '/*.rb').each do |cfile|
+              load cfile
+            end
+          else
+            load file
+          end
+        end
+      end
+    end
+  end
+end
+
+options = defined?(OPTIONS) ? OPTIONS : {}
+Backnob::Server.start(options)

data/lib/backnob/version.rb

@@ -0,0 +1,9 @@
+module Backnob #:nodoc:
+  module VERSION #:nodoc:
+    MAJOR = 0
+    MINOR = 0
+    TINY  = 1
+
+    STRING = [MAJOR, MINOR, TINY].join('.')
+  end
+end

data/lib/backnob/worker.rb

@@ -0,0 +1,103 @@
+require 'rubygems'
+require 'slave'
+require 'thread'
+require 'active_support'
+require 'monitor'
+require 'thread'
+require 'drb/observer'
+
+module Kernel
+  def singleton_class
+    class << self; self; end
+  end
+end
+
+module Backnob
+  class Worker
+    include DRb::DRbObservable
+    
+    def self.define_attr_method(name, value)
+      class_eval do
+        define_method(name) do
+          value
+        end
+      end
+    end
+    
+    def self.define_class_attr(name, default)
+      define_attr_method name, default
+      self.singleton_class.send :define_method, name do |value|
+        define_attr_method name, value
+      end
+    end
+    
+    define_class_attr :rails, false
+    
+    def initialize(key, options = {})
+      @key = key
+      @options = options
+      @results = {}
+      @results.extend(MonitorMixin)
+    end
+        
+    def logger
+      @logger ||= Logger.new($stdout)
+    end
+    
+    # Start this worker. Starts a new thread and calls execute
+    # for work to be performed.
+    def start
+      @thread = Thread.new do
+        Thread.pass
+        
+        logger.debug "Starting #{self.class.name} with key #{@key}"
+                  
+        results(:running, true)
+                
+        if rails
+          require 'config/environment'
+          ActiveRecord::Base.logger = logger
+        end
+
+        # require File.dirname(__FILE__) + "/../../config/environment"
+        # ActiveRecord::Base.logger = logger
+        
+        begin
+          execute
+        rescue Exception => e
+          results(:error, e)
+        ensure
+          results(:running, false)
+          logger.debug "Finished #{self.class.name} with key #{@key}"
+        end
+      end
+    end
+    
+    # Get or set results. Workers should access results
+    # through this method as it is synchronized
+    def results(key = nil,value = nil)
+      res = nil
+
+      @results.synchronize do
+        if key.nil?
+          res = @results.dup
+        elsif value.nil?      
+          res = @results[key]
+        else
+          if value != @results[key]
+            @results[key] = value
+            changed(true) # I've changed
+
+            # Marshal results into string
+            # for some reason marshal doesn't like hash, so
+            # convert to array and convert back on server side
+            obo = ::Marshal.dump([@key, key, value])     
+            notify_observers(obo)
+          end
+        end
+      end
+
+      res
+    end    
+  end
+end

data/lib/backnob.rb

@@ -0,0 +1,4 @@
+module Backnob
+end
+
+require 'backnob/version'

data/lib/backnob_control.rb

@@ -0,0 +1,84 @@
+#!/usr/bin/env ruby
+#
+#  Created by Jeremy Wells on 2007-9-18.
+#  Copyright (c) 2007 Boost New Media Ltd. All rights reserved.
+
+begin
+  require 'rubygems'
+rescue LoadError
+  # no rubygems to load, so we fail silently
+end
+
+require 'pathname'
+require 'optparse'
+require 'rubygems'
+require 'daemons'
+require 'yaml'
+require File.dirname(__FILE__) + '/backnob/options'
+require File.dirname(__FILE__) + '/backnob/version'
+
+OPTIONS = {:path => Pathname.new('.').realpath.to_s}
+OPTIONS.extend(Backnob::Options)
+
+parser = OptionParser.new do |opts|
+  opts.banner = <<BANNER
+Backnob server version #{Backnob::VERSION::STRING}
+
+Server Usage: #{File.basename($0)} start|stop|restart|run
+  start: Start the server
+  stop: Stop the server
+  restart: Restart the server
+  run: Run the server in the foreground
+  
+Client Usage: #{File.basename($0)} create|results|generate
+  create: Create and start a worker. Returns the worker key
+  results: Get the results for a worker key.
+  generate: Not sure yet
+
+Options are:
+BANNER
+  opts.separator ""
+  opts.on("-h", "--help",
+          "Show this help message.") { puts opts; exit }
+  opts.on("-cFILE", "--configuration=FILE", "Load a configuration file") do |c| 
+    if File.exists?(c)
+      file_options = YAML.load(File.read(c)) rescue {}
+      OPTIONS.merge!(file_options)
+    end
+  end
+  opts.on("-lADDRESS", "--listen=ADDRESS", "Set the listen address. Defaults to 127.0.0.1:6444") {|l| OPTIONS[:listen] = l }
+  opts.on("-v", "--version", "Show the version") { puts "Backnob server version #{Backnob::VERSION::STRING}"; exit }
+  opts.parse!(ARGV)
+  
+  OPTIONS.sanitize!
+  
+  case ARGV.first.to_s.downcase
+  when 'start', 'stop', 'run', 'restart'
+    Daemons.run(File.dirname(__FILE__) + '/backnob/server.rb', :dir_mode => :normal, :dir => '/tmp', :mode => :load, :log_output => true)
+  when 'create', 'generate', 'results'
+    require 'backnob/client'
+    client = Backnob::Client.new
+    command = ARGV.pop
+    
+    unless client.available?
+      puts <<-EOF
+You must have a server running before using the #{command} command.
+Run a server using `backnob start`
+EOF
+      exit
+    end
+    
+    case command.downcase
+      when 'generate'
+    
+      when 'create'
+        begin; puts 'You must specify a worker to create'; exit; end unless ARGV.first
+        puts client.create_worker(ARGV.first)
+      when 'results'
+        begin; puts 'You must specify a worker key to get results'; exit; end unless ARGV.first
+        puts client.results(ARGV.first)
+    end
+  else
+    puts opts
+  end  
+end