backnob 0.0.1 → 0.0.2

data/History.txt

@@ -1,3 +1,7 @@
+== 0.0.2 2007-09-24
+
+* Added singleton workers for processing queues
+
 == 0.0.1 2007-09-18
 
 * 1 major enhancement:

data/Manifest.txt

@@ -4,6 +4,8 @@ Manifest.txt
 README.txt
 Rakefile
 bin/backnob
+config/hoe.rb
+config/requirements.rb
 examples/backnob.yml
 lib/backnob.rb
 lib/backnob/client.rb
@@ -13,11 +15,32 @@ lib/backnob/server.rb
 lib/backnob/version.rb
 lib/backnob/worker.rb
 lib/backnob_control.rb
+log/debug.log
+plugin/LICENSE
+plugin/README
+plugin/conf/backnob.yml
+plugin/conf/test_worker.rb
+plugin/init.rb
+plugin/lib/backnob_ext.rb
+plugin/tasks/backnob_tasks.rake
+script/destroy
+script/generate
+script/txt2html
 scripts/txt2html
 setup.rb
+tasks/deployment.rake
+tasks/environment.rake
+tasks/website.rake
 test/backnob.yml
+test/single_worker.rb
 test/test_client.rb
 test/test_controller.rb
 test/test_helper.rb
 test/test_server.rb
+test/test_worker.rb
 test/worker.rb
+website/index.html
+website/index.txt
+website/javascripts/rounded_corners_lite.inc.js
+website/stylesheets/screen.css
+website/template.rhtml

data/README.txt

@@ -1,3 +1,182 @@
-README for backnob
-==================
+= README for backnob
 
+Backnob is a really simple background processor. Once started workers can be added. When created workers will fork off to do their processing, and return a hash of results for later inspection.
+
+Server Usage: backnob start|stop|restart|run
+[start:] Start the server
+[stop:] Stop the server
+[restart:] Restart the server
+[run:] Run the server in the foreground
+  
+Client Usage: backnob create|results
+[create:] Create and start a worker. Returns the worker key
+[results:] Get the results for a worker key.
+
+Options are:
+[-h, --help]                       Show this help message.
+[-c, --configuration=FILE]         Load a configuration file
+[-l, --listen=ADDRESS]             Set the listen address. Defaults to 127.0.0.1:6444
+[-v, --version]                    Show the version
+    
+== Configuration File Options
+
+[:workers]    Array of worker files or directories
+[:listen]     Address to listen on. Default is 127.0.0.1:6444
+
+
+== Using the library
+
+A simple worker example (simple.rb):
+
+  class SimpleWorker < Backnob::Worker
+    def execute
+      contents = File.read(@options[:file])
+      File.open('simple.txt', 'a') do |file|
+        file.write contents
+      end
+      results(:size, File.size('simple.txt'))
+    end
+  end
+ 
+Now start up a backnob server instance by running
+
+  backnob start
+
+Using the worker from irb:
+
+  require 'rubygems'
+  require 'backnob/client'
+
+  client = Backnob::Client.new
+  client.add_worker('simple.rb')
+  key = client.create_worker('simple', :file => '/tmp/to_be_read.txt')
+  while !client.results(key, :finished)
+    sleep 1
+  end
+  
+  unless client.results(key, :error)
+    puts "File length is now: #{client.results(key, :size)}"
+  end
+  
+= Using from Rails
+
+Install the plugin:
+
+  svn co svn://rubyforge.org/var/svn/backnob/trunk/plugin vendor/plugins/backnob
+  rake backnob:setup
+  
+Edit config/backnob.yml as appropriate. Create workers in lib/backnob_workers. Workers must
+specify "rails true" for the rails environment to be required in. It's not a great idea
+to require the environment manually as this will load rails into the server as well as the
+worker.
+
+A simple example of a rails worker that imports a CSV file formatted "email, name" into a users table:
+
+  require 'csv'
+
+  class ImportWorker < Backnob:Worker
+    rails true
+    
+    def execute
+      file = @options[:file]
+      return unless File.exists? file
+      
+      records = CSV.parse(File.read(file))
+
+      records.each_with_index do |record, index|
+        results(:progress, (index.to_f / records.length) * 100)
+      
+        unless User.find_by_email record[0]
+          User.create(:email => record[0],
+                      :name => record[1],
+                      :active => true)
+        end
+      end
+    end
+  end
+
+Now in a controller:
+
+  class ImportController < ApplicationController
+    def import
+      return unless request.post?
+      
+      file = Tempfile.new('import')
+      file.write params[:file].read
+      file.close
+      
+      session[:import_key] = backnob.create_worker('import', :file => file.path)
+      render :action => 'wait'
+    end
+    
+    def ajax_check_import_progress
+      results = backnob.results(session[:import_key])
+      unless results[:finished]
+        render :update do |page|
+          page.replace_html 'progress', results[:progress].to_i.to_s
+        end
+      else
+        render :update do |page|
+          page.redirect_to :action => 'success.rhtml'
+        end
+      end
+    end
+  end
+      
+Views, import.rhtml, wait.rhtml and success.rhtml:
+
+  # import.rhtml
+  <%- form_tag do %>
+    <%= file_field_tag :file %>
+    <%= submit_tag %>
+  <% end -%>
+  
+
+  # wait.rhtml
+  <p> Importing <span id='progress'>0</span>% complete, please wait... <%= image_tag('indicator.gif') %> </p>
+  <%= update_page_tag do |page|
+    page << periodically_call_remote :url => {:action => 'ajax_check_import_progress'}, :frequency => 5
+  end %>
+  
+
+  # success.rhtml
+  <p> File imported successfully </p>
+
+= Single Workers
+
+A single worker will stay running after execution so you can queue as much work up as you want and it will
+execute each bit of work sequentially. Each bit of work still has its own key and results.
+
+An example worker:
+
+  class SingleWorker < Backnob::Worker
+    single true
+    
+    def execute
+      @times_run ||= 0
+      @times_run += 1
+      
+      work = @options[:work]
+      logger.info work
+      
+      results(:run, @times_run)
+    end
+  end
+  
+Client:
+
+  client = Backnob::Client.new
+  client.add_worker('single_worker.rb')
+  key1 = client.create_worker('single', :work => "hello")
+  key2 = client.create_worker('single', :work => "world")
+  
+  while !client.results(key1, :finished)
+    sleep 0.25
+  end
+  
+  while !client.results(key2, :finished)
+    sleep 0.25
+  end
+    
+  puts client.results(key1, :run) => 1
+  puts client.results(key2, :run) => 2

data/Rakefile

@@ -1,125 +1,8 @@
-require 'rubygems'
-require 'rake'
-require 'rake/clean'
-require 'rake/testtask'
-require 'rake/packagetask'
-require 'rake/gempackagetask'
-require 'rake/rdoctask'
-require 'rake/contrib/rubyforgepublisher'
-require 'fileutils'
-require 'hoe'
-
-include FileUtils
-require File.join(File.dirname(__FILE__), 'lib', 'backnob', 'version')
-
-AUTHOR = 'Jeremy Wells'  # can also be an array of Authors
-EMAIL = "jeremy@boost.co.nz"
-DESCRIPTION = "Background workers"
-GEM_NAME = 'backnob' # what ppl will type to install your gem
-
-@config_file = "~/.rubyforge/user-config.yml"
-@config = nil
-def rubyforge_username
-  unless @config
-    begin
-      @config = YAML.load(File.read(File.expand_path(@config_file)))
-    rescue
-      puts <<-EOS
-ERROR: No rubyforge config file found: #{@config_file}"
-Run 'rubyforge setup' to prepare your env for access to Rubyforge
- - See http://newgem.rubyforge.org/rubyforge.html for more details
-      EOS
-      exit
-    end
-  end
-  @rubyforge_username ||= @config["username"]
-end
-
-RUBYFORGE_PROJECT = 'backnob' # The unix name for your project
-HOMEPATH = "http://#{RUBYFORGE_PROJECT}.rubyforge.org"
-DOWNLOAD_PATH = "http://rubyforge.org/projects/#{RUBYFORGE_PROJECT}"
-
-NAME = "backnob"
-REV = nil 
-# UNCOMMENT IF REQUIRED: 
-# REV = `svn info`.each {|line| if line =~ /^Revision:/ then k,v = line.split(': '); break v.chomp; else next; end} rescue nil
-VERS = Backnob::VERSION::STRING + (REV ? ".#{REV}" : "")
-CLEAN.include ['**/.*.sw?', '*.gem', '.config', '**/.DS_Store']
-RDOC_OPTS = ['--quiet', '--title', 'backnob documentation',
-    "--opname", "index.html",
-    "--line-numbers", 
-    "--main", "README",
-    "--inline-source"]
-
-class Hoe
-  def extra_deps 
-    @extra_deps.reject { |x| Array(x).first == 'hoe' } 
-  end 
-end
-
-# Generate all the Rake tasks
-# Run 'rake -T' to see list of generated tasks (from gem root directory)
-hoe = Hoe.new(GEM_NAME, VERS) do |p|
-  p.author = AUTHOR 
-  p.description = DESCRIPTION
-  p.email = EMAIL
-  p.summary = DESCRIPTION
-  p.url = HOMEPATH
-  p.rubyforge_name = RUBYFORGE_PROJECT if RUBYFORGE_PROJECT
-  p.test_globs = ["test/**/test_*.rb"]
-  p.clean_globs |= CLEAN  #An array of file patterns to delete on clean.
-  
-  # == Optional
-  p.changes = p.paragraphs_of("History.txt", 0..1).join("\n\n")
-  #p.extra_deps = []     # An array of rubygem dependencies [name, version], e.g. [ ['active_support', '>= 1.3.1'] ]
-  #p.spec_extras = {}    # A hash of extra values to set in the gemspec.
-end
-
-CHANGES = hoe.paragraphs_of('History.txt', 0..1).join("\n\n")
-PATH    = (RUBYFORGE_PROJECT == GEM_NAME) ? RUBYFORGE_PROJECT : "#{RUBYFORGE_PROJECT}/#{GEM_NAME}"
-hoe.remote_rdoc_dir = File.join(PATH.gsub(/^#{RUBYFORGE_PROJECT}\/?/,''), 'rdoc')
-
-desc 'Generate website files'
-task :website_generate do
-  Dir['website/**/*.txt'].each do |txt|
-    sh %{ ruby scripts/txt2html #{txt} > #{txt.gsub(/txt$/,'html')} }
-  end
-end
-
-desc 'Upload website files to rubyforge'
-task :website_upload do
-  host = "#{rubyforge_username}@rubyforge.org"
-  remote_dir = "/var/www/gforge-projects/#{PATH}/"
-  local_dir = 'website'
-  sh %{rsync -aCv #{local_dir}/ #{host}:#{remote_dir}}
-end
-
-desc 'Generate and upload website files'
-task :website => [:website_generate, :website_upload, :publish_docs]
-
-desc 'Release the website and new gem version'
-task :deploy => [:check_version, :website, :release] do
-  puts "Remember to create SVN tag:"
-  puts "svn copy svn+ssh://#{rubyforge_username}@rubyforge.org/var/svn/#{PATH}/trunk " +
-    "svn+ssh://#{rubyforge_username}@rubyforge.org/var/svn/#{PATH}/tags/REL-#{VERS} "
-  puts "Suggested comment:"
-  puts "Tagging release #{CHANGES}"
-end
-
-desc 'Runs tasks website_generate and install_gem as a local deployment of the gem'
-task :local_deploy => [:website_generate, :install_gem]
-
-task :check_version do
-  unless ENV['VERSION']
-    puts 'Must pass a VERSION=x.y.z release version'
-    exit
-  end
-  unless ENV['VERSION'] == VERS
-    puts "Please update your version.rb to match the release version, currently #{VERS}"
-    exit
-  end
-end
-
-task :test do
-  system('ruby lib/backnob_control.rb stop')
+require 'config/requirements'
+require 'config/hoe' # setup Hoe + all gem configuration
+
+Dir['tasks/**/*.rake'].each { |rake| load rake }
+
+task :test do
+  system('ruby lib/backnob_control.rb stop')
 end

data/config/hoe.rb

@@ -0,0 +1,70 @@
+require 'backnob/version'
+
+AUTHOR = 'Jeremy Wells'  # can also be an array of Authors
+EMAIL = "jeremy@boost.co.nz"
+DESCRIPTION = "Simple background worker library"
+GEM_NAME = 'backnob' # what ppl will type to install your gem
+RUBYFORGE_PROJECT = 'backnob' # The unix name for your project
+HOMEPATH = "http://#{RUBYFORGE_PROJECT}.rubyforge.org"
+DOWNLOAD_PATH = "http://rubyforge.org/projects/#{RUBYFORGE_PROJECT}"
+
+@config_file = "~/.rubyforge/user-config.yml"
+@config = nil
+RUBYFORGE_USERNAME = "jemmyw"
+def rubyforge_username
+  unless @config
+    begin
+      @config = YAML.load(File.read(File.expand_path(@config_file)))
+    rescue
+      puts <<-EOS
+ERROR: No rubyforge config file found: #{@config_file}
+Run 'rubyforge setup' to prepare your env for access to Rubyforge
+ - See http://newgem.rubyforge.org/rubyforge.html for more details
+      EOS
+      exit
+    end
+  end
+  RUBYFORGE_USERNAME.replace @config["username"]
+end
+
+
+REV = nil 
+# UNCOMMENT IF REQUIRED: 
+# REV = `svn info`.each {|line| if line =~ /^Revision:/ then k,v = line.split(': '); break v.chomp; else next; end} rescue nil
+VERS = Backnob::VERSION::STRING + (REV ? ".#{REV}" : "")
+RDOC_OPTS = ['--quiet', '--title', 'backnob documentation',
+    "--opname", "index.html",
+    "--line-numbers", 
+    "--main", "README",
+    "--inline-source"]
+
+class Hoe
+  def extra_deps 
+    @extra_deps.reject! { |x| Array(x).first == 'hoe' } 
+    @extra_deps
+  end 
+end
+
+# Generate all the Rake tasks
+# Run 'rake -T' to see list of generated tasks (from gem root directory)
+hoe = Hoe.new(GEM_NAME, VERS) do |p|
+  p.author = AUTHOR 
+  p.description = DESCRIPTION
+  p.email = EMAIL
+  p.summary = DESCRIPTION
+  p.url = HOMEPATH
+  p.rubyforge_name = RUBYFORGE_PROJECT if RUBYFORGE_PROJECT
+  p.test_globs = ["test/**/test_*.rb"]
+  p.clean_globs |= ['**/.*.sw?', '*.gem', '.config', '**/.DS_Store']  #An array of file patterns to delete on clean.
+  
+  # == Optional
+  p.changes = p.paragraphs_of("History.txt", 0..1).join("\\n\\n")
+  p.extra_deps = [['slave', '>= 1.2.1']]     # An array of rubygem dependencies [name, version], e.g. [ ['active_support', '>= 1.3.1'] ]
+  
+  #p.spec_extras = {}    # A hash of extra values to set in the gemspec.
+  
+end
+
+CHANGES = hoe.paragraphs_of('History.txt', 0..1).join("\\n\\n")
+PATH    = (RUBYFORGE_PROJECT == GEM_NAME) ? RUBYFORGE_PROJECT : "#{RUBYFORGE_PROJECT}/#{GEM_NAME}"
+hoe.remote_rdoc_dir = File.join(PATH.gsub(/^#{RUBYFORGE_PROJECT}\/?/,''), 'rdoc')

data/config/requirements.rb

@@ -0,0 +1,17 @@
+require 'fileutils'
+include FileUtils
+
+require 'rubygems'
+%w[rake hoe newgem rubigen].each do |req_gem|
+  begin
+    require req_gem
+  rescue LoadError
+    puts "This Rakefile requires the '#{req_gem}' RubyGem."
+    puts "Installation: gem install #{req_gem} -y"
+    exit
+  end
+end
+
+$:.unshift(File.join(File.dirname(__FILE__), %w[.. lib]))
+
+require 'backnob'

data/lib/backnob/client.rb

@@ -4,23 +4,36 @@ require 'pathname'
 module Backnob
   SERVER_URI = 'druby://127.0.0.1:6444'
   
-  class Client        
+  class Client
+    # Create a new client to connect to the server on the passed uri.
+    # The uri defaults to druby://127.0.0.1:6444
     def initialize(uri = nil)
       @uri = uri || SERVER_URI
     end
     
+    # Create and start a new worker. Pass the short name of the worker.
+    # For example if you have ImportWorker use 'import'
+    #
+    # The options hash is passed to the worker. The worker key is returned.
+    #   client.create_worker('import', {:file => '/tmp/import.csv'})
     def create_worker(name, options = {})    
       server do |s|
         s.create_worker(name, options)
       end
     end
     
+    # Add a worker file or directory. For example:
+    #   client.add_worker('workers/import_worker.rb')
+    # Will add just the import worker file, whereas:
+    #   client.add_worker('workers')
+    # Will add all worker files in the directory workers.
     def add_worker(file)
       server do |s|
         s.add_file(Pathname.new(file).realpath.to_s)
       end
     end
     
+    # Test to see if the server is available
     def available?
       begin
         server do |s|
@@ -31,12 +44,23 @@ module Backnob
       end
     end
     
+    # Retrieve as results hash for a given worker key. Can also
+    # take a key to retrieve just the results on a given key.
     def results(key, hk = nil)
       server do |s|
         s.results(key, hk)
       end
     end
     
+    # Return a DRb object reference to the server. If passed
+    # a block this will open the connection, yield then close
+    # the connection.
+    #
+    #   client.server do |server|
+    #     server.ping
+    #   end
+    #
+    # This will return true
     def server
       unless block_given?
         DRb.start_service

data/lib/backnob/hash.rb

@@ -1,4 +1,4 @@
-class Hash
+class Hash # :nodoc: all
   # Destructively convert all keys to symbols.
   def symbolize_keys!
     keys.each do |key|
@@ -9,4 +9,14 @@ class Hash
     end
     self
   end
+end
+
+class AttrHash < Hash
+  def method_missing(symbol, *args)
+    if self.has_key?(symbol)
+      self[symbol]
+    else
+      super
+    end
+  end
 end

data/lib/backnob/options.rb

@@ -1,8 +1,8 @@
 require 'pathname'
 require File.dirname(__FILE__) + '/hash'
 
-module Backnob
-  module Options
+module Backnob # :nodoc:
+  module Options # :nodoc: all
     def sanitize!
       self.symbolize_keys!
       self[:workers] = [self[:workers]].compact unless self[:workers].is_a?(Array)