backnob 0.0.1 → 0.0.2

data/lib/backnob/server.rb

@@ -11,7 +11,8 @@ require 'yaml'
 require File.dirname(__FILE__) + '/options'
 require File.dirname(__FILE__) + '/worker'
 
-module Backnob  
+module Backnob
+  # Backnob server.
   class Server
     include Singleton
     
@@ -24,7 +25,7 @@ module Backnob
     WORKER_KLASSES = []
   
     # Pass any calls to class to instance
-    def self.method_missing(method, *args, &block)
+    def self.method_missing(method, *args, &block) # :nodoc:
       self.instance.__send__(method, *args)
     end
     
@@ -32,20 +33,24 @@ module Backnob
     # If receieved twice then stops even if
     # workers are still running
     def stop
-      unless true #@stopping
-        @stopping = true
-        logger.info "Waiting for workers to exit"
-        @workers.values.each{|w| w.wait}
-      else
-        logger.info "Forcing workers to exit"
-        @workers.values.each{|w| w.shutdown}
-      end
+      @stopping = true
+      @rqueue << :stop
       
+      logger.info "Sending stop to all active workers"
+      @workers.each{|w| w.object.stop rescue nil }
+      @workers.each{|w| w.slave.shutdown rescue nil }
       logger.info "Stopping DRb service"
       DRb.stop_service
+      
+      exit!
     end
     
-    # Start the server
+    # Start the server with given options.
+    #
+    # Options:
+    # [:path]     Path to chdir to after starting
+    # [:workers]  Array of worker files or directories
+    # [:listen]   Address to listen on. Defaults to 127.0.0.1:6444
     def start(options = {})    
       options.symbolize_keys!        
       
@@ -62,7 +67,7 @@ module Backnob
       add_file File.dirname(__FILE__) + '/workers'
       
       # Set default variables
-      @workers = {}
+      @workers = []
       @results = {}
       @rqueue = Queue.new
 
@@ -84,27 +89,20 @@ module Backnob
       # This thread now waits for exit, and also
       # caches results off the results queue.
       # It also cleans up workers that have quit.
-      while DRb.thread && DRb.thread.alive?
-        sleep 0.25
-        
-        unless @rqueue.empty?
-          # Unmarshal results
-          data = ::Marshal.load(@rqueue.pop)
+      while DRb.thread && DRb.thread.alive? && !@stopping
+        rqueue = @rqueue.pop        
+        unless rqueue == :stop
+          data = ::Marshal.load(rqueue)
           key = data[0]
-          
+
           @results[key] = {} unless @results.has_key?(key)
           @results[key][data[1]] = data[2]        
-          
+
           if data[1] == :error
             logger.debug "Error occurred in worker #{key}"
             logger.debug data[2].to_s
             logger.debug data[2].backtrace if data[2].backtrace
           end
-          
-          if data[1] == :running && data[2] == false
-            @workers[key].shutdown(:quiet => true)
-            @workers.delete(key)
-          end
         end
       end
     end
@@ -112,12 +110,12 @@ module Backnob
     # Get results from a worker. Results are cached
     # if possible
     def results(key, hk = nil)
-      results = {}
-      if @results[key]
-        results = @results[key]
-      else
-        results = @workers[key].object.results() rescue {}      
-      end
+      results = @results[key]
+      # if @results[key]
+      #   results = @results[key]
+      # else
+      #   results = get_worker(key).object.results() rescue {}     
+      # end
       (hk ? results[hk] : results)
     end
     
@@ -147,28 +145,46 @@ module Backnob
       klass = WORKER_KLASSES.detect do |w|
         w.name =~ /#{name}/i
       end
-      
+            
       if klass
-        key = Digest::SHA1.hexdigest(Time.now.to_i.to_s + klass.name)
-
-        logger.debug "Creating worker #{klass.name} with key #{key}"
-        
-        worker = Slave.new :object => klass.new(key, options)
-        worker.object.add_observer(self)        
-        worker.object.start
-        @workers[key] = worker
+        # Generate a key
+        begin
+          key = Digest::SHA1.hexdigest(rand.to_s + Time.now.to_i.to_s + klass.name)
+        end until !get_worker(key)
+                    
+        # If the class is marked as single then find an existing one if possible
+        if klass.single && worker = @workers.detect{|w| w.klass == klass}
+          worker.keys << key
+          worker.object.enqueue(key, options)
+          logger.debug "Adding work to worker #{klass.name} with key #{key}"
+        else     
+          logger.debug "Creating worker #{klass.name} with key #{key}"
+          
+          slave = Slave.new :object => klass.new(key, options)
+          worker = AttrHash.new
+          worker[:slave] = slave
+          worker[:object] = slave.object
+          worker[:klass] = klass
+          worker[:keys] = [key]
+          slave.object.add_observer(self)
+          slave.object.start
+          @workers << worker
+        end
 
         key
+      else
+        raise Exception.new("Invalid worker class: #{name}")
       end
     end
-    
+        
     # Register a class as a worker
     def register(klass)
       WORKER_KLASSES.delete(klass) rescue nil
       WORKER_KLASSES << klass
     end
     
-    # Add a worker file
+    # Add a worker file or a directory of
+    # worker files
     def add_file(file)
       @options[:workers] ||= []
       
@@ -184,9 +200,14 @@ module Backnob
     
     private
     
+    # Get the worker with a given key
+    def get_worker(key)
+      @workers.detect{|w| w.keys.include?(key)}
+    end
+    
     # Load each worker file. Worker files should
     # have calls to register their classes.
-    def load_workers
+    def load_workers      
       @options[:workers].each do |file|
         if File.exists?(file)
           # If the file is a directory then load all

data/lib/backnob/version.rb

@@ -2,7 +2,7 @@ module Backnob #:nodoc:
   module VERSION #:nodoc:
     MAJOR = 0
     MINOR = 0
-    TINY  = 1
+    TINY  = 2
 
     STRING = [MAJOR, MINOR, TINY].join('.')
   end

data/lib/backnob/worker.rb

@@ -1,30 +1,69 @@
 require 'rubygems'
 require 'slave'
 require 'thread'
-require 'active_support'
+#require 'active_support'
 require 'monitor'
 require 'thread'
 require 'drb/observer'
 
-module Kernel
+module Kernel # :nodoc:
   def singleton_class
     class << self; self; end
   end
 end
 
 module Backnob
+  # Worker class. Workers should inherit from this class
+  # and implement the execute method.
+  #
+  # Example of a simple worker:
+  #   class SimpleWorker < Backnob::Worker
+  #     def execute
+  #       contents = File.read(@options[:file])
+  #       File.open('simple.txt', 'a') do |file|
+  #         file.write contents
+  #       end
+  #
+  #       results(:size, File.size('simple.txt'))
+  #     end
+  #   end
+  #
+  # Usage:
+  #   client = Backnob::Client.new
+  #   client.add_worker('simple.rb')
+  #   key = client.create_worker('simple', :file => '/tmp/to_be_read.txt')
+  #   while !client.results(key, :finished)
+  #     sleep 1
+  #   end
+  #   
+  #   unless client.results(key, :error)
+  #     puts "File length is now: #{client.results(key, :size)}"
+  #   else
+  #     puts client.results(key, :error).to_s
+  #   end
+  #
+  # Worker adds the key :running to the results automatically and sets it to
+  # true when it starts.
+  #
+  # If an error occurs during execution the worker will place the error object
+  # into the results under the key :error.
   class Worker
     include DRb::DRbObservable
+    attr_reader :key
     
-    def self.define_attr_method(name, value)
-      class_eval do
+    def self.define_attr_method(name, value) # :nodoc:
+      class_eval do      
         define_method(name) do
           value
         end
       end
+      
+      self.singleton_class.send :define_method, name do
+        value
+      end
     end
     
-    def self.define_class_attr(name, default)
+    def self.define_class_attr(name, default) # :nodoc:
       define_attr_method name, default
       self.singleton_class.send :define_method, name do |value|
         define_attr_method name, value
@@ -32,66 +71,91 @@ module Backnob
     end
     
     define_class_attr :rails, false
+    define_class_attr :single, false
     
+    # Server creates a new worker passing its key and
+    # any user options as a hash
     def initialize(key, options = {})
-      @key = key
       @options = options
       @results = {}
       @results.extend(MonitorMixin)
+      
+      @wqueue = Queue.new
+      enqueue(key, options)
     end
-        
+    
+    # Access to the logger
     def logger
       @logger ||= Logger.new($stdout)
     end
     
+    def enqueue(key, options = {})
+      @wqueue << [key, options]
+    end
+    
     # Start this worker. Starts a new thread and calls execute
     # for work to be performed.
     def start
       @thread = Thread.new do
-        Thread.pass
-        
-        logger.debug "Starting #{self.class.name} with key #{@key}"
-                  
-        results(:running, true)
+        Thread.pass      
+        @running = true
                 
         if rails
           require 'config/environment'
           ActiveRecord::Base.logger = logger
         end
-
-        # require File.dirname(__FILE__) + "/../../config/environment"
-        # ActiveRecord::Base.logger = logger
         
         begin
-          execute
-        rescue Exception => e
-          results(:error, e)
-        ensure
-          results(:running, false)
-          logger.debug "Finished #{self.class.name} with key #{@key}"
-        end
+          # Get work item off queue
+          @key, @options = @wqueue.pop                      
+          logger.debug "Starting #{self.class.name} with key #{@key}"
+          
+          begin
+            execute unless @key == :stop
+          rescue Exception => e
+            results(:error, e)
+          ensure
+            results(:finished, true)
+            logger.debug "Finished #{self.class.name} with key #{@key}"
+          end
+        end while @running && self.single
+        
+        logger.debug "Worker #{self.class.name} exiting"
+        exit!
       end
     end
     
+    # Stop this worker.
+    def stop
+      @running = false
+      @wqueue << [:stop, {}]
+    end
+    
     # Get or set results. Workers should access results
-    # through this method as it is synchronized
-    def results(key = nil,value = nil)
+    # through this method as it is synchronized.
+    #
+    # Results are marshalled back to the client, so if
+    # you are passing non-standard classes be sure to
+    # require the appropriate files on the client side.
+    def results(key = nil,value = nil)      
       res = nil
 
       @results.synchronize do
+        @results[@key] ||= {}
+        
         if key.nil?
-          res = @results.dup
+          res = @results[@key].dup
         elsif value.nil?      
-          res = @results[key]
+          res = @results[@key][key]
         else
-          if value != @results[key]
-            @results[key] = value
+          if value != @results[@key][key]
+            @results[@key][key] = value
             changed(true) # I've changed
 
             # Marshal results into string
             # for some reason marshal doesn't like hash, so
             # convert to array and convert back on server side
-            obo = ::Marshal.dump([@key, key, value])     
+            obo = ::Marshal.dump([@key, key, value])
             notify_observers(obo)
           end
         end

data/lib/backnob_control.rb

@@ -17,7 +17,7 @@ require 'yaml'
 require File.dirname(__FILE__) + '/backnob/options'
 require File.dirname(__FILE__) + '/backnob/version'
 
-OPTIONS = {:path => Pathname.new('.').realpath.to_s}
+OPTIONS = {:path => Pathname.new('.').realpath.to_s, :pid => '/tmp'}
 OPTIONS.extend(Backnob::Options)
 
 parser = OptionParser.new do |opts|
@@ -47,6 +47,7 @@ BANNER
     end
   end
   opts.on("-lADDRESS", "--listen=ADDRESS", "Set the listen address. Defaults to 127.0.0.1:6444") {|l| OPTIONS[:listen] = l }
+  opts.on("-pDIR", "--pid=DIR", "Set the directory to store the pid and log files. Defaults to /tmp") {|p| OPTIONS[:pid] = p }
   opts.on("-v", "--version", "Show the version") { puts "Backnob server version #{Backnob::VERSION::STRING}"; exit }
   opts.parse!(ARGV)
   
@@ -54,7 +55,11 @@ BANNER
   
   case ARGV.first.to_s.downcase
   when 'start', 'stop', 'run', 'restart'
-    Daemons.run(File.dirname(__FILE__) + '/backnob/server.rb', :dir_mode => :normal, :dir => '/tmp', :mode => :load, :log_output => true)
+    pid_dir = OPTIONS.delete(:pid)
+    Daemons.run(File.dirname(__FILE__) + '/backnob/server.rb', :dir_mode => :normal, 
+                                                               :dir => pid_dir, 
+                                                               :mode => :load, 
+                                                               :log_output => true)
   when 'create', 'generate', 'results'
     require 'backnob/client'
     client = Backnob::Client.new

data/plugin/LICENSE

@@ -0,0 +1,18 @@
+Copyright (c) 2007 Jeremy Wells, Boost New Media Ltd.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of 
+this software and associated documentation files (the "Software"), to deal in 
+the Software without restriction, including without limitation the rights to 
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of 
+the Software, and to permit persons to whom the Software is furnished to do so, 
+subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all 
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS 
+FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER 
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/plugin/README

@@ -0,0 +1,3 @@
+= Backnob
+
+Yeah

data/plugin/conf/test_worker.rb

@@ -0,0 +1,12 @@
+
+class TestWorker < Backnob::Worker
+  def execute
+    results(:data, 'Hello there')
+    sleep 2
+    results(:data, 'booha')
+    sleep 2
+    results(:data, 'bye now')
+  end
+end
+
+Backnob::Server.register(TestWorker)

data/plugin/init.rb

@@ -0,0 +1,5 @@
+require 'backnob/client'
+require 'backnob_ext'
+
+ActiveRecord::Base.send :include, BacknobExt
+ActionController::Base.send :include, BacknobExt

data/plugin/lib/backnob_ext.rb

@@ -0,0 +1,5 @@
+module BacknobExt
+  def backnob
+    @backnob_client ||= Backnob::Client.new
+  end
+end

data/plugin/tasks/backnob_tasks.rake

@@ -0,0 +1,41 @@
+require 'fileutils'
+
+def run
+  begin
+    require 'rubygems'
+  rescue LoadError
+    # no rubygems to load, so we fail silently
+  end
+
+  require 'backnob_control'
+end
+
+namespace :backnob do
+  desc "Start a backnob server"
+  task :start do
+    ARGV = ['-c', RAILS_ROOT + '/config/backnob.yml', 'start']
+    run
+  end
+  
+  desc "Stop a backnob server"
+  task :stop do
+    ARGV = ['stop']
+    run
+  end
+  
+  desc "Restart a backnob server"
+  task :restart do
+    ARGV = ['-c', RAILS_ROOT + '/config/backnob.yml', 'restart']
+    run
+  end
+  
+  desc "Set up the backnob plugin"
+  task :setup do
+    unless File.exists?(RAILS_ROOT + '/config/backnob.yml')
+      FileUtils.cp File.dirname(__FILE__) + '/../conf/backnob.yml', RAILS_ROOT + '/config/backnob.yml'
+    end
+    
+    FileUtils.mkdir_p RAILS_ROOT + '/lib/backnob_workers'
+    FileUtils.cp File.dirname(__FILE__) + '/../conf/test_worker.rb', RAILS_ROOT + '/lib/backnob_workers'
+  end
+end

data/script/destroy

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+APP_ROOT = File.join(File.dirname(__FILE__), '..')
+
+begin
+  require 'rubigen'
+rescue LoadError
+  require 'rubygems'
+  require 'rubigen'
+end
+require 'rubigen/scripts/destroy'
+
+ARGV.shift if ['--help', '-h'].include?(ARGV[0])
+RubiGen::Base.use_component_sources! [:rubygems, :newgem, :newgem_theme]
+RubiGen::Scripts::Destroy.new.run(ARGV)

data/script/generate

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+APP_ROOT = File.join(File.dirname(__FILE__), '..')
+
+begin
+  require 'rubigen'
+rescue LoadError
+  require 'rubygems'
+  require 'rubigen'
+end
+require 'rubigen/scripts/generate'
+
+ARGV.shift if ['--help', '-h'].include?(ARGV[0])
+RubiGen::Base.use_component_sources! [:rubygems, :newgem, :newgem_theme]
+RubiGen::Scripts::Generate.new.run(ARGV)

data/script/txt2html

@@ -0,0 +1,76 @@
+#!/usr/bin/env ruby
+
+require 'rubygems'
+begin
+  require 'newgem'
+rescue LoadError
+  puts "\n\nGenerating the website requires the newgem RubyGem"
+  puts "Install: gem install newgem\n\n"
+  exit(1)
+end
+require 'redcloth'
+require 'syntax/convertors/html'
+require 'erb'
+require File.dirname(__FILE__) + '/../lib/backnob/version.rb'
+
+version  = Backnob::VERSION::STRING
+download = 'http://rubyforge.org/projects/backnob'
+
+class Fixnum
+  def ordinal
+    # teens
+    return 'th' if (10..19).include?(self % 100)
+    # others
+    case self % 10
+    when 1: return 'st'
+    when 2: return 'nd'
+    when 3: return 'rd'
+    else    return 'th'
+    end
+  end
+end
+
+class Time
+  def pretty
+    return "#{mday}#{mday.ordinal} #{strftime('%B')} #{year}"
+  end
+end
+
+def convert_syntax(syntax, source)
+  return Syntax::Convertors::HTML.for_syntax(syntax).convert(source).gsub(%r!^<pre>|</pre>$!,'')
+end
+
+if ARGV.length >= 1
+  src, template = ARGV
+  template ||= File.join(File.dirname(__FILE__), '/../website/template.rhtml')
+  
+else
+  puts("Usage: #{File.split($0).last} source.txt [template.rhtml] > output.html")
+  exit!
+end
+
+template = ERB.new(File.open(template).read)
+
+title = nil
+title_text = nil
+body = nil
+File.open(src) do |fsrc|
+  title_text = fsrc.readline
+  body_text = fsrc.read
+  syntax_items = []
+  body_text.gsub!(%r!<(pre|code)[^>]*?syntax=['"]([^'"]+)[^>]*>(.*?)</\1>!m){
+    ident = syntax_items.length
+    element, syntax, source = $1, $2, $3
+    syntax_items << "<#{element} class='syntax'>#{convert_syntax(syntax, source)}</#{element}>"
+    "syntax-temp-#{ident}"
+  }
+  title = RedCloth.new(title_text).to_html.strip
+  title_text = RedCloth.new(title_text).to_html.gsub(%r!<.*?>!,'').strip
+  body = RedCloth.new(body_text).to_html
+  body.gsub!(%r!(?:<pre><code>)?syntax-temp-(\d+)(?:</code></pre>)?!){ syntax_items[$1.to_i] }
+end
+stat = File.stat(src)
+created = stat.ctime
+modified = stat.mtime
+
+$stdout << template.result(binding)

data/scripts/txt2html

@@ -45,6 +45,7 @@ end
 template = ERB.new(File.open(template).read)
 
 title = nil
+title_text = nil
 body = nil
 File.open(src) do |fsrc|
   title_text = fsrc.readline
@@ -56,7 +57,8 @@ File.open(src) do |fsrc|
     syntax_items << "<#{element} class='syntax'>#{convert_syntax(syntax, source)}</#{element}>"
     "syntax-temp-#{ident}"
   }
-  title = RedCloth.new(title_text).to_html.gsub(%r!<.*?>!,'').strip
+  title = RedCloth.new(title_text).to_html.strip
+  title_text = RedCloth.new(title_text).to_html.gsub(%r!<.*?>!,'').strip
   body = RedCloth.new(body_text).to_html
   body.gsub!(%r!(?:<pre><code>)?syntax-temp-(d+)(?:</code></pre>)?!){ syntax_items[$1.to_i] }
 end

data/tasks/deployment.rake

@@ -0,0 +1,27 @@
+desc 'Release the website and new gem version'
+task :deploy => [:check_version, :website, :release] do
+  puts "Remember to create SVN tag:"
+  puts "svn copy svn+ssh://#{rubyforge_username}@rubyforge.org/var/svn/#{PATH}/trunk " +
+    "svn+ssh://#{rubyforge_username}@rubyforge.org/var/svn/#{PATH}/tags/REL-#{VERS} "
+  puts "Suggested comment:"
+  puts "Tagging release #{CHANGES}"
+end
+
+desc 'Runs tasks website_generate and install_gem as a local deployment of the gem'
+task :local_deploy => [:website_generate, :install_gem]
+
+task :check_version do
+  unless ENV['VERSION']
+    puts 'Must pass a VERSION=x.y.z release version'
+    exit
+  end
+  unless ENV['VERSION'] == VERS
+    puts "Please update your version.rb to match the release version, currently #{VERS}"
+    exit
+  end
+end
+
+desc 'Install the package as a gem, without generating documentation(ri/rdoc)'
+task :install_gem_no_doc => [:clean, :package] do
+  sh "#{'sudo ' unless Hoe::WINDOZE }gem install pkg/*.gem --no-rdoc --no-ri"
+end

data/tasks/environment.rake

@@ -0,0 +1,7 @@
+task :ruby_env do
+  RUBY_APP = if RUBY_PLATFORM =~ /java/
+    "jruby"
+  else
+    "ruby"
+  end unless defined? RUBY_APP
+end