documentcloud-cloud-crowd 0.1.0 → 0.1.1

data/README

@@ -30,19 +30,19 @@
     * split -> process -> merge
     * As easy as `gem install cloud-crowd`
 
-		Well-suited for:
-		
-		* Generating or resizing images.
-		* Encoding video.
-		* Running text extraction or OCR on PDFs.
-		* Migrating a large file set or database.
-		* Web scraping.
+    Well-suited for:
+    
+    * Generating or resizing images.
+    * Encoding video.
+    * Running text extraction or OCR on PDFs.
+    * Migrating a large file set or database.
+    * Web scraping.
     
     
   ~ Documentation ~
   
     Wiki: http://wiki.github.com/documentcloud/cloud-crowd
-		Rdoc: http://rdoc.info/projects/documentcloud/cloud-crowd
+    Rdoc: http://rdoc.info/projects/documentcloud/cloud-crowd
   
   
   ~ Getting started ~

data/cloud-crowd.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
   s.name      = 'cloud-crowd'
-  s.version   = '0.1.0'         # Keep version in sync with cloud-cloud.rb
-  s.date      = '2009-09-01'
+  s.version   = '0.1.1'         # Keep version in sync with cloud-cloud.rb
+  s.date      = '2009-09-15'
 
   s.homepage    = "http://wiki.github.com/documentcloud/cloud-crowd"
   s.summary     = "Parallel Processing for the Rest of Us"
@@ -32,7 +32,7 @@ Gem::Specification.new do |s|
   s.add_dependency 'json',          ['>= 1.1.7']
   s.add_dependency 'rest-client',   ['>= 1.0.3']
   s.add_dependency 'right_aws',     ['>= 1.10.0']
-  s.add_dependency 'daemons',       ['>= 1.0.10']
+  s.add_dependency 'thin',          ['>= 1.2.4']
 
   if s.respond_to?(:add_development_dependency)
     s.add_development_dependency 'faker',               ['>= 0.3.1']
@@ -56,23 +56,22 @@ examples/process_pdfs_example.rb
 examples/word_count_example.rb
 lib/cloud-crowd.rb
 lib/cloud_crowd/action.rb
-lib/cloud_crowd/app.rb
 lib/cloud_crowd/asset_store/filesystem_store.rb
 lib/cloud_crowd/asset_store/s3_store.rb
 lib/cloud_crowd/asset_store.rb
 lib/cloud_crowd/command_line.rb
-lib/cloud_crowd/daemon.rb
 lib/cloud_crowd/exceptions.rb
 lib/cloud_crowd/helpers/authorization.rb
 lib/cloud_crowd/helpers/resources.rb
 lib/cloud_crowd/helpers.rb
 lib/cloud_crowd/inflector.rb
 lib/cloud_crowd/models/job.rb
+lib/cloud_crowd/models/node_record.rb
 lib/cloud_crowd/models/work_unit.rb
-lib/cloud_crowd/models/worker_record.rb
 lib/cloud_crowd/models.rb
-lib/cloud_crowd/runner.rb
+lib/cloud_crowd/node.rb
 lib/cloud_crowd/schema.rb
+lib/cloud_crowd/server.rb
 lib/cloud_crowd/worker.rb
 LICENSE
 public/css/admin_console.css
@@ -83,6 +82,7 @@ public/images/cloud_hand.png
 public/images/header_back.png
 public/images/logo.png
 public/images/queue_fill.png
+public/images/server.png
 public/images/server_error.png
 public/images/sidebar_bottom.png
 public/images/sidebar_top.png
@@ -93,7 +93,7 @@ public/js/excanvas.js
 public/js/flot.js
 public/js/jquery.js
 README
-test/acceptance/test_app.rb
+test/acceptance/test_server.rb
 test/acceptance/test_failing_work_units.rb
 test/acceptance/test_word_count.rb
 test/blueprints.rb

data/config/config.example.ru

@@ -4,7 +4,13 @@
 # using any Rack-compliant server handler. For example, start up three servers 
 # with a specified port number, using Thin:
 #
-# thin start -R config.ru -p 9173 --servers 3
+# thin start -R config.ru --servers 3
+#
+# Or a single server with Unicorn:
+#
+# unicorn config.ru
+#
+
 
 require 'rubygems'
 require 'cloud-crowd'
@@ -13,5 +19,5 @@ CloudCrowd.configure(File.dirname(__FILE__) + '/config.yml')
 CloudCrowd.configure_database(File.dirname(__FILE__) + '/database.yml')
 
 map '/' do
-  run CloudCrowd::App
+  run CloudCrowd::Server
 end

data/config/config.example.yml

@@ -1,6 +1,11 @@
 # The URL where you're planning on running the central server/queue/database.
 :central_server:          http://localhost:9173
 
+# Set the maximum number of workers allowed per-node. Workers only run while 
+# there's work to be done. It's best to set 'max_workers' below the point where 
+# you'd start to swap or peg your CPU (as determined by experiment).
+:max_workers:             5
+
 # The storage back-end that you'd like to use for intermediate and final results
 # of processing. 's3' and 'filesystem' are supported. 'filesystem' should only
 # be used in development, or on single-machine installations.
@@ -29,20 +34,6 @@
 # additional actions from a location of your choice.
 # :actions_path: /path/to/actions
 
-# Set the following numbers to tweak the configuration of your worker daemons. 
-# Optimum results will depend on proportion of the Memory/CPU/IO bottlenecks
-# in your actions, the number of central servers you have running, and your
-# desired balance between latency and traffic.
-  
-# The number of workers that `crowd workers start` spins up.
-:num_workers:             3
-
-# The minimum number of seconds a worker waits between checking the job queue.
-:min_worker_wait:         1
-
-# The maximum number of seconds a worker waits between checking the job queue.
-:max_worker_wait:         5
-
 # The number of separate attempts that will be made to process an individual
 # work unit, before marking it as having failed.
-:work_unit_retries:       3
+:work_unit_retries:       3

data/examples/process_pdfs_example.rb

@@ -17,7 +17,7 @@ RestClient.post('http://localhost:9173/jobs',
       'http://tigger.uic.edu/~victor/personal/futurism.pdf',
       'http://www.jonasmekas.com/Catalog_excerpt/The%20Avant-Garde%20From%20Futurism%20to%20Fluxus.pdf',
       'http://www.dzignism.com/articles/Futurist.Manifesto.pdf',
-      'http://benfry.com/phd/dissertation-050312b-acrobat.pdf'
+      'http://www.pitt.edu/~slavic/sisc/SISC4/dadswell.pdf'
     ],
     
     'options' => {

data/examples/word_count_example.rb

@@ -39,3 +39,4 @@ RestClient.post('http://localhost:9173/jobs',
 )
 
 # With 23 Workers running, and over Wifi, it counted all the words in 5.5 secs.
+# On a fast internet connection, you may not even see this job show up.

data/lib/cloud-crowd.rb

@@ -5,16 +5,15 @@ $LOAD_PATH.unshift File.expand_path(File.dirname(__FILE__))
 # Common Gems:
 require 'rubygems'
 gem 'activerecord'
-gem 'daemons'
 gem 'json'
 gem 'rest-client'
 gem 'right_aws'
 gem 'sinatra'
+gem 'thin'
 
 # Autoloading for all the pieces which may or may not be needed:
 autoload :ActiveRecord, 'activerecord'
 autoload :Benchmark,    'benchmark'
-autoload :Daemons,      'daemons'
 autoload :Digest,       'digest'
 autoload :ERB,          'erb'
 autoload :FileUtils,    'fileutils'
@@ -23,6 +22,7 @@ autoload :RestClient,   'restclient'
 autoload :RightAws,     'right_aws'
 autoload :Sinatra,      'sinatra'
 autoload :Socket,       'socket'
+autoload :Thin,         'thin'
 autoload :YAML,         'yaml'
 
 # Common code which should really be required in every circumstance.
@@ -31,21 +31,22 @@ require 'cloud_crowd/exceptions'
 module CloudCrowd
   
   # Autoload all the CloudCrowd classes which may not be required.
-  autoload :App,          'cloud_crowd/app'
   autoload :Action,       'cloud_crowd/action'
   autoload :AssetStore,   'cloud_crowd/asset_store'
   autoload :Helpers,      'cloud_crowd/helpers'
   autoload :Inflector,    'cloud_crowd/inflector'
   autoload :Job,          'cloud_crowd/models'
+  autoload :Node,         'cloud_crowd/node'
+  autoload :NodeRecord,   'cloud_crowd/models'
+  autoload :Server,       'cloud_crowd/server'
   autoload :Worker,       'cloud_crowd/worker'
   autoload :WorkUnit,     'cloud_crowd/models'
-  autoload :WorkerRecord, 'cloud_crowd/models'
   
   # Root directory of the CloudCrowd gem.
   ROOT        = File.expand_path(File.dirname(__FILE__) + '/..')
   
   # Keep the version in sync with the gemspec.
-  VERSION     = '0.1.0'
+  VERSION     = '0.1.1'
     
   # A Job is processing if its WorkUnits in the queue to be handled by workers.
   PROCESSING  = 1

data/lib/cloud_crowd/action.rb

@@ -38,12 +38,16 @@ module CloudCrowd
     
     # Download a file to the specified path.
     def download(url, path)
-      if url.match(FILE_URL)
-        FileUtils.cp(url.sub(FILE_URL, ''), path)
-      else
-        resp = RestClient::Request.execute(:url => url, :method => :get, :raw_response => true)
-        FileUtils.mv resp.file.path, path
-      end
+      URI.parse(url) # Sanity check.
+      `curl -s "#{url}" > "#{path}"`
+      # if url.match(FILE_URL)
+      #   FileUtils.cp(url.sub(FILE_URL, ''), path)
+      # else
+      #   # An alternative would be shelling out: `curl -s "#{url}" > "#{path}"`
+      #   puts url
+      #   resp = RestClient::Request.execute(:url => url, :method => :get, :raw_response => true)
+      #   FileUtils.mv resp.file.path, path
+      # end
       path
     end
     
@@ -55,7 +59,7 @@ module CloudCrowd
     end
     
     # After the Action has finished, we remove the work directory and return
-    # to the root directory (where daemons run by default).
+    # to the root directory (where workers run by default).
     def cleanup_work_directory
       FileUtils.rm_r(@work_directory) if File.exists?(@work_directory)
     end

data/lib/cloud_crowd/asset_store/filesystem_store.rb

@@ -6,6 +6,11 @@ module CloudCrowd
     # installation.
     module FilesystemStore
       
+      # Make sure that local storage is writeable before starting.
+      def setup
+        raise Error::StorageNotWritable, "#{LOCAL_STORAGE_PATH} is not writable" unless File.writable?(LOCAL_STORAGE_PATH)
+      end
+      
       # Save a file to somewhere semi-persistent on the filesystem. Can be used
       # in development, when offline, or if you happen to have a single-machine
       # CloudCrowd installation. To use, configure <tt>:storage => 'filesystem'</tt>.

data/lib/cloud_crowd/asset_store/s3_store.rb

@@ -5,11 +5,16 @@ module CloudCrowd
     # on S3 for all resulting files.
     module S3Store
       
+      # Configure authentication and establish a connection to S3, first thing.
+      def setup
+        @use_auth = CloudCrowd.config[:use_s3_authentication]
+        establish_s3_connection
+      end
+      
       # Save a finished file from local storage to S3. Save it publicly unless 
       # we're configured to use S3 authentication. Authenticated links expire
       # after one day by default.
       def save(local_path, save_path)
-        ensure_s3_connection
         if @use_auth
           @bucket.put(save_path, File.open(local_path), {}, 'private')
           @s3.interface.get_link(@bucket, save_path)
@@ -21,13 +26,12 @@ module CloudCrowd
             
       # Remove all of a Job's resulting files from S3, both intermediate and finished.
       def cleanup(job)
-        ensure_s3_connection
         @bucket.delete_folder("#{job.action}/job_#{job.id}")
       end
       
       # Workers, through the course of many WorkUnits, keep around an AssetStore.
       # Ensure we have a persistent S3 connection after first use.
-      def ensure_s3_connection
+      def establish_s3_connection
         unless @s3 && @bucket
           params = {:port => 80, :protocol => 'http'}
           @s3 = RightAws::S3.new(CloudCrowd.config[:aws_access_key], CloudCrowd.config[:aws_secret_key], params)

data/lib/cloud_crowd/asset_store.rb

@@ -25,9 +25,9 @@ module CloudCrowd
     
     # Creating the AssetStore ensures that its scratch directory exists.
     def initialize
-      @use_auth = CloudCrowd.config[:use_s3_authentication]
       FileUtils.mkdir_p temp_storage_path unless File.exists? temp_storage_path
       raise Error::StorageNotWritable, "#{temp_storage_path} is not writable" unless File.writable?(temp_storage_path)
+      setup if respond_to? :setup
     end
     
     # Get the path to CloudCrowd's temporary local storage. All actions run

data/lib/cloud_crowd/command_line.rb

@@ -9,9 +9,6 @@ module CloudCrowd
     # Reference the absolute path to the root.
     CC_ROOT = File.expand_path(File.dirname(__FILE__) + '/../..')
     
-    # Path to the Daemons gem script which launches workers.
-    WORKER_RUNNER = File.expand_path("#{CC_ROOT}/lib/cloud_crowd/runner.rb")
-    
     # Command-line banner for the usage message.
     BANNER = <<-EOS
 CloudCrowd is a MapReduce-inspired Parallel Processing System for Ruby.
@@ -24,7 +21,7 @@ Usage: crowd COMMAND OPTIONS
 Commands:
   install       Install the CloudCrowd configuration files to the specified directory
   server        Start up the central server (requires a database)
-  workers       Control worker daemons, use: (start | stop | restart | status | run)
+  node          Start up a worker node (only one node per machine, please)
   console       Launch a CloudCrowd console, connected to the central database
   load_schema   Load the schema into the database specified by database.yml
 
@@ -38,7 +35,7 @@ Options:
       case command
       when 'console'      then run_console
       when 'server'       then run_server
-      when 'workers'      then run_workers_command
+      when 'node'         then run_node
       when 'load_schema'  then run_load_schema
       when 'install'      then run_install
       else                     usage
@@ -63,6 +60,7 @@ Options:
     # (Mongrel, falling back to WEBrick). The equivalent of Rails' script/server.
     def run_server
       ensure_config
+      @options[:port] ||= 9173
       require 'rubygems'
       rackup_path = File.expand_path("#{@options[:config_path]}/config.ru")
       if Gem.available? 'thin'
@@ -72,6 +70,14 @@ Options:
       end
     end
     
+    # Launch a Node. Please only run a single node per machine. The Node process
+    # will be long-lived, although its workers will come and go.
+    def run_node
+      ENV['RACK_ENV'] = @options['environment']
+      load_code
+      Node.new(@options[:port])
+    end
+    
     # Load in the database schema to the database specified in 'database.yml'.
     def run_load_schema
       load_code
@@ -86,51 +92,11 @@ Options:
       install_path = ARGV.shift || '.'
       FileUtils.mkdir_p install_path unless File.exists?(install_path)
       install_file "#{CC_ROOT}/config/config.example.yml", "#{install_path}/config.yml"
-      install_file "#{CC_ROOT}/config/config.example.ru", "#{install_path}/config.ru"
       install_file "#{CC_ROOT}/config/database.example.yml", "#{install_path}/database.yml"
+      install_file "#{CC_ROOT}/config/config.example.ru", "#{install_path}/config.ru"
       install_file "#{CC_ROOT}/actions", "#{install_path}/actions", true
     end
     
-    # Manipulate worker daemons -- handles all commands that the Daemons gem
-    # provides: start, stop, restart, run, and status.
-    def run_workers_command
-      ensure_config
-      command = ARGV.shift
-      case command
-      when 'start'    then start_workers
-      when 'stop'     then stop_workers
-      when 'restart'  then stop_workers && start_workers
-      when 'run'      then run_worker
-      when 'status'   then show_worker_status
-      else                 usage
-      end
-    end
-    
-    # Start up N workers, specified by argument or the number of workers in
-    # config.yml.
-    def start_workers
-      load_code
-      num_workers = @options[:num_workers] || CloudCrowd.config[:num_workers]
-      num_workers.times do
-        `CLOUD_CROWD_CONFIG='#{File.expand_path(@options[:config_path] + "/config.yml")}' ruby #{WORKER_RUNNER} start`
-      end
-    end
-    
-    # For debugging, run a single worker in the current process, showing output.
-    def run_worker
-      exec "CLOUD_CROWD_CONFIG='#{File.expand_path(@options[:config_path] + "/config.yml")}' ruby #{WORKER_RUNNER} run"
-    end
-    
-    # Stop all active workers.
-    def stop_workers
-      `ruby #{WORKER_RUNNER} stop`
-    end
-
-    # Display the status of all active workers.
-    def show_worker_status
-      puts `ruby #{WORKER_RUNNER} status`
-    end
-    
     # Print `crowd` usage.
     def usage
       puts "\n#{@option_parser}\n"
@@ -150,7 +116,6 @@ Options:
     # Parse all options for all commands.
     def parse_options
       @options = {
-        :port         => 9173,
         :environment  => 'production',
         :config_path  => ENV['CLOUD_CROWD_CONFIG'] || '.'
       }
@@ -158,17 +123,14 @@ Options:
         opts.on('-c', '--config PATH', 'path to configuration directory') do |conf_path|
           @options[:config_path] = conf_path
         end
-        opts.on('-n', '--num-workers NUM', OptionParser::DecimalInteger, 'number of worker processes') do |num|
-          @options[:num_workers] = num
-        end
-        opts.on('-p', '--port PORT', 'central server port number') do |port_num|
+        opts.on('-p', '--port PORT', 'port number for server (central or node)') do |port_num|
           @options[:port] = port_num
         end
         opts.on('-e', '--environment ENV', 'server environment (sinatra)') do |env|
           @options[:environment] = env
         end
         opts.on_tail('-v', '--version', 'show version') do
-          load_code
+          require "#{CC_ROOT}/lib/cloud-crowd"
           puts "CloudCrowd version #{VERSION}"
           exit
         end
@@ -181,7 +143,6 @@ Options:
     # Not all commands require this.
     def load_code
       ensure_config
-      require 'rubygems'
       require "#{CC_ROOT}/lib/cloud-crowd"
       CloudCrowd.configure("#{@options[:config_path]}/config.yml")
     end

data/lib/cloud_crowd/exceptions.rb

@@ -8,6 +8,10 @@ module CloudCrowd
     # exist.
     class ActionNotFound < Error
     end
+    
+    # CentralServerUnavailable is used then the central server can't be reached.
+    class CentralServerUnavailable < Error
+    end
   
     # StorageNotFound is raised when config.yml specifies a storage back end that
     # doesn't exist.

data/lib/cloud_crowd/helpers/authorization.rb

@@ -23,7 +23,7 @@ module CloudCrowd
       # A request is authorized if its login and password match those stored
       # in config.yml, or if authentication is disabled. If authentication is
       # turned on, then every request is authenticated, including between 
-      # the worker daemons and the central server.
+      # the nodes and the central server.
       def authorize(login, password)
         return true unless CloudCrowd.config[:use_http_authentication]
         return CloudCrowd.config[:login] == login &&
@@ -37,7 +37,7 @@ module CloudCrowd
         @auth ||= Rack::Auth::Basic::Request.new(request.env)
       end
       
-      def unauthorized!(realm = App.authorization_realm)
+      def unauthorized!(realm = Server.authorization_realm)
         response['WWW-Authenticate'] = "Basic realm=\"#{realm}\""
         halt 401, 'Authorization Required'
       end

data/lib/cloud_crowd/helpers/resources.rb

@@ -20,26 +20,6 @@ module CloudCrowd
         @work_unit ||= WorkUnit.find_by_id(params[:work_unit_id]) or raise Sinatra::NotFound
       end
       
-      # Try to fetch a work unit from the queue. If none are pending, respond
-      # with no content.
-      def dequeue_work_unit(offset=0)
-        handle_conflicts do
-          worker, actions = params[:worker_name], params[:worker_actions].split(',')
-          WorkUnit.dequeue(worker, actions, offset)
-        end
-      end
-      
-      # We're using ActiveRecords optimistic locking, so stale work units
-      # may sometimes arise. handle_conflicts responds with a the HTTP status
-      # code of your choosing if the update failed to be applied.
-      def handle_conflicts(code=204)
-        begin
-          yield
-        rescue ActiveRecord::StaleObjectError => e
-          return status(code) && ''
-        end
-      end
-      
     end
   end
 end

data/lib/cloud_crowd/models/job.rb

@@ -31,30 +31,39 @@ module CloudCrowd
     # finished, if so, continue on to the next phase of the job. 
     def check_for_completion
       return unless all_work_units_complete?
-      transition_to_next_phase               
-      output_list = gather_outputs_from_work_units
-      
-      if complete?
-        self.outputs = output_list.to_json
-        self.time = Time.now - self.created_at
-      end
-      self.save
+      set_next_status    
+      outs = gather_outputs_from_work_units
+      update_attributes(:outputs => outs.to_json, :time => time_taken) if complete?
       
       case self.status
-      when PROCESSING then queue_for_workers(output_list.map {|o| JSON.parse(o) }.flatten)
-      when MERGING    then queue_for_workers(output_list.to_json)
+      when PROCESSING then queue_for_workers(outs.map {|o| JSON.parse(o) }.flatten)
+      when MERGING    then queue_for_workers(outs.to_json)
       else                 fire_callback
       end
       self
     end
     
+    # Transition this Job's status to the appropriate next status.
+    def set_next_status
+      update_attribute(:status,
+        any_work_units_failed? ? FAILED     :
+        self.splitting?        ? PROCESSING :
+        self.mergeable?        ? MERGING    :
+                                 SUCCEEDED
+      )
+    end
+    
     # If a <tt>callback_url</tt> is defined, post the Job's JSON to it upon 
     # completion. The <tt>callback_url</tt> may include HTTP basic authentication,
     # if you like:
     #   http://user:password@example.com/job_complete
+    # If the callback_url is successfully pinged, we proceed to cleanup the job.
+    # TODO: This should be moved into a Work Unit...
     def fire_callback
+      return unless callback_url
       begin
-        RestClient.post(callback_url, {:job => self.to_json}) if callback_url
+        RestClient.post(callback_url, {:job => self.to_json})
+        self.destroy
       rescue RestClient::Exception => e
         puts "Failed to fire job callback. Hmmm, what should happen here?"
       end
@@ -62,15 +71,12 @@ module CloudCrowd
     
     # Cleaning up after a job will remove all of its files from S3. Destroying
     # a Job calls cleanup_assets first.
+    # TODO: Convert this into a 'cleanup' work unit that gets run by a worker.
     def cleanup_assets
       AssetStore.new.cleanup(self)
     end
     
     # Have all of the WorkUnits finished? 
-    #--
-    # We could trade reads for writes here
-    # by keeping a completed_count on the Job itself.
-    #++
     def all_work_units_complete?
       self.work_units.incomplete.count <= 0
     end
@@ -98,10 +104,11 @@ module CloudCrowd
     end
     
     # How complete is this Job?
+    # Unfortunately, with the current processing sequence, the percent_complete
+    # can pull a fast one and go backwards.
     def percent_complete
-      return 0   if splitting?
-      return 100 if complete?
       return 99  if merging?
+      return 100 if complete?
       (work_units.complete.count / work_units.count.to_f * 100).round
     end
     
@@ -143,21 +150,13 @@ module CloudCrowd
       self.work_units.complete.destroy_all
       outs
     end
-    
-    # Transition this Job's status to the appropriate next status.
-    def transition_to_next_phase
-      self.status = any_work_units_failed? ? FAILED     :
-                    self.splitting?        ? PROCESSING :
-                    self.mergeable?        ? MERGING    :
-                                             SUCCEEDED
-    end
         
     # When starting a new job, or moving to a new stage, split up the inputs 
     # into WorkUnits, and queue them. Workers will start picking them up right
     # away.
     def queue_for_workers(input=nil)
       input ||= JSON.parse(self.inputs)
-      [input].flatten.each do |wu_input|
+      [input].flatten.map do |wu_input|
         WorkUnit.create(
           :job    => self, 
           :action => self.action,