mooktakim-cloud-crowd 0.3.4

data/lib/cloud_crowd/node.rb

@@ -0,0 +1,199 @@
+module CloudCrowd
+  
+  # A Node is a Sinatra/Thin application that runs a single instance per-machine
+  # It registers with the central server, receives WorkUnits, and forks off 
+  # Workers to process them. The actions are:
+  #
+  # [get /heartbeat] Returns 200 OK to let monitoring tools know the server's up.
+  # [post /work] The central server hits <tt>/work</tt> to dispatch a WorkUnit to this Node.
+  class Node < Sinatra::Default
+    
+    # A Node's default port. You only run a single node per machine, so they
+    # can all use the same port without any problems.
+    DEFAULT_PORT        = 9063
+    
+    # A list of regex scrapers, which let us extract the one-minute load 
+    # average and the amount of free memory on different flavors of UNIX.
+    
+    SCRAPE_UPTIME       = /\d+\.\d+/
+    SCRAPE_LINUX_MEMORY = /MemFree:\s+(\d+) kB/
+    SCRAPE_MAC_MEMORY   = /Pages free:\s+(\d+)./   
+    SCRAPE_MAC_PAGE     = /page size of (\d+) bytes/
+    
+    # The interval at which the node monitors the machine's load and memory use
+    # (if configured to do so in config.yml).
+    MONITOR_INTERVAL    = 3
+    
+    # The interval at which the node regularly checks in with central (5 min).
+    CHECK_IN_INTERVAL   = 300
+    
+    # The response sent back when this node is overloaded.
+    OVERLOADED_MESSAGE  = 'Node Overloaded'
+    
+    attr_reader :enabled_actions, :host, :port, :central
+            
+    set :root, ROOT
+    set :authorization_realm, "CloudCrowd"
+    
+    helpers Helpers
+    
+    # methodoverride allows the _method param.
+    enable :methodoverride
+    
+    # Enabling HTTP Authentication turns it on for all requests.
+    # This works the same way as in the central CloudCrowd::Server.
+    before do
+      login_required if CloudCrowd.config[:http_authentication]
+    end
+    
+    # To monitor a Node with Monit, God, Nagios, or another tool, you can hit 
+    # /heartbeat to make sure its still online.
+    get '/heartbeat' do
+      "buh-bump"
+    end
+    
+    # Posts a WorkUnit to this Node. Forks a Worker and returns the process id.
+    # Returns a 503 if this Node is overloaded.
+    post '/work' do
+      throw :halt, [503, OVERLOADED_MESSAGE] if @overloaded
+      unit = JSON.parse(params[:work_unit])
+      pid = fork { Worker.new(self, unit).run }
+      Process.detach(pid)
+      json :pid => pid
+    end
+    
+    # When creating a node, specify the port it should run on.
+    def initialize(port=nil, daemon=false)
+      require 'json'
+      CloudCrowd.identity = :node
+      @central          = CloudCrowd.central_server
+      @host             = Socket.gethostname
+      @enabled_actions  = CloudCrowd.actions.keys
+      @port             = port || DEFAULT_PORT
+      @daemon           = daemon
+      @overloaded       = false
+      @max_load         = CloudCrowd.config[:max_load]
+      @min_memory       = CloudCrowd.config[:min_free_memory]
+      start unless test?
+    end
+    
+    # Starting up a Node registers with the central server and begins to listen
+    # for incoming WorkUnits.
+    def start
+      FileUtils.mkdir_p(CloudCrowd.log_path) if @daemon && !File.exists?(CloudCrowd.log_path)
+      @server          = Thin::Server.new('0.0.0.0', @port, self, :signals => false)
+      @server.tag      = 'cloud-crowd-node'
+      @server.pid_file = CloudCrowd.pid_path('node.pid')
+      @server.log_file = CloudCrowd.log_path('node.log')
+      @server.daemonize if @daemon
+      trap_signals
+      asset_store
+      @server_thread   = Thread.new { @server.start }
+      check_in(true)
+      check_in_periodically
+      monitor_system if @max_load || @min_memory
+      @server_thread.join
+    end
+    
+    # Checking in with the central server informs it of the location and 
+    # configuration of this Node. If it can't check-in, there's no point in 
+    # starting.
+    def check_in(critical=false)
+      @central["/node/#{@host}"].put(
+        :port             => @port,
+        :busy             => @overloaded,
+        :max_workers      => CloudCrowd.config[:max_workers],
+        :enabled_actions  => @enabled_actions.join(',')
+      )
+    rescue RestClient::Exception, Errno::ECONNREFUSED
+      puts "Failed to connect to the central server (#{@central.to_s})."
+      raise SystemExit if critical
+    end
+    
+    # Before exiting, the Node checks out with the central server, releasing all
+    # of its WorkUnits for other Nodes to handle
+    def check_out
+      @central["/node/#{@host}"].delete
+    end
+    
+    # Lazy-initialize the asset_store, preferably after the Node has launched.
+    def asset_store
+      @asset_store ||= AssetStore.new
+    end
+    
+    # Is the node overloaded? If configured, checks if the load average is 
+    # greater than 'max_load', or if the available RAM is less than
+    # 'min_free_memory'.
+    def overloaded?
+      (@max_load && load_average > @max_load) ||
+      (@min_memory && free_memory < @min_memory)
+    end
+    
+    # The current one-minute load average.
+    def load_average
+      `uptime`.match(SCRAPE_UPTIME).to_s.to_f
+    end
+    
+    # The current amount of free memory in megabytes.
+    def free_memory
+      case RUBY_PLATFORM
+      when /darwin/
+        stats = `vm_stat`
+        @mac_page_size ||= stats.match(SCRAPE_MAC_PAGE)[1].to_f / 1048576.0
+        stats.match(SCRAPE_MAC_MEMORY)[1].to_f * @mac_page_size
+      when /linux/
+        `cat /proc/meminfo`.match(SCRAPE_LINUX_MEMORY)[1].to_f / 1024.0
+      else
+        raise NotImplementedError, "'min_free_memory' is not yet implemented on your platform"
+      end
+    end
+    
+    
+    private
+    
+    # Launch a monitoring thread that periodically checks the node's load 
+    # average and the amount of free memory remaining. If we transition out of 
+    # the overloaded state, let central know.
+    def monitor_system
+      @monitor_thread = Thread.new do
+        loop do
+          was_overloaded = @overloaded
+          @overloaded = overloaded?
+          check_in if was_overloaded && !@overloaded
+          sleep MONITOR_INTERVAL
+        end
+      end
+    end
+    
+    # If communication is interrupted for external reasons, the central server 
+    # will assume that the node has gone down. Checking in will let central know 
+    # it's still online.
+    def check_in_periodically
+      @check_in_thread = Thread.new do
+        loop do
+          sleep CHECK_IN_INTERVAL
+          check_in
+        end
+      end
+    end
+    
+    # Trap exit signals in order to shut down cleanly.
+    def trap_signals
+      Signal.trap('QUIT') { shut_down }
+      Signal.trap('INT')  { shut_down }
+      Signal.trap('KILL') { shut_down }
+      Signal.trap('TERM') { shut_down }
+    end
+    
+    # At shut down, de-register with the central server before exiting.
+    def shut_down
+      @check_in_thread.kill if @check_in_thread
+      @monitor_thread.kill if @monitor_thread
+      check_out
+      @server_thread.kill if @server_thread
+      Process.exit
+    end
+    
+  end
+  
+end

data/lib/cloud_crowd/schema.rb

@@ -0,0 +1,50 @@
+# Complete schema for CloudCrowd.
+ActiveRecord::Schema.define(:version => CloudCrowd::SCHEMA_VERSION) do
+
+  create_table "jobs", :force => true do |t|
+    t.integer  "status",                      :null => false
+    t.text     "inputs",                      :null => false
+    t.string   "action",                      :null => false
+    t.text     "options",                     :null => false
+    t.text     "outputs"
+    t.float    "time"
+    t.string   "callback_url"
+    t.string   "email"
+    t.datetime "created_at"
+    t.datetime "updated_at"
+  end
+  
+  create_table "node_records", :force => true do |t|
+    t.string   "host",                                :null => false
+    t.string   "ip_address",                          :null => false
+    t.integer  "port",                                :null => false
+    t.string   "enabled_actions", :default => '',     :null => false
+    t.boolean  "busy",            :default => false,  :null => false
+    t.integer  "max_workers"
+    t.datetime "created_at"
+    t.datetime "updated_at"
+  end
+
+  create_table "work_units", :force => true do |t|
+    t.integer  "status",                          :null => false
+    t.integer  "job_id",                          :null => false
+    t.text     "input",                           :null => false
+    t.string   "action",                          :null => false
+    t.integer  "attempts",      :default => 0,    :null => false
+    t.integer  "node_record_id"
+    t.integer  "worker_pid"
+    t.integer  "reservation"
+    t.float    "time"
+    t.text     "output"
+    t.datetime "created_at"
+    t.datetime "updated_at"
+  end
+
+  # Here be indices. After looking, it seems faster not to have them at all.
+  # 
+  # add_index "jobs", ["status"], :name => "index_jobs_on_status"
+  # add_index "work_units", ["job_id"], :name => "index_work_units_on_job_id"
+  # add_index "work_units", ["worker_pid"], :name => "index_work_units_on_worker_pid"
+  # add_index "work_units", ["worker_pid", "status"], :name => "index_work_units_on_worker_pid_and_status"
+  # add_index "work_units", ["worker_pid", "node_record_id"], :name => "index_work_units_on_worker_pid_and_node_record_id"
+end

data/lib/cloud_crowd/server.rb

@@ -0,0 +1,123 @@
+module CloudCrowd
+  
+  # The main CloudCrowd (Sinatra) application. The actions are:
+  #
+  # == Admin
+  # [get /] Render the admin console, with a progress meter for running jobs.
+  # [get /status] Get the combined JSON of every active job and worker.
+  # [get /worker/:name] Look up the details of a WorkUnit that a Worker is busy processing.
+  # [get /heartbeat] Returns 200 OK to let monitoring tools know the server's up.
+  # 
+  # == Public API
+  # [post /jobs] Begin a new Job. Post with a JSON representation of the job-to-be. (see examples).
+  # [get /jobs/:job_id] Check the status of a Job. Response includes output, if the Job has finished.
+  # [delete /jobs/:job_id] Clean up a Job when you're done downloading the results. Removes all intermediate files.
+  #
+  # == Internal Workers API
+  # [puts /node/:host] Registers a new Node, making it available for processing.
+  # [delete /node/:host] Removes a Node from the registry, freeing up any WorkUnits that it had checked out.
+  # [put /work/:unit_id] Mark a finished WorkUnit as completed or failed, with results.
+  class Server < Sinatra::Default
+    
+    set :root, ROOT
+    set :authorization_realm, "CloudCrowd"
+    
+    helpers Helpers
+    
+    # static serves files from /public, methodoverride allows the _method param.
+    enable :static, :methodoverride
+    
+    # Enabling HTTP Authentication turns it on for all requests.
+    before do
+      login_required if CloudCrowd.config[:http_authentication]
+    end
+    
+    # Render the admin console.
+    get '/' do
+      erb :operations_center
+    end
+    
+    # Get the JSON for every active job in the queue and every active worker
+    # in the system. This action may get a little worrisome as the system grows
+    # larger -- keep it in mind.
+    get '/status' do
+      json(
+        'jobs'            => Job.incomplete, 
+        'nodes'           => NodeRecord.all(:order => 'host desc'),
+        'work_unit_count' => WorkUnit.incomplete.count
+      )
+    end
+    
+    # Get the JSON for what a worker is up to.
+    get '/worker/:name' do
+      json WorkUnit.find_by_worker_name(params[:name]) || {}
+    end
+    
+    # To monitor the central server with Monit, God, Nagios, or another 
+    # monitoring tool, you can hit /heartbeat to make sure.
+    get '/heartbeat' do
+      "buh-bump"
+    end
+    
+    # PUBLIC API:
+    
+    # Start a new job. Accepts a JSON representation of the job-to-be.
+    # Distributes all work units to available nodes.
+    post '/jobs' do
+      job = Job.create_from_request(JSON.parse(params[:job]))
+      WorkUnit.distribute_to_nodes
+      json job
+    end
+    
+    # Check the status of a job, returning the output if finished, and the
+    # number of work units remaining otherwise. 
+    get '/jobs/:job_id' do
+      json current_job
+    end
+    
+    # Cleans up a Job's saved S3 files. Delete a Job after you're done 
+    # downloading the results.
+    delete '/jobs/:job_id' do
+      current_job.destroy
+      json nil
+    end
+    
+    # INTERNAL NODE API:
+    
+    # A new Node will this this action to register its location and 
+    # configuration with the central server. Triggers distribution of WorkUnits. 
+    put '/node/:host' do
+      NodeRecord.check_in(params, request)
+      WorkUnit.distribute_to_nodes
+      json nil
+    end
+    
+    # Deregisters a Node from the central server. Releases and redistributes any 
+    # WorkUnits it may have had checked out.
+    delete '/node/:host' do
+      NodeRecord.destroy_all(:host => params[:host])
+      json nil
+    end
+    
+    # When workers are done with their unit, either successfully on in failure,
+    # they mark it back on the central server and exit. Triggers distribution
+    # of pending work units.
+    put '/work/:work_unit_id' do
+      case params[:status]
+      when 'succeeded' then current_work_unit.finish(params[:output], params[:time])
+      when 'failed'    then current_work_unit.fail(params[:output], params[:time])
+      else             error(500, "Completing a work unit must specify status.")
+      end
+      WorkUnit.distribute_to_nodes
+      json nil
+    end
+    
+    # At initialization record the identity of this Ruby instance as a server.
+    def initialize(*args)
+      super(*args)
+      CloudCrowd.identity = :server
+    end
+    
+  end
+  
+end

data/lib/cloud_crowd/worker.rb

@@ -0,0 +1,149 @@
+module CloudCrowd
+
+  # The Worker, forked off from the Node when a new WorkUnit is received,
+  # launches an Action for processing. Workers will only ever receive WorkUnits
+  # that they are able to handle (for which they have a corresponding action in
+  # their actions directory). If communication with the central server is
+  # interrupted, the Worker will repeatedly attempt to complete its unit --
+  # every Worker::RETRY_WAIT seconds. Any exceptions that take place during
+  # the course of the Action will cause the Worker to mark the WorkUnit as
+  # having failed. When finished, the Worker's process exits, minimizing the
+  # potential for memory leaks.
+  class Worker
+
+    # Wait five seconds to retry, after internal communcication errors.
+    RETRY_WAIT = 5
+
+    attr_reader :pid, :node, :unit, :status
+
+    # A new Worker customizes itself to its WorkUnit at instantiation.
+    def initialize(node, unit)
+      @start_time = Time.now
+      @pid        = $$
+      @node       = node
+      @unit       = unit
+      @status     = @unit['status']
+      @retry_wait = RETRY_WAIT
+    end
+
+    # Return output to the central server, marking the WorkUnit done.
+    def complete_work_unit(result)
+      keep_trying_to "complete work unit" do
+        data = base_params.merge({:status => 'succeeded', :output => result})
+        @node.central["/work/#{data[:id]}"].put(data)
+        log "finished #{display_work_unit} in #{data[:time]} seconds"
+      end
+    end
+
+    # Mark the WorkUnit failed, returning the exception to central.
+    def fail_work_unit(exception)
+      keep_trying_to "mark work unit as failed" do
+        data = base_params.merge({:status => 'failed', :output => {'output' => exception.message}.to_json})
+        @node.central["/work/#{data[:id]}"].put(data)
+        log "failed #{display_work_unit} in #{data[:time]} seconds\n#{exception.message}\n#{exception.backtrace}"
+      end
+    end
+
+    # We expect and require internal communication between the central server
+    # and the workers to succeed. If it fails for any reason, log it, and then
+    # keep trying the same request.
+    def keep_trying_to(title)
+      begin
+        yield
+      rescue RestClient::ResourceNotFound => e
+        log "work unit ##{@unit['id']} doesn't exist. discarding..."
+      rescue Exception => e
+        log "failed to #{title} -- retry in #{@retry_wait} seconds"
+        log e.message
+        log e.backtrace
+        sleep @retry_wait
+        retry
+      end
+    end
+
+    # Loggable details describing what the Worker is up to.
+    def display_work_unit
+      "unit ##{@unit['id']} (#{@unit['action']}/#{CloudCrowd.display_status(@status)})"
+    end
+
+    # Executes the WorkUnit by running the Action, catching all exceptions as
+    # failures. We capture the thread so that we can kill it from the outside,
+    # when exiting.
+    def run_work_unit
+      begin
+        result = nil
+        action_class = CloudCrowd.actions[@unit['action']]
+        action = action_class.new(@status, @unit['input'], enhanced_unit_options, @node.asset_store)
+        Dir.chdir(action.work_directory) do
+          result = case @status
+          when PROCESSING then action.process
+          when SPLITTING  then action.split
+          when MERGING    then action.merge
+          else raise Error::StatusUnspecified, "work units must specify their status"
+          end
+        end
+        action.cleanup_work_directory if action
+        complete_work_unit({'output' => result}.to_json)
+      rescue Exception => e
+        action.cleanup_work_directory if action
+        fail_work_unit(e)
+      end
+    end
+
+    # Run this worker inside of a fork. Attempts to exit cleanly.
+    # Wraps run_work_unit to benchmark the execution time, if requested.
+    def run
+      trap_signals
+      log "starting #{display_work_unit}"
+      if @unit['options']['benchmark']
+        log("ran #{display_work_unit} in " + Benchmark.measure { run_work_unit }.to_s)
+      else
+        run_work_unit
+      end
+      Process.exit!
+    end
+
+    # There are some potentially important attributes of the WorkUnit that we'd
+    # like to pass into the Action -- in case it needs to know them. They will
+    # always be made available in the options hash.
+    def enhanced_unit_options
+      @unit['options'].merge({
+        'job_id'        => @unit['job_id'],
+        'work_unit_id'  => @unit['id'],
+        'attempts'      => @unit['attempts']
+      })
+    end
+
+    # How long has this worker been running for?
+    def time_taken
+      Time.now - @start_time
+    end
+
+
+    private
+
+    # Common parameters to send back to central upon unit completion,
+    # regardless of success or failure.
+    def base_params
+      { :pid  => @pid,
+        :id   => @unit['id'],
+        :time => time_taken }
+    end
+
+    # Log a message to the daemon log. Includes PID for identification.
+    def log(message)
+      puts "Worker ##{@pid}: #{message}" unless ENV['RACK_ENV'] == 'test'
+    end
+
+    # When signaled to exit, make sure that the Worker shuts down without firing
+    # the Node's at_exit callbacks.
+    def trap_signals
+      Signal.trap('QUIT') { Process.exit! }
+      Signal.trap('INT')  { Process.exit! }
+      Signal.trap('KILL') { Process.exit! }
+      Signal.trap('TERM') { Process.exit! }
+    end
+
+  end
+
+end

data/mooktakim-cloud-crowd.gemspec

@@ -0,0 +1,116 @@
+Gem::Specification.new do |s|
+  s.name      = 'mooktakim-cloud-crowd'
+  s.version   = '0.3.4'         # Keep version in sync with cloud-cloud.rb
+  s.date      = '2010-02-26'
+
+  s.homepage    = "http://wiki.github.com/documentcloud/cloud-crowd"
+  s.summary     = "Parallel Processing for the Rest of Us"
+  s.description = <<-EOS
+    The crowd, suddenly there where there was nothing before, is a mysterious and
+    universal phenomenon. A few people may have been standing together -- five, ten
+    or twelve, nor more; nothing has been announced, nothing is expected. Suddenly
+    everywhere is black with people and more come streaming from all sides as though
+    streets had only one direction.
+  EOS
+
+  s.authors           = ['Jeremy Ashkenas', 'Mooktakim Ahmed']
+  s.email             = 'jeremy@documentcloud.org'
+  s.rubyforge_project = 'cloud-crowd'
+
+  s.require_paths     = ['lib']
+  s.executables       = ['crowd']
+
+  s.has_rdoc          = true
+  s.extra_rdoc_files  = ['README']
+  s.rdoc_options      << '--title'    << 'CloudCrowd | Parallel Processing for the Rest of Us' <<
+                         '--exclude'  << 'test' <<
+                         '--main'     << 'README' <<
+                         '--all'
+
+  s.add_dependency 'sinatra',       ['>= 0.9.4']
+  s.add_dependency 'activerecord',  ['>= 2.3.3']
+  s.add_dependency 'json',          ['>= 1.1.7']
+  s.add_dependency 'rest-client',   ['>= 1.0.3']
+  s.add_dependency 'right_aws',     ['>= 1.10.0']
+  s.add_dependency 'thin',          ['>= 1.2.4']
+
+  if s.respond_to?(:add_development_dependency)
+    s.add_development_dependency 'faker',               ['>= 0.3.1']
+    s.add_development_dependency 'thoughtbot-shoulda',  ['>= 2.10.2']
+    s.add_development_dependency 'notahat-machinist',   ['>= 1.0.3']
+    s.add_development_dependency 'rack-test',           ['>= 0.4.1']
+    s.add_development_dependency 'mocha',               ['>= 0.9.7']
+  end
+
+  s.files = %w(
+actions/graphics_magick.rb
+actions/process_pdfs.rb
+actions/word_count.rb
+mooktakim-cloud-crowd.gemspec
+config/config.example.ru
+config/config.example.yml
+config/database.example.yml
+EPIGRAPHS
+examples/graphics_magick_example.rb
+examples/process_pdfs_example.rb
+examples/word_count_example.rb
+lib/cloud-crowd.rb
+lib/cloud_crowd/action.rb
+lib/cloud_crowd/asset_store/filesystem_store.rb
+lib/cloud_crowd/asset_store/s3_store.rb
+lib/cloud_crowd/asset_store.rb
+lib/cloud_crowd/command_line.rb
+lib/cloud_crowd/exceptions.rb
+lib/cloud_crowd/helpers/authorization.rb
+lib/cloud_crowd/helpers/resources.rb
+lib/cloud_crowd/helpers.rb
+lib/cloud_crowd/inflector.rb
+lib/cloud_crowd/models/job.rb
+lib/cloud_crowd/models/node_record.rb
+lib/cloud_crowd/models/work_unit.rb
+lib/cloud_crowd/models.rb
+lib/cloud_crowd/node.rb
+lib/cloud_crowd/schema.rb
+lib/cloud_crowd/server.rb
+lib/cloud_crowd/worker.rb
+LICENSE
+public/css/admin_console.css
+public/css/reset.css
+public/images/bullet_green.png
+public/images/bullet_white.png
+public/images/cloud_hand.png
+public/images/header_back.png
+public/images/logo.png
+public/images/queue_fill.png
+public/images/server.png
+public/images/server_busy.png
+public/images/server_error.png
+public/images/sidebar_bottom.png
+public/images/sidebar_top.png
+public/images/worker_info.png
+public/images/worker_info_loading.gif
+public/js/admin_console.js
+public/js/excanvas.js
+public/js/flot.js
+public/js/jquery.js
+README
+test/acceptance/test_node.rb
+test/acceptance/test_failing_work_units.rb
+test/acceptance/test_server.rb
+test/acceptance/test_word_count.rb
+test/blueprints.rb
+test/config/config.ru
+test/config/config.yml
+test/config/database.yml
+test/config/actions/failure_testing.rb
+test/test_helper.rb
+test/unit/test_action.rb
+test/unit/test_configuration.rb
+test/unit/test_node.rb
+test/unit/test_node_record.rb
+test/unit/test_job.rb
+test/unit/test_worker.rb
+test/unit/test_work_unit.rb
+views/operations_center.erb
+)
+end