mooktakim-cloud-crowd 0.3.4

data/lib/cloud_crowd/exceptions.rb

@@ -0,0 +1,46 @@
+module CloudCrowd
+  
+  # Base Error class which all custom CloudCrowd exceptions inherit from.
+  # Rescuing CloudCrowd::Error (or RuntimeError) will get all custom exceptions.
+  # If your cluster is correctly configured, you should never expect to see any
+  # of these.
+  class Error < RuntimeError
+    
+    # ActionNotFound is raised when a job is created for an action that doesn't 
+    # exist.
+    class ActionNotFound < Error
+    end
+  
+    # StorageNotFound is raised when config.yml specifies a storage back-end that
+    # doesn't exist.
+    class StorageNotFound < Error
+    end
+    
+    # If the AssetStore can't write to its scratch directory.
+    class StorageNotWritable < Error
+    end
+    
+    # StatusUnspecified is raised when a WorkUnit returns without a valid
+    # status code.
+    class StatusUnspecified < Error
+    end
+    
+    # MissingConfiguration is raised when we're trying to run a method that
+    # needs configuration not present in config.yml.
+    class MissingConfiguration < Error
+    end
+    
+    # CommandFailed is raised when an action shells out, and the external 
+    # command returns a non-zero exit code.
+    class CommandFailed < Error
+      attr_reader :exit_code
+      
+      def initialize(message, exit_code)
+        super(message)
+        @exit_code = exit_code
+      end
+    end
+    
+  end
+  
+end

data/lib/cloud_crowd/helpers/authorization.rb

@@ -0,0 +1,52 @@
+module CloudCrowd
+  module Helpers
+    
+    # Authorization takes after sinatra-authorization... See 
+    # http://github.com/integrity/sinatra-authorization
+    # for the original.
+    module Authorization
+      
+      # Ensure that the request includes the correct credentials.
+      def login_required
+        return if authorized?
+        unauthorized! unless auth.provided?
+        bad_request!  unless auth.basic?
+        unauthorized! unless authorize(*auth.credentials)
+        request.env['REMOTE_USER'] = auth.username
+      end
+      
+      # Has the request been authenticated?
+      def authorized?
+        !!request.env['REMOTE_USER']
+      end
+      
+      # A request is authorized if its login and password match those stored
+      # in config.yml, or if authentication is disabled. If authentication is
+      # turned on, then every request is authenticated, including between 
+      # the nodes and the central server.
+      def authorize(login, password)
+        return true unless CloudCrowd.config[:http_authentication]
+        return CloudCrowd.config[:login] == login &&
+               CloudCrowd.config[:password] == password
+      end
+      
+      
+      private
+      
+      # Provide a Rack Authorization object.
+      def auth
+        @auth ||= Rack::Auth::Basic::Request.new(request.env)
+      end
+      
+      # Unauthorized requests will prompt the browser to provide credentials.
+      def unauthorized!(realm = Server.authorization_realm)
+        response['WWW-Authenticate'] = "Basic realm=\"#{realm}\""
+        halt 401, 'Authorization Required'
+      end
+      
+      def bad_request!
+        halt 400, 'Bad Request'
+      end
+    end
+  end
+end

data/lib/cloud_crowd/helpers/resources.rb

@@ -0,0 +1,25 @@
+module CloudCrowd
+  module Helpers
+    module Resources
+      
+      # Convenience method for responding with JSON. Sets the content-type, 
+      # serializes, and allows empty responses.
+      def json(obj)
+        content_type :json
+        return status(204) && '' if obj.nil?
+        obj.to_json
+      end
+      
+      # Lazy-fetch the job specified by <tt>job_id</tt>.
+      def current_job
+        @job ||= Job.find_by_id(params[:job_id]) or raise Sinatra::NotFound
+      end
+      
+      # Lazy-fetch the WorkUnit specified by <tt>work_unit_id</tt>.
+      def current_work_unit
+        @work_unit ||= WorkUnit.find_by_id(params[:work_unit_id]) or raise Sinatra::NotFound
+      end
+      
+    end
+  end
+end

data/lib/cloud_crowd/helpers.rb

@@ -0,0 +1,8 @@
+require 'cloud_crowd/helpers/authorization'
+require 'cloud_crowd/helpers/resources'
+
+module CloudCrowd
+  module Helpers
+    include Authorization, Resources #, Rack::Utils
+  end
+end

data/lib/cloud_crowd/inflector.rb

@@ -0,0 +1,19 @@
+module CloudCrowd
+  
+  # Pilfered in parts from the ActiveSupport::Inflector.
+  module Inflector
+  
+    def self.camelize(word)
+      word.to_s.gsub(/\/(.?)/) { "::#{$1.upcase}" }.gsub(/(?:^|_)(.)/) { $1.upcase }
+    end
+    
+    def self.underscore(word)
+      word.to_s.gsub(/::/, '/').
+        gsub(/([A-Z]+)([A-Z][a-z])/,'\1_\2').
+        gsub(/([a-z\d])([A-Z])/,'\1_\2').
+        tr("-", "_").
+        downcase
+    end
+  
+  end
+end

data/lib/cloud_crowd/models/job.rb

@@ -0,0 +1,190 @@
+module CloudCrowd
+
+  # A chunk of work that will be farmed out into many WorkUnits to be processed
+  # in parallel by each active CloudCrowd::Worker. Jobs are defined by a list
+  # of inputs (usually public urls to files), an action (the name of a script that
+  # CloudCrowd knows how to run), and, eventually a corresponding list of output.
+  class Job < ActiveRecord::Base
+    include ModelStatus
+
+    CLEANUP_GRACE_PERIOD = 7 # That's a week.
+
+    has_many :work_units, :dependent => :destroy
+
+    validates_presence_of :status, :inputs, :action, :options
+
+    before_validation_on_create :set_initial_status
+    after_create                :queue_for_workers
+    before_destroy              :cleanup_assets
+
+    # Jobs that were last updated more than N days ago.
+    named_scope :older_than, lambda {|num| {:conditions => ['updated_at < ?', num.days.ago]} }
+
+    # Create a Job from an incoming JSON request, and add it to the queue.
+    def self.create_from_request(h)
+      self.create(
+        :inputs       => h['inputs'].to_json,
+        :action       => h['action'],
+        :options      => (h['options'] || {}).to_json,
+        :email        => h['email'],
+        :callback_url => h['callback_url']
+      )
+    end
+
+    # Clean up all jobs beyond a certain age.
+    def self.cleanup_all(opts = {})
+      days = opts[:days] || CLEANUP_GRACE_PERIOD
+      self.complete.older_than(days).find_in_batches(:batch_size => 100) do |jobs|
+        jobs.each {|job| job.destroy }
+      end
+    end
+
+    # After work units are marked successful, we check to see if all of them have
+    # finished, if so, continue on to the next phase of the job.
+    def check_for_completion
+      return unless all_work_units_complete?
+      set_next_status
+      outs = gather_outputs_from_work_units
+      return queue_for_workers([outs]) if merging?
+      if complete?
+        update_attributes(:outputs => outs, :time => time_taken)
+        Thread.new { fire_callback } if callback_url
+      end
+      self
+    end
+
+    # Transition this Job's current status to the appropriate next one, based
+    # on the state of the WorkUnits and the nature of the Action.
+    def set_next_status
+      update_attribute(:status,
+        any_work_units_failed? ? FAILED     :
+        self.splitting?        ? PROCESSING :
+        self.mergeable?        ? MERGING    :
+                                 SUCCEEDED
+      )
+    end
+
+    # If a <tt>callback_url</tt> is defined, post the Job's JSON to it upon
+    # completion. The <tt>callback_url</tt> may include HTTP basic authentication,
+    # if you like:
+    #   http://user:password@example.com/job_complete
+    # If the callback URL returns a '201 Created' HTTP status code, CloudCrowd
+    # will assume that the resource has been successfully created, and the Job
+    # will be cleaned up.
+    def fire_callback
+      begin
+        response = RestClient.post(callback_url, {:job => self.to_json})
+        Thread.new { self.destroy } if response && response.code == 201
+      rescue RestClient::Exception => e
+        puts "Job ##{id} failed to fire callback: #{callback_url}"
+      end
+    end
+
+    # Cleaning up after a job will remove all of its files from S3 or the
+    # filesystem. Destroying a Job will cleanup_assets first. Run this in a
+    # separate thread to get out of the transaction's way.
+    # TODO: Convert this into a 'cleanup' work unit that gets run by a worker.
+    def cleanup_assets
+      AssetStore.new.cleanup(self)
+    end
+
+    # Have all of the WorkUnits finished?
+    def all_work_units_complete?
+      self.work_units.incomplete.count <= 0
+    end
+
+    # Have any of the WorkUnits failed?
+    def any_work_units_failed?
+      self.work_units.failed.count > 0
+    end
+
+    # This job is splittable if its Action has a +split+ method.
+    def splittable?
+      self.action_class.public_instance_methods.map {|m| m.to_sym }.include? :split
+    end
+
+    # This job is done splitting if it's finished with its splitting work units.
+    def done_splitting?
+      splittable? && work_units.splitting.count <= 0
+    end
+
+    # This job is mergeable if its Action has a +merge+ method.
+    def mergeable?
+      self.processing? && self.action_class.public_instance_methods.map {|m| m.to_sym }.include?(:merge)
+    end
+
+    # Retrieve the class for this Job's Action.
+    def action_class
+      @action_class ||= CloudCrowd.actions[self.action]
+      return @action_class if @action_class
+      raise Error::ActionNotFound, "no action named: '#{self.action}' could be found"
+    end
+
+    # How complete is this Job?
+    # Unfortunately, with the current processing sequence, the percent_complete
+    # can pull a fast one and go backwards. This happens when there's a single
+    # large input that takes a long time to split, and when it finally does it
+    # creates a whole swarm of work units. This seems unavoidable.
+    def percent_complete
+      return 99  if merging?
+      return 100 if complete?
+      unit_count = work_units.count
+      return 100 if unit_count <= 0
+      (work_units.complete.count / unit_count.to_f * 100).round
+    end
+
+    # How long has this Job taken?
+    def time_taken
+      return self.time if self.time
+      Time.now - self.created_at
+    end
+
+    # Generate a stable 8-bit Hex color code, based on the Job's id.
+    def color
+      @color ||= Digest::MD5.hexdigest(self.id.to_s)[-7...-1]
+    end
+
+    # A JSON representation of this job includes the statuses of its component
+    # WorkUnits, as well as any completed outputs.
+    def to_json(opts={})
+      atts = {
+        'id'                => id,
+        'color'             => color,
+        'status'            => display_status,
+        'percent_complete'  => percent_complete,
+        'work_units'        => work_units.count,
+        'time_taken'        => time_taken
+      }
+      atts['outputs'] = JSON.parse(outputs) if outputs
+      atts['email']   = email               if email
+      atts.to_json
+    end
+
+
+    private
+
+    # When the WorkUnits are all finished, gather all their outputs together
+    # before removing them from the database entirely. Returns their merged JSON.
+    def gather_outputs_from_work_units
+      units = self.work_units.complete
+      outs = self.work_units.complete.map {|u| u.parsed_output }
+      self.work_units.complete.destroy_all
+      outs.to_json
+    end
+
+    # When starting a new job, or moving to a new stage, split up the inputs
+    # into WorkUnits, and queue them. Workers will start picking them up right
+    # away.
+    def queue_for_workers(input=nil)
+      input ||= JSON.parse(self.inputs)
+      input.each {|i| WorkUnit.start(self, action, i, status) }
+      self
+    end
+
+    # A Job starts out either splitting or processing, depending on its action.
+    def set_initial_status
+      self.status = self.splittable? ? SPLITTING : PROCESSING
+    end
+
+  end
+end

data/lib/cloud_crowd/models/node_record.rb

@@ -0,0 +1,107 @@
+module CloudCrowd
+
+  # A NodeRecord is the central server's record of a Node running remotely. We 
+  # can use it to assign WorkUnits to the Node, and keep track of its status.
+  # When a Node exits, it destroys this record.
+  class NodeRecord < ActiveRecord::Base
+        
+    has_many :work_units
+    
+    validates_presence_of :host, :ip_address, :port, :enabled_actions
+    
+    after_destroy :redistribute_work_units
+    
+    # Available Nodes haven't used up their maxiumum number of workers yet.
+    named_scope :available, {
+      :conditions => ['(max_workers is null or (select count(*) from work_units where node_record_id = node_records.id) < max_workers)'],
+      :order      => 'updated_at asc'
+    }
+    
+    # Register a Node with the central server. Currently this only happens at
+    # Node startup.
+    def self.check_in(params, request)
+      attrs = {
+        :ip_address       => request.ip,
+        :port             => params[:port],
+        :busy             => params[:busy],
+        :max_workers      => params[:max_workers],
+        :enabled_actions  => params[:enabled_actions]
+      }
+      self.find_or_create_by_host(params[:host]).update_attributes!(attrs)
+    end
+    
+    # Dispatch a WorkUnit to this node. Places the node at back at the end of
+    # the rotation. If we fail to send the WorkUnit, we consider the node to be
+    # down, and remove this record, freeing up all of its checked-out work units.
+    # If the Node responds that it's overloaded, we mark it as busy. Returns 
+    # true if the WorkUnit was dispatched successfully.
+    def send_work_unit(unit)
+      result = node['/work'].post(:work_unit => unit.to_json)
+      unit.assign_to(self, JSON.parse(result)['pid'])
+      touch && true
+    rescue RestClient::RequestFailed => e
+      raise e unless e.http_code == 503 && e.http_body == Node::OVERLOADED_MESSAGE
+      update_attribute(:busy, true) && false
+    rescue RestClient::Exception, Errno::ECONNREFUSED, Timeout::Error
+      # Couldn't post to node, assume it's gone away.
+      destroy && false
+    end
+    
+    # What Actions is this Node able to run?
+    def actions
+      @actions ||= enabled_actions.split(',')
+    end
+    
+    # Is this Node too busy for more work? Determined by number of workers, or 
+    # the Node's load average, as configured in config.yml.
+    def busy?
+      busy || (max_workers && work_units.count >= max_workers)
+    end
+    
+    # The URL at which this Node may be reached.
+    # TODO: Make sure that the host actually has externally accessible DNS.
+    def url
+      @url ||= "http://#{host}:#{port}"
+    end
+    
+    # Keep a RestClient::Resource handy for contacting the Node, including 
+    # HTTP authentication, if configured.
+    def node
+      @node ||= RestClient::Resource.new(url, CloudCrowd.client_options)
+    end
+    
+    # The printable status of the Node.
+    def display_status
+      busy? ? 'busy' : 'available'
+    end
+    
+    # A list of the process ids of the workers currently being run by the Node.
+    def worker_pids
+      work_units.all(:select => 'worker_pid').map(&:worker_pid)
+    end
+    
+    # Release all of this Node's WorkUnits for other nodes to take.
+    def release_work_units
+      WorkUnit.update_all('node_record_id = null, worker_pid = null', "node_record_id = #{id}")
+    end
+    
+    # The JSON representation of a NodeRecord includes its worker_pids.
+    def to_json(opts={})
+      { 'host'    => host,
+        'workers' => worker_pids,
+        'status'  => display_status
+      }.to_json
+    end
+    
+    
+    private
+    
+    # When a Node exits, release its WorkUnits and redistribute them to others. 
+    # Redistribute in a separate thread to avoid delaying shutdown.
+    def redistribute_work_units
+      release_work_units
+      Thread.new { WorkUnit.distribute_to_nodes }
+    end
+    
+  end
+end

data/lib/cloud_crowd/models/work_unit.rb

@@ -0,0 +1,170 @@
+module CloudCrowd
+
+  # A WorkUnit is an atomic chunk of work from a job, processing a single input
+  # through a single action. The WorkUnits are run in parallel, with each worker
+  # daemon processing one at a time. The splitting and merging stages of a job
+  # are each run as a single WorkUnit.
+  class WorkUnit < ActiveRecord::Base
+    include ModelStatus
+    
+    # We use a random number in (0...MAX_RESERVATION) to reserve work units.
+    # The size of the maximum signed integer in MySQL -- SQLite has no limit.
+    MAX_RESERVATION = 2147483647
+    
+    # We only reserve a certain number of WorkUnits in a single go, to avoid
+    # reserving the entire table.
+    RESERVATION_LIMIT = 25
+    
+    belongs_to :job
+    belongs_to :node_record
+    
+    validates_presence_of :job_id, :status, :input, :action
+    
+    # Available WorkUnits are waiting to be distributed to Nodes for processing.
+    named_scope :available, {:conditions => {:reservation => nil, :worker_pid => nil, :status => INCOMPLETE}}
+    # Reserved WorkUnits have been marked for distribution by a central server process.
+    named_scope :reserved,  lambda {|reservation| 
+      {:conditions => {:reservation => reservation}, :order => 'updated_at asc'}
+    }
+    
+    # Attempt to send a list of WorkUnits to nodes with available capacity.
+    # A single central server process stops the same WorkUnit from being
+    # distributed to multiple nodes by reserving it first. The algorithm used
+    # should be lock-free.
+    #
+    # We reserve WorkUnits for this process in chunks of RESERVATION_LIMIT size,
+    # and try to match them to Nodes that are capable of handling the Action. 
+    # WorkUnits get removed from the availability list when they are 
+    # successfully sent, and Nodes get removed when they are busy or have the 
+    # action in question disabled.
+    def self.distribute_to_nodes
+      reservation = nil
+      loop do
+        return unless reservation = WorkUnit.reserve_available(:limit => RESERVATION_LIMIT)
+        work_units = WorkUnit.reserved(reservation)
+        available_nodes = NodeRecord.available
+        while node = available_nodes.shift and unit = work_units.shift do
+          if node.actions.include? unit.action
+            if node.send_work_unit(unit)
+              available_nodes.push(node) unless node.busy?
+              next
+            end
+          end
+          work_units.push(unit)
+        end
+        return if work_units.any? || available_nodes.empty?
+      end
+    ensure
+      WorkUnit.cancel_reservations(reservation) if reservation
+    end
+    
+    # Reserves all available WorkUnits for this process. Returns false if there 
+    # were none available.
+    def self.reserve_available(options={})
+      reservation = ActiveSupport::SecureRandom.random_number(MAX_RESERVATION)
+      any = WorkUnit.available.update_all("reservation = #{reservation}", nil, options) > 0
+      any && reservation
+    end
+    
+    # Cancels all outstanding WorkUnit reservations for this process.
+    def self.cancel_reservations(reservation)
+      WorkUnit.reserved(reservation).update_all('reservation = null')
+    end
+    
+    # Cancels all outstanding WorkUnit reservations for all processes. (Useful
+    # in the console for debugging.)
+    def self.cancel_all_reservations
+      WorkUnit.update_all('reservation = null')
+    end
+    
+    # Look up a WorkUnit by the worker that's currently processing it. Specified
+    # by <tt>pid@host</tt>.
+    def self.find_by_worker_name(name)
+      pid, host = name.split('@')
+      node = NodeRecord.find_by_host(host)
+      node && node.work_units.find_by_worker_pid(pid)
+    end
+    
+    # Convenience method for starting a new WorkUnit.
+    def self.start(job, action, input, status)
+      input = input.to_json unless input.is_a? String
+      self.create(:job => job, :action => action, :input => input, :status => status)
+    end
+    
+    # Mark this unit as having finished successfully.
+    # Splitting work units are handled differently (an optimization) -- they 
+    # immediately fire off all of their resulting WorkUnits for processing, 
+    # without waiting for the rest of their splitting cousins to complete.
+    def finish(result, time_taken)
+      if splitting?
+        [parsed_output(result)].flatten.each do |new_input|
+          WorkUnit.start(job, action, new_input, PROCESSING)
+        end
+        self.destroy
+        job.set_next_status if job && job.done_splitting?
+      else
+        update_attributes({
+          :status         => SUCCEEDED,
+          :node_record    => nil,
+          :worker_pid     => nil,
+          :attempts       => attempts + 1,
+          :output         => result,
+          :time           => time_taken
+        })
+        job && job.check_for_completion
+      end
+    end
+    
+    # Mark this unit as having failed. May attempt a retry.
+    def fail(output, time_taken)
+      tries = self.attempts + 1
+      return try_again if tries < CloudCrowd.config[:work_unit_retries]
+      update_attributes({
+        :status         => FAILED,
+        :node_record    => nil,
+        :worker_pid     => nil,
+        :attempts       => tries,
+        :output         => output,
+        :time           => time_taken
+      })
+      job && job.check_for_completion
+    end
+    
+    # Ever tried. Ever failed. No matter. Try again. Fail again. Fail better.
+    def try_again
+      update_attributes({
+        :node_record  => nil,
+        :worker_pid   => nil,
+        :attempts     => self.attempts + 1
+      })
+    end
+    
+    # When a Node checks out a WorkUnit, establish the connection between
+    # WorkUnit and NodeRecord and record the worker_pid.
+    def assign_to(node_record, worker_pid)
+      update_attributes!(:node_record => node_record, :worker_pid => worker_pid)
+    end
+    
+    # All output needs to be wrapped in a JSON object for consistency 
+    # (unfortunately, JSON.parse needs the top-level to be an object or array). 
+    # Convenience method to provide the parsed version.
+    def parsed_output(out = self.output)
+      JSON.parse(out)['output']
+    end
+    
+    # The JSON representation of a WorkUnit shares the Job's options with all
+    # its cousin WorkUnits.
+    def to_json
+      {
+        'id'        => self.id,
+        'job_id'    => self.job_id,
+        'input'     => self.input,
+        'attempts'  => self.attempts,
+        'action'    => self.action,
+        'options'   => JSON.parse(self.job.options),
+        'status'    => self.status
+      }.to_json
+    end
+    
+  end
+end

data/lib/cloud_crowd/models.rb

@@ -0,0 +1,40 @@
+module CloudCrowd
+  
+  # Adds named scopes and query methods for every CloudCrowd status to
+  # both Jobs and WorkUnits.
+  module ModelStatus
+    
+    def self.included(klass)
+      
+      klass.class_eval do
+        # Note that COMPLETE and INCOMPLETE are unions of other states.
+        named_scope 'processing', :conditions => {:status => PROCESSING}
+        named_scope 'succeeded',  :conditions => {:status => SUCCEEDED}
+        named_scope 'failed',     :conditions => {:status => FAILED}
+        named_scope 'splitting',  :conditions => {:status => SPLITTING}
+        named_scope 'merging',    :conditions => {:status => MERGING}
+        named_scope 'complete',   :conditions => {:status => COMPLETE}
+        named_scope 'incomplete', :conditions => {:status => INCOMPLETE}
+      end
+      
+    end
+    
+    def processing?;  self.status == PROCESSING;          end
+    def succeeded?;   self.status == SUCCEEDED;           end
+    def failed?;      self.status == FAILED;              end
+    def splitting?;   self.status == SPLITTING;           end
+    def merging?;     self.status == MERGING;             end
+    def complete?;    COMPLETE.include?(self.status);     end
+    def incomplete?;  INCOMPLETE.include?(self.status);   end
+    
+    # Get the displayable status name of the model's status code.
+    def display_status
+      CloudCrowd.display_status(self.status)
+    end
+    
+  end
+end
+
+require 'cloud_crowd/models/job'
+require 'cloud_crowd/models/node_record'
+require 'cloud_crowd/models/work_unit'