cloud-crowd 0.1.0 → 0.2.0

data/lib/cloud_crowd/asset_store/filesystem_store.rb

@@ -2,15 +2,26 @@ module CloudCrowd
   class AssetStore
     
     # The FilesystemStore is an implementation of the AssetStore, good only for
-    # use in development, testing, or if you're only running a single-machine
-    # installation.
+    # use in development, testing, if you're only running a single-machine
+    # installation, or are using a networked drive.
     module FilesystemStore
       
-      # Save a file to somewhere semi-persistent on the filesystem. Can be used
-      # in development, when offline, or if you happen to have a single-machine
-      # CloudCrowd installation. To use, configure <tt>:storage => 'filesystem'</tt>.
+      DEFAULT_STORAGE_PATH = '/tmp/cloud_crowd_storage'
+      
+      attr_reader :local_storage_path
+      
+      # Make sure that local storage exists and is writeable before starting.
+      def setup
+        lsp = @local_storage_path = CloudCrowd.config[:local_storage_path] || DEFAULT_STORAGE_PATH
+        FileUtils.mkdir_p(lsp) unless File.exists?(lsp)
+        raise Error::StorageNotWritable, "#{lsp} is not writable" unless File.writable?(lsp)
+      end
+      
+      # Save a file to somewhere semi-persistent on the filesystem. To use, 
+      # configure <tt>:storage: 'filesystem'</tt> in *config.yml*, as well as
+      # <tt>:local_storage_path:</tt>.
       def save(local_path, save_path)
-        save_path = File.join(LOCAL_STORAGE_PATH, save_path)
+        save_path = File.join(@local_storage_path, save_path)
         save_dir = File.dirname(save_path)
         FileUtils.mkdir_p save_dir unless File.exists? save_dir
         FileUtils.cp(local_path, save_path)
@@ -19,7 +30,7 @@ module CloudCrowd
       
       # Remove all of a Job's result files from the filesystem.
       def cleanup(job)
-        path = "#{LOCAL_STORAGE_PATH}/#{job.action}/job_#{job.id}"
+        path = "#{@local_storage_path}/#{job.action}/job_#{job.id}"
         FileUtils.rm_r(path) if File.exists?(path)
       end
     end

data/lib/cloud_crowd/asset_store/s3_store.rb

@@ -5,11 +5,24 @@ module CloudCrowd
     # on S3 for all resulting files.
     module S3Store
       
+      # Configure authentication and establish a connection to S3, first thing.
+      def setup
+        @use_auth   = CloudCrowd.config[:s3_authentication]
+        bucket_name = CloudCrowd.config[:s3_bucket]
+        key, secret = CloudCrowd.config[:aws_access_key], CloudCrowd.config[:aws_secret_key]
+        valid_conf  = [bucket_name, key, secret].all? {|s| s.is_a? String }
+        raise Error::MissingConfiguration, "An S3 account must be configured in 'config.yml' before 's3' storage can be used" unless valid_conf
+        protocol    = @use_auth ? 'https' : 'http'
+        port        = @use_auth ? 443 : 80
+        @s3         = RightAws::S3.new(key, secret, :protocol => protocol, :port => port)
+        @bucket     = @s3.bucket(bucket_name)
+        @bucket     = @s3.bucket(bucket_name, true) unless @bucket
+      end
+      
       # Save a finished file from local storage to S3. Save it publicly unless 
       # we're configured to use S3 authentication. Authenticated links expire
       # after one day by default.
       def save(local_path, save_path)
-        ensure_s3_connection
         if @use_auth
           @bucket.put(save_path, File.open(local_path), {}, 'private')
           @s3.interface.get_link(@bucket, save_path)
@@ -21,19 +34,9 @@ module CloudCrowd
             
       # Remove all of a Job's resulting files from S3, both intermediate and finished.
       def cleanup(job)
-        ensure_s3_connection
         @bucket.delete_folder("#{job.action}/job_#{job.id}")
       end
       
-      # Workers, through the course of many WorkUnits, keep around an AssetStore.
-      # Ensure we have a persistent S3 connection after first use.
-      def ensure_s3_connection
-        unless @s3 && @bucket
-          params = {:port => 80, :protocol => 'http'}
-          @s3 = RightAws::S3.new(CloudCrowd.config[:aws_access_key], CloudCrowd.config[:aws_secret_key], params)
-          @bucket = @s3.bucket(CloudCrowd.config[:s3_bucket], true)
-        end
-      end
     end
     
   end

data/lib/cloud_crowd/command_line.rb

@@ -9,9 +9,6 @@ module CloudCrowd
     # Reference the absolute path to the root.
     CC_ROOT = File.expand_path(File.dirname(__FILE__) + '/../..')
     
-    # Path to the Daemons gem script which launches workers.
-    WORKER_RUNNER = File.expand_path("#{CC_ROOT}/lib/cloud_crowd/runner.rb")
-    
     # Command-line banner for the usage message.
     BANNER = <<-EOS
 CloudCrowd is a MapReduce-inspired Parallel Processing System for Ruby.
@@ -24,7 +21,7 @@ Usage: crowd COMMAND OPTIONS
 Commands:
   install       Install the CloudCrowd configuration files to the specified directory
   server        Start up the central server (requires a database)
-  workers       Control worker daemons, use: (start | stop | restart | status | run)
+  node          Start up a worker node (only one node per machine, please)
   console       Launch a CloudCrowd console, connected to the central database
   load_schema   Load the schema into the database specified by database.yml
 
@@ -38,7 +35,7 @@ Options:
       case command
       when 'console'      then run_console
       when 'server'       then run_server
-      when 'workers'      then run_workers_command
+      when 'node'         then run_node
       when 'load_schema'  then run_load_schema
       when 'install'      then run_install
       else                     usage
@@ -52,7 +49,7 @@ Options:
       require 'irb/completion'
       require 'pp'
       load_code
-      connect_to_database
+      connect_to_database(true)
       IRB.start
     end
     
@@ -63,6 +60,7 @@ Options:
     # (Mongrel, falling back to WEBrick). The equivalent of Rails' script/server.
     def run_server
       ensure_config
+      @options[:port] ||= 9173
       require 'rubygems'
       rackup_path = File.expand_path("#{@options[:config_path]}/config.ru")
       if Gem.available? 'thin'
@@ -72,10 +70,18 @@ Options:
       end
     end
     
+    # Launch a Node. Please only run a single node per machine. The Node process
+    # will be long-lived, although its workers will come and go.
+    def run_node
+      ENV['RACK_ENV'] = @options['environment']
+      load_code
+      Node.new(@options[:port])
+    end
+    
     # Load in the database schema to the database specified in 'database.yml'.
     def run_load_schema
       load_code
-      connect_to_database
+      connect_to_database(false)
       require 'cloud_crowd/schema.rb'
     end
     
@@ -91,46 +97,6 @@ Options:
       install_file "#{CC_ROOT}/actions", "#{install_path}/actions", true
     end
     
-    # Manipulate worker daemons -- handles all commands that the Daemons gem
-    # provides: start, stop, restart, run, and status.
-    def run_workers_command
-      ensure_config
-      command = ARGV.shift
-      case command
-      when 'start'    then start_workers
-      when 'stop'     then stop_workers
-      when 'restart'  then stop_workers && start_workers
-      when 'run'      then run_worker
-      when 'status'   then show_worker_status
-      else                 usage
-      end
-    end
-    
-    # Start up N workers, specified by argument or the number of workers in
-    # config.yml.
-    def start_workers
-      load_code
-      num_workers = @options[:num_workers] || CloudCrowd.config[:num_workers]
-      num_workers.times do
-        `CLOUD_CROWD_CONFIG='#{File.expand_path(@options[:config_path] + "/config.yml")}' ruby #{WORKER_RUNNER} start`
-      end
-    end
-    
-    # For debugging, run a single worker in the current process, showing output.
-    def run_worker
-      exec "CLOUD_CROWD_CONFIG='#{File.expand_path(@options[:config_path] + "/config.yml")}' ruby #{WORKER_RUNNER} run"
-    end
-    
-    # Stop all active workers.
-    def stop_workers
-      `ruby #{WORKER_RUNNER} stop`
-    end
-
-    # Display the status of all active workers.
-    def show_worker_status
-      puts `ruby #{WORKER_RUNNER} status`
-    end
-    
     # Print `crowd` usage.
     def usage
       puts "\n#{@option_parser}\n"
@@ -150,7 +116,6 @@ Options:
     # Parse all options for all commands.
     def parse_options
       @options = {
-        :port         => 9173,
         :environment  => 'production',
         :config_path  => ENV['CLOUD_CROWD_CONFIG'] || '.'
       }
@@ -158,17 +123,14 @@ Options:
         opts.on('-c', '--config PATH', 'path to configuration directory') do |conf_path|
           @options[:config_path] = conf_path
         end
-        opts.on('-n', '--num-workers NUM', OptionParser::DecimalInteger, 'number of worker processes') do |num|
-          @options[:num_workers] = num
-        end
-        opts.on('-p', '--port PORT', 'central server port number') do |port_num|
+        opts.on('-p', '--port PORT', 'port number for server (central or node)') do |port_num|
           @options[:port] = port_num
         end
         opts.on('-e', '--environment ENV', 'server environment (sinatra)') do |env|
           @options[:environment] = env
         end
         opts.on_tail('-v', '--version', 'show version') do
-          load_code
+          require "#{CC_ROOT}/lib/cloud-crowd"
           puts "CloudCrowd version #{VERSION}"
           exit
         end
@@ -181,26 +143,30 @@ Options:
     # Not all commands require this.
     def load_code
       ensure_config
-      require 'rubygems'
       require "#{CC_ROOT}/lib/cloud-crowd"
       CloudCrowd.configure("#{@options[:config_path]}/config.yml")
     end
     
     # Establish a connection to the central server's database. Not all commands
     # require this.
-    def connect_to_database
+    def connect_to_database(validate_schema)
       require 'cloud_crowd/models'
-      CloudCrowd.configure_database("#{@options[:config_path]}/database.yml")
+      CloudCrowd.configure_database("#{@options[:config_path]}/database.yml", validate_schema)
     end
     
     # Exit with an explanation if the configuration files couldn't be found.
     def config_not_found
-      puts "`crowd` can't find the CloudCrowd configuration directory. Please either run `crowd` from inside of the configuration directory, or use `crowd -c path/to/config`"
+      puts "`crowd` can't find the CloudCrowd configuration directory. Please use `crowd -c path/to/config`, or run `crowd` from inside of the configuration directory itself."
       exit(1)
     end
     
-    # Install a file and log the installation.
+    # Install a file and log the installation. If we're overwriting a file, 
+    # offer a chance to back out.
     def install_file(source, dest, is_dir=false)
+      if File.exists?(dest)
+        print "#{dest} already exists. Overwrite it? (yes/no) "
+        return unless ['y', 'yes', 'ok'].include? gets.chomp.downcase
+      end
       is_dir ? FileUtils.cp_r(source, dest) : FileUtils.cp(source, dest)
       puts "installed #{dest}"
     end

data/lib/cloud_crowd/exceptions.rb

@@ -2,6 +2,8 @@ module CloudCrowd
   
   # Base Error class which all custom CloudCrowd exceptions inherit from.
   # Rescuing CloudCrowd::Error (or RuntimeError) will get all custom exceptions.
+  # If your cluster is correctly configured, you should never expect to see any
+  # of these.
   class Error < RuntimeError
     
     # ActionNotFound is raised when a job is created for an action that doesn't 
@@ -23,6 +25,11 @@ module CloudCrowd
     class StatusUnspecified < Error
     end
     
+    # MissingConfiguration is raised when we're trying to run a method that
+    # needs configuration not present in config.yml.
+    class MissingConfiguration < Error
+    end
+    
   end
   
 end

data/lib/cloud_crowd/helpers/authorization.rb

@@ -23,9 +23,9 @@ module CloudCrowd
       # A request is authorized if its login and password match those stored
       # in config.yml, or if authentication is disabled. If authentication is
       # turned on, then every request is authenticated, including between 
-      # the worker daemons and the central server.
+      # the nodes and the central server.
       def authorize(login, password)
-        return true unless CloudCrowd.config[:use_http_authentication]
+        return true unless CloudCrowd.config[:http_authentication]
         return CloudCrowd.config[:login] == login &&
                CloudCrowd.config[:password] == password
       end
@@ -33,11 +33,13 @@ module CloudCrowd
       
       private
       
+      # Provide a Rack Authorization object.
       def auth
         @auth ||= Rack::Auth::Basic::Request.new(request.env)
       end
       
-      def unauthorized!(realm = App.authorization_realm)
+      # Unauthorized requests will prompt the browser to provide credentials.
+      def unauthorized!(realm = Server.authorization_realm)
         response['WWW-Authenticate'] = "Basic realm=\"#{realm}\""
         halt 401, 'Authorization Required'
       end

data/lib/cloud_crowd/helpers/resources.rb

@@ -20,26 +20,6 @@ module CloudCrowd
         @work_unit ||= WorkUnit.find_by_id(params[:work_unit_id]) or raise Sinatra::NotFound
       end
       
-      # Try to fetch a work unit from the queue. If none are pending, respond
-      # with no content.
-      def dequeue_work_unit(offset=0)
-        handle_conflicts do
-          worker, actions = params[:worker_name], params[:worker_actions].split(',')
-          WorkUnit.dequeue(worker, actions, offset)
-        end
-      end
-      
-      # We're using ActiveRecords optimistic locking, so stale work units
-      # may sometimes arise. handle_conflicts responds with a the HTTP status
-      # code of your choosing if the update failed to be applied.
-      def handle_conflicts(code=204)
-        begin
-          yield
-        rescue ActiveRecord::StaleObjectError => e
-          return status(code) && ''
-        end
-      end
-      
     end
   end
 end

data/lib/cloud_crowd/models.rb

@@ -36,5 +36,5 @@ module CloudCrowd
 end
 
 require 'cloud_crowd/models/job'
+require 'cloud_crowd/models/node_record'
 require 'cloud_crowd/models/work_unit'
-require 'cloud_crowd/models/worker_record'

data/lib/cloud_crowd/models/job.rb

@@ -31,30 +31,36 @@ module CloudCrowd
     # finished, if so, continue on to the next phase of the job. 
     def check_for_completion
       return unless all_work_units_complete?
-      transition_to_next_phase               
-      output_list = gather_outputs_from_work_units
-      
+      set_next_status    
+      outs = gather_outputs_from_work_units
+      return queue_for_workers(outs) if merging?
       if complete?
-        self.outputs = output_list.to_json
-        self.time = Time.now - self.created_at
-      end
-      self.save
-      
-      case self.status
-      when PROCESSING then queue_for_workers(output_list.map {|o| JSON.parse(o) }.flatten)
-      when MERGING    then queue_for_workers(output_list.to_json)
-      else                 fire_callback
+        update_attributes(:outputs => outs, :time => time_taken)
+        fire_callback if callback_url
       end
       self
     end
     
+    # Transition this Job's status to the appropriate next status.
+    def set_next_status
+      update_attribute(:status,
+        any_work_units_failed? ? FAILED     :
+        self.splitting?        ? PROCESSING :
+        self.mergeable?        ? MERGING    :
+                                 SUCCEEDED
+      )
+    end
+    
     # If a <tt>callback_url</tt> is defined, post the Job's JSON to it upon 
     # completion. The <tt>callback_url</tt> may include HTTP basic authentication,
     # if you like:
     #   http://user:password@example.com/job_complete
+    # If the callback_url is successfully pinged, we proceed to cleanup the job.
+    # TODO: This should be moved into a Work Unit...
     def fire_callback
       begin
-        RestClient.post(callback_url, {:job => self.to_json}) if callback_url
+        RestClient.post(callback_url, {:job => self.to_json})
+        self.destroy
       rescue RestClient::Exception => e
         puts "Failed to fire job callback. Hmmm, what should happen here?"
       end
@@ -62,15 +68,12 @@ module CloudCrowd
     
     # Cleaning up after a job will remove all of its files from S3. Destroying
     # a Job calls cleanup_assets first.
+    # TODO: Convert this into a 'cleanup' work unit that gets run by a worker.
     def cleanup_assets
       AssetStore.new.cleanup(self)
     end
     
     # Have all of the WorkUnits finished? 
-    #--
-    # We could trade reads for writes here
-    # by keeping a completed_count on the Job itself.
-    #++
     def all_work_units_complete?
       self.work_units.incomplete.count <= 0
     end
@@ -85,6 +88,11 @@ module CloudCrowd
       self.action_class.public_instance_methods.include? 'split'
     end
     
+    # This job is done splitting if it's finished with its splitting work units.
+    def done_splitting?
+      splittable? && work_units.splitting.count <= 0
+    end
+    
     # This job is mergeable if its Action has a +merge+ method.
     def mergeable?
       self.processing? && self.action_class.public_instance_methods.include?('merge')
@@ -92,16 +100,19 @@ module CloudCrowd
     
     # Retrieve the class for this Job's Action.
     def action_class
-      klass = CloudCrowd.actions[self.action]
-      return klass if klass
+      @action_class ||= CloudCrowd.actions[self.action]
+      return @action_class if @action_class
       raise Error::ActionNotFound, "no action named: '#{self.action}' could be found"
     end
     
     # How complete is this Job?
+    # Unfortunately, with the current processing sequence, the percent_complete
+    # can pull a fast one and go backwards. This happens when there's a single
+    # large input that takes a long time to split, and when it finally does it 
+    # creates a whole swarm of work units. This seems unavoidable.
     def percent_complete
-      return 0   if splitting?
-      return 100 if complete?
       return 99  if merging?
+      return 100 if complete?
       (work_units.complete.count / work_units.count.to_f * 100).round
     end
     
@@ -136,20 +147,12 @@ module CloudCrowd
     private
     
     # When the WorkUnits are all finished, gather all their outputs together
-    # before removing them from the database entirely.
+    # before removing them from the database entirely. Returns their merged JSON.
     def gather_outputs_from_work_units
       units = self.work_units.complete
-      outs = self.work_units.complete.map {|u| JSON.parse(u.output)['output'] }
+      outs = self.work_units.complete.map {|u| u.parsed_output }
       self.work_units.complete.destroy_all
-      outs
-    end
-    
-    # Transition this Job's status to the appropriate next status.
-    def transition_to_next_phase
-      self.status = any_work_units_failed? ? FAILED     :
-                    self.splitting?        ? PROCESSING :
-                    self.mergeable?        ? MERGING    :
-                                             SUCCEEDED
+      outs.to_json
     end
         
     # When starting a new job, or moving to a new stage, split up the inputs 
@@ -157,14 +160,8 @@ module CloudCrowd
     # away.
     def queue_for_workers(input=nil)
       input ||= JSON.parse(self.inputs)
-      [input].flatten.each do |wu_input|
-        WorkUnit.create(
-          :job    => self, 
-          :action => self.action, 
-          :input  => wu_input, 
-          :status => self.status
-        )
-      end
+      [input].flatten.each {|i| WorkUnit.start(self, action, i, status) }        
+      self
     end
     
     # A Job starts out either splitting or processing, depending on its action.