mooktakim-cloud-crowd 0.3.4

data/lib/cloud-crowd.rb

@@ -0,0 +1,188 @@
+# The Grand Central of code loading...
+
+$LOAD_PATH.unshift File.expand_path(File.dirname(__FILE__))
+
+# Common Gems:
+require 'rubygems'
+gem 'activerecord', '2.3.5'
+gem 'json'
+gem 'rest-client'
+gem 'right_aws'
+gem 'sinatra'
+gem 'thin'
+
+# Autoloading for all the pieces which may or may not be needed:
+autoload :ActiveRecord, 'active_record'
+autoload :Benchmark,    'benchmark'
+autoload :Digest,       'digest'
+autoload :ERB,          'erb'
+autoload :FileUtils,    'fileutils'
+autoload :JSON,         'json'
+autoload :RestClient,   'rest_client'
+autoload :RightAws,     'right_aws'
+autoload :Sinatra,      'sinatra'
+autoload :Thin,         'thin'
+autoload :YAML,         'yaml'
+
+# Common code which should really be required in every circumstance.
+require 'socket'
+require 'cloud_crowd/exceptions'
+
+module CloudCrowd
+
+  # Autoload all the CloudCrowd internals.
+  autoload :Action,       'cloud_crowd/action'
+  autoload :AssetStore,   'cloud_crowd/asset_store'
+  autoload :CommandLine,  'cloud_crowd/command_line'
+  autoload :Helpers,      'cloud_crowd/helpers'
+  autoload :Inflector,    'cloud_crowd/inflector'
+  autoload :Job,          'cloud_crowd/models'
+  autoload :Node,         'cloud_crowd/node'
+  autoload :NodeRecord,   'cloud_crowd/models'
+  autoload :Server,       'cloud_crowd/server'
+  autoload :Worker,       'cloud_crowd/worker'
+  autoload :WorkUnit,     'cloud_crowd/models'
+
+  # Keep this version in sync with the gemspec.
+  VERSION        = '0.3.3'
+
+  # Increment the schema version when there's a backwards incompatible change.
+  SCHEMA_VERSION = 3
+
+  # Root directory of the CloudCrowd gem.
+  ROOT           = File.expand_path(File.dirname(__FILE__) + '/..')
+
+  # Default folder to log daemonized servers and nodes into.
+  LOG_PATH       = 'log'
+
+  # Default folder to contain the pids of daemonized servers and nodes.
+  PID_PATH       = 'tmp/pids'
+
+  # A Job is processing if its WorkUnits are in the queue to be handled by nodes.
+  PROCESSING     = 1
+
+  # A Job has succeeded if all of its WorkUnits have finished successfully.
+  SUCCEEDED      = 2
+
+  # A Job has failed if even a single one of its WorkUnits has failed (they may
+  # be attempted multiple times on failure, however).
+  FAILED         = 3
+
+  # A Job is splitting if it's in the process of dividing its inputs up into
+  # multiple WorkUnits.
+  SPLITTING      = 4
+
+  # A Job is merging if it's busy collecting all of its successful WorkUnits
+  # back together into the final result.
+  MERGING        = 5
+
+  # A Job is considered to be complete if it succeeded or if it failed.
+  COMPLETE       = [SUCCEEDED, FAILED]
+
+  # A Job is considered incomplete if it's being processed, split up or merged.
+  INCOMPLETE     = [PROCESSING, SPLITTING, MERGING]
+
+  # Mapping of statuses to their display strings.
+  DISPLAY_STATUS_MAP = ['unknown', 'processing', 'succeeded', 'failed', 'splitting', 'merging']
+
+  class << self
+    attr_reader :config
+    attr_accessor :identity
+
+    # Configure CloudCrowd by passing in the path to <tt>config.yml</tt>.
+    def configure(config_path)
+      @config_path = File.expand_path(File.dirname(config_path))
+      @config = YAML.load_file(config_path)
+    end
+
+    # Configure the CloudCrowd central database (and connect to it), by passing
+    # in a path to <tt>database.yml</tt>. The file should use the standard
+    # ActiveRecord connection format.
+    def configure_database(config_path, validate_schema=true)
+      configuration = YAML.load_file(config_path)
+      ActiveRecord::Base.establish_connection(configuration)
+      if validate_schema
+        version = ActiveRecord::Base.connection.select_values('select max(version) from schema_migrations').first.to_i
+        return true if version == SCHEMA_VERSION
+        puts "Your database schema is out of date. Please use `crowd load_schema` to update it. This will wipe all the tables, so make sure that your jobs have a chance to finish first.\nexiting..."
+        exit
+      end
+    end
+
+    # Get a reference to the central server, including authentication if
+    # configured.
+    def central_server
+      @central_server ||= RestClient::Resource.new(CloudCrowd.config[:central_server], CloudCrowd.client_options)
+    end
+
+    # The path that daemonized servers and nodes will log to.
+    def log_path(log_file=nil)
+      @log_path ||= config[:log_path] || LOG_PATH
+      log_file ? File.join(@log_path, log_file) : @log_path
+    end
+
+    # The path in which daemonized servers and nodes will store their pids.
+    def pid_path(pid_file=nil)
+      @pid_path ||= config[:pid_path] || PID_PATH
+      pid_file ? File.join(@pid_path, pid_file) : @pid_path
+    end
+
+    # The standard RestClient options for the central server talking to nodes,
+    # as well as the other way around. There's a timeout of 5 seconds to open
+    # a connection, and a timeout of 30 to finish reading it.
+    def client_options
+      return @client_options if @client_options
+      @client_options = {:timeout => 30, :open_timeout => 5}
+      if CloudCrowd.config[:http_authentication]
+        @client_options[:user]      = CloudCrowd.config[:login]
+        @client_options[:password]  = CloudCrowd.config[:password]
+      end
+      @client_options
+    end
+
+    # Return the displayable status name of an internal CloudCrowd status number.
+    # (See the above constants).
+    def display_status(status)
+      DISPLAY_STATUS_MAP[status] || 'unknown'
+    end
+
+    # CloudCrowd::Actions are requested dynamically by name. Access them through
+    # this actions property, which behaves like a hash. At load time, we
+    # load all installed Actions and CloudCrowd's default Actions into it.
+    # If you wish to have certain nodes be specialized to only handle certain
+    # Actions, then install only those into the actions directory.
+    def actions
+      return @actions if @actions
+      @actions = action_paths.inject({}) do |memo, path|
+        name = File.basename(path, File.extname(path))
+        require path
+        memo[name] = Module.const_get(Inflector.camelize(name))
+        memo
+      end
+    rescue NameError => e
+      adjusted_message = "One of your actions failed to load. Please ensure that the name of your action class can be deduced from the name of the file. ex: 'word_count.rb' => 'WordCount'\n#{e.message}"
+      raise NameError.new(adjusted_message, e.name)
+    end
+
+    # Retrieve the list of every installed Action for this node or server.
+    def action_paths
+      default_actions   = Dir["#{ROOT}/actions/*.rb"]
+      installed_actions = Dir["#{@config_path}/actions/*.rb"]
+      custom_actions    = CloudCrowd.config[:actions_path] ? Dir["#{CloudCrowd.config[:actions_path]}/*.rb"] : []
+      default_actions + installed_actions + custom_actions
+    end
+
+    # Is this CloudCrowd instance a server? Useful for avoiding loading unneeded
+    # code from actions.
+    def server?
+      @identity == :server
+    end
+
+    # Or is it a node?
+    def node?
+      @identity == :node
+    end
+
+  end
+
+end

data/lib/cloud_crowd/action.rb

@@ -0,0 +1,125 @@
+module CloudCrowd
+  
+  # As you write your custom actions, have them inherit from CloudCrowd::Action.
+  # All actions must implement a +process+ method, which should return a 
+  # JSON-serializable object that will be used as the output for the work unit.
+  # See the default actions for examples.
+  #
+  # Optionally, actions may define +split+ and +merge+ methods to do mapping
+  # and reducing around the +input+. +split+ should return an array of URLs --
+  # to be mapped into WorkUnits and processed in parallel. In the +merge+ step,
+  # +input+ will be an array of all the resulting outputs from calling process.
+  #
+  # All actions have use of an individual +work_directory+, for scratch files,
+  # and spend their duration inside of it, so relative paths work well.
+  #
+  # Note that Actions inherit a backticks (`) method that raises an Exception
+  # if the external command fails.
+  class Action
+    
+    FILE_URL = /\Afile:\/\//
+    
+    attr_reader :input, :input_path, :file_name, :options, :work_directory
+    
+    # Initializing an Action sets up all of the read-only variables that
+    # form the bulk of the API for action subclasses. (Paths to read from and
+    # write to). It creates the +work_directory+ and moves into it.
+    # If we're not merging multiple results, it downloads the input file into
+    # the +work_directory+ before starting.
+    def initialize(status, input, options, store)
+      @input, @options, @store = input, options, store
+      @job_id, @work_unit_id = options['job_id'], options['work_unit_id']
+      @work_directory = File.expand_path(File.join(@store.temp_storage_path, storage_prefix))
+      FileUtils.mkdir_p(@work_directory) unless File.exists?(@work_directory)
+      parse_input
+      download_input
+    end
+    
+    # Each Action subclass must implement a +process+ method, overriding this.
+    def process
+      raise NotImplementedError, "CloudCrowd::Actions must override 'process' with their own processing code."
+    end
+    
+    # Download a file to the specified path.
+    def download(url, path)
+      `curl -s "#{url}" > "#{path}"`
+      return path
+      # The previous implementation is below, and, although it would be 
+      # wonderful not to shell out, RestClient wasn't handling URLs with encoded
+      # entities (%20, for example), and doesn't let you download to a given
+      # location. Getting a RestClient patch in would be ideal.
+      #
+      # if url.match(FILE_URL)
+      #   FileUtils.cp(url.sub(FILE_URL, ''), path)
+      # else
+      #   resp = RestClient::Request.execute(:url => url, :method => :get, :raw_response => true)
+      #   FileUtils.mv resp.file.path, path
+      # end
+    end
+    
+    # Takes a local filesystem path, saves the file to S3, and returns the 
+    # public (or authenticated) url on S3 where the file can be accessed. 
+    def save(file_path)
+      save_path = File.join(storage_prefix, File.basename(file_path))
+      @store.save(file_path, save_path)
+    end
+    
+    # After the Action has finished, we remove the work directory and return
+    # to the root directory (where workers run by default).
+    def cleanup_work_directory
+      FileUtils.rm_r(@work_directory) if File.exists?(@work_directory)
+    end
+    
+    # Actions have a backticks command that raises a CommandFailed exception 
+    # on failure, so that processing doesn't just blithely continue.
+    def `(command)
+      result    = super(command)
+      exit_code = $?.to_i
+      raise Error::CommandFailed.new(result, exit_code) unless exit_code == 0
+      result
+    end
+    
+    
+    private
+    
+    # Convert an unsafe URL into a filesystem-friendly filename.
+    def safe_filename(url)
+      ext  = File.extname(url)
+      name = URI.unescape(File.basename(url)).gsub(/[^a-zA-Z0-9_\-.]/, '-').gsub(/-+/, '-')
+      File.basename(name, ext).gsub('.', '-') + ext
+    end
+    
+    # The directory prefix to use for both local and S3 storage.
+    # [action]/job_[job_id]/unit_[work_unit_it]
+    def storage_prefix
+      path_parts = []
+      path_parts << Inflector.underscore(self.class)
+      path_parts << "job_#{@job_id}"
+      path_parts << "unit_#{@work_unit_id}" if @work_unit_id
+      @storage_prefix ||= File.join(path_parts)
+    end
+    
+    # If we think that the input is JSON, replace it with the parsed form.
+    # It would be great if the JSON module had an is_json? method.
+    def parse_input
+      return unless ['[', '{'].include? @input[0..0]
+      @input = JSON.parse(@input) rescue @input
+    end
+    
+    def input_is_url?
+      !URI.parse(@input).scheme.nil? rescue false
+    end
+    
+    # If the input is a URL, download the file before beginning processing.
+    def download_input
+      return unless input_is_url?
+      Dir.chdir(@work_directory) do
+        @input_path = File.join(@work_directory, safe_filename(@input))
+        @file_name = File.basename(@input_path, File.extname(@input_path))
+        download(@input, @input_path)
+      end
+    end
+    
+  end
+  
+end

data/lib/cloud_crowd/asset_store/filesystem_store.rb

@@ -0,0 +1,39 @@
+module CloudCrowd
+  class AssetStore
+    
+    # The FilesystemStore is an implementation of the AssetStore, good only for
+    # use in development, testing, if you're only running a single-machine
+    # installation, or are using a networked drive.
+    module FilesystemStore
+      
+      DEFAULT_STORAGE_PATH = '/tmp/cloud_crowd_storage'
+      
+      attr_reader :local_storage_path
+      
+      # Make sure that local storage exists and is writeable before starting.
+      def setup
+        lsp = @local_storage_path = CloudCrowd.config[:local_storage_path] || DEFAULT_STORAGE_PATH
+        FileUtils.mkdir_p(lsp) unless File.exists?(lsp)
+        raise Error::StorageNotWritable, "#{lsp} is not writable" unless File.writable?(lsp)
+      end
+      
+      # Save a file to somewhere semi-persistent on the filesystem. To use, 
+      # configure <tt>:storage: 'filesystem'</tt> in *config.yml*, as well as
+      # <tt>:local_storage_path:</tt>.
+      def save(local_path, save_path)
+        save_path = File.join(@local_storage_path, save_path)
+        save_dir = File.dirname(save_path)
+        FileUtils.mkdir_p save_dir unless File.exists? save_dir
+        FileUtils.cp(local_path, save_path)
+        "file://#{File.expand_path(save_path)}"
+      end
+      
+      # Remove all of a Job's result files from the filesystem.
+      def cleanup(job)
+        path = "#{@local_storage_path}/#{job.action}/job_#{job.id}"
+        FileUtils.rm_r(path) if File.exists?(path)
+      end
+    end
+    
+  end
+end

data/lib/cloud_crowd/asset_store/s3_store.rb

@@ -0,0 +1,43 @@
+module CloudCrowd
+  class AssetStore
+    
+    # The S3Store is an implementation of an AssetStore that uses a bucket
+    # on S3 for all resulting files.
+    module S3Store
+      
+      # Configure authentication and establish a connection to S3, first thing.
+      def setup
+        @use_auth   = CloudCrowd.config[:s3_authentication]
+        bucket_name = CloudCrowd.config[:s3_bucket]
+        key, secret = CloudCrowd.config[:aws_access_key], CloudCrowd.config[:aws_secret_key]
+        valid_conf  = [bucket_name, key, secret].all? {|s| s.is_a? String }
+        raise Error::MissingConfiguration, "An S3 account must be configured in 'config.yml' before 's3' storage can be used" unless valid_conf
+        protocol    = @use_auth ? 'https' : 'http'
+        port        = @use_auth ? 443 : 80
+        @s3         = RightAws::S3.new(key, secret, :protocol => protocol, :port => port)
+        @bucket     = @s3.bucket(bucket_name)
+        @bucket     = @s3.bucket(bucket_name, true) unless @bucket
+      end
+      
+      # Save a finished file from local storage to S3. Save it publicly unless 
+      # we're configured to use S3 authentication. Authenticated links expire
+      # after one day by default.
+      def save(local_path, save_path)
+        if @use_auth
+          @bucket.put(save_path, File.open(local_path), {}, 'private')
+          @s3.interface.get_link(@bucket, save_path)
+        else
+          @bucket.put(save_path, File.open(local_path), {}, 'public-read')
+          @bucket.key(save_path).public_link
+        end
+      end
+            
+      # Remove all of a Job's resulting files from S3, both intermediate and finished.
+      def cleanup(job)
+        @bucket.delete_folder("#{job.action}/job_#{job.id}")
+      end
+      
+    end
+    
+  end
+end

data/lib/cloud_crowd/asset_store.rb

@@ -0,0 +1,41 @@
+require 'tmpdir'
+
+module CloudCrowd
+
+  # The AssetStore provides a common API for storing files and returning URLs
+  # that can access them. At the moment, the files can be saved to either S3, or
+  # the local filesystem. You shouldn't need to use the AssetStore directly --
+  # Action's +download+ and +save+ methods use it behind the scenes.
+  #
+  # To implement a new back-end for the AssetStore, you must provide
+  # <tt>save(local_path, save_path)</tt>, <tt>cleanup(job)</tt>, and optionally,
+  # a <tt>setup</tt> method that will be called once at initialization.
+  class AssetStore
+
+    autoload :S3Store,         'cloud_crowd/asset_store/s3_store'
+    autoload :FilesystemStore, 'cloud_crowd/asset_store/filesystem_store'
+
+    # Configure the AssetStore with the specific storage implementation
+    # specified by 'storage' in <tt>config.yml</tt>.
+    case CloudCrowd.config[:storage]
+    when 's3'         then include S3Store
+    when 'filesystem' then include FilesystemStore
+    else raise Error::StorageNotFound, "#{CloudCrowd.config[:storage]} is not a valid storage back end"
+    end
+
+    # Creating the AssetStore ensures that its scratch directory exists.
+    def initialize
+      FileUtils.mkdir_p temp_storage_path unless File.exists? temp_storage_path
+      raise Error::StorageNotWritable, "#{temp_storage_path} is not writable" unless File.writable?(temp_storage_path)
+      setup if respond_to? :setup
+    end
+
+    # Get the path to CloudCrowd's temporary local storage. All actions run
+    # in subdirectories of this.
+    def temp_storage_path
+      @temp_storage_path ||= CloudCrowd.config[:temp_storage_path] ||  "#{Dir.tmpdir}/cloud_crowd_tmp"
+    end
+
+  end
+
+end

data/lib/cloud_crowd/command_line.rb

@@ -0,0 +1,242 @@
+require 'optparse'
+
+module CloudCrowd
+  class CommandLine
+    
+    # Configuration files required for the `crowd` command to function.
+    CONFIG_FILES = ['config.yml', 'database.yml']
+    
+    # Reference the absolute path to the root.
+    CC_ROOT = File.expand_path(File.dirname(__FILE__) + '/../..')
+    
+    # Command-line banner for the usage message.
+    BANNER = <<-EOS
+CloudCrowd is a MapReduce-inspired Parallel Processing System for Ruby.
+
+Wiki: http://wiki.github.com/documentcloud/cloud-crowd
+Rdoc: http://rdoc.info/projects/documentcloud/cloud-crowd
+
+Usage: crowd COMMAND OPTIONS
+
+Commands:
+  install       Install the CloudCrowd configuration files to the specified directory
+  server        Start up the central server (requires a database)
+  node          Start up a worker node (only one node per machine, please)
+  console       Launch a CloudCrowd console, connected to the central database
+  load_schema   Load the schema into the database specified by database.yml
+  cleanup       Removes jobs that were finished over --days (7 by default) ago
+  
+  server -d [start | stop | restart]    Servers and nodes can be launched as
+  node -d [start | stop | restart]      daemons, then stopped or restarted.
+
+Options:
+    EOS
+    
+    # Creating a CloudCrowd::CommandLine runs from the contents of ARGV.
+    def initialize
+      parse_options
+      command     = ARGV.shift
+      subcommand  = ARGV.shift
+      case command
+      when 'console'      then run_console
+      when 'server'       then run_server(subcommand)
+      when 'node'         then run_node(subcommand)
+      when 'load_schema'  then run_load_schema
+      when 'install'      then run_install(subcommand)
+      when 'cleanup'      then run_cleanup
+      else                     usage
+      end
+    end
+    
+    # Spin up an IRB session with the CloudCrowd code loaded in, and a database
+    # connection established. The equivalent of Rails' `script/console`.
+    def run_console
+      require 'irb'
+      require 'irb/completion'
+      require 'pp'
+      load_code
+      connect_to_database true
+      CloudCrowd::Server # Preload server to autoload classes.
+      Object.send(:include, CloudCrowd)
+      IRB.start
+    end
+    
+    # `crowd server` can either 'start', 'stop', or 'restart'.
+    def run_server(subcommand)
+      load_code
+      subcommand ||= 'start'
+      case subcommand
+      when 'start'    then start_server
+      when 'stop'     then stop_server
+      when 'restart'  then restart_server
+      end
+    end
+    
+    # Convenience command for quickly spinning up the central server. More
+    # sophisticated deployments, load-balancing across multiple app servers,
+    # should use the config.ru rackup file directly. This method will start
+    # a single Thin server.
+    def start_server
+      port        = @options[:port] || 9173
+      daemonize   = @options[:daemonize] ? '-d' : ''
+      log_path    = CloudCrowd.log_path('server.log')
+      pid_path    = CloudCrowd.pid_path('server.pid')
+      rackup_path = File.expand_path("#{@options[:config_path]}/config.ru")
+      FileUtils.mkdir_p(CloudCrowd.log_path) if @options[:daemonize] && !File.exists?(CloudCrowd.log_path)
+      puts "Starting CloudCrowd Central Server on port #{port}..."
+      exec "thin -e #{@options[:environment]} -p #{port} #{daemonize} --tag cloud-crowd-server --log #{log_path} --pid #{pid_path} -R #{rackup_path} start"
+    end
+    
+    # Stop the daemonized central server, if it exists.
+    def stop_server
+      Thin::Server.kill(CloudCrowd.pid_path('server.pid'), 0)
+    end
+    
+    # Restart the daemonized central server.
+    def restart_server
+      stop_server
+      sleep 1
+      start_server
+    end
+    
+    # `crowd node` can either 'start', 'stop', or 'restart'.
+    def run_node(subcommand)
+      load_code
+      ENV['RACK_ENV'] = @options[:environment]
+      case (subcommand || 'start')
+      when 'start'    then start_node
+      when 'stop'     then stop_node
+      when 'restart'  then restart_node
+      end
+    end
+    
+    # Launch a Node. Please only run a single node per machine. The Node process
+    # will be long-lived, although its workers will come and go.
+    def start_node
+      port = @options[:port] || Node::DEFAULT_PORT
+      puts "Starting CloudCrowd Node on port #{port}..."
+      Node.new(port, @options[:daemonize])
+    end
+    
+    # If the daemonized Node is running, stop it.
+    def stop_node
+      Thin::Server.kill CloudCrowd.pid_path('node.pid')
+    end
+    
+    # Restart the daemonized Node, if it exists.
+    def restart_node
+      stop_node
+      sleep 1
+      start_node
+    end
+    
+    # Load in the database schema to the database specified in 'database.yml'.
+    def run_load_schema
+      load_code
+      connect_to_database(false)
+      require 'cloud_crowd/schema.rb'
+    end
+    
+    # Install the required CloudCrowd configuration files into the specified
+    # directory, or the current one.
+    def run_install(install_path)
+      require 'fileutils'
+      install_path ||= '.'
+      FileUtils.mkdir_p install_path unless File.exists?(install_path)
+      install_file "#{CC_ROOT}/config/config.example.yml", "#{install_path}/config.yml"
+      install_file "#{CC_ROOT}/config/config.example.ru", "#{install_path}/config.ru"
+      install_file "#{CC_ROOT}/config/database.example.yml", "#{install_path}/database.yml"
+      install_file "#{CC_ROOT}/actions", "#{install_path}/actions", true
+    end
+    
+    # Clean up all Jobs in the CloudCrowd database older than --days old.
+    def run_cleanup
+      load_code
+      connect_to_database(true)
+      Job.cleanup_all(:days => @options[:days])
+    end
+    
+    # Print `crowd` usage.
+    def usage
+      puts "\n#{@option_parser}\n"
+    end
+    
+    
+    private
+    
+    # Check for configuration files, either in the current directory, or in
+    # the CLOUD_CROWD_CONFIG environment variable. Exit if they're not found.
+    def ensure_config
+      return if @config_found
+      found = CONFIG_FILES.all? {|f| File.exists? "#{@options[:config_path]}/#{f}" }
+      found ? @config_dir = true : config_not_found
+    end
+    
+    # Parse all options for all commands.
+    # Valid options are: --config --port --environment --daemonize --days.
+    def parse_options
+      @options = {
+        :environment  => 'production',
+        :config_path  => ENV['CLOUD_CROWD_CONFIG'] || '.',
+        :daemonize    => false
+      }
+      @option_parser = OptionParser.new do |opts|
+        opts.on('-c', '--config PATH', 'path to configuration directory') do |conf_path|
+          @options[:config_path] = conf_path
+        end
+        opts.on('-p', '--port PORT', 'port number for server (central or node)') do |port_num|
+          @options[:port] = port_num
+        end
+        opts.on('-e', '--environment ENV', 'server environment (defaults to production)') do |env|
+          @options[:environment] = env
+        end
+        opts.on('-d', '--daemonize', 'run as a background daemon') do |daemonize|
+          @options[:daemonize] = daemonize
+        end
+        opts.on('--days NUM_DAYS', 'grace period before cleanup (7 by default)') do |days|
+          @options[:days] = days.to_i if days.match(/\A\d+\Z/)
+        end
+        opts.on_tail('-v', '--version', 'show version') do
+          require "#{CC_ROOT}/lib/cloud-crowd"
+          puts "CloudCrowd version #{VERSION}"
+          exit
+        end
+      end
+      @option_parser.banner = BANNER
+      @option_parser.parse!(ARGV)
+    end
+    
+    # Load in the CloudCrowd module code, dependencies, lib files and models.
+    # Not all commands require this.
+    def load_code
+      ensure_config
+      require "#{CC_ROOT}/lib/cloud-crowd"
+      CloudCrowd.configure("#{@options[:config_path]}/config.yml")
+    end
+    
+    # Establish a connection to the central server's database. Not all commands
+    # require this.
+    def connect_to_database(validate_schema)
+      require 'cloud_crowd/models'
+      CloudCrowd.configure_database("#{@options[:config_path]}/database.yml", validate_schema)
+    end
+    
+    # Exit with an explanation if the configuration files couldn't be found.
+    def config_not_found
+      puts "`crowd` can't find the CloudCrowd configuration directory. Please use `crowd -c path/to/config`, or run `crowd` from inside of the configuration directory itself."
+      exit(1)
+    end
+    
+    # Install a file and log the installation. If we're overwriting a file, 
+    # offer a chance to back out.
+    def install_file(source, dest, is_dir=false)
+      if File.exists?(dest)
+        print "#{dest} already exists. Overwrite it? (yes/no) "
+        return unless ['y', 'yes', 'ok'].include? gets.chomp.downcase
+      end
+      is_dir ? FileUtils.cp_r(source, dest) : FileUtils.cp(source, dest)
+      puts "installed #{dest}" unless ENV['RACK_ENV'] == 'test'
+    end
+    
+  end
+end