documentcloud-cloud-crowd 0.0.1

data/actions/graphics_magick.rb

@@ -0,0 +1,44 @@
+# The GraphicsMagick action, dependent on the `gm` command, is able to perform
+# any number of GraphicsMagick conversions on an image passed in as an input.
+# The options hash should specify the +name+ for the particular step (which is
+# appended to the resulting image filename) the +command+ (eg. convert, mogrify), 
+# the +options+ (to the command, eg. -shadow -blur), and the +extension+ which 
+# will determine the resulting image type. Optionally, you may also specify
+# +input+ as the name of a previous step; doing this will use the result of
+# that step as the source image, otherwise each step uses the original image
+# as its source.
+class GraphicsMagick < CloudCrowd::Action
+  
+  # Download the initial image, and run each of the specified GraphicsMagick
+  # commands against it, returning the aggregate output.
+  def run
+    return options['steps'].map {|step| run_step(step) }
+  end
+  
+  # Run an individual step (single GraphicsMagick command) in a shell-injection
+  # safe way, uploading the result to the AssetStore, and returning the public
+  # URL as the result.
+  # TODO: +system+ wasn't working, figure out some other way to escape.
+  def run_step(step)
+    name, cmd, opts, ext = step['name'], step['command'], step['options'], step['extension']
+    in_path, out_path = input_path_for(step), output_path_for(step)
+    `gm #{cmd} #{opts} #{in_path} #{out_path}`
+    public_url = save(out_path)
+    {'name' => name, 'url' => public_url}.to_json
+  end
+  
+  # Where should the starting image be located?
+  # If you pass in an optional step, returns the path to that step's output
+  # as input for further processing.
+  def input_path_for(step)
+    in_step = step && step['input'] && options['steps'].detect {|s| s['name'] == step['input']}
+    return input_path unless in_step
+    return output_path_for(in_step)
+  end
+  
+  # Where should resulting images be saved locally?
+  def output_path_for(step)
+    "#{work_directory}/#{file_name}_#{step['name']}.#{step['extension']}"
+  end
+  
+end

data/bin/crowd

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+
+require "#{File.dirname(__FILE__)}/../lib/cloud_crowd/command_line"
+
+CloudCrowd::CommandLine.new

data/cloud-crowd.gemspec

@@ -0,0 +1,71 @@
+Gem::Specification.new do |s|
+  s.name      = 'cloud-crowd'
+  s.version   = '0.0.1'
+  s.date      = '2009-08-23'
+
+  s.homepage    = "http://documentcloud.org" # wiki page on github?  
+  s.summary     = "Better living through Map --> Ruby --> Reduce"
+  s.description = <<-EOS
+    The crowd, suddenly there where there was nothing before, is a mysterious and
+    universal phenomenon. A few people may have been standing together -- five, ten
+    or twelve, nor more; nothing has been announced, nothing is expected. Suddenly
+    everywhere is black with people and more come streaming from all sides as though
+    streets had only one direction.
+  EOS
+  
+  s.authors     = ['Jeremy Ashkenas']
+  s.email       = 'jeremy@documentcloud.org'
+  
+  s.require_paths = ['lib']
+  s.executables   = ['crowd']
+  
+  s.post_install_message = "Run `crowd help` for information on using CloudCrowd."
+  s.rubyforge_project    = 'cloud-crowd'
+  s.has_rdoc             = true
+  
+  s.add_dependency 'sinatra',       ['>= 0.9.4']
+  s.add_dependency 'activerecord',  ['>= 2.3.3']
+  s.add_dependency 'json',          ['>= 1.1.7']
+  s.add_dependency 'rest-client',   ['>= 1.0.3']
+  s.add_dependency 'right_aws',     ['>= 1.10.0']
+  s.add_dependency 'daemons',       ['>= 1.0.10']
+
+  if s.respond_to?(:add_development_dependency)
+    s.add_development_dependency 'faker',               ['>= 0.3.1']
+    s.add_development_dependency 'thoughtbot-shoulda',  ['>= 2.10.2']
+    s.add_development_dependency 'notahat-machinist',   ['>= 1.0.3']
+    s.add_development_dependency 'rack-test',           ['>= 0.4.1']
+    s.add_development_dependency 'mocha',               ['>= 0.9.7']
+  end
+  
+  s.files = %w(
+actions/graphics_magick.rb
+cloud-crowd.gemspec
+config/config.example.ru
+config/config.example.yml
+config/database.example.yml
+lib/cloud-crowd.rb
+lib/cloud_crowd/action.rb
+lib/cloud_crowd/app.rb
+lib/cloud_crowd/asset_store.rb
+lib/cloud_crowd/command_line.rb
+lib/cloud_crowd/core_ext.rb
+lib/cloud_crowd/daemon.rb
+lib/cloud_crowd/helpers/resources.rb
+lib/cloud_crowd/helpers/urls.rb
+lib/cloud_crowd/helpers.rb
+lib/cloud_crowd/models/job.rb
+lib/cloud_crowd/models/work_unit.rb
+lib/cloud_crowd/models.rb
+lib/cloud_crowd/runner.rb
+lib/cloud_crowd/schema.rb
+lib/cloud_crowd/worker.rb
+test/acceptance/test_failing_work_units.rb
+test/blueprints.rb
+test/config/test_config.yml
+test/config/test_database.yml
+test/test_helper.rb
+test/unit/test_job.rb
+test/unit/test_work_unit.rb
+)
+end

data/config/config.example.ru

@@ -0,0 +1,17 @@
+#!/usr/bin/env ruby
+
+# This rackup script can be used to start the central CloudCrowd server
+# using any Rack-compliant server handler. For example, start up three servers 
+# with a specified port number, using Thin:
+#
+# thin start -R config.ru -p 9173 --servers 3
+
+require 'rubygems'
+require 'cloud-crowd'
+
+CloudCrowd.configure(File.dirname(__FILE__) + '/config.yml')
+CloudCrowd.configure_database(File.dirname(__FILE__) + '/database.yml')
+
+map '/' do
+  run CloudCrowd::App
+end

data/config/config.example.yml

@@ -0,0 +1,11 @@
+:num_workers:             4
+:default_worker_wait:     1
+:max_worker_wait:         20
+:worker_wait_multiplier:  1.3
+:worker_retry_wait:       5
+:work_unit_retries:       3
+
+:central_server:          http://localhost:9173
+:s3_bucket:               [your CloudCrowd bucket]
+:aws_access_key:          [your AWS access key]
+:aws_secret_key:          [your AWS secret access key]

data/config/database.example.yml

@@ -0,0 +1,6 @@
+:adapter:  mysql
+:encoding: utf8
+:username: root
+:password:
+:socket:   /tmp/mysql.sock
+:database: cloud_crowd

data/lib/cloud-crowd.rb

@@ -0,0 +1,96 @@
+$LOAD_PATH.unshift File.expand_path(File.dirname(__FILE__))
+
+# Standard Library:
+require 'tmpdir'
+require 'erb'
+
+# Gems:
+require 'sinatra'
+require 'activerecord'
+require 'json'
+require 'daemons'
+require 'rest_client'
+require 'right_aws'
+
+module CloudCrowd
+  
+  class App < Sinatra::Default
+    set :root, File.expand_path(File.dirname(__FILE__) + '/..')
+  end
+  
+  # Keep the version in sync with the gemspec.
+  VERSION     = '0.0.1'
+    
+  # A Job is processing if its WorkUnits in the queue to be handled by workers.
+  PROCESSING  = 1
+  
+  # A Job has succeeded if all of its WorkUnits have finished successfully.
+  SUCCEEDED   = 2
+  
+  # A Job has failed if even a single one of its WorkUnits has failed (they may
+  # be attempted multiple times on failure, however).
+  FAILED      = 3
+  
+  # A Job is splitting if it's in the process of dividing its inputs up into
+  # multiple WorkUnits.
+  SPLITTING   = 4
+  
+  # A Job is merging if it's busy collecting all of its successful WorkUnits
+  # back together into the final result.
+  MERGING     = 5
+  
+  # A work unit is considered to be complete if it succeeded or if it failed.
+  COMPLETE    = [SUCCEEDED, FAILED]
+  
+  # A work unit is considered incomplete if it's being processed, split up or 
+  # merged together.
+  INCOMPLETE  = [PROCESSING, SPLITTING, MERGING]
+  
+  # Mapping of statuses to their display strings.
+  DISPLAY_STATUS_MAP = {
+    1 => 'processing', 2 => 'succeeded', 3 => 'failed', 4 => 'splitting', 5 => 'merging'
+  }
+  
+  class << self
+    attr_reader :config
+    
+    # Configure CloudCrowd by passing in the path to +config.yml+.
+    def configure(config_path)
+      @config = YAML.load_file(config_path)
+    end
+
+    # Configure the CloudCrowd central database (and connect to it), by passing
+    # in a path to +database.yml+.
+    def configure_database(config_path)
+      configuration = YAML.load_file(config_path)
+      ActiveRecord::Base.establish_connection(configuration)
+    end
+
+    # Return the readable status name of an internal CloudCrowd status number.
+    def display_status(status)
+      DISPLAY_STATUS_MAP[status]
+    end
+    
+    # Some workers might not ever need to load all the installed actions,
+    # so we lazy-load them. Think about a variant of this for installing and
+    # loading actions into a running CloudCrowd cluster on the fly.
+    def actions(name)
+      action_class = name.camelize
+      begin
+        Module.const_get(action_class)
+      rescue NameError => e
+        require "#{CloudCrowd::App.root}/actions/#{name}"
+        retry
+      end
+    end
+  end
+  
+end
+
+# CloudCrowd:
+require 'cloud_crowd/core_ext'
+require 'cloud_crowd/models'
+require 'cloud_crowd/asset_store'
+require 'cloud_crowd/action'
+require 'cloud_crowd/helpers'
+require 'cloud_crowd/app'

data/lib/cloud_crowd/action.rb

@@ -0,0 +1,88 @@
+module CloudCrowd
+  
+  # Base CloudCrowd::Action class. Override this with your custom action steps.
+  #
+  # Public API to CloudCrowd::Action subclasses:
+  # +input+, +input_path+, +file_name+, +work_directory+, +options+, +save+
+  #
+  # CloudCrowd::Actions must implement a +process+ method, which must return a 
+  # JSON-serializeable object that will be used as the output for the work unit.
+  # Optionally, actions may define +split+ and +merge+ methods to do mapping
+  # and reducing around the input.
+  # +split+ must return an array of inputs.
+  # +merge+ must return the output for the job.
+  # All actions run inside of their individual +work_directory+.
+  class Action
+    
+    attr_reader :input, :input_path, :file_name, :options, :work_directory
+    
+    # Configuring a new Action sets up all of the read-only variables that
+    # form the bulk of the API for action subclasses. (Paths to read from and
+    # write to).
+    def configure(status, input, options, store)
+      @input, @options, @store = input, options, store
+      @job_id, @work_unit_id = options['job_id'], options['work_unit_id']
+      @work_directory = File.expand_path(File.join(@store.temp_storage_path, storage_prefix))
+      FileUtils.mkdir_p(@work_directory) unless File.exists?(@work_directory)
+      Dir.chdir @work_directory
+      unless status == CloudCrowd::MERGING
+        @input_path = File.join(@work_directory, File.basename(@input))
+        @file_name = File.basename(@input_path, File.extname(@input_path))
+        download(@input, @input_path)
+      end
+    end
+    
+    # Each CloudCrowd::Action must implement a +process+ method.
+    def process
+      raise NotImplementedError.new("CloudCrowd::Actions must override 'run' with their own processing code.")
+    end
+    
+    # Download a file to the specified path using curl.
+    def download(url, path)
+      `curl -s "#{url}" > #{path}`
+      path
+    end
+    
+    # Takes a local filesystem path, and returns the public url on S3 where the 
+    # file was saved. 
+    def save(file_path)
+      save_path = File.join(s3_storage_path, File.basename(file_path))
+      @store.save(file_path, save_path)
+      return @store.url(save_path)
+    end
+    
+    # After the Action has finished, we remove the work directory.
+    def cleanup_work_directory
+      Dir.chdir '/'
+      FileUtils.rm_r(@work_directory)
+    end
+    
+    
+    private
+    
+    # The directory prefix to use for both local and S3 storage.
+    # [action_name]/job_[job_id]/unit_[work_unit_it]
+    def storage_prefix
+      path_parts = []
+      path_parts << underscore(self.class.to_s)
+      path_parts << "job_#{@job_id}"
+      path_parts << "unit_#{@work_unit_id}" if @work_unit_id
+      @storage_prefix ||= File.join(path_parts)
+    end
+    
+    def s3_storage_path
+      @s3_storage_path ||= storage_prefix
+    end
+    
+    # Pilfered from the ActiveSupport::Inflector.
+    def underscore(word)
+      word.to_s.gsub(/::/, '/').
+        gsub(/([A-Z]+)([A-Z][a-z])/,'\1_\2').
+        gsub(/([a-z\d])([A-Z])/,'\1_\2').
+        tr("-", "_").
+        downcase
+    end
+    
+  end
+  
+end

data/lib/cloud_crowd/app.rb

@@ -0,0 +1,54 @@
+module CloudCrowd
+  
+  class App < Sinatra::Default
+        
+    # static serves files from /public, methodoverride allows the _method param.
+    enable :static, :methodoverride
+    
+    helpers CloudCrowd::Helpers
+    
+    # Start a new job. Accepts a JSON representation of the job-to-be.
+    post '/jobs' do
+      Job.create_from_request(JSON.parse(params[:json])).to_json
+    end
+    
+    # Check the status of a job, returning the output if finished, and the
+    # number of work units remaining otherwise. 
+    get '/jobs/:job_id' do
+      current_job.to_json
+    end
+    
+    # Cleans up a Job's saved S3 files. Delete a Job after you're done 
+    # downloading the results.
+    delete '/jobs/:job_id' do
+      current_job.cleanup
+      ''
+    end
+    
+    # Internal method for worker daemons to fetch the work unit at the front
+    # of the queue. Work unit is marked as taken and handed off to the worker.
+    get '/work' do
+      begin
+        unit = WorkUnit.first(:conditions => {:status => CloudCrowd::INCOMPLETE, :taken => false}, :order => "created_at desc")
+        return status(204) && '' unless unit
+        unit.update_attributes(:taken => true)
+        unit.to_json
+      rescue ActiveRecord::StaleObjectError => e
+        return status(204) && ''
+      end
+    end
+    
+    # When workers are done with their unit, either successfully on in failure,
+    # they mark it back on the central server.
+    put '/work/:work_unit_id' do
+      case params[:status]
+      when 'succeeded' then current_work_unit.finish(params[:output], params[:time])
+      when 'failed'    then current_work_unit.fail(params[:output], params[:time])
+      else             return error(500, "Completing a work unit must specify status.")
+      end
+      return status(204) && ''
+    end
+    
+  end
+  
+end

data/lib/cloud_crowd/asset_store.rb

@@ -0,0 +1,58 @@
+module CloudCrowd
+
+  # The CloudCrowd::AssetStore should provide a common API for stashing and retrieving
+  # assets via URLs, in production this will be S3 but in development it may
+  # be the filesystem or /tmp.
+  class AssetStore
+    include FileUtils
+    
+    def initialize
+      mkdir_p temp_storage_path unless File.exists? temp_storage_path
+    end
+    
+    # Path to CloudCrowd's temporary local storage.
+    def temp_storage_path
+      "#{Dir.tmpdir}/cloud_crowd_tmp"
+    end
+    
+    # Copy a finished file from our local storage to S3.
+    def save(local_path, save_path)
+      ensure_s3_connection
+      @bucket.put(save_path, File.open(local_path), {}, 'public-read')
+    end
+    
+    # Cleanup all S3 files for a job that's been completed and retrieved.
+    def cleanup_job(job)
+      ensure_s3_connection
+      @bucket.delete_folder("#{job.action}/job_#{job.id}")
+    end
+    
+    # Return the S3 public URL for a finshed file.
+    def url(save_path)
+      @bucket.key(save_path).public_link
+    end
+    
+    private
+    
+    # Unused for the moment. Think about using the filesystem instead of S3
+    # in development.
+    def save_to_filesystem(local_path, save_path)
+      save_path = File.join("/tmp/cloud_crowd_storage", save_path)
+      save_dir = File.dirname(save_path)
+      mkdir_p save_dir unless File.exists? save_dir
+      cp(local_path, save_path)
+    end
+    
+    # Workers, through the course of many WorkUnits, keep around an AssetStore.
+    # Ensure we have a persistent S3 connection after first use.
+    def ensure_s3_connection
+      unless @s3 && @bucket
+        params = {:port => 80, :protocol => 'http'}
+        @s3 = RightAws::S3.new(CloudCrowd.config[:aws_access_key], CloudCrowd.config[:aws_secret_key], params)
+        @bucket = @s3.bucket(CloudCrowd.config[:s3_bucket], true)
+      end
+    end
+  
+  end
+
+end

data/lib/cloud_crowd/command_line.rb

@@ -0,0 +1,198 @@
+require 'optparse'
+
+module CloudCrowd
+  class CommandLine
+    
+    # Configuration files required for the `crowd` command to function.
+    CONFIG_FILES = ['config.yml', 'config.ru', 'database.yml']
+    
+    # Path to the Daemons gem script which launches workers.
+    WORKER_RUNNER = File.expand_path("#{File.dirname(__FILE__)}/runner.rb")
+    
+    # Command-line banner for the usage message.
+    BANNER = <<-EOS
+Usage: crowd COMMAND OPTIONS
+
+COMMANDS:
+  install       Install the CloudCrowd configuration files to the specified directory
+  server        Start up the central server (requires a database)
+  workers       Control worker daemons, use: (start | stop | restart | status | run)
+  console       Launch a CloudCrowd console, connected to the central database
+  load_schema   Load the schema into the database specified by database.yml
+
+OPTIONS:
+    EOS
+    
+    # Creating a CloudCrowd::CommandLine runs from the contents of ARGV.
+    def initialize
+      parse_options
+      command = ARGV.shift
+      case command
+      when 'console'      then run_console
+      when 'server'       then run_server
+      when 'workers'      then run_workers_command
+      when 'load_schema'  then run_load_schema
+      when 'install'      then run_install
+      else                     usage
+      end
+    end
+    
+    # Spin up an IRB session with the CloudCrowd code loaded in, and a database
+    # connection established. The equivalent of Rails' `script/console`.
+    def run_console
+      require 'irb'
+      require 'irb/completion'
+      load_code
+      connect_to_database
+      IRB.start
+    end
+    
+    # Convenience command for quickly spinning up the central server. More 
+    # sophisticated deployments, load-balancing across multiple app servers, 
+    # should use the config.ru rackup file directly. This method will start
+    # a single Thin server, if Thin is installed, otherwise the rackup defaults 
+    # (Mongrel, falling back to WEBrick). The equivalent of Rails' script/server.
+    def run_server
+      ensure_config
+      require 'rubygems'
+      rackup_path = File.expand_path('config.ru')
+      if Gem.available? 'thin'
+        exec "thin -e production -p #{@options[:port]} -R #{rackup_path} start"
+      else
+        exec "rackup -E production -p #{@options[:port]} #{rackup_path}"
+      end
+    end
+    
+    # Load in the database schema to the database specified in 'database.yml'.
+    def run_load_schema
+      load_code
+      connect_to_database
+      require 'cloud_crowd/schema.rb'
+    end
+    
+    # Install the required CloudCrowd configuration files into the specified
+    # directory, or the current one.
+    def run_install
+      require 'fileutils'
+      install_path = ARGV.shift || '.'
+      cc_root = File.dirname(__FILE__) + '/../..'
+      FileUtils.mkdir_p install_path unless File.exists?(install_path)
+      install_file "#{cc_root}/config/config.example.yml", "#{install_path}/config.yml"
+      install_file "#{cc_root}/config/config.example.ru", "#{install_path}/config.ru"
+      install_file "#{cc_root}/config/database.example.yml", "#{install_path}/database.yml"
+      install_file "#{cc_root}/actions", "#{install_path}/actions", true
+    end
+    
+    # Manipulate worker daemons -- handles all commands that the Daemons gem
+    # provides: start, stop, restart, run, and status.
+    def run_workers_command
+      ensure_config
+      command = ARGV.shift
+      case command
+      when 'start'    then start_workers
+      when 'stop'     then stop_workers
+      when 'restart'  then stop_workers && start_workers
+      when 'run'      then run_worker
+      when 'status'   then show_worker_status
+      else                 usage
+      end
+    end
+    
+    # Start up N workers, specified by argument or the number of workers in
+    # config.yml.
+    def start_workers
+      load_code
+      num_workers = @options[:num_workers] || CloudCrowd.config[:num_workers]
+      num_workers.times do
+        `CLOUD_CROWD_CONFIG='#{File.expand_path('config.yml')}' ruby #{WORKER_RUNNER} start`
+      end
+    end
+    
+    # For debugging, run a single worker in the current process, showing output.
+    def run_worker
+      exec "CLOUD_CROWD_CONFIG='#{File.expand_path('config.yml')}' ruby #{WORKER_RUNNER} run"
+    end
+    
+    # Stop all active workers.
+    def stop_workers
+      `ruby #{WORKER_RUNNER} stop`
+    end
+
+    # Display the status of all active workers.
+    def show_worker_status
+      puts `ruby #{WORKER_RUNNER} status`
+    end
+    
+    # Print `crowd` usage.
+    def usage
+      puts @option_parser
+    end
+    
+    
+    private
+    
+    # Check for configuration files, either in the current directory, or in
+    # the CLOUD_CROWD_CONFIG environment variable. Exit if they're not found.
+    def ensure_config
+      return if @config_found
+      config_dir = ENV['CLOUD_CROWD_CONFIG'] || '.'
+      Dir.chdir config_dir
+      CONFIG_FILES.all? {|f| File.exists? f } ? @config_dir = true : config_not_found
+    end
+    
+    # Parse all options for all actions.
+    # TODO: Think about parsing options per sub-command separately.
+    def parse_options
+      @options = {
+        :db_config => 'database.yml',
+        :port      => 9173,
+      }
+      @option_parser = OptionParser.new do |opts|
+        opts.on('-n', '--num-workers NUM', OptionParser::DecimalInteger, 'number of worker processes') do |num|
+          @options[:num_workers] = num
+        end
+        opts.on('-d', '--database-config PATH', 'path to database.yml') do |conf_path|
+          @options[:db_config] = conf_path
+        end
+        opts.on('-p', '--port PORT', 'central server port number') do |port_num|
+          @options[:port] = port_num
+        end
+        opts.on_tail('-v', '--version', 'show version') do
+          load_code
+          puts "CloudCrowd version #{CloudCrowd::VERSION}"
+          exit
+        end
+      end
+      @option_parser.banner = BANNER
+      @option_parser.parse!(ARGV)
+    end
+    
+    # Load in the CloudCrowd module code, dependencies, lib files and models.
+    # Not all commands require this.
+    def load_code
+      ensure_config
+      require 'rubygems'
+      require File.dirname(__FILE__) + '/../cloud-crowd'
+      CloudCrowd.configure('config.yml')
+    end
+    
+    # Establish a connection to the central server's database. Not all commands
+    # require this.
+    def connect_to_database
+      CloudCrowd.configure_database(@options[:db_config])
+    end
+    
+    # Exit with an explanation if the configuration files couldn't be found.
+    def config_not_found
+      puts "`crowd` can't find the CloudCrowd configuration directory. Please either run `crowd` from inside of the configuration directory, or add a CLOUD_CROWD_CONFIG variable to your environment."
+      exit(1)
+    end
+    
+    # Install a file and log the installation.
+    def install_file(source, dest, is_dir=false)
+      is_dir ? FileUtils.cp_r(source, dest) : FileUtils.cp(source, dest)
+      puts "installed #{dest}"
+    end
+    
+  end
+end

data/lib/cloud_crowd/core_ext.rb

@@ -0,0 +1,10 @@
+# Extensions to core Ruby.
+
+class String
+  
+  # Stolen-ish in parts from ActiveSupport::Inflector.
+  def camelize
+    self.gsub(/\/(.?)/) { "::#{$1.upcase}" }.gsub(/(?:^|_)(.)/) { $1.upcase }
+  end
+  
+end