documentcloud-cloud-crowd 0.0.1 → 0.0.2

data/actions/graphics_magick.rb

@@ -11,8 +11,10 @@ class GraphicsMagick < CloudCrowd::Action
   
   # Download the initial image, and run each of the specified GraphicsMagick
   # commands against it, returning the aggregate output.
-  def run
-    return options['steps'].map {|step| run_step(step) }
+  def process
+    result = {}
+    options['steps'].each {|step| result[step['name']] = run_step(step) }
+    result.to_json
   end
   
   # Run an individual step (single GraphicsMagick command) in a shell-injection
@@ -20,11 +22,10 @@ class GraphicsMagick < CloudCrowd::Action
   # URL as the result.
   # TODO: +system+ wasn't working, figure out some other way to escape.
   def run_step(step)
-    name, cmd, opts, ext = step['name'], step['command'], step['options'], step['extension']
+    cmd, opts = step['command'], step['options']
     in_path, out_path = input_path_for(step), output_path_for(step)
     `gm #{cmd} #{opts} #{in_path} #{out_path}`
-    public_url = save(out_path)
-    {'name' => name, 'url' => public_url}.to_json
+    save(out_path)    
   end
   
   # Where should the starting image be located?

data/cloud-crowd.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name      = 'cloud-crowd'
-  s.version   = '0.0.1'
+  s.version   = '0.0.2'         # Keep version in sync with cloud-cloud.rb
   s.date      = '2009-08-23'
 
   s.homepage    = "http://documentcloud.org" # wiki page on github?  
@@ -19,7 +19,7 @@ Gem::Specification.new do |s|
   s.require_paths = ['lib']
   s.executables   = ['crowd']
   
-  s.post_install_message = "Run `crowd help` for information on using CloudCrowd."
+  # s.post_install_message = "Run `crowd --help` for information on using CloudCrowd."
   s.rubyforge_project    = 'cloud-crowd'
   s.has_rdoc             = true
   
@@ -51,6 +51,7 @@ lib/cloud_crowd/asset_store.rb
 lib/cloud_crowd/command_line.rb
 lib/cloud_crowd/core_ext.rb
 lib/cloud_crowd/daemon.rb
+lib/cloud_crowd/helpers/authorization.rb
 lib/cloud_crowd/helpers/resources.rb
 lib/cloud_crowd/helpers/urls.rb
 lib/cloud_crowd/helpers.rb
@@ -62,8 +63,9 @@ lib/cloud_crowd/schema.rb
 lib/cloud_crowd/worker.rb
 test/acceptance/test_failing_work_units.rb
 test/blueprints.rb
-test/config/test_config.yml
-test/config/test_database.yml
+test/config/config.yml
+test/config/database.yml
+test/config/actions/failure_testing.rb
 test/test_helper.rb
 test/unit/test_job.rb
 test/unit/test_work_unit.rb

data/config/config.example.ru

@@ -8,6 +8,7 @@
 
 require 'rubygems'
 require 'cloud-crowd'
+require 'cloud_crowd/app'
 
 CloudCrowd.configure(File.dirname(__FILE__) + '/config.yml')
 CloudCrowd.configure_database(File.dirname(__FILE__) + '/database.yml')

data/config/config.example.yml

@@ -6,6 +6,11 @@
 :work_unit_retries:       3
 
 :central_server:          http://localhost:9173
+:use_http_authentication: no
+:login:                   [your login name]
+:password:                [your password]
+
+:use_s3_authentication:   no
 :s3_bucket:               [your CloudCrowd bucket]
 :aws_access_key:          [your AWS access key]
-:aws_secret_key:          [your AWS secret access key]
+:aws_secret_key:          [your AWS secret access key]

data/lib/cloud-crowd.rb

@@ -1,25 +1,21 @@
 $LOAD_PATH.unshift File.expand_path(File.dirname(__FILE__))
 
-# Standard Library:
-require 'tmpdir'
-require 'erb'
-
-# Gems:
-require 'sinatra'
-require 'activerecord'
+# Common Gems:
 require 'json'
-require 'daemons'
 require 'rest_client'
 require 'right_aws'
 
+# Common CloudCrowd libs:
+require 'cloud_crowd/core_ext'
+require 'cloud_crowd/action'
+
 module CloudCrowd
   
-  class App < Sinatra::Default
-    set :root, File.expand_path(File.dirname(__FILE__) + '/..')
-  end
+  # Root directory of the CloudCrowd gem.
+  ROOT        = File.expand_path(File.dirname(__FILE__) + '/..')
   
   # Keep the version in sync with the gemspec.
-  VERSION     = '0.0.1'
+  VERSION     = '0.0.2'
     
   # A Job is processing if its WorkUnits in the queue to be handled by workers.
   PROCESSING  = 1
@@ -56,6 +52,7 @@ module CloudCrowd
     
     # Configure CloudCrowd by passing in the path to +config.yml+.
     def configure(config_path)
+      @config_path = File.expand_path(File.dirname(config_path))
       @config = YAML.load_file(config_path)
     end
 
@@ -77,20 +74,16 @@ module CloudCrowd
     def actions(name)
       action_class = name.camelize
       begin
+        raise NameError, "can't find the #{action_class} Action" unless Module.constants.include?(action_class)
         Module.const_get(action_class)
       rescue NameError => e
-        require "#{CloudCrowd::App.root}/actions/#{name}"
-        retry
+        user_action     = "#{@config_path}/actions/#{name}"
+        default_action  = "#{CloudCrowd::ROOT}/actions/#{name}"
+        require user_action and retry    if File.exists? "#{user_action}.rb"
+        require default_action and retry if File.exists? "#{default_action}.rb"
+        raise e
       end
     end
   end
   
-end
-
-# CloudCrowd:
-require 'cloud_crowd/core_ext'
-require 'cloud_crowd/models'
-require 'cloud_crowd/asset_store'
-require 'cloud_crowd/action'
-require 'cloud_crowd/helpers'
-require 'cloud_crowd/app'
+end

data/lib/cloud_crowd/action.rb

@@ -34,7 +34,7 @@ module CloudCrowd
     
     # Each CloudCrowd::Action must implement a +process+ method.
     def process
-      raise NotImplementedError.new("CloudCrowd::Actions must override 'run' with their own processing code.")
+      raise NotImplementedError.new("CloudCrowd::Actions must override 'process' with their own processing code.")
     end
     
     # Download a file to the specified path using curl.

data/lib/cloud_crowd/app.rb

@@ -1,3 +1,8 @@
+require 'erb'
+require 'sinatra'
+require 'cloud_crowd/models'
+require 'cloud_crowd/helpers'
+
 module CloudCrowd
   
   class App < Sinatra::Default
@@ -5,8 +10,15 @@ module CloudCrowd
     # static serves files from /public, methodoverride allows the _method param.
     enable :static, :methodoverride
     
+    set :root,                CloudCrowd::ROOT
+    set :authorization_realm, "CloudCrowd"
+    
     helpers CloudCrowd::Helpers
     
+    before do
+      login_required if CloudCrowd.config[:use_http_authentication]
+    end
+    
     # Start a new job. Accepts a JSON representation of the job-to-be.
     post '/jobs' do
       Job.create_from_request(JSON.parse(params[:json])).to_json
@@ -49,6 +61,12 @@ module CloudCrowd
       return status(204) && ''
     end
     
+    # To monitor the central server with Monit, God, Nagios, or another 
+    # monitoring tool, you can hit /heartbeat to check.
+    get '/heartbeat' do
+      "buh-bump"
+    end
+    
   end
   
 end

data/lib/cloud_crowd/asset_store.rb

@@ -1,3 +1,5 @@
+require 'tmpdir'
+
 module CloudCrowd
 
   # The CloudCrowd::AssetStore should provide a common API for stashing and retrieving
@@ -7,6 +9,7 @@ module CloudCrowd
     include FileUtils
     
     def initialize
+      @use_auth = CloudCrowd.config[:use_s3_authentication]
       mkdir_p temp_storage_path unless File.exists? temp_storage_path
     end
     
@@ -15,10 +18,12 @@ module CloudCrowd
       "#{Dir.tmpdir}/cloud_crowd_tmp"
     end
     
-    # Copy a finished file from our local storage to S3.
+    # Copy a finished file from our local storage to S3. Save it publicly if
+    # we're not configured to use S3 authentication.
     def save(local_path, save_path)
       ensure_s3_connection
-      @bucket.put(save_path, File.open(local_path), {}, 'public-read')
+      permission = @use_auth ? 'private' : 'public-read'
+      @bucket.put(save_path, File.open(local_path), {}, permission)
     end
     
     # Cleanup all S3 files for a job that's been completed and retrieved.
@@ -27,9 +32,11 @@ module CloudCrowd
       @bucket.delete_folder("#{job.action}/job_#{job.id}")
     end
     
-    # Return the S3 public URL for a finshed file.
+    # Return the S3 public URL for a finshed file. Authenticated links expire
+    # after one day by default.
     def url(save_path)
-      @bucket.key(save_path).public_link
+      @use_auth ? @s3.interface.get_link(@bucket, save_path) :
+                  @bucket.key(save_path).public_link
     end
     
     private

data/lib/cloud_crowd/command_line.rb

@@ -6,8 +6,11 @@ module CloudCrowd
     # Configuration files required for the `crowd` command to function.
     CONFIG_FILES = ['config.yml', 'config.ru', 'database.yml']
     
+    # Reference the absolute path to the root, because we're about to chdir.
+    CC_ROOT = File.expand_path(File.dirname(__FILE__) + '/../..')
+    
     # Path to the Daemons gem script which launches workers.
-    WORKER_RUNNER = File.expand_path("#{File.dirname(__FILE__)}/runner.rb")
+    WORKER_RUNNER = File.expand_path("#{CC_ROOT}/lib/cloud_crowd/runner.rb")
     
     # Command-line banner for the usage message.
     BANNER = <<-EOS
@@ -75,12 +78,11 @@ OPTIONS:
     def run_install
       require 'fileutils'
       install_path = ARGV.shift || '.'
-      cc_root = File.dirname(__FILE__) + '/../..'
       FileUtils.mkdir_p install_path unless File.exists?(install_path)
-      install_file "#{cc_root}/config/config.example.yml", "#{install_path}/config.yml"
-      install_file "#{cc_root}/config/config.example.ru", "#{install_path}/config.ru"
-      install_file "#{cc_root}/config/database.example.yml", "#{install_path}/database.yml"
-      install_file "#{cc_root}/actions", "#{install_path}/actions", true
+      install_file "#{CC_ROOT}/config/config.example.yml", "#{install_path}/config.yml"
+      install_file "#{CC_ROOT}/config/config.example.ru", "#{install_path}/config.ru"
+      install_file "#{CC_ROOT}/config/database.example.yml", "#{install_path}/database.yml"
+      install_file "#{CC_ROOT}/actions", "#{install_path}/actions", true
     end
     
     # Manipulate worker daemons -- handles all commands that the Daemons gem
@@ -135,8 +137,7 @@ OPTIONS:
     # the CLOUD_CROWD_CONFIG environment variable. Exit if they're not found.
     def ensure_config
       return if @config_found
-      config_dir = ENV['CLOUD_CROWD_CONFIG'] || '.'
-      Dir.chdir config_dir
+      Dir.chdir @options[:config_path]
       CONFIG_FILES.all? {|f| File.exists? f } ? @config_dir = true : config_not_found
     end
     
@@ -144,16 +145,20 @@ OPTIONS:
     # TODO: Think about parsing options per sub-command separately.
     def parse_options
       @options = {
-        :db_config => 'database.yml',
-        :port      => 9173,
+        :db_config    => 'database.yml',
+        :port         => 9173,
+        :config_path  => ENV['CLOUD_CROWD_CONFIG'] || '.',
       }
       @option_parser = OptionParser.new do |opts|
-        opts.on('-n', '--num-workers NUM', OptionParser::DecimalInteger, 'number of worker processes') do |num|
-          @options[:num_workers] = num
+        opts.on('-c', '--config PATH', 'path to configuration directory') do |conf_path|
+          @options[:config_path] = conf_path
         end
         opts.on('-d', '--database-config PATH', 'path to database.yml') do |conf_path|
           @options[:db_config] = conf_path
         end
+        opts.on('-n', '--num-workers NUM', OptionParser::DecimalInteger, 'number of worker processes') do |num|
+          @options[:num_workers] = num
+        end
         opts.on('-p', '--port PORT', 'central server port number') do |port_num|
           @options[:port] = port_num
         end
@@ -172,19 +177,20 @@ OPTIONS:
     def load_code
       ensure_config
       require 'rubygems'
-      require File.dirname(__FILE__) + '/../cloud-crowd'
+      require "#{CC_ROOT}/lib/cloud-crowd"
       CloudCrowd.configure('config.yml')
     end
     
     # Establish a connection to the central server's database. Not all commands
     # require this.
     def connect_to_database
+      require 'cloud_crowd/models'
       CloudCrowd.configure_database(@options[:db_config])
     end
     
     # Exit with an explanation if the configuration files couldn't be found.
     def config_not_found
-      puts "`crowd` can't find the CloudCrowd configuration directory. Please either run `crowd` from inside of the configuration directory, or add a CLOUD_CROWD_CONFIG variable to your environment."
+      puts "`crowd` can't find the CloudCrowd configuration directory. Please either run `crowd` from inside of the configuration directory, or use `crowd -c path/to/config`"
       exit(1)
     end
     

data/lib/cloud_crowd/helpers/authorization.rb

@@ -0,0 +1,46 @@
+# After sinatra-authorization...
+
+module CloudCrowd
+  module Helpers
+    module Authorization
+      
+      def login_required
+        return if authorized?
+        unauthorized! unless auth.provided?
+        bad_request!  unless auth.basic?
+        unauthorized! unless authorize(*auth.credentials)
+        request.env['REMOTE_USER'] = auth.username
+      end
+      
+      def authorized?
+        !!request.env['REMOTE_USER']
+      end
+      
+      def current_user
+        request.env['REMOTE_USER']
+      end
+      
+      def authorize(login, password)
+        return true unless CloudCrowd.config[:use_http_authentication]
+        return CloudCrowd.config[:login] == login &&
+               CloudCrowd.config[:password] == password
+      end
+      
+      
+      private
+      
+      def auth
+        @auth ||= Rack::Auth::Basic::Request.new(request.env)
+      end
+      
+      def unauthorized!(realm = CloudCrowd::App.authorization_realm)
+        response['WWW-Authenticate'] = "Basic realm=\"#{realm}\""
+        halt 401, 'Authorization Required'
+      end
+      
+      def bad_request!
+        halt 400, 'Bad Request'
+      end
+    end
+  end
+end

data/lib/cloud_crowd/helpers.rb

@@ -1,8 +1,9 @@
+require 'cloud_crowd/helpers/authorization'
 require 'cloud_crowd/helpers/resources'
 require 'cloud_crowd/helpers/urls'
 
 module CloudCrowd
   module Helpers
-    include Resources, Urls #, Rack::Utils
+    include Authorization, Resources, Urls #, Rack::Utils
   end
 end

data/lib/cloud_crowd/models/job.rb

@@ -113,7 +113,7 @@ class Job < ActiveRecord::Base
   # WorkUnits, as well as any completed outputs.
   def to_json(opts={})
     atts = {'id' => self.id, 'status' => self.display_status, 'work_units_remaining' => self.work_units_remaining}
-    atts.merge!({'output' => JSON.parse(self.outputs)}) if self.outputs
+    atts.merge!({'outputs' => JSON.parse(self.outputs)}) if self.outputs
     atts.merge!({'time' => self.time}) if self.time
     atts.to_json
   end

data/lib/cloud_crowd/models.rb

@@ -1,3 +1,5 @@
+require 'activerecord'
+
 module CloudCrowd
   module ModelStatus
     

data/lib/cloud_crowd/runner.rb

@@ -2,24 +2,23 @@
 # daemons don't load the entire rails stack, this file functions like a mini
 # environment.rb, loading all the common gems that we need.
 
-# CloudCrowd::App.root = File.expand_path(File.dirname(__FILE__) + '/../..') unless defined?(CloudCrowd::App.root)
-
-# Standard Lib and Gems
+# Standard Libs
 require 'fileutils'
+require 'benchmark'
+require 'socket'
+
+# Gems
 require 'rubygems'
 require 'daemons'
-require 'socket'
 require 'yaml'
-require 'json'
-require 'rest_client'
-require 'right_aws'
 
 FileUtils.mkdir('log') unless File.exists?('log')
 
 # Daemon/Worker Dependencies.
 require "#{File.dirname(__FILE__)}/../cloud-crowd"
+require 'cloud_crowd/asset_store'
 
-Daemons.run("#{CloudCrowd::App.root}/lib/cloud_crowd/daemon.rb", {
+Daemons.run("#{CloudCrowd::ROOT}/lib/cloud_crowd/daemon.rb", {
   :app_name   => "cloud_crowd_worker",
   :dir_mode   => :normal,
   :dir        => 'log',

data/lib/cloud_crowd/worker.rb

@@ -11,15 +11,17 @@ module CloudCrowd
     # connection to S3. This AssetStore gets passed into each action, for use
     # as it is run.
     def initialize
-      @id = $$
+      @id       = $$
       @hostname = Socket.gethostname
-      @store = CloudCrowd::AssetStore.new
+      @store    = CloudCrowd::AssetStore.new
+      @server   = central_server_resource
+      log 'started'
     end
     
     # Ask the central server for a new WorkUnit.
     def fetch_work_unit
       keep_trying_to "fetch a new work unit" do
-        unit_json = RestClient.get("#{CENTRAL_URL}/work")
+        unit_json = @server['/work'].get
         return unless unit_json # No content means no work for us.
         @start_time = Time.now
         parse_work_unit unit_json
@@ -31,7 +33,7 @@ module CloudCrowd
     def complete_work_unit(result)
       keep_trying_to "complete work unit" do
         data = completion_params.merge({:status => 'succeeded', :output => result})
-        RestClient.put("#{CENTRAL_URL}/work/#{data[:id]}", data)
+        @server["/work/#{data[:id]}"].put(data)
         log "finished #{@action_name} in #{data[:time]} seconds"
       end
     end
@@ -40,7 +42,7 @@ module CloudCrowd
     def fail_work_unit(exception)
       keep_trying_to "mark work unit as failed" do
         data = completion_params.merge({:status => 'failed', :output => exception.message})
-        RestClient.put("#{CENTRAL_URL}/work/#{data[:id]}", data)
+        @server["/work/#{data[:id]}"].put(data)
         log "failed #{@action_name} in #{data[:time]} seconds\n#{exception.message}\n#{exception.backtrace}"
       end
     end
@@ -63,7 +65,7 @@ module CloudCrowd
     end
     
     # Executes the current work unit, catching all exceptions as failures.
-    def run
+    def run_work_unit
       begin
         @action = CloudCrowd.actions(@action_name).new
         @action.configure(@status, @input, @options, @store)
@@ -81,9 +83,24 @@ module CloudCrowd
       end
     end
     
+    # Wraps +run_work_unit+ to benchmark the execution time, if requested.
+    def run
+      return run_work_unit unless @options['benchmark']
+      status = CloudCrowd.display_status(@status)
+      log("ran #{@action_name}/#{status} in " + Benchmark.measure { run_work_unit }.to_s)
+    end
+    
     
     private
     
+    # Keep an authenticated (if configured to enable authentication) resource 
+    # for the central server.
+    def central_server_resource
+      params = [CENTRAL_URL]
+      params += [CloudCrowd.config[:login], CloudCrowd.config[:password]] if CloudCrowd.config[:use_http_authentication]
+      RestClient::Resource.new(*params)
+    end
+    
     # Common parameters to send back to central, regardless of success or failure.
     def completion_params
       {:id => @options['work_unit_id'], :time => Time.now - @start_time}

data/test/config/actions/failure_testing.rb

@@ -0,0 +1,13 @@
+# Simple Action that fails the work unit until it is just about to exhaust
+# all of its retries.
+class FailureTesting < CloudCrowd::Action
+  
+  def run
+    if options['attempts'] + 1 >= CloudCrowd.config[:work_unit_retries]
+      return 'made it!'
+    else
+      raise 'hell'
+    end    
+  end
+  
+end

data/test/test_helper.rb

@@ -2,8 +2,10 @@ require 'rubygems'
 
 here = File.dirname(__FILE__)
 require File.expand_path(here + "/../lib/cloud-crowd")
-CloudCrowd.configure(here + '/config/test_config.yml')
-CloudCrowd.configure_database(here + '/config/test_database.yml')
+require 'cloud_crowd/app'
+
+CloudCrowd.configure(here + '/config/config.yml')
+CloudCrowd.configure_database(here + '/config/database.yml')
 
 require 'faker'
 require 'sham'
@@ -11,7 +13,7 @@ require 'rack/test'
 require 'shoulda/active_record'
 require 'machinist/active_record'
 require 'mocha'
-require "#{CloudCrowd::App.root}/test/blueprints.rb"
+require "#{CloudCrowd::ROOT}/test/blueprints.rb"
 
 class Test::Unit::TestCase
   include CloudCrowd

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: documentcloud-cloud-crowd
 version: !ruby/object:Gem::Version 
-  version: 0.0.1
+  version: 0.0.2
 platform: ruby
 authors: 
 - Jeremy Ashkenas
@@ -143,6 +143,7 @@ files:
 - lib/cloud_crowd/command_line.rb
 - lib/cloud_crowd/core_ext.rb
 - lib/cloud_crowd/daemon.rb
+- lib/cloud_crowd/helpers/authorization.rb
 - lib/cloud_crowd/helpers/resources.rb
 - lib/cloud_crowd/helpers/urls.rb
 - lib/cloud_crowd/helpers.rb
@@ -154,15 +155,15 @@ files:
 - lib/cloud_crowd/worker.rb
 - test/acceptance/test_failing_work_units.rb
 - test/blueprints.rb
-- test/config/test_config.yml
-- test/config/test_database.yml
+- test/config/config.yml
+- test/config/database.yml
+- test/config/actions/failure_testing.rb
 - test/test_helper.rb
 - test/unit/test_job.rb
 - test/unit/test_work_unit.rb
 has_rdoc: true
 homepage: http://documentcloud.org
-licenses: 
-post_install_message: Run `crowd help` for information on using CloudCrowd.
+post_install_message: 
 rdoc_options: []
 
 require_paths: 
@@ -182,7 +183,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
 requirements: []
 
 rubyforge_project: cloud-crowd
-rubygems_version: 1.3.5
+rubygems_version: 1.2.0
 signing_key: 
 specification_version: 2
 summary: Better living through Map --> Ruby --> Reduce