cloud-crowd 0.1.0

data/test/config/config.ru

@@ -0,0 +1,17 @@
+#!/usr/bin/env ruby
+
+# This rackup script can be used to start the central CloudCrowd server
+# using any Rack-compliant server handler. For example, start up three servers 
+# with a specified port number, using Thin:
+#
+# thin start -R config.ru -p 9173 --servers 3
+
+require 'rubygems'
+require 'cloud-crowd'
+
+CloudCrowd.configure(File.dirname(__FILE__) + '/config.yml')
+CloudCrowd.configure_database(File.dirname(__FILE__) + '/database.yml')
+
+map '/' do
+  run CloudCrowd::App
+end

data/test/config/config.yml

@@ -0,0 +1,7 @@
+:num_workers:             4
+:min_worker_wait:         1
+:max_worker_wait:         20
+:work_unit_retries:       3
+
+:central_server:          http://localhost:9173
+:storage:                 filesystem

data/test/config/database.yml

@@ -0,0 +1,6 @@
+:adapter:  mysql
+:encoding: utf8
+:username: root
+:password:
+:socket:   /tmp/mysql.sock
+:database: cloud_crowd_test

data/test/test_helper.rb

@@ -0,0 +1,19 @@
+ENV['RACK_ENV'] = 'test'
+require 'rubygems'
+
+here = File.dirname(__FILE__)
+require File.expand_path(here + "/../lib/cloud-crowd")
+CloudCrowd.configure(here + '/config/config.yml')
+CloudCrowd.configure_database(here + '/config/database.yml')
+
+require 'faker'
+require 'sham'
+require 'rack/test'
+require 'shoulda/active_record'
+require 'machinist/active_record'
+require 'mocha'
+require "#{CloudCrowd::ROOT}/test/blueprints.rb"
+
+class Test::Unit::TestCase
+  include CloudCrowd
+end

data/test/unit/test_action.rb

@@ -0,0 +1,49 @@
+require 'test_helper'
+
+class CloudCrowd::Action
+  public :safe_filename
+end
+
+class EmptyAction < CloudCrowd::Action
+end
+
+class ActionTest < Test::Unit::TestCase
+  
+  context "A CloudCrowd Job" do
+        
+    setup do
+      @store = CloudCrowd::AssetStore.new
+      @args = [CloudCrowd::PROCESSING, 'file://' + File.expand_path(__FILE__), {'job_id' => 1, 'work_unit_id' => 1}, @store]
+      @action = CloudCrowd.actions['word_count'].new(*@args)
+    end
+    
+    should "throw an exception if the 'process' method isn't implemented" do
+      assert_raise(NotImplementedError) { EmptyAction.new(*@args).process }
+    end
+    
+    should "have downloaded the input URL to local storage" do
+      assert @action.input_path
+      assert File.read(@action.input_path) == File.read(File.expand_path(__FILE__))
+    end
+    
+    should "be able to save (to the filesystem while testing)" do
+      assert @action.save(@action.input_path) == "file://#{CloudCrowd::AssetStore::LOCAL_STORAGE_PATH}/word_count/job_1/unit_1/test_action.rb"
+    end
+    
+    should "be able to clean up after itself" do
+      @action.cleanup_work_directory
+      assert !File.exists?(@action.work_directory)
+    end
+    
+    should "be able to generate a safe filename for a URL to write to disk" do
+      name = @action.safe_filename("http://example.com/Some%20(Crazy'Kinda%7E)'Filename.txt")
+      assert name == 'Some-Crazy-Kinda-Filename.txt'
+    end
+    
+    should "be able to count the number of words in this file" do
+      assert @action.process == 149
+    end
+            
+  end
+  
+end

data/test/unit/test_configuration.rb

@@ -0,0 +1,28 @@
+require 'test_helper'
+
+class ConfigurationTest < Test::Unit::TestCase
+
+  context "CloudCrowd Configuration" do
+            
+    should "have read in config.yml" do
+      assert CloudCrowd.config[:num_workers] == 4
+      assert CloudCrowd.config[:storage] == 'filesystem'
+    end
+    
+    should "allow config.yml to configure the implementation of AssetStore" do
+      assert CloudCrowd::AssetStore.ancestors.include?(CloudCrowd::AssetStore::FilesystemStore)
+    end
+    
+    should "have properly configured the ActiveRecord database" do
+      assert ActiveRecord::Base.connection.active?
+    end
+    
+    should "have loaded in the default set of actions" do
+      assert CloudCrowd.actions['word_count']       == WordCount
+      assert CloudCrowd.actions['process_pdfs']     == ProcessPdfs
+      assert CloudCrowd.actions['graphics_magick']  == GraphicsMagick
+    end
+            
+  end
+  
+end

data/test/unit/test_job.rb

@@ -0,0 +1,78 @@
+require 'test_helper'
+
+class JobTest < Test::Unit::TestCase
+
+  context "A CloudCrowd Job" do
+        
+    setup do
+      @job = CloudCrowd::Job.make
+      @unit = @job.work_units.first
+    end
+    
+    subject { @job }
+    
+    should_have_many :work_units
+    
+    should_validate_presence_of :status, :inputs, :action, :options
+    
+    should "create all of its work units as soon as the job is created" do
+      assert @job.work_units.count >= 1
+      assert @job.percent_complete == 0
+      assert @job.processing?
+      assert @unit.processing?
+      assert !@job.all_work_units_complete?
+    end
+    
+    should "know its completion status" do
+      assert !@job.all_work_units_complete?
+      @unit.update_attributes(:status => CloudCrowd::SUCCEEDED, :output => '{"output":"hello"}')
+      assert @job.reload.all_work_units_complete?
+      assert @job.percent_complete == 100
+      assert @job.outputs == "[\"hello\"]"
+    end
+    
+    should "be able to create a job from a JSON request" do
+      job = CloudCrowd::Job.create_from_request(JSON.parse(<<-EOS
+      { "inputs"       : ["one", "two", "three"],
+        "action"       : "graphics_magick",
+        "email"        : "bob@example.com",
+        "callback_url" : "http://example.com/callback" }
+      EOS
+      ))
+      assert job.work_units.count == 3
+      assert job.action == 'graphics_magick'
+      assert job.action_class == GraphicsMagick
+      assert job.callback_url == "http://example.com/callback"
+    end
+    
+    should "be able to return a comprehensive JSON representation" do
+      json = JSON.parse(@job.to_json)
+      assert json['email'] == 'noone@example.com'
+      assert json['percent_complete'] == 0
+      assert json['work_units'] == 1
+      assert json['time_taken'] > 0
+    end
+    
+    should "create jobs with a SPLITTING status for actions that have a split method defined" do
+      job = CloudCrowd::Job.create_from_request({'inputs' => ['1'], 'action' => 'process_pdfs'})
+      assert job.splittable?
+      assert job.splitting?
+    end
+    
+    should "fire a callback when a job has finished, successfully or not" do
+      CloudCrowd::Job.any_instance.expects(:fire_callback)
+      @job.work_units.first.finish('{"output":"output"}', 10)
+      assert @job.all_work_units_complete?
+    end
+    
+    should "have a 'pretty' display of the Job's status" do
+      assert @job.display_status == 'processing'
+      @job.update_attribute(:status, CloudCrowd::FAILED)
+      assert @job.display_status == 'failed'
+      @job.update_attribute(:status, CloudCrowd::MERGING)
+      assert @job.display_status == 'merging'
+    end
+            
+  end
+  
+end

data/test/unit/test_work_unit.rb

@@ -0,0 +1,55 @@
+require 'test_helper'
+
+class WorkUnitTest < Test::Unit::TestCase
+
+  context "A WorkUnit" do
+    
+    setup do
+      @unit = CloudCrowd::WorkUnit.make
+      @job = @unit.job
+    end
+    
+    subject { @unit }
+    
+    should_belong_to :job
+    
+    should_validate_presence_of :job_id, :status, :input, :action
+    
+    should "know if its done" do
+      assert !@unit.complete?
+      @unit.status = CloudCrowd::SUCCEEDED
+      assert @unit.complete?
+      @unit.status = CloudCrowd::FAILED
+      assert @unit.complete?
+      @unit.expects :check_for_job_completion
+      @unit.save
+    end
+    
+    should "have JSON that includes job attributes" do
+      job = CloudCrowd::Job.make
+      unit_data = JSON.parse(job.work_units.first.to_json)
+      assert unit_data['job_id'] == job.id
+      assert unit_data['action'] == job.action
+      assert JSON.parse(job.inputs).include? unit_data['input']
+    end
+    
+    should "be able to retry, on failure" do
+      @unit.update_attribute :worker_record_id, 10
+      assert @unit.attempts == 0
+      @unit.fail('oops', 10)
+      assert @unit.worker_record == nil
+      assert @unit.attempts == 1
+      assert @unit.processing?
+      @unit.fail('oops again', 10)
+      assert @unit.attempts == 2
+      assert @unit.processing?
+      assert @unit.job.processing?
+      @unit.fail('oops one last time', 10)
+      assert @unit.attempts == 3
+      assert @unit.failed?
+      assert @unit.job.any_work_units_failed?
+    end
+    
+  end
+  
+end

data/views/index.erb

@@ -0,0 +1,77 @@
+<!DOCTYPE html>
+<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
+<head>
+  <meta http-equiv="content-type" content="text/html; charset=UTF-8" />
+  <title>Operations Center | CloudCrowd</title>
+  <link href="/css/reset.css" media="screen" rel="stylesheet" type="text/css" />
+  <link href="/css/admin_console.css" media="screen" rel="stylesheet" type="text/css" />
+  <script src="/js/jquery.js" type="text/javascript"></script>
+  <!--[if IE]><script src="/js/excanvas.js" type="text/javascript"></script><![endif]-->
+  <script src="/js/flot.js" type="text/javascript"></script>
+  <script src="/js/admin_console.js" type="text/javascript"></script>
+</head>
+
+<body>
+  <div id="container">
+    
+    <div id="header">
+      <div id="logo"></div>
+    </div>
+    
+    <div id="disconnected" class="small_caps" style="display:none;">
+      <div class="server_error"></div>
+      Disconnected. Attempting to reconnect&hellip;
+    </div>
+    
+    <div id="queue">
+      <div id="no_jobs" class="small_caps">no jobs running&hellip;</div>
+      <div id="jobs">
+        <%# Render target for jobs. %>
+      </div>
+      <div id="queue_fill"></div>
+    </div>
+    
+    <div id="sidebar">
+      <div id="sidebar_top" class="sidebar_back"></div>
+      <div id="sidebar_background"></div>
+      <div id="sidebar_bottom" class="sidebar_back"></div>
+      <div id="sidebar_header" class="small_caps">
+        <span class="has_workers">Active Worker Daemons</span>
+        <span class="no_workers">No Workers Running&hellip;</span>
+      </div>
+      <div id="workers">
+        <%# Render target for workers. %>
+      </div>
+    </div>
+    
+    <div id="graphs">
+      <div class="graph_container">
+        <div class="graph_title">Work Units in Queue:</div>
+        <div id="work_units_graph" class="graph"></div>
+      </div>
+      <div class="graph_container">
+        <div class="graph_title">Jobs in Queue:</div>
+        <div id="jobs_graph" class="graph"></div>
+      </div>
+      <div class="graph_container">
+        <div class="graph_title">Active Worker Daemons:</div>
+        <div id="workers_graph" class="graph"></div>
+      </div>
+    </div>
+    
+    <div id="worker_info" style="display:none;">
+      <div id="worker_info_inner" class="small_caps">
+        <div id="worker_details">
+          <div id="worker_status">status: <span class="status"></span></div>
+          <div id="worker_action">action: <span class="action"></span></div>
+          <div id="worker_job_id">job #<span class="job_id"></span> / work unit #<span class="work_unit_id"></span></div>
+        </div>
+        <div id="worker_sleeping">
+          worker is sleeping&hellip;
+        </div>
+      </div>
+    </div>
+    
+  </div>
+</body>
+</html>

metadata

@@ -0,0 +1,233 @@
+--- !ruby/object:Gem::Specification 
+name: cloud-crowd
+version: !ruby/object:Gem::Version 
+  version: 0.1.0
+platform: ruby
+authors: 
+- Jeremy Ashkenas
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2009-09-14 00:00:00 -04:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: sinatra
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 0.9.4
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: activerecord
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 2.3.3
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: json
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 1.1.7
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: rest-client
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 1.0.3
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: right_aws
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 1.10.0
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: daemons
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 1.0.10
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: faker
+  type: :development
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 0.3.1
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: thoughtbot-shoulda
+  type: :development
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 2.10.2
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: notahat-machinist
+  type: :development
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 1.0.3
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: rack-test
+  type: :development
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 0.4.1
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: mocha
+  type: :development
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 0.9.7
+    version: 
+description: "    The crowd, suddenly there where there was nothing before, is a mysterious and\n    universal phenomenon. A few people may have been standing together -- five, ten\n    or twelve, nor more; nothing has been announced, nothing is expected. Suddenly\n    everywhere is black with people and more come streaming from all sides as though\n    streets had only one direction.\n"
+email: jeremy@documentcloud.org
+executables: 
+- crowd
+extensions: []
+
+extra_rdoc_files: 
+- README
+files: 
+- actions/graphics_magick.rb
+- actions/process_pdfs.rb
+- actions/word_count.rb
+- cloud-crowd.gemspec
+- config/config.example.ru
+- config/config.example.yml
+- config/database.example.yml
+- EPIGRAPHS
+- examples/graphics_magick_example.rb
+- examples/process_pdfs_example.rb
+- examples/word_count_example.rb
+- lib/cloud-crowd.rb
+- lib/cloud_crowd/action.rb
+- lib/cloud_crowd/app.rb
+- lib/cloud_crowd/asset_store/filesystem_store.rb
+- lib/cloud_crowd/asset_store/s3_store.rb
+- lib/cloud_crowd/asset_store.rb
+- lib/cloud_crowd/command_line.rb
+- lib/cloud_crowd/daemon.rb
+- lib/cloud_crowd/exceptions.rb
+- lib/cloud_crowd/helpers/authorization.rb
+- lib/cloud_crowd/helpers/resources.rb
+- lib/cloud_crowd/helpers.rb
+- lib/cloud_crowd/inflector.rb
+- lib/cloud_crowd/models/job.rb
+- lib/cloud_crowd/models/work_unit.rb
+- lib/cloud_crowd/models/worker_record.rb
+- lib/cloud_crowd/models.rb
+- lib/cloud_crowd/runner.rb
+- lib/cloud_crowd/schema.rb
+- lib/cloud_crowd/worker.rb
+- LICENSE
+- public/css/admin_console.css
+- public/css/reset.css
+- public/images/bullet_green.png
+- public/images/bullet_white.png
+- public/images/cloud_hand.png
+- public/images/header_back.png
+- public/images/logo.png
+- public/images/queue_fill.png
+- public/images/server_error.png
+- public/images/sidebar_bottom.png
+- public/images/sidebar_top.png
+- public/images/worker_info.png
+- public/images/worker_info_loading.gif
+- public/js/admin_console.js
+- public/js/excanvas.js
+- public/js/flot.js
+- public/js/jquery.js
+- README
+- test/acceptance/test_app.rb
+- test/acceptance/test_failing_work_units.rb
+- test/acceptance/test_word_count.rb
+- test/blueprints.rb
+- test/config/config.ru
+- test/config/config.yml
+- test/config/database.yml
+- test/config/actions/failure_testing.rb
+- test/test_helper.rb
+- test/unit/test_action.rb
+- test/unit/test_configuration.rb
+- test/unit/test_job.rb
+- test/unit/test_work_unit.rb
+- views/index.erb
+has_rdoc: true
+homepage: http://wiki.github.com/documentcloud/cloud-crowd
+licenses: []
+
+post_install_message: 
+rdoc_options: 
+- --title
+- CloudCrowd | Parallel Processing for the Rest of Us
+- --exclude
+- test
+- --main
+- README
+- --all
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+requirements: []
+
+rubyforge_project: cloud-crowd
+rubygems_version: 1.3.5
+signing_key: 
+specification_version: 3
+summary: Parallel Processing for the Rest of Us
+test_files: []
+