mooktakim-cloud-crowd 0.3.4

data/test/unit/test_node.rb

@@ -0,0 +1,41 @@
+require 'test_helper'
+
+class NodeUnitTest < Test::Unit::TestCase
+  
+  context "A Node" do
+    
+    setup do
+      @node = Node.new(11011).instance_variable_get(:@app)
+    end
+    
+    should "set the identity of the Ruby instance" do
+      assert CloudCrowd.node?
+    end
+    
+    should "instantiate correctly" do
+      assert @node.central.to_s == "http://localhost:9173"
+      assert @node.port == 11011
+      assert @node.host == Socket.gethostname
+      assert @node.enabled_actions.length > 2
+      assert @node.asset_store.is_a? AssetStore::FilesystemStore
+    end
+    
+    should "trap signals and launch a server at start" do
+      Thin::Server.any_instance.expects(:start)
+      @node.expects(:check_in)
+      @node.start
+    end
+    
+    should "be able to determine if the node is overloaded" do
+      assert !@node.overloaded?
+      @node.instance_variable_set :@max_load, 0.01
+      assert @node.overloaded?
+      @node.instance_variable_set :@max_load, nil
+      assert !@node.overloaded?
+      @node.instance_variable_set :@min_memory, 8000
+      assert @node.overloaded?
+    end
+      
+  end
+  
+end

data/test/unit/test_node_record.rb

@@ -0,0 +1,42 @@
+require 'test_helper'
+
+class NodeRecordTest < Test::Unit::TestCase
+
+  context "A NodeRecord" do
+        
+    setup do
+      @node = CloudCrowd::NodeRecord.make
+    end
+    
+    subject { @node }
+    
+    should_have_many :work_units
+    
+    should_validate_presence_of :host, :ip_address, :port, :enabled_actions
+    
+    should "be available" do
+      assert NodeRecord.available.map(&:id).include? @node.id
+    end
+    
+    should "know its enabled actions" do
+      assert @node.actions.include? 'graphics_magick'
+      assert @node.actions.include? 'word_count'
+    end
+    
+    should "know if the node is busy" do
+      assert !@node.busy?
+      assert @node.display_status == 'available'
+      (@node.max_workers + 1).times { WorkUnit.make(:node_record => @node) }
+      assert @node.busy?
+      assert @node.display_status == 'busy'
+      @node.release_work_units
+      assert !@node.busy?
+    end
+    
+    should "be reachable at a URL" do
+      assert !!URI.parse(@node.url)
+    end
+    
+  end
+  
+end

data/test/unit/test_work_unit.rb

@@ -0,0 +1,53 @@
+require 'test_helper'
+
+class WorkUnitTest < Test::Unit::TestCase
+
+  context "A WorkUnit" do
+    
+    setup do
+      @unit = CloudCrowd::WorkUnit.make
+      @job = @unit.job
+    end
+    
+    subject { @unit }
+    
+    should_belong_to :job
+    
+    should_validate_presence_of :job_id, :status, :input, :action
+    
+    should "know if its done" do
+      assert !@unit.complete?
+      @unit.status = SUCCEEDED
+      assert @unit.complete?
+      @unit.status = FAILED
+      assert @unit.complete?
+    end
+    
+    should "have JSON that includes job attributes" do
+      job = Job.make
+      unit_data = JSON.parse(job.work_units.first.to_json)
+      assert unit_data['job_id'] == job.id
+      assert unit_data['action'] == job.action
+      assert JSON.parse(job.inputs).include? unit_data['input']
+    end
+    
+    should "be able to retry, on failure" do
+      @unit.update_attribute :worker_pid, 7337
+      assert @unit.attempts == 0
+      @unit.fail('oops', 10)
+      assert @unit.worker_pid == nil
+      assert @unit.attempts == 1
+      assert @unit.processing?
+      @unit.fail('oops again', 10)
+      assert @unit.attempts == 2
+      assert @unit.processing?
+      assert @unit.job.processing?
+      @unit.fail('oops one last time', 10)
+      assert @unit.attempts == 3
+      assert @unit.failed?
+      assert @unit.job.any_work_units_failed?
+    end
+    
+  end
+  
+end

data/test/unit/test_worker.rb

@@ -0,0 +1,48 @@
+require 'test_helper'
+
+class WorkerTest < Test::Unit::TestCase
+  
+  context "A CloudCrowd::Worker" do
+        
+    setup do
+      @node = Node.new.instance_variable_get(:@app)
+      @unit = WorkUnit.make
+      @worker = Worker.new(@node, JSON.parse(@unit.to_json))
+    end
+    
+    should "instantiate correctly" do
+      assert @worker.pid == $$
+      assert @worker.unit['id'] == @unit.id
+      assert @worker.status == @unit.status
+      assert @worker.node == @node
+      assert @worker.time_taken > 0
+    end
+    
+    should "be able to retry operations that must succeed" do
+      @worker.instance_variable_set :@retry_wait, 0.01
+      @worker.expects(:log).at_least(3)
+      tries = 0
+      @worker.keep_trying_to("do something critical") do
+        tries += 1;
+        raise 'hell' unless tries > 3
+        assert "made it through"
+      end    
+    end
+    
+    should "be able to run an action and try to complete it" do
+      GraphicsMagick.any_instance.expects(:process).returns('the answer')
+      GraphicsMagick.any_instance.expects(:cleanup_work_directory)
+      @worker.expects(:complete_work_unit).with({'output' => 'the answer'}.to_json)
+      @worker.run_work_unit
+    end
+    
+    should "enchance the options that an action receives with extra info" do
+      opts = @worker.enhanced_unit_options
+      assert opts['work_unit_id'] == @unit.id
+      assert opts['job_id'] == @unit.job.id
+      assert opts['attempts'] == @unit.attempts
+    end
+      
+  end
+  
+end

data/views/operations_center.erb

@@ -0,0 +1,82 @@
+<!DOCTYPE html>
+<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
+<head>
+  <meta http-equiv="content-type" content="text/html; charset=UTF-8" />
+  <title>Operations Center | CloudCrowd</title>
+  <link href="css/reset.css" media="screen" rel="stylesheet" type="text/css" />
+  <link href="css/admin_console.css" media="screen" rel="stylesheet" type="text/css" />
+  <script src="js/jquery.js" type="text/javascript"></script>
+  <!--[if IE]><script src="js/excanvas.js" type="text/javascript"></script><![endif]-->
+  <script src="js/flot.js" type="text/javascript"></script>
+  <script src="js/admin_console.js" type="text/javascript"></script>
+</head>
+
+<body>
+  <div id="container">
+    
+    <div id="header">
+      <div id="logo"></div>
+    </div>
+    
+    <div id="disconnected" class="small_caps" style="display:none;">
+      <div class="server_error"></div>
+      Disconnected. Attempting to reconnect&hellip;
+    </div>
+    
+    <div id="queue">
+      <div id="no_jobs" class="small_caps">no jobs running&hellip;</div>
+      <div id="jobs">
+        <%# Render target for jobs. %>
+      </div>
+      <div id="queue_fill"></div>
+    </div>
+    
+    <div id="sidebar">
+      <div id="sidebar_top" class="sidebar_back"></div>
+      <div id="sidebar_background"></div>
+      <div id="sidebar_bottom" class="sidebar_back"></div>
+      <div id="sidebar_header" class="small_caps">
+        <span class="has_nodes"></span>
+        <span class="no_nodes">No Nodes Online&hellip;</span>
+      </div>
+      <div id="nodes">
+        <%# Render target for nodes and workers. %>
+      </div>
+    </div>
+    
+    <div id="graphs">
+      <div class="graph_container">
+        <div class="graph_title">Work Units in Queue</div>
+        <div id="work_units_graph" class="graph"></div>
+      </div>
+      <div class="graph_container">
+        <div class="graph_title">Jobs in Queue</div>
+        <div id="jobs_graph" class="graph"></div>
+      </div>
+      <div class="graph_container">
+        <div class="graph_title">
+          Active Nodes
+          (<span id="nodes_legend" class="legend_box"></span>)
+          and Workers
+          (<span id="workers_legend" class="legend_box"></span>)
+        </div>
+        <div id="workers_graph" class="graph"></div>
+      </div>
+    </div>
+    
+    <div id="worker_info" style="display:none;">
+      <div id="worker_info_inner" class="small_caps">
+        <div id="worker_details">
+          <div id="worker_status">status: <span class="status"></span></div>
+          <div id="worker_action">action:&nbsp;<span class="action"></span></div>
+          <div id="worker_job_id">job #<span class="job_id"></span> / unit #<span class="work_unit_id"></span></div>
+        </div>
+        <div id="worker_sleeping">
+          worker exiting&hellip;
+        </div>
+      </div>
+    </div>
+    
+  </div>
+</body>
+</html>

metadata

@@ -0,0 +1,290 @@
+--- !ruby/object:Gem::Specification 
+name: mooktakim-cloud-crowd
+version: !ruby/object:Gem::Version 
+  prerelease: false
+  segments: 
+  - 0
+  - 3
+  - 4
+  version: 0.3.4
+platform: ruby
+authors: 
+- Jeremy Ashkenas
+- Mooktakim Ahmed
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2010-02-26 00:00:00 +00:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: sinatra
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 0
+        - 9
+        - 4
+        version: 0.9.4
+  type: :runtime
+  version_requirements: *id001
+- !ruby/object:Gem::Dependency 
+  name: activerecord
+  prerelease: false
+  requirement: &id002 !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 2
+        - 3
+        - 3
+        version: 2.3.3
+  type: :runtime
+  version_requirements: *id002
+- !ruby/object:Gem::Dependency 
+  name: json
+  prerelease: false
+  requirement: &id003 !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 1
+        - 1
+        - 7
+        version: 1.1.7
+  type: :runtime
+  version_requirements: *id003
+- !ruby/object:Gem::Dependency 
+  name: rest-client
+  prerelease: false
+  requirement: &id004 !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 1
+        - 0
+        - 3
+        version: 1.0.3
+  type: :runtime
+  version_requirements: *id004
+- !ruby/object:Gem::Dependency 
+  name: right_aws
+  prerelease: false
+  requirement: &id005 !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 1
+        - 10
+        - 0
+        version: 1.10.0
+  type: :runtime
+  version_requirements: *id005
+- !ruby/object:Gem::Dependency 
+  name: thin
+  prerelease: false
+  requirement: &id006 !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 1
+        - 2
+        - 4
+        version: 1.2.4
+  type: :runtime
+  version_requirements: *id006
+- !ruby/object:Gem::Dependency 
+  name: faker
+  prerelease: false
+  requirement: &id007 !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 0
+        - 3
+        - 1
+        version: 0.3.1
+  type: :development
+  version_requirements: *id007
+- !ruby/object:Gem::Dependency 
+  name: thoughtbot-shoulda
+  prerelease: false
+  requirement: &id008 !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 2
+        - 10
+        - 2
+        version: 2.10.2
+  type: :development
+  version_requirements: *id008
+- !ruby/object:Gem::Dependency 
+  name: notahat-machinist
+  prerelease: false
+  requirement: &id009 !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 1
+        - 0
+        - 3
+        version: 1.0.3
+  type: :development
+  version_requirements: *id009
+- !ruby/object:Gem::Dependency 
+  name: rack-test
+  prerelease: false
+  requirement: &id010 !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 0
+        - 4
+        - 1
+        version: 0.4.1
+  type: :development
+  version_requirements: *id010
+- !ruby/object:Gem::Dependency 
+  name: mocha
+  prerelease: false
+  requirement: &id011 !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 0
+        - 9
+        - 7
+        version: 0.9.7
+  type: :development
+  version_requirements: *id011
+description: "    The crowd, suddenly there where there was nothing before, is a mysterious and\n    universal phenomenon. A few people may have been standing together -- five, ten\n    or twelve, nor more; nothing has been announced, nothing is expected. Suddenly\n    everywhere is black with people and more come streaming from all sides as though\n    streets had only one direction.\n"
+email: jeremy@documentcloud.org
+executables: 
+- crowd
+extensions: []
+
+extra_rdoc_files: 
+- README
+files: 
+- actions/graphics_magick.rb
+- actions/process_pdfs.rb
+- actions/word_count.rb
+- mooktakim-cloud-crowd.gemspec
+- config/config.example.ru
+- config/config.example.yml
+- config/database.example.yml
+- EPIGRAPHS
+- examples/graphics_magick_example.rb
+- examples/process_pdfs_example.rb
+- examples/word_count_example.rb
+- lib/cloud-crowd.rb
+- lib/cloud_crowd/action.rb
+- lib/cloud_crowd/asset_store/filesystem_store.rb
+- lib/cloud_crowd/asset_store/s3_store.rb
+- lib/cloud_crowd/asset_store.rb
+- lib/cloud_crowd/command_line.rb
+- lib/cloud_crowd/exceptions.rb
+- lib/cloud_crowd/helpers/authorization.rb
+- lib/cloud_crowd/helpers/resources.rb
+- lib/cloud_crowd/helpers.rb
+- lib/cloud_crowd/inflector.rb
+- lib/cloud_crowd/models/job.rb
+- lib/cloud_crowd/models/node_record.rb
+- lib/cloud_crowd/models/work_unit.rb
+- lib/cloud_crowd/models.rb
+- lib/cloud_crowd/node.rb
+- lib/cloud_crowd/schema.rb
+- lib/cloud_crowd/server.rb
+- lib/cloud_crowd/worker.rb
+- LICENSE
+- public/css/admin_console.css
+- public/css/reset.css
+- public/images/bullet_green.png
+- public/images/bullet_white.png
+- public/images/cloud_hand.png
+- public/images/header_back.png
+- public/images/logo.png
+- public/images/queue_fill.png
+- public/images/server.png
+- public/images/server_busy.png
+- public/images/server_error.png
+- public/images/sidebar_bottom.png
+- public/images/sidebar_top.png
+- public/images/worker_info.png
+- public/images/worker_info_loading.gif
+- public/js/admin_console.js
+- public/js/excanvas.js
+- public/js/flot.js
+- public/js/jquery.js
+- README
+- test/acceptance/test_node.rb
+- test/acceptance/test_failing_work_units.rb
+- test/acceptance/test_server.rb
+- test/acceptance/test_word_count.rb
+- test/blueprints.rb
+- test/config/config.ru
+- test/config/config.yml
+- test/config/database.yml
+- test/config/actions/failure_testing.rb
+- test/test_helper.rb
+- test/unit/test_action.rb
+- test/unit/test_configuration.rb
+- test/unit/test_node.rb
+- test/unit/test_node_record.rb
+- test/unit/test_job.rb
+- test/unit/test_worker.rb
+- test/unit/test_work_unit.rb
+- views/operations_center.erb
+has_rdoc: true
+homepage: http://wiki.github.com/documentcloud/cloud-crowd
+licenses: []
+
+post_install_message: 
+rdoc_options: 
+- --title
+- CloudCrowd | Parallel Processing for the Rest of Us
+- --exclude
+- test
+- --main
+- README
+- --all
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: cloud-crowd
+rubygems_version: 1.3.6
+signing_key: 
+specification_version: 3
+summary: Parallel Processing for the Rest of Us
+test_files: []
+