skynet 0.9.2 → 0.9.3

Sign up to get free protection for your applications and to get access to all the features.
Files changed (128) hide show
  1. data/History.txt +49 -0
  2. data/Manifest.txt +84 -6
  3. data/README.txt +75 -64
  4. data/app_generators/skynet_install/skynet_install_generator.rb +14 -8
  5. data/app_generators/skynet_install/templates/migration.rb +1 -24
  6. data/app_generators/skynet_install/templates/skynet_config.rb +50 -0
  7. data/app_generators/skynet_install/templates/skynet_initializer.rb +1 -0
  8. data/app_generators/skynet_install/templates/{skynet_schema.sql → skynet_mysql_schema.sql} +1 -24
  9. data/bin/skynet +37 -10
  10. data/bin/skynet_install +5 -5
  11. data/bin/skynet_tuplespace_server +27 -19
  12. data/examples/dgrep/README +70 -0
  13. data/examples/dgrep/config/skynet_config.rb +26 -0
  14. data/examples/dgrep/data/shakespeare/README +2 -0
  15. data/examples/dgrep/data/shakespeare/poetry/loverscomplaint +381 -0
  16. data/examples/dgrep/data/shakespeare/poetry/rapeoflucrece +2199 -0
  17. data/examples/dgrep/data/shakespeare/poetry/sonnets +2633 -0
  18. data/examples/dgrep/data/shakespeare/poetry/various +640 -0
  19. data/examples/dgrep/data/shakespeare/poetry/venusandadonis +1423 -0
  20. data/examples/dgrep/data/testfile1.txt +1 -0
  21. data/examples/dgrep/data/testfile2.txt +1 -0
  22. data/examples/dgrep/data/testfile3.txt +1 -0
  23. data/examples/dgrep/data/testfile4.txt +1 -0
  24. data/examples/dgrep/lib/dgrep.rb +59 -0
  25. data/examples/dgrep/lib/mapreduce_test.rb +32 -0
  26. data/examples/dgrep/lib/most_common_words.rb +45 -0
  27. data/examples/dgrep/script/dgrep +75 -0
  28. data/examples/rails_mysql_example/README +66 -0
  29. data/examples/rails_mysql_example/Rakefile +10 -0
  30. data/examples/rails_mysql_example/app/controllers/application.rb +10 -0
  31. data/examples/rails_mysql_example/app/helpers/application_helper.rb +3 -0
  32. data/examples/rails_mysql_example/app/models/user.rb +21 -0
  33. data/examples/rails_mysql_example/app/models/user_favorite.rb +5 -0
  34. data/examples/rails_mysql_example/app/models/user_mailer.rb +12 -0
  35. data/examples/rails_mysql_example/app/views/user_mailer/welcome.erb +5 -0
  36. data/examples/rails_mysql_example/config/boot.rb +109 -0
  37. data/examples/rails_mysql_example/config/database.yml +42 -0
  38. data/examples/rails_mysql_example/config/environment.rb +59 -0
  39. data/examples/rails_mysql_example/config/environments/development.rb +18 -0
  40. data/examples/rails_mysql_example/config/environments/production.rb +19 -0
  41. data/examples/rails_mysql_example/config/environments/test.rb +22 -0
  42. data/examples/rails_mysql_example/config/initializers/inflections.rb +10 -0
  43. data/examples/rails_mysql_example/config/initializers/mime_types.rb +5 -0
  44. data/examples/rails_mysql_example/config/initializers/skynet.rb +1 -0
  45. data/examples/rails_mysql_example/config/routes.rb +35 -0
  46. data/examples/rails_mysql_example/config/skynet_config.rb +36 -0
  47. data/examples/rails_mysql_example/db/migrate/001_create_skynet_tables.rb +43 -0
  48. data/examples/rails_mysql_example/db/migrate/002_create_users.rb +16 -0
  49. data/examples/rails_mysql_example/db/migrate/003_create_user_favorites.rb +14 -0
  50. data/examples/rails_mysql_example/db/schema.rb +85 -0
  51. data/examples/rails_mysql_example/db/skynet_mysql_schema.sql +33 -0
  52. data/examples/rails_mysql_example/doc/README_FOR_APP +2 -0
  53. data/examples/rails_mysql_example/lib/tasks/rails_mysql_example.rake +20 -0
  54. data/examples/rails_mysql_example/public/.htaccess +40 -0
  55. data/examples/rails_mysql_example/public/404.html +30 -0
  56. data/examples/rails_mysql_example/public/422.html +30 -0
  57. data/examples/rails_mysql_example/public/500.html +30 -0
  58. data/examples/rails_mysql_example/public/dispatch.cgi +10 -0
  59. data/examples/rails_mysql_example/public/dispatch.fcgi +24 -0
  60. data/examples/rails_mysql_example/public/dispatch.rb +10 -0
  61. data/{log/debug.log → examples/rails_mysql_example/public/favicon.ico} +0 -0
  62. data/examples/rails_mysql_example/public/images/rails.png +0 -0
  63. data/examples/rails_mysql_example/public/index.html +277 -0
  64. data/examples/rails_mysql_example/public/javascripts/application.js +2 -0
  65. data/examples/rails_mysql_example/public/javascripts/controls.js +963 -0
  66. data/examples/rails_mysql_example/public/javascripts/dragdrop.js +972 -0
  67. data/examples/rails_mysql_example/public/javascripts/effects.js +1120 -0
  68. data/examples/rails_mysql_example/public/javascripts/prototype.js +4225 -0
  69. data/examples/rails_mysql_example/public/robots.txt +5 -0
  70. data/examples/rails_mysql_example/script/about +3 -0
  71. data/examples/rails_mysql_example/script/console +3 -0
  72. data/examples/rails_mysql_example/script/destroy +3 -0
  73. data/examples/rails_mysql_example/script/generate +3 -0
  74. data/examples/rails_mysql_example/script/performance/benchmarker +3 -0
  75. data/examples/rails_mysql_example/script/performance/profiler +3 -0
  76. data/examples/rails_mysql_example/script/performance/request +3 -0
  77. data/examples/rails_mysql_example/script/plugin +3 -0
  78. data/examples/rails_mysql_example/script/process/inspector +3 -0
  79. data/examples/rails_mysql_example/script/process/reaper +3 -0
  80. data/examples/rails_mysql_example/script/process/spawner +3 -0
  81. data/examples/rails_mysql_example/script/runner +3 -0
  82. data/examples/rails_mysql_example/script/server +3 -0
  83. data/examples/rails_mysql_example/test/fixtures/user_favorites.yml +9 -0
  84. data/examples/rails_mysql_example/test/fixtures/users.yml +11 -0
  85. data/examples/rails_mysql_example/test/test_helper.rb +38 -0
  86. data/examples/rails_mysql_example/test/unit/user_favorite_test.rb +8 -0
  87. data/examples/rails_mysql_example/test/unit/user_test.rb +8 -0
  88. data/extras/README +7 -0
  89. data/extras/init.d/skynet +87 -0
  90. data/extras/nagios/check_skynet.sh +121 -0
  91. data/extras/rails/controllers/skynet_controller.rb +43 -0
  92. data/extras/rails/views/skynet/index.rhtml +137 -0
  93. data/lib/skynet.rb +59 -1
  94. data/lib/skynet/mapreduce_helper.rb +2 -2
  95. data/lib/skynet/mapreduce_test.rb +32 -1
  96. data/lib/skynet/message_queue_adapters/mysql.rb +422 -539
  97. data/lib/skynet/message_queue_adapters/tuple_space.rb +45 -71
  98. data/lib/skynet/skynet_active_record_extensions.rb +22 -11
  99. data/lib/skynet/skynet_config.rb +54 -20
  100. data/lib/skynet/skynet_console.rb +4 -1
  101. data/lib/skynet/skynet_console_helper.rb +5 -1
  102. data/lib/skynet/skynet_debugger.rb +58 -4
  103. data/lib/skynet/skynet_job.rb +61 -24
  104. data/lib/skynet/skynet_launcher.rb +29 -3
  105. data/lib/skynet/skynet_logger.rb +11 -1
  106. data/lib/skynet/skynet_manager.rb +403 -240
  107. data/lib/skynet/skynet_message.rb +1 -3
  108. data/lib/skynet/skynet_message_queue.rb +42 -19
  109. data/lib/skynet/skynet_partitioners.rb +19 -15
  110. data/lib/skynet/skynet_ruby_extensions.rb +18 -0
  111. data/lib/skynet/skynet_tuplespace_server.rb +17 -14
  112. data/lib/skynet/skynet_worker.rb +132 -98
  113. data/lib/skynet/version.rb +1 -1
  114. data/script/destroy +0 -0
  115. data/script/generate +0 -0
  116. data/script/txt2html +0 -0
  117. data/test/test_helper.rb +2 -0
  118. data/test/test_skynet.rb +13 -5
  119. data/test/test_skynet_manager.rb +24 -9
  120. data/test/test_skynet_task.rb +1 -1
  121. data/website/index.html +77 -29
  122. data/website/index.txt +53 -24
  123. data/website/stylesheets/screen.css +12 -12
  124. metadata +156 -66
  125. data/app_generators/skynet_install/templates/skynet +0 -46
  126. data/log/skynet.log +0 -29
  127. data/log/skynet_tuplespace_server.log +0 -7
  128. data/log/skynet_worker.pid +0 -1
data/lib/skynet/version.rb CHANGED
@@ -2,7 +2,7 @@ class Skynet #:nodoc:
2
2
  module VERSION #:nodoc:
3
3
  MAJOR = 0
4
4
  MINOR = 9
5
- TINY = 2
5
+ TINY = 3
6
6
 
7
7
  STRING = [MAJOR, MINOR, TINY].join('.')
8
8
  end
data/script/destroy CHANGED
File without changes
data/script/generate CHANGED
File without changes
data/script/txt2html CHANGED
File without changes
data/test/test_helper.rb CHANGED
@@ -1,4 +1,6 @@
1
1
  ENV["RAILS_ENV"] = "test"
2
+ $VERBOSE=false
3
+
2
4
 
3
5
  require 'test/unit'
4
6
  require 'rubygems'
data/test/test_skynet.rb CHANGED
@@ -1,11 +1,19 @@
1
1
  require File.dirname(__FILE__) + '/test_helper.rb'
2
2
 
3
+ require 'tempfile'
3
4
  class TestSkynet < Test::Unit::TestCase
4
5
 
5
- def setup
6
- end
7
-
8
- def test_truth
9
- assert true
6
+ # this test doesn't work on a mac
7
+ def test_fork_and_exec
8
+ Tempfile.new('control').open
9
+ file = Tempfile.new('fork_exec')
10
+ Skynet.fork_and_exec("/usr/sbin/lsof -p $$ >#{file.path}")
11
+ sleep 1
12
+ open("#{file.path}", 'r') do |f|
13
+ lines = f.readlines
14
+ assert_equal 3, lines.grep(/null/).size, "fork_and_exec should redirect 0,1,2 to dev null"
15
+ assert_equal 0, lines.grep(/control/).size, "fork_and_exec should close parent's file descriptors"
16
+ end
10
17
  end
18
+
11
19
  end
data/test/test_skynet_manager.rb CHANGED
@@ -1,4 +1,5 @@
1
1
  require File.dirname(__FILE__) + '/test_helper.rb'
2
+ require 'functor'
2
3
 
3
4
  class SkynetManagerTest < Test::Unit::TestCase
4
5
 
@@ -6,6 +7,16 @@ class SkynetManagerTest < Test::Unit::TestCase
6
7
  PIDFILE = []
7
8
 
8
9
  def setup
10
+ connection = ActiveRecord::Base.establish_connection(
11
+ :adapter => "mysql",
12
+ :host => "localhost",
13
+ :username => "root",
14
+ :password => "",
15
+ :database => "skynet_test"
16
+ )
17
+
18
+ ActiveRecord::Base.connection.reconnect!
19
+
9
20
  Skynet.configure(
10
21
  :ENABLE => false,
11
22
  # :SKYNET_PIDS_FILE => File.expand_path("#{RAILS_ROOT}/log/skynet_#{RAILS_ENV}.pids"),
@@ -13,15 +24,9 @@ class SkynetManagerTest < Test::Unit::TestCase
13
24
  :SKYNET_LOG_LEVEL => 4,
14
25
  :MESSAGE_QUEUE_ADAPTER => "Skynet::MessageQueueAdapter::Mysql",
15
26
  :MYSQL_TEMPERATURE_CHANGE_SLEEP => 1,
16
- :MYSQL_NEXT_TASK_TIMEOUT => 1
27
+ :MYSQL_NEXT_TASK_TIMEOUT => 1,
28
+ :MYSQL_QUEUE_DATABASE => nil
17
29
  )
18
- ActiveRecord::Base.establish_connection(
19
- :adapter => "mysql",
20
- :host => "localhost",
21
- :username => "root",
22
- :password => "",
23
- :database => "skynet_test"
24
- )
25
30
 
26
31
  mq.clear_outstanding_tasks
27
32
  mq.clear_worker_status
@@ -62,7 +67,17 @@ class SkynetManagerTest < Test::Unit::TestCase
62
67
 
63
68
  pids = @pids
64
69
  hostname = @hostname
65
- @manager.define_method(:fork) do
70
+ Skynet.extend(Functor)
71
+ Skynet.define_method(:close_files) do
72
+ true
73
+ end
74
+
75
+ Skynet.define_method(:close_console) do
76
+ true
77
+ end
78
+
79
+
80
+ Skynet.define_method(:fork_and_exec) do |cmd|
66
81
  newpid = pids.size + 1
67
82
  pids << newpid
68
83
  worker_info = {
data/test/test_skynet_task.rb CHANGED
@@ -11,7 +11,7 @@ class SkynetTaskTest < Test::Unit::TestCase
11
11
  end
12
12
 
13
13
  def test_master_task
14
- job = Skynet::Job.new(:map_reduce_class => self.class, :async => true)
14
+ job = Skynet::Job.new(:map_reduce_class => self.class, :async => true, :local_master => false, :solo => true)
15
15
  master_task = Skynet::Task.master_task(job)
16
16
  master_job = Skynet::Job.new(master_task.process)
17
17
  assert_equal self.class.to_s, master_job.map
data/website/index.html CHANGED
@@ -5,7 +5,7 @@
5
5
  <link rel="stylesheet" href="stylesheets/screen.css" type="text/css" media="screen" />
6
6
  <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
7
7
  <title>
8
- skynet
8
+ space
9
9
  </title>
10
10
  <script src="javascripts/rounded_corners_lite.inc.js" type="text/javascript"></script>
11
11
  <style>
@@ -30,66 +30,114 @@
30
30
  <body>
31
31
  <div id="main">
32
32
 
33
- <h1>skynet</h1>
33
+ <h1>space</h1>
34
34
  <div id="version" class="clickable" onclick='document.location = "http://rubyforge.org/projects/skynet"; return false'>
35
35
  <p>Get Version</p>
36
- <a href="http://rubyforge.org/projects/skynet" class="numbers">0.9.2</a>
36
+ <a href="http://rubyforge.org/projects/skynet" class="numbers">0.9.3</a>
37
37
  </div>
38
- <p><a href="http://rubyforge.org/projects/skynet/">Skynet RubyForge Home</a><br />
38
+ <p><img src="skynet_logo.jpg" alt="" /></p>
39
+
40
+
41
+ <h2>A Ruby MapReduce Framework</h2>
42
+
43
+
44
+ <p><a href="http://rubyforge.org/projects/skynet/">Skynet RubyForge Home</a><br />
39
45
  <a href="doc/index.html">Documentation</a></p>
40
46
 
41
47
 
48
+ <h2>See Skynet at RailsConf 08</h2>
49
+
50
+
51
+ <h4>Sat Mar 31, 2008 4:25pm &#8211; 5:15pm <span class="caps">PDT</span> @ Portland Ballroom 255</h4>
52
+
53
+
54
+ <p><a href="http://en.oreilly.com/rails2008/public/schedule/detail/2022" target="_blank">More Info</a></p>
55
+
56
+
57
+ <p><a href="http://en.oreilly.com/rails2008/public/schedule/detail/2022" target="_blank">
58
+ <img src="http://conferences.oreillynet.com/banners/rails/speaker/728x90.jpg" width="728" height="90" border="0" alt="RailsConf 2008" title="RailsConf 2008" />
59
+ </a></p>
60
+
61
+
42
62
  <h2>What is Skynet</h2>
43
63
 
44
64
 
45
- <p>Skynet is an open source Ruby implementation of Google&#8217;s Map/Reduce framework, created at Geni.com. With Skynet, one can easily convert a time-consuming serial task, such as a computationally expensive Rails migration, into a distributed program running on many computers.</p>
65
+ <p>Skynet is an open source Ruby implementation of Google&#8217;s MapReduce framework, created at Geni. With Skynet, one can easily convert a time-consuming serial task, such as a computationally expensive Rails migration, into a distributed program running on many computers. If you&#8217;d like to learn more about MapReduce, see my intro at the bottom of this document.</p>
46
66
 
47
67
 
48
68
  <p>Skynet is an adaptive, self-upgrading, fault-tolerant, and fully distributed system with no single point of failure. It uses a &#8220;peer recovery&#8221; system where workers watch out for each other. If a worker dies or fails for any reason, another worker will notice and pick up that task. Skynet also has no special &#8216;master&#8217; servers, only workers which can act as a master for any task at any time. Even these master tasks can fail and will be picked up by other workers.</p>
49
69
 
50
70
 
51
- <h2>Installing</h2>
71
+ <p>For more detailed documentation see the following:</p>
72
+
73
+
74
+ <p>Skynet::Job &#8211; The main interface to Skynet; includes an example of how to use Skynet</p>
75
+
76
+
77
+ <p>Skynet::Config &#8211; Configuration options</p>
78
+
79
+
80
+ <p>bin/skynet[link:files/bin/skynet.html] &#8211; Starting Skynet</p>
81
+
82
+
83
+ <p>bin/skynet_install[link:files/bin/skynet_install.html] &#8211; Installing Skynet into a local project</p>
84
+
85
+
86
+ <p>There are also some examples in the examples/ directory included with Skynet.</p>
87
+
88
+
89
+ <h2>Installation</h2>
52
90
 
53
91
 
54
92
  <p>Skynet can be installed via RubyGems:</p>
55
93
 
56
94
 
57
- <p><pre class='syntax'><span class="ident">sudo</span> <span class="ident">gem</span> <span class="ident">install</span> <span class="ident">skynet</span></pre></p>
95
+ <pre><code>$ sudo gem install skynet</code></pre>
58
96
 
59
97
 
60
- <h2>More Info</h2>
98
+ <p>or grab the bleeding edge skynet in svn at
99
+ $ svn checkout svn+ssh://developername@rubyforge.org/var/svn/skynet
100
+ $ cd skynet; rake install_gem</p>
61
101
 
62
102
 
63
- <p>For more info check out the <a href="doc/index.html"><span class="caps">README</span></a></p>
103
+ <h2>Initial Setup</h2>
64
104
 
65
105
 
66
- <h2>Contact</h2>
106
+ <p>Skynet works by putting &#8220;tasks&#8221; on a message queue which are picked up by skynet workers. The workers execute tasks and put their results back on the message queue. Skynet workers need to load your code at startup in order to be able to execute your tasks. This loading is handled by installing a skynet config file into your app running skynet_install[link:files/bin/skynet_install.html]. </p>
67
107
 
68
108
 
69
- <p>Comments are welcome. Send an email to &#8220;Adam Pisoni&#8221; apisoni at geni.com</p>
109
+ <pre><code>$ skynet_install <a href="</del>-mysql"><del>-rails</a> APP_ROOT_DIR</code></pre>
110
+
111
+
112
+ <p>This creates a file called skynet_config.rb in <span class="caps">APP</span>_ROOT_DIR/config to which you can add the relevant requires. For example, you might have a rails app and want some of that code to run asynchronously or in a distributed way. Just run &#8216;skynet_install&#8212;rails&#8217; in your rails root, and it will automatically create config/skynet_config.rb and require environment.rb.</p>
113
+
114
+
115
+ <p>Skynet currently supports 2 message queue systems, TupleSpace and Mysql. By default, the TupleSpace queue is used as it is the easiest to set up, though it is less powerful and less scaleable for large installations. If you pass&#8212;mysql to skynet_install, it will assume you are using the mysql as your message queue.</p>
116
+
117
+
118
+ <h2>Starting Skynet</h2>
119
+
70
120
 
121
+ <p>Once it is installed in your application, you can run skynet from your applications root directory with:</p>
71
122
 
72
- <h2>Credits
73
- There are a number of people who either directly or indirectly worked on Skynet.
74
- John Beppu (wrote the original worker/manager code)
75
- Justin Balthrop
76
- Zack Parker
77
- Amos Elliston
78
- Zack Hobson
79
- Alan Braverman
80
- Mike Stangel
81
- Scott Steadman
82
- Andrew Arrow
83
- Jason Rojas</h2>
84
123
 
124
+ <pre><code>$ skynet start [--workers=N]</code></pre>
85
125
 
86
- <p>Skynet was inspired by and heavily influenced by Josh Carter and this blog post.
87
- http://multipart-mixed.com/software/simple_mapreduce_in_ruby.html</p>
88
126
 
127
+ <p>This starts a skynet tuple space message queue and 4 workers. You can control how many workers to start per machine
128
+ by passing&#8212;workers=N.</p>
89
129
 
90
- <p>Also by Starfish by Lucas Carlson
91
- http://tech.rufy.com/2006/08/mapreduce-for-ruby-ridiculously-easy.html
92
- http://rufy.com/starfish/doc/</p>
130
+
131
+ <h2>More Info</h2>
132
+
133
+
134
+ <p>For more info check out the <a href="doc/index.html"><span class="caps">README</span></a></p>
135
+
136
+
137
+ <h2>Contact</h2>
138
+
139
+
140
+ <p>Comments are welcome. Send an email to &#8220;Adam Pisoni&#8221; apisoni at geni.com</p>
93
141
 
94
142
 
95
143
  <h2><span class="caps">LICENSE</span>:</h2>
@@ -122,7 +170,7 @@ included in all copies or substantial portions of the Software.</p>
122
170
  <span class="caps">TORT OR OTHERWISE</span>, ARISING <span class="caps">FROM</span>, OUT <span class="caps">OF OR IN CONNECTION WITH THE</span>
123
171
  <span class="caps">SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE</span>.</p>
124
172
  <p class="coda">
125
- Adam Pisoni, Geni.com 25th January 2008<br>
173
+ Adam Pisoni, Geni.com 30th May 2008<br>
126
174
  Theme extended from <a href="http://rb2js.rubyforge.org/">Paul Battley</a>
127
175
  </p>
128
176
  </div>
data/website/index.txt CHANGED
@@ -1,19 +1,68 @@
1
- h1. skynet
1
+ h1. space
2
+
3
+ !skynet_logo.jpg!
4
+
5
+ h2. A Ruby MapReduce Framework
2
6
 
3
7
  <a href="http://rubyforge.org/projects/skynet/">Skynet RubyForge Home</a><br />
4
8
  <a href="doc/index.html">Documentation</a>
5
9
 
10
+ h2. See Skynet at RailsConf 08
11
+
12
+ h4. Sat Mar 31, 2008 4:25pm - 5:15pm PDT @ Portland Ballroom 255
13
+
14
+ <a href="http://en.oreilly.com/rails2008/public/schedule/detail/2022" target="_blank">More Info</a>
15
+
16
+ <a href="http://en.oreilly.com/rails2008/public/schedule/detail/2022" target="_blank">
17
+ <img src="http://conferences.oreillynet.com/banners/rails/speaker/728x90.jpg" width="728" height="90" border="0" alt="RailsConf 2008" title="RailsConf 2008" />
18
+ </a>
19
+
6
20
  h2. What is Skynet
7
21
 
8
- Skynet is an open source Ruby implementation of Google's Map/Reduce framework, created at Geni.com. With Skynet, one can easily convert a time-consuming serial task, such as a computationally expensive Rails migration, into a distributed program running on many computers.
22
+ Skynet is an open source Ruby implementation of Google's MapReduce framework, created at Geni. With Skynet, one can easily convert a time-consuming serial task, such as a computationally expensive Rails migration, into a distributed program running on many computers. If you'd like to learn more about MapReduce, see my intro at the bottom of this document.
9
23
 
10
24
  Skynet is an adaptive, self-upgrading, fault-tolerant, and fully distributed system with no single point of failure. It uses a "peer recovery" system where workers watch out for each other. If a worker dies or fails for any reason, another worker will notice and pick up that task. Skynet also has no special 'master' servers, only workers which can act as a master for any task at any time. Even these master tasks can fail and will be picked up by other workers.
11
25
 
12
- h2. Installing
26
+ For more detailed documentation see the following:
27
+
28
+ Skynet::Job - The main interface to Skynet; includes an example of how to use Skynet
29
+
30
+ Skynet::Config - Configuration options
31
+
32
+ bin/skynet[link:files/bin/skynet.html] - Starting Skynet
33
+
34
+ bin/skynet_install[link:files/bin/skynet_install.html] - Installing Skynet into a local project
35
+
36
+ There are also some examples in the examples/ directory included with Skynet.
37
+
38
+ h2. Installation
13
39
 
14
40
  Skynet can be installed via RubyGems:
15
41
 
16
- <pre syntax="ruby">sudo gem install skynet</pre>
42
+ $ sudo gem install skynet
43
+
44
+ or grab the bleeding edge skynet in svn at
45
+ $ svn checkout svn+ssh://developername@rubyforge.org/var/svn/skynet
46
+ $ cd skynet; rake install_gem
47
+
48
+ h2. Initial Setup
49
+
50
+ Skynet works by putting "tasks" on a message queue which are picked up by skynet workers. The workers execute tasks and put their results back on the message queue. Skynet workers need to load your code at startup in order to be able to execute your tasks. This loading is handled by installing a skynet config file into your app running skynet_install[link:files/bin/skynet_install.html].
51
+
52
+ $ skynet_install [--rails] [--mysql] APP_ROOT_DIR
53
+
54
+ This creates a file called skynet_config.rb in APP_ROOT_DIR/config to which you can add the relevant requires. For example, you might have a rails app and want some of that code to run asynchronously or in a distributed way. Just run 'skynet_install --rails' in your rails root, and it will automatically create config/skynet_config.rb and require environment.rb.
55
+
56
+ Skynet currently supports 2 message queue systems, TupleSpace and Mysql. By default, the TupleSpace queue is used as it is the easiest to set up, though it is less powerful and less scaleable for large installations. If you pass --mysql to skynet_install, it will assume you are using the mysql as your message queue.
57
+
58
+ h2. Starting Skynet
59
+
60
+ Once it is installed in your application, you can run skynet from your applications root directory with:
61
+
62
+ $ skynet start [--workers=N]
63
+
64
+ This starts a skynet tuple space message queue and 4 workers. You can control how many workers to start per machine
65
+ by passing --workers=N.
17
66
 
18
67
  h2. More Info
19
68
 
@@ -23,26 +72,6 @@ h2. Contact
23
72
 
24
73
  Comments are welcome. Send an email to "Adam Pisoni" apisoni at geni.com
25
74
 
26
- h2. Credits
27
- There are a number of people who either directly or indirectly worked on Skynet.
28
- John Beppu (wrote the original worker/manager code)
29
- Justin Balthrop
30
- Zack Parker
31
- Amos Elliston
32
- Zack Hobson
33
- Alan Braverman
34
- Mike Stangel
35
- Scott Steadman
36
- Andrew Arrow
37
- Jason Rojas
38
-
39
- Skynet was inspired by and heavily influenced by Josh Carter and this blog post.
40
- http://multipart-mixed.com/software/simple_mapreduce_in_ruby.html
41
-
42
- Also by Starfish by Lucas Carlson
43
- http://tech.rufy.com/2006/08/mapreduce-for-ruby-ridiculously-easy.html
44
- http://rufy.com/starfish/doc/
45
-
46
75
  h2. LICENSE:
47
76
 
48
77
  (The MIT License)
data/website/stylesheets/screen.css CHANGED
@@ -1,8 +1,8 @@
1
1
  body {
2
- background-color: #E1D1F1;
3
- font-family: "Georgia", sans-serif;
2
+ background-color: #fff;
3
+ font-family: "Lucida Grande", sans-serif;
4
4
  font-size: 16px;
5
- line-height: 1.6em;
5
+ line-height: 1.4em;
6
6
  padding: 1.6em 0 0 0;
7
7
  color: #333;
8
8
  }
@@ -12,19 +12,19 @@ h1, h2, h3, h4, h5, h6 {
12
12
  h1 {
13
13
  font-family: sans-serif;
14
14
  font-weight: normal;
15
- font-size: 4em;
16
- line-height: 0.8em;
15
+ font-size: 0em;
16
+ line-height: 0.0em;
17
17
  letter-spacing: -0.1ex;
18
18
  margin: 5px;
19
19
  }
20
+
20
21
  li {
21
22
  padding: 0;
22
23
  margin: 0;
23
24
  list-style-type: square;
24
25
  }
25
26
  a {
26
- color: #5E5AFF;
27
- background-color: #DAC;
27
+ color: #2255aa;
28
28
  font-weight: normal;
29
29
  text-decoration: underline;
30
30
  }
@@ -51,7 +51,7 @@ blockquote {
51
51
 
52
52
  table {
53
53
  font-size: 90%;
54
- line-height: 1.4em;
54
+ line-height: 1.2em;
55
55
  color: #ff8;
56
56
  background-color: #111;
57
57
  padding: 2px 10px 2px 10px;
@@ -103,18 +103,18 @@ pre, code {
103
103
  font-weight: normal;
104
104
  background-color: #B3ABFF;
105
105
  color: #141331;
106
- padding: 15px 20px 10px 20px;
106
+ padding: 10px 10px 10px 10px;
107
107
  margin: 0 auto;
108
- margin-top: 15px;
108
+ margin-top: 10px;
109
109
  border: 3px solid #141331;
110
110
  }
111
111
 
112
112
  #version .numbers {
113
113
  display: block;
114
114
  font-size: 4em;
115
- line-height: 0.8em;
115
+ line-height: 0.6em;
116
116
  letter-spacing: -0.1ex;
117
- margin-bottom: 15px;
117
+ margin-bottom: 10px;
118
118
  }
119
119
 
120
120
  #version p {
metadata CHANGED
@@ -1,33 +1,54 @@
1
1
  --- !ruby/object:Gem::Specification
2
- rubygems_version: 0.9.4
3
- specification_version: 1
4
2
  name: skynet
5
3
  version: !ruby/object:Gem::Version
6
- version: 0.9.2
7
- date: 2008-01-28 00:00:00 -08:00
8
- summary: Skynet - A Ruby Map/Reduce Framework
9
- require_paths:
10
- - lib
11
- email: apisoni@geni.com
12
- homepage: http://skynet.rubyforge.org
13
- rubyforge_project: skynet
14
- description: Skynet - A Ruby Map/Reduce Framework
15
- autorequire:
16
- default_executable:
17
- bindir: bin
18
- has_rdoc: true
19
- required_ruby_version: !ruby/object:Gem::Version::Requirement
20
- requirements:
21
- - - ">"
22
- - !ruby/object:Gem::Version
23
- version: 0.0.0
24
- version:
4
+ version: 0.9.3
25
5
  platform: ruby
26
- signing_key:
27
- cert_chain:
28
- post_install_message:
29
6
  authors:
30
7
  - Adam Pisoni
8
+ autorequire:
9
+ bindir: bin
10
+ cert_chain: []
11
+
12
+ date: 2008-05-31 00:00:00 -07:00
13
+ default_executable:
14
+ dependencies:
15
+ - !ruby/object:Gem::Dependency
16
+ name: daemons
17
+ version_requirement:
18
+ version_requirements: !ruby/object:Gem::Requirement
19
+ requirements:
20
+ - - ">="
21
+ - !ruby/object:Gem::Version
22
+ version: "1"
23
+ version:
24
+ - !ruby/object:Gem::Dependency
25
+ name: rubigen
26
+ version_requirement:
27
+ version_requirements: !ruby/object:Gem::Requirement
28
+ requirements:
29
+ - - ">="
30
+ - !ruby/object:Gem::Version
31
+ version: 1.1.1
32
+ version:
33
+ description: Skynet - A Ruby Map/Reduce Framework
34
+ email: apisoni@geni.com
35
+ executables:
36
+ - skynet
37
+ - skynet_install
38
+ - skynet_tuplespace_server
39
+ extensions: []
40
+
41
+ extra_rdoc_files:
42
+ - History.txt
43
+ - License.txt
44
+ - Manifest.txt
45
+ - README.txt
46
+ - examples/dgrep/data/testfile1.txt
47
+ - examples/dgrep/data/testfile2.txt
48
+ - examples/dgrep/data/testfile3.txt
49
+ - examples/dgrep/data/testfile4.txt
50
+ - examples/rails_mysql_example/public/robots.txt
51
+ - website/index.txt
31
52
  files:
32
53
  - History.txt
33
54
  - License.txt
@@ -37,13 +58,95 @@ files:
37
58
  - app_generators/skynet_install/USAGE
38
59
  - app_generators/skynet_install/skynet_install_generator.rb
39
60
  - app_generators/skynet_install/templates/migration.rb
40
- - app_generators/skynet_install/templates/skynet
41
- - app_generators/skynet_install/templates/skynet_schema.sql
61
+ - app_generators/skynet_install/templates/skynet_config.rb
62
+ - app_generators/skynet_install/templates/skynet_initializer.rb
63
+ - app_generators/skynet_install/templates/skynet_mysql_schema.sql
42
64
  - bin/skynet
43
65
  - bin/skynet_install
44
66
  - bin/skynet_tuplespace_server
45
67
  - config/hoe.rb
46
68
  - config/requirements.rb
69
+ - examples/dgrep/README
70
+ - examples/dgrep/config/skynet_config.rb
71
+ - examples/dgrep/data/shakespeare/README
72
+ - examples/dgrep/data/shakespeare/poetry/loverscomplaint
73
+ - examples/dgrep/data/shakespeare/poetry/rapeoflucrece
74
+ - examples/dgrep/data/shakespeare/poetry/sonnets
75
+ - examples/dgrep/data/shakespeare/poetry/various
76
+ - examples/dgrep/data/shakespeare/poetry/venusandadonis
77
+ - examples/dgrep/data/testfile1.txt
78
+ - examples/dgrep/data/testfile2.txt
79
+ - examples/dgrep/data/testfile3.txt
80
+ - examples/dgrep/data/testfile4.txt
81
+ - examples/dgrep/lib/dgrep.rb
82
+ - examples/dgrep/lib/mapreduce_test.rb
83
+ - examples/dgrep/lib/most_common_words.rb
84
+ - examples/dgrep/script/dgrep
85
+ - examples/rails_mysql_example/README
86
+ - examples/rails_mysql_example/Rakefile
87
+ - examples/rails_mysql_example/app/controllers/application.rb
88
+ - examples/rails_mysql_example/app/helpers/application_helper.rb
89
+ - examples/rails_mysql_example/app/models/user.rb
90
+ - examples/rails_mysql_example/app/models/user_favorite.rb
91
+ - examples/rails_mysql_example/app/models/user_mailer.rb
92
+ - examples/rails_mysql_example/app/views/user_mailer/welcome.erb
93
+ - examples/rails_mysql_example/config/boot.rb
94
+ - examples/rails_mysql_example/config/database.yml
95
+ - examples/rails_mysql_example/config/environment.rb
96
+ - examples/rails_mysql_example/config/environments/development.rb
97
+ - examples/rails_mysql_example/config/environments/production.rb
98
+ - examples/rails_mysql_example/config/environments/test.rb
99
+ - examples/rails_mysql_example/config/initializers/inflections.rb
100
+ - examples/rails_mysql_example/config/initializers/mime_types.rb
101
+ - examples/rails_mysql_example/config/initializers/skynet.rb
102
+ - examples/rails_mysql_example/config/routes.rb
103
+ - examples/rails_mysql_example/config/skynet_config.rb
104
+ - examples/rails_mysql_example/db/migrate/001_create_skynet_tables.rb
105
+ - examples/rails_mysql_example/db/migrate/002_create_users.rb
106
+ - examples/rails_mysql_example/db/migrate/003_create_user_favorites.rb
107
+ - examples/rails_mysql_example/db/schema.rb
108
+ - examples/rails_mysql_example/db/skynet_mysql_schema.sql
109
+ - examples/rails_mysql_example/doc/README_FOR_APP
110
+ - examples/rails_mysql_example/lib/tasks/rails_mysql_example.rake
111
+ - examples/rails_mysql_example/public/.htaccess
112
+ - examples/rails_mysql_example/public/404.html
113
+ - examples/rails_mysql_example/public/422.html
114
+ - examples/rails_mysql_example/public/500.html
115
+ - examples/rails_mysql_example/public/dispatch.cgi
116
+ - examples/rails_mysql_example/public/dispatch.fcgi
117
+ - examples/rails_mysql_example/public/dispatch.rb
118
+ - examples/rails_mysql_example/public/favicon.ico
119
+ - examples/rails_mysql_example/public/images/rails.png
120
+ - examples/rails_mysql_example/public/index.html
121
+ - examples/rails_mysql_example/public/javascripts/application.js
122
+ - examples/rails_mysql_example/public/javascripts/controls.js
123
+ - examples/rails_mysql_example/public/javascripts/dragdrop.js
124
+ - examples/rails_mysql_example/public/javascripts/effects.js
125
+ - examples/rails_mysql_example/public/javascripts/prototype.js
126
+ - examples/rails_mysql_example/public/robots.txt
127
+ - examples/rails_mysql_example/script/about
128
+ - examples/rails_mysql_example/script/console
129
+ - examples/rails_mysql_example/script/destroy
130
+ - examples/rails_mysql_example/script/generate
131
+ - examples/rails_mysql_example/script/performance/benchmarker
132
+ - examples/rails_mysql_example/script/performance/profiler
133
+ - examples/rails_mysql_example/script/performance/request
134
+ - examples/rails_mysql_example/script/plugin
135
+ - examples/rails_mysql_example/script/process/inspector
136
+ - examples/rails_mysql_example/script/process/reaper
137
+ - examples/rails_mysql_example/script/process/spawner
138
+ - examples/rails_mysql_example/script/runner
139
+ - examples/rails_mysql_example/script/server
140
+ - examples/rails_mysql_example/test/fixtures/user_favorites.yml
141
+ - examples/rails_mysql_example/test/fixtures/users.yml
142
+ - examples/rails_mysql_example/test/test_helper.rb
143
+ - examples/rails_mysql_example/test/unit/user_favorite_test.rb
144
+ - examples/rails_mysql_example/test/unit/user_test.rb
145
+ - extras/README
146
+ - extras/init.d/skynet
147
+ - extras/nagios/check_skynet.sh
148
+ - extras/rails/controllers/skynet_controller.rb
149
+ - extras/rails/views/skynet/index.rhtml
47
150
  - lib/skynet.rb
48
151
  - lib/skynet/mapreduce_helper.rb
49
152
  - lib/skynet/mapreduce_test.rb
@@ -68,10 +171,6 @@ files:
68
171
  - lib/skynet/skynet_tuplespace_server.rb
69
172
  - lib/skynet/skynet_worker.rb
70
173
  - lib/skynet/version.rb
71
- - log/debug.log
72
- - log/skynet.log
73
- - log/skynet_tuplespace_server.log
74
- - log/skynet_worker.pid
75
174
  - script/destroy
76
175
  - script/generate
77
176
  - script/txt2html
@@ -95,6 +194,33 @@ files:
95
194
  - website/javascripts/rounded_corners_lite.inc.js
96
195
  - website/stylesheets/screen.css
97
196
  - website/template.rhtml
197
+ has_rdoc: true
198
+ homepage: http://skynet.rubyforge.org
199
+ post_install_message:
200
+ rdoc_options:
201
+ - --main
202
+ - README.txt
203
+ require_paths:
204
+ - lib
205
+ required_ruby_version: !ruby/object:Gem::Requirement
206
+ requirements:
207
+ - - ">="
208
+ - !ruby/object:Gem::Version
209
+ version: "0"
210
+ version:
211
+ required_rubygems_version: !ruby/object:Gem::Requirement
212
+ requirements:
213
+ - - ">="
214
+ - !ruby/object:Gem::Version
215
+ version: "0"
216
+ version:
217
+ requirements: []
218
+
219
+ rubyforge_project: skynet
220
+ rubygems_version: 1.1.0
221
+ signing_key:
222
+ specification_version: 2
223
+ summary: Skynet - A Ruby Map/Reduce Framework
98
224
  test_files:
99
225
  - test/test_active_record_extensions.rb
100
226
  - test/test_generator_helper.rb
@@ -107,39 +233,3 @@ test_files:
107
233
  - test/test_skynet_message.rb
108
234
  - test/test_skynet_task.rb
109
235
  - test/test_tuplespace_message_queue.rb
110
- rdoc_options:
111
- - --main
112
- - README.txt
113
- extra_rdoc_files:
114
- - History.txt
115
- - License.txt
116
- - Manifest.txt
117
- - README.txt
118
- - website/index.txt
119
- executables:
120
- - skynet
121
- - skynet_install
122
- - skynet_tuplespace_server
123
- extensions: []
124
-
125
- requirements: []
126
-
127
- dependencies:
128
- - !ruby/object:Gem::Dependency
129
- name: daemons
130
- version_requirement:
131
- version_requirements: !ruby/object:Gem::Version::Requirement
132
- requirements:
133
- - - ">="
134
- - !ruby/object:Gem::Version
135
- version: "1"
136
- version:
137
- - !ruby/object:Gem::Dependency
138
- name: rubigen
139
- version_requirement:
140
- version_requirements: !ruby/object:Gem::Version::Requirement
141
- requirements:
142
- - - ">="
143
- - !ruby/object:Gem::Version
144
- version: 1.1.1
145
- version: