skynet 0.9.2 → 0.9.3

data/History.txt

@@ -1,3 +1,52 @@
+== 0.9.3 2008-05-22
+Skynet::Manager and Skynet Script Runner
+ - Rewrote how Skynet workers and skynet manager talks on each machine.  See below for more info 
+ 
+ - Added an examples/ directory with sample skynet apps.
+ 
+ - Support starting Skynet with 'skynet start/stop' to daemonize
+ - skynet_install now only installs a config/skynet_config.rb in your applications directory.  You are no longer supposed
+ to have a script/skynet to start skynet.  Instead, as long as you have a config/skynet_config.rb you can just run
+ 'skynet start/stop' from within your application_directory/
+ If you're installing in a rails application, it will install a default config/initializers/skynet.rb which merely requires config/skynet_config.rb
+ - The config file can also be specified with --config= skynet start
+
+ - Gracefully handles trying to start skynet more than once
+ - Close file handles on exec.
+     Skynet::Worker and Skynet::Manager now call Skynet.fork_and_exec instead of their own versions.
+     Skynet.fork_and_exec prevents file descriptor exhaustion by calling Skynet.close_file_handles.
+     Skynet::Manager detatches from console by calling Skynet.close_console
+
+ - Dramatically improved Skynet shutdown time
+ - Huge performance improvements in taking/putting tasks and results.  Much lower resource utilization, especially on mysql.
+ - You now have to specify your PIDFILE and LOGFILE dirs and files differently. This will break all old skynet runners. Sorry.
+ - Skynet runner handles default config variables better
+ - Fix serious bug in Skynet::Partitioners::RecombineAndSplit where it wouldn't handle empty results well.
+ - Fixed Skynet::Partitioners::ArrayDataSplitByFirstEntry to handle strings as keys better
+ - Added Skynet::Job.results_by_job_id to retrieve results from asyncronous jobs
+ - Added printlog logging method which always prints to the log as [LOG]
+ - Deprecated Skynet.new to Skynet.start 
+ - Mysql Message Queue Adapter - Make delete_expired_messages much safer.
+ - rename ActiveRecord::Base.distributed_find.each to ActiveRecord::Base.distributed_find.map
+ - ActiveRecord::Base.distributed_find - Patch submitted by Lourens Naude (lourens@methodmissing.com) which checks the model for the primary_key name as opposed to assuming it is
+  'id'
+ - We don't want to use rails constantize so I've temporarily borrowed the method from ActiveSupport inflector and added it to skynet_ruby_extensions.
+ - Fix bug in Job comment where it referenced MapreduceTest instead of Skynet::MapreduceTest 
+ - Fix tests. For some reason you still can't run ALL the test at once with rake test, but if the files are run individually they all pass.
+ - Change mysql text fields to longtext in migration and schema files
+ - Include some extras including our init.d script, our nagios monitoring script, rails controller and view for monitoring
+ - Created a new skynet rails initializer which gets installed from skynet_install --rails
+ - Modified the skynet_install skynet runner to take skynet initializer into account
+ - Skynet::ActiveRecordExtensions Fixed a bug where it would fail if your table had fewer than 1000 rows.  It performs a count
+  first now to make sure there are enough rows.
+ - Introduced some new Skynet::Config methods for getting logfile and pidfile locations   
+ 
+Skynet Manager/Worker Refactor
+  The workers used to publish their worker statuses to the skynet_worker_queue which lived in the same Q space as the skynet_message_queue.  skynet managers would then query that queue for their workers' statuses.   This was a very inefficient use of central resources.    NOW, workers communicate with their manager vir DRb, calling manager.worker_notify(status).   This adds the worker status hash onto a local Queue object stored in the manager.  A separate thread watches that queue and updates the internal manager information about it's workers.  This was in place of polling a queue every N seconds for new worker records.
+Managers now save their worker information to a file periodically so they can be reloaded on a restart.  This means managers can keep track of how many tasks all of their workers have done even after restarts.
+As a consequence of decentralizing worker stats, you now have to ask all your managers for their individual stats along with the main message queue stats.  Added a stats_for_hosts method to Skynet::Manager which aggregates stats accross many managers.
+
+
 == 0.9.2 2008-01-22
 Highlights: 
  - Multiple Message Queues

data/Manifest.txt

@@ -6,13 +6,95 @@ Rakefile
 app_generators/skynet_install/USAGE
 app_generators/skynet_install/skynet_install_generator.rb
 app_generators/skynet_install/templates/migration.rb
-app_generators/skynet_install/templates/skynet
-app_generators/skynet_install/templates/skynet_schema.sql
+app_generators/skynet_install/templates/skynet_config.rb
+app_generators/skynet_install/templates/skynet_initializer.rb
+app_generators/skynet_install/templates/skynet_mysql_schema.sql
 bin/skynet
 bin/skynet_install
 bin/skynet_tuplespace_server
 config/hoe.rb
 config/requirements.rb
+examples/dgrep/README
+examples/dgrep/config/skynet_config.rb
+examples/dgrep/data/shakespeare/README
+examples/dgrep/data/shakespeare/poetry/loverscomplaint
+examples/dgrep/data/shakespeare/poetry/rapeoflucrece
+examples/dgrep/data/shakespeare/poetry/sonnets
+examples/dgrep/data/shakespeare/poetry/various
+examples/dgrep/data/shakespeare/poetry/venusandadonis
+examples/dgrep/data/testfile1.txt
+examples/dgrep/data/testfile2.txt
+examples/dgrep/data/testfile3.txt
+examples/dgrep/data/testfile4.txt
+examples/dgrep/lib/dgrep.rb
+examples/dgrep/lib/mapreduce_test.rb
+examples/dgrep/lib/most_common_words.rb
+examples/dgrep/script/dgrep
+examples/rails_mysql_example/README
+examples/rails_mysql_example/Rakefile
+examples/rails_mysql_example/app/controllers/application.rb
+examples/rails_mysql_example/app/helpers/application_helper.rb
+examples/rails_mysql_example/app/models/user.rb
+examples/rails_mysql_example/app/models/user_favorite.rb
+examples/rails_mysql_example/app/models/user_mailer.rb
+examples/rails_mysql_example/app/views/user_mailer/welcome.erb
+examples/rails_mysql_example/config/boot.rb
+examples/rails_mysql_example/config/database.yml
+examples/rails_mysql_example/config/environment.rb
+examples/rails_mysql_example/config/environments/development.rb
+examples/rails_mysql_example/config/environments/production.rb
+examples/rails_mysql_example/config/environments/test.rb
+examples/rails_mysql_example/config/initializers/inflections.rb
+examples/rails_mysql_example/config/initializers/mime_types.rb
+examples/rails_mysql_example/config/initializers/skynet.rb
+examples/rails_mysql_example/config/routes.rb
+examples/rails_mysql_example/config/skynet_config.rb
+examples/rails_mysql_example/db/migrate/001_create_skynet_tables.rb
+examples/rails_mysql_example/db/migrate/002_create_users.rb
+examples/rails_mysql_example/db/migrate/003_create_user_favorites.rb
+examples/rails_mysql_example/db/schema.rb
+examples/rails_mysql_example/db/skynet_mysql_schema.sql
+examples/rails_mysql_example/doc/README_FOR_APP
+examples/rails_mysql_example/lib/tasks/rails_mysql_example.rake
+examples/rails_mysql_example/public/.htaccess
+examples/rails_mysql_example/public/404.html
+examples/rails_mysql_example/public/422.html
+examples/rails_mysql_example/public/500.html
+examples/rails_mysql_example/public/dispatch.cgi
+examples/rails_mysql_example/public/dispatch.fcgi
+examples/rails_mysql_example/public/dispatch.rb
+examples/rails_mysql_example/public/favicon.ico
+examples/rails_mysql_example/public/images/rails.png
+examples/rails_mysql_example/public/index.html
+examples/rails_mysql_example/public/javascripts/application.js
+examples/rails_mysql_example/public/javascripts/controls.js
+examples/rails_mysql_example/public/javascripts/dragdrop.js
+examples/rails_mysql_example/public/javascripts/effects.js
+examples/rails_mysql_example/public/javascripts/prototype.js
+examples/rails_mysql_example/public/robots.txt
+examples/rails_mysql_example/script/about
+examples/rails_mysql_example/script/console
+examples/rails_mysql_example/script/destroy
+examples/rails_mysql_example/script/generate
+examples/rails_mysql_example/script/performance/benchmarker
+examples/rails_mysql_example/script/performance/profiler
+examples/rails_mysql_example/script/performance/request
+examples/rails_mysql_example/script/plugin
+examples/rails_mysql_example/script/process/inspector
+examples/rails_mysql_example/script/process/reaper
+examples/rails_mysql_example/script/process/spawner
+examples/rails_mysql_example/script/runner
+examples/rails_mysql_example/script/server
+examples/rails_mysql_example/test/fixtures/user_favorites.yml
+examples/rails_mysql_example/test/fixtures/users.yml
+examples/rails_mysql_example/test/test_helper.rb
+examples/rails_mysql_example/test/unit/user_favorite_test.rb
+examples/rails_mysql_example/test/unit/user_test.rb
+extras/README
+extras/init.d/skynet
+extras/nagios/check_skynet.sh
+extras/rails/controllers/skynet_controller.rb
+extras/rails/views/skynet/index.rhtml
 lib/skynet.rb
 lib/skynet/mapreduce_helper.rb
 lib/skynet/mapreduce_test.rb
@@ -37,10 +119,6 @@ lib/skynet/skynet_task.rb
 lib/skynet/skynet_tuplespace_server.rb
 lib/skynet/skynet_worker.rb
 lib/skynet/version.rb
-log/debug.log
-log/skynet.log
-log/skynet_tuplespace_server.log
-log/skynet_worker.pid
 script/destroy
 script/generate
 script/txt2html

data/README.txt

@@ -2,63 +2,61 @@ Skynet
 	http://skynet.rubyforge.org/
 	by Adam Pisoni and Geni.com
 
-== DESCRIPTION:
+== DESCRIPTION
 
-Skynet is an open source Ruby implementation of Google's Map/Reduce framework, created at Geni. With Skynet, one can easily convert a time-consuming serial task, such as a computationally expensive Rails migration, into a distributed program running on many computers.
+Skynet is an open source Ruby implementation of Google's MapReduce framework, created at Geni. With Skynet, one can easily convert a time-consuming serial task, such as a computationally expensive Rails migration, into a distributed program running on many computers.  If you'd like to learn more about MapReduce, see my intro at the bottom of this document.
 
 Skynet is an adaptive, self-upgrading, fault-tolerant, and fully distributed system with no single point of failure. It uses a "peer recovery" system where workers watch out for each other. If a worker dies or fails for any reason, another worker will notice and pick up that task. Skynet also has no special 'master' servers, only workers which can act as a master for any task at any time. Even these master tasks can fail and will be picked up by other workers.
 
-== DOCUMENTATION
+For more detailed documentation see the following:
 
-Feel free to read on if you want more of an overview of Skynet with some specific examples. More specific documentation can be found here:
+Skynet::Job - The main interface to Skynet; includes an example of how to use Skynet
 
-Skynet::Job - The main interface to Skynet
-
-Skynet::Config - Configuration Options
+Skynet::Config - Configuration options
 
 bin/skynet[link:files/bin/skynet.html] - Starting Skynet
 
 bin/skynet_install[link:files/bin/skynet_install.html] - Installing Skynet into a local project
 
-== Map/Reduce
+There are also some examples in the examples/ directory included with Skynet.
 
-First of all, Skynet is merely a distributed computing system that allows you to follow the map/reduce steps.  You don't have to use it as a map/reduce framework.  You can use it as a simple distributed system, or even a simple asynchronous processing system.
+== INSTALLATION
 
-If you already know what Map/Reduce is, skip this section.                                                             
+Skynet can be installed via RubyGems:
 
-If you want to know where all this Map/Reduce hype started, you should read Google's paper on it.  http://labs.google.com/papers/mapreduce.html
+  $ sudo gem install skynet
 
-When I first read that Google paper some years ago, I was a little confused about what all the hypes was.   At the most basic level, it seemed too simple to be revolutionary.  So you've got a job with 3 steps,  you put some data in, it gets split out to a map step run no many machines, the returned data gets reshuffled and parceled out to a reduce step run on many machines.  All the results are then put together again.   You can see it as 5 steps actually.   Data -> Partition -> Map -> Partition -> Reduce.  Simple enough.  Almost too simple.   It was only years later when I began working on Skynet that I realize what the revolutionary part of Google's framwork was.  It made distributed computing accessible.   Any engineer could write a complex distributed system without needing to know about the complexities of such systems.   Also, since the distributed system was generalized, you would only need one class of machines to run ALL of your distributed processing, instead of specialized machines for specialized functions.  THAT was revolutionary.   
+or grab the bleeding edge skynet in svn at
+  $ svn checkout svn+ssh://developername@rubyforge.org/var/svn/skynet
+  $ cd skynet; rake install_gem
 
-There are a number of key differences between Google's MR system and skynet.  Firstly, currently you can not actually send raw code to the workers.  You are really only telling it where the code is.   At first this bothered me a lot.  Then I realized that in most OO systems, the amount of code you'd need duplicate and to send over the wire to every worker could be ridiculous.  For example, if you want to distribute a task you need to run in Rails, you'd have to send almost all of your app and rails to every worker with every chunk of data.  So, even if you COULD send code, you'd probably only be sending code that just called some other code in your system.  If you can't send ALL the code it needs, then you might as well just tell it where the code is.
+== INITIAL SETUP
 
-The second big difference is that Google's MR framework uses Master federater processes to dole out tasks, recombine them, and generally watch the system.  Skynet has not such masters.   Instead Skynet uses a standard message queue for all communication.  That same message queue allows workers to watch each other in the same way a master would, but without the single point of failure (except the queue itself). 
+Skynet works by putting "tasks" on a message queue which are picked up by skynet workers. The workers execute tasks and put their results back on the message queue. Skynet workers need to load your code at startup in order to be able to execute your tasks. This loading is handled by installing a skynet config file into your app running skynet_install[link:files/bin/skynet_install.html]. 
 
-At its simplest level, a single map reduce job defines a data set, a map method and a reduce method. It may also define a partition method. The map/reduce server evenly splits up (partitions) the data given to it and sends those chunks of data, along with a copy of the code in the map method, to workers that execute the map method against the data it was given. The output from each worker is sent back to the map/reduce server. At this point the Mapreduce server evenly partitions the RESULT data returned from the workers and sends those chunks of data along with the reduce code to the workers to be executed. The reducers return the final result which is returned to whomever requested the job be done in the first place. Not all job need a reduce step, some may just have a map step.
+  $ skynet_install [--rails] [--mysql] APP_ROOT_DIR
 
-The most common example of a mapreduce job is a distributed word counter. Say you wanted to determine how many times a single word appears in a 1GB text file. The map/reduce server would break up the 1GB file into reasonable chunks, say 100 lines per chunk (or partition) and then send each 100 line partition along with the code that looks for that word, to workers. Each worker would grab its partition of the data, count how many times the word appears in the data and return that number. It might take dozens of workers to complete the task. When the map step is done, you are left with a huge list of counts returned by the workers. In this example, the reduce step would involve sending that list of counts to another worker, with the code required to sum those counts and finally return the total. In this way a task that used to be done in a linear fashion can be parallelized easily.
+This creates a file called skynet_config.rb in APP_ROOT_DIR/config to which you can add the relevant requires. For example, you might have a rails app and want some of that code to run asynchronously or in a distributed way. Just run 'skynet_install --rails' in your rails root, and it will automatically create config/skynet_config.rb and require environment.rb.
 
-== INSTALLATION:
+Skynet currently supports 2 message queue systems, TupleSpace and Mysql. By default, the TupleSpace queue is used as it is the easiest to set up, though it is less powerful and less scaleable for large installations. If you pass --mysql to skynet_install, it will assume you are using the mysql as your message queue.
 
-Skynet can be installed via RubyGems:
+== STARTING SKYNET
 
-  $ sudo gem install skynet
+Once it is installed in your application, you can run skynet from your applications root directory with:
 
-== GETTING STARTED
+  $ skynet start [--workers=N]
 
-Skynet works by putting "tasks" on a message queue which are picked up by skynet workers, who execute the tasks, then put their results back on the message queue.   Skynet works best when it runs with your code.  For example, you might have a rails app and want some code you've already written to run asynchronously or in a distributed way.   Skynet can run within your code by installing a skynet launcher into your app.   Running this skynet launcher within your app guarantees all skynet workers will have access to your code.   This will be covered later.
+This starts a skynet tuple space message queue and 4 workers. You can control how many workers to start per machine
+by passing --workers=N.
 
-Skynet currently supports 2 message queue systems, TupleSpace and Mysql.   By default, the TupleSpace queue is used as it is the easiest to set up, though it is less powerful and less scaleable for large installations.  
+== SKYNET CONSOLE
 
-== RUNING SKYNET FOR THE FIRST TIME
-Since Skynet is a distributed system, it requires you have a skynet message queue as well as any number of skynet workers running.  To start a skynet message queue and a small number of workers:
+You can now run the skynet console to play with skynet a little. See Skynet::ConsoleHelper for commands.
 
-  $ skynet
+  $ skynet console
 
-This starts a skynet tuple space message queue and 4 workers.   You can now run the skynet console to play with skynet a little.  See Skynet::ConsoleHelper for commands.
+Remember, when you change your code, you must stop/start skynet.
 
-  $ skynet console
-  
 For help try:
   $ skynet --help
 or
@@ -69,67 +67,80 @@ Here are some commands you can run in the skynet console.
   > manager.worker_pids
   > [1,2,3,1,1,4].mapreduce(Skynet::MapreduceTest)
   
-That last command actually took whatever array you gave it and counted the number of times each element appeared in the array.  It's not a very useful task, but it shows how easy it is to use.
-
-== RUNING SKYNET IN YOUR APPLICATION
-
-To be really useful, you'll want to run skynet in your own application.   To do that run:
+That last command actually took whatever array you gave it and counted the number of times each element appeared in the array.  It's not a very useful task, but it shows how easy Skynet is to use.
 
-  $ skynet_install [--rails] YOUR_APP_DIRECTORY
+To see what Skynet is doing, you may want to tail the skynet logs being written to your log directory.
 
-If you pass --rails it will assume it is installing in a rails app.   Once it is installed in your application, you can run skynet with
+For more information on creating your own Skynet jobs read the Skynet::Job documentation.
 
-  $ ./script/skynet
-  $ ./script/skynet console
+== USAGE
 
-== USAGE:
-
-Skynet was designed to make doing easy things easy and hard things possible.   The easiest way to use skynet is to create a new class with a self.map class method.  You can optionally include self.reduce, self.reduce_partitioner, self.map_partitioner as well.  Each of those methods should expect a single array (regardless of what data you pass).   Then, simple create an array and call mapreduce on it passing your class name.   Skynet will figure out which methods your class supports and use them accordingly.
+Skynet was designed to make doing easy things easy and hard things possible. The easiest way to use skynet is to create a new class with a self.map class method. You can optionally include self.reduce and self.reduce_partitioner as well. Each of those methods should expect a single array (regardless of what data you pass). Then, simply create an array and call mapreduce on it passing your class name. Skynet will figure out which methods your class supports and use them accordingly.
 
 == USING SKYNET IN RAILS
 
-Skynet includes an addition to ActiveRecord that is very powerful.   
+Skynet includes an extension to ActiveRecord that is very powerful.   
 
 === distributed_find
-  $ YourModel.distributed_find(:all).each(YourClass)
-or
+
   $ YourModel.distributed_find(:all).each(:somemethod)
   
-In the first example, a find is 'virtually' run with your model class, and the results are distributed to the skynet workers.  If you've implemented a self.map method in YourClass, the retrieved objects will be passed (as arrays) on all the workers.
-
-In the second example, once the objects of YourModel are distributed, each worker merely calls :somemethod against each object.
+A find is 'virtually' run with your model class, and the results are distributed to the skynet workers. Each worker then calls :somemethod against each object.
 
 === send_later
 
-  $ model_object.send_later(:method,options,:save)
+  $ model_object.send_later(:method, options, :save)
   
-Sometimes you have a method you want to call on a model asynchronously.  Using :send_later you can call a method, pass it options, and decide whether you want Skynet to save that model or not once its done calling your method.
+Sometimes you have a method you want to call on a model asynchronously. Using :send_later you can call a method, pass it options, and decide whether you want Skynet to save that model or not once its done calling your method.
 
 == Creating Skynet Jobs
 
-The main interface to Skynet is through Skynet::AsyncJob or Skynet::Job
+The main interface to Skynet is through Skynet::Job
 
-  job = Skynet::AsyncJob.new(options)
-  job.run_master
+  job = Skynet::Job.new(options)
+  job.run
   
-There are many options you can pass or change once you have a job object.  See Skynet::Job for more info.
+There are many options you can pass or change once you have a job object. See Skynet::Job for more info.
   
-Most of the time, you will only need to pass a map_reduce_class and map_data.  All other options just give you finer grain control.   map_data must be an array.   The map_reduce_class must AT LEAST implement a self.map class method.  It may also implement self.reduce, self.reduce_partitioner, and self.map_partitioner.  Skynet will assume it can use all of those methods in the map_reduce_class you pass.
+Most of the time, you will only need to pass a :map_reduce_class and :map_data. All other options just give you finer grained control. The :map_data must be an array, and the :map_reduce_class must implement at least a self.map class method. It may optionally implement self.reduce and self.reduce_partitioner. Your map and reduce class methods should ALWAYS assume they are being passed an array. Your map method must always return an array as well.
+
+== Skynet Logging
+
+You might be interested in seeing what skynet is doing.  There are two Skynet::Config options which control logging: Skynet::CONFIG[:SKYNET_LOG_LEVEL] and Skynet::CONFIG[:SKYNET_LOG_FILE]. Skynet::CONFIG[:SKYNET_LOG_LEVEL] is set to Logger::ERROR by default. Other possibilities are Logger::DEBUG, Logger::INFO, Logger::WARN, Logger::ERROR, and Logger::FATAL. You might try Logger::INFO to see more of what's going on. To use the Skynet::Logger inside your own classes simply add:
+
+  include SkynetDebugger
+
+For more information see SkynetDebugger[link:files/lib/skynet/skynet_debugger_rb.html].
+
+== A Note on MapReduce
+
+At its simplest level, a MapReduce job defines a data set, a map method and a reduce method. It may also define a partition method. The MapReduce server evenly splits up (partitions) the data given to it and sends those chunks of data, along with a copy of the code in the map method, to workers that execute the map method against the data it was given. The output from each worker is sent back to the MapReduce server. At this point the MapReduce server evenly partitions the RESULT data returned from the workers and sends those chunks of data along with the reduce code to the workers to be executed. The reducers return the final result which is returned to the process that requested the job be done in the first place. Not all jobs need a reduce step, some may just have a map step.
 
-Your map and reduce class methods should ALWAYS assume they are being passed an array.  Your map method must always return an array as well.
+The most common example of a MapReduce job is a distributed word counter. Say you wanted to determine how many times a single word appears in a 1GB text file. The MapReduce server would break up the 1GB file into reasonable chunks, say 100 lines per chunk (or partition) and then send each 100 line partition along with the code that looks for that word, to workers. Each worker would grab its partition of the data, count how many times the word appears in the data and return that number. It might take dozens of workers to complete the task. When the map step is done, you are left with a huge list of counts returned by the workers. In this example, the reduce step would consist of sending that list of counts to another worker, with the code required to sum those counts and finally return the total. In this way a task that used to be done in a linear fashion can be parallelized easily.
+
+If you want more details on MapReduce, read Google's paper on it.  http://labs.google.com/papers/mapreduce.html
+
+When I first read that Google paper some years ago, I was a little confused about what was so unique about it. At the most basic level, it seemed too simple to be revolutionary. So you've got a job with two steps: map and reduce. You put some data in, it gets split out to a map step run on many machines. the returned data gets reshuffled and parceled out to a reduce step run on many machines. All the results are then put together again. You can see it as five steps actually. Data -> Partition -> Map -> Partition -> Reduce. Simple enough. Almost too simple. It was only years later when I began working on Skynet that I realized what the revolutionary part of Google's framework was. It made distributed computing accessible. Any engineer could write a complex distributed system without needing to know about the complexities of such systems. They just write a map function and a reduce function. Also, since the distributed system was generalized, you would only need one class of machines to run ALL of your distributed processing, instead of specialized machines for specialized jobs. That WAS revolutionary.
+
+Skynet is merely a distributed computing system that allows you to break your problem into map and reduce steps. You don't have to use it as a MapReduce framework though. You can use it as a simple distributed system, or even a simple asynchronous processing system.
+
+There are a number of key differences between Google's MapReduce system and skynet. First, currently you can not actually send raw code to the workers. You are really only telling it where the code is. At first this bothered me a lot. Then I realized that in most Object Oriented systems, the amount of code you'd need to duplicate and send over the wire to every worker could be ridiculous. For example, if you want to distribute a task you need to run in Rails, you'd have to send almost all of your app and rails code to every worker with every chunk of data. So, even if you COULD send code, that code would probably eventually jut call some other code in your system. If you can't send ALL the code needed for a task, then you might as well just tell Skynet where all the needed code is.
+
+The second big difference is that Google's MapReduce framework uses master federator processes to dole out tasks, recombine them, and generally watch the system. Skynet has no such masters. Instead Skynet uses a standard message queue for all communication. That message queue allows workers to watch each other in the same way a master would, but without the single point of failure (except for the queue itself).
 
 == CREDITS
+
 There are a number of people who either directly or indirectly worked on Skynet.
-John Beppu (wrote the original worker/manager code)
-Justin Balthrop
-Zack Parker
-Amos Elliston
-Zack Hobson
-Alan Braverman
-Mike Stangel
-Scott Steadman
-Andrew Arrow
-Jason Rojas
+* Justin Balthrop
+* Zack Parker
+* Amos Elliston
+* Zack Hobson
+* Alan Braverman
+* Mike Stangel
+* Scott Steadman
+* Andrew Arrow
+* John Beppu (wrote the original worker/manager code)
+* Jason Rojas
 
 Skynet was inspired by and heavily influenced by Josh Carter and this blog post.
 http://multipart-mixed.com/software/simple_mapreduce_in_ruby.html

data/app_generators/skynet_install/skynet_install_generator.rb

@@ -20,15 +20,19 @@ class SkynetInstallGenerator < RubiGen::Base
 
   def manifest
     record do |m|
+
       # Ensure appropriate folder(s) exists
-      m.directory 'script'
       BASEDIRS.each { |path| m.directory path }
       
       # Create stubs
-      m.template     "skynet",         "script/skynet", :collision => :ask, :chmod => 0775, :shebang => options[:shebang]
+      m.template  "skynet_config.rb", "config/skynet_config.rb", :collision => :ask, :chmod => 0655
+      if @in_rails
+        m.directory 'config/initializers'
+        m.template  "skynet_initializer.rb", "config/initializers/skynet.rb", :collision => :ask, :chmod => 0655
+      end
       if @mysql
-        m.template   "skynet_schema.sql", "db/skynet_schema.sql", :collision => :ask, :chmod => 0655
         m.directory 'db/migrate'
+        m.template   "skynet_mysql_schema.sql", "db/skynet_mysql_schema.sql", :collision => :ask, :chmod => 0655
         m.migration_template "migration.rb", "db/migrate", 
           :collision => :ask, 
           :assigns => {
@@ -45,7 +49,7 @@ Creates a ...
 
 USAGE: #{spec.name} [--rails] [--mysql] directory (can be '.' for current)"
 Installs: 
-  ./script/skynet
+  ./config/skynet_config.rb
 EOS
     end
 
@@ -58,13 +62,16 @@ EOS
       opts.on("--mysql", 
              "Include mysql migration if you want to use mysql as your message queue.  
              Installs:
-             ./db/skynet_schema.sql
+             ./db/skynet_mysql_schema.sql
              ./db/migrate/db/migrate/###_create_skynet_tables.rb
              ") do |mysql|
                options[:mysql] = true if mysql
              end
       opts.on("-r", "--rails",
-              "Install into rails app",
+              "Install into rails app.
+              Installs:
+              ./config/initializers/skynet.rb
+              (If using rails 1, make sure to add require 'skynet' to your environment.rb)",
               "Default: false") do |rails| 
                 options[:rails] = true if rails
               end
@@ -81,8 +88,7 @@ EOS
     # Installation skeleton.  Intermediate directories are automatically
     # created so don't sweat their absence here.
     BASEDIRS = %w(
-      db
+      config
       log
-      script
     )
 end

data/app_generators/skynet_install/templates/migration.rb

@@ -1,25 +1,5 @@
 class <%= migration_name %> < ActiveRecord::Migration
   def self.up
-    create_table :skynet_worker_queues do |t|
-      t.column  :id,            "bigint unsigned primary key"
-      t.column  :queue_id,      :integer, :default => 0      
-      t.column  :created_on,    :timestamp
-      t.column  :updated_on,    :timestamp
-      t.column  :tasktype,      :string
-      t.column  :tasksubtype,   :string
-      t.column  :worker_id,     'bigint unsigned'
-      t.column  :hostname,      :string
-      t.column  :process_id,    :integer
-      t.column  :job_id,        'bigint unsigned'
-      t.column  :task_id,       'bigint unsigned'
-      t.column  :iteration,     :integer
-      t.column  :name,          :string
-      t.column  :map_or_reduce, :string
-      t.column  :started_at,    "decimal(16,4)"
-      t.column  :version,       :integer
-      t.column  :processed,     :integer
-      t.column  :timeout,       "decimal(16,4)"            
-    end                                 
     create_table :skynet_message_queues do |t|
       t.column  :id,            "bigint unsigned primary key"
       t.column  :queue_id,      :integer, :default => 0
@@ -29,7 +9,7 @@ class <%= migration_name %> < ActiveRecord::Migration
       t.column  :tasktype,      :string
       t.column  :task_id,       'bigint unsigned'
       t.column  :job_id,        'bigint unsigned'
-      t.column  :raw_payload,   :text
+      t.column  :raw_payload,   "longtext"
       t.column  :payload_type,  :string
       t.column  :name,          :string
       t.column  :expiry,        :integer
@@ -51,15 +31,12 @@ class <%= migration_name %> < ActiveRecord::Migration
     add_index :skynet_message_queues, :task_id
     add_index :skynet_message_queues, :tran_id, :unique => true
     add_index :skynet_message_queues, [:queue_id,:tasktype,:payload_type,:expire_time], :name => "index_skynet_mqueue_for_take"
-    add_index :skynet_worker_queues, [:hostname, :process_id]
-    add_index :skynet_worker_queues, :worker_id, :unique=> true
     execute "insert into skynet_queue_temperature (queue_id,type) values (0,'master')"
     execute "insert into skynet_queue_temperature (queue_id,type) values (0,'any')"
     execute "insert into skynet_queue_temperature (queue_id,type) values (0,'task')"
   end
 
   def self.down
-    drop_table :skynet_worker_queues
     drop_table :skynet_queue_temperature
     drop_table :skynet_message_queues
   end