skynet 0.9.2 → 0.9.3

data/examples/dgrep/data/testfile1.txt

@@ -0,0 +1 @@
+This is a test file with some data

data/examples/dgrep/data/testfile2.txt

@@ -0,0 +1 @@
+Another file with more data and more data

data/examples/dgrep/data/testfile3.txt

@@ -0,0 +1 @@
+One more file with no D ata

data/examples/dgrep/data/testfile4.txt

@@ -0,0 +1 @@
+Some thing data

data/examples/dgrep/lib/dgrep.rb

@@ -0,0 +1,59 @@
+class Dgrep
+  include SkynetDebugger
+
+  # Takes an array that looks like [ [filename1,["word1","word2"]], [filename2,["word1","word2"]] ]
+  # Returns an array that looks like [ [word1, cnt], [word2,cnt], [word1, cnt] ]
+  # Map gets an array and should return an array
+  def self.map(map_datas)
+    results = []
+    map_datas.each do |filename,words|
+      begin
+        words.each do |word|
+          cnt = File.read(filename).scan(/\W#{word}\W/i).size
+          results << [word,cnt] if cnt and cnt > 0
+        end
+      rescue Errno::EISDIR
+        # skip directories
+      end
+    end
+    return results if results.any?
+  end
+
+  # The chosen reduce partiioner attempts group keys by partition.  For example, if you choose 2 words and 2 reducers it will make sure
+  # the first word goes to the first reducer and the second word to the second reducer.  If you only have 1 reducer (the defaukt)
+  # It doesn't bother partitioning.  You can write as complex a partiioner as you'd like.
+  def self.reduce_partition(post_map_data,new_partitions)
+    Skynet::Partitioners::ArrayDataSplitByFirstEntry.reduce_partition(post_map_data,new_partitions)
+
+    ## I've included the contents of the above method so you can see how to write a reduce_partitioner
+    # partitions = []
+    # (0..new_partitions - 1).each { |i| partitions[i] = Array.new }
+    # keys_seen = {}
+    # post_map_data.each do |partition|
+    #   partition.each do |array|
+    #     next unless array.is_a?(Array) and array.size >= 2
+    #     if array[0].kind_of?(Fixnum)
+    #       key = array[0]
+    #     else
+    #       keys_seen[array[0]] ||= keys_seen.keys.size
+    #       key = keys_seen[array[0]]
+    #     end
+    #     partitions[key % new_partitions] << array
+    #   end
+    # end
+    # partitions
+
+  end
+
+  # Takes an array that looks like  [ [word1, cnt], [word2,cnt], [word1, cnt] ]
+  # reduce also gets an array
+  def self.reduce(reduce_datas)
+    results = {}
+    reduce_datas.each do |reduce_data|
+      results[reduce_data[0]] ||= 0
+      results[reduce_data[0]] += reduce_data[1]
+    end
+    results
+  end
+
+end

data/examples/dgrep/lib/mapreduce_test.rb

@@ -0,0 +1,32 @@
+class MapreduceTest
+  
+  include SkynetDebugger
+  
+  def self.run
+    job = Skynet::Job.new(
+      :mappers          => 2, 
+      :reducers         => 1,
+      :map_reduce_class => self,
+      :map_data         => [OpenStruct.new({:created_by => 2}),OpenStruct.new({:created_by => 2}),OpenStruct.new({:created_by => 3})]
+    )    
+    results = job.run
+  end
+
+  def self.map(profiles)
+    result = Array.new
+    profiles.each do |profile|
+      result << [profile.created_by, 1] if profile.created_by
+    end
+    result
+  end
+
+  def self.reduce(pairs)
+    totals = Hash.new
+    pairs.each do |pair|
+      created_by, count = pair[0], pair[1]
+      totals[created_by] ||= 0
+      totals[created_by] += count
+    end
+    totals
+  end
+end

data/examples/dgrep/lib/most_common_words.rb

@@ -0,0 +1,45 @@
+class MostCommonWords
+  include SkynetDebugger
+
+  def self.map(map_datas)
+    results = Hash.new(0)
+    map_datas.each do |filename|
+      begin
+        File.open(filename).each_word do |word|
+          next if word.length <= 5
+          results[word.downcase] += 1
+        end
+      rescue Errno::EISDIR
+        # skip directories
+      end
+    end
+    ## For efficiency.  Not 100% accurate.
+    results.sort{|a,b| a[1]<=>b[1]}.reverse.first(100)
+  end
+
+  # The chosen reduce partiioner attempts group keys by partition.  For example, if you choose 2 words and 2 reducers it will make sure
+  # the first word goes to the first reducer and the second word to the second reducer.  If you only have 1 reducer (the defaukt)
+  # It doesn't bother partitioning.  You can write as complex a partiioner as you'd like.
+  def self.reduce_partition(post_map_data,new_partitions)
+    Skynet::Partitioners::ArrayDataSplitByFirstEntry.reduce_partition(post_map_data,new_partitions)
+  end
+
+  # Takes an array that looks like  [ [word1, cnt], [word2,cnt], [word1, cnt] ]
+  # reduce also gets an array
+  def self.reduce(reduce_datas)
+    results = Hash.new(0)
+    reduce_datas.each do |reduce_data|
+      results[reduce_data[0]] += reduce_data[1]
+    end
+    results.sort{|a,b| a[1]<=>b[1]}.reverse.first(10).sort
+  end
+
+end
+
+class IO
+  def each_word(&b)
+    each do |line|
+     line.scan(/\w+/, &b)
+    end
+  end
+end

data/examples/dgrep/script/dgrep

@@ -0,0 +1,75 @@
+#!/usr/bin/env ruby
+# USAGE:
+#
+# You can provide one or more words. The last argument is the directory.
+#
+# Eg:
+#
+# ruby ./script/dgrep [-r] word1 word2 word3 directory
+#
+# Passing -r means search recursively
+#
+# Most Common Words
+# I've included a second demo in the form of:
+# ruby ./script/dgrep -r -mcw directory
+# This will return the top 10 most common 5 or more letter words
+#
+# You can also choose to distribute the reduce step by passing --reducers=2  (or any other number. 1 is the default)
+
+require 'rubygems'
+require 'rdoc/usage'
+require 'skynet'
+require File.dirname(__FILE__) + '/../config/skynet_config.rb'
+
+puts "LOGGING TO #{Skynet.config.logfile_location}"
+
+RDoc::usage if ARGV.empty? or ARGV.include?("--help")
+
+reducers = 1
+if reduce_arg = ARGV.detect {|i| i =~ /--reducers/ }
+  reducers = reduce_arg.split('=').last.to_i
+  ARGV.delete_if {|i| i =~ /--reducers/ }  
+end
+
+user_dir = ARGV.pop
+directory = File.expand_path(user_dir)
+
+mcw = ARGV.delete('-mcw') ? true : false
+
+files = if ARGV.delete('-r')
+   Dir["#{directory}/**/*"]
+else
+  Dir["#{directory}/*"]
+end                             
+
+if files.empty?
+  puts "No files in #{user_dir}"
+  exit
+end
+
+words = ARGV
+
+results = []
+if mcw
+  puts "Looking for the most common words in #{user_dir}"
+  results = files.mapreduce(MostCommonWords, :reducers => reducers, :master_timeout => 60)
+  if results and results.any?
+    puts "RESULTS:"
+    results.sort_by{|a| a[1]}.reverse.each_with_index do |a,ii|
+      puts "#{ii}. #{a[1]} #{a[0]}"
+    end
+  else
+    puts "No results"
+  end
+else
+  map_data = files.collect{|file| [file,words]}
+  puts "Looking at all the files in #{user_dir} for the word or words '#{words.join(',')}'"
+  results = map_data.mapreduce(Dgrep, :data_debug => true)
+  if results and results.any?
+    results.sort_by{|a| a[1]}.reverse.each_with_index do |a,ii|
+      puts "#{ii}. #{a[1]} #{a[0]}"
+    end
+  else
+    puts "No results"
+  end
+end

data/examples/rails_mysql_example/README

@@ -0,0 +1,66 @@
+ ========================
+ = Skynet Rails Example =
+ ========================
+ 
+Below you'll find a few example skynet uses within a rails application.
+Rails itself is not included in the app so you will need Rails 2 installed locally.
+You will also need to have mysql running locally.  If you have special permissions, please edit config/database.yml.
+
+SETUP
+-------------------------------------
+> rake db:create
+> rake db:migrate
+> rake db:initialize_defaults
+
+
+EXAMPLE 1: Complex Migrations
+-------------------------------------
+Lets say we were storing the user's favorites in a single comma separated field in the users table, but want
+to normalize it into a user_favorites table.  This is something that would be difficult to do in straight SQL.
+We have a method in our User model to migrate a single user.  All we have to do is run it on every user.
+
+I added a rake task to run the skynet migration
+> rake db:migrate_favorites
+
+You can also choose to run the command (or other commands) yourself in the skynet console
+                                                                                         
+> ruby script/skynet console
+console> User.distributed_find(:all).map(:migrate_favorites)
+console> u = User.find(:first)
+console> u.user_favorites.size
+
+If you want to re-run the example, remember to run rake db:initialize_defaults again.
+
+
+
+Example 2: Sending Email Asyncronously
+-------------------------------------
+> ruby script/skynet console
+console> u = User.find(:first)
+console> u.send_welcome_email 
+# pauses 5 seconds while the user goes to another site and never comes back We don't want that!
+# try again
+console> u.send_later(:send_welcome_email)
+# returns immediately, but sends the email after 5 seconds anyway
+
+This sample application was created by creating in the following way
+
+1. Create new rails app
+ > rails -d mysql rails_mysql_example
+
+2. Install Skynet with rails and mysql support
+ > skynet_install --rails --mysql
+
+3. Run skynet migrations
+ > rake db:migrate
+ 
+4. Create other migrations models
+ > ./script/generate model user name:string password:string favorites:string email:string
+ > ./script/generate model user_favorite user_id:integer favorite:string
+
+------------------------------------------------------
+If you'd like get a peak at what Skynet is doing, edit ./script/skynet and set
+
+Skynet::CONFIG[:SKYNET_LOG_LEVEL] = Logger::INFO # or Logger::INFO
+
+and tail ./log/skynet_development.log

data/examples/rails_mysql_example/Rakefile

@@ -0,0 +1,10 @@
+# Add your own tasks in files placed in lib/tasks ending in .rake,
+# for example lib/tasks/capistrano.rake, and they will automatically be available to Rake.
+
+require(File.join(File.dirname(__FILE__), 'config', 'boot'))
+
+require 'rake'
+require 'rake/testtask'
+require 'rake/rdoctask'
+
+require 'tasks/rails'

data/examples/rails_mysql_example/app/controllers/application.rb

@@ -0,0 +1,10 @@
+# Filters added to this controller apply to all controllers in the application.
+# Likewise, all the methods added will be available for all controllers.
+
+class ApplicationController < ActionController::Base
+  helper :all # include all helpers, all the time
+
+  # See ActionController::RequestForgeryProtection for details
+  # Uncomment the :secret if you're not using the cookie session store
+  protect_from_forgery # :secret => '8a54ccda625a3eeb4b2746aaaa094406'
+end

data/examples/rails_mysql_example/app/helpers/application_helper.rb

@@ -0,0 +1,3 @@
+# Methods added to this helper will be available to all templates in the application.
+module ApplicationHelper
+end

data/examples/rails_mysql_example/app/models/user.rb

@@ -0,0 +1,21 @@
+class User < ActiveRecord::Base
+  
+  has_many :user_favorites, :dependent => :destroy
+  
+  # Moves comma seperated column into seperate relationship table
+  def migrate_favorites
+    return unless self.favorites
+    self.favorites.split(",").each do |favorite|
+      self.user_favorites.create(:favorite => favorite)
+    end
+    self.favorites = nil
+    self.save
+  end
+  
+  # Slow running welcome email sender
+  def send_welcome_email
+    sleep 5
+    UserMailer.deliver_welcome(self)
+  end
+end
+                       

data/examples/rails_mysql_example/app/models/user_favorite.rb

@@ -0,0 +1,5 @@
+class UserFavorite < ActiveRecord::Base
+  
+  belongs_to :user
+  
+end

data/examples/rails_mysql_example/app/models/user_mailer.rb

@@ -0,0 +1,12 @@
+class UserMailer < ActionMailer::Base  
+  class Error < StandardError; end
+
+  def welcome(user)
+    @recipients   = "#{user.email}"
+    @from         = "Mycompany <root@localhost>"
+    @sent_on      = Time.now
+    @subject      = 'Your account has been activated!'
+    @body[:url]   = "http://localhost:3000/"
+    @body[:user]  = user
+  end
+end

data/examples/rails_mysql_example/app/views/user_mailer/welcome.erb

@@ -0,0 +1,5 @@
+Dear <%= @user.name %>,
+
+Welcome to this site!
+
+<%= @url %>

data/examples/rails_mysql_example/config/boot.rb

@@ -0,0 +1,109 @@
+# Don't change this file!
+# Configure your app in config/environment.rb and config/environments/*.rb
+
+RAILS_ROOT = "#{File.dirname(__FILE__)}/.." unless defined?(RAILS_ROOT)
+
+module Rails
+  class << self
+    def boot!
+      unless booted?
+        preinitialize
+        pick_boot.run
+      end
+    end
+
+    def booted?
+      defined? Rails::Initializer
+    end
+
+    def pick_boot
+      (vendor_rails? ? VendorBoot : GemBoot).new
+    end
+
+    def vendor_rails?
+      File.exist?("#{RAILS_ROOT}/vendor/rails")
+    end
+
+    # FIXME : Ruby 1.9
+    def preinitialize
+      load(preinitializer_path) if File.exists?(preinitializer_path)
+    end
+
+    def preinitializer_path
+      "#{RAILS_ROOT}/config/preinitializer.rb"
+    end
+  end
+
+  class Boot
+    def run
+      load_initializer
+      Rails::Initializer.run(:set_load_path)
+    end
+  end
+
+  class VendorBoot < Boot
+    def load_initializer
+      require "#{RAILS_ROOT}/vendor/rails/railties/lib/initializer"
+    end
+  end
+
+  class GemBoot < Boot
+    def load_initializer
+      self.class.load_rubygems
+      load_rails_gem
+      require 'initializer'
+    end
+
+    def load_rails_gem
+      if version = self.class.gem_version
+        gem 'rails', version
+      else
+        gem 'rails'
+      end
+    rescue Gem::LoadError => load_error
+      $stderr.puts %(Missing the Rails #{version} gem. Please `gem install -v=#{version} rails`, update your RAILS_GEM_VERSION setting in config/environment.rb for the Rails version you do have installed, or comment out RAILS_GEM_VERSION to use the latest version installed.)
+      exit 1
+    end
+
+    class << self
+      def rubygems_version
+        Gem::RubyGemsVersion if defined? Gem::RubyGemsVersion
+      end
+
+      def gem_version
+        if defined? RAILS_GEM_VERSION
+          RAILS_GEM_VERSION
+        elsif ENV.include?('RAILS_GEM_VERSION')
+          ENV['RAILS_GEM_VERSION']
+        else
+          parse_gem_version(read_environment_rb)
+        end
+      end
+
+      def load_rubygems
+        require 'rubygems'
+
+        unless rubygems_version >= '0.9.4'
+          $stderr.puts %(Rails requires RubyGems >= 0.9.4 (you have #{rubygems_version}). Please `gem update --system` and try again.)
+          exit 1
+        end
+
+      rescue LoadError
+        $stderr.puts %(Rails requires RubyGems >= 0.9.4. Please install RubyGems and try again: http://rubygems.rubyforge.org)
+        exit 1
+      end
+
+      def parse_gem_version(text)
+        $1 if text =~ /^[^#]*RAILS_GEM_VERSION\s*=\s*["']([!~<>=]*\s*[\d.]+)["']/
+      end
+
+      private
+        def read_environment_rb
+          File.read("#{RAILS_ROOT}/config/environment.rb")
+        end
+    end
+  end
+end
+
+# All that for this:
+Rails.boot!