brendan-skynet 0.9.33 → 0.9.303

data/examples/dgrep/data/testfile1.txt

@@ -1 +0,0 @@
-This is a test file with some data

data/examples/dgrep/data/testfile2.txt

@@ -1 +0,0 @@
-Another file with more data and more data

data/examples/dgrep/data/testfile3.txt

@@ -1 +0,0 @@
-One more file with no D ata

data/examples/dgrep/data/testfile4.txt

@@ -1 +0,0 @@
-Some thing data

data/examples/dgrep/lib/dgrep.rb

@@ -1,59 +0,0 @@
-class Dgrep
-  include SkynetDebugger
-
-  # Takes an array that looks like [ [filename1,["word1","word2"]], [filename2,["word1","word2"]] ]
-  # Returns an array that looks like [ [word1, cnt], [word2,cnt], [word1, cnt] ]
-  # Map gets an array and should return an array
-  def self.map(map_datas)
-    results = []
-    map_datas.each do |filename,words|
-      begin
-        words.each do |word|
-          cnt = File.read(filename).scan(/\W#{word}\W/i).size
-          results << [word,cnt] if cnt and cnt > 0
-        end
-      rescue Errno::EISDIR
-        # skip directories
-      end
-    end
-    return results if results.any?
-  end
-
-  # The chosen reduce partiioner attempts group keys by partition.  For example, if you choose 2 words and 2 reducers it will make sure
-  # the first word goes to the first reducer and the second word to the second reducer.  If you only have 1 reducer (the defaukt)
-  # It doesn't bother partitioning.  You can write as complex a partiioner as you'd like.
-  def self.reduce_partition(post_map_data,new_partitions)
-    Skynet::Partitioners::ArrayDataSplitByFirstEntry.reduce_partition(post_map_data,new_partitions)
-
-    ## I've included the contents of the above method so you can see how to write a reduce_partitioner
-    # partitions = []
-    # (0..new_partitions - 1).each { |i| partitions[i] = Array.new }
-    # keys_seen = {}
-    # post_map_data.each do |partition|
-    #   partition.each do |array|
-    #     next unless array.is_a?(Array) and array.size >= 2
-    #     if array[0].kind_of?(Fixnum)
-    #       key = array[0]
-    #     else
-    #       keys_seen[array[0]] ||= keys_seen.keys.size
-    #       key = keys_seen[array[0]]
-    #     end
-    #     partitions[key % new_partitions] << array
-    #   end
-    # end
-    # partitions
-
-  end
-
-  # Takes an array that looks like  [ [word1, cnt], [word2,cnt], [word1, cnt] ]
-  # reduce also gets an array
-  def self.reduce(reduce_datas)
-    results = {}
-    reduce_datas.each do |reduce_data|
-      results[reduce_data[0]] ||= 0
-      results[reduce_data[0]] += reduce_data[1]
-    end
-    results
-  end
-
-end

data/examples/dgrep/lib/mapreduce_test.rb

@@ -1,32 +0,0 @@
-class MapreduceTest
-  
-  include SkynetDebugger
-  
-  def self.run
-    job = Skynet::Job.new(
-      :mappers          => 2, 
-      :reducers         => 1,
-      :map_reduce_class => self,
-      :map_data         => [OpenStruct.new({:created_by => 2}),OpenStruct.new({:created_by => 2}),OpenStruct.new({:created_by => 3})]
-    )    
-    results = job.run
-  end
-
-  def self.map(profiles)
-    result = Array.new
-    profiles.each do |profile|
-      result << [profile.created_by, 1] if profile.created_by
-    end
-    result
-  end
-
-  def self.reduce(pairs)
-    totals = Hash.new
-    pairs.each do |pair|
-      created_by, count = pair[0], pair[1]
-      totals[created_by] ||= 0
-      totals[created_by] += count
-    end
-    totals
-  end
-end

data/examples/dgrep/lib/most_common_words.rb

@@ -1,45 +0,0 @@
-class MostCommonWords
-  include SkynetDebugger
-
-  def self.map(map_datas)
-    results = Hash.new(0)
-    map_datas.each do |filename|
-      begin
-        File.open(filename).each_word do |word|
-          next if word.length <= 5
-          results[word.downcase] += 1
-        end
-      rescue Errno::EISDIR
-        # skip directories
-      end
-    end
-    ## For efficiency.  Not 100% accurate.
-    results.sort{|a,b| a[1]<=>b[1]}.reverse.first(100)
-  end
-
-  # The chosen reduce partiioner attempts group keys by partition.  For example, if you choose 2 words and 2 reducers it will make sure
-  # the first word goes to the first reducer and the second word to the second reducer.  If you only have 1 reducer (the defaukt)
-  # It doesn't bother partitioning.  You can write as complex a partiioner as you'd like.
-  def self.reduce_partition(post_map_data,new_partitions)
-    Skynet::Partitioners::ArrayDataSplitByFirstEntry.reduce_partition(post_map_data,new_partitions)
-  end
-
-  # Takes an array that looks like  [ [word1, cnt], [word2,cnt], [word1, cnt] ]
-  # reduce also gets an array
-  def self.reduce(reduce_datas)
-    results = Hash.new(0)
-    reduce_datas.each do |reduce_data|
-      results[reduce_data[0]] += reduce_data[1]
-    end
-    results.sort{|a,b| a[1]<=>b[1]}.reverse.first(10).sort
-  end
-
-end
-
-class IO
-  def each_word(&b)
-    each do |line|
-     line.scan(/\w+/, &b)
-    end
-  end
-end

data/examples/dgrep/script/dgrep

@@ -1,75 +0,0 @@
-#!/usr/bin/env ruby
-# USAGE:
-#
-# You can provide one or more words. The last argument is the directory.
-#
-# Eg:
-#
-# ruby ./script/dgrep [-r] word1 word2 word3 directory
-#
-# Passing -r means search recursively
-#
-# Most Common Words
-# I've included a second demo in the form of:
-# ruby ./script/dgrep -r -mcw directory
-# This will return the top 10 most common 5 or more letter words
-#
-# You can also choose to distribute the reduce step by passing --reducers=2  (or any other number. 1 is the default)
-
-require 'rubygems'
-require 'rdoc/usage'
-require 'skynet'
-require File.dirname(__FILE__) + '/../config/skynet_config.rb'
-
-puts "LOGGING TO #{Skynet.config.logfile_location}"
-
-RDoc::usage if ARGV.empty? or ARGV.include?("--help")
-
-reducers = 1
-if reduce_arg = ARGV.detect {|i| i =~ /--reducers/ }
-  reducers = reduce_arg.split('=').last.to_i
-  ARGV.delete_if {|i| i =~ /--reducers/ }  
-end
-
-user_dir = ARGV.pop
-directory = File.expand_path(user_dir)
-
-mcw = ARGV.delete('-mcw') ? true : false
-
-files = if ARGV.delete('-r')
-   Dir["#{directory}/**/*"]
-else
-  Dir["#{directory}/*"]
-end                             
-
-if files.empty?
-  puts "No files in #{user_dir}"
-  exit
-end
-
-words = ARGV
-
-results = []
-if mcw
-  puts "Looking for the most common words in #{user_dir}"
-  results = files.mapreduce(MostCommonWords, :reducers => reducers, :master_timeout => 60)
-  if results and results.any?
-    puts "RESULTS:"
-    results.sort_by{|a| a[1]}.reverse.each_with_index do |a,ii|
-      puts "#{ii}. #{a[1]} #{a[0]}"
-    end
-  else
-    puts "No results"
-  end
-else
-  map_data = files.collect{|file| [file,words]}
-  puts "Looking at all the files in #{user_dir} for the word or words '#{words.join(',')}'"
-  results = map_data.mapreduce(Dgrep, :data_debug => true)
-  if results and results.any?
-    results.sort_by{|a| a[1]}.reverse.each_with_index do |a,ii|
-      puts "#{ii}. #{a[1]} #{a[0]}"
-    end
-  else
-    puts "No results"
-  end
-end

data/examples/rails_mysql_example/README

@@ -1,66 +0,0 @@
- ========================
- = Skynet Rails Example =
- ========================
- 
-Below you'll find a few example skynet uses within a rails application.
-Rails itself is not included in the app so you will need Rails 2 installed locally.
-You will also need to have mysql running locally.  If you have special permissions, please edit config/database.yml.
-
-SETUP
--------------------------------------
-> rake db:create
-> rake db:migrate
-> rake db:initialize_defaults
-
-
-EXAMPLE 1: Complex Migrations
--------------------------------------
-Lets say we were storing the user's favorites in a single comma separated field in the users table, but want
-to normalize it into a user_favorites table.  This is something that would be difficult to do in straight SQL.
-We have a method in our User model to migrate a single user.  All we have to do is run it on every user.
-
-I added a rake task to run the skynet migration
-> rake db:migrate_favorites
-
-You can also choose to run the command (or other commands) yourself in the skynet console
-                                                                                         
-> ruby script/skynet console
-console> User.distributed_find(:all).map(:migrate_favorites)
-console> u = User.find(:first)
-console> u.user_favorites.size
-
-If you want to re-run the example, remember to run rake db:initialize_defaults again.
-
-
-
-Example 2: Sending Email Asyncronously
--------------------------------------
-> ruby script/skynet console
-console> u = User.find(:first)
-console> u.send_welcome_email 
-# pauses 5 seconds while the user goes to another site and never comes back We don't want that!
-# try again
-console> u.send_later(:send_welcome_email)
-# returns immediately, but sends the email after 5 seconds anyway
-
-This sample application was created by creating in the following way
-
-1. Create new rails app
- > rails -d mysql rails_mysql_example
-
-2. Install Skynet with rails and mysql support
- > skynet_install --rails --mysql
-
-3. Run skynet migrations
- > rake db:migrate
- 
-4. Create other migrations models
- > ./script/generate model user name:string password:string favorites:string email:string
- > ./script/generate model user_favorite user_id:integer favorite:string
-
-------------------------------------------------------
-If you'd like get a peak at what Skynet is doing, edit ./script/skynet and set
-
-Skynet::CONFIG[:SKYNET_LOG_LEVEL] = Logger::INFO # or Logger::INFO
-
-and tail ./log/skynet_development.log

data/examples/rails_mysql_example/Rakefile

@@ -1,10 +0,0 @@
-# Add your own tasks in files placed in lib/tasks ending in .rake,
-# for example lib/tasks/capistrano.rake, and they will automatically be available to Rake.
-
-require(File.join(File.dirname(__FILE__), 'config', 'boot'))
-
-require 'rake'
-require 'rake/testtask'
-require 'rake/rdoctask'
-
-require 'tasks/rails'

data/examples/rails_mysql_example/app/controllers/application.rb

@@ -1,10 +0,0 @@
-# Filters added to this controller apply to all controllers in the application.
-# Likewise, all the methods added will be available for all controllers.
-
-class ApplicationController < ActionController::Base
-  helper :all # include all helpers, all the time
-
-  # See ActionController::RequestForgeryProtection for details
-  # Uncomment the :secret if you're not using the cookie session store
-  protect_from_forgery # :secret => '8a54ccda625a3eeb4b2746aaaa094406'
-end

data/examples/rails_mysql_example/app/helpers/application_helper.rb

@@ -1,3 +0,0 @@
-# Methods added to this helper will be available to all templates in the application.
-module ApplicationHelper
-end

data/examples/rails_mysql_example/app/models/user.rb

@@ -1,21 +0,0 @@
-class User < ActiveRecord::Base
-  
-  has_many :user_favorites, :dependent => :destroy
-  
-  # Moves comma seperated column into seperate relationship table
-  def migrate_favorites
-    return unless self.favorites
-    self.favorites.split(",").each do |favorite|
-      self.user_favorites.create(:favorite => favorite)
-    end
-    self.favorites = nil
-    self.save
-  end
-  
-  # Slow running welcome email sender
-  def send_welcome_email
-    sleep 5
-    UserMailer.deliver_welcome(self)
-  end
-end
-                       

data/examples/rails_mysql_example/app/models/user_favorite.rb

@@ -1,5 +0,0 @@
-class UserFavorite < ActiveRecord::Base
-  
-  belongs_to :user
-  
-end

data/examples/rails_mysql_example/app/models/user_mailer.rb

@@ -1,12 +0,0 @@
-class UserMailer < ActionMailer::Base  
-  class Error < StandardError; end
-
-  def welcome(user)
-    @recipients   = "#{user.email}"
-    @from         = "Mycompany <root@localhost>"
-    @sent_on      = Time.now
-    @subject      = 'Your account has been activated!'
-    @body[:url]   = "http://localhost:3000/"
-    @body[:user]  = user
-  end
-end

data/examples/rails_mysql_example/app/views/user_mailer/welcome.erb

@@ -1,5 +0,0 @@
-Dear <%= @user.name %>,
-
-Welcome to this site!
-
-<%= @url %>

data/examples/rails_mysql_example/config/boot.rb

@@ -1,109 +0,0 @@
-# Don't change this file!
-# Configure your app in config/environment.rb and config/environments/*.rb
-
-RAILS_ROOT = "#{File.dirname(__FILE__)}/.." unless defined?(RAILS_ROOT)
-
-module Rails
-  class << self
-    def boot!
-      unless booted?
-        preinitialize
-        pick_boot.run
-      end
-    end
-
-    def booted?
-      defined? Rails::Initializer
-    end
-
-    def pick_boot
-      (vendor_rails? ? VendorBoot : GemBoot).new
-    end
-
-    def vendor_rails?
-      File.exist?("#{RAILS_ROOT}/vendor/rails")
-    end
-
-    # FIXME : Ruby 1.9
-    def preinitialize
-      load(preinitializer_path) if File.exists?(preinitializer_path)
-    end
-
-    def preinitializer_path
-      "#{RAILS_ROOT}/config/preinitializer.rb"
-    end
-  end
-
-  class Boot
-    def run
-      load_initializer
-      Rails::Initializer.run(:set_load_path)
-    end
-  end
-
-  class VendorBoot < Boot
-    def load_initializer
-      require "#{RAILS_ROOT}/vendor/rails/railties/lib/initializer"
-    end
-  end
-
-  class GemBoot < Boot
-    def load_initializer
-      self.class.load_rubygems
-      load_rails_gem
-      require 'initializer'
-    end
-
-    def load_rails_gem
-      if version = self.class.gem_version
-        gem 'rails', version
-      else
-        gem 'rails'
-      end
-    rescue Gem::LoadError => load_error
-      $stderr.puts %(Missing the Rails #{version} gem. Please `gem install -v=#{version} rails`, update your RAILS_GEM_VERSION setting in config/environment.rb for the Rails version you do have installed, or comment out RAILS_GEM_VERSION to use the latest version installed.)
-      exit 1
-    end
-
-    class << self
-      def rubygems_version
-        Gem::RubyGemsVersion if defined? Gem::RubyGemsVersion
-      end
-
-      def gem_version
-        if defined? RAILS_GEM_VERSION
-          RAILS_GEM_VERSION
-        elsif ENV.include?('RAILS_GEM_VERSION')
-          ENV['RAILS_GEM_VERSION']
-        else
-          parse_gem_version(read_environment_rb)
-        end
-      end
-
-      def load_rubygems
-        require 'rubygems'
-
-        unless rubygems_version >= '0.9.4'
-          $stderr.puts %(Rails requires RubyGems >= 0.9.4 (you have #{rubygems_version}). Please `gem update --system` and try again.)
-          exit 1
-        end
-
-      rescue LoadError
-        $stderr.puts %(Rails requires RubyGems >= 0.9.4. Please install RubyGems and try again: http://rubygems.rubyforge.org)
-        exit 1
-      end
-
-      def parse_gem_version(text)
-        $1 if text =~ /^[^#]*RAILS_GEM_VERSION\s*=\s*["']([!~<>=]*\s*[\d.]+)["']/
-      end
-
-      private
-        def read_environment_rb
-          File.read("#{RAILS_ROOT}/config/environment.rb")
-        end
-    end
-  end
-end
-
-# All that for this:
-Rails.boot!