ProcessorPool 1.0.0

data/History.txt

@@ -0,0 +1,6 @@
+== 1.0.0a / 2008-01-27
+
+* 1 major enhancement
+  * Birthday!
+  * Initial check-in of gem
+

data/Manifest.txt

@@ -0,0 +1,13 @@
+.DS_Store
+History.txt
+Manifest.txt
+README.txt
+Rakefile
+bin/processor_pool
+lib/processor/processor.rb
+lib/processor/processor_instance_info.rb
+lib/processor/remote_directory.rb
+lib/processor_pool.rb
+test/helper.rb
+test/test_processor.rb
+test/test_processor_instance_info.rb

data/README.txt

@@ -0,0 +1,123 @@
+ProcessorPool
+    by Ari Lerner and Ron Evans
+    CitrusByte
+    http://blog.citrusbyte.com
+
+== DESCRIPTION:
+  
+Provides a simple load-balancing solution for Amazon's EC2 - S3 backend.
+
+Usage would include an upload server, a image and video processing and transcoding server, all non-blocking separate from an application server.
+
+== FEATURES/PROBLEMS:
+  
+* Need to find a keep-alive solution
+
+== SYNOPSIS:
+
+  require "rubygems"
+  require "lib/processor.rb"
+
+  # Set the access_key_id and the secret_access_key provided by amazon
+  access_key_id, secret_access_key = '11H31W1044QZ9QQF48G2', 'a8Xb46qggIeDfm1y/RUL48Ullq87MxgOKCzCmw1c'
+
+  # And start the processors!
+  ProcessorPool.start(access_key_id, secret_access_key)
+
+  # ProcessorPool is built on top of Sinatra (http://sinatra.rubyforge.org/) and uses the same idioms. 
+  # For more information, check out Sinatra (at http://sinatra.rubyforge.org/):
+
+  get '/' do
+   "Processing data"
+  end
+  
+== IMPLEMENTATION
+  # Uploading and processing  
+  # To use it without inline uploading  
+  # Upload like you normally would do, but in the controller, just pass the variables to the offsite storage and continue on as you would. This won't block the rails mongrel from continuing on. Of course, you also need to check to make sure the file does get uploaded and processed at this point.  
+  # Inline uploading
+  # In Rails, you can redirect the the upload to the processor pool in the form. For instance:
+  
+  <% form_for @media, :id => "fileform", :url => upload_url, :multipart => true do %>
+  
+  # To do the inline uploading, you should call a javascript method to interact with the user to let them know the uploading has begun. For instance:
+  
+  <button class="button" id="buttonUpload" onclick="return beginFileUpload();">Upload</button>
+  
+  <script type="text/javascript">
+  function beginFileUpload()
+  {
+  	// todo: some kind of feedback that upload has started...
+  	$("#uploading").show();
+  ...
+  
+  # For ease of use, we recommend embedding this helper function somewhere:
+  
+  def url_for_upload_server
+    if RAILS_ENV == 'production'
+      # determine the upload server to try to use
+      p = Processors.get_random_processor(::SERVER_POOL_BUCKET)
+      "http://#{p.hostname}:4567/new"
+    else
+      p = Processors.get_random_processor(::SERVER_POOL_BUCKET)
+      "http://localhost:4567/new"
+    end
+  end
+    
+  # The reset is an exercise for you to complete.
+  
+  # Other ideas, using similar methodology would be to upload videos and do the processing there, to run awy sort of processor intensive computations off-site.
+  
+== CONVENIENCE METHODS
+
+  # There are a number of convenience methods included with processor_pool. It is intended to work with Amazon's S3 service.
+  
+  AWS::S3::S3Object.store('/folder/to/greeting.txt', 'hello world!', 'ron', :use_virtual_directories => true)
+  
+  # Alternatively, you can use the included RemoteDirectory class
+  
+  dir = RemoteDirectory.new("/folder/name")
+  
+  # Save one file to the S3 directory
+  dir.save_to_s3("movie.mpg", "movies", "encoded")
+  
+  # Copy all the files in the directory to the s3 bucket
+  dir.copy_to_s3("movies", "encoded")
+  
+  dir.files # Returns a list of all the files in the bucket
+
+== REQUIREMENTS:
+
+* aws/s3
+* net/ping
+* open-uri
+* sinatra
+
+== INSTALL:
+
+* sudo gem install processor_pool
+
+== LICENSE:
+
+(The MIT License)
+
+Copyright (c) 2008 Ari Lerner, CitrusByte and Ron Evans
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+'Software'), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/Rakefile

@@ -0,0 +1,19 @@
+# -*- ruby -*-
+
+require 'rubygems'
+require 'hoe'
+$:.unshift(File.dirname(__FILE__) + "/lib")
+require 'processor_pool.rb'
+
+Hoe.new('ProcessorPool', ProcessorPool::VERSION) do |p|
+  p.rubyforge_name = 'processorpool'
+  p.name = "ProcessorPool"
+  p.author = 'Ari Lerner and Ron Evans'
+  p.email = 'ari.lerner@citrusbyte.com'
+  p.summary = 'Provides a simple load-balancing solution for Amazon\'s EC2 - S3 backend'
+  p.description = p.paragraphs_of('README.txt', 2..5).join("\n\n")
+  p.url = p.paragraphs_of('README.txt', 0).first.split(/\n/)[1..-1]
+  p.changes = p.paragraphs_of('History.txt', 0..1).join("\n\n")
+end
+
+# vim: syntax=Ruby

data/lib/processor/processor.rb

@@ -0,0 +1,107 @@
+require 'rubygems'
+require 'aws/s3'
+require 'net/ping'
+require File.join(File.dirname(__FILE__), *%w[processor_instance_info])
+
+=begin rdoc
+  Processors is the list of Processors from which 
+  a process can choose from.
+=end
+class Processors
+  attr_reader :bucket
+    
+  # returns an array of the available active processors
+  def self.registered(bucket)
+    @bucket = bucket
+    processors = []
+    # get the list of servers that have registered themselves
+    AWS::S3::Bucket.objects(@bucket).each {|o|
+      processors << Processor.new(@bucket, o.key)
+    }
+    processors
+  end
+  
+  # returns a random processor from the pool of processors
+  def self.get_random_processor(bucket)
+    processors = self.registered(bucket)
+    return nil if processors.empty?
+    processors[rand(processors.size)]
+  end
+end
+
+=begin rdoc
+  A Processor is a link in the processing chain. This is where
+  the processor gets chosen and the application runs on top of 
+  the chosen processor
+=end
+class Processor
+  attr_reader :bucket, :hostname
+  
+  def initialize(bucket, hostname = Processor.get_current_hostname)
+    @bucket = bucket
+    @hostname = hostname
+  end
+  
+  # make this processor available to the pool
+  def register
+    AWS::S3::S3Object.store(@hostname, 'ready', @bucket) if @hostname && @bucket
+  end
+  
+  # remove this processor from the pool
+  def unregister
+    AWS::S3::S3Object.delete(@hostname, @bucket) if @hostname && @bucket
+  end
+  
+  # check to see if this processor has been properly registered
+  def registered?
+    AWS::S3::S3Object.exists?(@hostname, @bucket)
+  end
+  
+  # is this processor working?
+  def active?
+    # check to see if this server is pingable
+    return Net::PingExternal.new(@hostname).ping
+  end
+  
+  # equality operator
+  def ==(other)
+    return @hostname == other.hostname
+  end
+  
+  # inequality operator
+  def not_equal?(other)
+    return @hostname != other.hostname
+  end
+  
+  # return the current unix hostname
+  def self.get_current_hostname
+    return `hostname`.chomp
+  end
+  
+  # return the URL needed to try to upload to this processor
+  def url_for
+    "http://#{@hostname}:4567/new"
+  end
+end
+
+=begin rdoc
+  MissingInformation exception is thrown when there is missing information
+  required to run the processor
+=end
+class MissingInformation < Exception
+  @@missing_info_messages = {
+    :access_key_id => "Must provide an access_key_id",
+    :secret_access_key => "Must provide a secret_access_key"
+  }
+  attr_reader :i
+  # Add the missing information key here
+  def initialize(i=nil)
+    super
+    @i = i
+  end
+  # Takes the missing symbol key and translates it to a message
+  def message
+    "Missing information: #{@@missing_info_messages[@i]}"
+  end
+  alias :to_s :message
+end

data/lib/processor/processor_instance_info.rb

@@ -0,0 +1,23 @@
+require 'open-uri'
+
+class ProcessorInstanceInfo
+  def self.host_name
+    @@host_name ||= open('http://169.254.169.254/latest/meta-data/public-hostname').read
+    @@host_name
+  end
+
+  def self.server_pool_bucket
+    @@server_pool_bucket ||= open('http://169.254.169.254/latest/user-data').read.split('|')[0]
+    @@server_pool_bucket
+  end
+  
+  def self.server_host_name
+    @@server_host_name ||= open('http://169.254.169.254/latest/user-data').read.split('|')[1]
+    @@server_host_name
+  end
+  
+  def self.storage_bucket
+    @@storage_bucket ||= open('http://169.254.169.254/latest/user-data').read.split('|')[2]
+    @@storage_bucket
+  end
+end

data/lib/processor/remote_directory.rb

@@ -0,0 +1,85 @@
+require 'fileutils'
+require 'aws/s3'
+
+
+# This is an extension to S3Object that supports the emerging 'standard' for virtual folders on S3.
+# For example:
+#   S3Object.store('/folder/to/greeting.txt', 'hello world!', 'ron', :use_virtual_directories => true)
+#
+# This will create an object in S3 that mimics a folder, as far as the S3 GUI browsers like
+# the S3 Firefox Extension or Bucket Explorer are concerned.
+module AWS
+  module S3
+    class S3Object
+       class << self
+         
+        alias :original_store :store
+        def store(key, data, bucket = nil, options = {})
+          store_folders(key, bucket, options) if options[:use_virtual_directories]
+          original_store(key, data, bucket, options)
+        end
+  
+        def store_folders(key, bucket = nil, options = {})
+          folders = key.split("/")
+          folders.slice!(0)
+          folders.pop
+          current_folder = "/"
+          folders.each {|folder|
+            current_folder += folder
+            store_folder(current_folder, bucket, options)
+            current_folder += "/"
+          }
+        end
+  
+        def store_folder(key, bucket = nil, options = {})
+          original_store(key + "_$folder$", "", bucket, options) # store the magic entry that emulates a folder
+        end
+      end
+    end
+  end
+end
+
+
+
+class RemoteDirectory
+  # pass in the string with the name of directory like "/path/to/dir".
+  def initialize(dirname)
+    @dirname = dirname
+    @deleted = false
+  end
+  
+  # returns all of the files in this directory. wraps Dir.glob.
+  def files
+    result = []
+    return result if @dirname.nil? || @deleted
+    Dir.chdir(@dirname) do
+      result = Dir.glob("*")
+    end
+    result   
+  end
+  
+  # copy each of the files in this directory up to S3. pass in the bucket like "my-bucket", 
+  # and the prefix for storing files within the bucket like "path/to"
+  def copy_to_s3(bucket, prefix, use_virtual_directories=false)
+    files.each {|f|
+      save_file_to_s3(f, bucket, prefix, use_virtual_directories)
+    }
+  end
+  
+  # delete all of the files in this directory, and the directory itself
+  def delete!
+    # todo: implement this convenience method
+    
+    # return if @deleted
+    # FileUtils.rm files
+    # Dir.rmdir(@dirname)
+    # @deleted = true
+  end
+  
+  # save an individual file up to S3 storage
+  def save_file_to_s3(filename, bucket, prefix, use_virtual_directories=false)
+    options = {:access => :public_read}
+    options[:use_virtual_directories] = true if use_virtual_directories
+    AWS::S3::S3Object.store("/" + prefix + "/" + filename, open(File.join(@dirname, filename)), bucket, options)
+  end
+end

data/lib/processor_pool.rb

@@ -0,0 +1,57 @@
+require 'open-uri'
+require "rubygems"
+require "builder"
+require File.join(File.dirname(__FILE__), *%w[processor processor])
+
+class ProcessorPool
+  VERSION = '1.0.0'
+  attr_reader :started
+  
+  Dir["processor/**.rb"].each {|f| require f}
+    
+  class << self
+    # Initiates the S3 connection
+    def start(access_key_id, secret_access_key)
+      raise MissingInformation.new(:access_key_id) unless access_key_id
+      raise MissingInformation.new(:secret_access_key) unless secret_access_key
+
+      get_instance_data
+      establish_connection_with_s3
+
+      current_processor = Processor.new(@@server_pool_bucket, @@host_name)
+      processors = Processors.registered(@@server_pool_bucket)
+      
+      register_self_to_pool
+      check_other_processors_in_pool
+
+      @started = true
+      # Require sinatra for the user to use
+      require "sinatra"
+    end
+    # determine which server pool to use by making a REST query to get the instance metadata
+    def get_instance_data      
+      @@server_pool_bucket = ProcessorInstanceInfo.server_pool_bucket
+      @@host_name = ProcessorInstanceInfo.host_name
+    end    
+    # Establish connection with s3 and user defined details
+    def establish_connection_with_s3
+      AWS::S3::Base.establish_connection!(
+        :access_key_id     => access_key_id,
+        :secret_access_key => secret_access_key,
+        :persistent => false
+      )
+    end
+    # make ourselves available in the pool, if not yet registered
+    def register_self_to_pool      
+      if !processors.include?(current_processor)
+        current_processor.register
+      end
+    end
+    # check all of the registered processors in pool, except ourselves
+    def check_other_processors_in_pool      
+      processors.each {|p|
+        p.unregister if p.not_equal?(current_processor) && !p.active?
+      }
+    end
+  end  
+end

data/test/helper.rb

@@ -0,0 +1,74 @@
+require File.dirname(__FILE__) + '/../lib/processor_pool'
+
+%w(test/spec).each do |library|
+  begin
+    require library
+  rescue
+    STDERR.puts "== Cannot run test without #{library}"
+  end
+end
+
+PRODUCTION_MODE = false
+SERVER_POOL_BUCKET = "processor_pool_bucket_test" # SET THIS
+ACCESS_KEY_ID = '' # AND THIS
+SECRET_ACCESS_KEY = '' # AND FINALLY THIS
+
+AWS::S3::Base.establish_connection!(
+  :access_key_id     => ACCESS_KEY_ID,
+  :secret_access_key => SECRET_ACCESS_KEY
+)
+
+begin
+  Processor.new(SERVER_POOL_BUCKET, "test1").register
+rescue
+  raise Exception.new "You must set the access_key_id and secret_access_key in test/helper.rb to run these"
+end
+
+class Test::Unit::TestCase
+  def file_exists?(temp_directory_prefix, test_id, file)
+    return File.exists?(File.join(temp_directory_prefix, test_id, file))
+  end
+  
+  def temp_directory_prefix
+    File.dirname(__FILE__) + "/temp"
+  end
+  
+  def copy_working_file(test_id, file, newfile=nil)
+    test_id = test_id + "/" + newfile if newfile
+    FileUtils.cp file, temp_directory_prefix + "/" + test_id
+  end
+  
+  def setup_temp_directory(test_id)
+    FileUtils.makedirs(temp_directory_prefix + "/" + test_id) if not File.exists?(temp_directory_prefix + "/" + test_id)
+  end
+  
+  def clean_temp_directory(test_id)
+    FileUtils.rm Dir.glob(File.join(temp_directory_prefix, test_id, "/*"))
+  end
+end
+module Test::Unit::AssertDifference
+  def assert_difference(object, method = nil, difference = 1)
+    initial_value = object.send(method)
+    yield
+    assert_equal initial_value + difference, object.send(method), "#{object}##{method}"
+  end
+
+  def assert_no_difference(object, method, &block)
+    assert_difference object, method, 0, &block
+  end
+end
+# Weird gem fix
+module Kernel
+  def memoize(reload = true, storage = nil)
+    storage = "@#{storage || __method__(1)}"
+    if reload 
+      instance_variable_set(storage, nil)
+    else
+      instance_variable_set(storage, nil)
+      if cache = instance_variable_get(storage)
+        return cache
+      end
+    end
+    instance_variable_set(storage, yield)
+  end
+end

data/test/test_processor.rb

@@ -0,0 +1,67 @@
+require File.dirname(__FILE__) + '/helper'
+
+context "Processors when there are no processors registered" do  
+  specify "should return an array of registered pools when requested" do
+    Processors.registered(SERVER_POOL_BUCKET).class.should == Array
+  end
+  specify "should return an empty array if there are no registered pools" do
+    Processors.registered(SERVER_POOL_BUCKET).empty?.should == true
+  end
+  specify "should return nil if there are no processors when asking for a random processor" do
+    Processors.get_random_processor(SERVER_POOL_BUCKET).should == nil
+  end
+end
+context "Processors when there are processors registered" do
+  before(:all) do
+    Processor.new(SERVER_POOL_BUCKET, "test1").register
+    Processor.new(SERVER_POOL_BUCKET, "test2").register
+    Processor.new(SERVER_POOL_BUCKET, "test3").register
+  end
+  after(:all) do
+    Processors.registered(SERVER_POOL_BUCKET).each {|p|
+      p.unregister
+    }
+  end  
+  specify "should return a non-empty array when there are registered pools" do
+    Processors.registered(SERVER_POOL_BUCKET).empty?.should == false
+  end
+  specify "should return a random processor when asking for a random processor" do
+    Processors.get_random_processor(SERVER_POOL_BUCKET).class.should == Processor
+  end
+end
+context "Processor in general, when registering and unregistering" do
+  setup do
+    @processor = Processor.new(SERVER_POOL_BUCKET)
+  end
+  after(:all) do
+    Processors.registered(SERVER_POOL_BUCKET).each {|p|
+      p.unregister
+    }
+  end
+  specify "should initialize with a bucket" do
+    @processor.bucket.should == SERVER_POOL_BUCKET
+  end
+  specify "should increase the pool size when registering" do
+    cnt = Processors.registered(SERVER_POOL_BUCKET).size
+    @processor.register
+    (cnt + 1).should == Processors.registered(SERVER_POOL_BUCKET).size
+  end
+  specify "should decrease the pool size when unregistering" do
+    @processor.register
+    cnt = Processors.registered(SERVER_POOL_BUCKET).size    
+    @processor.unregister
+    (cnt - 1).should == Processors.registered(SERVER_POOL_BUCKET).size
+  end
+  specify "should register within the pool" do
+    @processor.register
+    Processors.registered(SERVER_POOL_BUCKET).include?(@processor).should == true
+  end
+  specify "should be able to tell if it is registered" do
+    @processor.register
+    @processor.registered?.should == true
+  end
+  specify "should be able to tell that it is not registered" do
+    @processor.unregister
+    @processor.registered?.should == false
+  end
+end

data/test/test_processor_instance_info.rb

@@ -0,0 +1,17 @@
+require File.dirname(__FILE__) + '/helper'
+
+context "Processor Instance Info class methods" do
+  specify "it should respond to the method host_name" do
+    ProcessorInstanceInfo.respond_to?(:host_name).should == true
+  end
+  specify "should respond to the method server_pool_bucket" do
+    ProcessorInstanceInfo.respond_to?(:server_pool_bucket).should == true
+  end
+  specify "should respond to the method server_host_name" do
+    ProcessorInstanceInfo.respond_to?(:server_host_name).should == true
+  end
+  specify "should respond to the method storage_bucket" do
+    ProcessorInstanceInfo.respond_to?(:storage_bucket).should == true
+  end
+end
+

metadata

@@ -0,0 +1,70 @@
+--- !ruby/object:Gem::Specification 
+rubygems_version: 0.9.4
+specification_version: 1
+name: ProcessorPool
+version: !ruby/object:Gem::Version 
+  version: 1.0.0
+date: 2008-02-12 00:00:00 -08:00
+summary: Provides a simple load-balancing solution for Amazon's EC2 - S3 backend
+require_paths: 
+- lib
+email: ari.lerner@citrusbyte.com
+homepage: "    by Ari Lerner and Ron Evans"
+rubyforge_project: processorpool
+description: "Usage would include an upload server, a image and video processing and transcoding server, all non-blocking separate from an application server.  == FEATURES/PROBLEMS:  * Need to find a keep-alive solution  == SYNOPSIS:  require \"rubygems\" require \"lib/processor.rb\""
+autorequire: 
+default_executable: 
+bindir: bin
+has_rdoc: true
+required_ruby_version: !ruby/object:Gem::Version::Requirement 
+  requirements: 
+  - - ">"
+    - !ruby/object:Gem::Version 
+      version: 0.0.0
+  version: 
+platform: ruby
+signing_key: 
+cert_chain: 
+post_install_message: 
+authors: 
+- Ari Lerner and Ron Evans
+files: 
+- .DS_Store
+- History.txt
+- Manifest.txt
+- README.txt
+- Rakefile
+- bin/processor_pool
+- lib/processor/processor.rb
+- lib/processor/processor_instance_info.rb
+- lib/processor/remote_directory.rb
+- lib/processor_pool.rb
+- test/helper.rb
+- test/test_processor.rb
+- test/test_processor_instance_info.rb
+test_files: 
+- test/test_processor.rb
+- test/test_processor_instance_info.rb
+rdoc_options: 
+- --main
+- README.txt
+extra_rdoc_files: 
+- History.txt
+- Manifest.txt
+- README.txt
+executables: 
+- processor_pool
+extensions: []
+
+requirements: []
+
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: hoe
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Version::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 1.5.0
+    version: