hot_potato 0.12.1

data/lib/hot_potato/templates/boot.rb

@@ -0,0 +1,21 @@
+require 'rubygems'
+
+APP_PATH ||= File.expand_path('../..', __FILE__)
+RACK_ENV ||= ENV['RACK_ENV'] || 'development'
+
+# Set up gems listed in the Gemfile.
+gemfile = File.expand_path('../../Gemfile', __FILE__)
+begin
+  ENV['BUNDLE_GEMFILE'] = gemfile
+  require 'bundler'
+  Bundler.setup
+  Bundler.require(:default)
+  require File.expand_path('../routes',  __FILE__)
+  require File.expand_path("../environments/#{RACK_ENV}",  __FILE__)
+  Dir["#{File.expand_path('../../app',  __FILE__)}/*.rb"].each { |f| require f }
+rescue Bundler::GemNotFound => e
+  STDERR.puts e.message
+  STDERR.puts "Try running `bundle install`."
+  exit!
+end if File.exist?(gemfile)
+

data/lib/hot_potato/templates/config.yml

@@ -0,0 +1,11 @@
+development:
+  redis_hostname: localhost
+  redis_port: 6379
+
+test:
+  redis_hostname: localhost
+  redis_port: 6379
+
+production:
+  redis_hostname: localhost
+  redis_port: 6379

data/lib/hot_potato/templates/generate

@@ -0,0 +1,4 @@
+#!/usr/bin/env ruby
+
+require File.expand_path('../../config/boot',  __FILE__)
+HotPotato::GenerateAppTask.new

data/lib/hot_potato/templates/routes.rb

@@ -0,0 +1,3 @@
+HotPotato::Route.build do
+
+end

data/lib/hot_potato/templates/supervisor

@@ -0,0 +1,4 @@
+#!/usr/bin/env ruby
+
+require File.expand_path('../../config/boot',  __FILE__)
+HotPotato::SupervisorServer.new

data/lib/hot_potato/templates/template_faucet.rb

@@ -0,0 +1,8 @@
+class __NAME__ < HotPotato::Faucet
+  
+  def perform
+    # TODO: Implement
+    # Be sure to call: send_message(message)
+  end
+  
+end

data/lib/hot_potato/templates/template_sink.rb

@@ -0,0 +1,7 @@
+class __NAME__ < HotPotato::Sink
+  
+  def perform(message)
+    # TODO: Implement
+  end
+  
+end

data/lib/hot_potato/templates/template_worker.rb

@@ -0,0 +1,8 @@
+class __NAME__ < HotPotato::Worker
+  
+  def perform(message)
+    # TODO: Implement
+    # Be sure to call: send_message(message)
+  end
+  
+end

data/lib/hot_potato/utils.rb

@@ -0,0 +1,43 @@
+module Process
+  
+  # Return true if the process is still running, false otherwise.
+  def self.alive?(pid)
+    begin
+      Process.getpgid(pid)
+      return true
+    rescue Errno::ESRCH
+      return false
+    end
+  end
+  
+end
+
+module Kernel
+  
+  # Taken from Rails ActiveSupport::Inflector
+  def classify(name)
+    # strip out any leading schema name
+    camelize(name.to_s.sub(/.*\./, ''))
+  end
+  
+  # Taken from Rails ActiveSupport::Inflector
+  def camelize(lower_case_and_underscored_word, first_letter_in_uppercase = true)
+    if first_letter_in_uppercase
+      lower_case_and_underscored_word.to_s.gsub(/\/(.?)/) { "::#{$1.upcase}" }.gsub(/(?:^|_)(.)/) { $1.upcase }
+    else
+      lower_case_and_underscored_word.to_s[0].chr.downcase + camelize(lower_case_and_underscored_word)[1..-1]
+    end
+  end
+  
+  # Taken from Rails ActiveSupport::Inflector
+  def underscore(camel_cased_word)
+    word = camel_cased_word.to_s.dup
+    word.gsub!(/::/, '/')
+    word.gsub!(/([A-Z]+)([A-Z][a-z])/,'\1_\2')
+    word.gsub!(/([a-z\d])([A-Z])/,'\1_\2')
+    word.tr!("-", "_")
+    word.downcase!
+    word
+  end
+  
+end

data/lib/hot_potato/version.rb

@@ -0,0 +1,3 @@
+module HotPotato
+  VERSION = "0.12.1"
+end

data/lib/hot_potato/worker.rb

@@ -0,0 +1,40 @@
+module HotPotato
+  
+  # Workers manipulate data from other workers or faucets.  Examples include: Calculate Scores, Merge 
+  # Data, and Filter Data.  Each worker is a ruby file in the app directory that extends HotPotato::Worker
+  # and implements the perform(message) method.  For each message the worker wants to send to the next AppTask,
+  # the send_message method should be called.
+  # 
+  #     class Influencer < HotPotato::Worker
+  # 
+  #       def perform(message)
+  #         message["influence"] = rand(100)
+  #         send_message message
+  #       end
+  # 
+  #     end
+  class Worker
+    
+    include HotPotato::AppTask
+    
+    def start(queue_in)
+      @queue_out = underscore(self.class.name.to_sym)
+      if !self.respond_to?('perform')
+        log.error "The Worker #{self.class.name} does not implement a perform method."
+        exit 1
+      end
+      start_heartbeat_service      
+      queue_subscribe(queue_in) do |m|
+        count_message_in
+        perform m
+      end
+    end
+    
+    def send_message(m)
+      queue_inject @queue_out, m
+      count_message_out
+    end
+    
+  end
+  
+end

data/lib/hot_potato.rb

@@ -0,0 +1,20 @@
+APP_PATH ||= '.'
+RACK_ENV ||= 'development'
+
+require 'hot_potato/core'
+require 'hot_potato/cache'
+require 'hot_potato/dsl'
+require 'hot_potato/admin'
+require 'hot_potato/app_task'
+require 'hot_potato/app_task_info'
+require 'hot_potato/app_task_server'
+require 'hot_potato/supervisor_server'
+require 'hot_potato/supervisor_info'
+require 'hot_potato/queue_logger'
+require 'hot_potato/faucet'
+require 'hot_potato/worker'
+require 'hot_potato/sink'
+require 'hot_potato/utils'
+require 'hot_potato/generate'
+require 'hot_potato/generate_app_task'
+require 'hot_potato/version'

data/readme.md

@@ -0,0 +1,219 @@
+# Hot Potato
+
+A Real-time Processing Framework
+
+## Description
+
+Hot Potato is an open source real-time processing framework written in Ruby. Originally designed to process the Twitter firehose at 3,000+ tweets per second, it has been extended to support any type of streaming data as input or output to the framework. The framework excels with applications such as, social media analysis, log processing, fraud prevention, spam detection, instant messaging, and many others that include the processing of streaming data.
+
+## What is it?
+
+* Written in Ruby (requires 1.9)
+* Handles streaming data
+* Designed for scale (can easily handle the twitter firehose on one server)
+* Simple interface for writing AppTasks
+
+# Getting Started
+
+Start by downloading the gem (this requires Ruby 1.9):
+
+		$ gem install hotpotato
+
+Next create a project:
+
+		$ hotpotato sample
+		Hot Potato (v0.12.1)
+		Generating application sample...
+		    create  sample
+		    add     sample/Gemfile
+		    add     sample/Rakefile
+		    create  sample/app
+		    create  sample/bin
+		    add     sample/bin/admin
+		    add     sample/bin/app_task
+		    add     sample/bin/supervisor
+		    create  sample/config
+		    add     sample/config/boot.rb
+		    add     sample/config/config.yml
+		    add     sample/config/routes.rb
+		    create  sample/docs
+		    create  sample/logs
+		    create  sample/test
+		    create  sample/tmp
+
+## Generating AppTasks
+
+To help with creating AppTasks, there is a generator available:
+
+		$ bin/generate [faucet|worker|sink] name
+
+# The Details
+
+## Environments
+
+By setting the RACK_ENV environment variable, one can control which environment is loaded.
+By default there are three: development, test, and production.
+
+## AppTasks
+
+AppTasks are the controllers in the framework.  The Supervisor (See below) is responsible for
+starting AppTasks.  There are three types:
+
+### Faucets
+
+Faucets inject data into the system.  Examples include: Twitter Reader, SMTP, and Tail Log File.
+Each faucet is a ruby file in the app directory that extends HotPotato::Faucet and implements
+the perform method.  For each message received, the method should call the send_message to send
+it to the next AppTask.
+
+		class TwitterFaucet < HotPotato::Faucet
+
+		  def perform
+		    TweetStream::Client.new("user", "secret").sample do |s|
+		      message = {}
+		      message["username"] = s.user.screen_name
+		      message["text"]     = s.text
+		      send_message message
+		    end
+		  end
+
+		end
+
+### Workers
+
+Workers manipulate data from other workers or faucets.  Examples include: Calculate Scores, Merge 
+Data, and Filter Data.  Each worker is a ruby file in the app directory that extends HotPotato::Worker
+and implements the perform(message) method.  For each message the worker wants to send to the next AppTask,
+the send_message method should be called.
+
+		class Influencer < HotPotato::Worker
+  
+		  def perform(message)
+		    message["influence"] = rand(100)
+		    send_message message
+		  end
+
+		end
+
+### Sinks
+
+Sinks send data out of the system.  Examples include: WebSocket, Database (Data Warehouse), and 
+File Writer.  Each sink is a ruby file in the app directory that extends HotPotato::Sink and implements
+the perform(message) method.  There is no send_message for the sink to call since it is a final destination
+for the message.
+
+		class LogWriter < HotPotato::Sink
+  
+		  def perform(message)
+		    log.debug "#{message["username"]}:#{message["influence"]}"
+		  end
+
+		end
+
+## Supervisor
+
+The supervisor is a process that runs on each machine that participates in the cluster.
+When it starts it does the following:
+
+0. Read the routes file
+1. Connect to the Redis server and get the appTask process ID table
+2. Acquire the global lock
+3. If a process is needed, fork a new process for AppTask
+4. Release the global lock
+5. Rinse and Repeat
+
+The supervisor also starts the Heartbeat service and logging service as background threads.
+
+The supervisor can be managed from the command line:
+
+    $ bin/supervisor [run|start|stop|restart]
+
+If started without any settings, it will default to run.
+
+## Admin Server
+
+The admin server is a Sinatra-based application to display statistical and diagnostic information.
+
+The admin server can be managed from the command line:
+
+    $ bin/admin --help
+
+		Usage: ./admin [options]
+
+		Vegas options:
+		  -K, --kill               kill the running process and exit
+		  -S, --status             display the current running PID and URL then quit
+		  -s, --server SERVER      serve using SERVER (thin/mongrel/webrick)
+		  -o, --host HOST          listen on HOST (default: 0.0.0.0)
+		  -p, --port PORT          use PORT (default: 5678)
+		  -x, --no-proxy           ignore env proxy settings (e.g. http_proxy)
+		  -e, --env ENVIRONMENT    use ENVIRONMENT for defaults (default: development)
+		  -F, --foreground         don't daemonize, run in the foreground
+		  -L, --no-launch          don't launch the browser
+		  -d, --debug              raise the log level to :debug (default: :info)
+		      --app-dir APP_DIR    set the app dir where files are stored (default: ~/.vegas/Hot_Potato_Admin_Server)/)
+		  -P, --pid-file PID_FILE  set the path to the pid file (default: app_dir/Hot_Potato_Admin_Server.pid)
+		      --log-file LOG_FILE  set the path to the log file (default: app_dir/Hot_Potato_Admin_Server.log)
+		      --url-file URL_FILE  set the path to the URL file (default: app_dir/Hot_Potato_Admin_Server.url)
+
+		Common options:
+		  -h, --help               Show this message
+		      --version            Show version
+		
+The page can be accessed at http://localhost:5678
+
+## Routes
+
+The routes file (config/routes.rb) is a Ruby DSL that does the following:
+
+* Defines AppTasks (Faucets, Workers, Sinks)
+* Defines processing chain for AppTasks
+* Restrict AppTasks to a host group
+* Limit number of instances
+
+Example:
+
+		HotPotato::Route.build do
+
+		  faucet :twitter_faucet
+		  worker :influencer, :source => :twitter_faucet
+		  sink :log_writer, :source => :influencer
+
+		end
+
+Multiple sources can be attached to a worker or sink:
+
+		worker :influencer, :source => [:twitter_faucet. :other_source]
+
+The number of instances is set to 1.  This can be changed by setting the number of instances:
+
+    worker :influencer, :source => :twitter_faucet, :instances => 2
+
+AppTasks can be limited to a specific server (or set of servers) by creating a group in the 
+config/config.yml file:
+
+		development:
+		  redis_hostname: localhost
+		  redis_port: 6379
+		  servers:
+		    - hostname: worker01
+		      group: incoming
+		      max_app_tasks: 15
+		    - hostname: worker02
+		      group: worker
+		      max_app_tasks: 15
+
+and specifying the group in the routes files:
+
+		faucet :twitter_faucet, :group => :incoming
+
+# Support
+
+If you have a question try our group: http://groups.google.com/group/hotpotato-rb
+
+If you have a bug, file it in GitHub.
+
+# Contributing
+
+[Fork the project](http://github.com/dshimy/HotPotato) and send pull requests.
+

data/test/helper.rb

@@ -0,0 +1,7 @@
+require 'test/unit'
+$LOAD_PATH << File.dirname(__FILE__) + '/../lib/'
+
+APP_PATH ||= File.expand_path('..', __FILE__)
+RACK_ENV ||= 'test'
+
+require 'hot_potato'

data/test/version_test.rb

@@ -0,0 +1,9 @@
+require 'helper'
+
+class VersionTest < Test::Unit::TestCase
+  
+  def test_version
+    assert_not_nil HotPotato::VERSION
+  end
+  
+end

metadata

@@ -0,0 +1,166 @@
+--- !ruby/object:Gem::Specification 
+name: hot_potato
+version: !ruby/object:Gem::Version 
+  prerelease: 
+  version: 0.12.1
+platform: ruby
+authors: 
+- Darian Shimy
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2011-07-26 00:00:00 -07:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: json
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+  type: :runtime
+  version_requirements: *id001
+- !ruby/object:Gem::Dependency 
+  name: redis
+  prerelease: false
+  requirement: &id002 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+  type: :runtime
+  version_requirements: *id002
+- !ruby/object:Gem::Dependency 
+  name: bunny
+  prerelease: false
+  requirement: &id003 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+  type: :runtime
+  version_requirements: *id003
+- !ruby/object:Gem::Dependency 
+  name: sinatra
+  prerelease: false
+  requirement: &id004 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+  type: :runtime
+  version_requirements: *id004
+- !ruby/object:Gem::Dependency 
+  name: vegas
+  prerelease: false
+  requirement: &id005 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+  type: :runtime
+  version_requirements: *id005
+- !ruby/object:Gem::Dependency 
+  name: snappy
+  prerelease: false
+  requirement: &id006 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+  type: :runtime
+  version_requirements: *id006
+description: Hot Potato is an open source real-time processing framework written in Ruby. Originally designed to process the Twitter firehose at 3,000+ tweets per second, it has been extended to support any type of streaming data as input or output to the framework. The framework excels with applications such as, social media analysis, log processing, fraud prevention, spam detection, instant messaging, and many others that include the processing of streaming data.
+email: 
+- dshimy@gmail.com
+executables: 
+- hotpotato
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- .gitignore
+- Gemfile
+- LICENSE
+- Rakefile
+- bin/hotpotato
+- hot_potato.gemspec
+- lib/hot_potato.rb
+- lib/hot_potato/admin.rb
+- lib/hot_potato/admin/public/admin.css
+- lib/hot_potato/admin/views/index.erb
+- lib/hot_potato/app_task.rb
+- lib/hot_potato/app_task_info.rb
+- lib/hot_potato/app_task_server.rb
+- lib/hot_potato/cache.rb
+- lib/hot_potato/core.rb
+- lib/hot_potato/dsl.rb
+- lib/hot_potato/faucet.rb
+- lib/hot_potato/generate.rb
+- lib/hot_potato/generate_app_task.rb
+- lib/hot_potato/queue_logger.rb
+- lib/hot_potato/sink.rb
+- lib/hot_potato/supervisor_info.rb
+- lib/hot_potato/supervisor_server.rb
+- lib/hot_potato/templates/Gemfile
+- lib/hot_potato/templates/Rakefile
+- lib/hot_potato/templates/admin
+- lib/hot_potato/templates/app_task
+- lib/hot_potato/templates/boot.rb
+- lib/hot_potato/templates/config.yml
+- lib/hot_potato/templates/development.rb
+- lib/hot_potato/templates/generate
+- lib/hot_potato/templates/production.rb
+- lib/hot_potato/templates/routes.rb
+- lib/hot_potato/templates/supervisor
+- lib/hot_potato/templates/template_faucet.rb
+- lib/hot_potato/templates/template_sink.rb
+- lib/hot_potato/templates/template_worker.rb
+- lib/hot_potato/templates/test.rb
+- lib/hot_potato/utils.rb
+- lib/hot_potato/version.rb
+- lib/hot_potato/worker.rb
+- readme.md
+- test/helper.rb
+- test/version_test.rb
+has_rdoc: true
+homepage: http://github.com/dshimy/hotpotato
+licenses: []
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+requirements: []
+
+rubyforge_project: hot_potato
+rubygems_version: 1.6.2
+signing_key: 
+specification_version: 3
+summary: A Real-time Processing Framework
+test_files: 
+- test/helper.rb
+- test/version_test.rb