hot_potato 0.12.1 → 0.12.2

data/hot_potato.gemspec

@@ -12,12 +12,12 @@ Gem::Specification.new do |s|
   s.summary     = %q{A Real-time Processing Framework}
   s.description = %q{Hot Potato is an open source real-time processing framework written in Ruby. Originally designed to process the Twitter firehose at 3,000+ tweets per second, it has been extended to support any type of streaming data as input or output to the framework. The framework excels with applications such as, social media analysis, log processing, fraud prevention, spam detection, instant messaging, and many others that include the processing of streaming data.}
 
-  s.add_dependency 'json'
-  s.add_dependency 'redis'
-  s.add_dependency 'bunny'
-  s.add_dependency 'sinatra'
-  s.add_dependency "vegas"
-  s.add_dependency "snappy"
+  s.add_dependency 'json',    "~> 1.5.3"
+  s.add_dependency 'redis',   "~> 2.2.1"
+  s.add_dependency 'bunny',   "~> 0.6.0"
+  s.add_dependency 'sinatra', "~> 1.2.6"
+  s.add_dependency "vegas",   "~> 0.1.8"
+  s.add_dependency "snappy",  "~> 0.0.2"
 
   s.rubyforge_project = "hot_potato"
 

data/lib/hot_potato/app_task.rb

@@ -6,32 +6,34 @@ module HotPotato
   # starting AppTasks.  There are three types: Faucets, Workers, and Sinks.
   module AppTask
    
-   HEARTBEAT_INTERVAL = 20
+   HEARTBEAT_INTERVAL   = 20
+   HEARTBEAT_EXPIRE     = 30
+   MESSAGE_COUNT_EXPIRE = 600
    
    include HotPotato::Core
    
    # Used to keep AppTask statistics when a message is received.
    def count_message_in
      stat.incr "hotpotato.counter.apptask.#{Socket.gethostname}.#{self.class.name}.#{Process.pid}.messages_in"
-     stat.expire "hotpotato.counter.apptask.#{Socket.gethostname}.#{self.class.name}.#{Process.pid}.messages_in", 600
+     stat.expire "hotpotato.counter.apptask.#{Socket.gethostname}.#{self.class.name}.#{Process.pid}.messages_in", MESSAGE_COUNT_EXPIRE
    end
    
    # Used to keep AppTask statistics when a message is sent.
    def count_message_out
      stat.incr "hotpotato.counter.apptask.#{Socket.gethostname}.#{self.class.name}.#{Process.pid}.messages_out"     
-     stat.expire "hotpotato.counter.apptask.#{Socket.gethostname}.#{self.class.name}.#{Process.pid}.messages_out", 600
+     stat.expire "hotpotato.counter.apptask.#{Socket.gethostname}.#{self.class.name}.#{Process.pid}.messages_out", MESSAGE_COUNT_EXPIRE
    end
    
    # Starts the HeartBeat service to maintain the process id table.
    def start_heartbeat_service
      ati = AppTaskInfo.new(:classname => self.class.name)
-     stat.set ati.key, ati.to_json, 120
+     stat.set ati.key, ati.to_json, HEARTBEAT_EXPIRE
 
      Thread.new do
        log.info "Thread created for AppTask [Heartbeat]"
        loop do
          ati.touch
-         stat.set ati.key, ati.to_json, 120
+         stat.set ati.key, ati.to_json, HEARTBEAT_EXPIRE
          sleep HEARTBEAT_INTERVAL
        end
      end

data/lib/hot_potato/supervisor_server.rb

@@ -209,14 +209,15 @@ module HotPotato
       options.hostname      = Socket.gethostname
       options.running       = true
       
-      config["servers"].each do |server|
-        if server["hostname"] == options.hostname
-          options.max_app_tasks = server["max_app_tasks"] || MAX_APP_TASKS
-          options.group         = server["group"]         || ""
-          break
+      if config["servers"]
+        config["servers"].each do |server|
+          if server["hostname"] == options.hostname
+            options.max_app_tasks = server["max_app_tasks"] || MAX_APP_TASKS
+            options.group         = server["group"]         || ""
+            break
+          end
         end
       end
-
       return options
     end
 

data/lib/hot_potato/version.rb

@@ -1,3 +1,3 @@
 module HotPotato
-  VERSION = "0.12.1"
+  VERSION = "0.12.2"
 end

data/readme.md

@@ -17,35 +17,41 @@ Hot Potato is an open source real-time processing framework written in Ruby. Ori
 
 Start by downloading the gem (this requires Ruby 1.9):
 
-		$ gem install hotpotato
+```bash
+$ gem install hotpotato
+```
 
 Next create a project:
 
-		$ hotpotato sample
-		Hot Potato (v0.12.1)
-		Generating application sample...
-		    create  sample
-		    add     sample/Gemfile
-		    add     sample/Rakefile
-		    create  sample/app
-		    create  sample/bin
-		    add     sample/bin/admin
-		    add     sample/bin/app_task
-		    add     sample/bin/supervisor
-		    create  sample/config
-		    add     sample/config/boot.rb
-		    add     sample/config/config.yml
-		    add     sample/config/routes.rb
-		    create  sample/docs
-		    create  sample/logs
-		    create  sample/test
-		    create  sample/tmp
+```bash
+$ hotpotato sample
+Hot Potato (v0.12.1)
+Generating application sample...
+    create  sample
+    add     sample/Gemfile
+    add     sample/Rakefile
+    create  sample/app
+    create  sample/bin
+    add     sample/bin/admin
+    add     sample/bin/app_task
+    add     sample/bin/supervisor
+    create  sample/config
+    add     sample/config/boot.rb
+    add     sample/config/config.yml
+    add     sample/config/routes.rb
+    create  sample/docs
+    create  sample/logs
+    create  sample/test
+    create  sample/tmp
+```
 
 ## Generating AppTasks
 
 To help with creating AppTasks, there is a generator available:
 
-		$ bin/generate [faucet|worker|sink] name
+```bash
+$ bin/generate [faucet|worker|sink] name
+```
 
 # The Details
 
@@ -66,18 +72,20 @@ Each faucet is a ruby file in the app directory that extends HotPotato::Faucet a
 the perform method.  For each message received, the method should call the send_message to send
 it to the next AppTask.
 
-		class TwitterFaucet < HotPotato::Faucet
+```ruby
+class TwitterFaucet < HotPotato::Faucet
 
-		  def perform
-		    TweetStream::Client.new("user", "secret").sample do |s|
-		      message = {}
-		      message["username"] = s.user.screen_name
-		      message["text"]     = s.text
-		      send_message message
-		    end
-		  end
+  def perform
+    TweetStream::Client.new("user", "secret").sample do |s|
+      message = {}
+      message["username"] = s.user.screen_name
+      message["text"]     = s.text
+      send_message message
+    end
+  end
 
-		end
+end
+```
 
 ### Workers
 
@@ -86,14 +94,16 @@ Data, and Filter Data.  Each worker is a ruby file in the app directory that ext
 and implements the perform(message) method.  For each message the worker wants to send to the next AppTask,
 the send_message method should be called.
 
-		class Influencer < HotPotato::Worker
-  
-		  def perform(message)
-		    message["influence"] = rand(100)
-		    send_message message
-		  end
+```ruby
+class Influencer < HotPotato::Worker
 
-		end
+  def perform(message)
+    message["influence"] = rand(100)
+    send_message message
+  end
+
+end
+```
 
 ### Sinks
 
@@ -102,13 +112,15 @@ File Writer.  Each sink is a ruby file in the app directory that extends HotPota
 the perform(message) method.  There is no send_message for the sink to call since it is a final destination
 for the message.
 
-		class LogWriter < HotPotato::Sink
-  
-		  def perform(message)
-		    log.debug "#{message["username"]}:#{message["influence"]}"
-		  end
+```ruby
+class LogWriter < HotPotato::Sink
+
+  def perform(message)
+    log.debug "#{message["username"]}:#{message["influence"]}"
+  end
 
-		end
+end
+```
 
 ## Supervisor
 
@@ -126,7 +138,9 @@ The supervisor also starts the Heartbeat service and logging service as backgrou
 
 The supervisor can be managed from the command line:
 
-    $ bin/supervisor [run|start|stop|restart]
+```bash
+$ bin/supervisor [run|start|stop|restart]
+```
 
 If started without any settings, it will default to run.
 
@@ -136,30 +150,32 @@ The admin server is a Sinatra-based application to display statistical and diagn
 
 The admin server can be managed from the command line:
 
-    $ bin/admin --help
-
-		Usage: ./admin [options]
-
-		Vegas options:
-		  -K, --kill               kill the running process and exit
-		  -S, --status             display the current running PID and URL then quit
-		  -s, --server SERVER      serve using SERVER (thin/mongrel/webrick)
-		  -o, --host HOST          listen on HOST (default: 0.0.0.0)
-		  -p, --port PORT          use PORT (default: 5678)
-		  -x, --no-proxy           ignore env proxy settings (e.g. http_proxy)
-		  -e, --env ENVIRONMENT    use ENVIRONMENT for defaults (default: development)
-		  -F, --foreground         don't daemonize, run in the foreground
-		  -L, --no-launch          don't launch the browser
-		  -d, --debug              raise the log level to :debug (default: :info)
-		      --app-dir APP_DIR    set the app dir where files are stored (default: ~/.vegas/Hot_Potato_Admin_Server)/)
-		  -P, --pid-file PID_FILE  set the path to the pid file (default: app_dir/Hot_Potato_Admin_Server.pid)
-		      --log-file LOG_FILE  set the path to the log file (default: app_dir/Hot_Potato_Admin_Server.log)
-		      --url-file URL_FILE  set the path to the URL file (default: app_dir/Hot_Potato_Admin_Server.url)
-
-		Common options:
-		  -h, --help               Show this message
-		      --version            Show version
-		
+```
+$ bin/admin --help
+
+Usage: ./admin [options]
+
+Vegas options:
+  -K, --kill               kill the running process and exit
+  -S, --status             display the current running PID and URL then quit
+  -s, --server SERVER      serve using SERVER (thin/mongrel/webrick)
+  -o, --host HOST          listen on HOST (default: 0.0.0.0)
+  -p, --port PORT          use PORT (default: 5678)
+  -x, --no-proxy           ignore env proxy settings (e.g. http_proxy)
+  -e, --env ENVIRONMENT    use ENVIRONMENT for defaults (default: development)
+  -F, --foreground         don't daemonize, run in the foreground
+  -L, --no-launch          don't launch the browser
+  -d, --debug              raise the log level to :debug (default: :info)
+      --app-dir APP_DIR    set the app dir where files are stored (default: ~/.vegas/Hot_Potato_Admin_Server)/)
+  -P, --pid-file PID_FILE  set the path to the pid file (default: app_dir/Hot_Potato_Admin_Server.pid)
+      --log-file LOG_FILE  set the path to the log file (default: app_dir/Hot_Potato_Admin_Server.log)
+      --url-file URL_FILE  set the path to the URL file (default: app_dir/Hot_Potato_Admin_Server.url)
+
+Common options:
+  -h, --help               Show this message
+      --version            Show version
+```
+	
 The page can be accessed at http://localhost:5678
 
 ## Routes
@@ -173,39 +189,49 @@ The routes file (config/routes.rb) is a Ruby DSL that does the following:
 
 Example:
 
-		HotPotato::Route.build do
+```ruby
+HotPotato::Route.build do
 
-		  faucet :twitter_faucet
-		  worker :influencer, :source => :twitter_faucet
-		  sink :log_writer, :source => :influencer
+  faucet :twitter_faucet
+  worker :influencer, :source => :twitter_faucet
+  sink :log_writer, :source => :influencer
 
-		end
+end
+```
 
 Multiple sources can be attached to a worker or sink:
 
-		worker :influencer, :source => [:twitter_faucet. :other_source]
+```ruby
+worker :influencer, :source => [:twitter_faucet. :other_source]
+```
 
 The number of instances is set to 1.  This can be changed by setting the number of instances:
 
-    worker :influencer, :source => :twitter_faucet, :instances => 2
+```ruby
+worker :influencer, :source => :twitter_faucet, :instances => 2
+```
 
 AppTasks can be limited to a specific server (or set of servers) by creating a group in the 
 config/config.yml file:
 
-		development:
-		  redis_hostname: localhost
-		  redis_port: 6379
-		  servers:
-		    - hostname: worker01
-		      group: incoming
-		      max_app_tasks: 15
-		    - hostname: worker02
-		      group: worker
-		      max_app_tasks: 15
+```yaml
+development:
+  redis_hostname: localhost
+  redis_port: 6379
+  servers:
+    - hostname: worker01
+      group: incoming
+      max_app_tasks: 15
+    - hostname: worker02
+      group: worker
+      max_app_tasks: 15
+```
 
 and specifying the group in the routes files:
 
-		faucet :twitter_faucet, :group => :incoming
+```ruby
+faucet :twitter_faucet, :group => :incoming
+```
 
 # Support
 

data/test/helper.rb

@@ -1,7 +1,11 @@
+require 'rubygems'
+require 'bundler'
+Bundler.setup(:default, :test)
+Bundler.require(:default, :test)
+
 require 'test/unit'
-$LOAD_PATH << File.dirname(__FILE__) + '/../lib/'
 
+$LOAD_PATH << File.dirname(__FILE__) + '/../lib/'
 APP_PATH ||= File.expand_path('..', __FILE__)
 RACK_ENV ||= 'test'
 
-require 'hot_potato'

data/test/utils_test.rb

@@ -0,0 +1,23 @@
+require 'helper'
+
+class UtilsTest < Test::Unit::TestCase
+  
+  def test_process_alive
+    pid = Process.pid
+    assert Process.alive?(pid)
+  end
+  
+  def test_clasify
+    assert_equal "MyTask", classify("my_task")
+    assert_equal "Mytask", classify("mytask")
+    assert_equal "MyTask", classify("MyTask")
+    
+  end
+
+  def test_underscore
+    assert_equal "my_task", underscore("MyTask")
+    assert_equal "mytask", underscore("Mytask")
+    assert_equal "my_task", underscore("my_task")
+  end
+  
+end

metadata

@@ -2,7 +2,7 @@
 name: hot_potato
 version: !ruby/object:Gem::Version 
   prerelease: 
-  version: 0.12.1
+  version: 0.12.2
 platform: ruby
 authors: 
 - Darian Shimy
@@ -19,9 +19,9 @@ dependencies:
   requirement: &id001 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
-    - - ">="
+    - - ~>
       - !ruby/object:Gem::Version 
-        version: "0"
+        version: 1.5.3
   type: :runtime
   version_requirements: *id001
 - !ruby/object:Gem::Dependency 
@@ -30,9 +30,9 @@ dependencies:
   requirement: &id002 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
-    - - ">="
+    - - ~>
       - !ruby/object:Gem::Version 
-        version: "0"
+        version: 2.2.1
   type: :runtime
   version_requirements: *id002
 - !ruby/object:Gem::Dependency 
@@ -41,9 +41,9 @@ dependencies:
   requirement: &id003 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
-    - - ">="
+    - - ~>
       - !ruby/object:Gem::Version 
-        version: "0"
+        version: 0.6.0
   type: :runtime
   version_requirements: *id003
 - !ruby/object:Gem::Dependency 
@@ -52,9 +52,9 @@ dependencies:
   requirement: &id004 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
-    - - ">="
+    - - ~>
       - !ruby/object:Gem::Version 
-        version: "0"
+        version: 1.2.6
   type: :runtime
   version_requirements: *id004
 - !ruby/object:Gem::Dependency 
@@ -63,9 +63,9 @@ dependencies:
   requirement: &id005 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
-    - - ">="
+    - - ~>
       - !ruby/object:Gem::Version 
-        version: "0"
+        version: 0.1.8
   type: :runtime
   version_requirements: *id005
 - !ruby/object:Gem::Dependency 
@@ -74,9 +74,9 @@ dependencies:
   requirement: &id006 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
-    - - ">="
+    - - ~>
       - !ruby/object:Gem::Version 
-        version: "0"
+        version: 0.0.2
   type: :runtime
   version_requirements: *id006
 description: Hot Potato is an open source real-time processing framework written in Ruby. Originally designed to process the Twitter firehose at 3,000+ tweets per second, it has been extended to support any type of streaming data as input or output to the framework. The framework excels with applications such as, social media analysis, log processing, fraud prevention, spam detection, instant messaging, and many others that include the processing of streaming data.
@@ -132,6 +132,7 @@ files:
 - lib/hot_potato/worker.rb
 - readme.md
 - test/helper.rb
+- test/utils_test.rb
 - test/version_test.rb
 has_rdoc: true
 homepage: http://github.com/dshimy/hotpotato
@@ -163,4 +164,5 @@ specification_version: 3
 summary: A Real-time Processing Framework
 test_files: 
 - test/helper.rb
+- test/utils_test.rb
 - test/version_test.rb