hot_potato 0.12.1

data/.gitignore

@@ -0,0 +1,4 @@
+*.gem
+.bundle
+Gemfile.lock
+pkg/*

data/Gemfile

@@ -0,0 +1,4 @@
+source "http://rubygems.org"
+
+# Specify your gem's dependencies in hot_potato.gemspec
+gemspec

data/LICENSE

@@ -0,0 +1,19 @@
+Copyright (c) 2010-2011 Darian Shimy
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/Rakefile

@@ -0,0 +1,25 @@
+require 'bundler'
+Bundler::GemHelper.install_tasks
+
+require 'rake'
+require 'rake/testtask'
+require 'rake/rdoctask'
+
+desc 'Default: run unit tests.'
+task :default => :test
+
+desc 'Run the Hot Potato tests.'
+Rake::TestTask.new(:test) do |t|
+  t.libs << 'lib' << 'test'
+  t.pattern = 'test/*_test.rb'
+  t.verbose = true
+end
+
+desc 'Generate documentation for Hot Potato.'
+Rake::RDocTask.new(:rdoc) do |rdoc|
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title    = 'Hot Potato'
+  rdoc.options << '--line-numbers' << '--inline-source'
+  rdoc.rdoc_files.include('README')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end

data/bin/hotpotato

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require 'rubygems'
+require 'hot_potato/version'
+require 'hot_potato/generate'
+
+app_path = ARGV.first
+unless app_path
+	puts "Usage: hotpotato [-v|--version] app_name"
+	exit(1)
+end
+
+puts "Hot Potato (v #{HotPotato::VERSION})"
+HotPotato::Generate.new app_path

data/hot_potato.gemspec

@@ -0,0 +1,28 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+require "hot_potato/version"
+
+Gem::Specification.new do |s|
+  s.name        = "hot_potato"
+  s.version     = HotPotato::VERSION
+  s.platform    = Gem::Platform::RUBY
+  s.authors     = ["Darian Shimy"]
+  s.email       = ["dshimy@gmail.com"]
+  s.homepage    = "http://github.com/dshimy/hotpotato"
+  s.summary     = %q{A Real-time Processing Framework}
+  s.description = %q{Hot Potato is an open source real-time processing framework written in Ruby. Originally designed to process the Twitter firehose at 3,000+ tweets per second, it has been extended to support any type of streaming data as input or output to the framework. The framework excels with applications such as, social media analysis, log processing, fraud prevention, spam detection, instant messaging, and many others that include the processing of streaming data.}
+
+  s.add_dependency 'json'
+  s.add_dependency 'redis'
+  s.add_dependency 'bunny'
+  s.add_dependency 'sinatra'
+  s.add_dependency "vegas"
+  s.add_dependency "snappy"
+
+  s.rubyforge_project = "hot_potato"
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+end

data/lib/hot_potato/admin/public/admin.css

@@ -0,0 +1,30 @@
+body {
+  font-family: helvetica, sans-serif;
+  font-size: 12px;
+}
+h1 {
+  margin: 0 0 20px 0;
+  font-size: 24px;
+}
+h2 {
+  margin: 0 0 5px 0;
+  font-size: 20px;
+}
+table {
+  width: 900px;
+  border: 1px solid #ccc;
+  border-collapse: collapse;
+  margin-bottom: 20px;
+}
+table tr th {
+  border: 1px solid #ccc;
+  background: #EC7800;
+  color: #fff;
+  padding: 4px;
+
+}
+table tr td {
+  border: 1px solid #ccc;
+  padding: 2px 8px;
+  text-align: center;
+}

data/lib/hot_potato/admin/views/index.erb

@@ -0,0 +1,58 @@
+<!DOCTYPE html>
+<html>
+  <head>
+    <meta http-equiv="Content-type" content="text/html; charset=utf-8">
+    <title>Hot Potato Admin Server</title>
+    <link rel='stylesheet' href='/admin.css' type='text/css' media="screen">
+    <meta http-equiv="refresh" content="30">
+  </head>
+  <body>
+    <h1>Hot Potato Admin Server</h1>
+    <h2>Machines</h2>
+    <table>
+      <tr>
+        <th>Hostname</th>
+        <th>Process ID</th>
+        <th>Started</th>
+        <th>Last Heartbeat</th>
+        <th>AppTasks</th>
+      </tr>
+      <% @machines.each do |m| %>
+      <tr>
+        <td><%= m.hostname %></td>
+        <td><%= m.pid %></td>
+        <td><%= format_date(m.started_at) %></td>
+        <td><%= format_date(m.updated_at) %></td>
+        <td><%= m.app_tasks %></td>
+      </tr>
+      <% end %>
+    </table>
+    
+    <h2>App Tasks</h2>
+    <table>
+      <tr>
+        <th>Hostname</th>
+        <th>Class</th>
+        <th>Type</th>
+        <th>Process ID</th>
+        <th>Started</th>
+        <th>Last Heartbeat</th>
+        <th>Requests In</th>
+        <th>Requests Out</th>
+      </tr>
+      <% @apptasks.each do |a| %>
+      <tr>
+        <td><%= a.hostname %></td>
+        <td><%= a.classname %></td>
+        <td><%= a.type %></td>
+        <td><%= a.pid %></td>
+        <td><%= format_date(a.started_at) %></td>
+        <td><%= format_date(a.updated_at) %></td>
+        <td><%= a.requests_in %></td>
+        <td><%= a.requests_out %></td>
+      </tr>
+      <% end %>
+    </table>
+    
+  </body>
+</html>

data/lib/hot_potato/admin.rb

@@ -0,0 +1,67 @@
+require 'sinatra/base'
+require 'redis'
+require 'erb'
+
+module HotPotato
+
+  # The admin server is a Sinatra-based application to display statistical and diagnostic information.
+  # 
+  # The admin server can be managed from the command line:
+  # 
+  #     $ bin/admin --help
+  # 
+  #     Usage: ./admin [options]
+  # 
+  #     Vegas options:
+  #       -K, --kill               kill the running process and exit
+  #       -S, --status             display the current running PID and URL then quit
+  #       -s, --server SERVER      serve using SERVER (thin/mongrel/webrick)
+  #       -o, --host HOST          listen on HOST (default: 0.0.0.0)
+  #       -p, --port PORT          use PORT (default: 5678)
+  #       -x, --no-proxy           ignore env proxy settings (e.g. http_proxy)
+  #       -e, --env ENVIRONMENT    use ENVIRONMENT for defaults (default: development)
+  #       -F, --foreground         don't daemonize, run in the foreground
+  #       -L, --no-launch          don't launch the browser
+  #       -d, --debug              raise the log level to :debug (default: :info)
+  #           --app-dir APP_DIR    set the app dir where files are stored (default: ~/.vegas/Hot_Potato_Admin_Server)/)
+  #       -P, --pid-file PID_FILE  set the path to the pid file (default: app_dir/Hot_Potato_Admin_Server.pid)
+  #           --log-file LOG_FILE  set the path to the log file (default: app_dir/Hot_Potato_Admin_Server.log)
+  #           --url-file URL_FILE  set the path to the URL file (default: app_dir/Hot_Potato_Admin_Server.url)
+  # 
+  #     Common options:
+  #       -h, --help               Show this message
+  #           --version            Show version
+  # 
+  # The page can be accessed at http://localhost:5678
+  class Admin < Sinatra::Base
+    
+    include HotPotato::Core
+    
+    dir = File.dirname(File.expand_path(__FILE__))
+
+    set :views,  "#{dir}/admin/views"
+    set :public, "#{dir}/admin/public"
+    set :static, true
+    
+    helpers do
+      def format_date(d)
+        return "" unless d
+        d.strftime("%b %d, %Y %H:%M")
+      end
+    end
+    
+    get "/" do
+      @machines = []
+      stat.keys('hotpotato.supervisor.*').each do |key|
+        @machines << JSON.parse(stat.get(key))
+      end
+      @apptasks = []
+      stat.keys('hotpotato.apptask.*').each do |key|
+        @apptasks << JSON.parse(stat.get(key))
+      end
+      erb :index
+    end    
+    
+  end
+  
+end

data/lib/hot_potato/app_task.rb

@@ -0,0 +1,42 @@
+require "socket"
+
+module HotPotato
+  
+  # AppTasks are the controllers in the framework.  The Supervisor (See below) is responsible for
+  # starting AppTasks.  There are three types: Faucets, Workers, and Sinks.
+  module AppTask
+   
+   HEARTBEAT_INTERVAL = 20
+   
+   include HotPotato::Core
+   
+   # Used to keep AppTask statistics when a message is received.
+   def count_message_in
+     stat.incr "hotpotato.counter.apptask.#{Socket.gethostname}.#{self.class.name}.#{Process.pid}.messages_in"
+     stat.expire "hotpotato.counter.apptask.#{Socket.gethostname}.#{self.class.name}.#{Process.pid}.messages_in", 600
+   end
+   
+   # Used to keep AppTask statistics when a message is sent.
+   def count_message_out
+     stat.incr "hotpotato.counter.apptask.#{Socket.gethostname}.#{self.class.name}.#{Process.pid}.messages_out"     
+     stat.expire "hotpotato.counter.apptask.#{Socket.gethostname}.#{self.class.name}.#{Process.pid}.messages_out", 600
+   end
+   
+   # Starts the HeartBeat service to maintain the process id table.
+   def start_heartbeat_service
+     ati = AppTaskInfo.new(:classname => self.class.name)
+     stat.set ati.key, ati.to_json, 120
+
+     Thread.new do
+       log.info "Thread created for AppTask [Heartbeat]"
+       loop do
+         ati.touch
+         stat.set ati.key, ati.to_json, 120
+         sleep HEARTBEAT_INTERVAL
+       end
+     end
+   end
+    
+  end
+  
+end

data/lib/hot_potato/app_task_info.rb

@@ -0,0 +1,80 @@
+require "socket"
+
+class AppTaskInfo 
+  
+  include HotPotato::Core
+  
+  def initialize(options = {})
+    @data = {}
+    @data[:started_at] = options[:started_at] || Time.now
+    @data[:updated_at] = options[:updated_at] || @data[:started_at]
+    @data[:hostname]   = options[:hostname]   || Socket.gethostname
+    @data[:classname]  = options[:classname]  || "AppTask"
+    @data[:pid]        = options[:pid]        || Process.pid
+  end
+  
+  def key 
+    "hotpotato.apptask.#{@data[:hostname]}.#{@data[:classname]}.#{@data[:pid]}"
+  end
+
+  def to_s
+    @data.to_s
+  end
+  
+  def type
+    return "Faucet" if Kernel.const_get(@data[:classname]).ancestors.include?(HotPotato::Faucet)
+    return "Worker" if Kernel.const_get(@data[:classname]).ancestors.include?(HotPotato::Worker)
+    return "Sink"   if Kernel.const_get(@data[:classname]).ancestors.include?(HotPotato::Sink)    
+  end
+  
+  def classname
+    @data[:classname]
+  end
+  
+  def hostname
+    @data[:hostname]
+  end
+
+  def requests_in
+    stat.get("hotpotato.counter.apptask.#{@data[:hostname]}.#{@data[:classname]}.#{@data[:pid]}.messages_in") || 0
+  end
+
+  def requests_out
+    stat.get("hotpotato.counter.apptask.#{@data[:hostname]}.#{@data[:classname]}.#{@data[:pid]}.messages_out") || 0
+  end
+
+  def pid
+    @data[:pid].to_i
+  end
+  
+  def started_at
+    return nil unless @data[:started_at]
+    DateTime.parse(@data[:started_at])
+  end
+  
+  def updated_at
+    return nil unless @data[:updated_at]
+    DateTime.parse(@data[:updated_at])
+  end
+  
+  def touch
+    @data[:updated_at] = Time.now
+  end
+  
+  def to_json(*a)
+    result = @data
+    result["json_class"] = self.class.name
+    result.to_json(*a)
+  end
+  
+  def self.json_create(o)
+    options = {}
+    options[:started_at] = o["started_at"]
+    options[:updated_at] = o["updated_at"]
+    options[:pid]        = o["pid"]
+    options[:classname]  = o["classname"]    
+    options[:hostname]   = o["hostname"]    
+    new options
+  end
+  
+end

data/lib/hot_potato/app_task_server.rb

@@ -0,0 +1,92 @@
+require 'ostruct'
+require 'optparse'
+require 'socket'
+
+module HotPotato
+  
+  # This is used internally.
+  class AppTaskServer
+    
+    include HotPotato::Core
+    
+    def initialize
+      @options = load_options
+      @options.app_task_name, @options.mode = parse_options 
+      trap("INT") { shutdown }
+      self.send(@options.mode)
+    end
+    
+    def start
+      set_logger(:queue_logger, :classname => classify(app_task_name))
+      Process.daemon
+      STDIN.reopen '/dev/null'
+      STDOUT.sync = true
+      STDERR.sync = true
+      run
+    end
+    
+    def run
+      $0 = "Hot Potato AppTask [#{classify(@options.app_task_name)}]"
+      log.info "Starting Hot Potato AppTask #{HotPotato::VERSION} #{classify(@options.app_task_name)}"
+      app_task = @options.routes.find @options.app_task_name
+      if app_task
+        obj = Kernel.const_get(classify(app_task.classname)).new
+        begin
+          if app_task.source
+            obj.start app_task.source
+          else
+            obj.start
+          end
+        rescue Exception
+          log.fatal $!
+          exit 1
+        end
+      else
+        puts "Could not find #{@options.app_task_name} in the route definition"
+        exit 1
+      end
+    end
+    
+    def parse_options
+      mode = :run
+      app_task_name = ""
+      op = OptionParser.new do |opts|
+        opts.banner = "Usage: #{$0} app_task_name [run|start]"
+        opts.on_tail("-h", "--help", "Show this message") do
+          puts op
+          exit!
+        end
+      end
+      begin
+        op.parse!
+        app_task_name = ARGV.shift
+        mode = (ARGV.shift || "run").to_sym
+        if ![:start, :run].include?(mode)
+          puts op
+          exit!
+        end
+      rescue
+        puts op
+        exit!
+      end
+      return app_task_name, mode
+    end
+    
+    def shutdown
+      log.info "Stopping AppTask..."
+      exit!
+    end
+    
+    def load_options
+      options = OpenStruct.new
+      options.mode          = :run
+      options.hostname      = Socket.gethostname
+      options.routes        = HotPotato::Route.routes
+      options.app_task_name = ""
+      
+      return options
+    end
+    
+  end
+  
+end

data/lib/hot_potato/cache.rb

@@ -0,0 +1,45 @@
+module HotPotato
+  
+  class Cache
+    
+    include HotPotato::Core
+    
+    def initialize
+      log.info "Initializing connection to Redis (#{config['redis_hostname']}:#{config['redis_port']})"
+      @@redis ||= Redis.new :host => config['redis_hostname'], :port => config['redis_port']
+    end
+    
+    def get(k)
+      @@redis.get(k)
+    end
+    
+    def set(k, v, ttl = nil)
+      @@redis.multi do
+        @@redis.set(k, v)
+        @@redis.expire(k, ttl) if ttl
+      end
+    end
+    
+    def getset(k, v)
+      @@redis.getset(k, v)
+    end
+    
+    def keys(k)
+      @@redis.keys(k)
+    end
+    
+    def del(k)
+      @@redis.del(k)
+    end
+    
+    def expire(k, ttl)
+      @@redis.expire(k, ttl)
+    end
+    
+    def incr(k)
+      @@redis.incr(k)
+    end
+    
+  end
+  
+end

data/lib/hot_potato/core.rb

@@ -0,0 +1,62 @@
+require 'logger'
+require 'snappy'
+
+module HotPotato
+  
+  module Core
+
+    CONFIG_PATH = "#{APP_PATH}/config/config.yml"
+
+    def set_logger(provider, options = {})
+      if provider == :queue_logger
+        @@log ||= QueueLogger.new(options)
+      else
+        @@log ||= Logger.new(STDOUT)
+      end
+    end
+
+    def log
+      @@log ||= Logger.new(STDOUT)
+      @@log.level = Logger::INFO
+      return @@log
+    end
+
+    def config
+      @@config ||= YAML.load_file(CONFIG_PATH)[RACK_ENV]
+    end
+      
+    def stat
+      @@cache ||= Cache.new
+    end
+      
+    def queue_inject(name, message)
+      @@queue ||= Redis.new :host => config['redis_hostname'], :port => config['redis_port']
+      @@queue.publish name.to_sym, Snappy.deflate(message.to_json)
+    end
+    
+    def queue_subscribe(name, &block) 
+      queue ||= Redis.new :host => config['redis_hostname'], :port => config['redis_port']
+      queue.subscribe(*name) do |on|
+        on.message do |channel, message|
+          yield JSON.parse(Snappy.inflate(message))
+        end
+      end
+    end
+      
+    def acquire_lock(key, duration = 10)
+      if !stat.getset("hotpotato.lock.#{key.to_s}", "1")
+        stat.expire "hotpotato.lock.#{key.to_s}", duration
+        return true
+      else
+        stat.expire "hotpotato.lock.#{key.to_s}", duration
+        return false
+      end
+    end
+    
+    def release_lock(key)
+      stat.del("hotpotato.lock.#{key.to_s}")
+    end  
+      
+  end
+  
+end