hot_potato 0.12.1

data/lib/hot_potato/dsl.rb

@@ -0,0 +1,172 @@
+module HotPotato
+  
+  # The routes file (config/routes.rb) is a Ruby DSL that does the following:
+  # 
+  # * Defines AppTasks (Faucets, Workers, Sinks)
+  # * Defines processing chain for AppTasks
+  # * Restrict AppTasks to a host group
+  # * Limit number of instances
+  # 
+  # Example:
+  # 
+  #     HotPotato::Route.build do
+  # 
+  #       faucet :twitter_faucet
+  #       worker :influencer, :source => :twitter_faucet
+  #       sink :log_writer, :source => :influencer
+  # 
+  #     end
+  #
+  # Multiple sources can be attached to a worker or sink:
+  # 
+  #     worker :influencer, :source => [:twitter_faucet. :other_source]
+  #   
+  # The number of instances is set to 1.  This can be changed by setting the number of instances:
+  # 
+  #     worker :influencer, :source => :twitter_faucet, :instances => 2
+  # 
+  # AppTasks can be limited to a specific server (or set of servers) by creating a group in the 
+  # config/config.yml file:
+  # 
+  #     development:
+  #       redis_hostname: localhost
+  #       redis_port: 6379
+  #       servers:
+  #         - hostname: worker01
+  #           group: incoming
+  #           max_app_tasks: 15
+  #         - hostname: worker02
+  #           group: worker
+  #           max_app_tasks: 15
+  # 
+  # and specifying the group in the routes files:
+  # 
+  #     faucet :twitter_faucet, :group => :incoming
+  module Route
+    
+    def self.build(&block)
+      @@routes = Routes.new
+      @@routes.instance_eval(&block)
+    end
+
+    def self.routes
+      @@routes
+    end
+
+    class Routes
+      
+      attr_accessor :faucets, :workers, :sinks
+  
+      def initialize
+        @faucets = []
+        @workers = []
+        @sinks = []      
+      end
+  
+      def find(name)
+        app_tasks.each do |app_task|
+          if app_task.classname.to_s == name.to_s
+            return app_task
+          end
+        end
+        return nil
+      end
+  
+      def app_tasks
+        @faucets + @workers + @sinks
+      end
+  
+      def faucet(classname, options = {}, &block)
+        f = Faucet.new(classname, options)      
+        f.instance_eval(&block) if block
+        @faucets << f
+      end
+  
+      def worker(classname, options = {}, &block)
+        w = Worker.new(classname, options)      
+        w.instance_eval(&block) if block
+        @workers << w
+      end
+  
+      def sink(classname, options = {}, &block)
+        s = Sink.new(classname, options)      
+        s.instance_eval(&block) if block
+        @sinks << s
+      end
+  
+      def to_s
+        str = "Faucets\n"
+        @faucets.each do |f|
+          str << f.to_s
+        end
+        str << "Workers\n"
+        @workers.each do |f|
+          str << f.to_s
+        end
+        str << "Sinks\n"
+        @sinks.each do |f|
+          str << f.to_s
+        end
+        str
+      end
+    end
+
+    class AppTask
+      
+      include HotPotato::Core
+      
+      attr_accessor :classname, :instances, :source, :group
+      def initialize(classname, options = {})
+        @classname = classname.to_s
+        @instances = options[:instances] || 1
+        @group     = options[:group] || ""
+        if options[:source]
+          if options[:source].class == Array
+            @source = options[:source].map { |s| s.to_s }
+          else
+            @source = [options[:source].to_s]
+          end
+        else
+          @source = nil
+        end
+      end
+      
+      def allow_group(name)
+        return true if @group == ""
+        return name == @group.to_s
+      end
+      
+      def running_instances
+        stat.keys("hotpotato.apptask.*.#{classify(@classname)}.*").count || 0
+      end
+      
+      def type
+        return "Faucet" if Kernel.const_get(@data[:classname]).ancestors.include?(HotPotato::Faucet)
+        return "Worker" if Kernel.const_get(@data[:classname]).ancestors.include?(HotPotato::Worker)
+        return "Sink"   if Kernel.const_get(@data[:classname]).ancestors.include?(HotPotato::Sink)  
+        return "AppTask"  
+      end
+      
+    end
+
+    class Faucet < AppTask
+      def to_s
+        "  Faucet class [#{@classname}]\n"
+      end
+    end
+
+    class Worker < AppTask
+      def to_s
+        "  Worker class [#{@classname}]\n"
+      end
+    end
+
+    class Sink < AppTask
+      def to_s
+        "  Sink class [#{@classname}]\n"
+      end
+    end
+
+  end
+  
+end

data/lib/hot_potato/faucet.rb

@@ -0,0 +1,41 @@
+module HotPotato
+  
+  # Faucets inject data into the system.  Examples include: Twitter Reader, SMTP, and Tail Log File.
+  # Each faucet is a ruby file in the app directory that extends HotPotato::Faucet and implements
+  # the perform method.  For each message received, the method should call the send_message to send
+  # it to the next AppTask.
+  # 
+  #     class TwitterFaucet < HotPotato::Faucet
+  # 
+  #       def perform
+  #         TweetStream::Client.new("user", "secret").sample do |s|
+  #           message = {}
+  #           message["username"] = s.user.screen_name
+  #           message["text"]     = s.text
+  #           send_message message
+  #         end
+  #       end
+  # 
+  #     end
+  class Faucet
+    
+    include HotPotato::AppTask
+    
+    def start
+      @queue_out = underscore(self.class.name.to_sym)
+      if self.respond_to?('perform')
+        start_heartbeat_service
+        perform
+      else
+        log.error "The Faucet #{self.class.name} does not implement a perform method."
+      end      
+    end
+    
+    def send_message(m)
+      queue_inject @queue_out, m
+      count_message_out
+    end
+    
+  end
+  
+end

data/lib/hot_potato/generate.rb

@@ -0,0 +1,55 @@
+require 'fileutils'
+
+module HotPotato
+  
+  class Generate
+    
+    def initialize(app_path) 
+      @app_path = app_path   
+      log "Generating application #{@app_path}..."
+      mkdir app_path    
+      copy_file "Gemfile"
+      copy_file "Rakefile"
+      mkdir "app"
+      mkdir "bin"
+      copy_file "admin", 'bin'
+      copy_file "app_task", 'bin'
+      copy_file "generate", 'bin'
+      copy_file "supervisor", 'bin'
+      mkdir "config"
+      mkdir "config/environments"
+      copy_file "development.rb", 'config/environments'
+      copy_file "test.rb", 'config/environments'
+      copy_file "production.rb", 'config/environments'
+      copy_file "boot.rb", 'config'
+      copy_file "config.yml", 'config'
+      copy_file "routes.rb", 'config'   
+      mkdir "docs"
+      mkdir "logs"
+      mkdir "test"
+      mkdir "tmp"
+    end
+
+    def mkdir(path)
+      dir = path == @app_path ? @app_path : "#{@app_path}/#{path}"
+      if Dir.exists?(dir)
+        log "     exist  #{dir}"
+      else
+        Dir.mkdir(dir)
+        log "    create  #{dir}"
+      end
+    end
+    
+    def copy_file(src, dest = "")
+      dest = "#{dest}/" unless dest == ""
+      log "    add     #{@app_path}/#{dest}#{src}"
+      FileUtils.cp "#{File.expand_path('..', __FILE__)}/templates/#{src}", "#{@app_path}/#{dest}#{src}"
+    end
+    
+    def log(message)
+      puts "#{message}"
+    end
+    
+  end
+  
+end

data/lib/hot_potato/generate_app_task.rb

@@ -0,0 +1,41 @@
+module HotPotato
+  
+  class GenerateAppTask
+   
+    def initialize
+      usage unless ARGV[1]
+      name = underscore(ARGV[1])
+      
+      case ARGV[0]
+      when "sink"
+        process_template "template_sink.rb", "#{name}.rb", classify(name)
+      when "faucet"
+        process_template "template_faucet.rb", "#{name}.rb", classify(name)
+      when "worker"
+        process_template "template_worker.rb", "#{name}.rb", classify(name)
+      else
+        usage
+      end
+    end
+    
+    def usage
+      puts "Usage: generate [faucet|worker|sink] name"
+      exit 1      
+    end
+    
+    def process_template(src, dest, name)
+
+      template_file = File.open("#{File.expand_path('..', __FILE__)}/templates/#{src}")
+      contents = ""
+      template_file.each { |line| contents << line}
+
+      result = contents.gsub("__NAME__", name)
+      dest_file = File.new("#{APP_PATH}/app/#{dest}", "w")
+      dest_file.write(result)
+      dest_file.close
+      puts "Writing #{APP_PATH}/app/#{dest}"
+    end
+    
+  end
+  
+end

data/lib/hot_potato/queue_logger.rb

@@ -0,0 +1,33 @@
+module HotPotato
+  
+  class QueueLogger
+      
+    include HotPotato::Core
+        
+    attr_accessor :level, :formatter
+        
+    def initialize(options = {})
+      @classname = options[:classname]
+    end
+        
+    def log(message, severity)
+      log_entry = {}
+      log_entry[:created_at] = Time.now
+      log_entry[:message]    = message
+      log_entry[:severity]   = severity
+      log_entry[:host]       = Socket.gethostname
+      log_entry[:pid]        = Process.pid
+      log_entry[:classname]  = @classname
+      
+      queue_inject "hotpotato.log.#{log_entry[:host]}", log_entry.to_json
+    end
+  
+    def debug(m); log m, __method__; end
+    def info(m);  log m, __method__; end
+    def warn(m);  log m, __method__; end
+    def error(m); log m, __method__; end
+    def fatal(m); log m, __method__; end
+        
+  end
+
+end

data/lib/hot_potato/sink.rb

@@ -0,0 +1,33 @@
+module HotPotato
+  
+  # Sinks send data out of the system.  Examples include: WebSocket, Database (Data Warehouse), and 
+  # File Writer.  Each sink is a ruby file in the app directory that extends HotPotato::Sink and implements
+  # the perform(message) method.  There is no send_message for the sink to call since it is a final destination
+  # for the message.
+  # 
+  #     class LogWriter < HotPotato::Sink
+  # 
+  #       def perform(message)
+  #         log.debug "#{message["username"]}:#{message["influence"]}"
+  #       end
+  # 
+  #     end
+  class Sink
+    
+    include HotPotato::AppTask
+        
+    def start(queue_in)
+      if !self.respond_to?('perform')
+        log.error "The Sink #{self.class.name} does not implement a perform method."
+        exit 1
+      end
+      start_heartbeat_service      
+      queue_subscribe(queue_in) do |m|
+        count_message_in
+        perform m
+      end
+    end
+    
+  end
+  
+end

data/lib/hot_potato/supervisor_info.rb

@@ -0,0 +1,64 @@
+require "socket"
+
+class SupervisorInfo 
+  
+  include HotPotato::Core
+    
+  def initialize(options = {})
+    @data = {}
+    @data[:started_at] = options[:started_at] || Time.now
+    @data[:updated_at] = options[:updated_at] || @data[:started_at]
+    @data[:hostname]   = options[:hostname]   || Socket.gethostname
+    @data[:pid]        = options[:pid]        || Process.pid
+  end
+  
+  def key 
+    "hotpotato.supervisor.#{@data[:hostname]}.#{@data[:pid]}"
+  end
+
+  def to_s
+    @data.to_s
+  end
+  
+  def app_tasks
+    stat.keys("hotpotato.apptask.#{@data[:hostname]}.*").count
+  end
+  
+  def hostname
+    @data[:hostname]
+  end
+
+  def pid
+    @data[:pid]
+  end
+  
+  def started_at
+    return nil unless @data[:started_at]
+    DateTime.parse(@data[:started_at])
+  end
+  
+  def updated_at
+    return nil unless @data[:updated_at]
+    DateTime.parse(@data[:updated_at])
+  end
+  
+  def touch
+    @data[:updated_at] = Time.now
+  end
+  
+  def to_json(*a)
+    result = @data
+    result["json_class"] = self.class.name
+    result.to_json(*a)
+  end
+  
+  def self.json_create(o)
+    options = {}
+    options[:started_at] = o["started_at"]
+    options[:updated_at] = o["updated_at"]
+    options[:pid]        = o["pid"]   
+    options[:hostname]   = o["hostname"]    
+    new options
+  end
+  
+end

data/lib/hot_potato/supervisor_server.rb

@@ -0,0 +1,225 @@
+require 'ostruct'
+require 'optparse'
+require 'socket'
+
+module HotPotato
+  
+  # The supervisor is a process that runs on each machine that participates in the cluster.
+  # When it starts it does the following:
+  # 
+  # 0. Read the routes file
+  # 1. Connect to the Redis server and get the appTask process ID table
+  # 2. Acquire the global lock
+  # 3. If a process is needed, fork a new process for AppTask
+  # 4. Release the global lock
+  # 5. Rinse and Repeat
+  # 
+  # The supervisor also starts the Heartbeat service and logging service as background threads.
+  # 
+  # The supervisor can be managed from the command line:
+  # 
+  #     $ bin/supervisor [run|start|stop|restart]
+  # 
+  # If started without any settings, it will default to run.
+  class SupervisorServer
+
+    include HotPotato::Core
+
+    MAX_APP_TASKS      = 32
+    HEARTBEAT_INTERVAL = 20
+    PID_FILE           = "#{APP_PATH}/tmp/supervisor.pid"
+    LOG_FILE           = "#{APP_PATH}/logs/supervisor.log"
+    
+    def initialize
+      @options      = load_options
+      @options.mode = parse_options 
+      trap("INT") { shutdown }
+      self.send(@options.mode)
+    end
+        
+    def run
+      $0 = "Hot Potato Supervisor"
+      log.info "Starting Hot Potato Supervisor #{HotPotato::VERSION}"
+      begin
+        start_heartbeat_service
+        start_log_service
+        routes = HotPotato::Route.routes
+        while @options.running do       
+          if acquire_lock :supervisor
+            log.debug "Lock acquired"          
+            routes.app_tasks.each do |app_task|
+              if app_task.running_instances < app_task.instances && app_task.allow_group(@options.group)
+                if has_capacity
+                  log.info "Starting AppTask [#{classify(app_task.classname)}]"
+                  pid = fork do
+                    Process.setsid
+                    exec "#{APP_PATH}/bin/app_task #{app_task.classname.to_s}"
+                  end
+                  Process.detach pid
+                  sleep 2
+                else
+                  log.warn "Cannot start AppTask [#{app_task.classname}] - Server at Capacity (Increase max_app_tasks)"
+                end
+              end
+            end            
+            release_lock :supervisor        
+          end
+          sleep (5 + rand(5))
+        end
+      rescue Exception
+        log.error $!
+        log.error $@
+        exit 1
+      end
+    end    
+    
+    def start
+      # Check if we are running
+      if File.exists?(PID_FILE)
+         pid = 0;
+         File.open(PID_FILE, 'r') do |f|
+            pid = f.read.to_i
+         end
+         # Check if we are REALLY running
+         if Process.alive?(pid)
+           log.fatal "Supervisor is already running on this machine.  Only one instance can run per machine."
+           exit 1
+         else
+           log.info "Supervisor is not running despite the presence of the pid file.  I will overwrite the pid file and start the supervisor."
+         end
+      end
+      Process.daemon
+      File.open(PID_FILE, 'w') do |f|
+        f.write "#{Process.pid}\n"
+      end    
+      STDIN.reopen '/dev/null'
+      STDOUT.reopen LOG_FILE, 'a'
+      STDERR.reopen STDOUT
+      STDOUT.sync = true
+      STDERR.sync = true
+      run
+    end
+    
+    # Stops the Supervisor.  Requires the existance of a PID file.
+    # Calls the shutdown hook to stop AppTasks.
+    def stop
+      pid = 0
+      shutdown      
+      if File.exists?(PID_FILE)
+        File.open(PID_FILE, 'r') do |f|
+          pid = f.read.to_i
+        end              
+        Process.kill("INT", pid) if Process.alive?(pid)
+        File.delete(PID_FILE)
+      else
+        log.fatal "Supervisor PID file does not exist."
+        exit 1
+      end
+    end
+    
+    # Restarts the Supervisor
+    def restart
+      stop
+      sleep 2
+      start
+    end
+    
+    def parse_options
+      mode = :run
+      op = OptionParser.new do |opts|
+        opts.banner = "Usage: #{$0} [run|start|stop|restart]"
+        opts.on_tail("-h", "--help", "Show this message") do
+          puts op
+          exit
+        end
+      end
+      begin
+        op.parse!
+        mode = (ARGV.shift || "run").to_sym
+        if ![:start, :stop, :restart, :run].include?(mode)
+          puts op
+          exit 1
+        end
+      rescue
+        puts op
+        exit 1
+      end
+      return mode
+    end
+    
+    # Kills any running AppTasks on this machine and removes entries from the 
+    # process table cache.  Removes entry for the supervisor in the process
+    # table cache.
+    def shutdown
+      @options.running = false      
+      stat.keys("hotpotato.apptask.#{@options.hostname}.*").each do |app_task_key|
+        app_task = JSON.parse(stat.get(app_task_key))
+        log.info "Killing PID #{app_task.pid} [#{app_task.classname}]"
+        Process.kill("INT", app_task.pid) if Process.alive?(app_task.pid)
+        stat.del app_task_key
+      end
+      stat.keys("hotpotato.supervisor.#{@options.hostname}.*").each do |supervisor_key|
+        stat.del supervisor_key
+      end
+      log.info "Stopping Supervisor..."
+    end
+   
+    # Determines if this host reached the limit of the number of AppTasks it
+    # can run
+    def has_capacity
+      return stat.keys("hotpotato.apptask.#{@options.hostname}.*").count < @options.max_app_tasks
+    end
+    
+    # OK, this is not really a log service, but it is responsible for subscribing to the log messages
+    # from the AppTasks on this server.
+    def start_log_service
+      Thread.new do
+        log.info "Thread created for Supervisor [Log]"
+        queue_subscribe("hotpotato.log.#{@options.hostname}") do |m|
+          log_entry = JSON.parse(m)
+          if log.respond_to?(log_entry["severity"].to_sym)
+            log.send(log_entry["severity"].to_sym, "#{log_entry['classname']} [#{log_entry['pid']}] - #{log_entry['message']}")
+          end
+        end
+      end
+    end
+    
+    # Starts a background thread to update the process list in redis.
+    def start_heartbeat_service
+      si = SupervisorInfo.new
+      stat.set si.key, si.to_json
+      stat.expire si.key, 120
+
+      Thread.new do
+        log.info "Thread created for Supervisor [Heartbeat]"
+        loop do
+          si.touch
+          stat.set si.key, si.to_json, 120
+          sleep HEARTBEAT_INTERVAL
+        end
+      end
+    end
+
+    # Loads the options, mainly from the config.yml file, into an OpenStruct object.
+    def load_options
+      options = OpenStruct.new
+      options.max_app_tasks = MAX_APP_TASKS
+      options.group         = ""
+      options.mode          = :run
+      options.hostname      = Socket.gethostname
+      options.running       = true
+      
+      config["servers"].each do |server|
+        if server["hostname"] == options.hostname
+          options.max_app_tasks = server["max_app_tasks"] || MAX_APP_TASKS
+          options.group         = server["group"]         || ""
+          break
+        end
+      end
+
+      return options
+    end
+
+  end
+
+end

data/lib/hot_potato/templates/Gemfile

@@ -0,0 +1,6 @@
+source "http://rubygems.org"
+
+gem "hot_potato"
+gem "redis"
+gem "json"
+gem "vegas"

data/lib/hot_potato/templates/Rakefile

@@ -0,0 +1,9 @@
+require 'rake'
+require 'rake/testtask'
+require 'rake/rdoctask'
+require File.expand_path('../config/boot',  __FILE__)
+
+desc 'Print the routing table'
+task :routes do
+  puts HotPotato::Route.routes.to_s
+end

data/lib/hot_potato/templates/admin

@@ -0,0 +1,4 @@
+#!/usr/bin/env ruby
+
+require File.expand_path('../../config/boot',  __FILE__)
+Vegas::Runner.new(HotPotato::Admin, 'Hot Potato Admin Server')

data/lib/hot_potato/templates/app_task

@@ -0,0 +1,4 @@
+#!/usr/bin/env ruby
+
+require File.expand_path('../../config/boot',  __FILE__)
+HotPotato::AppTaskServer.new